# robotstxt.org
User-agent: Googlebot         # All Google services
User-agent: Bingbot           # Bing’s crawler
User-agent: Slurp             # Yahoo’s crawler
User-agent: DuckDuckBot       # DuckDuckGo’s crawler
User-agent: YandexBot         # Russian Yandex’s crawler
User-agent: Baiduspider       # Chinese Baidu’s crawler
Disallow: /                   # Disallow everything
Allow: /$                     # Allow domain withoud any subdomain
Allow: /login$                 # Allow login page
Allow: /unsubscribe$           # Allow unsubscribe page