# Tell search engines to index these pages
User-agent: *
Disallow: /assets
Disallow: /xml

User-agent: msnbot
Crawl-delay: 10

User-agent: Slurp
Crawl-delay: 5

User-agent: Googlebot
Crawl-delay: 5