# robots.txt

User-agent: Slurp
Crawl-Delay: 10

User-agent: msnbot
Crawl-delay: 120

User-agent: dotbot
Crawl-delay: 120
Disallow: /

User-agent: *
Disallow: /install/
Disallow: /lib/
Disallow: /modules/
Disallow: /themes/
Disallow: /upgrade/