# These pages contain links to relevant information, but don't themselves contain any content worth indexing.

User-agent: *
Disallow: /404.html
Disallow: /favicon.ico
Disallow: /index.xml
Disallow: /index.rdf
Disallow: /atom.xml
Disallow: /mt/
Disallow: /rsd.xml
Disallow: /include/

# Sorry, we no longer allow badly behaved robots.

User-agent: asterias
User-agent: baiduspider
User-agent: Caddbot
User-agent: cosmos
User-agent: curl
User-agent: Generic
# User-agent: ia_archiver
User-agent: larbin
User-agent: LinkWalker
User-agent: LNSpiderguy
User-agent: lwp-trivial
User-agent: MIIxpc
User-agent: MSIECrawler
User-agent: NaverRobot
User-agent: NaverBot
User-agent: NationalDirectory
User-agent: NPbot
User-agent: pompos
User-agent: psbot
User-agent: sitecheck.internetseer.com
User-agent: Teleport
User-agent: TurnitinBot
User-agent: Wget
Disallow: /