# Allow Archiver
User-agent: ia_archiver
Allow: /


User-agent: slurp
Crawl-delay: 60


User-agent: *

Disallow: *.php