# robots.txt for Disallow: /
User-agent: *
Disallow: /cgi-bin/
Disallow: /spidertest/
User-agent: asterias
Allow: /spidertest/