#robots.txt file to prevent robots from scanning this site #Disallowing access to all robots User-agent: * Disallow: / # This should be redundant Disallow: /may_2005/