# # robots.txt # # information from http://info.webcrawler.com/mak/projects/robots/norobots.html # User-agent: * Disallow: /cgi-bin/ # Disallow: /errors/ # Disallow: /forms/ # Disallow: /jobs/ # Disallow: /logs/ # Not needed Disallow: /oldjobs/ # Disallow: /pending/ # Disallow: /templates/ #