# /robots.txt # # Note: "Disallow: " without a /anything after it actually means allow # #User-agent:* #Disallow: / #to disallow all User-agent:* Disallow: /cgi-bin/ Disallow: /Core/