# robots.txt for search engine robots # examples # User-agent: * # Disallow: /folder/folder/ # This is an infinite virtual URL space # Disallow: /temp/ # these will soon disappear # Disallow: /page.html #User-Agent: msnbot #Disallow: /*.PDF$ #Disallow: /*.jpeg$ #Disallow: /*.exe$ #User-agent: Googlebot #Disallow: /*.PDF$ # Yahoo Slurp #User-agent: Slurp #Disallow: /*exe$ #Disallow: /*.PDF$ #Disallow: /*.swf$ User-agent: * Disallow: /cgi-bin/