User-agent: * Disallow: /stores/az_link.phtml Disallow: /stores/atdbredir.phtml Disallow: /stores/buylink.phtml Disallow: /stores/buylink.html Disallow: /members.phtml #Disallow: /at_shared/link_finder.php Disallow: /stores/question.phtml Disallow: /send_page.phtml Disallow: /contact.phtml Crawl-Delay: 20 #User-agent: BecomeBot #Crawl-Delay: 20 #This one would make repeated calls to relative links #/page.html, then ../page.html, then ../../page.html, etc User-agent: Pompos Disallow: / #picsearch.com picture search User-agent: psbot Disallow: / #Korean Telecom User-agent: nhnbot Disallow: / #Korean Telecom User-agent: NaverBot Disallow: / #Ask Jeeves/Teoma User-agent: Ask Jeeves Disallow: /stores/search.phtml #MSN User-agent: msnbot Disallow: /stores/search.phtml #These things walk through the stores all day #User-agent: Wget #Disallow: / #These things walk through the stores all day User-agent: Httrack Disallow: / #Japan internet multifeed User-agent: gazz Disallow: /stores/ #Nameprotect.com - DAMNED IP crawler User-agent: NPBot Disallow: / #ShopWiki private comparison shopper User-agent: ShopWiki Disallow: /