# robots.txt 
User-agent: Googlebot 
User-agent: Googlebot-Image
User-agent: MSNBot 
User-agent: Slurp 
User-agent: Teoma
User-agent: Zeusbot
User-agent: friendly
User-agent: Gigabot
User-agent: Scrubby
User-agent: Robozilla
User-agent: Nutch
User-agent: ia_archiver 
User-agent: baiduspider
User-agent: yahoo-mmcrawler
User-agent: psbot
User-agent: asterias
User-agent: yahoo-blogs/v3.9
User-agent: InfoNaviRobot
User-agent: TV33_Mercator
User-agent: AVSearch
User-agent: Mercator
User-agent: Scooter
User-agent: SearchengineLicenceSheep
User-agent: shadow
User-agent: MultiText
User-agent: FAST-WebCrawler
User-agent: Lycos_Spider
User-agent: Atomz
User-agent: htdig
User-agent: spider00.logika.net
User-agent: NetMechanic
User-agent: libwww-perl
User-agent: Teleport Pro
User-agent: BizBot04 kirk.overleaf.com
User-agent: HappyBot 
User-agent: CaliforniaBrownSpider
User-agent: EI*Net/0.1  libwww/0.1
User-agent: Ibot/1.0 libwww-perl/0.40    
User-agent: Merritt/1.0
User-agent: StatFetcher/1.0
User-agent: TeacherSoft/1.0  libwww/2.17
User-agent: WWW Collector
User-agent: processor/0.0ALPHA libwww-perl/0.20
User-agent: wobot/1.0 from 206.214.202.45
User-agent: Libertech-Rover         
User-agent: WhoWhere Robot
User-agent: ITI Spider
User-agent: w3index
User-agent: MyCNNSpider
User-agent: SummyCrawler
User-agent: OGspider
User-agent: linklooker
User-agent: CyberSpyder 
User-agent: SlowBot
User-agent: heraSpider
User-agent: Surfbot
User-agent: Bizbot003
User-agent: WebWalker
User-agent: SandBot
User-agent: EnigmaBot
User-agent: spyder3.microsys.com
User-agent: Inktomi spider
Disallow: /searchtools-rss.xml

User-agent: *
Disallow: /test/robots/noindex/
Disallow: /test/robots/disal/
Disallow: /test/robots/partial
Disallow: /background/
Disallow: /blog/
Disallow: /cgi-bin/
Disallow: /info/articles/
Disallow: /info/conferences-past.html
Disallow: /info/meetings/examples/
Disallow: /info/meetings/thunderlizard/examples/
Disallow: /info/robots/
Disallow: /info/slides/
Disallow: /lists/
Disallow: /related/
Disallow: /reviews/
Disallow: /test/relativelinks/2ndlevel/http://
Disallow: /test/relativelinks/rtestprob/http://searchtools/about/
Disallow: /test/relativelinks/rtestprob/http://searchtools/analysis/
Disallow: /test/relativelinks/rtestprob/http://searchtools/guide/
Disallow: /test/relativelinks/rtestprob/http://searchtools/info/
Disallow: /test/relativelinks/rtestprob/http://searchtools/pub/
Disallow: /test/relativelinks/rtestprob/http://searchtools/robots/
Disallow: /test/relativelinks/rtestprob/http://searchtools/search/
Disallow: /test/relativelinks/rtestprob/http://searchtools/site/
Disallow: /test/relativelinks/rtestprob/http://searchtools/slides/
Disallow: /test/relativelinks/rtestprob/http://searchtools/surveys/
Disallow: /test/relativelinks/rtestprob/http://searchtools/tools/
Disallow: /searchtools/
Disallow: /slides/examples/
Disallow: /ST/
Disallow: /st/
Disallow: /St/
Disallow: /wr/

# updated 2002-03-22 (disallow rtestprob links)
# updated 2002-06-25 (disallow info/slides links, info/robots/)
# updated 2002-07-25 (disallow /searchtools/ which is an alias)
# updated 2005-09-09 (rearranged as per Enrico's advice)