User-agent:
User-agent: *
User-agent:a1.google.com
User-agent:a2.google.com
User-agent:a2z.lycos.com
User-agent:AbachoBOT
User-agent:abcdatos_botlink/1.0.2
User-agent:Acme.Spider
User-agent:Acoon Robot
User-agent:Ad Muncher
User-agent:ADB_Web/1.1
User-agent:add-url.altavista.com Scooter/1.0 
User-agent:AESOP_com_SpiderMan
User-agent:ah-ha.com crawler
User-agent:Ahoy! The Homepage Finder
User-agent:Aladin/3.324
User-agent:Albert Indexer
User-agent:Alexa
User-agent:Alkaline
User-agent:AltaVista Intranet V1.0 AltaVista Search Intranet PX V1.0 
User-agent:AltaVista Intranet V1.0 bluewindow.ch
User-agent:AltaVista Intranet V1.0 pta.at
User-agent:AltaVista Intranet V1.0 Re-Index
User-agent:AltaVista Intranet V2.0 pta.at
User-agent:AltaVista Intranet V2.0 Sear.ch
User-agent:AltaVista Intranet V2.0 www.altavista.de
User-agent:AltaVista-Intranet
User-agent:ananzi 
User-agent:AnswerChase
User-agent:AnthillV1.1
User-agent:AntiBot
User-agent:AnzwersCrawl/2.0
User-agent:appie/1.1
User-agent:Arachnoidea
User-agent:Arachnophilia
User-agent:Arale
User-agent:Araneo
User-agent:Aranha
User-agent:AraybOt/1.0 (+http://www.araykoo.com/araybot.html)
User-agent:ArchitextSpider
User-agent:Aretha
User-agent:ARIADNE
User-agent:arks/1.0
User-agent:AskJeeves 
User-agent:Asp seek
User-agent:ASpider (Associative Spider)
User-agent:ASPSeek
User-agent:asterias
User-agent:ATA-Translation-Service
User-agent:ATN_Worldwide
User-agent:Atomz/1.0
User-agent:AURESYS/1.0
User-agent:av-dev3.av.pa-x.dec.com Scooter/2.0 G.R.A.B V1.0
User-agent:av-dev4.pa.alta-vista.net Scooter/2.0 G.R.A.B. V1.1.0 
User-agent:AVSearch 
User-agent:BackRub
User-agent:BackRub/2.1
User-agent:bbot/0.100
User-agent:Big Brother
User-agent:Bizbot003
User-agent:BizBot04 kirk.overleaf.com
User-agent:Bjaaland
User-agent:BlackWidow
User-agent:Bloodhound
User-agent:Borg-Bot
User-agent:borg-bot/0.9
User-agent:bos-catalog1.bos.lycos.com   
User-agent:bos-catalog2.bos.lycos.com
User-agent:bos-catalog4.bos.lycos.com   
User-agent:bos-fe1-1.bos.lycos.com
User-agent:bos-fe11-1.bos.lycos.com
User-agent:bos-fe12-1.bos.lycos.com
User-agent:bos-fe13-1.bos.lycos.com   
User-agent:bos-fe2-1.bos.lycos.com
User-agent:bos-fe3-1.bos.lycos.com
User-agent:bos-fe4-1.bos.lycos.com
User-agent:bos-fe5-1.bos.lycos.com
User-agent:bos-fe6-1.bos.lycos.com
User-agent:bos-fe6-4.bos.lycos.com   
User-agent:bos-fe8-1.bos.lycos.com
User-agent:bos-fe8-1.bos.lycos.com
User-agent:bos-fe9-1.bos.lycos.com   
User-agent:bos-spider1.bos.lycos.com   
User-agent:bos-spider10b.bos.lycos.com   
User-agent:bos-spider11b.bos.lycos.com
User-agent:bos-spider2.bos.lycos.com   
User-agent:bos-spider3.bos.lycos.com   
User-agent:bos-spider4.bos.lycos.com   
User-agent:bos-spider5.bos.lycos.com   
User-agent:bos-spider6.bos.lycos.com   
User-agent:bos-spider7.bos.lycos.com
User-agent:bos-spider9b.bos.lycos.com
User-agent:BoxSeaBot/0.5 (http://boxsea.com/crawler)
User-agent:bright.net caching robot 
User-agent:brillo.av.pa-x.dec.com Scooter/2.0 G.R.A.B. V1.1.0
User-agent:BSpider 
User-agent:c1.googlebot.com
User-agent:c2.googlebot.com
User-agent:c3.googlebot.com
User-agent:CACTVS Chemistry Spider 
User-agent:Calif/0.6 (kosarev@tnps.net; http://www.tnps.dp.ua)
User-agent:CaliforniaBrownSpider
User-agent:Cassandra 
User-agent:Checkbot/x.xx LWP/5.x
User-agent:ChristCrawler.com 
User-agent:churl 
User-agent:cIeNcIaFiCcIoN.nEt 
User-agent:CMC/0.01 
User-agent:Collective 
User-agent:Combine System 
User-agent:ComputingSite Robi/1.0
User-agent:Conceptbot 
User-agent:Confuzzledbot/X.X (+http://www.confuzzled.lu/bot/)
User-agent:CoolBot 
User-agent:C-PBWF-ip3000.com-crawler
User-agent:crawl1.googlebot.com
User-agent:crawler1.bos2.fast-search.net
User-agent:crawler1.googlebot.com
User-agent:crawler2.bos2.fast-search.net
User-agent:crawler2.googlebot.com
User-agent:crawler3.bos2.fast-search.net
User-agent:crawler4.bos2.fast-search.net
User-agent:crawler5.bos2.fast-search.net
User-agent:Cusco 
User-agent:CyberSpyder
User-agent:CyberSpyder (amant@www.cyberspyder.com)
User-agent:CyberSpyder Link Test 
User-agent:DaviesBot
User-agent:DeepIndex
User-agent:deepweb
User-agent:DeWeb(c) Katalog/Index 
User-agent:Die Blinde Kuh
User-agent:DienstSpider 
User-agent:Digger 
User-agent:Digimarc MarcSpider
User-agent:Digimarc Marcspider/CGI 
User-agent:Digital Integrity Robot 
User-agent:DigOut4U
User-agent:DIIbot
User-agent:Direct Hit Grabber 
User-agent:DirectHitGrabber
User-agent:DISCoFinder
User-agent:DittoSpyder
User-agent:DNAbot 
User-agent:DownLoad Express 
User-agent:DragonBot 
User-agent:DWCP (Dridus' Web Cataloging Project) 
User-agent:EbiNess 
User-agent:Echo2
User-agent:e-collector 
User-agent:EI*Net/0.1  libwww/0.1
User-agent:EIT Link Verifier Robot 
User-agent:ELFINBOT 
User-agent:Emacs-w3 Search Engine 
User-agent:EmailWolf
User-agent:EnigmaBot
User-agent:e-sense
User-agent:Esther 
User-agent:Eule-Robot
User-agent:Evliya Celebi 
User-agent:Excalibur Internet Spider
User-agent:EZResult
User-agent:Fast
User-agent:FastCrawler 
User-agent:FAST-WebCrawler/3.7/FirstPage 
User-agent:Felix IDE 
User-agent:FetchRover 
User-agent:fido 
User-agent:fido/1.0 Harvest/1.4.pl2
User-agent:Fish search 
User-agent:Fluffy the spider
User-agent:Fluid Dynamics Search Engine robot 
User-agent:Fouineur 
User-agent:Freecrawl
User-agent:FunnelWeb 
User-agent:GAIS Robot/1.0B2
User-agent:gammaSpider, FocusedCrawler 
User-agent:gazz 
User-agent:GCreep 
User-agent:geckobot
User-agent:GetBot 
User-agent:GetterroboPlus Puu
User-agent:GetURL 
User-agent:GigaBaz
User-agent:GigaBazVStheWeb
User-agent:Giskard
User-agent:Golem 
User-agent:Googlebot
User-agent:Googlebot/2.1
User-agent:Googlebot/2.1d
User-agent:Googlebot/2.1w
User-agent:Googlebot-Image1.0
User-agent:Googlebot-w/2.1
User-agent:Grapnel/0.01 Experiment 
User-agent:griffon
User-agent:Gromit 
User-agent:grub-client
User-agent:Gulliver
User-agent:Gulliver/1.1
User-agent:Gulliver/1.2
User-agent:Gulliver/1.3
User-agent:Gulper Bot 
User-agent:Hämähäkki 
User-agent:HamBot 
User-agent:HappyBot
User-agent:HappyBot (gserver.kw.net)
User-agent:Harvest 
User-agent:havIndex 
User-agent:HenryTheMiragoRobot
User-agent:heraSpider
User-agent:heraSpider
User-agent:HI (HTML Index) Search 
User-agent:HKU WWW Octopus 
User-agent:Hometown Spider Pro 
User-agent:ht://Dig 
User-agent:html_analyzer
User-agent:HTMLgobble 
User-agent:Hubater
User-agent:Hyper-Decontextualizer 
User-agent:I, Robot 
User-agent:ia_archiver
User-agent:iajaBot 
User-agent:IBM_Planetwide 
User-agent:Ibot/1.0 libwww-perl/0.40    
User-agent:icreport.lycos.com
User-agent:IlTrovatore-Setaccio
User-agent:image.kapsi.net 
User-agent:Imagelock 
User-agent:IncyWincy
User-agent:infoGIST
User-agent:Informant 
User-agent:InfoSeek Robot 1.0 
User-agent:Infoseek Sidewinder 
User-agent:Infoseek Sidewinder/0.9
User-agent:InfoSpiders 
User-agent:Ingrid 
User-agent:Inktomi Slurp
User-agent:Inspector Web 
User-agent:IntelliAgent 
User-agent:InterGO
User-agent:Internet Cruiser Robot 
User-agent:Internet Ninja
User-agent:Internet Shinchakubin 
User-agent:InternetAmi IOR
User-agent:InternetArchive
User-agent:InternetSeer
User-agent:ip3000.com-crawler
User-agent:ipiumBot
User-agent:Iron33
User-agent:Israeli-search 
User-agent:ITI Spider
User-agent:JavaBee 
User-agent:JBot Java Web Robot 
User-agent:JCrawler 
User-agent:JoBo Java Web Robot 
User-agent:Jobot 
User-agent:JoeBot 
User-agent:JumpStation 
User-agent:Katipo 
User-agent:Katriona
User-agent:KDD-Explorer 
User-agent:Kenjin Spider
User-agent:Kilroy 
User-agent:KIT_Fireball/2.0
User-agent:KIT-Fireball 
User-agent:KIT-Fireball/2.0
User-agent:KO_Yappo_Robot 
User-agent:Konqueror/1.1.1
User-agent:LabelGrabber 
User-agent:larbin 
User-agent:legs 
User-agent:LEIA
User-agent:LexiBot 
User-agent:Libertech-Rover
User-agent:Link Validator 
User-agent:linklooker
User-agent:linklooker
User-agent:LinkScan 
User-agent:LinkWalker 
User-agent:LNSpiderguy
User-agent:Lockon 
User-agent:logikabot
User-agent:logo.gif Crawler 
User-agent:lwp-trivial/1.27
User-agent:Lycos 
User-agent:Lycos_Spider_(T-Rex)
User-agent:Lycos_Spider_(T-Rex)/3.0 
User-agent:lycosidae.lycos.com
User-agent:lycosinc.NorthRoyalton.cw.net Mozilla/4.0
User-agent:Mac WWWWorm 
User-agent:Magpie 
User-agent:MantraAgent
User-agent:marvin/infoseek 
User-agent:Mata Hari
User-agent:Mattie 
User-agent:MediaFox 
User-agent:Mercator
User-agent:Mercator-1.1
User-agent:Mercator-1.2
User-agent:Merritt/1.0
User-agent:MerzScope 
User-agent:metabot
User-agent:MindCrawler 
User-agent:Mister Jack
User-agent:Mister Pix II
User-agent:mnoGoSearch search engine software 
User-agent:moget 
User-agent:moget/2.0
User-agent:MOMspider 
User-agent:Monster 
User-agent:MOSES 2.0 Spider
User-agent:Motor 
User-agent:Mozilla/2.0 (compatible; EZResult -- Internet Search Engine)
User-agent:Mozilla/2.0 (compatible; T-H-U-N-D-E-R-S-T-O-N-E)
User-agent:Mozilla/3.0 (compatible; MuscatFerret/1.4.1; olly@muscat.co.uk) 
User-agent:Mozilla/3.0 (compatible; MuscatFerret/1.5.2; olly@muscat.co.uk) 
User-agent:Mozilla/3.0 (compatible; MuscatFerret/1.5.3; olly@muscat.co.uk) 
User-agent:Mozilla/3.01 (Win95; I)
User-agent:Mozilla/4.0
User-agent:Mozilla/4.04 [de] (Win95; I ;Kolibri gncwebbot) 
User-agent:Mozilla/4.04 [de] (Win95; I ;Nav; Kolibri gncwebbot) 
User-agent:MSIECrawler
User-agent:msnbot
User-agent:MSProxy
User-agent:Muncher 
User-agent:Muscat Ferret 
User-agent:Mwd.Search 
User-agent:MyCNNSpider
User-agent:MyCNNSpider
User-agent:NEC-MeshExplorer 
User-agent:Nederland.zoek
User-agent:NetCarta WebMap Engine 
User-agent:NetCruiser 
User-agent:NetMechanic 
User-agent:NetResearchServer
User-agent:NetScoop 
User-agent:NetZippy
User-agent:newscan-online 
User-agent:News-gathering agent 
User-agent:NHSE Web Forager 
User-agent:Nomad 
User-agent:NPBot 
User-agent:NZBot
User-agent:nzexplorer 
User-agent:Occam 
User-agent:Offline Explorer
User-agent:OGspider
User-agent:Open Text Index Robot
User-agent:Opencola
User-agent:Openfind data gatherer 
User-agent:ORA_checksite
User-agent:Orb Search 
User-agent:Oxxbot1
User-agent:Pack Rat 
User-agent:PageBoy 
User-agent:ParaSite 
User-agent:Patric 
User-agent:pegasus
User-agent:PerlCrawler 1.0
User-agent:Phantom
User-agent:Phoaks
User-agent:PhpDig
User-agent:phpMySearch-Crawler
User-agent:PICgrabber
User-agent:PictureOfInternet www.malfunction.org/poi 
User-agent:PiltdownMan
User-agent:Pimptrain.com's robot
User-agent:PintaSpider
User-agent:Pioneer
User-agent:Pita (Chub.Stanford.EDU)
User-agent:PitSpyder Thread<n>0
User-agent:PlumtreeWebAccessor
User-agent:PolyBot
User-agent:Pompos
User-agent:Poppi
User-agent:Popular Iconoclast 
User-agent:Portal Juice Spider
User-agent:PortalB Spider
User-agent:processor/0.0ALPHA libwww-perl/0.20
User-agent:psbot
User-agent:PureSight
User-agent:RaBot
User-agent:Raven Search
User-agent:RBSE Spider
User-agent:recrawler3.bos2.fastsearch.net
User-agent:recrawler4.bos2.fastsearch.net
User-agent:res5.eng.pgh.lycos.com
User-agent:Resume Robot
User-agent:Road Runner: The ImageScape Robot
User-agent:RoadHouse Crawling System
User-agent:Robbie the Robot
User-agent:RoboCrawl Spider
User-agent:RoboFox
User-agent:Robot Francoroute 
User-agent:Robot2.0(PingSoft)
User-agent:Robozilla
User-agent:Roverbot
User-agent:RuLeS
User-agent:Rumours-Agent
User-agent:SafetyNet Robot
User-agent:SandBot
User-agent:Scooter/1.0
User-agent:Scooter/1.1 (custom) AltaVista 
User-agent:Scooter/2.0 G.R.A.B. V1.1.0
User-agent:Scooter/2.0 G.R.A.B. V1.1.0 AltaVista 
User-agent:Scooter/2.0 G.R.A.B. X2.0 AltaVista 
User-agent:Scooter_Merc_lk-1.0
User-agent:Scooter2_Mercator_lcb-1.0
User-agent:Scooter-3.0.g12-23
User-agent:scoutmaster
User-agent:search.at V1.2 search.at 
User-agent:Search.Aus-AU.COM
User-agent:SearchProcess
User-agent:Senrigan
User-agent:sexsearcher Sexsearcher 
User-agent:SG-Scout
User-agent:ShagSeeker
User-agent:Shai'Hulud
User-agent:Sift
User-agent:SilentSurf
User-agent:Simmany Robot Ver1.0
User-agent:Site Searcher
User-agent:Site Valet
User-agent:SiteTech-Rover
User-agent:sjc-accipiter1f.sjc.lycos.com
User-agent:sjc-fe10-1.sjc.lycos.com   
User-agent:sjc-fe12-1.sjc.lycos.com   
User-agent:sjc-fe13-1.sjc.lycos.com   
User-agent:sjc-fe2-2.sjc.lycos.com
User-agent:sjc-fe3-1.sjc.lycos.com   
User-agent:sjc-fe3-2.sjc.lycos.com
User-agent:sjc-fe4-1.sjc.lycos.com
User-agent:sjc-fe4-2.sjc.lycos.com
User-agent:sjc-fe5-1.sjc.lycos.com 
User-agent:sjc-fe5-2.sjc.lycos.com
User-agent:sjc-fe8-1.sjc.lycos.com
User-agent:sjc-fe9-1.sjc.lycos.com   
User-agent:Skymob.com
User-agent:SLCrawler
User-agent:Sleek
User-agent:SlowBot
User-agent:Slurp
User-agent:Slurp.so/1.0 
User-agent:Slurp.so/1.0 (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
User-agent:Slurp/2.0
User-agent:Slurp/2.0 (slurp@inktomi.com; http://www.inktomi.com/slurp.html)
User-agent:Slurp/2.0-KiteHourly
User-agent:Slurp/2.0-OwlWeekly
User-agent:Slurp/3.0-AU
User-agent:Slurp/si
User-agent:SlySearch
User-agent:Smart Spider
User-agent:Snooper
User-agent:Solbot
User-agent:SpaceBison
User-agent:speedfind ramBot xtreme 8.1
User-agent:Speedy Spider
User-agent:spider_monkey
User-agent:spider4.srv.pgh.lycos.com
User-agent:SpiderBot
User-agent:Spiderline Crawler
User-agent:SpiderMan
User-agent:SpiderView(tm)
User-agent:SpotOn
User-agent:Spry Wizard Robot
User-agent:spyder3.microsys.com
User-agent:SQ Webscanner
User-agent:Squid
User-agent:Sqworm
User-agent:srd06.ma.lycos.com   
User-agent:srd07.ma.lycos.com   
User-agent:srd08.ma.lycos.com   
User-agent:srd09.ma.lycos.com   
User-agent:srd10.ma.lycos.com   
User-agent:StatFetcher/1.0
User-agent:StatFetcher/1.0
User-agent:Suchknecht(Der_Knecht)
User-agent:Suchknecht.at/Spider
User-agent:Suke
User-agent:SummyCrawler
User-agent:suntek search engine
User-agent:Surfbot
User-agent:SurfControl
User-agent:Surfnomore Spider v1.1
User-agent:Sven
User-agent:SwissSearch V1.2 
User-agent:TACH Black Widow
User-agent:Tarantula
User-agent:tarspider
User-agent:TaWWWantula
User-agent:Tcl W3 Robot
User-agent:TeacherSoft/1.0  libwww/2.17
User-agent:TechBOT
User-agent:Templeton
User-agent:TeraCrawl
User-agent:The Informant
User-agent:The Jubii Indexing Robot 
User-agent:The NorthStar Robot 
User-agent:The NWI Robot
User-agent:The Peregrinator
User-agent:The Python Robot
User-agent:The TkWWW Robot
User-agent:The Web Moose
User-agent:The Web Wombat
User-agent:The Webfoot Robot
User-agent:the World Wide Web Wanderer
User-agent:The World Wide Web Worm
User-agent:TITAN
User-agent:TitIn
User-agent:TLSpider
User-agent:Toutatis 2.5-2
User-agent:Turnitin
User-agent:TurnitinBot
User-agent:UCmore
User-agent:UCSD Crawl
User-agent:UdmSearch
User-agent:UdmSearch
User-agent:unnamed
User-agent:URL Check
User-agent:URL Spider Pro
User-agent:URLBlaze
User-agent:UrlScope 
User-agent:User-agent:vscooter.av.pa-x.dec.com vscooter/2.0 G.R.A.B. V1.1.0
User-agent:UtilMind HTTPGet
User-agent:Vagabondo
User-agent:Valkyrie
User-agent:VCI WebViewer
User-agent:Verticrawl
User-agent:Victoria
User-agent:vision-search
User-agent:Voilabot
User-agent:Voyager
User-agent:vscooter/2.0 G.R.A.B. V1.1.0
User-agent:vspider
User-agent:VWbot
User-agent:w@pSpider by wap4.com
User-agent:w3index
User-agent:w3index
User-agent:W3M2
User-agent:w3mir
User-agent:Walhello appie
User-agent:WallPaper
User-agent:WAVETools
User-agent:Web Core / Roots 
User-agent:Web Hound
User-agent:Web Magnet
User-agent:Webbandit
User-agent:WebBandit Web Spider
User-agent:Webbase
User-agent:WebCatcher
User-agent:Webclipping.com
User-agent:webcollage
User-agent:WebCompass
User-agent:WebCopy
User-agent:WebCrawler/3.0 Robot libwww/5.0a WebCrawler 
User-agent:WebCrawler-AddURL/2.0 WebCrawler 
User-agent:webfetcher
User-agent:WebGenie
User-agent:Webinator
User-agent:weblayers
User-agent:WebLinker
User-agent:Weblog Monitor
User-agent:WebMiner
User-agent:WebMirror
User-agent:Webpush
User-agent:WebQuest
User-agent:webrank
User-agent:WebReaper
User-agent:webs
User-agent:Website eXtractor
User-agent:Websnarf
User-agent:WebSpider
User-agent:WebStolperer
User-agent:WebSymmetrix
User-agent:WebVac
User-agent:webwalk
User-agent:WebWalker
User-agent:webwasher
User-agent:WebWatch
User-agent:WebZinger
User-agent:Wget
User-agent:whatUseek Winona
User-agent:WhizBang! Lab
User-agent:WhosTalking
User-agent:WhoWhere Robot
User-agent:Wild Ferret Web Hopper #1, #2, #3 
User-agent:WIRE WebRefiner:
User-agent:Wired Digital 
User-agent:WiseWire WiseWire 
User-agent:WiseWire-Alpha-1.0 WiseWire 
User-agent:WiseWire-Alpha12-Spider971219a WiseWire 
User-agent:WiseWire-Alpha-Spider WiseWire 
User-agent:WiseWire-HotSpider-1.0 WiseWire 
User-agent:WiseWire-Spider WiseWire 
User-agent:WiseWire-Spider-1.0 WiseWire 
User-agent:WiseWire-Spider2 WiseWire 
User-agent:WiseWire-Widow-1.0 WiseWire 
User-agent:WiseWire-Widow-1.0-ALPHA12 
User-agent:WiseWire-Widow-1.0r WiseWire 
User-agent:wobot/1.0
User-agent:wobot/1.0 from 206.214.202.45
User-agent:WSCbot
User-agent:WWW Collector
User-agent:WWWC Ver 0.2.5
User-agent:www-jp.lycos.com
User-agent:Xaldon WebSpider
User-agent:XGET
User-agent:XupiterToolbar
User-agent:XYLEME Robot 
User-agent:YahooFeedSeeker/1.0 
User-agent:Yahoo-MMCrawler/3.x
User-agent:Yahoo-VerticalCrawler-FormerWebCrawler/3.9 
User-agent:Yandex
User-agent:Yellopet-Spider
User-agent:zebra.eng.pgh.lycos.com
User-agent:ZyBorg
Disallow:/mail/
Disallow:/cgi-bin/
Disallow:/stats/