User-agent: 12soso 
User-agent: 192.comAgent 
User-agent: 1Noonbot 
User-agent: 1on1searchBot 
User-agent: 3D_SEARCH 
User-agent: 3DE_SEARCH2 
User-agent: 3GSE 
User-agent: 50.nu 
User-agent: A1 Sitemap Generator 
User-agent: A1 Website Download 
User-agent: A6-Indexer 
User-agent: AASP 
User-agent: ABACHOBot 
User-agent: Abonti 
User-agent: abot 
User-agent: AbotEmailSearch 
User-agent: Aboundex/0.3 
User-agent: AboutUsBot 
User-agent: AccMonitor Compliance Server 
User-agent: accoona 
User-agent: AChulkov.NET page walker 
User-agent: Acme.Spider 
User-agent: AcoonBot 
User-agent: acquia-crawler 
User-agent: ActiveTouristBot 
User-agent: Ad Muncher 
User-agent: AdamM Bot 
User-agent: adbeat_bot 
User-agent: adminshop.com 
User-agent: Advanced Email Extractor 
User-agent: AESOP_com_SpiderMan 
User-agent: AESpider 
User-agent: AF Knowledge Now Verity Spider 
User-agent: aggregator:Vocus 
User-agent: ah-ha.com crawler 
User-agent: AhrefsBot
User-agent: AIBOT 
User-agent: aiHitBot 
User-agent: aipbot 
User-agent: AISIID 
User-agent: AITCSRobot/1.1 
User-agent: Akamai-SiteSnapshot 
User-agent: AlexaWebSearchPlatform 
User-agent: AlexfDownload 
User-agent: Alexibot 
User-agent: AlkalineBOT 
User-agent: All Acronyms Bot 
User-agent: Alpha Search Agent 
User-agent: Amerla Search Bot 
User-agent: Amfibibot 
User-agent: AmPmPPC.com 
User-agent: AMZNKAssocBot 
User-agent: Anemone 
User-agent: Anonymous/3G bot 
User-agent: Anonymouse.org 
User-agent: AnotherBot 
User-agent: AnswerBot 
User-agent: AnswerBus 
User-agent: AnswerChase PROve 
User-agent: AntBot 
User-agent: antibot- 
User-agent: AntiSantyWorm 
User-agent: Antro.Net 
User-agent: AONDE-Spider 
User-agent: Aport 
User-agent: appid: s~stremor-crawler- 
User-agent: Aqua_Products 
User-agent: AraBot 
User-agent: Arachmo 
User-agent: Arachnophilia 
User-agent: aria eQualizer 
User-agent: arianna.libero.it 
User-agent: Arikus_Spider 
User-agent: ArtavisBot 
User-agent: Artera 
User-agent: Art-Online.com 
User-agent: ASAHA Search Engine Turkey 
User-agent: ASpider/0.09 
User-agent: ASPSeek 
User-agent: asterias 
User-agent: AstroFind 
User-agent: athenusbot 
User-agent: AtlocalBot 
User-agent: Atomic_Email_Hunter 
User-agent: attach 
User-agent: attrakt 
User-agent: attributor 
User-agent: Attributor.comBot 
User-agent: Attributor/Dejan- 
User-agent: augurfind 
User-agent: AURESYS 
User-agent: AutoBaron crawler 
User-agent: autoemailspider 
User-agent: autowebdir 
User-agent: AVSearch- 
User-agent: axfeedsbot 
User-agent: Axonize-bot 
User-agent: Ayna 
User-agent: b2w 
User-agent: b2w/0.1 
User-agent: BackDoorBot 
User-agent: BackRub/ 
User-agent: BackStreet Browser 
User-agent: BackWeb 
User-agent: Bandit 
User-agent: BatchFTP 
User-agent: baypup 
User-agent: BDFetch 
User-agent: BecomeBot 
User-agent: BecomeJPBot 
User-agent: BeetleBot 
User-agent: Bender 
User-agent: besserscheitern-crawl 
User-agent: betaBot 
User-agent: Big Brother 
User-agent: Big Data 
User-agent: Bigado.com 
User-agent: BigCliqueBot 
User-agent: Bigfoot 
User-agent: BIGLOTRON 
User-agent: Bilbo 
User-agent: BilgiBetaBot 
User-agent: BilgiBot 
User-agent: bintellibot 
User-agent: bitlybot 
User-agent: BitvoUserAgent 
User-agent: Bizbot003 
User-agent: BizBot04 kirk.overleaf.com 
User-agent: BizWorks Retriever 
User-agent: Black Hole 
User-agent: Blackbird 
User-agent: BlackMask.Net Search Engine 
User-agent: BlackWidow 
User-agent: bladder fusion 
User-agent: Blaiz-Bee 
User-agent: BLEXBot 
User-agent: Blinkx/DFS-Fetch 
User-agent: BlitzBOT 
User-agent: Blog Conversation Project 
User-agent: BlogMyWay 
User-agent: BlogPulseLive 
User-agent: BlogRefsBot 
User-agent: BlogScope 
User-agent: Blogslive 
User-agent: BloobyBot 
User-agent: BlowFish 
User-agent: BLT 
User-agent: bnf.fr_bot 
User-agent: BoaConstrictor 
User-agent: BoardReader-Image-Fetcher 
User-agent: BOI_crawl_00 
User-agent: BOIA.ORG-Scan-Agent 
User-agent: BOIA-Scan-Agent 
User-agent: boitho.com-dc 
User-agent: Bookmark Buddy bookmark checker 
User-agent: Bookmark search tool 
User-agent: bosug 
User-agent: Bot 
User-agent: Bot Apoena 
User-agent: BotALot 
User-agent: BotRightHere 
User-agent: Botswana 
User-agent: bottybot 
User-agent: BpBot 
User-agent: BRAINTIME_SEARCH 
User-agent: BrokenLinkCheck.com 
User-agent: BrowserEmulator 
User-agent: BrowserMob 
User-agent: BruinBot 
User-agent: BSearchR&D 
User-agent: BSpider 
User-agent: btbot 
User-agent: Btsearch 
User-agent: Buddy 
User-agent: Buibui 
User-agent: BuildCMS crawler 
User-agent: BuiltBotTough 
User-agent: Bullseye 
User-agent: bumblebee 
User-agent: BunnySlippers 
User-agent: BuscadorClarin 
User-agent: Buscaplus Robi 
User-agent: Butterfly 
User-agent: BuyHawaiiBot 
User-agent: BuzzBot 
User-agent: byindia 
User-agent: BySpider 
User-agent: byteserver 
User-agent: bzBot 
User-agent: c r a w l 3 r 
User-agent: CacheBlaster 
User-agent: Caddbot 
User-agent: Cafi 
User-agent: Camcrawler 
User-agent: CamelStampede 
User-agent: Canon-WebRecord 
User-agent: Canon-WebRecordPro 
User-agent: CareerBot 
User-agent: cataguru 
User-agent: CatchBot 
User-agent: CazoodleBot 
User-agent: CCBot 
User-agent: CCGCrawl 
User-agent: ccubee 
User-agent: CD-Preload 
User-agent: Cegbfeieh 
User-agent: CE-Preload 
User-agent: Cerberian Drtrs 
User-agent: CERT FigleafBot 
User-agent: cfetch 
User-agent: CFNetwork 
User-agent: Chameleon 
User-agent: Charlotte 
User-agent: Check&Get 
User-agent: Checkbot 
User-agent: CheckLinks 
User-agent: CheeseBot 
User-agent: ChemieDE-NodeBot 
User-agent: CherryPicker 
User-agent: CherryPickerElite 
User-agent: CherryPickerSE 
User-agent: Chilkat 
User-agent: ChinaClaw 
User-agent: CipinetBot 
User-agent: cis455crawler 
User-agent: citeseerxbot 
User-agent: cizilla.com/Cizilla- 
User-agent: ClariaBot 
User-agent: Clushbot 
User-agent: COAST scan engine 
User-agent: COAST WebMaster Pro 
User-agent: coccoc 
User-agent: CollapsarWEB 
User-agent: Collector 
User-agent: Combine 
User-agent: Compatible 
User-agent: CompSpyBot
User-agent: ConnectSearch 
User-agent: conpilot 
User-agent: ContentSmartz 
User-agent: ContextAd Bot 
User-agent: contype 
User-agent: cookieNET 
User-agent: CoolBot  
User-agent: CoolCheck 
User-agent: Copernic 
User-agent: Copier 
User-agent: CopyRightCheck 
User-agent: core-project 
User-agent: cosmos 
User-agent: Covario-IDS 
User-agent: Cowbot- 
User-agent: Cowdog Bot 
User-agent: crabbyBot 
User-agent: Crawl 
User-agent: crawl.UserAgent 
User-agent: Crawl_Application 
User-agent: CrawlConvera 
User-agent: Crawler 
User-agent: crawler.kpricorn.org 
User-agent: crawler@ 
User-agent: crawler_for_infomine 
User-agent: crawler43.ejupiter.com 
User-agent: crawler4j 
User-agent: CRAWLER-ALTSE.VUNET.ORG-Lynx 
User-agent: crawler-upgrade-config 
User-agent: Crawly 
User-agent: CreativeCommons 
User-agent: Crescent 
User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0 
User-agent: cs-crawler 
User-agent: CSE HTML Validator 
User-agent: CSHttpClient 
User-agent: Cuasarbot 
User-agent: culsearch/culs 
User-agent: Curious George
User-agent: Curl 
User-agent: Custo 
User-agent: Cutbot 
User-agent: cvaulev 
User-agent: Cyberdog 
User-agent: CyberNavi_WebGet 
User-agent: CyberPatrol SiteCat Webbot 
User-agent: CyberSpyder 
User-agent: CybEye.com
User-agent: CydralSpider 
User-agent: D1GArabicEngine 
User-agent: DA 
User-agent: DataCha0s 
User-agent: DataFountains 
User-agent: DataFountains/DMOZ Downloader 
User-agent: DataparkSearch 
User-agent: datascape robot 
User-agent: DataSpearSpiderBot 
User-agent: DataSpider 
User-agent: Dattatec.com 
User-agent: Dattatec.com-Sitios-Top 
User-agent: Daumoa 
User-agent: DAUMOA-video 
User-agent: DAUMOA-web 
User-agent: Declumbot 
User-agent: deepcrawl
User-agent: Deepindex 
User-agent: deepnet crawler 
User-agent: DeepTrawl 
User-agent: dejan 
User-agent: del.icio.us-thumbnails 
User-agent: DelvuBot 
User-agent: Der große BilderSauger 
User-agent: DiaGem 
User-agent: Diamond 
User-agent: DiamondBot 
User-agent: DiBot 
User-agent: didaxusbot 
User-agent: DigExt 
User-agent: Digger 
User-agent: DiGi-RSSBot 
User-agent: DigitalArchivesBot 
User-agent: DigOut4U 
User-agent: DIIbot 
User-agent: Dillo 
User-agent: Dir_Snatch.exe 
User-agent: DISCo 
User-agent: Distilled-Reputation-Monitor 
User-agent: Dit 
User-agent: DittoSpyder 
User-agent: DjangoTraineeBot 
User-agent: DKIMRepBot 
User-agent: DOC
User-agent: DoCoMo
User-agent: DOF-Verify 
User-agent: Domaincrawler 
User-agent: DomainScan 
User-agent: DomainWatcher Bot 
User-agent: Dotbot
User-agent: DotSpotsBot 
User-agent: Dow Jones Searchbot 
User-agent: Download 
User-agent: Download Ninja
User-agent: Downloader 
User-agent: DOY 
User-agent: dragonfly 
User-agent: Drip 
User-agent: drone 
User-agent: DTAAgent 
User-agent: dtSearchSpider 
User-agent: dumbot 
User-agent: Dwaar 
User-agent: Dwaarbot 
User-agent: DXSeeker 
User-agent: EAH 
User-agent: EasouSpider 
User-agent: EasyDL 
User-agent: ebingbong 
User-agent: EC2LinkFinder 
User-agent: eCairn-Grabber 
User-agent: eCatch 
User-agent: eChooseBot 
User-agent: EdisterBot (http://www.edister.com/bot.html) 
User-agent: EduGovSearch 
User-agent: egothor 
User-agent: eidetica.com/spider 
User-agent: EirGrabber 
User-agent: Elblindo the Blind Bot 
User-agent: ElisaBot 
User-agent: EllerdaleBot 
User-agent: EMail Exractor 
User-agent: EmailCollector 
User-agent: EmailLeach 
User-agent: EmailSiphon 
User-agent: EmailWolf 
User-agent: EMPAS_ROBOT 
User-agent: EnaBot 
User-agent: endeca 
User-agent: EnigmaBot 
User-agent: Enswer Neuro Bot 
User-agent: EntityCubeBot 
User-agent: EroCrawler 
User-agent: es 
User-agent: eStyleSearch 
User-agent: eSyndiCat Bot 
User-agent: Eurosoft-Bot 
User-agent: Evaal 
User-agent: Eventware 
User-agent: Everest-Vulcan Inc 
User-agent: Exabot 
User-agent: Exabot Test 
User-agent: Exabot-Images 
User-agent: ExaBotTest 
User-agent: Exabot-Test 
User-agent: Exabot-XXX 
User-agent: ExactSearch 
User-agent: exactseek.com 
User-agent: ExB Language Crawler
User-agent: exooba/exooba crawler 
User-agent: Exploder 
User-agent: Express WebPictures 
User-agent: Extractor 
User-agent: EyeNetIE 
User-agent: Ezooms 
User-agent: ez-robot 
User-agent: factbot 
User-agent: FairAd Client 
User-agent: falcon 
User-agent: Falconsbot 
User-agent: FAST Data Search Document Retriever 
User-agent: FAST ESP 
User-agent: fastbot crawler 
User-agent: fastbot.de crawler 
User-agent: fast-search-engine 
User-agent: FatBot 
User-agent: Favcollector 
User-agent: Faviconizer 
User-agent: Favorites Sweeper 
User-agent: f-bot test pilot 
User-agent: FDM 
User-agent: FDSE robot 
User-agent: FedContractorBot 
User-agent: fembot 
User-agent: Fetch
User-agent: Fetch API Request 
User-agent: fetch_ici 
User-agent: fgcrawler 
User-agent: Filangy 
User-agent: FileHound 
User-agent: FindAnISP.com_ISP_Finder 
User-agent: findlinks 
User-agent: FindWeb 
User-agent: Firebat 
User-agent: FirstGov.gov Search 
User-agent: Flaming AttackBot 
User-agent: Flamingo_SearchEngine
User-agent: Flamingo_SearchEngine 
User-agent: FlashCapture 
User-agent: FlashGet 
User-agent: FlashGet WebWasher 3.2 
User-agent: FlickySearchBot 
User-agent: Fluffy the spider 
User-agent: flunky 
User-agent: focused_crawler 
User-agent: FollowSite 
User-agent: Foobot 
User-agent: Fooooo_Web_Video_Crawl 
User-agent: Fopper 
User-agent: FormulaFinderBot 
User-agent: Forschungsportal 
User-agent: fr_crawler 
User-agent: Francis 
User-agent: FreeWebMonitoring SiteChecker 
User-agent: FreshDownload 
User-agent: freshlinks.exe 
User-agent: FriendFeedBot 
User-agent: frodo.at 
User-agent: froGgle 
User-agent: FrontPage 
User-agent: Froola Bot 
User-agent: full_breadth_crawler 
User-agent: FU-NBI/FU-NBI- 
User-agent: FunnelBack 
User-agent: FurlBot 
User-agent: G10-Bot 
User-agent: Gaisbot 
User-agent: GalaxyBot 
User-agent: gazz 
User-agent: generate_infomine_category_classifiers 
User-agent: genevabot 
User-agent: genieBot 
User-agent: GenieBotRD_SmallCrawl 
User-agent: Genieo
User-agent: Genio
User-agent: Geomaxenginebot 
User-agent: geometabot 
User-agent: GeonaBot 
User-agent: GeoVisu 
User-agent: GermCrawler 
User-agent: GetHTMLContents 
User-agent: Getleft 
User-agent: GetRight 
User-agent: GetSmart 
User-agent: GetURL.rexx v1.05 
User-agent: GetWeb! 
User-agent: Giant 
User-agent: Gib
User-agent: GigablastOpenSource 
User-agent: Gigabot 
User-agent: Girafabot 
User-agent: GleameBot 
User-agent: gnome-vfs 
User-agent: Go!Zilla 
User-agent: Go-Ahead-Got-It 
User-agent: GoForIt.com 
User-agent: GOFORITBOT 
User-agent: gold crawler 
User-agent: Goldfire Server 
User-agent: Golem/1.1 
User-agent: GoodJelly 
User-agent: Gordon-College-Google-Mini 
User-agent: goroam 
User-agent: GoSeebot 
User-agent: gotit 
User-agent: Govbot 
User-agent: GPU p2p crawler 
User-agent: Grabber 
User-agent: GrabNet 
User-agent: Grafula 
User-agent: grapeFX 
User-agent: grapeshot 
User-agent: GrapeshotCrawler/2.0 
User-agent: grbot 
User-agent: GreenYogi [ZSEBOT] 
User-agent: Gromit 
User-agent: GroupMe 
User-agent: grub 
User-agent: grub crawler 
User-agent: grub-client
User-agent: Grubclient- 
User-agent: grub-client 
User-agent: GrubNG 
User-agent: GruBot 
User-agent: gsa 
User-agent: gsa-crawler
User-agent: GSLFbot 
User-agent: Gulliver 
User-agent: GulperBot 
User-agent: GurujiBot 
User-agent: GVC BUSINESS crawler 
User-agent: GVC crawler 
User-agent: GVC SEARCH BOT 
User-agent: GVC WEB crawler 
User-agent: GVC Weblink crawler 
User-agent: GVC WORLD LINKS 
User-agent: gvcbot.com 
User-agent: HappyFunBot 
User-agent: Harvest 
User-agent: HarvestMan 
User-agent: Hatena Antenna 
User-agent: Hawler 
User-agent: hcat 
User-agent: hclsreport-crawler 
User-agent: HD nutch agent 
User-agent: Header_Test_Client 
User-agent: healia/healia 
User-agent: Helix 
User-agent: here will be link to crawler site 
User-agent: heritrix 
User-agent: hijbul-heritrix-crawler 
User-agent: HiScan 
User-agent: HiSoftware AccMonitor Server 
User-agent: HiSoftware AccVerify 
User-agent: hitcrawler_ 
User-agent: hivaBot 
User-agent: hloader 
User-agent: HMSEbot 
User-agent: HMView 
User-agent: hoge 
User-agent: holmes 
User-agent: HomePageSearch 
User-agent: Hooblybot-Image 
User-agent: HooWWWer 
User-agent: Hostcrawler 
User-agent: HSFT - Link Scanner 
User-agent: HSFT - LVU Scanner 
User-agent: HSlide 
User-agent: ht://check 
User-agent: htdig 
User-agent: Html Link Validator 
User-agent: HTMLParser 
User-agent: httplib 
User-agent: HTTrack 
User-agent: Huaweisymantecspider 
User-agent: hul-wax 
User-agent: humanlinks 
User-agent: HyperEstraier 
User-agent: Hyperix 
User-agent: IAArchiver- 
User-agent: ibuena 
User-agent: iCab 
User-agent: ICDS-Ingestion 
User-agent: ichiro 
User-agent: iCopyright Conductor 
User-agent: IEAutoDiscovery 
User-agent: IECheck 
User-agent: iHWebChecker 
User-agent: IIITBOT 
User-agent: iim_405 
User-agent: IlseBot 
User-agent: IlTrovatore 
User-agent: Iltrovatore-Setaccio 
User-agent: Image Stripper 
User-agent: Image Sucker 
User-agent: ImageBot 
User-agent: imagefortress 
User-agent: ImagesHereImagesThereImagesEverywhere 
User-agent: ImageVisu 
User-agent: imds_monitor 
User-agent: imo-google-robot-intelink 
User-agent: inagist.com url crawler 
User-agent: Industry Cortex Webcrawler 
User-agent: Indy Library 
User-agent: indylabs_marius 
User-agent: InelaBot 
User-agent: Inet32 Ctrl 
User-agent: inetbot 
User-agent: info seeker 
User-agent: InfoLink 
User-agent: INFOMINE 
User-agent: infomine.ucr.edu 
User-agent: InfoNaviRobot 
User-agent: Informant 
User-agent: Infoseek 
User-agent: InfoTekies 
User-agent: InfoUSABot 
User-agent: INGRID 
User-agent: INGRID/0.1 
User-agent: Inktomi 
User-agent: InsightsCollector 
User-agent: InsightsWorksBot 
User-agent: InspireBot 
User-agent: InsumaScout 
User-agent: Intelix 
User-agent: Intelliseek 
User-agent: InterGET 
User-agent: Internet Ninja 
User-agent: INTERNET RADIO crawler 
User-agent: InternetLinkAgent 
User-agent: Interseek 
User-agent: IOI 
User-agent: IPAdd Bot 
User-agent: IpselonBot 
User-agent: ip-web-crawler.com 
User-agent: Iria 
User-agent: IRLbot 
User-agent: Iron33/1.0.2 
User-agent: Isara/Isara- 
User-agent: iSearch 
User-agent: iSiloX 
User-agent: IstellaBot 
User-agent: its-learning crawler 
User-agent: IU_CSCI_B659_class_crawler 
User-agent: iVia 
User-agent: iVia Page Fetcher 
User-agent: iVia Site Checker 
User-agent: JadynAve 
User-agent: JadynAveBot 
User-agent: Jakarta 
User-agent: Jakarta Commons-HttpClient 
User-agent: Jbot 
User-agent: JemmaTheTourist 
User-agent: JennyBot 
User-agent: Jetbot 
User-agent: JetBrains Omea Pro 
User-agent: JetCar 
User-agent: Jim 
User-agent: JoBo 
User-agent: JobSpider_BA 
User-agent: JOC 
User-agent: JoeDog 
User-agent: JoyScapeBot 
User-agent: JSpyda 
User-agent: Junut Bot 
User-agent: JustView 
User-agent: Jyxobot 
User-agent: K.S.Bot 
User-agent: k2spider
User-agent: KakcleBot 
User-agent: kalooga 
User-agent: KaloogaBot 
User-agent: KATATUDO-Spider 
User-agent: kbeta1 
User-agent: KeepNI web site monitor 
User-agent: Kenjin.Spider 
User-agent: Keybot Translation-Search-Machine 
User-agent: KeywenBot 
User-agent: Keyword Density/0.9 
User-agent: Keyword.Density 
User-agent: kinjabot 
User-agent: Kitenga-crawler-bot 
User-agent: KiwiStatus 
User-agent: kmbot- 
User-agent: kmccrew Bot Search 
User-agent: Knight 
User-agent: KnowItAll 
User-agent: Knowledge Engine 
User-agent: Knowledge.com 
User-agent: KoepaBot 
User-agent: Koninklijke 
User-agent: KrOWLer 
User-agent: KSbot 
User-agent: kuloko-bot 
User-agent: kulturarw3 
User-agent: KummHttp 
User-agent: Kurzor 
User-agent: Kyluka crawl 
User-agent: L.webis 
User-agent: Labhoo 
User-agent: labourunions411 
User-agent: Lachesis 
User-agent: Lament 
User-agent: LamerExterminator 
User-agent: LapozzBot 
User-agent: larbin
User-agent: LARBIN-EXPERIMENTAL 
User-agent: LBot 
User-agent: LeapTag 
User-agent: LeechFTP 
User-agent: LeechGet 
User-agent: LetsCrawl.com 
User-agent: LexiBot 
User-agent: LexxeBot
User-agent: lftp 
User-agent: libcrawl
User-agent: libiViaCore 
User-agent: libWeb/clsHTTP 
User-agent: libwww
User-agent: libwww-perl 
User-agent: likse 
User-agent: Linguee Bot 
User-agent: Link Checker 
User-agent: Link Validator 
User-agent: link_checker 
User-agent: LinkAlarm 
User-agent: linkbot 
User-agent: LinkCheck by Siteimprove.com 
User-agent: LinkCheck Scanner 
User-agent: LinkChecker 
User-agent: linkdex
User-agent: linkdex.com 
User-agent: LinkextractorPro 
User-agent: LinkLint 
User-agent: linklooker 
User-agent: Linkman 
User-agent: linko
User-agent: Links SQL 
User-agent: LinkScan 
User-agent: LinkScan/8.1a.Unix 
User-agent: LinksManager.com_bot 
User-agent: LinkSweeper 
User-agent: LinkWalker 
User-agent: LiteFinder
User-agent: LitlrBot 
User-agent: Little Grabber at Skanktale.com 
User-agent: Livelapbot 
User-agent: LM Harvester 
User-agent: LMQueueBot 
User-agent: LNSpiderguy 
User-agent: LoadTimeBot 
User-agent: LocalcomBot 
User-agent: locust 
User-agent: LolongBot 
User-agent: LookBot 
User-agent: Lsearch/sondeur 
User-agent: lssbot 
User-agent: LT Scotland Checklink 
User-agent: LWNutch
User-agent: LWP/natweb-bad-link-mailer 
User-agent: LWP::Simple 
User-agent: lwp-request 
User-agent: lwp-trivial 
User-agent: Lycos_Spider 
User-agent: Lydia Entity Spider 
User-agent: LynnBot 
User-agent: Lytranslate 
User-agent: Magnet 
User-agent: Mag-Net 
User-agent: magpie-crawler
User-agent: Magus Bot 
User-agent: MAINSEEK_BOT 
User-agent: Mammoth 
User-agent: Map robot 
User-agent: MarkWatch 
User-agent: MaSagool 
User-agent: masidani_bot_ 
User-agent: Mass Downloader 
User-agent: Mata Hari 
User-agent: matentzn at cs dot man dot ac dot uk 
User-agent: maxamine.com-robot 
User-agent: maxamine.com--robot 
User-agent: maxomobot 
User-agent: McBot 
User-agent: Mediapartners-Google*
User-agent: medrabbit 
User-agent: Megite 
User-agent: MemacBot 
User-agent: Memo 
User-agent: MendeleyBot 
User-agent: Mercator- 
User-agent: mercuryboard_user_agent_sql_injection.nasl 
User-agent: metacarta 
User-agent: Metaeuro Web Search 
User-agent: Metager2 
User-agent: metager2-verification-bot 
User-agent: MetaGloss 
User-agent: metal crawler 
User-agent: metaquerier.cs.uiuc.edu 
User-agent: METASpider 
User-agent: Metaspinner 
User-agent: MetaURI 
User-agent: MetaURI API/2.0  metauri.com 
User-agent: MetaURI API/2.0 +metauri.com 
User-agent: MFcrawler 
User-agent: MFHttpScan 
User-agent: Microsoft.URL.Control
User-agent: MIDown tool 
User-agent: MIIxpc 
User-agent: MIIxpc/4.2 
User-agent: minibot 
User-agent: miniRank 
User-agent: mini-robot 
User-agent: Mirror 
User-agent: Missigua Locator 
User-agent: Mister PiX 
User-agent: Mister Pix II 2.01 
User-agent: Mister Pix II 2.02a 
User-agent: Mister PiX version.dll 
User-agent: Mister.PiX 
User-agent: Miva 
User-agent: MJ12bot
User-agent: MJ12bot/v1.4.5 
User-agent: mnoGoSearch 
User-agent: Moatbot
User-agent: mod_accessibility 
User-agent: moduna.com 
User-agent: moget 
User-agent: moget/2.1 
User-agent: MojeekBot 
User-agent: MojeekBot/0.6 
User-agent: MonkeyCrawl 
User-agent: MOSES 
User-agent: mowserbot 
User-agent: MQbot 
User-agent: MSE360 
User-agent: MSIECrawler
User-agent: MSIndianWebcrawl 
User-agent: MSMOBOT 
User-agent: MSNPTC 
User-agent: MSRBOT 
User-agent: MT-Soft 
User-agent: MultiText 
User-agent: My_Little_SearchEngine_Project 
User-agent: MyApp 
User-agent: MYCOMPANYBOT 
User-agent: mycrawler 
User-agent: MyEngines-US-Bot 
User-agent: MyFamilyBot 
User-agent: my-heritrix-crawler 
User-agent: Myra 
User-agent: nabot 
User-agent: nabot_ 
User-agent: Najdi.si 
User-agent: Nambu 
User-agent: NAMEPROTECT 
User-agent: NASA Search 
User-agent: NatchCVS 
User-agent: Navroad 
User-agent: NearSite 
User-agent: NEC-MeshExplorer 
User-agent: NeoScioCrawler 
User-agent: NerdByNature.Bot 
User-agent: NerdyBot 
User-agent: Nerima-crawl- 
User-agent: Nessus 
User-agent: NESSUS::SOAP 
User-agent: nestReader 
User-agent: Net Vampire 
User-agent: Net::Trackback 
User-agent: NetAnts 
User-agent: NetCarta CyberPilot Pro 
User-agent: Netcraft 
User-agent: NetID.com Bot 
User-agent: NetMechanic 
User-agent: Netprospector 
User-agent: NetResearchServer 
User-agent: NetSeer 
User-agent: NetSeer crawler 
User-agent: NetSeer crawler/2.0 
User-agent: NetShift= 
User-agent: NetSongBot 
User-agent: Netsparker 
User-agent: NetSpider 
User-agent: NetSrcherP 
User-agent: NetZIP 
User-agent: NewMedhunt 
User-agent: news bot  
User-agent: News_Search_App 
User-agent: NewsGatherer 
User-agent: Newsgroupreporter 
User-agent: NewsTroveBot 
User-agent: NextGenSearchBot 
User-agent: nextthing.org 
User-agent: NG 
User-agent: nicebot 
User-agent: NICErsPRO 
User-agent: niki-bot 
User-agent: NimbleCrawler 
User-agent: nimbus-1 
User-agent: ninetowns 
User-agent: Ninja 
User-agent: NjuiceBot 
User-agent: NLese 
User-agent: NLESE USEPA 
User-agent: Nogate 
User-agent: Norbert the Spider 
User-agent: NoteworthyBot 
User-agent: NPbot 
User-agent: NPBot- 
User-agent: NRCan intranet crawler 
User-agent: NSDL_Search_Bot 
User-agent: nu_tch-princeton/Nu_tch 
User-agent: nuggetize.com BOT 
User-agent: nuSearch Spider 
User-agent: Nutch
User-agent: Nutch 
User-agent: nutch1/huntsman 
User-agent: NutchCVS 
User-agent: NutchOrg 
User-agent: NWSpider 
User-agent: Nymesis 
User-agent: nys-crawler 
User-agent: ObjectsSearch 
User-agent: oBot 
User-agent: Obvius external linkcheck 
User-agent: Ocelli 
User-agent: Octopus 
User-agent: ODP entries t_st 
User-agent: oegp v 
User-agent: Offline Explorer 
User-agent: Offline Navigator 
User-agent: Offline.Explorer 
User-agent: OGspider 
User-agent: OmiExplorer_Bot 
User-agent: OmniExplorer_Bot 
User-agent: OmniFind 
User-agent: OmniWeb 
User-agent: OnetSzukaj 
User-agent: online link validator 
User-agent: OOZBOT 
User-agent: Openbot 
User-agent: Openfind 
User-agent: OpenHoseBot/2.1 
User-agent: OpenIntelligenceData 
User-agent: OpenISearch 
User-agent: OpenLink Virtuoso RDF crawler 
User-agent: OpenSearchServer_Bot 
User-agent: OpenWebIndex
User-agent: OpiDig 
User-agent: optidiscover 
User-agent: Oracle Secure Enterprise Search 
User-agent: Oracle Ultra Search 
User-agent: OrangeBot 
User-agent: ORISBot 
User-agent: ornl_crawler_1 
User-agent: ORNL_Mercury 
User-agent: osis-project.jp 
User-agent: OsO 
User-agent: OutfoxBot 
User-agent: OutfoxMelonBot 
User-agent: OWLER-BOT 
User-agent: owsBot 
User-agent: ozelot 
User-agent: P3P Client 
User-agent: page_verifier 
User-agent: PageBitesHyperBot 
User-agent: Pagebull 
User-agent: PageDown 
User-agent: PageFetcher 
User-agent: PageGrabber 
User-agent: PagePeeker 
User-agent: PageRank Monitor 
User-agent: pamsnbot.htm 
User-agent: Panopy Bot 
User-agent: panscient.com 
User-agent: Pansophica 
User-agent: Papa Foto 
User-agent: PaperLiBot 
User-agent: PaperLiBot/2.1 
User-agent: parasite 
User-agent: parsijoo 
User-agent: Pathtraq 
User-agent: Pattern 
User-agent: Patwebbot 
User-agent: pavuk 
User-agent: PaxleFramework 
User-agent: PBBOT 
User-agent: pcBrowser 
User-agent: pd-crawler 
User-agent: penthesila 
User-agent: perform_crawl 
User-agent: PerMan 
User-agent: PerMan Surfer 
User-agent: personal ultimate crawler 
User-agent: PHP version tracker 
User-agent: PHPCrawl 
User-agent: PhpDig 
User-agent: PicoSearch 
User-agent: PIENO robot 
User-agent: pipBot 
User-agent: pipeLiner 
User-agent: Pita 
User-agent: pixfinder 
User-agent: PiyushBot 
User-agent: plaNETWORK Bot Search 
User-agent: Plucker 
User-agent: Plukkie 
User-agent: Plumtree 6.0 
User-agent: Pockey 
User-agent: Pockey-GetHTML 
User-agent: PoCoHTTP 
User-agent: pogodak.ba 
User-agent: Pogodak.co.yu 
User-agent: Poirot 
User-agent: polybot 
User-agent: Pompos 
User-agent: Poodle predictor 
User-agent: PopScreenBot 
User-agent: PostPost 
User-agent: PrivacyFinder 
User-agent: ProjectWF-java-test-crawler 
User-agent: ProPowerBot/2.14 
User-agent: ProWebWalker 
User-agent: Proxem WebSearch 
User-agent: PROXY crawler 
User-agent: psbot 
User-agent: psbot-page 
User-agent: PSS-Bot 
User-agent: psycheclone 
User-agent: pub-crawler 
User-agent: pucl 
User-agent: pulseBot (pulse Web Miner) 
User-agent: Pump 
User-agent: PWeBot 
User-agent: pythonic-crawler 
User-agent: Python-urllib 
User-agent: PythonWikipediaBot 
User-agent: q1 
User-agent: QEAVis agent 
User-agent: QFKBot 
User-agent: qualidade 
User-agent: Qualidator.com Bot 
User-agent: QuepasaCreep 
User-agent: QueryN Metasearch 
User-agent: QueryN.Metasearch 
User-agent: quest.durato 
User-agent: Quintura-Crw 
User-agent: QunarBot 
User-agent: Qweery_robot.txt_CheckBot 
User-agent: QweeryBot 
User-agent: r2iBot 
User-agent: R6_CommentReader 
User-agent: R6_FeedFetcher 
User-agent: R6_VoteReader 
User-agent: RaBot 
User-agent: Radian6 
User-agent: radian6_linkcheck 
User-agent: Radiation Retriever 1.1 
User-agent: RAMPyBot 
User-agent: RankurBot/Rankur 
User-agent: Rational SiteCheck 
User-agent: RcStartBot 
User-agent: RealDownload 
User-agent: Reaper 
User-agent: REBI-Shoveler 
User-agent: Recorder 
User-agent: RedBot 
User-agent: RedCarpet 
User-agent: RediffNewsBot
User-agent: ReGet 
User-agent: RepoMonkey 
User-agent: repparser
User-agent: research robot 
User-agent: Riddler 
User-agent: RIIGHTBOT/RIIGHT- 
User-agent: RiseNetBot 
User-agent: RiverGlassScanner 
User-agent: RMA 
User-agent: RoboPal 
User-agent: Robosourcer 
User-agent: Robot 
User-agent: robotek 
User-agent: robots 
User-agent: Robozilla 
User-agent: Rogerbot
User-agent: Rome Client 
User-agent: Rondello 
User-agent: Rotondo 
User-agent: Roverbot 
User-agent: RPT-HTTPClient 
User-agent: rtgibot 
User-agent: RufusBot 
User-agent: Runnk online rss reader 
User-agent: Runnk RSS aggregator 
User-agent: s~stremor-crawler 
User-agent: S2Bot 
User-agent: SafariBookmarkChecker 
User-agent: SafetyNet Robot 0.1 
User-agent: SaladSpoon/ShopSalad 
User-agent: Sapienti/Indexer 
User-agent: SBIder 
User-agent: SBL-BOT 
User-agent: Scanmine 
User-agent: SCFCrawler/Nutch-1.8 
User-agent: Scich 
User-agent: ScientificCommons.org 
User-agent: ScollSpider 
User-agent: ScooperBot 
User-agent: Scooter 
User-agent: Scoutjet
User-agent: ScrapeBox 
User-agent: Scrapy 
User-agent: SCrawlTest 
User-agent: Screaming Frog SEO Spider
User-agent: Scrubby 
User-agent: scSpider 
User-agent: Scumbot 
User-agent: Search Publisher 
User-agent: search x-bot 
User-agent: search.KumKie.com 
User-agent: search.msn.com/msnbot.htm 
User-agent: search.updated.com 
User-agent: search.usgs.gov 
User-agent: Searcharoo.NET 
User-agent: SearchBlox 
User-agent: searchbot 
User-agent: Search-Channel 
User-agent: searchengine 
User-agent: Search-Engine-Studio 
User-agent: searchhippo.com 
User-agent: SearchIt-Bot 
User-agent: searchmarking 
User-agent: searchmarks 
User-agent: Searchmee! Spider 
User-agent: searchmee_v 
User-agent: searchmining 
User-agent: SearchnowBot_v1 
User-agent: searchpreview 
User-agent: SearchSpider.com 
User-agent: SearQuBot/SearQuBot v1.0 
User-agent: SEB Spider 
User-agent: Seekbot 
User-agent: Seeker.lookseek.com 
User-agent: SeeqBot 
User-agent: seeqpod-vertical-crawler 
User-agent: Selflinkchecker 
User-agent: Semager 
User-agent: semanticdiscovery 
User-agent: Semantifire1 
User-agent: semisearch 
User-agent: SemrushBot
User-agent: SEOENGWorldBot
User-agent: SeznamBot 
User-agent: SeznamBot/2.0 
User-agent: SeznamBot/3.2-test1 
User-agent: ShablastBot 
User-agent: ShadowWebAnalyzer 
User-agent: Shareaza 
User-agent: Shelob 
User-agent: sherlock 
User-agent: ShopWiki
User-agent: ShowLinks 
User-agent: ShowyouBot
User-agent: siclab 
User-agent: silk 
User-agent: sindice-site-manager
User-agent: Siphon 
User-agent: SiteArchive 
User-agent: sitecheck.internetseer.com
User-agent: SiteCheck-sitecrawl by Siteimprove.com 
User-agent: SiteFinder 
User-agent: SiteGuardBot 
User-agent: SiteOrbiter 
User-agent: SiteSnagger
User-agent: SiteSucker 
User-agent: SiteSweeper 
User-agent: SiteXpert 
User-agent: SkimBot 
User-agent: SkimWordsBot 
User-agent: SkreemRBot 
User-agent: Skywalker 
User-agent: Sleipnir 
User-agent: slow-crawler 
User-agent: SlySearch 
User-agent: smart-crawler 
User-agent: SmartDownload 
User-agent: Smarte Bot 
User-agent: smartwit.com 
User-agent: Snake 
User-agent: snap.com beta crawler 
User-agent: Snapbot 
User-agent: SnapPreviewBot 
User-agent: Snappy 
User-agent: snookit/Snookit 
User-agent: Snooper/b97_01 
User-agent: Snoopy 
User-agent: SocialSearcher/0.1 
User-agent: SocSciBot 
User-agent: SOFT411 Directory 
User-agent: sogou 
User-agent: sohu agent 
User-agent: sohu-search 
User-agent: Sokitomi crawl 
User-agent: Solbot/1.0 LWP/5.07 
User-agent: sootle 
User-agent: Sosospider
User-agent: Space Bison 
User-agent: Space Fung 
User-agent: SpaceBison 
User-agent: SpankBot 
User-agent: spanner 
User-agent: Spanner/1.0 (Linux 2.0.27 i586) 
User-agent: Spatineo Monitor Controller 
User-agent: Spatineo Serval Controller 
User-agent: Spatineo Serval GetMapBot 
User-agent: special_archiver 
User-agent: SpeedySpider 
User-agent: Sphere Scout 
User-agent: Sphere Scout&v 
User-agent: Sphider 
User-agent: Sphider2 
User-agent: spider 
User-agent: Spider.TerraNautic.net 
User-agent: SpiderEngine 
User-agent: SpiderKU 
User-agent: SpiderMan 
User-agent: Spinn3r 
User-agent: Spinne 
User-agent: sportcrew-Bot 
User-agent: spyder3.microsys.com 
User-agent: SQ Webscanner 
User-agent: sqlmap 
User-agent: SquidClamAV_Redirector 
User-agent: Squid-Prefetch 
User-agent: Sqworm 
User-agent: SrevBot 
User-agent: sslbot 
User-agent: SSM Agent 
User-agent: StackRambler 
User-agent: StarDownloader 
User-agent: statbot 
User-agent: statcrawler 
User-agent: statedept-crawler 
User-agent: Steeler 
User-agent: STEGMANN-Bot 
User-agent: stero 
User-agent: Stripper 
User-agent: Stumbler 
User-agent: suchclip 
User-agent: Sucker 
User-agent: SumeetBot 
User-agent: SumitBot 
User-agent: SummizeBot 
User-agent: SummizeFeedReader 
User-agent: Sunrise XP 
User-agent: SuperBot 
User-agent: superbot.com 
User-agent: SuperHTTP 
User-agent: SuperLumin Downloader 
User-agent: SuperPagesBot 
User-agent: Supybot 
User-agent: SURF 
User-agent: Surfbot 
User-agent: SurfControl 
User-agent: SurveyBot 
User-agent: suzuran 
User-agent: SWEBot 
User-agent: swish-e 
User-agent: SygolBot 
User-agent: SynapticWalker 
User-agent: Syntryx ANT Scout Chassis Pheromone 
User-agent: SystemSearch-robot 
User-agent: Szukacz 
User-agent: Tailrank 
User-agent: tAkeOut 
User-agent: Talkro Web-Shot 
User-agent: TAMU_CRAWLER 
User-agent: TapuzBot 
User-agent: Tarantula 
User-agent: targetblaster.com 
User-agent: TargetYourNews.com bot 
User-agent: TAUSDataBot 
User-agent: taxinomiabot 
User-agent: TeamSoft WinInet Component 
User-agent: Tecomi Bot 
User-agent: TeezirBot 
User-agent: Teleport
User-agent: TeleportPro
User-agent: Telesoft 
User-agent: Teradex Mapper 
User-agent: TERAGRAM_CRAWLER 
User-agent: TerrawizBot 
User-agent: TestBot 
User-agent: testing of bot 
User-agent: TextBot 
User-agent: thatrobotsite.com 
User-agent: The Dyslexalizer 
User-agent: The Intraformant 
User-agent: The.Intraformant 
User-agent: TheNomad 
User-agent: Theophrastus 
User-agent: theusefulbot 
User-agent: TheUsefulbot_ 
User-agent: ThumbBot 
User-agent: Thumbnail.CZ robot 
User-agent: thumbshots-de-bot 
User-agent: T-H-U-N-D-E-R-S-T-O-N-E 
User-agent: tigerbot 
User-agent: TightTwatBot 
User-agent: TinEye 
User-agent: Titan 
User-agent: toCrawl/UrlDispatcher 
User-agent: to-dress_ru_bot_ 
User-agent: to-night-Bot 
User-agent: Topicalizer 
User-agent: topicblogs 
User-agent: Toplistbot 
User-agent: TopServer PHP 
User-agent: topyx-crawler 
User-agent: Touche 
User-agent: TourlentaScanner 
User-agent: TPSystem 
User-agent: TRAAZI 
User-agent: TranSGeniKBot 
User-agent: TravelBot 
User-agent: TravelLazerBot 
User-agent: travel-search 
User-agent: Treezy 
User-agent: TREX 
User-agent: TridentSpider 
User-agent: Trovator 
User-agent: True_Robot 
User-agent: tScholarsBot 
User-agent: TSM Translation-Search-Machine 
User-agent: TsWebBot 
User-agent: TulipChain 
User-agent: turingos 
User-agent: TurnitinBot 
User-agent: TurnitinBot/1.5 
User-agent: TutorGigBot 
User-agent: TweetedTimes Bot 
User-agent: TweetmemeBot 
User-agent: TwengaBot 
User-agent: TwengaBot-Discover 
User-agent: Twiceler 
User-agent: Twikle 
User-agent: twinuffbot 
User-agent: Twisted PageGetter 
User-agent: Twitturls 
User-agent: Twitturly  
User-agent: TygoBot 
User-agent: TygoProwler 
User-agent: Typhoeus 
User-agent: U.S. Government Printing Office 
User-agent: uberbot 
User-agent: UbiCrawler
User-agent: ucb-nutch 
User-agent: UdmSearch 
User-agent: UFAM-crawler- 
User-agent: Ultraseek 
User-agent: UnChaos 
User-agent: unchaos_crawler_ 
User-agent: UnisterBot 
User-agent: Unitek UniEngine 
User-agent: UniversalSearch 
User-agent: UnwindFetchor 
User-agent: UofTDB_experiment 
User-agent: updated 
User-agent: URL Control 
User-agent: url_gather 
User-agent: URLAppendBot 
User-agent: URLBlaze 
User-agent: urlchecker 
User-agent: URL-Checker 
User-agent: urlck/1.2.3 
User-agent: UrlDispatcher 
User-agent: URLSpiderPro 
User-agent: URLy Warning 
User-agent: URLy.Warning 
User-agent: USAF AFKN K2SPIDER 
User-agent: usasearch 
User-agent: USS-Cosmix 
User-agent: USyd-NLP-Spider 
User-agent: Vacobot 
User-agent: Vacuum 
User-agent: VadixBot 
User-agent: Vagabondo 
User-agent: Validator 
User-agent: Valkyrie/1.0 libwww-perl/0.40 
User-agent: vBSEO 
User-agent: VCI 
User-agent: VCI WebViewer VCI WebViewer Win32 
User-agent: VerbstarBot 
User-agent: VeriCiteCrawler/Nutch-2.2.1 
User-agent: Verifactrola 
User-agent: Verity-URL-Gateway 
User-agent: Vermut 
User-agent: versus crawler 
User-agent: versus.integis.ch 
User-agent: viasarchivinginformation.html 
User-agent: VIP 
User-agent: VIPr 
User-agent: virus-detector 
User-agent: VisBot 
User-agent: Vishal For CLIA 
User-agent: VisWeb 
User-agent: Vital Search\\\'n Urchin 
User-agent: vlad 
User-agent: vlsearch 
User-agent: VMBot 
User-agent: VocusBot 
User-agent: VoidEYE 
User-agent: VoilaBot 
User-agent: Vortex 
User-agent: voyager 
User-agent: voyager-hc 
User-agent: voyager-partner-deep 
User-agent: VSE 
User-agent: vspider 
User-agent: W3C_Unicorn 
User-agent: W3C-WebCon 
User-agent: w3m 
User-agent: w3search 
User-agent: wacbot 
User-agent: wastrix 
User-agent: Water Conserve Portal 
User-agent: Water Conserve Spider 
User-agent: WatzBot 
User-agent: wauuu engine/Wauuu 
User-agent: Wavefire 
User-agent: Waypath 
User-agent: Wazzup 
User-agent: Wazzup1.0.4800 
User-agent: wbdbot 
User-agent: Web CEO Online robot 
User-agent: web crawler 
User-agent: Web Downloader 
User-agent: Web Image Collector 
User-agent: Web Link Validator 
User-agent: Web Magnet 
User-agent: Web Site Downloader 
User-agent: Web Sucker 
User-agent: Web.Image.Collector 
User-agent: web-agent 
User-agent: WebaltBot 
User-agent: WebAuto 
User-agent: WebBandit 
User-agent: WebBandit/3.50 
User-agent: WebBOT 
User-agent: webbul-bot 
User-agent: WebCapture 
User-agent: WebCapture 2.0 
User-agent: webcheck 1 
User-agent: Webclipping.com 
User-agent: webcollage 
User-agent: WebCopier
User-agent: WebCopy 
User-agent: WebCorp 
User-agent: webcrawl.net 
User-agent: WebCrawler 
User-agent: WebCrawler/3.0 Robot libwww/5.0a 
User-agent: WebDownloader for X 
User-agent: Webdup 
User-agent: WebEMailExtrac 
User-agent: WebEnhancer 
User-agent: WebFetch 
User-agent: WebFetcher/0.8, 
User-agent: WebGather 
User-agent: WebGo IS 
User-agent: webGobbler 
User-agent: WebImages 
User-agent: Webinator-search2.fasthealth.com 
User-agent: Webinator-WBI 
User-agent: WebIndex 
User-agent: WebIndexer 
User-agent: weblayers/0.0 
User-agent: WebLeacher 
User-agent: WeblexBot 
User-agent: WebLinker/0.0 libwww-perl/0.1 
User-agent: webLyzard 
User-agent: WebmasterCoffee 
User-agent: WebmasterWorld Extractor 
User-agent: WebmasterWorldForumBot 
User-agent: WebMiner 
User-agent: WebMoose/0.0.0000 
User-agent: WeBot 
User-agent: WebPix 
User-agent: WebReaper
User-agent: WebRipper 
User-agent: WebSauger 
User-agent: Webscan 
User-agent: websearchbench 
User-agent: WebSite 
User-agent: websitemirror 
User-agent: Web-Sniffer 
User-agent: WebSpear 
User-agent: websphinx.test 
User-agent: WebSpider 
User-agent: Webster 
User-agent: WebStripper 
User-agent: WebTrafficExpress 
User-agent: WebTrends Link Analyzer 
User-agent: WebVac 
User-agent: webwalk 
User-agent: WebWalker 
User-agent: WebWasher 
User-agent: WebWatch 
User-agent: WebWhacker 
User-agent: WebXM 
User-agent: WebZIP
User-agent: Weddings.info Bot 
User-agent: wenbin/search 
User-agent: WEP Search 
User-agent: WEPA 
User-agent: WeRelateBot 
User-agent: wget
User-agent: Whacker 
User-agent: Whirlpool Web Engine 
User-agent: WhoWhere Robot 
User-agent: Widow 
User-agent: WikiaBot 
User-agent: Wikio 
User-agent: wikiwix-bot- 
User-agent: WinHTTP Example 
User-agent: WinHttp.WinHttpRequest 
User-agent: WIRE 
User-agent: wired-digital-newsbot/1.5 
User-agent: WISEbot 
User-agent: WISENutbot 
User-agent: wish-la 
User-agent: wish-project 
User-agent: wisponbot 
User-agent: WMCAI-robot 
User-agent: wminer 
User-agent: WMSBot 
User-agent: WocBot
User-agent: woriobot
User-agent: worldshop 
User-agent: WorQmada 
User-agent: Wotbox 
User-agent: wume_crawler 
User-agent: WWW Collector 
User-agent: WWW-Mechanize 
User-agent: WWWOFFLE 
User-agent: wwwrobot 
User-agent: wwwster 
User-agent: WWWWanderer v3.0 
User-agent: wwwxref 
User-agent: Wysigot 
User-agent: Xaldon 
User-agent: X-clawler 
User-agent: Xenu
User-agent: Xerka MetaBot 
User-agent: Xerka WebBot 
User-agent: XGET/0.7 
User-agent: xirq 
User-agent: XmarksFetch 
User-agent: XoviBot 
User-agent: XoviBot/2.0 
User-agent: xqrobot 
User-agent: Y!J 
User-agent: Y!J-BRI 
User-agent: Y!J-BRJ/YATS crawler 
User-agent: Y!J-BRW 
User-agent: Y!J-BSC 
User-agent: Y!J-PSC 
User-agent: Y!J-SRD 
User-agent: Y!TunnelPro 
User-agent: yacy.net 
User-agent: yacybot
User-agent: YahooSeeker
User-agent: Yanga WorldSearch Bot 
User-agent: yarienavoir.net 
User-agent: Yasaklibot 
User-agent: yBot 
User-agent: YebolBot 
User-agent: yellowJacket 
User-agent: yes 
User-agent: YesupBot 
User-agent: Yeti 
User-agent: YioopBot 
User-agent: yolinkBot 
User-agent: yoogliFetchAgent 
User-agent: Yoono 
User-agent: Yoriwa 
User-agent: YottaCars_Bot 
User-agent: YoudaoBot
User-agent: you-dir 
User-agent: Z-Add Link Checker 
User-agent: zagrebin 
User-agent: Zao 
User-agent: Zealbot
User-agent: zedzo.digest 
User-agent: zedzo.validate 
User-agent: zermelo 
User-agent: Zeus 
User-agent: zibber-v 
User-agent: zimeno 
User-agent: Zing-BottaBot 
User-agent: ZipppBot 
User-agent: ZoomSpider 
User-agent: Zotag Search 
User-agent: ZuiBot 
User-agent: ZumBot 
User-agent: Zyborg 
User-agent: Zyte 
Disallow: /

User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /widget-small/
Disallow: /widget-medium/
Disallow: /our-scripts/