{ "360Spider": "360Spider", "AddThis": "AddThis", "ADmantX": "ADmantX Platform Semantic Analyzer", "AdsBot-Google": "Google Adwords", "AhrefsBot": "Ahrefs backlinks research tool", "alexa": "Alexa Crawler", "Apache-HttpClient": "Java http library", "ApacheBench": "ApacheBench (ab)", "AppEngine-Google": "Google App Engine", "Applebot": "Apple Bot", "Ask Jeeves": "Ask Jeeves", "AsyncHttpClient": "Java http and WebSocket client library", "awe.sm": "Awe.sm URL expander", "Baidu": "baidu", "Baiduspider": "Chinese search engine", "bingbot": "Microsoft bing bot", "bitlybot": "bit.ly bot", "BLEXBot": "webmeup-crawler.com", "bot@linkfluence.net": "Linkfluence bot", "BuiBui-CheckBot": "email: buibui[at]dadapro[dot]com", "Butterfly": "Topsy Labs", "buzztalk": "buzztalk", "check_http": "Nagios monitor", "CMRadar/0.1": "CMRadar/0.1", "ColdFusion": "ColdFusion http library", "Crowsnest": "crowsnest.tv news portal bot", "curl": "curl unix CLI http client", "Daumoa": "Korean portal and search engine indexing bot", "DAP/NetHTTP": "DAP/NetHTTP", "developers.google.com/+/web/snippet/": "Google Plus", "DigitalPersona Fingerprint Software": "HP Fingerprint scanner", "Domain Re-Animator Bot": "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com", "DotBot": "Dot Bot", "DuckDuckBot": "Duck Duck Go", "Embedly": "Embedly", "EOAAgent": "EOAAgent", "EventMachine HttpClient": "Ruby http library", "EvriNid": "Evri bot", "Exabot": "Exalead's bot", "ExaleadCloudView": "ExaleadCloudView", "facebookexternalhit": "Facebook Bot", "Facebot": "Facebook Bot", "FeedBurner": "RSS bot", "Feedfetcher-Google": "Google Feedfetcher", "FlipboardProxy": "FlipboardProxy", "Genieo": "Genieo Web filter bot", "getprismatic.com": "getprismatic.com", "Gigabot": "Gigabot spider", "Go http package": "Go http library", "Google Web Preview": "Google Instant Previews crawler", "Google Page Speed Insights": "Google Page Speed Insights", "Googlebot": "Google spider", "GrapeshotCrawler": "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)", "Hatena::Bookmark": "Hatena::Bookmark", "heritrix": "heritrix", "HTMLParser": "HTMLParser", "HTTPClient": "HTTPClient", "https://developers.google.com/+/web/snippet": "Google+ Snippet Fetcher", "HTTP_Request2": "HTTP_Request2", "HubSpot Connect": "HubSpot Connect", "ia_archiver(OS-Wayback)": "ia_archiver(OS-Wayback)", "iCoreService": "iCoreService", "InAGist": "URL resolver", "Insieve Bot": "Insieve Bot", "InsieveBot": "InsieveBot", "Instapaper": "Instapaper", "Insitesbot": "Insitesbot", "IstellaBot": "IstellaBot", "jack": "jack", "Jakarta Commons-HttpClient": "Jakarta Commons-HttpClient", "Jakarta Commons": "Jakarta Commons HttpClient", "Java": "Generic Java http library", "Jetslide": "Jetslide", "JS-Kit": "URL resolver", "Kimengi/nineconnections.com": "Kimengi/nineconnections.com", "Kimengi": "nineconnections.com bot", "knows.is": "knows.is", "kraken": "kraken", "Laconica": "Laconica", "libwww-perl": "Perl client-server library loved by script kids", "linkdexbot": "Linkdex Bot", "LinkedInBot": "LinkedIn", "LinksCrawler": "LinksCrawler", "Linode": "Linode Longview", "Lipperhey": "Lipperhey", "Livelapbot": "Livelapbot", "LoadTimeBot": "Load Time Bot", "LongURL": "URL expander service", "Lumibot": "Lumibot", "lwp-trivial": "Another Perl library loved by script kids", "magpie-crawler": "magpie-crawler", "Mail.RU_Bot": "Mail.ru Bot", "Mediapartners-Google": "Google Adsense bot", "MegaIndex.ru": "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)", "MetaURI": "URI meta information extractor bot", "MFE_expand": "Mcafee spider", "MJ12bot": "Majestic-12 spider", "MojeekBot": "Mojeek UK search crawler", "msnbot": "Microsoft bot", "netState": "netEstate NE Crawler", "Netvibes": "Personalized dashboard bot", "NewRelicPinger": "NewRelic monitor", "newsme": "newsme", "NING": "NING - Yet Another Twitter Swarmer", "Nutch": "Apache search spider", "Panopta": "Monitoring service", "PaperLiBot": "PaperLi is another content curation service", "peerindex": "peerindex", "PercolateCrawler": "PercolateCrawler", "PhantomJS": "PhantomJS", "Pingdom": "Pingdom monitoring", "Pinterest": "Pinterest", "publiclibraryarchive.org": "publiclibraryarchive.org", "PycURL": "Python http library", "Python-httplib2": "Python-httplib2", "python-requests": "python-requests", "Python-urllib": "Python http library", "QuerySeeker": "QuerySeekerSpider ( http://queryseeker.com/bot.html )", "QuickLook": "QuickLook", "Readability": "Readability", "RebelMouse": "RebelMouse", "redditbot": "Reddit Bot", "RelateIQ": "RelateIQ", "Riddler": "Riddler Bot", "rogerbot": "SeoMoz spider", "Ruby/1.9.3": "Ruby/1.9.3", "Ruby": "Ruby", "Scrapy": "Scrapy", "Screaming Frog SEO Spider": "Screaming Frog SEO Spider", "SearchmetricsBot": "SearchmetricsBot", "SemrushBot": "SEO analysis bot", "SeznamBot": "SeznamBot", "ShopWiki": "ShopWiki", "ShortLinkTranslate": "Link shortener", "ShowyouBot": "Showyou iOS app spider", "Siege": "Joe Dog Siege", "SiteUptime": "Site monitoring services", "Slack": "Slackbot-LinkExpanding", "Slurp": "Yahoo spider", "Sogou": "Chinese search engine", "spider": "generic web spider", "Spinn3r": "Spinn3r aggregator", "StatusCake": "StatusCake", "Test Certificate Info": "C http library?", "TinEye": "TinEye-bot/0.51 (see http://www.tineye.com/crawler.html)", "trendictionbot": "trendiction search", "TurnitinBot": "TurnitinBot", "TweetmemeBot": "TweetMeMe Crawler", "Twikle": "Social web search bot", "TwitJobSearch": "TwitJobSearch", "Twitmunin": "Twitmunin", "Twitterbot": "Twitter URL expander", "Twurly": "Twurly", "Typhoeus": "Typhoeus", "UnwindFetch": "Gnip URL expander", "UnwindFetchor": "Gnip crawler", "Vagabondo": "Vagabondo", "VB Project": "Visual Basic", "vkShare": "VKontake Sharer", "VoilaBot": "VoilaBot", "Webscout": "Webscout", "Wget": "wget unix CLI http client", "WordPress": "WordPress spider", "Wormly": "WormlyBot", "Xenu Link Sleuth": "Xenu Link Sleuth", "XoviBot": "XoviBot", "YandexBot": "Yandex spider", "YandexDirect": "Yandex Direct spider", "YandexMetrika": "Yandex Metrika spider", "YOURLS": "YOURLS", "zelist.ro": "feed parser", "ZIBB": "ZIBB spider", "ZyBorg": "Zyborg? Hmmm....", "StructuredDataTestingTool": "Google-StructuredDataTestingTool" }