200PleaseBot: "200PleaseBot (200please.com/bot)"
360Spider: "360Spider"
abot: "CrawlDaddy, abot (code.google.com/p/abot)"
AddThis: "AddThis"
ADmantX: "ADmantX Platform Semantic Analyzer"
AdsBot-Google: "Google Adwords"
AdvBot: "AdvBot (advbot.net/bot.htmlm, advbot.org/bot.html)"
AhrefsBot: "Ahrefs backlinks research tool"
alexa: "Alexa Crawler"
Apache-HttpClient: "Java http library"
ApacheBench: "ApacheBench (ab)"
AppEngine-Google: "Google App Engine"
Applebot: "Apple Bot"
archive.org_bot: "Internet Archive (archive.org)"
Ask Jeeves: "Ask Jeeves"
AsyncHttpClient: "Java http and WebSocket client library"
awe.sm: "Awe.sm URL expander"
Baidu: "baidu"
Baiduspider: "Chinese search engine"
BDCbot: "Big Data Corp (ecommerce.bigdatacorp.com.br/faq.aspx)"
bingbot: "Microsoft bing bot"
bitlybot: "bit.ly bot"
Blekkobot: "Blekkobot (blekko.com/about/blekkobot)"
BLEXBot: "webmeup-crawler.com"
bot@linkfluence.net: "Linkfluence bot"
BufferBot: "BufferBot"
BuiBui-CheckBot: "email: buibui[at]dadapro[dot]com"
Butterfly: "Topsy Labs"
buzztalk: "buzztalk"
CatchBot: "CatchBot (catchbot.com)"
check_http: "Nagios monitor"
Cliqzbot: "Cliqzbot (cliqz.com/company/cliqzbot)"
CMRadar/0.1: "CMRadar/0.1"
ColdFusion: "ColdFusion http library"
Comodo-Webinspector-Crawler: "Comodo (comodorobot.com)"
Crowsnest: "Crowsnest"
curabot: "cura.yt"
curl: "curl unix CLI http client"
DAP/NetHTTP: "DAP/NetHTTP"
datagnionbot: "datagnion.com/bot.html"
Daumoa: "Korean portal and search engine indexing bot"
developers.google.com/+/web/snippet/: "Google Plus"
diffbot: "Diffbot (diffbot.com)"
DigitalPersona Fingerprint Software: "HP Fingerprint scanner"
Domain Re-Animator Bot: "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com"
domainsbot: "DomainsBot (domainsbot.com)"
DomainTunoCrawler: "DomainTuno (domaintuno.com/robot)"
DotBot: "Dot Bot"
DuckDuckBot: "Duck Duck Go"
DuckDuckGo: "Duck Duck Go"
Embedly: "Embedly"
EOAAgent: "EOAAgent"
EventMachine HttpClient: "Ruby http library"
EveryoneSocialBot: "EveryoneSocial (everyonesocial.com)"
EvriNid: "Evri bot"
Exabot: "Exalead's bot"
ExaleadCloudView: "ExaleadCloudView"
facebookexternalhit: "Facebook Bot"
Facebot: "Facebook Bot"
FeedBurner: "RSS bot"
Feedfetcher-Google: "Google Feedfetcher"
Findxbot: "Findxbot (http://www.findxbot.com)"
FlipboardProxy: "FlipboardProxy"
FriendFeedBot: "FriendFeed (friendfeed.com/about/bot)"
Genieo: "Genieo Web filter bot"
getprismatic.com: "getprismatic.com"
Gigabot: "Gigabot spider"
Gimme60bot: "Gimme60 (gimme60.com)"
gimme60bot: "Gimme60 (gimme60.com)"
GimmeUSAbot: "Gimme60 (gimme60.com)"
Go http package: "Go http library"
Google Page Speed Insights: "Google Page Speed Insights"
Google Web Preview: "Google Instant Previews crawler"
Googlebot: "Google spider"
GrapeshotCrawler: "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)"
Gravitybot: "Gravitybot (gravity.com)"
Hatena::Bookmark: "Hatena::Bookmark"
heritrix: "heritrix"
HTMLParser: "HTMLParser"
HTTP_Request2: "HTTP_Request2"
HTTPClient: "HTTPClient"
https://developers.google.com/+/web/snippet: "Google+ Snippet Fetcher"
HubSpot: "HubSpot"
ia_archiver: "Internet Archive (WayBackMachine)"
iCoreService: "iCoreService"
idmarch: "idmarch.org/bot.html"
InAGist: "URL resolver"
Insieve: "Insieve Bot"
Insitesbot: "Insitesbot"
Instapaper: "Instapaper"
IstellaBot: "IstellaBot"
jack: "jack"
Jakarta Commons: "Jakarta Commons HttpClient"
Java: "Generic Java http library"
Jetslide: "Jetslide"
JS-Kit: "URL resolver"
Kemvibot: "Kemvi (kemvi.com)"
Kimengi: "Kimengi/nineconnections.com bot"
knows.is: "knows.is"
Kojitsubot: "Kojitsubot"
KomodiaBot: "KomodiaBot"
kraken: "kraken"
Laconica: "Laconica"
libwww-perl: "Perl client-server library loved by script kids"
Lijit Crawler: "Lijit (lijit.com/robot/crawler)"
linkdexbot: "Linkdex Bot"
LinkedInBot: "LinkedIn"
LinksCrawler: "LinksCrawler"
Linode: "Linode Longview"
Lipperhey: "Lipperhey"
Livelapbot: "Livelapbot"
LoadTimeBot: "Load Time Bot"
LongURL: "URL expander service"
Lumibot: "Lumibot"
lwp-trivial: "Another Perl library loved by script kids"
magpie-crawler: "magpie-crawler"
Mail.RU_Bot: "Mail.ru Bot"
meanpathbot: "meanpath (meanpath.com)"
Mediapartners-Google: "Google Adsense bot"
MegaIndex.ru: "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)"
memoryBot: "mignify.com/bot.html"
memorybot: "mignify.com/bot.html"
MetaURI: "MetaURI"
MFE_expand: "Mcafee spider"
MIR web crawler: "MIR web crawler (mirproject15@gmail.com)"
MJ12bot: "Majestic-12 spider"
MojeekBot: "Mojeek UK search crawler"
MS Search 6.0 Robot: "MS Search 6.0 Robot"
msnbot: "Microsoft bot"
NerdyBot: "NerdyBot"
netState: "netEstate NE Crawler"
Netvibes: "Personalized dashboard bot"
NetzCheckBot: "bot@netzcheck.com"
NewRelicPinger: "NewRelic monitor"
newsme: "newsme"
niki-bot: "niki-bot"
NING: "NING - Yet Another Twitter Swarmer"
Nutch: "Apache search spider"
OpenHoseBot: "OpenHoseBot (http://www.openhose.org/bot.html)"
OrangeBot: "OrangeBot (support.orangebot@orange.com)"
PagesInventory: "pagesinventory.com"
Panopta: "Monitoring service"
PaperLiBot: "PaperLi is another content curation service"
peerindex: "peerindex"
PercolateCrawler: "PercolateCrawler"
PerfectMarketKWTbot: "PerfectMarket (perfectmarket.com)"
PhantomJS: "PhantomJS"
Pingdom: "Pingdom monitoring"
Pinterest: "Pinterest"
Plukkie: "botje.com/plukkie.htm"
psbot-page: "picsearch.com/bot.html"
publiclibraryarchive.org: "publiclibraryarchive.org"
PycURL: "Python http library"
Python-httplib2: "Python-httplib2"
python-requests: "Python http library"
Python-urllib: "Python http library"
QuerySeeker: "QuerySeekerSpider ( http://queryseeker.com/bot.html )"
QuickLook: "QuickLook"
Re-Animator: "Domain Re-Animator Bot (http://domainreanimator.com)"
Readability: "Readability"
RebelMouse: "RebelMouse"
redditbot: "Reddit Bot"
RelateIQ: "RelateIQ"
Riddler: "Riddler Bot"
rogerbot: "SeoMoz spider"
RSSMicro: "RSS/Atom Feed Robot (rssmicro.com)"
Ruby: "Ruby"
Scrapy: "Scrapy"
Screaming Frog SEO Spider: Screaming Frog SEO Spider
SearchmetricsBot: "SearchmetricsBot"
SemrushBot: "SEO analysis bot"
SEOkicks-Robot: "SEOKicks (seokicks.de/robot.html)"
SeznamBot: "SeznamBot"
ShopWiki: "ShopWiki"
ShortLinkTranslate: "Link shortener"
ShowyouBot: "Showyou iOS app spider"
Siege: "Joe Dog Siege"
SISTRIX: "Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)"
SiteUptime: "Site monitoring services"
Slack: "Slackbot-LinkExpanding"
Slackbot: "api.slack.com/robots"
Slurp: "Yahoo spider"
SMTBot: "SimilarTech (http://www.similartech.com/smtbot)"
Sogou: "Chinese search engine"
spbot: "OpenLinkProfiler (http://openlinkprofiler.org/bot)"
spider: "generic web spider"
Spinn3r: "Spinn3r aggregator"
SputnikBot: "SputnikBot (http://corp.sputnik.ru/webmaster)"
StatusCake: "StatusCake"
StructuredDataTestingTool: 'Google-StructuredDataTestingTool'
Test Certificate Info: "C http library?"
TinEye: "TinEye-bot/0.51 (see http://www.tineye.com/crawler.html)"
trendictionbot: "trendiction search"
TurnitinBot: "TurnitinBot"
tweetedtimes: "The Tweeted Times"
TweetmemeBot: "TweetMeMe Crawler"
Twikle: "Social web search bot"
TwitJobSearch: "TwitJobSearch"
Twitmunin: "Twitmunin"
Twitterbot: "Twitter URL expander"
Twurly: "Twurly"
Typhoeus: "Typhoeus"
uMBot: "uberMetrics"
UnwindFetch: "Gnip"
UptimeRobot: "Uptime Robot"
Vagabondo: "Vagabondo"
VB Project: "Visual Basic"
Vigil: "Vigil (vigil-app.com/bot.html)"
vkShare: "VKontake Sharer"
VoilaBot: "VoilaBot"
VRCrawler: "ventureradar.com/crawler"
WASALive-Bot: "blog.wasalive.com/wasalive-bots"
WBSearchBot: "Ware Bay Best Buys (http://www.warebay.com/bot.html)"
Webscout: "Webscout"
WeSEE: "WeSEE (www.wesee.com/bot)"
Wget: "wget unix CLI http client"
wget: "wget unix CLI http client"
WordPress: "WordPress spider"
Wormly: "WormlyBot"
Wotbox: "Wotbox (http://www.wotbox.com/bot)"
Xenu Link Sleuth: "Xenu Link Sleuth"
XoviBot: "XoviBot"
yacybot: "YaCy (yacy.net)"
Yandex: "Yandex, Yandex Direct, Yandex Metrika, Yandex News (yandex.com)"
Yeti: "Naver Corp (help.naver.com/robots)"
YOURLS: "YOURLS"
zelist.ro: "feed parser"
ZIBB: "ZIBB spider"
zitebot: "Zite"
ZyBorg: "Zyborg? Hmmm...."