crawlers.rb in crawler_detect-0.1.4

- old
+ new

@@ -33,11 +33,10 @@
 ^NG\/[0-9\.]
 ^NING\/
 ^PHP\/[0-9]
 ^RMA\/
 ^Ruby|Ruby\/[0-9]
-^scrutiny\/
 ^VSE\/[0-9]
 ^WordPress\.com
 ^XRL\/[0-9]
 ^ZmEu
 a3logics\.in
@@ -74,14 +73,16 @@
 amagit
 Anarchie
 AndroidDownloadManager
 Anemone
 AngleSharp\/
+annotate_google
 Ant\.com
 Anturis Agent
 AnyEvent-HTTP\/
 Apache Droid
+Apache OpenOffice
 Apache-HttpAsyncClient\/
 Apache-HttpClient\/
 ApacheBench\/
 Apexoo
 APIs-Google
@@ -120,10 +121,11 @@
 BazQux
 BBBike
 BCKLINKS
 BDFetch
 BegunAdvertising\/
+Bidtellect\/
 BigBozz
 Bigfoot
 biglotron
 BingLocalSearch
 BingPreview
@@ -132,10 +134,12 @@
 Bitacle
 biz_Directory
 Black\ Hole
 Blackboard Safeassign
 BlackWidow
+BlockNote\.Net
+Bloglines\/
 Bloglovin
 BlogPulseLive
 BlogSearch
 Blogtrottr
 BlowFish
@@ -217,14 +221,16 @@
 cuwhois\/[0-9]
 cybo\.com
 DareBoost
 DatabaseDriverMysqli
 DataCha0s
+Datanyze
 DataparkSearch
 dataprovider
 DataXu
 Daum(oa)?[ \/][0-9]
+DemandbasePublisherAnalyzer\/
 Demon
 DeuSu
 developers\.google\.com\/\+\/web\/snippet\/
 Devil
 Digg
@@ -271,10 +277,11 @@
 Email%20Extractor%20Lite
 EMail\ Wolf
 EmailWolf
 Embed PHP Library
 Embedly
+endo\/
 europarchive\.org
 evc-batch
 EventMachine HttpClient
 Everwall Link Expander
 Evidon
@@ -297,19 +304,23 @@
 fasthttp
 Faveeo
 Favicon downloader
 FavOrg
 Feed Wrangler
+Feedable\/
 Feedbin
 FeedBooster
 FeedBucket
 FeedBunch\/[0-9]
 FeedBurner
 FeedChecker
 Feedly
+Feedreader
+FeedshowOnline
 Feedspot
 Feedwind\/[0-9]
+FeedZcollector
 feeltiptop
 Fetch API
 Fetch\/[0-9]
 Fever\/[0-9]
 FHscan
@@ -422,10 +433,11 @@
 hosterstats
 HostTracker
 ht:\/\/check
 htdig
 HTMLparser
+htmlyse\.com
 http-get
 HTTP-Header-Abfrage
 http-kit
 http-request\/
 HTTP-Tiny
@@ -455,19 +467,21 @@
 IDG Twitter Links Resolver
 IDwhois\/[0-9]
 Iframely
 igdeSpyder
 IlTrovatore
+ImageVisu\/
 Image\ Fetch
 Image\ Sucker
 ImageEngine\/
 Imagga
 imgsizer
 InAGist
 inbound\.li parser
 InDesign%20CC
 Indy\ Library
+InetURL
 infegy
 infohelfer
 InfoTekies
 InfoWizards Reciprocal Link System PRO
 inpwrd\.com
@@ -561,10 +575,11 @@
 LinkWalker
 Lipperhey
 Litemage_walker
 livedoor ScreenShot
 LoadImpactRload
+localsearch-web
 LongURL API
 looksystems\.net
 ltx71
 lua-resty-http
 lwp-request
@@ -607,10 +622,11 @@
 Miniature.io\/
 Miniflux
 Mister\ PiX
 mixdata dot com
 mixed-content-scan
+Mixmax-LinkPreview
 mixnode
 Mnogosearch
 mogimogi
 Mojeek
 Mojolicious \(Perl\)
@@ -643,10 +659,11 @@
 Net\ Vampire
 NetAnts
 NETCRAFT
 NetLyzer
 NetMechanic
+NetNewsWire
 Netpursual
 netresearch
 NetShelter ContentScan
 Netsparker
 NetTrack
@@ -681,10 +698,11 @@
 Nymesis
 NYU
 Ocelli\/[0-9]
 Octopus
 oegp
+og-scraper\/
 Offline Explorer
 Offline\ Navigator
 okhttp
 Omea Reader
 omgili
@@ -746,10 +764,11 @@
 Pingoscope
 PingSpot
 pinterest\.com
 Pixray
 Pizilla
+Plagger\/
 PleaseCrawl
 Ploetz \+ Zeller
 Plukkie
 plumanalytics
 PocketParser
@@ -793,10 +812,11 @@
 Qwantify
 Radian6
 RankActive
 RankFlex
 RankSonicSiteAuditor
+raynette_httprequest
 Readability
 RealDownload
 RealPlayer%20Downloader
 RebelMouse
 Recorder
@@ -814,29 +834,33 @@
 Robosourcer
 Robozilla\/[0-9]
 ROI Hunter
 RPT-HTTPClient
 RSSOwl
+RssReader\/
 safe-agent-scanner
 SalesIntelligent
 Saleslift
 SauceNAO
 SBIder
 scalaj-http
 scan\.lol
 ScanAlert
+ScrapeBox Page Scanner
 Scoop
 scooter
 ScoutJet
 ScoutURLMonitor
 Scrapy
 Screaming
 ScreenShotService\/[0-9]
 Scrubby
+Scrutiny\/
 Search37\/
 search\.thunderstone
 Searchestate
+SearchExpress
 SearchSight
 Seeker
 semanticdiscovery
 semanticjuice
 Semiocast HTTP client
@@ -897,10 +921,11 @@
 slurp
 SlySearch
 SmartDownload
 SMRF URL Expander
 SMUrlExpander
+Snarfer\/
 Snake
 Snappy
 SniffRSS
 sniptracker
 Snoopy
@@ -996,10 +1021,11 @@
 Typhoeus
 ubermetrics-technologies
 uclassify
 uCrawlr\/
 UdmSearch
+unirest-java
 UniversalFeedParser
 Unshorten\.It
 Untiny
 UnwindFetchor
 updated
@@ -1109,14 +1135,16 @@
 Whibse
 WhoRunsCoinHive
 Whynder Magnet
 Windows-RSS-Platform
 WinHttpRequest
+WinPodder
 wkhtmlto
 wmtips
 Woko
 Word\/
 WordPress\/
+WordupinfoSearch
 wotbox
 WP Engine Install Performance API
 wpif
 wprecon\.com survey
 WPScan