crawlers.rb in crawler_detect-0.1.12

- old
+ new

@@ -54,10 +54,11 @@
 adbeat
 AddThis
 ADmantX
 AdminLabs
 adressendeutschland
+adreview\/
 adscanner
 Adstxtaggregator
 adstxt-worker
 adstxt\.com
 agentslug
@@ -93,11 +94,10 @@
 APIs-Google
 AportWorm\/
 AppBeat\/
 AppEngine-Google
 AppleSyndication
-AppStoreScraperZ
 Aprc\/[0-9]
 Arachmo
 arachnode
 Arachnophilia
 aria2
@@ -108,10 +108,11 @@
 ASPSeek
 Asterias
 Astute
 asynchttp
 Attach
+attohttpc
 autocite
 AutomaticWPTester
 Autonomy
 axios\/
 AWS Security Scanner
@@ -132,10 +133,11 @@
 BazQux
 BBBike
 BCKLINKS
 BDFetch
 BegunAdvertising
+Bewica-security-scan
 Bidtellect
 BigBozz
 Bigfoot
 biglotron
 BingLocalSearch
@@ -146,17 +148,19 @@
 biz_Directory
 Black Hole
 Blackboard Safeassign
 BlackWidow
 BlockNote\.Net
+BlogBridge
 Bloglines
 Bloglovin
 BlogPulseLive
 BlogSearch
 Blogtrottr
 BlowFish
 boitho\.com-dc
+Boost\.Beast
 BPImageWalker
 Braintree-Webhooks
 Branch Metrics API
 Branch-Passthrough
 Brandprotect
@@ -203,10 +207,11 @@
 Clarsentia
 clips\.ua\.ac\.be
 Cloud mapping
 CloudEndure
 CloudFlare-AlwaysOnline
+Cloudflare-Healthchecks
 Cloudinary
 cmcm\.com
 coccoc
 cognitiveseo
 colly -
@@ -245,10 +250,11 @@
 DataXu
 Daum(oa)?[ \/][0-9]
 dBpoweramp
 ddline
 deeris
+delve\.ai
 Demon
 DeuSu
 developers\.google\.com\/\+\/web\/snippet\/
 Devil
 Digg
@@ -263,10 +269,11 @@
 DMBrowser
 DNSPod-reporting
 docoloc
 Dolphin http client
 DomainAppender
+DomainLabz
 Donuts Content Explorer
 dotMailer content retrieval
 dotSemantic
 downforeveryoneorjustme
 Download Wonder
@@ -276,10 +283,11 @@
 drupact
 Drupal \(\+http:\/\/drupal\.org\/\)
 DTS Agent
 dubaiindex
 DuplexWeb-Google
+DynatraceSynthetic
 EARTHCOM
 Easy-Thumb
 EasyDL
 Ebingbong
 ec2linkfinder
@@ -306,10 +314,11 @@
 Evrinid
 ExactSearch
 ExaleadCloudview
 Excel\/
 exif
+ExoRank
 Exploratodo
 Express WebPictures
 Extreme Picture Finder
 EyeNetIE
 ezooms
@@ -340,10 +349,11 @@
 feeltiptop
 Fetch API
 Fetch\/[0-9]
 Fever\/[0-9]
 FHscan
+Filestack
 Fimap
 findlink
 findthatfile
 FlashGet
 FlipboardBrowserProxy
@@ -357,10 +367,11 @@
 FoundSeoTool
 http:\/\/www.neomo.de\/
 free thumbnails
 Freeuploader
 Funnelback
+Fuzz Faster U Fool
 G-i-g-a-b-o-t
 g00g1e\.net
 ganarvisitas
 geek-tools
 Genieo
@@ -377,10 +388,11 @@
 Ghost Inspector
 GigablastOpenSource
 GIS-LABS
 github-camo
 github\.com
+Goldfire Server
 Go [\d\.]* package http
 Go http package
 Go-Ahead-Got-It
 Go-http-client
 Go!Zilla
@@ -396,29 +408,30 @@
 Google Keyword Tool
 Google Page Speed Insights
 Google PP Default
 Google Search Console
 Google Web Preview
+Google-Ads-Overview
 Google-Adwords
 Google-Apps-Script
 Google-Calendar-Importer
 Google-HotelAdsVerifier
 Google-HTTP-Java-Client
 Google-Publisher-Plugin
 Google-Read-Aloud
 Google-SearchByImage
 Google-Site-Verification
+Google-speakr
 Google-Structured-Data-Testing-Tool
 Google-Youtube-Links
 google-xrawler
 GoogleDocs
 GoogleHC\/
 GoogleProducer
 GoogleSites
 Google-Transparency-Report
 Gookey
-GoScraper
 GoSpotCheck
 gosquared-thumbnailer
 Gotit
 GoZilla
 grabify
@@ -448,10 +461,11 @@
 HeadlessChrome
 HEADMasterSEO
 HeartRails_Capture
 help@dataminr\.com
 heritrix
+Hexometer
 historious
 hkedcity
 hledejLevne\.cz
 Hloader
 HMView
@@ -594,10 +608,11 @@
 Licorne
 Liferea\/
 Lightspeedsystems
 Lighthouse
 Likse
+limber\.io
 Link Valet
 link_thumbnailer
 LinkAlarm\/
 linkCheck
 linkdex
@@ -613,10 +628,11 @@
 Litemage_walker
 livedoor ScreenShot
 LoadImpactRload
 localsearch-web
 LongURL API
+longurl-r-package
 looid\.com
 looksystems\.net
 ltx71
 lua-resty-http
 lwp-request
@@ -647,10 +663,11 @@
 Melvil Rawi
 MemGator
 Metaspinner
 MetaURI
 MFC_Tear_Sample
+MicroMessenger\/
 Microsearch
 Microsoft Office
 Microsoft Outlook
 Microsoft Windows Network Diagnostics
 Microsoft-WebDAV-MiniRedir
@@ -677,10 +694,11 @@
 Moreover
 Morfeus Fucking Scanner
 Morning Paper
 MovableType
 mowser
+Mr\.4x3 Powered
 Mrcgiguy
 MS Web Services Client Protocol
 MSFrontPage
 mShots
 MuckRack\/
@@ -746,20 +764,20 @@
 Octopus
 oegp
 Offline Explorer
 Offline Navigator
 OgScrper
-og-scraper
 okhttp
 omgili
 OMSC
 Online Domain Tools
 OpenCalaisSemanticProxy
 Openfind
 OpenLinkProfiler
 Openstat\/
 OpenVAS
+OPPO A33
 Optimizer
 Orbiter
 OrgProbe\/
 orion-semantics
 Outlook-Express
@@ -770,10 +788,11 @@
 OxfordCloudService
 Page Valet
 page_verifier
 page scorer
 page2rss
+PageFreezer
 PageGrabber
 PagePeeker
 PageScorer
 Pagespeed\/
 Panopta
@@ -823,10 +842,11 @@
 postano
 PostmanRuntime
 PostPost
 postrank
 PowerPoint\/
+Prebid
 Priceonomics Analysis Engine
 PrintFriendly
 PritTorrent
 Prlog
 probethenet
@@ -837,12 +857,12 @@
 proximic
 PRTG Network Monitor
 pshtt, https scanning
 PTST
 PTST\/[0-9]+
-Pulsepoint XT3 web scraper
 Pump
+python-httpx
 Python-httplib2
 python-requests
 Python-urllib
 Qirina Hurdler
 QQDownload
@@ -879,25 +899,27 @@
 Robosourcer
 Robozilla
 ROI Hunter
 RPT-HTTPClient
 RSSOwl
+RyowlEngine
 safe-agent-scanner
 SalesIntelligent
 Saleslift
 Sendsay\.Ru
 SauceNAO
 SBIder
+sc-downloader
 scalaj-http
+Scamadviser-Frontend
 scan\.lol
 ScanAlert
 Scoop
 scooter
 ScoutJet
 ScoutURLMonitor
 ScrapeBox Page Scanner
-SimpleScraper
 Scrapy
 Screaming
 ScreenShotService
 Scrubby
 Scrutiny\/
@@ -919,10 +941,11 @@
 seo4ajax
 Seobility
 SEOCentro
 SeoCheck
 SEOkicks
+SEOlizer
 Seomoz
 SEOprofiler
 SEOsearch
 seoscanners
 seositecheckup
@@ -932,10 +955,11 @@
 Seznam
 Shelob
 Shodan
 Shoppimon
 ShopWiki
+shortURL lengthener
 ShortLinkTranslate
 shrinktheweb
 Sideqik
 SimplePie
 SimplyFast
@@ -1011,11 +1035,10 @@
 Sucuri
 summify
 SuperHTTP
 Surphace Scout
 Suzuran
-SwiteScraper
 Symfony BrowserKit
 Symfony2 BrowserKit
 SynHttpClient-Built
 Sysomos
 sysscan
@@ -1100,11 +1123,10 @@
 Vagabondo
 VB Project
 vBSEO
 VCI
 via ggpht\.com GoogleImageProxy
-VidibleScraper
 Virusdie
 visionutils
 vkShare
 VoidEYE
 Voil
@@ -1189,15 +1211,17 @@
 WhatWeb
 WhereGoes\?
 Whibse
 WhoRunsCoinHive
 Whynder Magnet
+WinHttp-Autoproxy-Service
 Windows-RSS-Platform
 WinPodder
 wkhtmlto
 wmtips
 Woko
+Wolfram HTTPClient
 woorankreview
 Word\/
 WordPress\/
 worldping-api
 WordupinfoSearch
@@ -1252,10 +1276,10 @@
 ZnajdzFoto
 ZnHTTP
 Zombie\.js
 Zoom\.Mac
 ZyBorg
-[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)
+[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)
       ].strip.split(/\n+/).freeze
     end
   end
 end