crawlers.rb in crawler_detect-0.1.9

- old
+ new

@@ -56,10 +56,12 @@
 ADmantX
 AdminLabs
 adressendeutschland
 adscanner
 Adstxtaggregator
+adstxt-worker
+adstxt\.com
 agentslug
 AHC
 aihit
 aiohttp\/
 Airmail
@@ -71,10 +73,11 @@
 Alligator
 allloadin
 AllSubmitter
 alyze\.info
 amagit
+^Amazon Simple Notification Service Agent$
 Anarchie
 AndroidDownloadManager
 Anemone
 AngleSharp
 annotate_google
@@ -214,12 +217,14 @@
 convera
 CookieReports
 copyright sheriff
 CopyRightCheck
 Copyscape
+cortex\/
 Cosmos4j\.feedback
 Covario-IDS
+Craw\/
 Crescent
 Crowsnest
 Criteo
 CSHttp
 CSSCheck
@@ -269,10 +274,11 @@
 Drip
 drupact
 Drupal \(\+http:\/\/drupal\.org\/\)
 DTS Agent
 dubaiindex
+DuplexWeb-Google
 EARTHCOM
 Easy-Thumb
 EasyDL
 Ebingbong
 ec2linkfinder
@@ -305,10 +311,11 @@
 Express WebPictures
 Extreme Picture Finder
 EyeNetIE
 ezooms
 facebookexternalhit
+facebookexternalua
 facebookplatform
 fairshare
 Faraday v
 fasthttp
 Faveeo
@@ -363,10 +370,11 @@
 getprismatic
 GetRight
 getroot
 GetURLInfo\/
 GetWeb
+Geziyor
 Ghost Inspector
 GigablastOpenSource
 GIS-LABS
 github-camo
 github\.com
@@ -393,10 +401,11 @@
 Google-Apps-Script
 Google-Calendar-Importer
 Google-HotelAdsVerifier
 Google-HTTP-Java-Client
 Google-Publisher-Plugin
+Google-Read-Aloud
 Google-SearchByImage
 Google-Site-Verification
 Google-Structured-Data-Testing-Tool
 Google-Youtube-Links
 google-xrawler
@@ -467,10 +476,11 @@
 http\.rb\/
 http_get
 HttpComponents
 httphr
 HTTPMon
+HTTPie
 httpRequest
 httpscheck
 httpssites_power
 httpunit
 HttpUrlConnection
@@ -513,10 +523,11 @@
 InterGET
 internet_archive
 Internet Ninja
 InternetSeer
 internetVista monitor
+internetwache
 intraVnews
 IODC
 IOI
 iplabel
 ips-agent
@@ -527,10 +538,11 @@
 Irokez
 isitup\.org
 iskanie
 isUp\.li
 iThemes Sync\/
+IZaBEE
 iZSearch
 JAHHO
 janforman
 Jaunt\/
 Jbrofuzz
@@ -553,10 +565,11 @@
 Kelny\/
 Kerrigan\/
 KeyCDN
 Keyword Density
 Keywords Research
+khttp\/
 KickFire
 KimonoLabs\/
 Kml-Google
 knows\.is
 KOCMOHABT
@@ -597,10 +610,11 @@
 Litemage_walker
 livedoor ScreenShot
 LoadImpactRload
 localsearch-web
 LongURL API
+looid\.com
 looksystems\.net
 ltx71
 lua-resty-http
 lwp-request
 lwp-trivial
@@ -714,10 +728,11 @@
 nominet\.org\.uk
 nominet\.uk
 Norton-Safeweb
 Notifixious
 notifyninja
+NotionEmbedder
 nuhk
 nutch
 Nuzzel
 nWormFeedFinder
 nyawc\/
@@ -726,10 +741,11 @@
 Ocelli\/
 Octopus
 oegp
 Offline Explorer
 Offline Navigator
+OgScrper
 og-scraper
 okhttp
 omgili
 OMSC
 Online Domain Tools
@@ -1177,10 +1193,11 @@
 wmtips
 Woko
 woorankreview
 Word\/
 WordPress\/
+worldping-api
 WordupinfoSearch
 wotbox
 WP Engine Install Performance API
 wpif
 wprecon\.com survey
@@ -1227,9 +1244,10 @@
 Zend\\\Http\\\Client
 Zermelo
 Zeus 
 zgrab
 ZnajdzFoto
+ZnHTTP
 Zombie\.js
 Zoom\.Mac
 ZyBorg
 [a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)
       ].strip.split(/\n+/).freeze