lib/miteru/http_client.rb in miteru-0.10.0 vs lib/miteru/http_client.rb in miteru-0.10.1

@@ -1,14 +1,17 @@
 # frozen_string_literal: true
 
-require "http"
 require "down/http"
+require "http"
 require "securerandom"
+require "uri"
 
 module Miteru
   class HTTPClient
     DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
+    URLSCAN_UA = "miteru/#{Miteru::VERSION}"
+
     attr_reader :ssl_context
     def initialize
       ctx = OpenSSL::SSL::SSLContext.new
       ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
       @ssl_context = ctx
@@ -24,11 +27,15 @@
       new.download(url, base_dir)
     end
 
     def get(url, options = {})
       options = options.merge default_options
-      HTTP.follow.timeout(write: 2, connect: 5, read: 10).headers(default_headers).get(url, options)
+
+      HTTP.follow
+          .timeout(write: 2, connect: 5, read: 10)
+          .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
+          .get(url, options)
     end
 
     def self.get(url, options = {})
       new.get url, options
     end
@@ -47,8 +54,17 @@
       { user_agent: DEFAULT_UA }
     end
 
     def default_options
       { ssl_context: ssl_context }
+    end
+
+    def urlscan_headers
+      { user_agent: URLSCAN_UA }
+    end
+
+    def urlscan_url?(url)
+      uri = URI(url)
+      uri.hostname == "urlscan.io"
     end
   end
 end