Contents

# frozen_string_literal: true

require "down/http"
require "http"
require "securerandom"
require "uri"

module Miteru
  class HTTPClient
    DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
    URLSCAN_UA = "miteru/#{Miteru::VERSION}"

    attr_reader :ssl_context
    def initialize
      ctx = OpenSSL::SSL::SSLContext.new
      ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
      @ssl_context = ctx
    end

    def download(url, destination)
      down = Down::Http.new(default_options) { |client| client.headers(default_headers) }
      down.download(url, destination: destination)
      destination
    end

    def self.download(url, base_dir = "/tmp")
      new.download(url, base_dir)
    end

    def get(url, options = {})
      options = options.merge default_options

      HTTP.follow
          .timeout(write: 2, connect: 5, read: 10)
          .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
          .get(url, options)
    end

    def self.get(url, options = {})
      new.get url, options
    end

    def post(url, options = {})
      HTTP.post url, options
    end

    def self.post(url, options = {})
      new.post url, options
    end

    private

    def default_headers
      { user_agent: DEFAULT_UA }
    end

    def default_options
      { ssl_context: ssl_context }
    end

    def urlscan_headers
      { user_agent: URLSCAN_UA }
    end

    def urlscan_url?(url)
      uri = URI(url)
      uri.hostname == "urlscan.io"
    end
  end
end

Version data entries

17 entries across 17 versions & 1 rubygems

Version	Path
miteru-0.12.10	lib/miteru/http_client.rb
miteru-0.12.9	lib/miteru/http_client.rb
miteru-0.12.8	lib/miteru/http_client.rb
miteru-0.12.7	lib/miteru/http_client.rb
miteru-0.12.6	lib/miteru/http_client.rb
miteru-0.12.5	lib/miteru/http_client.rb
miteru-0.12.4	lib/miteru/http_client.rb
miteru-0.12.3	lib/miteru/http_client.rb
miteru-0.12.2	lib/miteru/http_client.rb
miteru-0.12.1	lib/miteru/http_client.rb
miteru-0.12.0	lib/miteru/http_client.rb
miteru-0.11.3	lib/miteru/http_client.rb
miteru-0.11.2	lib/miteru/http_client.rb
miteru-0.11.1	lib/miteru/http_client.rb
miteru-0.11.0	lib/miteru/http_client.rb
miteru-0.10.2	lib/miteru/http_client.rb
miteru-0.10.1	lib/miteru/http_client.rb