Contents

# frozen_string_literal: true

require "down/http"
require "http"
require "uri"

module Miteru
  class HTTPClient
    DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
    URLSCAN_UA = "miteru/#{Miteru::VERSION}".freeze

    attr_reader :ssl_context

    def initialize
      ctx = OpenSSL::SSL::SSLContext.new
      ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
      @ssl_context = ctx
    end

    def download(url, destination)
      down = Down::Http.new(**default_options) { |client| client.headers(**default_headers) }
      down.download(url, destination: destination)
      destination
    end

    def head(url, options = {})
      options = options.merge default_options

      HTTP.follow
        .timeout(3)
        .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
        .head(url, options)
    end

    def get(url, options = {})
      options = options.merge default_options

      HTTP.follow
        .timeout(write: 2, connect: 5, read: 10)
        .headers(urlscan_url?(url) ? urlscan_headers : default_headers)
        .get(url, options)
    end

    def post(url, options = {})
      HTTP.post url, options
    end

    class << self
      def download(url, base_dir = "/tmp")
        new.download(url, base_dir)
      end

      def get(url, options = {})
        new.get url, options
      end

      def post(url, options = {})
        new.post url, options
      end

      def head(url, options = {})
        new.head url, options
      end
    end

    private

    def default_headers
      { user_agent: DEFAULT_UA }
    end

    def default_options
      { ssl_context: ssl_context }
    end

    def urlscan_headers
      { user_agent: URLSCAN_UA }
    end

    def urlscan_url?(url)
      uri = URI(url)
      uri.hostname == "urlscan.io"
    end
  end
end

Version data entries

4 entries across 4 versions & 1 rubygems

Version	Path
miteru-1.1.0	lib/miteru/http_client.rb
miteru-1.0.2	lib/miteru/http_client.rb
miteru-1.0.1	lib/miteru/http_client.rb
miteru-1.0.0	lib/miteru/http_client.rb