Contents

module Simple
  module Scraper
    class Finder
      def self.find(url:, query: {}, headers: {})
        default_options[:query] = query
        default_options[:headers] = headers
        yield(Nokogiri::HTML(HTTParty.get(url, default_options)))
      rescue StandardError => e
        Simple::Scraper.logger&.error e
      end

      def self.default_options
        @default_options ||= {
          http_proxyaddr: Simple::Scraper.configuration.proxy_addr,
          http_proxyport: Simple::Scraper.configuration.proxy_port,
          http_proxyuser: Simple::Scraper.configuration.proxy_user,
          http_proxypass: Simple::Scraper.configuration.proxy_pass,
          verify: false
        }
      end
    end
  end
end

Version data entries

1 entries across 1 versions & 1 rubygems

Version	Path
simple-scraper-1.0.0	lib/simple/scraper/finder.rb