Contents

module CMSScanner
  # Options available in the Browser
  class Browser
    OPTIONS = [
      :cache_ttl,
      :cookie_jar,
      :cookie_string,
      :connect_timeout,
      :http_auth,
      :max_threads,
      :proxy,
      :proxy_auth,
      :random_user_agent,
      :request_timeout,
      :throttle,
      :user_agent,
      :user_agents_list,
      :vhost
    ].freeze

    attr_accessor(*OPTIONS)

    # @return [ String ]
    def default_user_agent
      "#{NS} v#{NS::VERSION}"
    end

    def hydra
      @hydra ||= Typhoeus::Hydra.new(max_concurrency: max_threads || 1)
    end

    # @param [ Hash ] options
    def load_options(options = {})
      OPTIONS.each do |sym|
        send("#{sym}=", options[sym]) if options.key?(sym)
      end
    end

    # Set the threads attribute and update
    # the max_concurrency of Typhoeus::Hydra
    #
    # If the throttle attribute is > 0, max_threads will be forced to 1
    #
    # @param [ Integer ] number
    def max_threads=(number)
      @max_threads = number.to_i > 0 && throttle.zero? ? number.to_i : 1

      hydra.max_concurrency = @max_threads
    end

    # @return [ String ] The user agent
    def user_agent
      @user_agent ||= random_user_agent ? user_agents.sample : default_user_agent
    end

    # @return [ Array<String> ]
    def user_agents
      return @user_agents if @user_agents

      @user_agents = []

      File.open(user_agents_list).each do |line|
        next if line == "\n" || line[0, 1] == '#'
        @user_agents << line.chomp
      end

      @user_agents
    end

    # @return [ String ] The path to the user agents list
    def user_agents_list
      @user_agents_list ||= File.join(APP_DIR, 'user_agents.txt')
    end

    # @param [ value ] The throttle time in milliseconds
    #
    # if value > 0, the max_threads will be set to 1
    def throttle=(value)
      @throttle = value.to_i.abs / 1000.0

      self.max_threads = 1 if @throttle > 0
    end

    def trottle!
      sleep(throttle) if throttle > 0
    end
  end
end

Version data entries

6 entries across 6 versions & 1 rubygems

Version	Path
cms_scanner-0.0.37.9	lib/cms_scanner/browser/options.rb
cms_scanner-0.0.37.8	lib/cms_scanner/browser/options.rb
cms_scanner-0.0.37.7	lib/cms_scanner/browser/options.rb
cms_scanner-0.0.37.6	lib/cms_scanner/browser/options.rb
cms_scanner-0.0.37.5	lib/cms_scanner/browser/options.rb
cms_scanner-0.0.37.4	lib/cms_scanner/browser/options.rb