Contents

module Sites
  class Base

    attr_reader :doc, :url, :rows_count, :jobs_count

    def initialize
      @url = "#{self.class::HOST}#{self.class::PATH}"
      @doc = Nokogiri::HTML(open_page(@url))
      @current_time = Time.now
      @timestamp = @current_time.strftime("%Y%m%d%H%M%S")
      @rows_count = 0
      @jobs_count = get_jobs_count
    end

    private

    def open_page(url)
      sleep(rand(delay_range)) unless ENV['RAILS_ENV'] == 'test' # less mechanical behaviour
      options = ENV['RAILS_ENV'] == 'test' ? {} : { 'User-Agent' => user_agent }
      open(url, options)
    end

    def delay_range
      RemoteJobScraper.configuration.delay_range
    end

    def user_agent
      Support::UserAgent::LIST.sample
    end

    def filepath
      return test_filepath if ENV["RAILS_ENV"] == 'test'
      "#{self.class::STORE_DIR}/#{@timestamp}.csv"
    end

    def test_filepath
      "spec/fixtures/data/#{underscore(self.class.name.split('::').last)}/#{@timestamp}.csv"
    end

    # https://stackoverflow.com/a/5622585
    def underscore(camel_cased_word)
      word = camel_cased_word.dup
      word.gsub!(/::/, '/')
      word.gsub!(/([A-Z]+)([A-Z][a-z])/,'\1_\2')
      word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
      word.tr!("-", "_")
      word.downcase!
      word
    end
  end
end

Version data entries

3 entries across 3 versions & 1 rubygems

Version	Path
remote_job_scraper-0.6.0	lib/sites/base.rb
remote_job_scraper-0.5.0	lib/sites/base.rb
remote_job_scraper-0.4.4	lib/sites/base.rb