Contents

require 'thread'
require 'watir-webdriver'
require_relative 'request_scraper'
require_relative 'user_operations'

class ParallelRequestScraper

  def initialize(url, user, thread_limit)
    @url = url
    @user = user
    @thread_limit = thread_limit
  end

  def scrape(request_ids)
    queue = request_ids.clone
    threads = []
    requests = []
    mutex = Mutex.new

    @thread_limit.times do
      threads << Thread.new do
        b = Watir::Browser.new
        navigator = Navigator.new b, @url
        user_operations = UserOperations.new b, navigator
        user_operations.login @user
        request_scraper = RequestScraper.new b, navigator
        until queue.empty?
          id = nil
          mutex.synchronize do
            id = queue.pop
          end
          break unless id
          requests << request_scraper.scrape(id)
        end
        b.close
      end
    end
    threads.each { |t| t.join }
    requests
  end

end

Version data entries

3 entries across 3 versions & 1 rubygems

Version	Path
help_desk_dashboard-0.0.3	lib/help-desk-dashboard/providers/track-it/2003.10.1/parallel_request_scraper.rb
help_desk_dashboard-0.0.2	lib/help-desk-dashboard/providers/track-it/2003.10.1/parallel_request_scraper.rb
help_desk_dashboard-0.0.1	lib/help-desk-dashboard/providers/track-it/2003.10.1/parallel_request_scraper.rb