Contents

require 'tjcrawler/page'
module Tjcrawler
  class Scheduler
    def start
      @queue = Queue.new
      @queue << @seed_url
      @crawler_number.times do
        @threads << Thread.new do
          crawler = Cralwer.new(css: '.entry-title > a')
          urls = crawler.crawl(@queue.pop)
          urls.each{ |url| @queue << url }
        end
      end
      @threads.each(&:join)
    end
  end
end

Version data entries

2 entries across 2 versions & 1 rubygems

Version	Path
tjcrawler-0.0.2	lib/tjcrawler/scheduler.rb
tjcrawler-0.0.1	lib/tjcrawler/scheduler.rb