Contents

require 'tjcrawler/version'
require 'tjcrawler/crawler'
require 'tjcrawler/parser'
require 'tjcrawler/config'

module Tjcrawler
  module_function
  def start_crawler
    config = Config.new
    yield config
    set_db(config)
    threads = []
    config.threads.times.each do
      threads << Thread.new{ Tjcrawler::Crawler.new(config.css_selector).start }
    end
    threads.each(&:join)
  end

  def start_parser
    config = Config.new
    yield config
    set_db(config)
    threads = []
    config.threads.times.each do
      threads << Thread.new{ Tjcrawler::Parser.new(&config.proc).start }
    end
    threads.each(&:join)
  end

  def set_db config
    ActiveRecord::Base.establish_connection config.db_setting
  end
end

Version data entries

2 entries across 2 versions & 1 rubygems

Version	Path
tjcrawler-0.0.2	lib/tjcrawler.rb
tjcrawler-0.0.1	lib/tjcrawler.rb