Contents

require 'aws-sdk'
require 'active_record'
require 'active_support'
require 'elasticity'
require 'highline/import'
require 'thor'

module Elasticrawl
  # S3 locations
  COMMON_CRAWL_BUCKET = 'commoncrawl'
  COMMON_CRAWL_PATH = 'crawl-data'
  SEGMENTS_PATH = 'segments'
  WARC_PATHS = 'warc.paths.gz'
  MAX_SEGMENTS = 256

  require 'elasticrawl/version'

  require 'elasticrawl/config'
  require 'elasticrawl/error'

  require 'elasticrawl/cluster'
  require 'elasticrawl/crawl'
  require 'elasticrawl/crawl_segment'
  require 'elasticrawl/job'
  require 'elasticrawl/combine_job'
  require 'elasticrawl/parse_job'
  require 'elasticrawl/job_step'
end

Version data entries

3 entries across 3 versions & 1 rubygems

Version	Path
elasticrawl-1.1.8	lib/elasticrawl.rb
elasticrawl-1.1.7	lib/elasticrawl.rb
elasticrawl-1.1.6	lib/elasticrawl.rb