Contents

require 'aws-sdk'
require 'active_record'
require 'active_support'
require 'elasticity'
require 'highline/import'
require 'thor'

module Elasticrawl
  # S3 locations
  COMMON_CRAWL_BUCKET = 'aws-publicdatasets'
  COMMON_CRAWL_PATH = 'common-crawl/crawl-data'
  SEGMENTS_PATH = 'segments'
  WARC_PATHS = 'warc.paths.gz'
  MAX_SEGMENTS = 256

  require 'elasticrawl/version'

  require 'elasticrawl/config'
  require 'elasticrawl/error'

  require 'elasticrawl/cluster'
  require 'elasticrawl/crawl'
  require 'elasticrawl/crawl_segment'
  require 'elasticrawl/job'
  require 'elasticrawl/combine_job'
  require 'elasticrawl/parse_job'
  require 'elasticrawl/job_step'
end

Version data entries

6 entries across 6 versions & 1 rubygems

Version	Path
elasticrawl-1.1.5	lib/elasticrawl.rb
elasticrawl-1.1.4	lib/elasticrawl.rb
elasticrawl-1.1.3	lib/elasticrawl.rb
elasticrawl-1.1.2	lib/elasticrawl.rb
elasticrawl-1.1.1	lib/elasticrawl.rb
elasticrawl-1.1.0	lib/elasticrawl.rb