# frozen_string_literal: true # rubocop:disable Style/Documentation require "logger" require "uri" # Plumbing require_relative "wayfarer/configuration" # Routing require_relative "wayfarer/routing/rule" require_relative "wayfarer/routing/uri_rule" require_relative "wayfarer/routing/host_rule" require_relative "wayfarer/routing/path_rule" require_relative "wayfarer/routing/query_rule" require_relative "wayfarer/routing/protocol_rule" require_relative "wayfarer/routing/filetypes_rule" require_relative "wayfarer/routing/custom_rule" require_relative "wayfarer/routing/router" # Networking require_relative "wayfarer/http_adapters/net_http_adapter" require_relative "wayfarer/http_adapters/selenium_adapter" require_relative "wayfarer/http_adapters/adapter_pool" # Parsers require_relative "wayfarer/parsers/xml_parser" require_relative "wayfarer/parsers/json_parser" # Frontiers require_relative "wayfarer/frontiers/frontier" require_relative "wayfarer/frontiers/memory_frontier" require_relative "wayfarer/frontiers/redis_frontier" require_relative "wayfarer/frontiers/normalize_uris" unless RUBY_PLATFORM == "java" require_relative "wayfarer/frontiers/memory_trie_frontier" require_relative "wayfarer/frontiers/memory_bloomfilter" require_relative "wayfarer/frontiers/redis_bloomfilter" end # Processing require_relative "wayfarer/crawl" require_relative "wayfarer/crawl_observer" require_relative "wayfarer/locals" require_relative "wayfarer/job" require_relative "wayfarer/finders" require_relative "wayfarer/page" require_relative "wayfarer/dispatcher" require_relative "wayfarer/processor" module Wayfarer VERSION = "0.0.3" def self.logger return @logger if @logger @logger = Logger.new(STDOUT) @logger.level = Logger::WARN @logger end def self.config @config ||= Configuration.new yield(@config) if block_given? @config end end