Sha256: 14da8da820f03683b40e63f5a580283aab336f7f6b37bf5cf30f88a7da3fcf55
Contents?: true
Size: 1.58 KB
Versions: 9
Compression:
Stored size: 1.58 KB
Contents
require 'chronicle/etl'
module Chronicle
module ETL
# Abstract class representing an Extractor for an ETL job
class Extractor
extend Chronicle::ETL::Registry::SelfRegistering
include Chronicle::ETL::Configurable
setting :since, type: :time
setting :until, type: :time
setting :limit, type: :numeric
setting :load_after_id
setting :input
# Construct a new instance of this extractor. Options are passed in from a Runner
# == Parameters:
# options::
# Options for configuring this Extractor
def initialize(options = {})
apply_options(options)
end
# Hook called before #extract. Useful for gathering data, initailizing proxies, etc
def prepare; end
# An optional method to calculate how many records there are to extract. Used primarily for
# building the progress bar
def results_count; end
# Entrypoint for this Extractor. Called by a Runner. Expects a series of records to be yielded
def extract
raise NotImplementedError
end
private
# TODO: reimplemenet this
# def handle_continuation
# return unless @config.continuation
# @config.since = @config.continuation.highest_timestamp if @config.continuation.highest_timestamp
# @config.load_after_id = @config.continuation.last_id if @config.continuation.last_id
# end
end
end
end
require_relative 'helpers/input_reader'
require_relative 'csv_extractor'
require_relative 'file_extractor'
require_relative 'json_extractor'
require_relative 'stdin_extractor'
Version data entries
9 entries across 9 versions & 1 rubygems