lib/chronicle/etl/job.rb in chronicle-etl-0.5.5 vs lib/chronicle/etl/job.rb in chronicle-etl-0.6.1
- old
+ new
@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
require 'forwardable'
module Chronicle
module ETL
# A runner job
@@ -10,17 +12,17 @@
extend Forwardable
def_delegators :@job_definition, :dry_run?
attr_accessor :name,
- :extractor_klass,
- :extractor_options,
- :transformer_klass,
- :transformer_options,
- :loader_klass,
- :loader_options,
- :job_definition
+ :extractor_klass,
+ :extractor_options,
+ :transformer_klasses,
+ :transformer_options,
+ :loader_klass,
+ :loader_options,
+ :job_definition
# TODO: build a proper id system
alias id name
def initialize(job_definition)
@@ -37,34 +39,50 @@
def instantiate_extractor
@extractor_klass = @job_definition.extractor_klass
@extractor_klass.new(@extractor_options)
end
- def instantiate_transformer(extraction)
- @transformer_klass = @job_definition.transformer_klass
- @transformer_klass.new(extraction, @transformer_options)
+ def instantiate_transformers
+ @job_definition.transformer_klasses.each_with_index.map do |klass, i|
+ klass.new(@transformer_options[i] || {})
+ end
end
def instantiate_loader
@loader_klass = @job_definition.loader_klass
@loader_klass.new(@loader_options)
end
def save_log?
# TODO: this needs more nuance
- return !id.nil?
+ !id.nil?
end
def to_s
- output = "Job"
- output += " '#{name}'".bold if name
- output += "\n"
- output += " → Extracting from #{@job_definition.extractor_klass.description}\n"
- output += " → Transforming #{@job_definition.transformer_klass.description}\n"
- output += " → Loading to #{@job_definition.loader_klass.description}\n"
+ output = "Job summary\n".upcase.bold
+ # output = ""
+ output += "#{name}:\n" if name
+ output += "→ Extracting from #{@job_definition.extractor_klass.description}\n"
+ output += options_to_s(@extractor_options)
+
+ @job_definition.transformer_klasses.each do |klass|
+ output += "→ Transforming #{klass.description}\n"
+ end
+ # TODO: transformer options
+ output += "→ Loading to #{@job_definition.loader_klass.description}\n"
+ output += options_to_s(@loader_options)
+ output
end
private
+
+ def options_to_s(options, indent: 4)
+ output = ''
+ options.each do |k, v|
+ output += "#{' ' * indent}#{k.to_s.light_blue}: #{v}\n"
+ end
+ output
+ end
def set_continuation
continuation = Chronicle::ETL::JobLogger.load_latest(@id)
@extractor_options[:continuation] = continuation
end