lib/chronicle/etl/job.rb in chronicle-etl-0.5.5 vs lib/chronicle/etl/job.rb in chronicle-etl-0.6.1

- old
+ new

@@ -1,5 +1,7 @@ +# frozen_string_literal: true + require 'forwardable' module Chronicle module ETL # A runner job @@ -10,17 +12,17 @@ extend Forwardable def_delegators :@job_definition, :dry_run? attr_accessor :name, - :extractor_klass, - :extractor_options, - :transformer_klass, - :transformer_options, - :loader_klass, - :loader_options, - :job_definition + :extractor_klass, + :extractor_options, + :transformer_klasses, + :transformer_options, + :loader_klass, + :loader_options, + :job_definition # TODO: build a proper id system alias id name def initialize(job_definition) @@ -37,34 +39,50 @@ def instantiate_extractor @extractor_klass = @job_definition.extractor_klass @extractor_klass.new(@extractor_options) end - def instantiate_transformer(extraction) - @transformer_klass = @job_definition.transformer_klass - @transformer_klass.new(extraction, @transformer_options) + def instantiate_transformers + @job_definition.transformer_klasses.each_with_index.map do |klass, i| + klass.new(@transformer_options[i] || {}) + end end def instantiate_loader @loader_klass = @job_definition.loader_klass @loader_klass.new(@loader_options) end def save_log? # TODO: this needs more nuance - return !id.nil? + !id.nil? end def to_s - output = "Job" - output += " '#{name}'".bold if name - output += "\n" - output += " → Extracting from #{@job_definition.extractor_klass.description}\n" - output += " → Transforming #{@job_definition.transformer_klass.description}\n" - output += " → Loading to #{@job_definition.loader_klass.description}\n" + output = "Job summary\n".upcase.bold + # output = "" + output += "#{name}:\n" if name + output += "→ Extracting from #{@job_definition.extractor_klass.description}\n" + output += options_to_s(@extractor_options) + + @job_definition.transformer_klasses.each do |klass| + output += "→ Transforming #{klass.description}\n" + end + # TODO: transformer options + output += "→ Loading to #{@job_definition.loader_klass.description}\n" + output += options_to_s(@loader_options) + output end private + + def options_to_s(options, indent: 4) + output = '' + options.each do |k, v| + output += "#{' ' * indent}#{k.to_s.light_blue}: #{v}\n" + end + output + end def set_continuation continuation = Chronicle::ETL::JobLogger.load_latest(@id) @extractor_options[:continuation] = continuation end