jobs.rb in chronicle-etl-0.4.0

- old
+ new

@@ -1,23 +1,38 @@
 require 'pp'
+
 module Chronicle
   module ETL
     module CLI
       # CLI commands for working with ETL jobs
       class Jobs < SubcommandBase
         default_task "start"
         namespace :jobs
 
+        class_option :name, aliases: '-j', desc: 'Job configuration name'
+
         class_option :extractor, aliases: '-e', desc: "Extractor class. Default: stdin", banner: 'extractor-name'
         class_option :'extractor-opts', desc: 'Extractor options', type: :hash, default: {}
         class_option :transformer, aliases: '-t', desc: 'Transformer class. Default: null', banner: 'transformer-name'
         class_option :'transformer-opts', desc: 'Transformer options', type: :hash, default: {}
         class_option :loader, aliases: '-l', desc: 'Loader class. Default: stdout', banner: 'loader-name'
         class_option :'loader-opts', desc: 'Loader options', type: :hash, default: {}
-        class_option :name, aliases: '-j', desc: 'Job configuration name'
 
-        map run: :start  # Thor doesn't like `run` as a command name
+        # This is an array to deal with shell globbing
+        class_option :input, aliases: '-i', desc: 'Input filename or directory', default: [], type: 'array', banner: 'FILENAME'
+        class_option :since, desc: "Load records SINCE this date. Overrides job's `load_since` configuration option in extractor's options", banner: 'DATE'
+        class_option :until, desc: "Load records UNTIL this date", banner: 'DATE'
+        class_option :limit, desc: "Only extract the first LIMIT records", banner: 'N'
+
+        class_option :output, aliases: '-o', desc: 'Output filename', type: 'string'
+        class_option :fields, desc: 'Output only these fields', type: 'array', banner: 'field1 field2 ...'
+
+        class_option :log_level, desc: 'Log level (debug, info, warn, error, fatal)', default: 'info'
+        class_option :verbose, aliases: '-v', desc: 'Set log level to verbose', type: :boolean
+
+        # Thor doesn't like `run` as a command name
+        map run: :start
         desc "run", "Start a job"
         option :log_level, desc: 'Log level (debug, info, warn, error, fatal)', default: 'info'
         option :verbose, aliases: '-v', desc: 'Set log level to verbose', type: :boolean
         option :dry_run, desc: 'Only run the extraction and transform steps, not the loading', type: :boolean
         long_desc <<-LONG_DESC
@@ -67,11 +82,11 @@
             loader = r[:loader][:name] if r[:loader]
 
             [job, extractor, transformer, loader]
           end
 
-          headers = ['name', 'extractor', 'transformer', 'loader'].map{|h| h.upcase.bold }
+          headers = ['name', 'extractor', 'transformer', 'loader'].map { |h| h.upcase.bold }
 
           table = TTY::Table.new(headers, job_details)
           puts table.render(indent: 0, padding: [0, 2])
         end
 
@@ -88,32 +103,46 @@
 
         # Create job definition by reading config file and then overwriting with flag options
         def build_job_definition(options)
           definition = Chronicle::ETL::JobDefinition.new
           definition.add_config(load_job_config(options[:name]))
-          definition.add_config(process_flag_options(options))
+          definition.add_config(process_flag_options(options).transform_keys(&:to_sym))
           definition
         end
 
         def load_job_config name
           Chronicle::ETL::Config.load_job_from_config(name)
         end
 
         # Takes flag options and turns them into a runner config
         def process_flag_options options
+          extractor_options = options[:'extractor-opts'].merge({
+            filename: (options[:input] if options[:input].any?),
+            since: options[:since],
+            until: options[:until],
+            limit: options[:limit],
+          }.compact)
+
+          transformer_options = options[:'transformer-opts']
+
+          loader_options = options[:'loader-opts'].merge({
+            output: options[:output],
+            fields: options[:fields]
+          }.compact)
+
           {
             dry_run: options[:dry_run],
             extractor: {
               name: options[:extractor],
-              options: options[:'extractor-opts']
+              options: extractor_options
             }.compact,
             transformer: {
               name: options[:transformer],
-              options: options[:'transformer-opts']
+              options: transformer_options
             }.compact,
             loader: {
               name: options[:loader],
-              options: options[:'loader-opts']
+              options: loader_options
             }.compact
           }
         end
       end
     end