metadata_job.rb in remi-0.3.0

- old
+ new
@@ -1,38 +1,43 @@
 require_relative 'all_jobs_shared'
 ENV['TZ'] = 'UTC'
 
-class MetadataJob
-  include AllJobsShared
+class MetadataJob < Remi::Job
+  source :source_data do
+    fields(
+      {
+        :activity_id      => { from: 'in', in: true, cdc_type: 2 },
+        :student_id       => { from: 'in', in: true, type: :string, cdc_type: 2 },
+        :student_dob      => { from: 'in', in: true, type: :date, in_format: '%m/%d/%Y', out_format: '%Y-%m-%d', cdc_type: 2 },
+        :activity_type    => { from: 'in', in: true, type: :string, valid_values: ['A', 'B', 'C'], cdc_type: 2 },
+        :activity_counter => { from: 'in', in: true, type: :integer, cdc_type: 2 },
+        :activity_score   => { from: 'in', in: true, type: :float, cdc_type: 2 },
+        :activity_cost    => { from: 'in', in: true, type: :decimal, precision: 8, scale: 2, cdc_type: 2 },
+        :activity_date    => { from: 'in', in: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S', cdc_type: 2 },
+        :source_filename  => { from: 'in', in: true, type: :string, cdc_type: 1 }
+      }
+    )
+  end
 
-  define_source :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :activity_id      => { from: 'in', in: true, cdc_type: 2 },
-      :student_id       => { from: 'in', in: true, type: :string, cdc_type: 2 },
-      :student_dob      => { from: 'in', in: true, type: :date, in_format: '%m/%d/%Y', out_format: '%Y-%m-%d', cdc_type: 2 },
-      :activity_type    => { from: 'in', in: true, type: :string, valid_values: ['A', 'B', 'C'], cdc_type: 2 },
-      :activity_counter => { from: 'in', in: true, type: :integer, cdc_type: 2 },
-      :activity_score   => { from: 'in', in: true, type: :float, cdc_type: 2 },
-      :activity_cost    => { from: 'in', in: true, type: :decimal, precision: 8, scale: 2, cdc_type: 2 },
-      :activity_date    => { from: 'in', in: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S', cdc_type: 2 },
-      :source_filename  => { from: 'in', in: true, type: :string, cdc_type: 1 }
-    }
+  target :target_data do
+    encoder Remi::Encoder::CsvFile.new path: "#{Remi::Settings.work_dir}/target_data.csv"
 
-  define_target :target_data, Remi::DataTarget::CsvFile,
-    path: "#{Remi::Settings.work_dir}/target_data.csv",
-    fields: {
-      :activity_id      => { from: 'out', out: true },
-      :student_id       => { from: 'out', out: true, type: :string },
-      :student_dob      => { from: 'out', out: true, type: :date, in_format: '%m/%d/%Y', out_format: '%Y-%m-%d' },
-      :activity_type    => { from: 'out', out: true, type: :string, valid_values: ['A', 'B', 'C'] },
-      :activity_counter => { from: 'out', out: true, type: :integer },
-      :activity_score   => { from: 'out', out: true, type: :float },
-      :activity_cost    => { from: 'out', out: true, type: :decimal, precision: 8, scale: 2 },
-      :activity_date    => { from: 'out', out: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S' },
-      :source_filename  => { from: 'out', out: true, type: :string, cdc_type: 1 }
-    }
+    fields(
+      {
+        :activity_id      => { from: 'out', out: true },
+        :student_id       => { from: 'out', out: true, type: :string },
+        :student_dob      => { from: 'out', out: true, type: :date, in_format: '%m/%d/%Y', out_format: '%Y-%m-%d' },
+        :activity_type    => { from: 'out', out: true, type: :string, valid_values: ['A', 'B', 'C'] },
+        :activity_counter => { from: 'out', out: true, type: :integer },
+        :activity_score   => { from: 'out', out: true, type: :float },
+        :activity_cost    => { from: 'out', out: true, type: :decimal, precision: 8, scale: 2 },
+        :activity_date    => { from: 'out', out: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S' },
+        :source_filename  => { from: 'out', out: true, type: :string, cdc_type: 1 }
+      }
+    )
+  end
 
-  define_transform :main do
+  transform :main do
     source_data.enforce_types
 
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df, source_metadata: source_data.fields, target_metadata: target_data.fields) do
       target_data.fields.keys.each do |field|
         map source(field) .target(field)