Contents

module ETL
  class Engine
  
    class << self
      def process(control_file)
        new().process(control_file)
      end
      
      attr_accessor :logger
      
      def logger
        unless @logger
          @logger = Logger.new('etl.log')
          @logger.level = Logger::DEBUG
        end
        @logger
      end
    end
  
    # Process a control file or object.
    def process(control)
      control = ETL::Control::Control.resolve(control)
      
      pre_process(control)
      
      sources = control.sources
      destinations = control.destinations
      
      sources.each do |source|
        source.each_with_index do |row, index|
          row.each do |name, value|
            # execute transforms
            row[name] = ETL::Transform::Transform.transform(name, value, control.get_transform(name))
          end
          # write the row to the destination
          destinations.each do |destination|
            destination.write(row)
          end
        end
        destinations.each do |destination|
          destination.close
        end
      end
      
      post_process(control)
    end
    
    private
    def pre_process(control)
      control.pre_processors.each do |processor|
        processor.process
      end
    end
    
    def post_process(control)
      control.post_processors.each do |processor|
        processor.process
      end
    end
  end
end

Version data entries

4 entries across 4 versions & 1 rubygems

Version	Path
activewarehouse-etl-0.1.0	lib/etl/engine.rb
activewarehouse-etl-0.2.0	lib/etl/engine.rb
activewarehouse-etl-0.3.0	lib/etl/engine.rb
activewarehouse-etl-0.4.0	lib/etl/engine.rb