Contents

module ETL #:nodoc:
  module Processor #:nodoc:
    # Processor which is used to bulk import data into a target database
    class BulkImportProcessor < ETL::Processor::Processor
      attr_reader :file, :target, :truncate, :columns
      def initialize(control, configuration)
        super
        @file = File.join(File.dirname(control.file), configuration[:file])
        @target = configuration[:target]
        @truncate = configuration[:truncate] ||= false
        @columns = configuration[:columns]
        connect
      end
      def process
        # columns = control.destinations.first.order.join(',') # TODO: support multiple destinations?
        conn = ETL::ActiveRecord::Base.connection
        conn.transaction do
          # TODO: Support all database types
          # Since LOCAL is used this must be allowed by both the client and server
          conn.truncate(target[:table]) if truncate
          options = {}
          options[:columns] = columns
          conn.bulk_load(file, target[:table], options)
        end
      end
      
      private
      # Connect to the database
      def connect
        ETL::ActiveRecord::Base.establish_connection(
            :adapter  => (target[:adapter] || :mysql),
            :username => (target[:username] || 'root'),
            :host     => (target[:host] || 'localhost'),
            :password => target[:password],
            :database => target[:database] 
        )
      end
    end
  end
end

Version data entries

3 entries across 3 versions & 1 rubygems

Version	Path
activewarehouse-etl-0.5.0	lib/etl/processor/bulk_import_processor.rb
activewarehouse-etl-0.5.1	lib/etl/processor/bulk_import_processor.rb
activewarehouse-etl-0.5.2	lib/etl/processor/bulk_import_processor.rb