lib/loader/loader.rb in myreplicator-0.0.16 vs lib/loader/loader.rb in myreplicator-0.0.17

@@ -1,39 +1,69 @@
 require "exporter"
 
 module Myreplicator
   class Loader
     
+    @queue = :myreplicator_load # Provided for Resque
+    
     def initialize *args
       options = args.extract_options!
     end
     
     def tmp_dir
       @tmp_dir ||= File.join(Myreplicator.app_root,"tmp", "myreplicator")
     end
 
-    def load
+    ##
+    # Main method provided for resque
+    ##
+    def self.perform
+      load # Kick off the load process
+    end
+
+    ##
+    # Kicks off all initial loads first and then all incrementals
+    # Looks at metadata files stored locally
+    ##
+    def self.load
       initials = []
       incrementals = []
 
+      # Read all metadata files
       metadata_files.each do |metadata|
         if metadata.export_type == "initial"
           initials << metadata
         else
           incrementals << metadata
         end
       end
-      
+
+      # Load all new tables
       initials.each do |metadata| 
         puts metadata.table
-        initial_load metadata
+
+        Log.run(:job_type => "loader", 
+                :name => "initial_import", 
+                :file => metadata.filename, 
+                :export_id => metadata.export_id) do |log|
+          
+          initial_load metadata
+        end
+
         cleanup metadata
       end
 
+      # Load all incremental files
       incrementals.each do |metadata|
         puts metadata.table
-        incremental_load metadata
+        Log.run(:job_type => "loader", 
+                :name => "incremental_import", 
+                :file => metadata.filename, 
+                :export_id => metadata.export_id) do |log|
+
+          incremental_load metadata
+        end
         cleanup metadata
       end
     end
 
     ##
@@ -45,31 +75,42 @@
       metadata.zipped = false
 
       cmd = ImportSql.initial_load(:db => exp.destination_schema,
                                    :filepath => metadata.destination_filepath(tmp_dir))      
       puts cmd
+
       result = `#{cmd}` # execute
       unless result.nil?
         if result.size > 0
           raise Exceptions::LoaderError.new("Initial Load #{metadata.filename} Failed!\n#{result}") 
         end
       end
+      
     end
 
     ##
     # Loads data incrementally
     # Uses the values specified in the metadatta object
     ##
     def incremental_load metadata
       exp = Export.find(metadata.export_id)
       unzip(metadata.filename)
       metadata.zipped = false
+      
+      options = {:table_name => exp.table_name, :db => exp.destination_schema,
+        :filepath => metadata.destination_filepath(tmp_dir)}
 
-      cmd = ImportSql.load_data_infile(:table_name => exp.table_name, 
-                                       :db => exp.destination_schema,
-                                       :filepath => metadata.destination_filepath(tmp_dir))
+      if metadata.export_type == "incremental_outfile"
+        options[:fields_terminated_by] = ";~;"
+        options[:lines_terminated_by] = "\\n"
+      end
+
+      cmd = ImportSql.load_data_infile(options)
+
       puts cmd
+      
       result = `#{cmd}` # execute
+      
       unless result.nil?
         if result.size > 0
           raise Exceptions::LoaderError.new("Incremental Load #{metadata.filename} Failed!\n#{result}") 
         end
       end