solr_indexer.rb in supernova-0.4.15

- old
+ new

@@ -1,11 +1,12 @@
 require "json"
 require "fileutils"
+require "time"
 
 class Supernova::SolrIndexer
   attr_accessor :options, :db, :ids, :max_rows_to_direct_index, :local_solr
-  attr_writer :index_file_path
+  attr_writer :index_file_path, :debug
   
   MAX_ROWS_TO_DIRECT_INDEX = 100
   
   include Supernova::Solr
   
@@ -68,10 +69,21 @@
     self.max_rows_to_direct_index ||= MAX_ROWS_TO_DIRECT_INDEX
     self.options = options
     self.ids ||= :all
   end
   
+  def debug(message)
+    response = true
+    time = Benchmark.realtime do
+      response = yield if block_given?
+    end
+    if @debug == true
+      puts "%s: %s" % [Time.now.iso8601(3), message.gsub("%TIME%", "%.3f" % time)]
+    end
+    response
+  end
+  
   def index!
     index_query(query_to_index)
   end
   
   def map_for_solr(row)
@@ -97,11 +109,11 @@
     self.class.field_definitions.each do |field, options|
       if hash.has_key?(field.to_s)
         value = hash.delete(field.to_s)
         if options[:type] == :date 
           value = Time.utc(value.year, value.month, value.day) if value.is_a?(Date)
-          value = value.utc.iso8601 
+          value = value.utc.iso8601 if value
         end
         hash["#{field}_#{self.class.suffix_from_type(options[:type])}"] = value
       end
     end
     hash["type"] = self.class.clazz.to_s if self.class.clazz
@@ -169,37 +181,54 @@
   def sql_date_column_from_field(field)
     %(IF(#{field} IS NULL, NULL, CONCAT(REPLACE(#{field}, " ", "T"), "Z")) AS #{field}_dt)
   end
   
   def query_db(query)
-    db.send(db.respond_to?(:query) ? :query : :select_all, query)
+    if db.respond_to?(:query)
+      db.query(query, :as => :hash)
+    else
+      db.select_all(query)
+    end
   end
   
+  def rows(query = nil)
+    query_db(query || query_to_index)
+  end
+  
   def solr_rows_to_index_for_query(query)
     query_db(query).map do |row|
       map_for_solr(row)
     end
   end
   
   def index_query(query)
-    rows = solr_rows_to_index_for_query(query)
+    debug "getting rows for #{query[0,100]}"
+    rows = debug "got all rows in %TIME%" do
+      solr_rows_to_index_for_query(query)
+    end
     if self.max_rows_to_direct_index < rows.count
-      index_with_json_file(rows)
+      debug "indexed #{rows.length} rows with json in %TIME%" do
+        index_with_json_file(rows)
+      end
     else
-      index_directly(rows)
+      debug "indexed #{rows.length} rows directly in %TIME%" do
+        index_directly(rows)
+      end
     end
   end
   
-  def index_directly(rows, &block)
+  def index_directly(rows)
     rows.each do |row|
       row = Supernova::Solr.connection.add(row)
     end
     Supernova::Solr.connection.commit if rows.any?
   end
   
-  def index_with_json_file(rows, &block)
-    rows.each do |row|
-      write_to_file(row)
+  def index_with_json_file(rows)
+    debug "wrote #{rows.count} rows to the json file in %TIME%" do
+      rows.each do |row|
+        write_to_file(row)
+      end
     end
     finish
   end
   
   def ids_given?
\ No newline at end of file