reso.rake in reso-0.1.1.0

- old
+ new

@@ -30,20 +30,23 @@
       return stream[0..pos-1] if pos
     end
     nil # Just in cases
   end
 
-  def create_queued_listing doc, import
+  def create_queued_listing_and_return_listing_key doc, import
     begin
       doc.css(import.repeating_element).each do |o|
         listing_data = {}
         Hash.from_xml(o.to_xml)[import.repeating_element].each_pair{|key, value| listing_data[key] = value }
-        QueuedListing.create(import: import, listing_data: listing_data)
+        queued_listing = QueuedListing.new(import: import, listing_data: listing_data)
+        queued_listing.save
+        return Mapper::unique_identifier(queued_listing)
       end
     rescue Exception => e
       puts e.inspect
       exit if Rails.env.development?
+      return nil
     end
   end
   
   def uncompress_and_return_new_filepath filepath
     output_path = [filepath, '.xml'].join
@@ -91,11 +94,11 @@
 
     args.with_defaults(:import_token => "reso")
     import = Import.find_by(token: args.import_token)
 
     unless import.blank?
-      l, count, stream = 0, 0, ''
+      l, count, incoming_listing_keys, stream = 0, 0, [], ''
       open_tag, close_tag = get_open_and_closing_tag_for import.repeating_element
 
       # Grab a file to work with
       filepath = download_feed_to_import import
       filepath = uncompress_and_return_new_filepath(filepath) if filepath.split('.').last.downcase == 'gz'
@@ -108,18 +111,21 @@
       File.foreach(filepath) do |line|
         stream += line
         while (from_here = stream.index(open_tag)) && (to_there = stream.index(close_tag))
           xml = stream[from_here..to_there + (close_tag.length-1)]
           doc = Nokogiri::XML([xml_header, xml].join).remove_namespaces!
-          create_queued_listing doc, import
+          incoming_listing_keys << create_queued_listing_and_return_listing_key(doc, import)
           stream.gsub!(xml, '')
           if ((l += 1) % 1000).zero?
             puts "#{l}\t#{l/(Time.now - start)}" if Rails.env.development?
           end
           GC.start if (l % 100).zero?
         end
       end
-      puts "#{l} - #{l/(Time.now - start)} listings/s" if Rails.env.development?
+      puts "Import speed: #{l/(Time.now - start)} listings/s" if Rails.env.development?
+      puts "Found #{l} new listings." if Rails.env.development?
+      stale_listing_keys = import.remove_listings_no_longer_present(incoming_listing_keys)
+      puts "Removed #{stale_listing_keys.count} old listings." if Rails.env.development?
       File.delete(filepath)
     end
   end
 
 end