lib/sup/poll.rb in sup-0.0.6 vs lib/sup/poll.rb in sup-0.0.7

@@ -22,11 +22,11 @@
     BufferManager.flash "Polling for new messages..."
     num, numi = buffer.mode.poll
     if num > 0
       BufferManager.flash "Loaded #{num} new messages, #{numi} to inbox." 
     else
-      BufferManager.flash "No new messages."
+      BufferManager.flash "No new messages." 
     end
     [num, numi]
   end
 
   def start_thread
@@ -36,69 +36,82 @@
         poll if @last_poll.nil? || (Time.now - @last_poll) >= DELAY
       end
     end
   end
 
-  ## TODO: merge this with sup-import
   def do_poll
     total_num = total_numi = 0
     @mutex.synchronize do
-      found = {}
       Index.usual_sources.each do |source|
-        next if source.broken? || source.done?
-
-        yield "Loading from #{source}... "
-        start_offset = nil
+#        yield "source #{source} is done? #{source.done?} (cur_offset #{source.cur_offset} >= #{source.end_offset})"
+        yield "Loading from #{source}... " unless source.done? || source.broken?
         num = 0
-        num_inbox = 0
-
-        source.each do |offset, labels|
-          break if source.broken?
-          start_offset ||= offset
-          yield "Found message at #{offset} with labels #{labels * ', '}"
-
-          begin
-            begin
-              m = Redwood::Message.new :source => source, :source_info => offset, :labels => labels
-            rescue MessageFormatError => e
-              yield "Non-fatal error loading message #{source}##{offset}: #{e.message}"
-              next
-            end
-
-            if found[m.id]
-              yield "Skipping duplicate message #{m.id}"
-              next
-            end
-            found[m.id] = true
-          
-            if Index.add_message m
-              UpdateManager.relay :add, m
-              num += 1
-              total_num += 1
-              total_numi += 1 if m.labels.include? :inbox
-            end
-        
-            if num % 1000 == 0 && num > 0
-              elapsed = Time.now - start
-              pctdone = source.pct_done
-              remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
-              yield "## #{num} (#{pctdone}% done) read; #{elapsed.to_time_s} elapsed; est. #{remaining.to_time_s} remaining"
-            end
-          rescue SourceError => e
-            msg = "Fatal error loading from #{source}: #{e.message}"
-            Redwood::log msg
-            yield msg
-            break
+        numi = 0
+        add_new_messages_from source do |m, offset, entry|
+          ## always preserve the labels on disk.
+          m.labels = entry[:label].split(/\s+/).map { |x| x.intern } if entry
+          yield "Found message at #{offset} with labels {#{m.labels * ', '}}"
+          unless entry
+            num += 1
+            numi += 1 if m.labels.include? :inbox
           end
+          m
         end
-        yield "Found #{num} messages" unless num == 0
+        yield "Found #{num} messages, #{numi} to inbox" unless num == 0
+        total_num += num
+        total_numi += numi
       end
 
       yield "Done polling; loaded #{total_num} new messages total"
       @last_poll = Time.now
       @polling = false
     end
     [total_num, total_numi]
+  end
+
+  ## this is the main mechanism for adding new messages to the
+  ## index. it's called both by sup-import and by PollMode.
+  ##
+  ## for each new message in the source, this yields the message, the
+  ## source offset, and the index entry on disk (if any). it expects
+  ## the yield to return the message (possibly altered in some way),
+  ## and then adds it (if new) or updates it (if previously seen).
+  ##
+  ## the labels of the yielded message are the source labels. it is
+  ## likely that callers will want to replace these with the index
+  ## labels, if they exist, so that state is not lost when e.g. a new
+  ## version of a message from a mailing list comes in.
+  def add_new_messages_from source
+    return if source.done? || source.broken?
+
+    source.each do |offset, labels|
+      if source.broken?
+        Redwood::log "error loading messages from #{source}: #{source.broken_msg}"
+        return
+      end
+      
+      labels.each { |l| LabelManager << l }
+
+      begin
+        m = Message.new :source => source, :source_info => offset, :labels => labels
+        if m.source_marked_read?
+          m.remove_label :unread
+          labels.delete :unread
+        end
+
+        docid, entry = Index.load_entry_for_id m.id
+        m = yield m, offset, entry
+        next unless m
+        if entry
+          Index.update_message m, docid, entry
+        else
+          Index.add_message m
+          UpdateManager.relay self, :add, m
+        end
+      rescue MessageFormatError, SourceError => e
+        Redwood::log "ignoring erroneous message at #{source}##{offset}: #{e.message}"
+      end
+    end
   end
 end
 
 end