index.rb in sup-0.15.0

- old
+ new

@@ -166,10 +166,36 @@
     xapian_query = build_xapian_query query
     matchset = run_query xapian_query, 0, 0, 100
     matchset.matches_estimated
   end
 
+  ## check if a message is part of a killed thread
+  ## (warning: duplicates code below)
+  ## NOTE: We can be more efficient if we assume every
+  ## killed message that hasn't been initially added
+  ## to the indexi s this way
+  def message_joining_killed? m
+    return false unless doc = find_doc(m.id)
+    queue = doc.value(THREAD_VALUENO).split(',')
+    seen_threads = Set.new
+    seen_messages = Set.new [m.id]
+    while not queue.empty?
+      thread_id = queue.pop
+      next if seen_threads.member? thread_id
+      return true if thread_killed?(thread_id)
+      seen_threads << thread_id
+      docs = term_docids(mkterm(:thread, thread_id)).map { |x| @xapian.document x }
+      docs.each do |doc|
+        msgid = doc.value MSGID_VALUENO
+        next if seen_messages.member? msgid
+        seen_messages << msgid
+        queue.concat doc.value(THREAD_VALUENO).split(',')
+      end
+    end
+    false
+  end
+
   ## yield all messages in the thread containing 'm' by repeatedly
   ## querying the index. yields pairs of message ids and
   ## message-building lambdas, so that building an unwanted message
   ## can be skipped in the block if desired.
   ##
@@ -246,26 +272,30 @@
     contacts.to_a.compact[0...num].map { |n,e| Person.from_name_and_email n, e }
   end
 
   ## Yield each message-id matching query
   EACH_ID_PAGE = 100
-  def each_id query={}
+  def each_id query={}, ignore_neg_terms = true
     offset = 0
     page = EACH_ID_PAGE
 
-    xapian_query = build_xapian_query query
+    xapian_query = build_xapian_query query, ignore_neg_terms
     while true
       ids = run_query_ids xapian_query, offset, (offset+page)
       ids.each { |id| yield id }
       break if ids.size < page
       offset += page
     end
   end
 
   ## Yield each message matching query
-  def each_message query={}, &b
-    each_id query do |id|
+  ## The ignore_neg_terms parameter is used to display result even if
+  ## it contains "forbidden" labels such as :deleted, it is used in
+  ## Poll#poll_from when we need to get the location of a message that
+  ## may contain these labels
+  def each_message query={}, ignore_neg_terms = true, &b
+    each_id query, ignore_neg_terms do |id|
       yield build_message(id)
     end
   end
 
   # Search messages. Returns an Enumerator.
@@ -311,13 +341,13 @@
   end
 
   ## Yields (in lexicographical order) the source infos of all locations from
   ## the given source with the given source_info prefix
   def each_source_info source_id, prefix='', &b
-    prefix = mkterm :location, source_id, prefix
-    each_prefixed_term prefix do |x|
-      yield x[prefix.length..-1]
+    p = mkterm :location, source_id, prefix
+    each_prefixed_term p do |x|
+      yield prefix + x[p.length..-1]
     end
   end
 
   class ParseError < StandardError; end
 
@@ -490,26 +520,30 @@
       xapian_query = qp.parse_query(subs, Xapian::QueryParser::FLAG_PHRASE|Xapian::QueryParser::FLAG_BOOLEAN|Xapian::QueryParser::FLAG_LOVEHATE|Xapian::QueryParser::FLAG_WILDCARD)
     rescue RuntimeError => e
       raise ParseError, "xapian query parser error: #{e}"
     end
 
-    debug "parsed xapian query: #{Util::Query.describe(xapian_query)}"
+    debug "parsed xapian query: #{Util::Query.describe(xapian_query, subs)}"
 
     raise ParseError if xapian_query.nil? or xapian_query.empty?
     query[:qobj] = xapian_query
     query[:text] = s
     query
   end
 
+  def save_message m
+    if @sync_worker
+      @sync_queue << m
+    else
+      update_message_state m
+    end
+    m.clear_dirty
+  end
+
   def save_thread t
     t.each_dirty_message do |m|
-      if @sync_worker
-        @sync_queue << m
-      else
-        update_message_state m
-      end
-      m.clear_dirty
+      save_message m
     end
   end
 
   def start_sync_worker
     @sync_worker = Redwood::reporting_thread('index sync') { run_sync_worker }
@@ -612,11 +646,11 @@
     matchset = run_query xapian_query, offset, limit
     matchset.matches.map { |r| r.document.value MSGID_VALUENO }
   end
 
   Q = Xapian::Query
-  def build_xapian_query opts
+  def build_xapian_query opts, ignore_neg_terms = true
     labels = ([opts[:label]] + (opts[:labels] || [])).compact
     neglabels = [:spam, :deleted, :killed].reject { |l| (labels.include? l) || opts.member?("load_#{l}".intern) }
     pos_terms, neg_terms = [], []
 
     pos_terms << mkterm(:type, 'mail')
@@ -628,11 +662,11 @@
     if opts[:participants]
       participant_terms = opts[:participants].map { |p| [:from,:to].map { |d| mkterm(:email, d, (Redwood::Person === p) ? p.email : p) } }.flatten
       pos_terms << Q.new(Q::OP_OR, participant_terms)
     end
 
-    neg_terms.concat(neglabels.map { |l| mkterm(:label,l) })
+    neg_terms.concat(neglabels.map { |l| mkterm(:label,l) }) if ignore_neg_terms
 
     pos_query = Q.new(Q::OP_AND, pos_terms)
     neg_query = Q.new(Q::OP_OR, neg_terms)
 
     if neg_query.empty?
@@ -641,9 +675,13 @@
       Q.new(Q::OP_AND_NOT, [pos_query, neg_query])
     end
   end
 
   def sync_message m, overwrite
+    ## TODO: we should not save the message if the sync_back failed
+    ## since it would overwrite the location field
+    m.sync_back
+
     doc = synchronize { find_doc(m.id) }
     existed = doc != nil
     doc ||= Xapian::Document.new
     do_index_static = overwrite || !existed
     old_entry = !do_index_static && doc.entry