index.rb in sup-0.0.7

- old
+ new

@@ -1,11 +1,10 @@
 ## the index structure for redwood. interacts with ferret.
 
 require 'thread'
 require 'fileutils'
 require 'ferret'
-#require_gem 'ferret', ">= 0.10.13"
 
 module Redwood
 
 class Index
   include Singleton
@@ -43,11 +42,11 @@
     ##TODO: why was this necessary?
     ##source.id += 1 while @sources.member? source.id
     @sources[source.id] = source
   end
 
-  def source_for name; @sources.values.find { |s| s.is_source_for? name }; end
+  def source_for uri; @sources.values.find { |s| s.is_source_for? uri }; end
   def usual_sources; @sources.values.find_all { |s| s.usual? }; end
   def sources; @sources.values; end
 
   def load_index dir=File.join(@dir, "ferret")
     if File.exists? dir
@@ -71,39 +70,43 @@
       field_infos.create_index dir
       @index = Ferret::Index::Index.new(:path => dir, :analyzer => @analyzer)
     end
   end
 
-  ## update the message by deleting and re-adding
-  def update_message m, source=nil, source_info=nil
-    docid, entry = load_entry_for_id m.id
-    if entry
-      source ||= entry[:source_id].to_i
-      source_info ||= entry[:source_info].to_i
+  ## Update the message state on disk, by deleting and re-adding it.
+  ## The message must exist in the index. docid and entry are found
+  ## unless given.
+  ##
+  ## Overwrites the labels on disk with the new labels in 'm', so that
+  ## we can actually change message state.
+  def update_message m, docid=nil, entry=nil
+    unless docid && entry
+      docid, entry = load_entry_for_id m.id
+      raise ArgumentError, "cannot find #{m.id} in the index" unless entry
     end
 
-    ## this happens sometimes. i'm not sure why. ferret bug?
-    raise "no entry and no source info for message #{m.id}: source #{source.inspect}, info #{source_info.inspect}, entry #{entry.inspect}, query #{Ferret::Search::TermQuery.new(:message_id, m.id)}, results #{@index.search(Ferret::Search::TermQuery.new(:message_id, m.id)).inspect}" unless source && source_info
+    raise "no entry and no source info for message #{m.id}" unless m.source && m.source_info
 
     raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id
+
     @index.delete docid
     add_message m
+    docid, entry = load_entry_for_id m.id
   end
 
   def save_index fn=File.join(@dir, "ferret")
-    # don't have to do anything apparently
+    # don't have to do anything,  apparently
   end
 
   def contains_id? id
     @index.search(Ferret::Search::TermQuery.new(:message_id, id)).total_hits > 0
   end
   def contains? m; contains_id? m.id; end
   def size; @index.size; end
 
   ## you should probably not call this on a block that doesn't break
-  ## rather quickly because the results will probably be, as we say
-  ## in scotland, frikkin' huuuge.
+  ## rather quickly because the results can be very large.
   EACH_BY_DATE_NUM = 100
   def each_id_by_date opts={}
     return if @index.size == 0 # otherwise ferret barfs ###TODO: remove this once my ferret patch is accepted
     query = build_query opts
     offset = 0
@@ -121,36 +124,37 @@
     q = build_query opts
     index.search(q).total_hits
   end
 
   ## yield all messages in the thread containing 'm' by repeatedly
-  ## querying the index.  yields pairs of message ids and
+  ## querying the index. uields pairs of message ids and
   ## message-building lambdas, so that building an unwanted message
   ## can be skipped in the block if desired.
+  ##
+  ## stops loading any thread if a message with a :killed flag is found.
+
   SAME_SUBJECT_DATE_LIMIT = 7
   def each_message_in_thread_for m, opts={}
     messages = {}
     searched = {}
     num_queries = 0
 
-    ## temporarily disabling subject searching because it's a
-    ## significant slowdown.
-    ##
-    ## TODO: make this configurable, i guess
-    if true
+    ## todo: make subject querying configurable
+    if true # do subject queries
       date_min = m.date - (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
       date_max = m.date + (SAME_SUBJECT_DATE_LIMIT * 12 * 3600)
 
       q = Ferret::Search::BooleanQuery.new true
       sq = Ferret::Search::PhraseQuery.new(:subject)
       wrap_subj(Message.normalize_subj(m.subj)).split(/\s+/).each do |t|
         sq.add_term t
       end
       q.add_query sq, :must
-      q.add_query Ferret::Search::TermQuery.new(:label, "spam"), :must_not
       q.add_query Ferret::Search::RangeQuery.new(:date, :>= => date_min.to_indexable_s, :<= => date_max.to_indexable_s), :must
 
+      q = build_query :qobj => q
+
       pending = @index.search(q).hits.map { |hit| @index[hit.doc][:message_id] }
       Redwood::log "found #{pending.size} results for subject query #{q}"
     else
       pending = [m.id]
     end
@@ -161,22 +165,25 @@
       searched[id] = true
       q = Ferret::Search::BooleanQuery.new true
       q.add_query Ferret::Search::TermQuery.new(:message_id, id), :should
       q.add_query Ferret::Search::TermQuery.new(:refs, id), :should
 
+      q = build_query :qobj => q, :load_killed => true
+
       num_queries += 1
       @index.search_each(q, :limit => :all) do |docid, score|
         break if opts[:limit] && messages.size >= opts[:limit]
+        break if @index[docid][:label].split(/\s+/).include? "killed" unless opts[:load_killed]
         mid = @index[docid][:message_id]
         unless messages.member? mid
           messages[mid] ||= lambda { build_message docid }
           refs = @index[docid][:refs].split(" ")
           pending += refs
         end
       end
     end
-    Redwood::log "ran #{num_queries} queries to build thread of #{messages.size} messages for #{m.id}"
+    Redwood::log "ran #{num_queries} queries to build thread of #{messages.size} messages for #{m.id}" if num_queries > 0
     messages.each { |mid, builder| yield mid, builder }
   end
 
   ## builds a message object from a ferret result
   def build_message docid
@@ -201,12 +208,14 @@
 
   def fresh_thread_id; @next_thread_id += 1; end
   def wrap_subj subj; "__START_SUBJECT__ #{subj} __END_SUBJECT__"; end
   def unwrap_subj subj; subj =~ /__START_SUBJECT__ (.*?) __END_SUBJECT__/ && $1; end
 
+  ## Adds a message to the index. The message cannot already exist in
+  ## the index.
   def add_message m
-    return false if contains? m
+    raise ArgumentError, "index already contains #{m.id}" if contains? m
 
     source_id = 
       if m.source.is_a? Integer
         m.source
       else
@@ -228,13 +237,13 @@
       :refs => (m.refs + m.replytos).uniq.join(" "),
     }
 
     @index.add_document d
     
-    ## TODO: figure out why this is sometimes triggered
-    #docid, entry = load_entry_for_id m.id
-    #raise "just added message #{m.id} but couldn't find it in a search" unless docid
+    docid, entry = load_entry_for_id m.id
+    ## this hasn't been triggered in a long time. TODO: decide whether it's still a problem.
+    raise "just added message #{m.id} but couldn't find it in a search" unless docid
     true
   end
 
   def drop_entry docno; @index.delete docno; end
 
@@ -292,9 +301,10 @@
       end
       query.add_query q2, :must
     end
         
     query.add_query Ferret::Search::TermQuery.new("label", "spam"), :must_not unless opts[:load_spam] || labels.include?(:spam)
+    query.add_query Ferret::Search::TermQuery.new("label", "deleted"), :must_not unless opts[:load_deleted] || labels.include?(:deleted)
     query.add_query Ferret::Search::TermQuery.new("label", "killed"), :must_not unless opts[:load_killed] || labels.include?(:killed)
     query
   end
 
   def load_sources fn=Redwood::SOURCE_FN