mongoid_fulltext.rb in mongoid_fulltext-0.5.1

- old
+ new

@@ -27,11 +27,23 @@
         :ngram_width => 3,
         :max_ngrams_to_search => 6,
         :apply_prefix_scoring_to_all_words => true,
         :index_full_words => true,
         :max_candidate_set_size => 1000,
-        :remove_accents => true
+        :remove_accents => true,
+        :stop_words => Hash[['i', 'a', 's', 't', 'me', 'my', 'we', 'he', 'it', 'am', 'is', 'be', 'do', 'an', 'if', 
+                             'or', 'as', 'of', 'at', 'by', 'to', 'up', 'in', 'on', 'no', 'so', 'our', 'you', 'him', 
+                             'his', 'she', 'her', 'its', 'who', 'are', 'was', 'has', 'had', 'did', 'the', 'and', 
+                             'but', 'for', 'out', 'off', 'why', 'how', 'all', 'any', 'few', 'nor', 'not', 'own', 
+                             'too', 'can', 'don', 'now', 'ours', 'your', 'hers', 'they', 'them', 'what', 'whom', 
+                             'this', 'that', 'were', 'been', 'have', 'does', 'with', 'into', 'from', 'down', 'over', 
+                             'then', 'once', 'here', 'when', 'both', 'each', 'more', 'most', 'some', 'such', 'only', 
+                             'same', 'than', 'very', 'will', 'just', 'yours', 'their', 'which', 'these', 'those', 
+                             'being', 'doing', 'until', 'while', 'about', 'after', 'above', 'below', 'under', 
+                             'again', 'there', 'where', 'other', 'myself', 'itself', 'theirs', 'having', 'during', 
+                             'before', 'should', 'himself', 'herself', 'because', 'against', 'between', 'through', 
+                             'further', 'yourself', 'ourselves', 'yourselves', 'themselves'].map{ |x| [x,true] }]
       }
       
       config.update(options)
 
       args = [:to_s] if args.empty?
@@ -178,11 +190,11 @@
           filtered_str = CGI.unescape(filtered_str)
         end
         filtered_str = UnicodeUtils.nfkd(filtered_str).gsub(/[^\x00-\x7F]/,'')
       end
 
-      filtered_str = filtered_str.downcase.split('').map{ |ch| config[:alphabet][ch] }.compact.join('')
+      filtered_str = filtered_str.mb_chars.downcase.to_s.split('').map{ |ch| config[:alphabet][ch] }.compact.join('')
       
       if bound_number_returned
         step_size = [((filtered_str.length - config[:ngram_width]).to_f / config[:max_ngrams_to_search]).ceil, 1].max
       else
         step_size = 1
@@ -201,12 +213,14 @@
 
       # If an ngram appears multiple times in the query string, keep the max score
       ngram_ary = ngram_ary.group_by{ |h| h[:ngram] }.map{ |key, values| {:ngram => key, :score => values.map{ |v| v[:score] }.max} }
       
       if (config[:index_full_words])
+        full_words_seen = {}
         filtered_str.split(Regexp.compile(config[:word_separators].keys.join)).each do |word|
-          if word.length >= config[:ngram_width]
+          if word.length >= config[:ngram_width] and full_words_seen[word].nil? and config[:stop_words][word].nil?
             ngram_ary << {:ngram => word, :score => 1}
+            full_words_seen[word] = true
           end
         end
       end
 
       # If an ngram appears as a full word and an ngram, keep the sum of the two scores