lib/picky/indexers/serial.rb in picky-3.0.1 vs lib/picky/indexers/serial.rb in picky-3.1.0

@@ -26,10 +26,11 @@
 
           category.prepared_index_file do |file|
             result = []
 
             source.harvest(category) do |indexed_id, text|
-              tokenizer.tokenize(text).each do |token_text|
+              tokens, _ = tokenizer.tokenize text # Note: Originals not needed.
+              tokens.each do |token_text|
                 next unless token_text
                 result << indexed_id << comma << token_text << newline
               end
               file.write(result.join) && result.clear if result.size > 100_000
             end
\ No newline at end of file