lib/picky/query/tokens.rb in picky-3.5.0 vs lib/picky/query/tokens.rb in picky-3.5.1

@@ -25,19 +25,9 @@
       #
       def self.processed words, originals, ignore_unassigned = false
         new words.zip(originals).collect! { |word, original| Token.processed word, original }, ignore_unassigned
       end
 
-      # Tokenizes each token.
-      #
-      # Note: Passed tokenizer needs to offer #normalize(text).
-      #
-      # TODO Still needed?
-      #
-      def tokenize_with tokenizer
-        @tokens.each { |token| token.tokenize_with(tokenizer) }
-      end
-
       # Generates an array in the form of
       # [
       #  [combination],                           # of token 1
       #  [combination, combination, combination], # of token 2
       #  [combination, combination]               # of token 3
\ No newline at end of file