lib/search_lingo/tokenizer.rb in search_lingo-2.0.0 vs lib/search_lingo/tokenizer.rb in search_lingo-3.0.0.pre1

@@ -14,22 +14,10 @@
   #   Tokenizer.foo 'foo "bar baz" froz: quux'
   class Tokenizer
     include Enumerable
     extend Forwardable
 
-    ##
-    # Pattern for matching a simple token (a term without a modifier).
-    SIMPLE_TOKEN   = /#{TERM}/
-
-    ##
-    # Pattern for matching a compound token (a term with an optional modifier).
-    COMPOUND_TOKEN = /(?:#{MODIFIER}:[[:space:]]*)?#{TERM}/
-
-    ##
-    # Pattern for matching the delimiter between tokens.
-    DELIMITER      = /[[:space:]]*/
-
     def initialize(query) # :nodoc:
       @scanner = StringScanner.new query.strip
     end
 
     ##
@@ -40,14 +28,14 @@
 
       yield self.next until scanner.eos?
     end
 
     ##
-    # Returns a Token for the next token in the query string. When the end of
+    # Returns a +Token+ for the next token in the query string. When the end of
     # the query string is reached raises +StopIteration+.
     def next
       scanner.skip DELIMITER
-      token = scanner.scan COMPOUND_TOKEN
+      token = scanner.scan SIMPLE_OR_COMPOUND_TOKEN
       raise StopIteration unless token
 
       Token.new token
     end