lib/search_lingo/tokenizer.rb in search_lingo-1.0.0.beta2 vs lib/search_lingo/tokenizer.rb in search_lingo-1.0.0.beta3

@@ -1,41 +1,42 @@
 require 'forwardable'
 require 'strscan'
+require 'search_lingo/constants'
 require 'search_lingo/token'
 
 module SearchLingo
   class Tokenizer
     include Enumerable
     extend Forwardable
 
-    SIMPLE    = %r{"[^"]*"|[[:graph:]]+}
-    COMPOUND  = %r{(?:[[:graph:]]+:[[:space:]]*)?#{SIMPLE}}
-    DELIMITER = %r{[[:space:]]*}
+    SIMPLE_TOKEN   = /#{TERM}/
+    COMPOUND_TOKEN = /(?:#{OPERATOR}:[[:space:]]*)?#{TERM}/
+    DELIMITER      = /[[:space:]]*/
 
     def initialize(query)
       @scanner = StringScanner.new query.strip
     end
 
-    def enum
-      Enumerator.new do |yielder|
-        until scanner.eos?
-          token = scanner.scan COMPOUND
-          if token
-            yielder << Token.new(token)
-          end
-          scanner.skip DELIMITER
-        end
+    def each
+      return to_enum(__callee__) unless block_given?
+
+      until scanner.eos?
+        yield self.next
       end
     end
 
+    def next
+      scanner.skip DELIMITER
+      token = scanner.scan COMPOUND_TOKEN
+      raise StopIteration unless token
+      Token.new token
+    end
+
     def_delegator :scanner, :reset
-    def_delegators :enum, :each, :next
 
     def simplify
       scanner.unscan
-      Token.new(scanner.scan(SIMPLE)).tap do
-        scanner.skip DELIMITER
-      end
+      Token.new scanner.scan SIMPLE_TOKEN
     end
 
     private
 
     attr_reader :scanner