lib/search_lingo/tokenizer.rb in search_lingo-2.0.0.pre2 vs lib/search_lingo/tokenizer.rb in search_lingo-2.0.0.pre3

@@ -1,11 +1,19 @@
+# frozen-string-literal: true
+
 require 'forwardable'
 require 'strscan'
 require 'search_lingo/constants'
 require 'search_lingo/token'
 
 module SearchLingo
+  ##
+  # Tokenizer breaks down a query string into individual tokens.
+  #
+  #   Tokenizer.new 'foo'
+  #   Tokenizer.foo 'foo "bar baz"'
+  #   Tokenizer.foo 'foo "bar baz" froz: quux'
   class Tokenizer
     include Enumerable
     extend Forwardable
 
     ##
@@ -28,21 +36,20 @@
     # Iterates over the query string. If called with a block, it yields each
     # token. If called without a block, it returns an +Enumerator+.
     def each
       return to_enum(__callee__) unless block_given?
 
-      until scanner.eos?
-        yield self.next
-      end
+      yield self.next until scanner.eos?
     end
 
     ##
     # Returns a Token for the next token in the query string. When the end of
     # the query string is reached raises +StopIteration+.
     def next
       scanner.skip DELIMITER
       token = scanner.scan COMPOUND_TOKEN
       raise StopIteration unless token
+
       Token.new token
     end
 
     def_delegator :scanner, :reset