lib/dentaku/tokenizer.rb in dentaku-1.2.0 vs lib/dentaku/tokenizer.rb in dentaku-1.2.1

@@ -2,11 +2,11 @@
 require 'dentaku/token_matcher'
 require 'dentaku/token_scanner'
 
 module Dentaku
   class Tokenizer
-    LPAREN = TokenMatcher.new(:grouping, :open)
+    LPAREN = TokenMatcher.new(:grouping, [:open, :fopen])
     RPAREN = TokenMatcher.new(:grouping, :close)
 
     def tokenize(string)
       @nesting = 0
       @tokens  = []
@@ -23,19 +23,22 @@
 
       @tokens
     end
 
     def scan(string, scanner)
-      if token = scanner.scan(string)
-        raise "unexpected zero-width match (:#{ token.category }) at '#{ string }'" if token.length == 0
+      if tokens = scanner.scan(string)
+        tokens.each do |token|
+          raise "unexpected zero-width match (:#{ token.category }) at '#{ string }'" if token.length == 0
 
-        @nesting += 1 if LPAREN == token
-        @nesting -= 1 if RPAREN == token
-        raise "too many closing parentheses" if @nesting < 0
+          @nesting += 1 if LPAREN == token
+          @nesting -= 1 if RPAREN == token
+          raise "too many closing parentheses" if @nesting < 0
 
-        @tokens << token unless token.is?(:whitespace)
+          @tokens << token unless token.is?(:whitespace)
+        end
 
-        [true, string[token.length..-1]]
+        match_length = tokens.map(&:length).reduce(:+)
+        [true, string[match_length..-1]]
       else
         [false, string]
       end
     end
   end