lib/dentaku/tokenizer.rb in dentaku-1.2.6 vs lib/dentaku/tokenizer.rb in dentaku-2.0.0

@@ -8,11 +8,11 @@
     RPAREN = TokenMatcher.new(:grouping, :close)
 
     def tokenize(string)
       @nesting = 0
       @tokens  = []
-      input    = string.to_s.dup
+      input    = strip_comments(string.to_s.dup)
 
       until input.empty?
         raise "parse error at: '#{ input }'" unless TokenScanner.scanners.any? do |scanner|
           scanned, input = scan(input, scanner)
           scanned
@@ -22,12 +22,16 @@
       raise "too many opening parentheses" if @nesting > 0
 
       @tokens
     end
 
+    def last_token
+      @tokens.last
+    end
+
     def scan(string, scanner)
-      if tokens = scanner.scan(string)
+      if tokens = scanner.scan(string, last_token)
         tokens.each do |token|
           raise "unexpected zero-width match (:#{ token.category }) at '#{ string }'" if token.length == 0
 
           @nesting += 1 if LPAREN == token
           @nesting -= 1 if RPAREN == token
@@ -39,8 +43,12 @@
         match_length = tokens.map(&:length).reduce(:+)
         [true, string[match_length..-1]]
       else
         [false, string]
       end
+    end
+
+    def strip_comments(input)
+      input.gsub(/\/\*[^*]*\*+(?:[^*\/][^*]*\*+)*\//, '')
     end
   end
 end