lib/dentaku/tokenizer.rb in dentaku-0.1.0 vs lib/dentaku/tokenizer.rb in dentaku-0.1.1

@@ -13,12 +13,16 @@
         when '-' then :subtract
         when '*' then :multiply
         when '/' then :divide
         end
       end),
-      TokenScanner.new(:grouping,   '\(|\)', lambda do |raw|
-        raw == '(' ? :open : :close
+      TokenScanner.new(:grouping,   '\(|\)|,', lambda do |raw|
+        case raw
+        when '(' then :open
+        when ')' then :close
+        when ',' then :comma
+        end
       end),
       TokenScanner.new(:comparator, '<=|>=|!=|<>|<|>|=', lambda do |raw|
         case raw
         when '<=' then :le
         when '>=' then :ge
@@ -28,9 +32,10 @@
         when '>'  then :gt
         when '='  then :eq
         end
       end),
       TokenScanner.new(:combinator, '(and|or)\b', lambda {|raw| raw.strip.to_sym }),
+      TokenScanner.new(:function,   '(if)\b',     lambda {|raw| raw.strip.to_sym }),
       TokenScanner.new(:identifier, '[A-Za-z_]+', lambda {|raw| raw.to_sym })
     ]
 
     LPAREN = TokenMatcher.new(:grouping, :open)
     RPAREN = TokenMatcher.new(:grouping, :close)