lib/dentaku/tokenizer.rb in dentaku-0.2.4 vs lib/dentaku/tokenizer.rb in dentaku-0.2.5

@@ -4,15 +4,16 @@
 
 module Dentaku
   class Tokenizer
     SCANNERS = [
       TokenScanner.new(:whitespace, '\s+'),
-      TokenScanner.new(:numeric,    '(\d+(\.\d+)?|\.\d+)', lambda { |raw| raw =~ /\./ ? raw.to_f : raw.to_i }),
-      TokenScanner.new(:string,     '"[^"]*"',    lambda { |raw| raw.gsub(/^"|"$/, '') }),
-      TokenScanner.new(:string,     "'[^']*'",    lambda { |raw| raw.gsub(/^'|'$/, '') }),
-      TokenScanner.new(:operator,   '\+|-|\*|\/', lambda do |raw|
+      TokenScanner.new(:numeric,    '(\d+(\.\d+)?|\.\d+)\b', lambda { |raw| raw =~ /\./ ? raw.to_f : raw.to_i }),
+      TokenScanner.new(:string,     '"[^"]*"',               lambda { |raw| raw.gsub(/^"|"$/, '') }),
+      TokenScanner.new(:string,     "'[^']*'",               lambda { |raw| raw.gsub(/^'|'$/, '') }),
+      TokenScanner.new(:operator,   '\^|\+|-|\*|\/', lambda do |raw|
         case raw
+        when '^' then :pow
         when '+' then :add
         when '-' then :subtract
         when '*' then :multiply
         when '/' then :divide
         end
@@ -33,12 +34,12 @@
         when '<'  then :lt
         when '>'  then :gt
         when '='  then :eq
         end
       end),
-      TokenScanner.new(:combinator, '(and|or)\b',   lambda { |raw| raw.strip.downcase.to_sym }),
-      TokenScanner.new(:function,   '(if|round)\b', lambda { |raw| raw.strip.to_sym }),
-      TokenScanner.new(:identifier, '[a-z_]+',      lambda { |raw| raw.downcase.to_sym })
+      TokenScanner.new(:combinator, '(and|or)\b',   lambda {|raw| raw.strip.downcase.to_sym }),
+      TokenScanner.new(:function,   '(if|round)\b', lambda {|raw| raw.strip.to_sym }),
+      TokenScanner.new(:identifier, '\w+\b',        lambda {|raw| raw.downcase.to_sym })
     ]
 
     LPAREN = TokenMatcher.new(:grouping, :open)
     RPAREN = TokenMatcher.new(:grouping, :close)