lib/dentaku/tokenizer.rb in dentaku-0.1.2 vs lib/dentaku/tokenizer.rb in dentaku-0.1.3

@@ -5,9 +5,11 @@
 module Dentaku
   class Tokenizer
     SCANNERS = [
       TokenScanner.new(:whitespace, '\s+'),
       TokenScanner.new(:numeric,    '(\d+(\.\d+)?|\.\d+)', lambda{|raw| raw =~ /\./ ? raw.to_f : raw.to_i }),
+      TokenScanner.new(:string,     '"[^"]*"', lambda{|raw| raw.gsub(/^"|"$/, '') }),
+      TokenScanner.new(:string,     "'[^']*'", lambda{|raw| raw.gsub(/^'|'$/, '') }),
       TokenScanner.new(:operator,   '\+|-|\*|\/', lambda do |raw|
         case raw
         when '+' then :add
         when '-' then :subtract
         when '*' then :multiply