lib/dentaku/tokenizer.rb in dentaku-3.0.0 vs lib/dentaku/tokenizer.rb in dentaku-3.1.0

@@ -2,20 +2,25 @@
 require 'dentaku/token_matcher'
 require 'dentaku/token_scanner'
 
 module Dentaku
   class Tokenizer
+    attr_reader :case_sensitive, :aliases
+
     LPAREN = TokenMatcher.new(:grouping, :open)
     RPAREN = TokenMatcher.new(:grouping, :close)
 
-    def tokenize(string)
+    def tokenize(string, options = {})
       @nesting = 0
       @tokens  = []
+      @aliases = options.fetch(:aliases, Dentaku.aliases)
       input    = strip_comments(string.to_s.dup)
+      input    = replace_aliases(input)
+      @case_sensitive = options.fetch(:case_sensitive, false)
 
       until input.empty?
-        scanned = TokenScanner.scanners.any? do |scanner|
+        scanned = TokenScanner.scanners(case_sensitive: case_sensitive).any? do |scanner|
           scanned, input = scan(input, scanner)
           scanned
         end
 
         unless scanned
@@ -54,9 +59,28 @@
       end
     end
 
     def strip_comments(input)
       input.gsub(/\/\*[^*]*\*+(?:[^*\/][^*]*\*+)*\//, '')
+    end
+
+    def replace_aliases(string)
+      return string unless @aliases.any?
+
+      string.gsub!(alias_regex) do |match|
+        match_regex = /^#{Regexp.escape(match)}$/i
+
+        @aliases.detect do |(_key, aliases)|
+          !aliases.grep(match_regex).empty?
+        end.first
+      end
+
+      string
+    end
+
+    def alias_regex
+      values = @aliases.values.flatten.join('|')
+      /(?<=\p{Punct}|[[:space:]]|\A)(#{values})(?=\()/i
     end
 
     private
 
     def fail!(reason, **meta)