loxxy_tokenizer.rb in rley-0.8.06

- old
+ new

@@ -1,26 +1,37 @@
 # frozen_string_literal: true
 
 require 'rley'
 require_relative 'loxxy_raw_scanner.rex'
 
+# Tokenizer for the Lox language that is compatible with a Rley parser.
+# It works as an adapter between the parser and the scanner generated 
+# with `oedipus_lex` scanner generator.
+# The purpose is to transform the output of a `LoxxyRawScanner` instance
+# into o sequence of tokens in the format expected by Rley.
 class LoxxyTokenizer
   # @return [LoxxyRawScanner] Scanner generated by `oedipus_lex`gem.
   attr_reader :scanner
 
-   # @return [String] Input text to tokenize
+  # @return [String] Lox input text to tokenize
   attr_reader :input
 
+  # A Hash that maps a Lox keyword to its uppercase version
+  # (a convention used in the grammar).
+  # If a search is unsuccessful, it returns the string 'IDENTIFIER'
+  # @return [{String => String}]
   Keyword2name = begin
     lookup = %w[
       and class else false fun for if nil or
       print return super this true var while
     ].map { |x| [x, x.upcase] }.to_h
     lookup.default = 'IDENTIFIER'
     lookup.freeze
   end
 
+  # A Hash that maps a special character of Lox into a name
+  # @return [{String => String}]
   Special2name = {
     '(' => 'LEFT_PAREN',
     ')' => 'RIGHT_PAREN',
     '{' => 'LEFT_BRACE',
     '}' => 'RIGHT_BRACE',
@@ -39,19 +50,26 @@
     '>=' => 'GREATER_EQUAL',
     '<' => 'LESS',
     '<=' => 'LESS_EQUAL'
   }.freeze
 
+  # Constructor
+  # @param source [String, NilClass] Optional. The text to tokenize
   def initialize(source = nil)
     @scanner = LoxxyRawScanner.new
     start_with(source)
   end
 
+  # Set the text to tokenize.
+  # @param source [String] The text to tokenize  
   def start_with(source)
     @input = source
   end
 
+  # Launch the scanning/tokenization of input text.
+  # Returns an array of Rley token objects.
+  # @return [Array<Rley::Lexical::Token>] array of token objects.
   def tokens
     raw_tokens = scanner.parse(input)
     cooked = raw_tokens.map do |(raw_type, raw_text, line, col)|
       pos = Rley::Lexical::Position.new(line, col + 1)
       convert(raw_type, raw_text, pos)
@@ -63,9 +81,10 @@
     cooked
   end
 
   private
 
+  # Convert the raw tokens from generated scanner into something edible by Rley.
   def convert(token_kind, token_text, pos)
     result = case token_kind
     when :SPECIAL
       Rley::Lexical::Token.new(token_text, Special2name[token_text])
     when :FALSE