lib/rley/rgn/tokenizer.rb in rley-0.8.10 vs lib/rley/rgn/tokenizer.rb in rley-0.8.11

- old
+ new

@@ -44,11 +44,11 @@ }.freeze # Here are all the implemented Rley notation keywords @@keywords = %w[ match_closest repeat - ].map { |x| [x, x] }.to_h + ].to_h { |x| [x, x] } # Constructor. Initialize a tokenizer for RGN input. # @param source [String] RGN text to tokenize. def initialize(source = nil) reset @@ -127,13 +127,10 @@ raise ScanError, "Error: [line #{lineno}:#{col}]: Unexpected character." end ws_found = false end # until - # unterminated(@string_start.line, @string_start.column) if state == :multiline token - - # return token end def build_token(aSymbolName, aLexeme) begin lex_length = aLexeme ? aLexeme.size : 0