Sha256: de9f11c6646ea1840c84baf581b4eae3a22e379d9e8c3e91e5b4108500c6e916
Contents?: true
Size: 1.05 KB
Versions: 7
Compression:
Stored size: 1.05 KB
Contents
require 'dentaku/token'
require 'dentaku/token_matcher'
require 'dentaku/token_scanner'
module Dentaku
class Tokenizer
LPAREN = TokenMatcher.new(:grouping, :open)
RPAREN = TokenMatcher.new(:grouping, :close)
def tokenize(string)
@nesting = 0
@tokens = []
input = string.dup
until input.empty?
raise "parse error at: '#{ input }'" unless TokenScanner.scanners.any? do |scanner|
scanned, input = scan(input, scanner)
scanned
end
end
raise "too many opening parentheses" if @nesting > 0
@tokens
end
def scan(string, scanner)
if token = scanner.scan(string)
raise "unexpected zero-width match (:#{ token.category }) at '#{ string }'" if token.length == 0
@nesting += 1 if LPAREN == token
@nesting -= 1 if RPAREN == token
raise "too many closing parentheses" if @nesting < 0
@tokens << token unless token.is?(:whitespace)
[true, string[token.length..-1]]
else
[false, string]
end
end
end
end
Version data entries
7 entries across 7 versions & 1 rubygems