Sha256: 2a08ccdfeb0a3ddcf814fc7236d3af90979031c8547d1d8903540a89f0f11f11
Contents?: true
Size: 964 Bytes
Versions: 65
Compression:
Stored size: 964 Bytes
Contents
# encoding: UTF-8
# Copyright 2012 Twitter, Inc
# http://www.apache.org/licenses/LICENSE-2.0
module TwitterCldr
module Tokenizers
class PatternTokenizer
attr_reader :data_reader, :tokenizer
def initialize(data_reader, tokenizer)
@data_reader = data_reader
@tokenizer = tokenizer
end
def tokenize(pattern)
tokenizer.tokenize(expand(pattern))
end
private
def expand(pattern)
if pattern.is_a?(Symbol)
# symbols mean another path was given
path = pattern.to_s.split(".").map(&:to_sym)
data = data_reader.pattern_at_path(path)
next_pattern = data.is_a?(Hash) ? data[:pattern] : data
expand_pattern(next_pattern)
elsif pattern.is_a?(Hash)
pattern.inject({}) do |ret, (key, val)|
ret[key] = expand(val)
ret
end
else
pattern
end
end
end
end
end
Version data entries
65 entries across 65 versions & 3 rubygems