Sha256: 8e17bf800647b5ffaa7fb2a9f9eb6079213d02c0c6fe06a60587626d28945f38
Contents?: true
Size: 1.93 KB
Versions: 29
Compression:
Stored size: 1.93 KB
Contents
# encoding: UTF-8
# Copyright 2012 Twitter, Inc
# http://www.apache.org/licenses/LICENSE-2.0
module TwitterCldr
module Tokenizers
class DateTimeTokenizer
attr_reader :data_reader
def initialize(data_reader)
@data_reader = data_reader
end
def tokenize(pattern)
expand_tokens(
PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern)
)
end
# Tokenizes mixed date and time pattern strings,
# used to tokenize the additional date format patterns.
def full_tokenize(pattern)
PatternTokenizer.new(data_reader, full_tokenizer).tokenize(pattern)
end
protected
def expand_tokens(tokens)
tokens.inject([]) do |ret, token|
ret + case token.type
when :date
expand_date(token)
when :time
expand_time(token)
else
[token]
end
end
end
def expand_date(token)
date_reader = data_reader.date_reader
date_reader.tokenizer.tokenize(date_reader.pattern)
end
def expand_time(token)
time_reader = data_reader.time_reader
time_reader.tokenizer.tokenize(time_reader.pattern)
end
def full_tokenizer
@@full_tokenizer ||= begin
new_tok = Tokenizer.union(
data_reader.date_reader.tokenizer.tokenizer,
data_reader.time_reader.tokenizer.tokenizer
) do |recognizer|
recognizer.token_type != :plaintext
end
new_tok.recognizers << TokenRecognizer.new(:plaintext, //)
new_tok
end
end
def tokenizer
@tokenizer ||= Tokenizer.new([
TokenRecognizer.new(:date, /\{\{date\}\}/),
TokenRecognizer.new(:time, /\{\{time\}\}/),
TokenRecognizer.new(:plaintext, /'.*'/),
TokenRecognizer.new(:plaintext, //)
])
end
end
end
end
Version data entries
29 entries across 29 versions & 3 rubygems