lib/cldr-plurals/compiler/tokenizer.rb in cldr-plurals-1.1.1 vs lib/cldr-plurals/compiler/tokenizer.rb in cldr-plurals-1.2.0
- old
+ new
@@ -12,22 +12,22 @@
end
class Tokenizer
TOKENS = {
- /@integer/ => :int_sample,
- /@decimal/ => :dec_sample,
- /\u2026/ => :infinite_set,
- /~/ => :sample_range,
- /and/ => :and,
- /or/ => :or,
- /[niftvw]/ => :operand,
- /,/ => :comma,
- /\.\./ => :range,
- /%/ => :modulo,
- /=/ => :equals,
- /\!=/ => :not_equals,
- /[\d]+/ => :number
+ /@integer/ => :int_sample,
+ /@decimal/ => :dec_sample,
+ /\u2026/ => :infinite_set,
+ /~/ => :sample_range,
+ /and/ => :and,
+ /or/ => :or,
+ /[niftvwe]/ => :operand,
+ /,/ => :comma,
+ /\.\./ => :range,
+ /%/ => :modulo,
+ /=/ => :equals,
+ /\!=/ => :not_equals,
+ /[\d]+/ => :number
}
ALL_TOKENS = Regexp.compile(
TOKENS.map { |r, _| r.source }.join('|')
)