Contents

# encoding: UTF-8

# Copyright 2012 Twitter, Inc
# http://www.apache.org/licenses/LICENSE-2.0

module TwitterCldr
  module Tokenizers
    class NumberTokenizer

      attr_reader :data_reader

      def initialize(data_reader)
        @data_reader = data_reader
      end

      def tokenize(pattern)
        tokens = PatternTokenizer.new(data_reader, tokenizer).tokenize(pattern)

        if tokens.first.value == ""
          tokens[1..-1]
        else
          tokens
        end
      end

      private

      def tokenizer
        @tokenizer ||= Tokenizer.new([
          TokenRecognizer.new(:pattern, /[0?#,\.]+/),
          TokenRecognizer.new(:plaintext, //),
        ], /([^0*#,\.]*)([0#,\.]+)([^0*#,\.]*)$/, false)
      end

    end
  end
end

Version data entries

5 entries across 5 versions & 1 rubygems

Version	Path
twitter_cldr-3.0.4	lib/twitter_cldr/tokenizers/numbers/number_tokenizer.rb
twitter_cldr-3.0.3	lib/twitter_cldr/tokenizers/numbers/number_tokenizer.rb
twitter_cldr-3.0.2	lib/twitter_cldr/tokenizers/numbers/number_tokenizer.rb
twitter_cldr-3.0.1	lib/twitter_cldr/tokenizers/numbers/number_tokenizer.rb
twitter_cldr-3.0.0	lib/twitter_cldr/tokenizers/numbers/number_tokenizer.rb