Sha256: d2bb3bd9951d8b0ff6a460312293424e5bad0f01f039c17a554a4940c4dbb07e

Contents?: true

Size: 1.59 KB

Versions: 29

Compression:

Stored size: 1.59 KB

Contents

# encoding: UTF-8

# Copyright 2012 Twitter, Inc
# http://www.apache.org/licenses/LICENSE-2.0

module TwitterCldr
  module Transforms

    class Tokenizer
      def tokenize(rule_text)
        tokenizer.tokenize(rule_text)
      end

      private

      def tokenizer
        TwitterCldr::Tokenizers::Tokenizer.new(recognizers)
      end

      def recognizers
        @recognizers ||= [
          TwitterCldr::Tokenizers::TokenRecognizer.new(:capture, /\$[\d]+/),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:variable, /\$[\w]+/),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:doubled_quote, /''/),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:quoted_string, /'[^']*'/),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:direction, /[<>]{1,2}/),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:before_context, /[{]/),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:after_context, /[}]/),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:cursor, /\|/),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:unicode_char, /\\u[a-fA-F0-9]{1,6}/),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:unicode_char, /\\u\{[a-fA-F0-9]{1,6}\}/),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:escaped_char, /\\./),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:escaped_backslash, /\\\\/),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:equals, /=/),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:semicolon, /;/),
          TwitterCldr::Tokenizers::TokenRecognizer.new(:string, /[^ ]{1}/)
        ]
      end
    end

  end
end

Version data entries

29 entries across 29 versions & 1 rubygems

Version Path
twitter_cldr-6.13.0 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.12.1 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.12.0 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.11.5 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.11.4 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.11.3 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.11.2 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.11.1 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.11.0 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.10.0 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.9.0 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.8.0 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.7.0 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.6.2 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.6.1 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.6.0 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.5.0 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.4.0 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.3.0 lib/twitter_cldr/transforms/tokenizer.rb
twitter_cldr-6.2.0 lib/twitter_cldr/transforms/tokenizer.rb