Sha256: fa945907905e949836035fe6c740d69addd5b46e1947cfb32dcf25507f698887
Contents?: true
Size: 776 Bytes
Versions: 4
Compression:
Stored size: 776 Bytes
Contents
# Copyright 2012 Twitter, Inc
# http://www.apache.org/licenses/LICENSE-2.0
class TwitterCldr.SegmentationTokenizer
constructor : ->
recognizers = [
new TwitterCldr.TokenRecognizer("break", /\u00f7/, ((val) ->
TwitterCldr.Utilities.trim_string(val))) # ÷ character
new TwitterCldr.TokenRecognizer("no_break", /\u00d7/, ((val) ->
TwitterCldr.Utilities.trim_string(val))) # × character
]
ur_tokenizer = new TwitterCldr.UnicodeRegexTokenizer
ur_tokenizer.insert_before("string", recognizers)
@tokenizer = ur_tokenizer
tokenize : (pattern) ->
result = []
tokens = @tokenizer.tokenize pattern
for token in tokens
if token.value.replace(/^\s+|\s+$/g, "").length isnt 0
result.push token
result
Version data entries
4 entries across 4 versions & 1 rubygems