Sha256: 13f30a546ce74698c7429a5d76b1e46034ce86393fa64531664ba5bfe32907d6
Contents?: true
Size: 963 Bytes
Versions: 4
Compression:
Stored size: 963 Bytes
Contents
// Copyright 2012 Twitter, Inc
// http://www.apache.org/licenses/LICENSE-2.0
var TwitterCldr = require('../../../lib/assets/javascripts/twitter_cldr/core.js');
var data = require('../../../lib/assets/javascripts/twitter_cldr/en.js');
TwitterCldr.set_data(data);
describe("SegmentationTokenizer", function() {
var tokenizer = new TwitterCldr.SegmentationTokenizer();
it("should tokenize an expression with a non-break", function() {
expect(tokenizer.tokenize("$CB ÷ $SP")).toEqualTokenList([
{ 'value' : "$CB", 'type' : "variable" },
{ 'value' : "÷", 'type' : "break" },
{ 'value' : "$SP", 'type' : "variable" }
]);
});
it("should tokenize an expression with a non-break", function() {
expect(tokenizer.tokenize("$ATerm × $Numeric")).toEqualTokenList([
{ 'value' : "$ATerm", 'type' : "variable" },
{ 'value' : "×", 'type' : "no_break" },
{ 'value' : "$Numeric", 'type' : "variable" }
]);
});
});
Version data entries
4 entries across 4 versions & 1 rubygems