Contents

// Copyright 2012 Twitter, Inc
// http://www.apache.org/licenses/LICENSE-2.0

var TwitterCldr = require('../../../lib/assets/javascripts/twitter_cldr/en.js');

describe("SegmentationTokenizer", function() {
  var tokenizer = new TwitterCldr.SegmentationTokenizer();
  it("should tokenize an expression with a non-break", function() {
    expect(tokenizer.tokenize("$CB ÷ $SP")).toEqualTokenList([
      { 'value' : "$CB", 'type' : "variable" },
      { 'value' : "÷", 'type' : "break" },
      { 'value' : "$SP", 'type' : "variable" }
    ]);
  });
  it("should tokenize an expression with a non-break", function() {
    expect(tokenizer.tokenize("$ATerm × $Numeric")).toEqualTokenList([
      { 'value' : "$ATerm", 'type' : "variable" },
      { 'value' : "×", 'type' : "no_break" },
      { 'value' : "$Numeric", 'type' : "variable" }
    ]);
  });
});

Version data entries

1 entries across 1 versions & 1 rubygems

Version	Path
twitter_cldr_js-2.4.0	spec/js/tokenizers/segmentation_tokenizer.spec.js