Contents

# encoding: utf-8
require "spec_helper"

describe TextNlp::Tokenizer do
  
  it "should tokenize text" do
    n = TextNlp::Tokenizer.new
    n.tokenize("Comment q'ça    se fait ?   blabla?").should eq ["Comment","q'ça","se","fait","?","blabla?"]
  end
  
end

Version data entries

3 entries across 3 versions & 1 rubygems

Version	Path
text_nlp-0.0.3	spec/tokenizer_spec.rb
text_nlp-0.0.2	spec/tokenizer_spec.rb
text_nlp-0.0.1	spec/tokenizer_spec.rb