# encoding: utf-8
require "spec_helper"

describe TextNlp::Tokenizer do
  
  it "should tokenize text" do
    n = TextNlp::Tokenizer.new
    n.tokenize("Comment q'ça    se fait ?   blabla?").should eq ["Comment","q'ça","se","fait","?","blabla?"]
  end
  
end