Contents

require 'spec_helper'

describe PragmaticTokenizer do
  context 'Language: French (fr)' do
    it 'tokenizes a string #001' do
      text = "D'art de l'univers, c'est un art"
      pt = PragmaticTokenizer::Tokenizer.new(
          language: 'fr'
      )
      expect(pt.tokenize(text)).to eq(["d'", "art", "de", "l'", "univers", ",", "c'" ,"est", "un", "art"])
    end
  end
end

Version data entries

3 entries across 3 versions & 1 rubygems

Version	Path
pragmatic_tokenizer-3.2.1	spec/languages/french_spec.rb
pragmatic_tokenizer-3.2.0	spec/languages/french_spec.rb
pragmatic_tokenizer-3.1.0	spec/languages/french_spec.rb