Contents

require 'spec_helper'

describe PragmaticTokenizer do
  context 'Language: French (fr)' do
    it 'tokenizes a string #001' do
      text = "L'art de l'univers, c'est un art"
      pt = PragmaticTokenizer::Tokenizer.new(
          text,
          language: 'fr'
      )
      expect(pt.tokenize).to eq(["l'", "art", "de", "l'", "univers", ",", "c'est", "un", "art"])
    end
  end
end

Version data entries

5 entries across 5 versions & 1 rubygems

Version	Path
pragmatic_tokenizer-1.6.0	spec/languages/french_spec.rb
pragmatic_tokenizer-2.1.0	spec/languages/french_spec.rb
pragmatic_tokenizer-1.5.1	spec/languages/french_spec.rb
pragmatic_tokenizer-2.0.0	spec/languages/french_spec.rb
pragmatic_tokenizer-1.5.0	spec/languages/french_spec.rb