Contents

require 'spec_helper'

describe PragmaticTokenizer do
  context 'Language: French (fr)' do
    it 'tokenizes a string #001' do
      text = "L'art de l'univers, c'est un art"
      pt = PragmaticTokenizer::Tokenizer.new(
          language: 'fr'
      )
      expect(pt.tokenize(text)).to eq(["l'", "art", "de", "l'", "univers", ",", "c'est", "un", "art"])
    end
  end
end

Version data entries

10 entries across 10 versions & 1 rubygems

Version	Path
pragmatic_tokenizer-3.0.7	spec/languages/french_spec.rb
pragmatic_tokenizer-3.0.6	spec/languages/french_spec.rb
pragmatic_tokenizer-3.0.5	spec/languages/french_spec.rb
pragmatic_tokenizer-3.0.4	spec/languages/french_spec.rb
pragmatic_tokenizer-2.2.1	spec/languages/french_spec.rb
pragmatic_tokenizer-3.0.3	spec/languages/french_spec.rb
pragmatic_tokenizer-3.0.2	spec/languages/french_spec.rb
pragmatic_tokenizer-3.0.1	spec/languages/french_spec.rb
pragmatic_tokenizer-2.2.0	spec/languages/french_spec.rb
pragmatic_tokenizer-3.0.0	spec/languages/french_spec.rb