Contents

# coding: utf-8
require 'spec_helper'

describe TfIdfSimilarity::Token do
  describe '#valid?' do
    it 'should return false if all of its characters are numbers, punctuation or whitespace characters' do
      TfIdfSimilarity::Token.new('1 2 3 ! @ #').valid?.should == false
    end

    it 'should return true if not all of its characters are numbers, punctuation or whitespace characters' do
      TfIdfSimilarity::Token.new('1 2 3 ! @ # a').valid?.should == true
    end
  end

  describe '#lowercase_filter' do
    it 'should lowercase the token' do
      TfIdfSimilarity::Token.new('HÉTÉROGÉNÉITÉ').lowercase_filter.should == 'hétérogénéité'
    end
  end

  describe '#classic_filter' do
    it 'should remove all periods' do
      TfIdfSimilarity::Token.new('X.Y.Z.').classic_filter.should == 'XYZ'
    end

    it 'should remove ending possessives' do
      TfIdfSimilarity::Token.new("foo's").classic_filter.should == 'foo'
    end

    it 'should not remove infix possessives' do
      TfIdfSimilarity::Token.new("foo's bar").classic_filter.should == "foo's bar"
    end
  end
end

Version data entries

4 entries across 4 versions & 1 rubygems

Version	Path
tf-idf-similarity-0.1.3	spec/token_spec.rb
tf-idf-similarity-0.1.2	spec/token_spec.rb
tf-idf-similarity-0.1.1	spec/token_spec.rb
tf-idf-similarity-0.1.0	spec/token_spec.rb