lib/picky/tokenizer.rb in picky-3.6.7 vs lib/picky/tokenizer.rb in picky-3.6.8

- old
+ new

@@ -133,18 +133,18 @@ end def downcase? !@case_sensitive end - def maximum_tokens amount - @maximum_tokens = amount + def max_words amount + @max_words = amount end def cap words - words.slice!(@maximum_tokens..-1) if cap?(words) + words.slice!(@max_words..-1) if cap?(words) end def cap? words - @maximum_tokens && words.size > @maximum_tokens + @max_words && words.size > @max_words end # Checks if the right argument type has been given. # def check_argument_in method, type, argument, &condition @@ -158,11 +158,11 @@ substitutes_characters_with options[:substitutes_characters_with] if options[:substitutes_characters_with] removes_characters options[:removes_characters] if options[:removes_characters] stopwords options[:stopwords] if options[:stopwords] splits_text_on options[:splits_text_on] || /\s/ normalizes_words options[:normalizes_words] if options[:normalizes_words] - maximum_tokens options[:maximum_tokens] + max_words options[:max_words] rejects_token_if &(options[:rejects_token_if] || :blank?) case_sensitive options[:case_sensitive] unless options[:case_sensitive].nil? end # Returns a number of tokens, generated from the given text, @@ -201,10 +201,10 @@ # Pretokenizing. # # Does: # * Split the text into words. - # * Cap the amount of tokens if maximum_tokens is set. + # * Cap the amount of tokens if max_words is set. # def pretokenize text words = split text words.collect! { |word| normalize_with_patterns word } if normalize_with_patterns? reject words \ No newline at end of file