lib/picky/tokenizer.rb in picky-3.6.7 vs lib/picky/tokenizer.rb in picky-3.6.8
- old
+ new
@@ -133,18 +133,18 @@
end
def downcase?
!@case_sensitive
end
- def maximum_tokens amount
- @maximum_tokens = amount
+ def max_words amount
+ @max_words = amount
end
def cap words
- words.slice!(@maximum_tokens..-1) if cap?(words)
+ words.slice!(@max_words..-1) if cap?(words)
end
def cap? words
- @maximum_tokens && words.size > @maximum_tokens
+ @max_words && words.size > @max_words
end
# Checks if the right argument type has been given.
#
def check_argument_in method, type, argument, &condition
@@ -158,11 +158,11 @@
substitutes_characters_with options[:substitutes_characters_with] if options[:substitutes_characters_with]
removes_characters options[:removes_characters] if options[:removes_characters]
stopwords options[:stopwords] if options[:stopwords]
splits_text_on options[:splits_text_on] || /\s/
normalizes_words options[:normalizes_words] if options[:normalizes_words]
- maximum_tokens options[:maximum_tokens]
+ max_words options[:max_words]
rejects_token_if &(options[:rejects_token_if] || :blank?)
case_sensitive options[:case_sensitive] unless options[:case_sensitive].nil?
end
# Returns a number of tokens, generated from the given text,
@@ -201,10 +201,10 @@
# Pretokenizing.
#
# Does:
# * Split the text into words.
- # * Cap the amount of tokens if maximum_tokens is set.
+ # * Cap the amount of tokens if max_words is set.
#
def pretokenize text
words = split text
words.collect! { |word| normalize_with_patterns word } if normalize_with_patterns?
reject words
\ No newline at end of file