Sha256: d6c1ddc6de9bd11a07afa18746d043d8bd87969a9904bc319a065540f20eab94
Contents?: true
Size: 607 Bytes
Versions: 13
Compression:
Stored size: 607 Bytes
Contents
module Internals
module Tokenizers
# The base indexing tokenizer.
#
# Override in indexing subclasses and define in configuration.
#
class Index < Base
def self.default= new_default
@default = new_default
end
def self.default
@default ||= new
end
# Does not actually return a token, but a
# symbol "token".
#
def tokens_for words
words.collect! { |word| word.downcase! if downcase?; word.to_sym }
end
# Returns empty tokens.
#
def empty_tokens
[]
end
end
end
end
Version data entries
13 entries across 13 versions & 1 rubygems