module Searchkick
class IndexOptions
attr_reader :options
def initialize(index)
@options = index.options
end
def index_options
# mortal symbols are garbage collected in Ruby 2.2+
custom_settings = (options[:settings] || {}).deep_symbolize_keys
custom_mappings = (options[:mappings] || {}).deep_symbolize_keys
if options[:mappings] && !options[:merge_mappings]
settings = custom_settings
mappings = custom_mappings
else
settings = generate_settings.deep_symbolize_keys.deep_merge(custom_settings)
mappings = generate_mappings.deep_symbolize_keys.deep_merge(custom_mappings)
end
set_deep_paging(settings) if options[:deep_paging]
{
settings: settings,
mappings: mappings
}
end
def generate_settings
language = options[:language]
language = language.call if language.respond_to?(:call)
settings = {
analysis: {
analyzer: {
searchkick_keyword: {
type: "custom",
tokenizer: "keyword",
filter: ["lowercase"] + (options[:stem_conversions] ? ["searchkick_stemmer"] : [])
},
default_analyzer => {
type: "custom",
# character filters -> tokenizer -> token filters
# https://www.elastic.co/guide/en/elasticsearch/guide/current/analysis-intro.html
char_filter: ["ampersand"],
tokenizer: "standard",
# synonym should come last, after stemming and shingle
# shingle must come before searchkick_stemmer
filter: ["lowercase", "asciifolding", "searchkick_index_shingle", "searchkick_stemmer"]
},
searchkick_search: {
type: "custom",
char_filter: ["ampersand"],
tokenizer: "standard",
filter: ["lowercase", "asciifolding", "searchkick_search_shingle", "searchkick_stemmer"]
},
searchkick_search2: {
type: "custom",
char_filter: ["ampersand"],
tokenizer: "standard",
filter: ["lowercase", "asciifolding", "searchkick_stemmer"]
},
# https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb
searchkick_autocomplete_search: {
type: "custom",
tokenizer: "keyword",
filter: ["lowercase", "asciifolding"]
},
searchkick_word_search: {
type: "custom",
tokenizer: "standard",
filter: ["lowercase", "asciifolding"]
},
searchkick_suggest_index: {
type: "custom",
tokenizer: "standard",
filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"]
},
searchkick_text_start_index: {
type: "custom",
tokenizer: "keyword",
filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
},
searchkick_text_middle_index: {
type: "custom",
tokenizer: "keyword",
filter: ["lowercase", "asciifolding", "searchkick_ngram"]
},
searchkick_text_end_index: {
type: "custom",
tokenizer: "keyword",
filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
},
searchkick_word_start_index: {
type: "custom",
tokenizer: "standard",
filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"]
},
searchkick_word_middle_index: {
type: "custom",
tokenizer: "standard",
filter: ["lowercase", "asciifolding", "searchkick_ngram"]
},
searchkick_word_end_index: {
type: "custom",
tokenizer: "standard",
filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"]
}
},
filter: {
searchkick_index_shingle: {
type: "shingle",
token_separator: ""
},
# lucky find https://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7
searchkick_search_shingle: {
type: "shingle",
token_separator: "",
output_unigrams: false,
output_unigrams_if_no_shingles: true
},
searchkick_suggest_shingle: {
type: "shingle",
max_shingle_size: 5
},
searchkick_edge_ngram: {
type: "edge_ngram",
min_gram: 1,
max_gram: 50
},
searchkick_ngram: {
type: "ngram",
min_gram: 1,
max_gram: 50
},
searchkick_stemmer: {
# use stemmer if language is lowercase, snowball otherwise
type: language == language.to_s.downcase ? "stemmer" : "snowball",
language: language || "English"
}
},
char_filter: {
# https://www.elastic.co/guide/en/elasticsearch/guide/current/custom-analyzers.html
# &_to_and
ampersand: {
type: "mapping",
mappings: ["&=> and "]
}
}
}
}
raise ArgumentError, "Can't pass both language and stemmer" if options[:stemmer] && language
update_language(settings, language)
update_stemming(settings)
if Searchkick.env == "test"
settings[:number_of_shards] = 1
settings[:number_of_replicas] = 0
end
if options[:similarity]
settings[:similarity] = {default: {type: options[:similarity]}}
end
settings[:index] = {
max_ngram_diff: 49,
max_shingle_diff: 4
}
if options[:case_sensitive]
settings[:analysis][:analyzer].each do |_, analyzer|
analyzer[:filter].delete("lowercase")
end
end
add_synonyms(settings)
add_search_synonyms(settings)
if options[:special_characters] == false
settings[:analysis][:analyzer].each_value do |analyzer_settings|
analyzer_settings[:filter].reject! { |f| f == "asciifolding" }
end
end
settings
end
def update_language(settings, language)
case language
when "chinese"
settings[:analysis][:analyzer].merge!(
default_analyzer => {
type: "ik_smart"
},
searchkick_search: {
type: "ik_smart"
},
searchkick_search2: {
type: "ik_max_word"
}
)
when "chinese2", "smartcn"
settings[:analysis][:analyzer].merge!(
default_analyzer => {
type: "smartcn"
},
searchkick_search: {
type: "smartcn"
},
searchkick_search2: {
type: "smartcn"
}
)
when "japanese", "japanese2"
analyzer = {
type: "custom",
tokenizer: "kuromoji_tokenizer",
filter: [
"kuromoji_baseform",
"kuromoji_part_of_speech",
"cjk_width",
"ja_stop",
"searchkick_stemmer",
"lowercase"
]
}
settings[:analysis][:analyzer].merge!(
default_analyzer => analyzer.deep_dup,
searchkick_search: analyzer.deep_dup,
searchkick_search2: analyzer.deep_dup
)
settings[:analysis][:filter][:searchkick_stemmer] = {
type: "kuromoji_stemmer"
}
when "korean"
settings[:analysis][:analyzer].merge!(
default_analyzer => {
type: "openkoreantext-analyzer"
},
searchkick_search: {
type: "openkoreantext-analyzer"
},
searchkick_search2: {
type: "openkoreantext-analyzer"
}
)
when "korean2"
settings[:analysis][:analyzer].merge!(
default_analyzer => {
type: "nori"
},
searchkick_search: {
type: "nori"
},
searchkick_search2: {
type: "nori"
}
)
when "vietnamese"
settings[:analysis][:analyzer].merge!(
default_analyzer => {
type: "vi_analyzer"
},
searchkick_search: {
type: "vi_analyzer"
},
searchkick_search2: {
type: "vi_analyzer"
}
)
when "polish", "ukrainian"
settings[:analysis][:analyzer].merge!(
default_analyzer => {
type: language
},
searchkick_search: {
type: language
},
searchkick_search2: {
type: language
}
)
end
end
def update_stemming(settings)
if options[:stemmer]
stemmer = options[:stemmer]
# could also support snowball and stemmer
case stemmer[:type]
when "hunspell"
# supports all token filter options
settings[:analysis][:filter][:searchkick_stemmer] = stemmer
else
raise ArgumentError, "Unknown stemmer: #{stemmer[:type]}"
end
end
stem = options[:stem]
# language analyzer used
stem = false if settings[:analysis][:analyzer][default_analyzer][:type] != "custom"
if stem == false
settings[:analysis][:filter].delete(:searchkick_stemmer)
settings[:analysis][:analyzer].each do |_, analyzer|
analyzer[:filter].delete("searchkick_stemmer") if analyzer[:filter]
end
end
if options[:stemmer_override]
stemmer_override = {
type: "stemmer_override"
}
if options[:stemmer_override].is_a?(String)
stemmer_override[:rules_path] = options[:stemmer_override]
else
stemmer_override[:rules] = options[:stemmer_override]
end
settings[:analysis][:filter][:searchkick_stemmer_override] = stemmer_override
settings[:analysis][:analyzer].each do |_, analyzer|
stemmer_index = analyzer[:filter].index("searchkick_stemmer") if analyzer[:filter]
analyzer[:filter].insert(stemmer_index, "searchkick_stemmer_override") if stemmer_index
end
end
if options[:stem_exclusion]
settings[:analysis][:filter][:searchkick_stem_exclusion] = {
type: "keyword_marker",
keywords: options[:stem_exclusion]
}
settings[:analysis][:analyzer].each do |_, analyzer|
stemmer_index = analyzer[:filter].index("searchkick_stemmer") if analyzer[:filter]
analyzer[:filter].insert(stemmer_index, "searchkick_stem_exclusion") if stemmer_index
end
end
end
def generate_mappings
mapping = {}
keyword_mapping = {type: "keyword"}
keyword_mapping[:ignore_above] = options[:ignore_above] || 30000
# conversions
Array(options[:conversions]).each do |conversions_field|
mapping[conversions_field] = {
type: "nested",
properties: {
query: {type: default_type, analyzer: "searchkick_keyword"},
count: {type: "integer"}
}
}
end
mapping_options =
[:suggest, :word, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end, :highlight, :searchable, :filterable]
.to_h { |type| [type, (options[type] || []).map(&:to_s)] }
word = options[:word] != false && (!options[:match] || options[:match] == :word)
mapping_options[:searchable].delete("_all")
analyzed_field_options = {type: default_type, index: true, analyzer: default_analyzer.to_s}
mapping_options.values.flatten.uniq.each do |field|
fields = {}
if options.key?(:filterable) && !mapping_options[:filterable].include?(field)
fields[field] = {type: default_type, index: false}
else
fields[field] = keyword_mapping
end
if !options[:searchable] || mapping_options[:searchable].include?(field)
if word
fields[:analyzed] = analyzed_field_options
if mapping_options[:highlight].include?(field)
fields[:analyzed][:term_vector] = "with_positions_offsets"
end
end
mapping_options.except(:highlight, :searchable, :filterable, :word).each do |type, f|
if options[:match] == type || f.include?(field)
fields[type] = {type: default_type, index: true, analyzer: "searchkick_#{type}_index"}
end
end
end
mapping[field] = fields[field].merge(fields: fields.except(field))
end
(options[:locations] || []).map(&:to_s).each do |field|
mapping[field] = {
type: "geo_point"
}
end
options[:geo_shape] = options[:geo_shape].product([{}]).to_h if options[:geo_shape].is_a?(Array)
(options[:geo_shape] || {}).each do |field, shape_options|
mapping[field] = shape_options.merge(type: "geo_shape")
end
if options[:inheritance]
mapping[:type] = keyword_mapping
end
routing = {}
if options[:routing]
routing = {required: true}
unless options[:routing] == true
routing[:path] = options[:routing].to_s
end
end
dynamic_fields = {
# analyzed field must be the default field for include_in_all
# http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
# however, we can include the not_analyzed field in _all
# and the _all index analyzer will take care of it
"{name}" => keyword_mapping
}
if options.key?(:filterable)
dynamic_fields["{name}"] = {type: default_type, index: false}
end
unless options[:searchable]
if options[:match] && options[:match] != :word
dynamic_fields[options[:match]] = {type: default_type, index: true, analyzer: "searchkick_#{options[:match]}_index"}
end
if word
dynamic_fields[:analyzed] = analyzed_field_options
end
end
# http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
multi_field = dynamic_fields["{name}"].merge(fields: dynamic_fields.except("{name}"))
mappings = {
properties: mapping,
_routing: routing,
# https://gist.github.com/kimchy/2898285
dynamic_templates: [
{
string_template: {
match: "*",
match_mapping_type: "string",
mapping: multi_field
}
}
]
}
mappings
end
def add_synonyms(settings)
synonyms = options[:synonyms] || []
synonyms = synonyms.call if synonyms.respond_to?(:call)
if synonyms.any?
settings[:analysis][:filter][:searchkick_synonym] = {
type: "synonym",
# only remove a single space from synonyms so three-word synonyms will fail noisily instead of silently
synonyms: synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.map { |s2| s2.sub(/\s+/, "") }.join(",") : s }.map(&:downcase)
}
# choosing a place for the synonym filter when stemming is not easy
# https://groups.google.com/forum/#!topic/elasticsearch/p7qcQlgHdB8
# TODO use a snowball stemmer on synonyms when creating the token filter
# http://elasticsearch-users.115913.n3.nabble.com/synonym-multi-words-search-td4030811.html
# I find the following approach effective if you are doing multi-word synonyms (synonym phrases):
# - Only apply the synonym expansion at index time
# - Don't have the synonym filter applied search
# - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general.
settings[:analysis][:analyzer][default_analyzer][:filter].insert(2, "searchkick_synonym")
%w(word_start word_middle word_end).each do |type|
settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_synonym")
end
end
end
def add_search_synonyms(settings)
search_synonyms = options[:search_synonyms] || []
search_synonyms = search_synonyms.call if search_synonyms.respond_to?(:call)
if search_synonyms.is_a?(String) || search_synonyms.any?
if search_synonyms.is_a?(String)
synonym_graph = {
type: "synonym_graph",
synonyms_path: search_synonyms
}
synonym_graph[:updateable] = true unless below73?
else
synonym_graph = {
type: "synonym_graph",
# TODO confirm this is correct
synonyms: search_synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.join(",") : s }.map(&:downcase)
}
end
settings[:analysis][:filter][:searchkick_synonym_graph] = synonym_graph
if ["japanese", "japanese2"].include?(options[:language])
[:searchkick_search, :searchkick_search2].each do |analyzer|
settings[:analysis][:analyzer][analyzer][:filter].insert(4, "searchkick_synonym_graph")
end
else
[:searchkick_search2, :searchkick_word_search].each do |analyzer|
unless settings[:analysis][:analyzer][analyzer].key?(:filter)
raise Error, "Search synonyms are not supported yet for language"
end
settings[:analysis][:analyzer][analyzer][:filter].insert(2, "searchkick_synonym_graph")
end
end
end
end
def set_deep_paging(settings)
if !settings.dig(:index, :max_result_window) && !settings[:"index.max_result_window"]
settings[:index] ||= {}
settings[:index][:max_result_window] = 1_000_000_000
end
end
def index_type
@index_type ||= begin
index_type = options[:_type]
index_type = index_type.call if index_type.respond_to?(:call)
index_type
end
end
def default_type
"text"
end
def default_analyzer
:searchkick_index
end
def below73?
Searchkick.server_below?("7.3.0")
end
end
end