module Searchkick class IndexOptions attr_reader :options def initialize(index) @options = index.options end def index_options # mortal symbols are garbage collected in Ruby 2.2+ custom_settings = (options[:settings] || {}).deep_symbolize_keys custom_mappings = (options[:mappings] || {}).deep_symbolize_keys if options[:mappings] && !options[:merge_mappings] settings = custom_settings mappings = custom_mappings else settings = generate_settings.deep_symbolize_keys.deep_merge(custom_settings) mappings = generate_mappings.deep_symbolize_keys.deep_merge(custom_mappings) end set_deep_paging(settings) if options[:deep_paging] { settings: settings, mappings: mappings } end def generate_settings language = options[:language] language = language.call if language.respond_to?(:call) settings = { analysis: { analyzer: { searchkick_keyword: { type: "custom", tokenizer: "keyword", filter: ["lowercase"] + (options[:stem_conversions] ? ["searchkick_stemmer"] : []) }, default_analyzer => { type: "custom", # character filters -> tokenizer -> token filters # https://www.elastic.co/guide/en/elasticsearch/guide/current/analysis-intro.html char_filter: ["ampersand"], tokenizer: "standard", # synonym should come last, after stemming and shingle # shingle must come before searchkick_stemmer filter: ["lowercase", "asciifolding", "searchkick_index_shingle", "searchkick_stemmer"] }, searchkick_search: { type: "custom", char_filter: ["ampersand"], tokenizer: "standard", filter: ["lowercase", "asciifolding", "searchkick_search_shingle", "searchkick_stemmer"] }, searchkick_search2: { type: "custom", char_filter: ["ampersand"], tokenizer: "standard", filter: ["lowercase", "asciifolding", "searchkick_stemmer"] }, # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb searchkick_autocomplete_search: { type: "custom", tokenizer: "keyword", filter: ["lowercase", "asciifolding"] }, searchkick_word_search: { type: "custom", tokenizer: "standard", filter: ["lowercase", "asciifolding"] }, searchkick_suggest_index: { type: "custom", tokenizer: "standard", filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"] }, searchkick_text_start_index: { type: "custom", tokenizer: "keyword", filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"] }, searchkick_text_middle_index: { type: "custom", tokenizer: "keyword", filter: ["lowercase", "asciifolding", "searchkick_ngram"] }, searchkick_text_end_index: { type: "custom", tokenizer: "keyword", filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"] }, searchkick_word_start_index: { type: "custom", tokenizer: "standard", filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"] }, searchkick_word_middle_index: { type: "custom", tokenizer: "standard", filter: ["lowercase", "asciifolding", "searchkick_ngram"] }, searchkick_word_end_index: { type: "custom", tokenizer: "standard", filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"] } }, filter: { searchkick_index_shingle: { type: "shingle", token_separator: "" }, # lucky find https://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7 searchkick_search_shingle: { type: "shingle", token_separator: "", output_unigrams: false, output_unigrams_if_no_shingles: true }, searchkick_suggest_shingle: { type: "shingle", max_shingle_size: 5 }, searchkick_edge_ngram: { type: "edge_ngram", min_gram: 1, max_gram: 50 }, searchkick_ngram: { type: "ngram", min_gram: 1, max_gram: 50 }, searchkick_stemmer: { # use stemmer if language is lowercase, snowball otherwise type: language == language.to_s.downcase ? "stemmer" : "snowball", language: language || "English" } }, char_filter: { # https://www.elastic.co/guide/en/elasticsearch/guide/current/custom-analyzers.html # &_to_and ampersand: { type: "mapping", mappings: ["&=> and "] } } } } raise ArgumentError, "Can't pass both language and stemmer" if options[:stemmer] && language update_language(settings, language) update_stemming(settings) if Searchkick.env == "test" settings[:number_of_shards] = 1 settings[:number_of_replicas] = 0 end if options[:similarity] settings[:similarity] = {default: {type: options[:similarity]}} end settings[:index] = { max_ngram_diff: 49, max_shingle_diff: 4 } if options[:case_sensitive] settings[:analysis][:analyzer].each do |_, analyzer| analyzer[:filter].delete("lowercase") end end add_synonyms(settings) add_search_synonyms(settings) if options[:special_characters] == false settings[:analysis][:analyzer].each_value do |analyzer_settings| analyzer_settings[:filter].reject! { |f| f == "asciifolding" } end end settings end def update_language(settings, language) case language when "chinese" settings[:analysis][:analyzer].merge!( default_analyzer => { type: "ik_smart" }, searchkick_search: { type: "ik_smart" }, searchkick_search2: { type: "ik_max_word" } ) when "chinese2", "smartcn" settings[:analysis][:analyzer].merge!( default_analyzer => { type: "smartcn" }, searchkick_search: { type: "smartcn" }, searchkick_search2: { type: "smartcn" } ) when "japanese", "japanese2" analyzer = { type: "custom", tokenizer: "kuromoji_tokenizer", filter: [ "kuromoji_baseform", "kuromoji_part_of_speech", "cjk_width", "ja_stop", "searchkick_stemmer", "lowercase" ] } settings[:analysis][:analyzer].merge!( default_analyzer => analyzer.deep_dup, searchkick_search: analyzer.deep_dup, searchkick_search2: analyzer.deep_dup ) settings[:analysis][:filter][:searchkick_stemmer] = { type: "kuromoji_stemmer" } when "korean" settings[:analysis][:analyzer].merge!( default_analyzer => { type: "openkoreantext-analyzer" }, searchkick_search: { type: "openkoreantext-analyzer" }, searchkick_search2: { type: "openkoreantext-analyzer" } ) when "korean2" settings[:analysis][:analyzer].merge!( default_analyzer => { type: "nori" }, searchkick_search: { type: "nori" }, searchkick_search2: { type: "nori" } ) when "vietnamese" settings[:analysis][:analyzer].merge!( default_analyzer => { type: "vi_analyzer" }, searchkick_search: { type: "vi_analyzer" }, searchkick_search2: { type: "vi_analyzer" } ) when "polish", "ukrainian" settings[:analysis][:analyzer].merge!( default_analyzer => { type: language }, searchkick_search: { type: language }, searchkick_search2: { type: language } ) end end def update_stemming(settings) if options[:stemmer] stemmer = options[:stemmer] # could also support snowball and stemmer case stemmer[:type] when "hunspell" # supports all token filter options settings[:analysis][:filter][:searchkick_stemmer] = stemmer else raise ArgumentError, "Unknown stemmer: #{stemmer[:type]}" end end stem = options[:stem] # language analyzer used stem = false if settings[:analysis][:analyzer][default_analyzer][:type] != "custom" if stem == false settings[:analysis][:filter].delete(:searchkick_stemmer) settings[:analysis][:analyzer].each do |_, analyzer| analyzer[:filter].delete("searchkick_stemmer") if analyzer[:filter] end end if options[:stemmer_override] stemmer_override = { type: "stemmer_override" } if options[:stemmer_override].is_a?(String) stemmer_override[:rules_path] = options[:stemmer_override] else stemmer_override[:rules] = options[:stemmer_override] end settings[:analysis][:filter][:searchkick_stemmer_override] = stemmer_override settings[:analysis][:analyzer].each do |_, analyzer| stemmer_index = analyzer[:filter].index("searchkick_stemmer") if analyzer[:filter] analyzer[:filter].insert(stemmer_index, "searchkick_stemmer_override") if stemmer_index end end if options[:stem_exclusion] settings[:analysis][:filter][:searchkick_stem_exclusion] = { type: "keyword_marker", keywords: options[:stem_exclusion] } settings[:analysis][:analyzer].each do |_, analyzer| stemmer_index = analyzer[:filter].index("searchkick_stemmer") if analyzer[:filter] analyzer[:filter].insert(stemmer_index, "searchkick_stem_exclusion") if stemmer_index end end end def generate_mappings mapping = {} keyword_mapping = {type: "keyword"} keyword_mapping[:ignore_above] = options[:ignore_above] || 30000 # conversions Array(options[:conversions]).each do |conversions_field| mapping[conversions_field] = { type: "nested", properties: { query: {type: default_type, analyzer: "searchkick_keyword"}, count: {type: "integer"} } } end mapping_options = [:suggest, :word, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end, :highlight, :searchable, :filterable] .to_h { |type| [type, (options[type] || []).map(&:to_s)] } word = options[:word] != false && (!options[:match] || options[:match] == :word) mapping_options[:searchable].delete("_all") analyzed_field_options = {type: default_type, index: true, analyzer: default_analyzer.to_s} mapping_options.values.flatten.uniq.each do |field| fields = {} if options.key?(:filterable) && !mapping_options[:filterable].include?(field) fields[field] = {type: default_type, index: false} else fields[field] = keyword_mapping end if !options[:searchable] || mapping_options[:searchable].include?(field) if word fields[:analyzed] = analyzed_field_options if mapping_options[:highlight].include?(field) fields[:analyzed][:term_vector] = "with_positions_offsets" end end mapping_options.except(:highlight, :searchable, :filterable, :word).each do |type, f| if options[:match] == type || f.include?(field) fields[type] = {type: default_type, index: true, analyzer: "searchkick_#{type}_index"} end end end mapping[field] = fields[field].merge(fields: fields.except(field)) end (options[:locations] || []).map(&:to_s).each do |field| mapping[field] = { type: "geo_point" } end options[:geo_shape] = options[:geo_shape].product([{}]).to_h if options[:geo_shape].is_a?(Array) (options[:geo_shape] || {}).each do |field, shape_options| mapping[field] = shape_options.merge(type: "geo_shape") end if options[:inheritance] mapping[:type] = keyword_mapping end routing = {} if options[:routing] routing = {required: true} unless options[:routing] == true routing[:path] = options[:routing].to_s end end dynamic_fields = { # analyzed field must be the default field for include_in_all # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/ # however, we can include the not_analyzed field in _all # and the _all index analyzer will take care of it "{name}" => keyword_mapping } if options.key?(:filterable) dynamic_fields["{name}"] = {type: default_type, index: false} end unless options[:searchable] if options[:match] && options[:match] != :word dynamic_fields[options[:match]] = {type: default_type, index: true, analyzer: "searchkick_#{options[:match]}_index"} end if word dynamic_fields[:analyzed] = analyzed_field_options end end # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/ multi_field = dynamic_fields["{name}"].merge(fields: dynamic_fields.except("{name}")) mappings = { properties: mapping, _routing: routing, # https://gist.github.com/kimchy/2898285 dynamic_templates: [ { string_template: { match: "*", match_mapping_type: "string", mapping: multi_field } } ] } mappings end def add_synonyms(settings) synonyms = options[:synonyms] || [] synonyms = synonyms.call if synonyms.respond_to?(:call) if synonyms.any? settings[:analysis][:filter][:searchkick_synonym] = { type: "synonym", # only remove a single space from synonyms so three-word synonyms will fail noisily instead of silently synonyms: synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.map { |s2| s2.sub(/\s+/, "") }.join(",") : s }.map(&:downcase) } # choosing a place for the synonym filter when stemming is not easy # https://groups.google.com/forum/#!topic/elasticsearch/p7qcQlgHdB8 # TODO use a snowball stemmer on synonyms when creating the token filter # http://elasticsearch-users.115913.n3.nabble.com/synonym-multi-words-search-td4030811.html # I find the following approach effective if you are doing multi-word synonyms (synonym phrases): # - Only apply the synonym expansion at index time # - Don't have the synonym filter applied search # - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general. settings[:analysis][:analyzer][default_analyzer][:filter].insert(2, "searchkick_synonym") %w(word_start word_middle word_end).each do |type| settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_synonym") end end end def add_search_synonyms(settings) search_synonyms = options[:search_synonyms] || [] search_synonyms = search_synonyms.call if search_synonyms.respond_to?(:call) if search_synonyms.is_a?(String) || search_synonyms.any? if search_synonyms.is_a?(String) synonym_graph = { type: "synonym_graph", synonyms_path: search_synonyms } synonym_graph[:updateable] = true unless below73? else synonym_graph = { type: "synonym_graph", # TODO confirm this is correct synonyms: search_synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.join(",") : s }.map(&:downcase) } end settings[:analysis][:filter][:searchkick_synonym_graph] = synonym_graph if ["japanese", "japanese2"].include?(options[:language]) [:searchkick_search, :searchkick_search2].each do |analyzer| settings[:analysis][:analyzer][analyzer][:filter].insert(4, "searchkick_synonym_graph") end else [:searchkick_search2, :searchkick_word_search].each do |analyzer| unless settings[:analysis][:analyzer][analyzer].key?(:filter) raise Error, "Search synonyms are not supported yet for language" end settings[:analysis][:analyzer][analyzer][:filter].insert(2, "searchkick_synonym_graph") end end end end def set_deep_paging(settings) if !settings.dig(:index, :max_result_window) && !settings[:"index.max_result_window"] settings[:index] ||= {} settings[:index][:max_result_window] = 1_000_000_000 end end def index_type @index_type ||= begin index_type = options[:_type] index_type = index_type.call if index_type.respond_to?(:call) index_type end end def default_type "text" end def default_analyzer :searchkick_index end def below73? Searchkick.server_below?("7.3.0") end end end