index_options.rb in searchkick-3.1.3

- old
+ new

@@ -38,23 +38,23 @@
                 # https://www.elastic.co/guide/en/elasticsearch/guide/current/analysis-intro.html
                 char_filter: ["ampersand"],
                 tokenizer: "standard",
                 # synonym should come last, after stemming and shingle
                 # shingle must come before searchkick_stemmer
-                filter: ["standard", "lowercase", "asciifolding", "searchkick_index_shingle", "searchkick_stemmer"]
+                filter: ["lowercase", "asciifolding", "searchkick_index_shingle", "searchkick_stemmer"]
               },
               searchkick_search: {
                 type: "custom",
                 char_filter: ["ampersand"],
                 tokenizer: "standard",
-                filter: ["standard", "lowercase", "asciifolding", "searchkick_search_shingle", "searchkick_stemmer"]
+                filter: ["lowercase", "asciifolding", "searchkick_search_shingle", "searchkick_stemmer"]
               },
               searchkick_search2: {
                 type: "custom",
                 char_filter: ["ampersand"],
                 tokenizer: "standard",
-                filter: ["standard", "lowercase", "asciifolding", "searchkick_stemmer"]
+                filter: ["lowercase", "asciifolding", "searchkick_stemmer"]
               },
               # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb
               searchkick_autocomplete_search: {
                 type: "custom",
                 tokenizer: "keyword",
@@ -142,10 +142,21 @@
               }
             }
           }
         }
 
+        if below60
+          # ES docs say standard token filter does nothing in ES 5
+          # (and therefore isn't needed at at), but tests say otherwise
+          # https://www.elastic.co/guide/en/elasticsearch/reference/5.0/analysis-standard-tokenfilter.html
+          [default_analyzer, :searchkick_search, :searchkick_search2].each do |analyzer|
+            settings[:analysis][:analyzer][analyzer][:filter].unshift("standard")
+          end
+        end
+
+        stem = options[:stem]
+
         case language
         when "chinese"
           settings[:analysis][:analyzer].merge!(
             default_analyzer => {
               type: "ik_smart"
@@ -156,11 +167,11 @@
             searchkick_search2: {
               type: "ik_max_word"
             }
           )
 
-          settings[:analysis][:filter].delete(:searchkick_stemmer)
+          stem = false
         when "japanese"
           settings[:analysis][:analyzer].merge!(
             default_analyzer => {
               type: "kuromoji"
             },
@@ -169,10 +180,12 @@
             },
             searchkick_search2: {
               type: "kuromoji"
             }
           )
+
+          stem = false
         when "korean"
           settings[:analysis][:analyzer].merge!(
             default_analyzer => {
               type: "openkoreantext-analyzer"
             },
@@ -181,10 +194,12 @@
             },
             searchkick_search2: {
               type: "openkoreantext-analyzer"
             }
           )
+
+          stem = false
         when "vietnamese"
           settings[:analysis][:analyzer].merge!(
             default_analyzer => {
               type: "vi_analyzer"
             },
@@ -193,10 +208,12 @@
             },
             searchkick_search2: {
               type: "vi_analyzer"
             }
           )
+
+          stem = false
         when "polish", "ukrainian", "smartcn"
           settings[:analysis][:analyzer].merge!(
             default_analyzer => {
               type: language
             },
@@ -205,10 +222,12 @@
             },
             searchkick_search2: {
               type: language
             }
           )
+
+          stem = false
         end
 
         if Searchkick.env == "test"
           settings[:number_of_shards] = 1
           settings[:number_of_replicas] = 0
@@ -229,12 +248,13 @@
           settings[:analysis][:analyzer].each do |_, analyzer|
             analyzer[:filter].delete("lowercase")
           end
         end
 
-        if options[:stem] == false
+        if stem == false
+          settings[:analysis][:filter].delete(:searchkick_stemmer)
           settings[:analysis][:analyzer].each do |_, analyzer|
-            analyzer[:filter].delete("searchkick_stemmer")
+            analyzer[:filter].delete("searchkick_stemmer") if analyzer[:filter]
           end
         end
 
         settings = settings.symbolize_keys.deep_merge((options[:settings] || {}).symbolize_keys)
 