out_cloudsearch.rb in fluent-plugin-cloudsearch-0.1.1

- old
+ new

@@ -13,10 +13,13 @@
     config_param :profile_name, :string, :default => nil
 
     # message packをJSONにした時に5MBを超えないように
     MAX_SIZE_LIMIT = 4.5 * 1024 * 1024
 
+    # https://docs.aws.amazon.com/en_us/cloudsearch/latest/developerguide/preparing-data.html
+    INVALID_CHAR_REGEX = /[^\u0009\u000a\u000d\u0020-\uD7FF\uE000-\uFFFD]/
+
     def initialize
       super
 
       require 'aws-sdk'
       require 'json'
@@ -65,10 +68,23 @@
       elsif record['type'] != 'delete' then
         log.warn "type is add or delete #{record.to_s}"
         return ''
       end
 
-      "#{record.to_json},"
+      r = record.dup
+      f = r['fields']
+      if f.kind_of? Hash
+        # replace invalid char to white space
+        f.each do |key, value|
+          if value.kind_of? String
+            f[key] = value.gsub(INVALID_CHAR_REGEX, ' ')
+          else
+            f[key] = value
+          end
+        end
+      end
+
+      return "#{r.to_json},"
     end
 
     def write(chunk)
       documents = '['
       documents << chunk.read.chop  # chop last ','