lib/fluent/plugin/out_s3.rb in fluent-plugin-s3-0.3.6 vs lib/fluent/plugin/out_s3.rb in fluent-plugin-s3-0.3.7

@@ -33,10 +33,11 @@
   config_param :aws_sec_key, :string, :default => nil
   config_param :s3_bucket, :string
   config_param :s3_endpoint, :string, :default => nil
   config_param :s3_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
   config_param :store_as, :string, :default => "gzip"
+  config_param :command_parameter, :string, :default => nil
   config_param :auto_create_bucket, :bool, :default => true
   config_param :check_apikey_on_start, :bool, :default => true
   config_param :proxy_uri, :string, :default => nil
   config_param :reduced_redundancy, :bool, :default => false
 
@@ -67,21 +68,25 @@
         end
       end
     end
 
     @ext, @mime_type = case @store_as
-      when 'gzip' then ['gz', 'application/x-gzip']
-      when 'lzo' then
-        begin
-          Open3.capture3('lzop -V')
-        rescue Errno::ENOENT
-          raise ConfigError, "'lzop' utility must be in PATH for LZO compression"
-        end
-        ['lzo', 'application/x-lzop']
-      when 'json' then ['json', 'application/json']
-      else ['txt', 'text/plain']
-    end
+                       when 'gzip'
+                         ['gz', 'application/x-gzip']
+                       when 'lzo'
+                         check_command('lzop', 'LZO')
+                         @command_parameter = '-qf1' if @command_parameter.nil?
+                         ['lzo', 'application/x-lzop']
+                       when 'lzma2'
+                         check_command('xz', 'LZMA2')
+                         @command_parameter = '-qf0' if @command_parameter.nil?
+                         ['xz', 'application/x-xz']
+                       when 'json'
+                         ['json', 'application/json']
+                       else
+                         ['txt', 'text/plain']
+                       end
 
     @timef = TimeFormatter.new(@time_format, @localtime)
 
     if @localtime
       @path_slicer = Proc.new {|path|
@@ -107,12 +112,12 @@
     options[:use_ssl] = @use_ssl
 
     @s3 = AWS::S3.new(options)
     @bucket = @s3.buckets[@s3_bucket]
 
-    ensure_bucket
     check_apikeys if @check_apikey_on_start
+    ensure_bucket
   end
 
   def format(tag, time, record)
     if @include_time_key || !@format_json
       time_str = @timef.format(time)
@@ -160,11 +165,17 @@
         w = Tempfile.new("chunk-tmp")
         chunk.write_to(w)
         w.close
         tmp.close
         # We don't check the return code because we can't recover lzop failure.
-        system "lzop -qf1 -o #{tmp.path} #{w.path}"
+        system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
+      elsif @store_as == "lzma2"
+        w = Tempfile.new("chunk-xz-tmp")
+        chunk.write_to(w)
+        w.close
+        tmp.close
+        system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
       else
         chunk.write_to(tmp)
         tmp.close
       end
       @bucket.objects[s3path].write(Pathname.new(tmp.path), {:content_type => @mime_type,
@@ -191,9 +202,17 @@
 
   def check_apikeys
     @bucket.empty?
   rescue
     raise "aws_key_id or aws_sec_key is invalid. Please check your configuration"
+  end
+
+  def check_command(command, algo)
+    begin
+      Open3.capture3("#{command} -V")
+    rescue Errno::ENOENT
+      raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
+    end
   end
 end
 
 
 end