require 'pathname' module Uricp class Segmenter include Methadone::CLILogging include Methadone::SH include Uricp::CurlPrimitives def initialize(options) @options = options source = options['from'] if @source = source && open(source) @stream = File.pipe?(source) || File.chardev?(source) else @source = STDIN @stream = true end split_path end def split_path elements = Pathname.new(to.path).enum_for(:each_filename).to_a elements.shift @account = elements.shift @container = elements.shift @object_path = File.join(elements) end def upload if large_upload? segmented_upload else sh! curl_upload_from(options['from']), on_fail: "Upload to #{to} failed" end end def segment_size options['segment-size'].to_i end def stream? @stream end def large_upload? stream? || @source.stat.size > segment_size end def manifest_suffix @manifest_suffix ||= [Time.now.to_f, @source.stat.size, segment_size].join('/') end def object_manifest @object_manifest ||= File.join(@container, @object_path, manifest_suffix) + '/' end def segmented_upload debug "#{self.class.name}: Large upload detected - segmenting into #{segment_size} byte chunks." suffix = 0 until @source.eof? debug "#{self.class.name}: Uploading segment #{suffix}" upload_segment(suffix) suffix = suffix.next end add_manifest end def upload_segment(segment_number) segment_name = File.join(to.to_s, manifest_suffix, '%08d' % segment_number) debug "Uploading with #{curl_upload_from('-', segment_name)}" open('|' + curl_upload_from('-', segment_name), 'w') do |destination| copy_length = IO.copy_stream(@source, destination, segment_size) debug "#{self.class.name}: Uploaded #{copy_length} bytes to #{segment_name}" end end def add_manifest debug "Adding DLO object_manifest #{object_manifest}" sh! curl_manifest(object_manifest), on_fail: "Upload to #{to} failed" end end end