module Ecoportal module API class V2 class S3 class Files # Class service to upload multiple files in one go class BatchUpload include Ecoportal::API::Common::Concerns::Benchmarkable include Ecoportal::API::Common::Concerns::Threadable SIZE_UNITS = 1_024 # KB MIN_UNIT = 1 attr_reader :files, :files_api FileResult = Struct.new(:file) do attr_accessor :poll, :s3_file_reference, :error def error? !error.nil? end def success? poll&.success? end def container_id return unless success? poll.container_id end end # @param files [Array] the files to be uploaded # @param files_api [API::S3::Files] the api object. def initialize(files, files_api:) @files = [files].flatten.compact @files_api = files_api end # Do the actual upload of the file def upload!(benchmarking: false, threads: 1, **kargs) # rubocop:disable Metrics/AbcSize bench = benchmarking && benchmark_enabled? bench_mth = "#{self.class}##{__method__}" thr_children = [] benchmarking("#{bench_mth}.#{files.count}_files", print: bench) do with_preserved_thread_globals(report: false) do files.each do |file| new_thread(thr_children, max: threads) do file_results << (result = FileResult.new(file)) s3_ref = nil kbytes = bench ? file_size(file) : 1 benchmarking("#{bench_mth}.s3_upload (KB)", units: kbytes, print: bench) do s3_ref = result.s3_file_reference = s3_api.upload_file(file) end benchmarking("##{bench_mth}.ep_poll (KB)", units: kbytes, print: bench) do files_api.poll!(s3_ref, **kargs) do |poll| result.poll = poll end end yield(result) if block_given? rescue *rescued_errors => err # each_file result.error = err yield(result) if block_given? end end end end thr_children.each(&:join) file_results ensure puts benchmark_summary(:all) if bench end private def file_size(file, bytes: SIZE_UNITS) return 1 unless File.exist?(file) units = (File.size(file) / bytes).round(10) [MIN_UNIT, units].max end def s3_api files_api.s3_api end def file_results @file_results ||= [] end def rescued_errors @rescued_errors ||= [ Ecoportal::API::V2::S3::MissingLocalFile, Ecoportal::API::V2::S3::CredentialsGetFailed, Ecoportal::API::V2::S3::Files::FailedPollRequest, Ecoportal::API::V2::S3::Files::CantCheckStatus, Ecoportal::API::Errors::TimeOut ].freeze end end end end end end end