require_relative 'launcher/valid_methods' require_relative 'launcher/options' require_relative 'launcher/mode_size' require_relative 'launcher/benchmarking' require_relative 'launcher/status_handling' require_relative 'launcher/retry' module Eco module API class Session class Batch module Launcher def self.included(base) unless base <= Eco::API::Common::Session::BaseSession msg = "To be included only in Eco::API::Common::Session::BaseSession. " msg << "Tried on '#{base}'" raise msg end super base.send(:include, ValidMethods) end include Options include ModeSize include Benchmarking include StatusHandling include Retry TIMEOUT_RETRIES = 2 RETRY_ON = [ Ecoportal::API::Errors::TimeOut, Ecoportal::API::Errors::StartTimeOut ].freeze private def batch_from( data, method:, params: {}, silent: false, options: self.options ) fatal "Invalid batch method: #{method}." unless valid_method?(method) return unless data.is_a?(Enumerable) msg = "cannot batch #{method} without api connnection, please provide a valid api connection!" fatal msg unless (people_api = api&.people) launch_batch( data, method: method, people_api: people_api, silent: silent, options: options ) end def launch_batch( # rubocop:disable Metrics/AbcSize data, method:, status: nil, job_mode: true, options: self.options, people_api: api&.people, silent: false ) tap_status(status: status, enviro: enviro, queue: data, method: method) do |overall_status| pending_for_server_error = data.to_a[0..] batch_mode_on(*RETRY_ON, options: options, allow_job_mode: job_mode) do |job_mode, per_page| iteration = 0 done = 0 iterations = (data.length.to_f / per_page).ceil start_time = Time.now data.each_slice(per_page) do |slice| iteration += 1 msg = "starting batch '#{method}' iteration #{iteration}/#{iterations}, " msg << "with #{slice.length} entries of #{data.length} -- #{done} done" msg << (" " * 20) log(:info) { msg } unless silent start_slice = Time.now offer_retry_on(*RETRY_ON, retries_left: TIMEOUT_RETRIES) do people_api.batch(job_mode: job_mode) do |batch| slice.each do |person| batch.public_send(method, person) do |response| faltal("Request with no response") unless response next if server_error?(response) pending_for_server_error.delete(person) overall_status[person] = response end end end # end batch end done += slice.length msg = " ... iteration #{iteration}/#{iterations} done " msg << "in #{str_per_sec(start_slice, slice.length)} " msg << "(average: #{str_per_sec(start_time, done)})" msg << (" " * 20) log(:info) { msg } unless silent end # next slice end # temporary working around (due to back-end problems with batch/jobs) unless pending_for_server_error.empty? msg = "Going to re-try #{pending_for_server_error.count} due to server errors" log(:info) { msg } unless silent launch_batch( pending_for_server_error, status: overall_status, method: method, job_mode: false, per_page: per_page, people_api: people_api, silent: silent, options: options ) end end end def server_error?(response) res_status = response.status server_error = !res_status || res_status.server_error? other_error = !server_error && (!res_status.code || res_status.code < 100) no_body = !server_error && !other_error && !response.body server_error || other_error || no_body end end end end end end