module Eco module API class Session # important! the handler should probably only create logs and save the update with same timestamp class Batch < Common::Session::BaseSession DEFAULT_BATCH_BLOCK = 100 VALID_METHODS = ['get', 'create', 'update', 'upsert', 'delete'] # people can be: empty, an api object, a people object, or an array with id's or persons def get_people(people = nil, api: nil, params: {per_page: DEFAULT_BATCH_BLOCK}) api = api || self.api return batch_from(people, 'get', api, params: params) if people.is_a?(Array) return batch_get(api, params: params) end def launch(people = nil, method:, api: nil, params: {per_page: DEFAULT_BATCH_BLOCK}) if !valid_method?(method) msg = "Invalid batch method: #{method}." logger.fatal(msg) raise msg end return nil if !people || !people.is_a?(Array) batch_from(people, method, api || self.api, params: params) end def valid_method?(value) VALID_METHODS.include?(value) end private def new_status(queue, method) BatchStatus.new(enviro, queue: queue, method: method) end def batch_get(api, params: {}) unless people_api = api&.people msg = "cannot batch get without api connnection, please provide a valid api connection!" logger.fatal(msg) raise msg end params = {per_page: DEFAULT_BATCH_BLOCK}.merge(params) client = people_api.client people = []; if page = params[:page] res, response = get(client, params: params) people +=res logger.info("page number: #{page}, got num people #{people.length}") else page = 1 loop do params = params.merge({page: page}) people_res, response = get(client, params: params) people += people_res total_pages = response.body["total_pages"] logger.info("page number: #{page}/#{total_pages}, got num people #{people_res.length}, with total #{people.length} people got") break if page >= total_pages page += 1 end end return people end def get(client, params:) response = client.get("/people", params: params) unless response.success? msg = "Request failed - params: #{params}" msg += "\n Error message: - Status #{response.status}: #{response.body}" logger.fatal(msg) raise msg end people = [] response.body["results"].each do |person_hash| person = Ecoportal::API::Internal::Person.new(person_hash) yield person if block_given? people.push(person) end [people, response] end def batch_from(people, method, api, params: {}) return nil if !people || !people.is_a?(Array) unless people_api = api&.people msg = "cannot batch #{method} without api connnection, please provide a valid api connection!" logger.fatal(msg) raise msg end # batch Status status = new_status(people, method) # param q does not make sense here, even for GET method params = {per_page: DEFAULT_BATCH_BLOCK}.merge(params) per_page = params.fetch(:per_page) iteration = 1; done = 0 iterations = (people.length.to_f / per_page).ceil people.each_slice(per_page) do |slice| msg = "starting batch '#{method}' iteration #{iteration}/#{iterations}, with #{slice.length} entries of #{people.length} -- #{done} done" logger.info(msg) people_api.batch do |batch| slice.each do |person| # valid method checked before batch.public_send(method, person) do |response| status[person] = response #status.print_error(person) end # current person end # next person end # next batch iteration += 1 done += slice.length end # next slice status.print_errors return status end end end end end