module Eco module API class Session class Batch < Common::Session::BaseSession DEFAULT_BATCH_BLOCK = 100 VALID_METHODS = [:get, :create, :update, :upsert, :delete] class << self # @return [Boolean] `true` if the method is supported, `false` otherwise. def valid_method?(value) VALID_METHODS.include?(value) end end # Gets the _people_ of the organization according `params`. # If `people` is not `nil`, scopes to only the people specified. # @note # - If `people` is given keys `page:` and `q` of `params:`. # @param people [Nil, People, Enumerable, Enumerable] target _People_ to launch the batch against. # @param params [Hash] api request options. # @option params [String] :page the page number `page` based on `:per_page`. # @option params [String] :per_page the number of people included per each batch api request. # @option params [String] :q some text to search. Omit this parameter to target all the people. # @return [Array] all the people based on `params` def get_people(people = nil, params: {}, silent: false) return launch(people, method: :get, params: params, silent: silent) if people.is_a?(Enumerable) return get(params: params, silent: silent) end # launches a batch of `method` type using `people` and the specified `params` # @raise Exception # - if `people` is `nil` or is not an `Enumerable`. # - if there's no `api` connection linked to the current `Batch`. # @param people [People, Enumerable, Enumerable] target _People_ to launch the batch against. # @param method [Symbol] the method to launch the batch api request with. # @param params [Hash] api request options. # @option params [String] :per_page the number of people included per each batch api request. # @return [Batch::Status] the `status` of this batch launch. def launch(people, method:, params: {} , silent: false) batch_from(people, method: method, params: params, silent: silent) end def search(data, silent: false, params: {}) params = {per_page: DEFAULT_BATCH_BLOCK}.merge(params) launch(data, method: :get, params: params, silent: silent).tap do |status| status.type = :search entries = status.queue puts "\n" entries.each_with_index do |entry, i| if (i % 10 == 0) percent = i * 100 / entries.length print "Searching: #{percent.round}% (#{i}/#{entries.length} entries)\r" $stdout.flush end unless status.success?(entry) email = nil case when entry.respond_to?(:email) email = entry.email when entry.respond_to?(:to_h) email = entry.to_h["email"] end people_matching = [] email = email.to_s.strip.downcase unless email.empty? people_matching = get(params: params.merge(q: email), silent: silent).select do |person| person.email == email end end case people_matching.length when 1 status.set_person_match(entry, people_matching.first) when 2..Float::INFINITY status.set_people_match(entry, people.matching) end end end end end private def new_status(queue, method) Batch::Status.new(enviro, queue: queue, method: method) end def get(params: {}, silent: false) fatal "cannot batch get without api connnection, please provide a valid api connection!" unless people_api = api&.people params = {per_page: DEFAULT_BATCH_BLOCK}.merge(params) client = people_api.client looping = !params.key?(:page) page = params[:page] || 1 people = []; total_pages = nil results_from = nil loop do params.merge!(results_from: results_from) unless !results_from people_res, response = client_get(client, params: params.merge(page: page), silent: silent) people += people_res total_iterations ||= response.body["total_pages"] no_pages = !response.body["total_pages"] total_results ||= response.body["total_results"] if !total_iterations total_iterations ||= (total_results.to_f / params[:per_page]).ceil end msg = "iteration number: #{page}/#{total_iterations}, got num people #{people_res.length}, with total #{people.length} people got" msg = "search(q=#{params[:q]}) " + msg if params.key?(:q) logger.info(msg) unless silent iterate = (no_pages && results_from = response.body["next_results_from"]) || (looping && page < total_iterations) break unless iterate page += 1 end return people end def client_get(client, params:, silent: false) response = client.get("/people", params: params) unless response.success? msg = "Request failed - params: #{params}" msg += "\n Error message: - Status #{response.status}: #{response.body}" fatal msg end people = [] response.body["results"].each do |person_hash| person = Ecoportal::API::Internal::Person.new(person_hash) yield person if block_given? people.push(person) end [people, response] end def batch_from(data, method:, params: {}, silent: false) fatal "Invalid batch method: #{method}." if !self.class.valid_method?(method) return nil if !data || !data.is_a?(Enumerable) fatal "cannot batch #{method} without api connnection, please provide a valid api connection!" unless people_api = api&.people # batch Status status = new_status(data, method) # param q does not make sense here, even for GET method params = {per_page: DEFAULT_BATCH_BLOCK}.merge(params) per_page = params[:per_page] || DEFAULT_BATCH_BLOCK iteration = 1; done = 0 iterations = (data.length.to_f / per_page).ceil data.each_slice(per_page) do |slice| msg = "starting batch '#{method}' iteration #{iteration}/#{iterations}, with #{slice.length} entries of #{data.length} -- #{done} done" logger.info(msg) unless silent people_api.batch do |batch| slice.each do |person| batch.public_send(method, person) do |response| faltal("Request with no response") unless !!response status[person] = response end end end # next batch iteration += 1 done += slice.length end # next slice status.errors.print unless silent return status end end end end end require_relative 'batch/job' require_relative 'batch/status' require_relative 'batch/errors' require_relative 'batch/jobs' require_relative 'batch/jobs_groups'