batch.rb in eco-helpers-0.8.1

- old
+ new

@@ -39,16 +39,25 @@
         # @return [BatchStatus] the `status` of this batch launch.
         def launch(people, method:, params: {} , silent: false)
           batch_from(people, method: method, params: params, silent: silent)
         end
 
-        def search(data, params: {})
+        def search(data, silent: false, params: {})
           params = {per_page: DEFAULT_BATCH_BLOCK}.merge(params)
 
-          launch(data, method: :get, params: params, silent: true).tap do |status|
+          launch(data, method: :get, params: params, silent: silent).tap do |status|
             status.type = :search
-            status.queue.each do |entry|
+
+            entries = status.queue
+            puts "\n"
+            entries.each_with_index do |entry, i|
+              if (i % 500 == 0)
+                percent = i * 100 / entries.length
+                print "Searching: #{percent.round}% (#{i}/#{entries.length} entries)\r"
+                $stdout.flush
+              end
+
               unless status.success?(entry)
                 email = nil
                 case
                 when entry.respond_to?(:email)
                   email = entry.email
@@ -57,11 +66,11 @@
                 end
 
                 people_matching = []
                 email = email.to_s.strip.downcase
                 unless email.empty?
-                  people_matching = get(params: params.merge(q: email), silent: true).select do |person|
+                  people_matching = get(params: params.merge(q: email), silent: silent).select do |person|
                     person.email == email
                   end
                 end
 
                 case people_matching.length
@@ -89,20 +98,29 @@
 
           looping = !params.key?(:page)
           page = params[:page] || 1
 
           people = []; total_pages = nil
+          results_from = nil
           loop do
+            params.merge!(results_from: results_from) unless !results_from
             people_res, response = client_get(client, params: params.merge(page: page), silent: silent)
             people += people_res
 
-            total_pages ||= response.body["total_pages"]
+            total_iterations ||= response.body["total_pages"]
+            no_pages = !response.body["total_pages"]
+            total_results    ||= response.body["total_results"]
 
-            msg = "page number: #{page}/#{total_pages}, got num people #{people_res.length}, with total #{people.length} people got"
+            if !total_iterations
+              total_iterations ||= (total_results.to_f / params[:per_page]).ceil
+            end
+
+            msg = "iteration number: #{page}/#{total_iterations}, got num people #{people_res.length}, with total #{people.length} people got"
             logger.info(msg) unless silent
 
-            break if page >= total_pages || !looping
+            iterate = (no_pages && results_from = response.body["next_results_from"]) || (looping && page < total_iterations)
+            break unless iterate
             page += 1
           end
 
           return people
         end
@@ -121,26 +139,26 @@
             people.push(person)
           end
           [people, response]
         end
 
-        def batch_from(people, method:, params: {}, silent: false)
+        def batch_from(data, method:, params: {}, silent: false)
           fatal "Invalid batch method: #{method}." if !self.class.valid_method?(method)
-          return nil if !people || !people.is_a?(Enumerable)
+          return nil if !data || !data.is_a?(Enumerable)
           fatal "cannot batch #{method} without api connnection, please provide a valid api connection!" unless people_api = api&.people
 
           # batch Status
-          status = new_status(people, method)
+          status = new_status(data, method)
 
           # param q does not make sense here, even for GET method
           params   = {per_page: DEFAULT_BATCH_BLOCK}.merge(params)
           per_page = params[:per_page] || DEFAULT_BATCH_BLOCK
 
           iteration  = 1; done = 0
-          iterations = (people.length.to_f / per_page).ceil
+          iterations = (data.length.to_f / per_page).ceil
 
-          people.each_slice(per_page) do |slice|
-            msg = "starting batch '#{method}' iteration #{iteration}/#{iterations}, with #{slice.length} entries of #{people.length} -- #{done} done"
+          data.each_slice(per_page) do |slice|
+            msg = "starting batch '#{method}' iteration #{iteration}/#{iterations}, with #{slice.length} entries of #{data.length} -- #{done} done"
             logger.info(msg) unless silent
 
             people_api.batch do |batch|
               slice.each do |person|
                 batch.public_send(method, person) do |response|