lib/bearcat/client.rb in bearcat-1.4.13 vs lib/bearcat/client.rb in bearcat-1.5.0.beta1

@@ -1,76 +1,104 @@
 require 'active_support/core_ext/hash'
 require 'footrest/client'
-require 'paul_walker'
+require_relative 'rate_limiting'
+require_relative 'client_module'
 
 module Bearcat
   class Client < Footrest::Client
     require 'bearcat/api_array'
 
+    @added_modules = []
+
     Dir[File.join(__dir__, 'client', '*.rb')].each do |mod|
       mname = File.basename(mod, '.*').camelize
       mname = 'GraphQL' if mname == 'GraphQl'
       require mod
-      include "Bearcat::Client::#{mname}".constantize
+      lmod = "Bearcat::Client::#{mname}".constantize
+      include lmod
+      @added_modules << lmod
     end
 
-    # Override Footrest request for ApiArray support
+    def self.registered_endpoints
+      @registered_endpoints ||= @added_modules.reduce({}) do |h, m|
+        h.merge!(m._registered_endpoints) rescue h
+      end
+      @registered_endpoints
+    end
+
     def request(method, &block)
-      enforce_rate_limits
-      response = connection.send(method, &block)
-      apply_rate_limits(response.headers['x-rate-limit-remaining'])
+      response = rate_limited_request do
+        connection.send(method, &block)
+      end
       ApiArray.process_response(response, self)
     end
 
-    def enforce_rate_limits
-      return unless limit_remaining.present?
-      return unless limit_remaining < Bearcat.rate_limit_min
+    protected
 
-      tts = ((Bearcat.rate_limit_min - limit_remaining) / 5).ceil
-      tts = Bearcat.min_sleep_seconds if tts < Bearcat.min_sleep_seconds
-      tts = Bearcat.max_sleep_seconds if tts > Bearcat.max_sleep_seconds
+    def rate_limited_request
+      return yield unless rate_limiter
 
+      canvas_rate_limits= 0
+      response = nil
 
-      message = "Canvas API rate limit minimum #{Bearcat.rate_limit_min} reached. "\
-        "Sleeping for #{tts} second(s) to catch up ~zzZZ~. "\
-        "Limit Remaining: #{limit_remaining}"
-      Bearcat.logger.debug(message)
+      begin
+        rate_limiter.apply(
+          rate_limit_key,
+          max_sleep: Bearcat.max_sleep_seconds,
+          on_sleep: ->(tts) {
+            message = "Canvas API rate limit reached; sleeping for #{tts.to_i} second(s) to catch up."
+            Bearcat.logger.debug(message)
+          },
+        ) do
+          response = yield
+          0
+        end
+      rescue Footrest::HttpError::Forbidden => err
+        # Somehow our rate-limiting didn't limit enough and Canvas stopped us.
+        response = err.response
+        if canvas_rate_limits < 2 && err.message.include?("(Rate Limit Exceeded)")
+          canvas_rate_limits += 1
+          rate_limiter.checkin_known(rate_limit_key, 0)
 
-      sleep(tts)
-    end
+          message = "Canvas API applied rate limit; upticking Bearcat rate-limit avoidance and retrying (Retry #{canvas_rate_limits})."
+          Bearcat.logger.debug(message)
 
-    def apply_rate_limits(limit)
-      return if limit.nil?
-      self.limit_remaining = limit.to_i
-    end
+          retry
+        end
+        raise
+      ensure
+        headers = response.try(:response_headers) || response.try(:headers) || {}
+        # -50 to provide a little extra leeway and hopefully be more proactive, making sure we don't even get close to Canvas throwing a 403, even if an out-of-band process is involved
+        rate_limiter.checkin_known(rate_limit_key, headers['x-rate-limit-remaining'].to_f - 100) if response
+      end
 
-    def using_master_rate_limit?
-      config[:master_rate_limit].present? || Bearcat.master_rate_limit.present?
+      response
     end
 
-    def limit_remaining
-      if using_master_rate_limit?
-        Bearcat.master_mutex.synchronize do
-          limit = PaulWalker::RateLimit.get(config[:token], config[:token])
-          if limit.nil?
-            PaulWalker::RateLimit.add(config[:token], config[:token], 0, Bearcat::rate_limit_min)
-            limit = { current: 0 }.with_indifferent_access
-          end
-          Bearcat.logger.debug limit['current'].to_s
-          limit['current']
+    def rate_limiter
+      @rate_limiter ||= begin
+        rl = config[:rate_limiter] || Bearcat.rate_limiter
+        master_rate_limit = config[:master_rate_limit].present? ? config[:master_rate_limit] : Bearcat.master_rate_limit
+
+        if rl.nil? && master_rate_limit.nil? && defined?(Rails) && Rails.env.production?
+          master_rate_limit = true if defined?(::Sidekiq)
         end
-      else
-        Bearcat.rate_limits[config[:token]]
-      end
-    end
 
-    def limit_remaining=(value)
-      if using_master_rate_limit?
-        Bearcat.master_mutex.synchronize do
-          PaulWalker::RateLimit.add(config[:token], config[:token], value, Bearcat::rate_limit_min)
+        if rl.nil? && master_rate_limit
+          rl = RateLimiting::RedisLimiter
         end
-      else
-        Bearcat.rate_limits[config[:token]] = value
+
+        if rl.is_a?(Class)
+          rl.new()
+        elsif rl.present?
+          rl
+        end
       end
+    end
+
+    private
+
+    def rate_limit_key
+      Digest::SHA1.hexdigest(config[:token])
     end
   end
 end