begin require 'sidekiq' rescue LoadError end require_relative './batch_aware_job' require_relative "./callback" require_relative "./context_hash" require_relative "./status" Dir[File.dirname(__FILE__) + "/jobs/*.rb"].each { |file| require file } require_relative "./chain_builder" # Implement Job Batching similar to Sidekiq::Batch. Supports ActiveJob and Sidekiq, or a mix thereof. # Much of this code is modifed/extended from https://github.com/breamware/sidekiq-batch module CanvasSync module JobBatches class Batch class NoBlockGivenError < StandardError; end def self.batch_attr(key, read_only: true) class_eval <<-RUBY, __FILE__, __LINE__ + 1 def #{key}=(value) raise "#{key} is read-only once the batch has been started" if #{read_only.to_s} && (@initialized || @existing) @#{key} = value persist_bid_attr('#{key}', value) end def #{key} return @#{key} if defined?(@#{key}) if (@initialized || @existing) @#{key} = read_bid_attr('#{key}') end end RUBY end delegate :redis, to: :class BID_EXPIRE_TTL = 2_592_000 attr_reader :bid def initialize(existing_bid = nil) @bid = existing_bid || SecureRandom.urlsafe_base64(10) @existing = !(!existing_bid || existing_bid.empty?) # Basically existing_bid.present? @initialized = false @bidkey = "BID-" + @bid.to_s @pending_attrs = {} @ready_to_queue = nil self.created_at = Time.now.utc.to_f unless @existing end batch_attr :description batch_attr :created_at batch_attr :callback_queue, read_only: false batch_attr :callback_batch, read_only: false batch_attr :allow_context_changes def context return @context if defined?(@context) if (@initialized || @existing) @context = ContextHash.new(bid) else @context = ContextHash.new(bid, {}) end end def context=(value) raise "context is read-only once the batch has been started" if (@initialized || @existing) # && !allow_context_changes raise "context must be a Hash" unless value.is_a?(Hash) || value.nil? return nil if value.nil? && @context.nil? value = {} if value.nil? value = value.local if value.is_a?(ContextHash) @context ||= ContextHash.new(bid, {}) @context.set_local(value) # persist_bid_attr('context', JSON.unparse(@context.local)) end def save_context_changes @context&.save! end def on(event, callback, options = {}) return unless Callback::VALID_CALLBACKS.include?(event.to_s) callback_key = "#{@bidkey}-callbacks-#{event}" redis do |r| r.multi do r.sadd(callback_key, JSON.unparse({ callback: callback, opts: options })) r.expire(callback_key, BID_EXPIRE_TTL) end end end def jobs raise NoBlockGivenError unless block_given? if !@existing && !@initialized parent_bid = Thread.current[:batch]&.bid redis do |r| r.multi do r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid r.expire(@bidkey, BID_EXPIRE_TTL) if parent_bid r.hincrby("BID-#{parent_bid}", "children", 1) r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL) r.sadd("BID-#{parent_bid}-bids", bid) end end end flush_pending_attrs @context&.save! @initialized = true end job_queue = @ready_to_queue = [] puts "Beginning Batch #{bid}" begin parent = Thread.current[:batch] Thread.current[:batch] = self yield ensure @ready_to_queue = nil append_jobs(job_queue, parent_bid) Thread.current[:batch] = parent end job_queue end def increment_job_queue(jid) if @ready_to_queue @ready_to_queue << jid else append_jobs([jid]) end end def invalidate_all redis do |r| r.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1) end end def parent_bid redis do |r| r.hget(@bidkey, "parent_bid") end end def parent if parent_bid Batch.new(parent_bid) end end def valid?(batch = self) valid = !redis { |r| r.exists?("invalidated-bid-#{batch.bid}") } batch.parent ? valid && valid?(batch.parent) : valid end # Any Batches or Jobs created in the given block won't be assocaiated to the current batch def self.without_batch parent = Thread.current[:batch] Thread.current[:batch] = nil yield ensure Thread.current[:batch] = parent end private def persist_bid_attr(attribute, value) if @initialized || @existing redis do |r| r.multi do r.hset(@bidkey, attribute, value) r.expire(@bidkey, BID_EXPIRE_TTL) end end else @pending_attrs[attribute] = value end end def read_bid_attr(attribute) redis do |r| r.hget(@bidkey, attribute) end end def flush_pending_attrs redis do |r| r.mapped_hmset(@bidkey, @pending_attrs) end @pending_attrs = {} end def append_jobs(jids, parent_bid = self.parent_bid) redis do |r| r.multi do if parent_bid r.hincrby("BID-#{parent_bid}", "total", jids.size) end r.hincrby(@bidkey, "pending", jids.size) r.hincrby(@bidkey, "total", jids.size) r.expire(@bidkey, BID_EXPIRE_TTL) if jids.size > 0 r.sadd(@bidkey + "-jids", jids) r.expire(@bidkey + "-jids", BID_EXPIRE_TTL) end end end end class << self def process_failed_job(bid, jid) _, pending, failed, children, complete, parent_bid = redis do |r| r.multi do r.sadd("BID-#{bid}-failed", jid) r.hincrby("BID-#{bid}", "pending", 0) r.scard("BID-#{bid}-failed") r.hincrby("BID-#{bid}", "children", 0) r.scard("BID-#{bid}-batches-complete") r.hget("BID-#{bid}", "parent_bid") r.expire("BID-#{bid}-failed", BID_EXPIRE_TTL) end end # if the batch failed, and has a parent, update the parent to show one pending and failed job if parent_bid redis do |r| r.multi do r.hincrby("BID-#{parent_bid}", "pending", 1) r.sadd("BID-#{parent_bid}-failed", jid) r.expire("BID-#{parent_bid}-failed", BID_EXPIRE_TTL) end end end if pending.to_i == failed.to_i && children == complete enqueue_callbacks(:complete, bid) end end def process_dead_job(bid, jid) _, failed, children, complete, parent_bid = redis do |r| r.multi do r.sadd("BID-#{bid}-dead", jid) r.scard("BID-#{bid}-dead") r.hincrby("BID-#{bid}", "children", 0) r.scard("BID-#{bid}-batches-complete") r.hget("BID-#{bid}", "parent_bid") r.expire("BID-#{bid}-dead", BID_EXPIRE_TTL) end end if parent_bid redis do |r| r.multi do r.sadd("BID-#{parent_bid}-dead", jid) r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL) end end end enqueue_callbacks(:dead, bid) end def process_successful_job(bid, jid) _, failed, pending, children, complete, success, total, parent_bid = redis do |r| r.multi do r.srem("BID-#{bid}-failed", jid) r.scard("BID-#{bid}-failed") r.hincrby("BID-#{bid}", "pending", -1) r.hincrby("BID-#{bid}", "children", 0) r.scard("BID-#{bid}-batches-complete") r.scard("BID-#{bid}-batches-success") r.hget("BID-#{bid}", "total") r.hget("BID-#{bid}", "parent_bid") r.srem("BID-#{bid}-jids", jid) r.expire("BID-#{bid}", BID_EXPIRE_TTL) end end all_success = pending.to_i.zero? && children == success # if complete or successfull call complete callback (the complete callback may then call successful) if (pending.to_i == failed.to_i && children == complete) || all_success enqueue_callbacks(:complete, bid) enqueue_callbacks(:success, bid) if all_success end end def enqueue_callbacks(event, bid) batch_key = "BID-#{bid}" callback_key = "#{batch_key}-callbacks-#{event}" already_processed, _, callbacks, queue, parent_bid, callback_batch = redis do |r| r.multi do r.hget(batch_key, event) r.hset(batch_key, event, true) r.smembers(callback_key) r.hget(batch_key, "callback_queue") r.hget(batch_key, "parent_bid") r.hget(batch_key, "callback_batch") end end return if already_processed == 'true' queue ||= "default" parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid # Basically parent_bid.blank? callback_args = callbacks.reduce([]) do |memo, jcb| cb = JSON.load(jcb) memo << [cb['callback'], event.to_s, cb['opts'], bid, parent_bid] end opts = {"bid" => bid, "event" => event} # Run callback batch finalize synchronously if callback_batch # Extract opts from cb_args or use current # Pass in stored event as callback finalize is processed on complete event cb_opts = callback_args.first&.at(2) || opts logger.debug {"Run callback batch bid: #{bid} event: #{event} args: #{callback_args.inspect}"} # Finalize now finalizer = Batch::Callback::Finalize.new status = Status.new bid finalizer.dispatch(status, cb_opts) return end logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"} if callback_args.empty? # Finalize now finalizer = Batch::Callback::Finalize.new status = Status.new bid finalizer.dispatch(status, opts) else # Otherwise finalize in sub batch complete callback cb_batch = self.new cb_batch.callback_batch = true logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"} cb_batch.on(:complete, "#{Batch::Callback::Finalize.to_s}#dispatch", opts) cb_batch.jobs do push_callbacks callback_args, queue end end end def cleanup_redis(bid) logger.debug {"Cleaning redis of batch #{bid}"} # redis do |r| # r.del( # "BID-#{bid}", # "BID-#{bid}-callbacks-complete", # "BID-#{bid}-callbacks-success", # "BID-#{bid}-failed", # "BID-#{bid}-batches-success", # "BID-#{bid}-batches-complete", # "BID-#{bid}-batches-failed", # "BID-#{bid}-jids", # ) # end end def redis(*args, &blk) defined?(::Sidekiq) ? ::Sidekiq.redis(*args, &blk) : nil # TODO end def logger defined?(::Sidekiq) ? ::Sidekiq.logger : Rails.logger end private def push_callbacks(args, queue) Batch::Callback::Worker.enqueue_all(args, queue) end end end ActiveJob::Base.include BatchAwareJob end end # Automatically integrate with Sidekiq if it is present. if defined?(::Sidekiq) require_relative './sidekiq' CanvasSync::JobBatches::Sidekiq.configure end