require "bearcat" require "canvas_sync/version" require "canvas_sync/engine" require "canvas_sync/misc_helper" require "canvas_sync/class_callback_executor" require "canvas_sync/job" require "canvas_sync/job_chain" require "canvas_sync/sidekiq_job" require "canvas_sync/api_syncable" require "canvas_sync/record" require "canvas_sync/jobs/report_starter" require "canvas_sync/jobs/report_checker" require "canvas_sync/jobs/report_processor_job" require "canvas_sync/config" Dir[File.dirname(__FILE__) + "/canvas_sync/jobs/*.rb"].each { |file| require file } Dir[File.dirname(__FILE__) + "/canvas_sync/processors/*.rb"].each { |file| require file } Dir[File.dirname(__FILE__) + "/canvas_sync/importers/*.rb"].each { |file| require file } Dir[File.dirname(__FILE__) + "/canvas_sync/generators/*.rb"].each { |file| require file } Dir[File.dirname(__FILE__) + "/canvas_sync/concerns/**/*.rb"].each { |file| require file } module CanvasSync SUPPORTED_MODELS = %w[ users pseudonyms courses groups group_memberships accounts terms enrollments sections assignments submissions roles admins assignment_groups context_modules context_module_items xlist ].freeze SUPPORTED_LIVE_EVENTS = %w[ course enrollment submission assignment user syllabus grade module module_item course_section ].freeze SUPPORTED_NON_PROV_REPORTS = %w[ graded_submissions ].freeze class << self # Runs a standard provisioning sync job with no extra report types. # Terms will be synced first using the API. If you are syncing users/roles/admins # and have also specified a Term scope, Users/Roles/Admins will by synced first, before # every other model (as Users/Roles/Admins are never scoped to Term). # # @param models [Array] A list of models to sync. e.g., ['users', 'courses']. # must be one of SUPPORTED_MODELS # @param term_scope [Symbol, nil] An optional symbol representing a scope that exists on the Term model. # The provisioning report will be run for each of the terms contained in that scope. # @param legacy_support [Boolean | Array, false] This enables legacy_support, where rows are not bulk inserted. # For this to work your models must have a `create_or_udpate_from_csv` class method that takes a row # and inserts it into the database. If an array of model names is provided then only those models will use legacy support. # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and # canvas_sync_client methods require an account ID. def provisioning_sync(models, term_scope: nil, legacy_support: false, account_id: nil) validate_models!(models) invoke_next(default_provisioning_report_chain(models, term_scope, legacy_support, account_id)) end # Runs a report different from provisioning sync job with no extra report types. # # @param reports_mapping [Array] An Array of hash that list the model and params with their report you # want to import: # [{model: 'submissions', report_name: 'my_report_name_csv', params: { "parameters[include_deleted]" => true } }, ...] # @param term_scope [Symbol, nil] An optional symbol representing a scope that exists on the Term model. # The provisioning report will be run for each of the terms contained in that scope. # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and # canvas_sync_client methods require an account ID. def simple_report_sync(reports_mapping, term_scope: nil, account_id: nil) invoke_next(simple_report_chain(reports_mapping, term_scope, account_id)) end # Runs a chain of ordered jobs # # See the README for usage and examples # # @param job_chain [Hash] def process_jobs(job_chain) invoke_next(job_chain) end # @deprecated def duplicate_chain(job_chain) Marshal.load(Marshal.dump(job_chain)) end # Invokes the next job in a chain of jobs. # # This should typically be called automatically by the gem where necessary. # # @param job_chain [Hash] A chain of jobs to execute def invoke_next(job_chain, extra_options: {}) job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain) job_chain.perform_next(extra_options) end def fork(job_log, job_chain, keys: [], &blk) job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain) job_chain.fork(job_log, keys: keys, &blk) end # Given a Model or Relation, scope it down to items that should be synced def sync_scope(scope) terms = %i[should_canvas_sync active_for_canvas_sync should_sync active_for_sync active] terms.each do |t| return scope.send(t) if scope.respond_to?(t) end model = scope.try(:model) || scope if model.try(:column_names)&.include?(:workflow_state) return scope.where.not(workflow_state: %w[deleted]) end Rails.logger.warn("Could not filter Syncable Scope for model '#{scope.try(:model)&.name || scope.name}'") scope end # Syn any report to an specific set of models # # @param reports_mapping [Array] List of reports with their specific model and params # @param term_scope [String] # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and # canvas_sync_client methods require an account ID. def simple_report_chain(reports_mapping, term_scope=nil, account_id=nil) jobs = reports_mapping.map do |report| { job: CanvasSync::Jobs::SyncSimpleTableJob.to_s, options: { report_name: report[:report_name], model: report[:model], mapping: report[:model], klass: report[:model].singularize.capitalize.to_s, term_scope: term_scope, params: report[:params] } } end global_options = {} global_options[:account_id] = account_id if account_id.present? JobChain.new(jobs: jobs, global_options: global_options) end # Syncs terms, users/roles/admins if necessary, then the rest of the specified models. # # @param models [Array] # @param term_scope [String] # @param legacy_support [Boolean, false] This enables legacy_support, where rows are not bulk inserted. # For this to work your models must have a `create_or_udpate_from_csv` class method that takes a row # and inserts it into the database. # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and # canvas_sync_client methods require an account ID. # @return [Hash] def default_provisioning_report_chain(models, term_scope=nil, legacy_support=false, account_id=nil, options: {}) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength return unless models.present? models.map! &:to_s term_scope = term_scope.to_s if term_scope options = options.deep_symbolize_keys! model_job_map = { terms: CanvasSync::Jobs::SyncTermsJob, accounts: CanvasSync::Jobs::SyncAccountsJob, roles: CanvasSync::Jobs::SyncRolesJob, admins: CanvasSync::Jobs::SyncAdminsJob, assignments: CanvasSync::Jobs::SyncAssignmentsJob, submissions: CanvasSync::Jobs::SyncSubmissionsJob, assignment_groups: CanvasSync::Jobs::SyncAssignmentGroupsJob, context_modules: CanvasSync::Jobs::SyncContextModulesJob, context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob, }.with_indifferent_access jobs = [] try_add_model_job = ->(model) { return unless models.include?(model) jobs.push(job: model_job_map[model].to_s, options: options[model.to_sym] || {}) models -= [model] } ############################## # Pre provisioning report jobs ############################## # Always sync Terms first models.unshift('terms') unless models.include?('terms') try_add_model_job.call('terms') # Accounts, users, roles, and admins are synced before provisioning because they cannot be scoped to term try_add_model_job.call('accounts') # These Models use the provisioning report, but are not term-scoped, # so we sync them before to ensure work is not duplicated if term_scope.present? models -= (first_provisioning_models = models & ['users', 'pseudonyms']) jobs.concat( generate_provisioning_jobs(first_provisioning_models, options) ) end try_add_model_job.call('roles') try_add_model_job.call('admins') pre_provisioning_jobs = jobs ############################### # Post provisioning report jobs ############################### jobs = [] try_add_model_job.call('assignments') try_add_model_job.call('submissions') try_add_model_job.call('assignment_groups') try_add_model_job.call('context_modules') try_add_model_job.call('context_module_items') post_provisioning_jobs = jobs ############################### # Main provisioning job and queueing ############################### jobs = [ *pre_provisioning_jobs, *generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users']), *post_provisioning_jobs, ] global_options = { legacy_support: legacy_support } global_options[:account_id] = account_id if account_id.present? global_options.merge!(options[:global]) if options[:global].present? JobChain.new(jobs: jobs, global_options: global_options) end def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning) dup_models = [ *model_list ] unique_option_models = {} filtered_models = only_split ? (only_split & model_list) : model_list filtered_models.each do |m| mopts = options_hash[m.to_sym] || options_hash[default_key] unique_option_models[mopts] ||= [] unique_option_models[mopts] << m dup_models.delete(m) end if dup_models.present? mopts = options_hash[default_key] unique_option_models[mopts] ||= [] unique_option_models[mopts].concat(dup_models) end unique_option_models end def generate_provisioning_jobs(model_list, options_hash, job_options: {}, only_split: nil, default_key: :provisioning) # Group the model options as best we can. # This is mainly for backwards compatibility, since 'users' was previously it's own job unique_option_models = group_by_job_options( model_list, options_hash, only_split: only_split, default_key: default_key, ) unique_option_models.map do |mopts, models| opts = { models: models } opts.merge!(job_options) opts.merge!(mopts) if mopts.present? { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: opts, } end end # Calls the canvas_sync_client in your app. If you have specified an account # ID when starting the job it will pass the account ID to your canvas_sync_client method. # # @param options [Hash] def get_canvas_sync_client(options) if options[:account_id] canvas_sync_client(options[:account_id]) else canvas_sync_client end end # Configure options for CanvasSync. See config.rb for valid configuration options. # # Example: # # CanvasSync.configure do |config| # config.classes_to_only_log_errors_on << "Blah" # end def configure yield config config end # Returns the CanvasSync config def config @config ||= CanvasSync::Config.new end def validate_models!(models) invalid = models - SUPPORTED_MODELS return if invalid.empty? raise "Invalid model(s) specified: #{invalid.join(', ')}. Only #{SUPPORTED_MODELS.join(', ')} are supported." end def validate_live_events!(events) invalid = events - SUPPORTED_LIVE_EVENTS return if invalid.empty? raise "Invalid live event(s) specified: #{invalid.join(', ')}. Only #{SUPPORTED_LIVE_EVENTS.join(', ')} are supported." end end end