require 'pulp_rpm_client' module Katello module Pulp3 class Repository # rubocop:disable Metrics/ClassLength class Yum < ::Katello::Pulp3::Repository include Katello::Util::Errata include Katello::Util::PulpcoreContentFilters UNIT_LIMIT = 10_000 def remote_options options = common_remote_options uri = URI(root.url) unless root.upstream_authentication_token.blank? options.merge!(sles_auth_token: root.upstream_authentication_token) end options.merge!(url: uri.to_s, policy: root.download_policy) end def publication_options(repository_version) options = super(repository_version) options.merge( { metadata_checksum_type: root.checksum_type, package_checksum_type: root.checksum_type } ) end def specific_create_options { retain_package_versions: retain_package_versions_count } end def skip_types skip_types = [] if root.ignorable_content.try(:include?, "srpm") skip_types << "srpm" end skip_types end def distribution_options(path) { base_path: path, publication: repo.publication_href, name: "#{generate_backend_object_name}" } end def import_distribution_data distribution = ::Katello::Pulp3::Distribution.fetch_content_list(repository_version: repo.version_href) if distribution.results.present? repo.update!( :distribution_version => distribution.results.first.release_version, :distribution_arch => distribution.results.first.arch, :distribution_family => distribution.results.first.release_name, :distribution_bootable => self.class.distribution_bootable?(distribution) ) unless distribution.results.first.variants.empty? unless distribution.results.first.variants.first.name.nil? repo.update!(:distribution_variant => distribution.results.first.variants.map(&:name).join(',')) end end end end def sync_url_params(sync_options) params = super params.delete(:mirror) params[:sync_policy] = repo.mirroring_policy params[:optimize] = sync_options[:optimize] if sync_options.key?(:optimize) params end def self.distribution_bootable?(distribution) file_paths = distribution.results.first.images.map(&:path) file_paths.any? do |path| path.include?('vmlinuz') || path.include?('pxeboot') || path.include?('kernel.img') || path.include?('initrd.img') || path.include?('boot.iso') end end def partial_repo_path "/pulp/content/#{repo.relative_path}/".sub('//', '/') end # rubocop:disable Metrics/MethodLength def multi_copy_units(repo_id_map, dependency_solving) tasks = [] if repo_id_map.values.pluck(:content_unit_hrefs).flatten.any? data = PulpRpmClient::Copy.new data.dependency_solving = dependency_solving data.config = [] repo_id_map.each do |source_repo_ids, dest_repo_id_map| dest_repo = ::Katello::Repository.find(dest_repo_id_map[:dest_repo]) dest_repo_href = ::Katello::Pulp3::Repository::Yum.new(dest_repo, SmartProxy.pulp_primary).repository_reference.repository_href content_unit_hrefs = dest_repo_id_map[:content_unit_hrefs] # Not needed during incremental update due to dest_base_version # -> Unless incrementally updating a CV repo that is a soft copy of its library instance. # -> I.e. no filters and not an incremental version. unless dest_repo_id_map[:base_version] source_repo_for_content = ::Katello::Repository.find(source_repo_ids.first) unless source_repo_for_content.library_instance? source_repo_for_content = source_repo_for_content.library_instance end modulemd_defaults_hrefs = modulemd_defaults({ :repository_version => source_repo_for_content.version_href }).map(&:pulp_href).sort package_env_hrefs = packageenvironments({ :repository_version => source_repo_for_content.version_href }).map(&:pulp_href).sort # Don't perform extra content actions if the repo is a soft copy of its library instance. # Taken care of by the IncrementalUpdate action. unless dest_repo.soft_copy_of_library? tasks << remove_all_content_from_repo(dest_repo_href) tasks << add_content_for_repo(dest_repo_href, package_env_hrefs) unless package_env_hrefs.empty? tasks << add_content_for_repo(dest_repo_href, modulemd_defaults_hrefs) unless modulemd_defaults_hrefs.empty? end end source_repo_ids.each do |source_repo_id| source_repo_version = ::Katello::Repository.find(source_repo_id).version_href config = { source_repo_version: source_repo_version, dest_repo: dest_repo_href, content: content_unit_hrefs } config[:dest_base_version] = dest_repo_id_map[:base_version] if dest_repo_id_map[:base_version] data.config << config end end tasks << copy_content_chunked(data) else tasks << remove_all_content_from_mapping(repo_id_map) end tasks.flatten end # rubocop:enable Metrics/MethodLength def copy_api_data_dup(data) data_dup = PulpRpmClient::Copy.new data_dup.dependency_solving = data.dependency_solving data_dup.config = [] data.config.each do |repo_config| config_hash = { source_repo_version: repo_config[:source_repo_version], dest_repo: repo_config[:dest_repo], content: [] } config_hash[:dest_base_version] = repo_config[:dest_base_version] if repo_config[:dest_base_version] data_dup.config << config_hash end data_dup end def copy_content_chunked(data) tasks = [] # Don't chunk if there aren't enough content units if data.config.sum { |repo_config| repo_config[:content].size } <= UNIT_LIMIT return api.copy_api.copy_content(data) end unit_copy_counter = 0 i = 0 leftover_units = data.config.first[:content].deep_dup # Copy data and clear its content fields data_dup = copy_api_data_dup(data) while i < data_dup.config.size # Copy all units within repo or only some? if leftover_units.length < UNIT_LIMIT - unit_copy_counter copy_amount = leftover_units.length else copy_amount = UNIT_LIMIT - unit_copy_counter end data_dup.config[i][:content] = leftover_units.pop(copy_amount) unit_copy_counter += copy_amount if unit_copy_counter != 0 tasks << api.copy_api.copy_content(data_dup) unit_copy_counter = 0 end if leftover_units.empty? # Nothing more to copy -- clear current config's content data_dup.config[i][:content] = [] i += 1 # Fetch unit list for next data config leftover_units = data.config[i][:content].deep_dup unless i == data_dup.config.size end end tasks end def remove_all_content_from_mapping(repo_id_map) tasks = [] repo_id_map.each do |_source_repo_ids, dest_repo_id_map| dest_repo = ::Katello::Repository.find(dest_repo_id_map[:dest_repo]) dest_repo_href = ::Katello::Pulp3::Repository::Yum.new(dest_repo, SmartProxy.pulp_primary).repository_reference.repository_href tasks << remove_all_content_from_repo(dest_repo_href) end tasks end def copy_units(source_repository, content_unit_hrefs, remove_all) remove_all = true if remove_all.nil? tasks = [] if content_unit_hrefs.sort!.any? content_unit_hrefs += packageenvironments({ :repository_version => source_repository.version_href }).map(&:pulp_href).sort content_unit_hrefs += modulemd_defaults({ :repository_version => source_repository.version_href }).map(&:pulp_href).sort first_slice = remove_all content_unit_hrefs.each_slice(UNIT_LIMIT) do |slice| tasks << add_content(slice, first_slice) first_slice = false end else tasks << remove_all_content end tasks end def remove_all_content_from_repo(repo_href) data = PulpRpmClient::RepositoryAddRemoveContent.new( remove_content_units: ['*']) api.repositories_api.modify(repo_href, data) end def remove_all_content data = PulpRpmClient::RepositoryAddRemoveContent.new( remove_content_units: ['*']) api.repositories_api.modify(repository_reference.repository_href, data) end def packageenvironments(options = {}) Katello::Pulp3::Api::Core.fetch_from_list { |page_opts| api.content_package_environments_api.list(page_opts.merge(options)) } end def modulemd_defaults(options = {}) Katello::Pulp3::Api::Core.fetch_from_list { |page_opts| api.content_modulemd_defaults_api.list(page_opts.merge(options)) } end def metadatafiles(options = {}) api.content_repo_metadata_files_api.list(options) end def distributiontrees(options = {}) api.content_distribution_trees_api.list(options) end def add_filter_content(source_repo_ids, filters, filter_list_map, additional_included_errata) filters.each do |filter| if filter.inclusion source_repo_ids.each do |repo_id| filter_list_map[:whitelist_ids] += filter.content_unit_pulp_ids(::Katello::Repository.find(repo_id)) end elsif filter.class == ContentViewErratumFilter source_repo_ids.each do |repo_id| filter_list_map[:blacklist_ids] += filter.content_unit_pulp_ids(::Katello::Repository.find(repo_id), additional_included_errata) end else source_repo_ids.each do |repo_id| filter_list_map[:blacklist_ids] += filter.content_unit_pulp_ids(::Katello::Repository.find(repo_id)) end end end filter_list_map end def add_un_modular_rpms(source_repo_ids, filters, filter_list_map) if (filter_list_map[:whitelist_ids].empty? && filters.select { |filter| filter.inclusion }.empty?) filter_list_map[:whitelist_ids] += source_repo_ids.collect do |source_repo_id| source_repo = ::Katello::Repository.find(source_repo_id) source_repo.rpms.where(:modular => false).pluck(:pulp_id).sort end end filter_list_map end def add_modular_content(source_repo_ids, filters, modular_filters, filter_list_map) inclusion_modular_filters = modular_filters.select { |filter| filter.inclusion } exclusion_modular_filters = modular_filters - inclusion_modular_filters if inclusion_modular_filters.empty? && !(filters.any? { |filter| filter.class == ContentViewErratumFilter && filter.inclusion }) source_repo_ids.each do |source_repo_id| source_repo = ::Katello::Repository.find(source_repo_id) filter_list_map[:whitelist_ids] += source_repo.rpms.where(:modular => true).pluck(:pulp_id).sort filter_list_map[:whitelist_ids] += source_repo.module_streams.pluck(:pulp_id).sort end end unless inclusion_modular_filters.empty? filter_list_map[:whitelist_ids] += source_repo_ids.collect do |source_repo_id| source_repo = ::Katello::Repository.find(source_repo_id) modular_packages(source_repo, inclusion_modular_filters) end end unless exclusion_modular_filters.empty? filter_list_map[:blacklist_ids] += source_repo_ids.collect do |source_repo_id| source_repo = ::Katello::Repository.find(source_repo_id) modular_packages(source_repo, exclusion_modular_filters) end end filter_list_map end # rubocop:disable Metrics/MethodLength def copy_content_from_mapping(repo_id_map, options = {}) repo_id_map.each do |source_repo_ids, dest_repo_map| filters = [ContentViewErratumFilter, ContentViewPackageGroupFilter, ContentViewPackageFilter].collect do |filter_class| filter_class.where(:id => dest_repo_map[:filter_ids]) end modular_filters = ContentViewModuleStreamFilter.where(:id => dest_repo_map[:filter_ids]) filters.flatten!.compact! errata_filters = ContentViewErratumFilter.where(:id => dest_repo_map[:filter_ids]) # Only filter out RPMs & modules via an erratum filter if they're not included in other allowed errata. # Begin this by calculating all the errata that should be copied. # If there are exclude filters, send the errata to the relevant methods to ensure their contents aren't excluded. additional_included_errata = [] all_excluded_errata = [] unless errata_filters.blacklist.empty? all_included_errata, all_excluded_errata = filtered_errata(errata_filters, source_repo_ids) additional_included_errata = all_included_errata - all_excluded_errata end filter_list_map = { whitelist_ids: [], blacklist_ids: [] } filter_list_map = add_filter_content(source_repo_ids, filters, filter_list_map, additional_included_errata) filter_list_map = add_un_modular_rpms(source_repo_ids, filters, filter_list_map) filter_list_map = add_modular_content(source_repo_ids, filters, modular_filters, filter_list_map) whitelist_ids = filter_list_map[:whitelist_ids].flatten&.uniq blacklist_ids = filter_list_map[:blacklist_ids].flatten&.uniq content_unit_hrefs = whitelist_ids - blacklist_ids source_repo_ids.each do |source_repo_id| content_unit_hrefs += ::Katello::Repository.find(source_repo_id).srpms.pluck(:pulp_id) end if content_unit_hrefs.any? source_repo_ids.each do |source_repo_id| content_unit_hrefs += additional_content_hrefs(::Katello::Repository.find(source_repo_id), content_unit_hrefs, all_excluded_errata) end end dest_repo_map[:content_unit_hrefs] = content_unit_hrefs.uniq.sort end dependency_solving = options[:solve_dependencies] || false multi_copy_units(repo_id_map, dependency_solving) end # rubocop:enable Metrics/MethodLength # Calculate errata that should be included and excluded. # https://projects.theforeman.org/issues/34437 def filtered_errata(errata_filters, source_repository_ids) if errata_filters.whitelist.empty? all_included_errata = ::Katello::Erratum.joins("inner join katello_repository_errata on katello_errata.id = katello_repository_errata.erratum_id"). joins("inner join katello_repositories on katello_repositories.id = katello_repository_errata.repository_id"). where("katello_repositories.id in (?)", source_repository_ids) else all_included_errata = errata_filters.whitelist.collect do |filter| ::Katello::Erratum.joins("inner join katello_repository_errata on katello_errata.id = katello_repository_errata.erratum_id"). where(filter.generate_clauses(nil)).where("katello_repository_errata.repository_id in (?)", source_repository_ids) end all_included_errata.flatten! end all_excluded_errata = errata_filters.blacklist.collect do |filter| ::Katello::Erratum.joins("inner join katello_repository_errata on katello_errata.id = katello_repository_errata.erratum_id"). where(filter.generate_clauses(nil)).where("katello_repository_errata.repository_id in (?)", source_repository_ids) end all_excluded_errata.flatten! return all_included_errata, all_excluded_errata end # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity def copy_content_for_source(source_repository, options = {}) package_filters = [ContentViewPackageGroupFilter, ContentViewPackageFilter].collect do |filter_class| filter_class.where(:id => options[:filter_ids]) end package_filters.flatten!.compact! errata_filters = ContentViewErratumFilter.where(:id => options[:filter_ids]) whitelist_ids = [] blacklist_ids = [] # Only filter out RPMs & modules via an erratum filter if they're not included in other allowed errata. # Begin this by calculating all the errata that should be copied. # If there are exclude filters, send the errata to the relevant methods to ensure their contents aren't excluded. additional_included_errata = [] all_excluded_errata = [] unless errata_filters.blacklist.empty? all_included_errata, all_excluded_errata = filtered_errata(errata_filters, [source_repository.id]) additional_included_errata = all_included_errata - all_excluded_errata end (errata_filters + package_filters).each do |filter| if filter.inclusion whitelist_ids += filter.content_unit_pulp_ids(source_repository) elsif filter.class == ContentViewErratumFilter blacklist_ids += filter.content_unit_pulp_ids(source_repository, additional_included_errata) else blacklist_ids += filter.content_unit_pulp_ids(source_repository) end end whitelist_ids = source_repository.rpms.where(:modular => false).pluck(:pulp_id).sort if (whitelist_ids.empty? && (errata_filters + package_filters).select { |filter| filter.inclusion }.empty?) modular_filters = ContentViewModuleStreamFilter.where(:id => options[:filter_ids]) inclusion_modular_filters = modular_filters.select { |filter| filter.inclusion } exclusion_modular_filters = modular_filters - inclusion_modular_filters if inclusion_modular_filters.empty? && errata_filters.whitelist.empty? whitelist_ids += source_repository.rpms.where(:modular => true).pluck(:pulp_id).sort whitelist_ids += source_repository.module_streams.pluck(:pulp_id).sort end whitelist_ids += modular_packages(source_repository, inclusion_modular_filters) unless inclusion_modular_filters.empty? blacklist_ids += modular_packages(source_repository, exclusion_modular_filters) unless exclusion_modular_filters.empty? content_unit_hrefs = whitelist_ids - blacklist_ids content_unit_hrefs += source_repository.srpms.pluck(:pulp_id) if content_unit_hrefs.any? content_unit_hrefs += additional_content_hrefs(source_repository, content_unit_hrefs, all_excluded_errata) end copy_units(source_repository, content_unit_hrefs.uniq, options[:remove_all]) end # rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity def modular_packages(source_repository, filters) list_ids = [] filters.each do |filter| list_ids += filter.content_unit_pulp_ids(source_repository, true) end list_ids end def additional_content_hrefs(source_repository, content_unit_hrefs, all_excluded_errata) repo_service = source_repository.backend_service(SmartProxy.pulp_primary) options = { :repository_version => source_repository.version_href } errata_to_include = filter_errata_by_pulp_href(source_repository.errata, content_unit_hrefs, source_repository.rpms.pluck(:filename) + source_repository.srpms.pluck(:filename)) - all_excluded_errata content_unit_hrefs += errata_to_include.collect do |erratum| erratum.repository_errata.where(repository_id: source_repository.id).pluck(:erratum_pulp3_href) end content_unit_hrefs.flatten! package_groups_to_include = filter_package_groups_by_pulp_href(source_repository.package_groups, content_unit_hrefs) content_unit_hrefs += package_groups_to_include.pluck(:pulp_id) metadata_file_hrefs_to_include = filter_metadatafiles_by_pulp_hrefs( repo_service.metadatafiles(options)&.results, content_unit_hrefs) content_unit_hrefs += metadata_file_hrefs_to_include distribution_tree_hrefs_to_include = filter_distribution_trees_by_pulp_hrefs( repo_service.distributiontrees(options)&.results, content_unit_hrefs) content_unit_hrefs + distribution_tree_hrefs_to_include end end end end end