[PR] Fixes #25978 - debian errata support

Co-Authored-by: Markus Bucher <[email protected]>
Co-Authored-by: Bernhard Suttner <[email protected]>
Co-Authored-by: Manisha Singhal <[email protected]>

from Katello#7961

Author: Matthias Dellweg

app/controllers/katello/api/v2/repositories_controller.rb

@@ -62,6 +62,7 @@ def custom_index_relation(collection)
       param :deb_releases, String, :desc => N_("whitespace-separated list of releases to be synced from deb-archive")
       param :deb_components, String, :desc => N_("whitespace-separated list of repo components to be synced from deb-archive")
       param :deb_architectures, String, :desc => N_("whitespace-separated list of architectures to be synced from deb-archive")
+      param :deb_errata_url, String, :desc => N_("URL to a deb errata service (use only on the security repositories)")
       param :ignorable_content, Array, :desc => N_("List of content units to ignore while syncing a yum repository. Must be subset of %s") % RootRepository::IGNORABLE_CONTENT_UNIT_TYPES.join(",")
       param :ansible_collection_requirements, String, :desc => N_("Contents of requirement yaml file to sync from URL")
       param :ansible_collection_auth_url, String, :desc => N_("The URL to receive a session token from, e.g. used with Automation Hub.")
@@ -586,7 +587,7 @@ def repository_params
       keys = [:download_policy, :mirroring_policy, :sync_policy, :arch, :verify_ssl_on_sync, :upstream_password,
               :upstream_username, :download_concurrency, :upstream_authentication_token, :metadata_expire,
               {:os_versions => []}, :deb_releases, :deb_components, :deb_architectures, :description,
-              :http_proxy_policy, :http_proxy_id, :retain_package_versions_count, {:ignorable_content => []}
+              :http_proxy_policy, :http_proxy_id, :retain_package_versions_count, {:ignorable_content => []}, :deb_errata_url
              ]
       keys += [{:include_tags => []}, {:exclude_tags => []}, :docker_upstream_name] if params[:action] == 'create' || @repository&.docker?
       keys += [:ansible_collection_requirements, :ansible_collection_auth_url, :ansible_collection_auth_token] if params[:action] == 'create' || @repository&.ansible_collection?
@@ -651,6 +652,7 @@ def construct_repo_from_params(repo_params) # rubocop:disable Metrics/AbcSize
         root.deb_releases = repo_params[:deb_releases] if repo_params[:deb_releases]
         root.deb_components = repo_params[:deb_components] if repo_params[:deb_components]
         root.deb_architectures = repo_params[:deb_architectures] if repo_params[:deb_architectures]
+        root.deb_errata_url = repo_params[:deb_errata_url] if repo_params[:deb_errata_url]
       end
 
       if root.ansible_collection?

app/lib/actions/katello/content_view/incremental_updates.rb

@@ -104,6 +104,7 @@ def total_counts(input)
                 total_count[:errata_count] = added_units[:erratum].try(:count)
                 total_count[:modulemd_count] = added_units[:modulemd].try(:count)
                 total_count[:rpm_count] = added_units[:rpm].try(:count)
+                total_count[:deb_count] = added_units[:deb].try(:count)
               end
             end
           end
@@ -147,6 +148,10 @@ def content_output_collection(total_count)
             rpm = _(" %{package_count} Package(s)" % {:package_count => total_count[:rpm_count]})
             content << rpm
           end
+          if total_count[:deb_count] && total_count[:deb_count] > 0
+            deb = _(" %{deb_package_count} Package(s)" % {:deb_package_count => total_count[:deb_count]})
+            content << deb
+          end
           content
         end
 

app/lib/actions/katello/content_view/presenters/incremental_updates_presenter.rb

@@ -6,7 +6,8 @@ class IncrementalUpdatesPresenter < Helpers::Presenter::Base
           HUMANIZED_TYPES = {
             ::Katello::Erratum::CONTENT_TYPE => "Errata",
             ::Katello::ModuleStream::CONTENT_TYPE => "Module Streams",
-            ::Katello::Rpm::CONTENT_TYPE => "Packages"
+            ::Katello::Rpm::CONTENT_TYPE => "RPM Packages",
+            ::Katello::Deb::CONTENT_TYPE => "Deb Packages"
           }.freeze
 
           def humanized_output
@@ -25,7 +26,7 @@ def humanized_content
               if cvv
                 humanized_lines << "Content View: #{cvv.content_view.name} version #{cvv.version}"
                 humanized_lines << _("Added Content:")
-                [::Katello::Erratum, ::Katello::ModuleStream, ::Katello::Rpm].each do |content_type|
+                [::Katello::Erratum, ::Katello::ModuleStream, ::Katello::Rpm, ::Katello::Deb].each do |content_type|
                   unless output[:added_units][content_type::CONTENT_TYPE].blank?
                     humanized_lines << "  #{HUMANIZED_TYPES[content_type::CONTENT_TYPE]}:"
                     humanized_lines += output[:added_units][content_type::CONTENT_TYPE].sort.map { |unit| "        #{unit}" }

app/lib/actions/katello/content_view_version/incremental_update.rb

@@ -1,7 +1,7 @@
 module Actions
   module Katello
     module ContentViewVersion
-      class IncrementalUpdate < Actions::EntryAction
+      class IncrementalUpdate < Actions::EntryAction # rubocop:disable Metrics/ClassLength
         include ::Katello::ContentViewHelper
         attr_accessor :new_content_view_version, :new_content_view_version_id
 
@@ -77,10 +77,21 @@ def plan(old_version, environments, options = {})
                 [:pulp3_deb_multicopy, :pulp3_yum_multicopy].each do |mapping|
                   if separated_repo_map[mapping].keys.flatten.present?
                     extended_repo_mapping = pulp3_repo_mapping(separated_repo_map[mapping], old_version)
-                    unit_map = pulp3_content_mapping(content)
+                    local_content = {}
+                    if mapping == :pulp3_deb_multicopy
+                      local_content[:errata_ids], local_content[:deb_ids] = resolve_deb_errata(content, extended_repo_mapping)
+                    end
+
+                    # makes sure that all keys in local_content are symbols!
+                    # content is of type ActionController::Parameters and uses string and symbol keys synonymically.
+                    # Hash (local_content) does not and we only access with symbol-keys, so make sure we only use symbols.
+                    local_content.keys.union(content.keys.map(&:to_sym)).each do |content_type|
+                      local_content[content_type] = local_content.fetch(content_type, []) + content.fetch(content_type, [])
+                    end
+                    unit_map = pulp3_content_mapping(local_content)
 
-                    unless extended_repo_mapping.empty? || unit_map.values.flatten.empty?
-                      sequence do
+                    sequence do
+                      unless extended_repo_mapping.empty? || unit_map.values.flatten.empty?
                         # Pre-copy content if dest_repo is a soft copy of its library instance.
                         # Don't use extended_repo_mapping because the source repositories are library instances.
                         # We want the old CV snapshot repositories here so as to not pull in excess new content.
@@ -101,6 +112,19 @@ def plan(old_version, environments, options = {})
                           end
                         end
                       end
+
+                      separated_repo_map[mapping].each do |_source_repos, dest_repo|
+                        next unless dest_repo.deb?
+
+                        # find errata belonging to this repo
+                        errata = ::Katello::Erratum.with_identifiers(local_content[:errata_ids]).joins(:root_repositories).where(::Katello::RootRepository.table_name => {id: dest_repo.root}).distinct
+                        if errata.present?
+                          # attach deb errata to the dest_repo
+                          copy_action_outputs << plan_action(Actions::Katello::Repository::CopyDebErratum,
+                                                             target_repo_id: dest_repo.id,
+                                                             erratum_ids: errata.pluck(:errata_id)).output
+                        end
+                      end
                     end
                   end
                 end
@@ -217,6 +241,36 @@ def components_repo_instances(old_version_repo, new_component_versions)
           end
         end
 
+        def resolve_deb_errata(content, extended_repo_mapping)
+          needed_errata = []
+          needed_debs = []
+          content[:errata_ids].each do |erratum_id|
+            extended_repo_mapping.each do |source_repos, _dest_repo|
+              source_repos.each do |source_repo|
+                re = ::Katello::RepositoryErratum.joins(:erratum).find_by(::Katello::Erratum.table_name => { errata_id: erratum_id }, repository_id: source_repo)
+
+                next if re.nil? # Erratum not in Repository
+
+                # find packages
+                # FIXME: if multiple packages with the same name exist, we have to make sure we install the newest version
+                #       but (for now) at least a version bigger or equal the one from the erratum!
+                pkgs = ::Katello::Deb.joins(repositories: { repository_errata: { erratum: :deb_packages }})
+                  .where("#{::Katello::Deb.table_name}.name = #{::Katello::ErratumDebPackage.table_name}.name AND deb_version_cmp(#{::Katello::Deb.table_name}.version,#{::Katello::ErratumDebPackage.table_name}.version) >=0")
+                  .where(::Katello::RepositoryErratum.table_name => { id: re })
+                  .distinct.pluck(:id)
+                errata = ::Katello::Erratum.joins({ repositories: :debs }, :deb_packages)
+                  .where(::Katello::RepositoryErratum.table_name => { repository_id: source_repo })
+                  .where(::Katello::Deb.table_name => { id: pkgs })
+                  .where("#{::Katello::Deb.table_name}.name = #{::Katello::ErratumDebPackage.table_name}.name AND deb_version_cmp(#{::Katello::Deb.table_name}.version,#{::Katello::ErratumDebPackage.table_name}.version) >=0")
+                  .distinct.pluck(:errata_id)
+                needed_errata.concat errata
+                needed_debs.concat pkgs
+              end
+            end
+          end
+          return needed_errata, needed_debs
+        end
+
         def run
           content = { ::Katello::Erratum::CONTENT_TYPE => [],
                       ::Katello::Rpm::CONTENT_TYPE => [],
@@ -234,6 +288,7 @@ def run
               new_errata = new_repo.errata - (matched_old_repo&.errata || [])
               new_module_streams = new_repo.module_streams - (matched_old_repo&.module_streams || [])
               new_rpms = new_repo.rpms - (matched_old_repo&.rpms || [])
+              new_debs = new_repo.debs - (matched_old_repo&.debs || [])
 
               new_errata.each do |erratum|
                 content[::Katello::Erratum::CONTENT_TYPE] << erratum.errata_id
@@ -245,6 +300,9 @@ def run
               new_rpms.each do |rpm|
                 content[::Katello::Rpm::CONTENT_TYPE] << rpm.nvra
               end
+              new_debs.each do |deb|
+                content[::Katello::Deb::CONTENT_TYPE] << deb.nva
+              end
             end
           end
           output[:added_units] = content
@@ -286,10 +344,11 @@ def calculate_components(old_version, new_components)
 
         def generate_description(version, content)
           humanized_lines = []
-          [::Katello::Erratum, ::Katello::Rpm].each do |content_type|
+          [::Katello::Erratum, ::Katello::Rpm, ::Katello::Deb].each do |content_type|
             unless content[content_type::CONTENT_TYPE].blank?
               humanized_lines << "#{HUMANIZED_TYPES[content_type::CONTENT_TYPE]}:"
-              humanized_lines += content[content_type::CONTENT_TYPE].sort.map { |unit| "    #{unit}" }
+              #FIXME: solves duplicate Deb-Errata displayed, here (might need deeper inspection)
+              humanized_lines += content[content_type::CONTENT_TYPE].uniq.sort.map { |unit| "    #{unit}" }
             end
             humanized_lines << ''
           end

app/lib/actions/katello/repository/clone_contents.rb

@@ -18,6 +18,13 @@ def plan(source_repositories, new_repository, options)
                           SmartProxy.pulp_primary,
                           source_repositories,
                           filters: filters, rpm_filenames: rpm_filenames, solve_dependencies: solve_dependencies)
+
+              source_repositories.select(&:deb?).each do |repository|
+                plan_action(Actions::Katello::Repository::CopyDebErratum,
+                            source_repo_id: repository.id,
+                            target_repo_id: new_repository.id,
+                            clean_target_errata: true)
+              end
             end
 
             matching_content = check_matching_content(new_repository, source_repositories)

app/lib/actions/katello/repository/copy_deb_erratum.rb

@@ -0,0 +1,43 @@
+module Actions
+  module Katello
+    module Repository
+      class CopyDebErratum < Actions::Base
+        input_format do
+          param :source_repo_id
+          param :target_repo_id
+          param :erratum_ids
+          param :clean_target_errata
+        end
+
+        def run
+          target_repo = ::Katello::Repository.find(input[:target_repo_id])
+
+          # drop all existing errata from target_repo (e.g. promoting LCENV back to an earlier version)
+          target_repo.repository_errata.destroy_all if input[:clean_target_errata] == true
+
+          erratum_ids_to_copy = []
+          if input[:source_repo_id].present?
+            erratum_ids_to_copy = ::Katello::Repository.find(input[:source_repo_id])&.erratum_ids
+          elsif input[:erratum_ids].present?
+            erratum_ids_to_copy = ::Katello::Erratum.where(errata_id: input[:erratum_ids]).pluck(:id)
+          end
+          erratum_ids_to_copy -= target_repo.erratum_ids
+          target_repo.erratum_ids |= erratum_ids_to_copy
+          target_repo.save
+
+          # fake output to make foreman task presenter happy
+          if input[:erratum_ids].present?
+            units = []
+            ::Katello::Erratum.find(erratum_ids_to_copy).each do |erratum|
+              units << { 'type_id' => 'erratum', 'unit_key' => { 'id' => erratum.pulp_id } }
+              erratum.deb_packages.map do |pkg|
+                units << { 'type_id' => 'deb', 'unit_key' => { 'name' => pkg.name, 'version' => pkg.version } }
+              end
+            end
+            output[:pulp_tasks] = [{ :result => { :units_successful => units } }]
+          end
+        end
+      end
+    end
+  end
+end

app/lib/actions/katello/repository/multi_clone_contents.rb

@@ -21,6 +21,12 @@ def plan(extended_repo_mapping, options)
               extended_repo_mapping.each do |source_repos, dest_repo_map|
                 dest_repo_map[:matching_content] = check_matching_content(dest_repo_map[:dest_repo], source_repos)
 
+                if source_repos.first.deb?
+                  plan_action(Actions::Katello::Repository::CopyDebErratum,
+                              source_repo_id: source_repos.first.id,
+                              target_repo_id: dest_repo_map[:dest_repo].id)
+                end
+
                 if generate_metadata
                   metadata_generate(source_repos, dest_repo_map[:dest_repo], dest_repo_map[:filters], dest_repo_map[:matching_content])
                 end

app/lib/actions/katello/repository/sync.rb

@@ -51,6 +51,7 @@ def plan(repo, options = {})
               plan_action(Katello::Foreman::ContentUpdate, repo.environment, repo.content_view, repo)
               plan_action(Katello::Repository::FetchPxeFiles, :id => repo.id)
               concurrence do
+                plan_action(Katello::Repository::SyncDebErrata, repo, skip_metadata_check) if repo.deb? && repo.root.deb_errata_url.present?
                 plan_action(Katello::Repository::ErrataMail, repo)
                 plan_action(Actions::Katello::Applicability::Repository::Regenerate, :repo_ids => [repo.id]) if generate_applicability
               end