Fix dnf repoquery checks only latest kube* packages (#3123) (#3126)

ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/almalinux-8/packages.yml

@@ -7,4 +7,5 @@ prereq-packages:
 
 packages:
   from_repo: []
+  multiple_versioned: []
   from_url: {}

ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/packages.yml

@@ -120,6 +120,7 @@ packages:
     - 'xorg-x11-font-utils'  # for grafana
     - 'xorg-x11-server-utils'  # for grafana
 
+  multiple_versioned:
     # K8s v1.18.6 (Epiphany >= v0.7.1)
     - 'kubeadm-1.18.6'
     - 'kubectl-1.18.6'

ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/rhel-8/packages.yml

@@ -7,4 +7,5 @@ prereq-packages:
 
 packages:
   from_repo: []
+  multiple_versioned: []
   from_url: {}

ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_repoquery.py

@@ -17,7 +17,8 @@ def __query(self, packages: List[str],
                 archlist: List[str],
                 requires: bool,
                 resolve: bool,
-                output_handler: Callable) -> List[str]:
+                output_handler: Callable,
+                only_newest: bool = True) -> List[str]:
         """
         Run generic query using `dnf repoquery` command.
 
@@ -27,6 +28,7 @@ def __query(self, packages: List[str],
         :param requires: get capabilities that the packages depend on
         :param resolve: resolve capabilities to originating package(s)
         :param output_handler: different queries produce different outputs, use specific output handler
+        :param only_newest: if there are more than one candidate packages, download only the newest one
         :raises:
             :class:`CriticalError`: can be raised on exceeding retries or when error occurred
             :class:`PackageNotfound`: when query did not return any package info
@@ -37,7 +39,8 @@ def __query(self, packages: List[str],
         args.append('repoquery')
         args.append(f'--archlist={",".join(archlist)}')
         args.append('--disableplugin=subscription-manager')  # to speed up querying
-        args.append('--latest-limit=1')
+        if only_newest:
+            args.append('--latest-limit=1')
         args.append(f'--queryformat={queryformat}')
         args.append('--quiet')
 
@@ -62,13 +65,14 @@ def __query(self, packages: List[str],
 
         return packages
 
-    def query(self, packages: List[str], queryformat: str, archlist: List[str]) -> List[str]:
+    def query(self, packages: List[str], queryformat: str, archlist: List[str], only_newest: bool = True) -> List[str]:
         """
         Generic query to dnf database.
 
         :param packages: data will be returned for those `packages`
         :param queryformat: specify custom query output format
         :param archlist: limit results to these architectures
+        :param only_newest: if there are more than one candidate packages, download only the newest one
         :raises:
             :class:`CriticalError`: can be raised on exceeding retries or when error occurred
             :class:`PackageNotfound`: when query did not return any package info
@@ -82,15 +86,16 @@ def output_handler(output: str):
             elif 'error' in output:
                 raise CriticalError(f'repoquery failed for packages `{packages}`, reason: `{output}`')
 
-        return self.__query(packages, queryformat, archlist, False, False, output_handler)
+        return self.__query(packages, queryformat, archlist, False, False, output_handler, only_newest)
 
-    def get_dependencies(self, packages: List[str], queryformat: str, archlist: List[str]) -> List[str]:
+    def get_dependencies(self, packages: List[str], queryformat: str, archlist: List[str], only_newest: bool = True) -> List[str]:
         """
         Get all dependencies for `packages`.
 
         :param packages: data will be returned for those `packages`
         :param queryformat: specify custom query output format
         :param archlist: limit results to these architectures
+        :param only_newest: if there are more than one candidate packages, download only the newest one
         :raises:
             :class:`CriticalError`: can be raised on exceeding retries or when error occurred
             :class:`ValueError`: when `packages` list is empty
@@ -105,4 +110,4 @@ def output_handler(output: str):
             if 'error' in output:
                 raise CriticalError(f'dnf repoquery failed for packages `{packages}`, reason: `{output}`')
 
-        return self.__query(packages, queryformat, archlist, True, True, output_handler)
+        return self.__query(packages, queryformat, archlist, True, True, output_handler, only_newest)

ansible/playbooks/roles/repository/files/download-requirements/src/mode/red_hat_family_mode.py

@@ -2,7 +2,7 @@
 import logging
 import shutil
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Set
 
 from src.command.command import Command
 from src.config.config import Config
@@ -16,6 +16,7 @@ class RedHatFamilyMode(BaseMode):
 
     def __init__(self, config: Config):
         super().__init__(config)
+        self.__all_queried_packages: Set[str] = set()
         self.__archs: List[str] = [config.os_arch.value, 'noarch']
         self.__base_packages: List[str] = ['curl', 'python3-dnf-plugins-core', 'wget']
         self.__installed_packages: List[str] = []
@@ -136,13 +137,14 @@ def _parse_packages(self) -> Dict[str, Any]:
         }
 
         reqs['packages']['from_repo'] += family_doc['packages']['from_repo']
+        reqs['packages']['multiple_versioned'] += family_doc['packages']['multiple_versioned']
 
         # distro level has precedence
         reqs['packages']['from_url'] = {**family_doc['packages']['from_url'], **distro_doc['packages']['from_url']}
 
         return reqs
 
-    def __download_prereq_packages(self) -> List[str]:
+    def __download_prereq_packages(self):
         # download requirements (fixed versions)
         prereqs_dir = self._cfg.dest_packages / 'repo-prereqs'
         prereqs_dir.mkdir(exist_ok=True, parents=True)
@@ -159,36 +161,42 @@ def __download_prereq_packages(self) -> List[str]:
                                                    archlist=self.__archs,
                                                    exclude='*i686',
                                                    destdir=prereqs_dir)
-        return collected_prereqs
 
-    def _download_packages(self):
-        downloaded_prereq_packages: List[str] = self.__download_prereq_packages()
-
-        packages: List[str] = sorted(set(self._requirements['packages']['from_repo']))
+        self.__all_queried_packages.update(collected_prereqs)
 
+    def __download_redhat_packages(self, packages: List[str], only_newest: bool = True):
         # packages
         queried_packages: List[str] = self._tools.repoquery.query(packages,
                                                                   queryformat='%{name}-%{version}-%{release}.%{arch}',
-                                                                  archlist=self.__archs)
+                                                                  archlist=self.__archs,
+                                                                  only_newest=only_newest)
 
-        packages_to_download: List[str] = sorted(set(queried_packages) - set(downloaded_prereq_packages))
+        packages_to_download: List[str] = sorted(set(queried_packages) - self.__all_queried_packages)
 
         logging.info(f'- packages to download: {packages_to_download}')
 
         # dependencies
         dependencies: List[str] = self._tools.repoquery.get_dependencies(packages_to_download,
                                                                          queryformat='%{name}.%{arch}',
-                                                                         archlist=self.__archs)
+                                                                         archlist=self.__archs,
+                                                                         only_newest=only_newest)
 
         logging.info(f'- dependencies to download: {dependencies}')
 
         packages_to_download = sorted(packages_to_download + dependencies)
+        self.__all_queried_packages.update(packages_to_download)
 
         self._tools.dnf_download.download_packages(packages_to_download,
                                                    archlist=self.__archs,
                                                    exclude='*i686',
                                                    destdir=self._cfg.dest_packages)
 
+    def _download_packages(self):
+        self.__download_prereq_packages()
+        self.__download_redhat_packages(sorted(set(self._requirements['packages']['from_repo'])))
+        self.__download_redhat_packages(sorted(set(self._requirements['packages']['multiple_versioned'])),
+                                        False)
+
     def _download_file(self, url: str, dest: Path):
         self._tools.wget.download(url, output_document=dest, additional_params=False)