diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/almalinux-8/packages.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/almalinux-8/packages.yml index 36c0271804..9027c32189 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/almalinux-8/packages.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/almalinux-8/packages.yml @@ -7,4 +7,5 @@ prereq-packages: packages: from_repo: [] + multiple_versioned: [] from_url: {} diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/packages.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/packages.yml index 0b0537d31e..a3b1735b24 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/packages.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/packages.yml @@ -120,6 +120,7 @@ packages: - 'xorg-x11-font-utils' # for grafana - 'xorg-x11-server-utils' # for grafana + multiple_versioned: # K8s v1.18.6 (Epiphany >= v0.7.1) - 'kubeadm-1.18.6' - 'kubectl-1.18.6' diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/rhel-8/packages.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/rhel-8/packages.yml index 3e9fca408b..2e6ed4f2ad 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/rhel-8/packages.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/rhel-8/packages.yml @@ -7,4 +7,5 @@ prereq-packages: packages: from_repo: [] + multiple_versioned: [] from_url: {} diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_repoquery.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_repoquery.py index 5aed53c1af..2d9991659f 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_repoquery.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_repoquery.py @@ -17,7 +17,8 @@ def __query(self, packages: List[str], archlist: List[str], requires: bool, resolve: bool, - output_handler: Callable) -> List[str]: + output_handler: Callable, + only_newest: bool = True) -> List[str]: """ Run generic query using `dnf repoquery` command. @@ -27,6 +28,7 @@ def __query(self, packages: List[str], :param requires: get capabilities that the packages depend on :param resolve: resolve capabilities to originating package(s) :param output_handler: different queries produce different outputs, use specific output handler + :param only_newest: if there are more than one candidate packages, download only the newest one :raises: :class:`CriticalError`: can be raised on exceeding retries or when error occurred :class:`PackageNotfound`: when query did not return any package info @@ -37,7 +39,8 @@ def __query(self, packages: List[str], args.append('repoquery') args.append(f'--archlist={",".join(archlist)}') args.append('--disableplugin=subscription-manager') # to speed up querying - args.append('--latest-limit=1') + if only_newest: + args.append('--latest-limit=1') args.append(f'--queryformat={queryformat}') args.append('--quiet') @@ -62,13 +65,14 @@ def __query(self, packages: List[str], return packages - def query(self, packages: List[str], queryformat: str, archlist: List[str]) -> List[str]: + def query(self, packages: List[str], queryformat: str, archlist: List[str], only_newest: bool = True) -> List[str]: """ Generic query to dnf database. :param packages: data will be returned for those `packages` :param queryformat: specify custom query output format :param archlist: limit results to these architectures + :param only_newest: if there are more than one candidate packages, download only the newest one :raises: :class:`CriticalError`: can be raised on exceeding retries or when error occurred :class:`PackageNotfound`: when query did not return any package info @@ -82,15 +86,16 @@ def output_handler(output: str): elif 'error' in output: raise CriticalError(f'repoquery failed for packages `{packages}`, reason: `{output}`') - return self.__query(packages, queryformat, archlist, False, False, output_handler) + return self.__query(packages, queryformat, archlist, False, False, output_handler, only_newest) - def get_dependencies(self, packages: List[str], queryformat: str, archlist: List[str]) -> List[str]: + def get_dependencies(self, packages: List[str], queryformat: str, archlist: List[str], only_newest: bool = True) -> List[str]: """ Get all dependencies for `packages`. :param packages: data will be returned for those `packages` :param queryformat: specify custom query output format :param archlist: limit results to these architectures + :param only_newest: if there are more than one candidate packages, download only the newest one :raises: :class:`CriticalError`: can be raised on exceeding retries or when error occurred :class:`ValueError`: when `packages` list is empty @@ -105,4 +110,4 @@ def output_handler(output: str): if 'error' in output: raise CriticalError(f'dnf repoquery failed for packages `{packages}`, reason: `{output}`') - return self.__query(packages, queryformat, archlist, True, True, output_handler) + return self.__query(packages, queryformat, archlist, True, True, output_handler, only_newest) diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/mode/red_hat_family_mode.py b/ansible/playbooks/roles/repository/files/download-requirements/src/mode/red_hat_family_mode.py index 95dda825bf..046a174ae0 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/mode/red_hat_family_mode.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/mode/red_hat_family_mode.py @@ -2,7 +2,7 @@ import logging import shutil from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Set from src.command.command import Command from src.config.config import Config @@ -16,6 +16,7 @@ class RedHatFamilyMode(BaseMode): def __init__(self, config: Config): super().__init__(config) + self.__all_queried_packages: Set[str] = set() self.__archs: List[str] = [config.os_arch.value, 'noarch'] self.__base_packages: List[str] = ['curl', 'python3-dnf-plugins-core', 'wget'] self.__installed_packages: List[str] = [] @@ -136,13 +137,14 @@ def _parse_packages(self) -> Dict[str, Any]: } reqs['packages']['from_repo'] += family_doc['packages']['from_repo'] + reqs['packages']['multiple_versioned'] += family_doc['packages']['multiple_versioned'] # distro level has precedence reqs['packages']['from_url'] = {**family_doc['packages']['from_url'], **distro_doc['packages']['from_url']} return reqs - def __download_prereq_packages(self) -> List[str]: + def __download_prereq_packages(self): # download requirements (fixed versions) prereqs_dir = self._cfg.dest_packages / 'repo-prereqs' prereqs_dir.mkdir(exist_ok=True, parents=True) @@ -159,36 +161,42 @@ def __download_prereq_packages(self) -> List[str]: archlist=self.__archs, exclude='*i686', destdir=prereqs_dir) - return collected_prereqs - def _download_packages(self): - downloaded_prereq_packages: List[str] = self.__download_prereq_packages() - - packages: List[str] = sorted(set(self._requirements['packages']['from_repo'])) + self.__all_queried_packages.update(collected_prereqs) + def __download_redhat_packages(self, packages: List[str], only_newest: bool = True): # packages queried_packages: List[str] = self._tools.repoquery.query(packages, queryformat='%{name}-%{version}-%{release}.%{arch}', - archlist=self.__archs) + archlist=self.__archs, + only_newest=only_newest) - packages_to_download: List[str] = sorted(set(queried_packages) - set(downloaded_prereq_packages)) + packages_to_download: List[str] = sorted(set(queried_packages) - self.__all_queried_packages) logging.info(f'- packages to download: {packages_to_download}') # dependencies dependencies: List[str] = self._tools.repoquery.get_dependencies(packages_to_download, queryformat='%{name}.%{arch}', - archlist=self.__archs) + archlist=self.__archs, + only_newest=only_newest) logging.info(f'- dependencies to download: {dependencies}') packages_to_download = sorted(packages_to_download + dependencies) + self.__all_queried_packages.update(packages_to_download) self._tools.dnf_download.download_packages(packages_to_download, archlist=self.__archs, exclude='*i686', destdir=self._cfg.dest_packages) + def _download_packages(self): + self.__download_prereq_packages() + self.__download_redhat_packages(sorted(set(self._requirements['packages']['from_repo']))) + self.__download_redhat_packages(sorted(set(self._requirements['packages']['multiple_versioned'])), + False) + def _download_file(self, url: str, dest: Path): self._tools.wget.download(url, output_document=dest, additional_params=False)