From 728613a8e72fc82c313d20154045b46934f9b079 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Mon, 28 Aug 2023 22:38:32 +0200 Subject: [PATCH 1/3] podman: fallback to normal bootstrap When Podman isn't installed, or the image can not be downloaded, or the image isn't compatible with host arch - Mock newly falls-back to a normal 'dnf --installroot' when bootstrap_image_fallback=True (default). Closes: #1200 --- mock/docs/site-defaults.cfg | 7 ++++++ mock/py/mockbuild/buildroot.py | 44 +++++++++++++++++++++++++++------- mock/py/mockbuild/config.py | 1 + mock/py/mockbuild/podman.py | 20 ++++++++++------ 4 files changed, 57 insertions(+), 15 deletions(-) diff --git a/mock/docs/site-defaults.cfg b/mock/docs/site-defaults.cfg index b5051b415..f60456079 100644 --- a/mock/docs/site-defaults.cfg +++ b/mock/docs/site-defaults.cfg @@ -158,6 +158,13 @@ # options invalidate the effect of this option. #config_opts['bootstrap_image_ready'] = False +# If 'use_bootstrap_image' is True, Mock is instructed download the configured +# container image from image registry. This option controls the behavior when +# the image can not be downloaded. When set to False, Mock fails hard. When +# set to True, Mock falls-back to normal bootstrap chroot installation using +# package manager (e.g. using dnf --installroot). +#config_opts['bootstrap_image_fallback'] = True + # anything you specify with 'bootstrap_*' will be copied to bootstrap config # e.g. config_opts['bootstrap_system_yum_command'] = '/usr/bin/yum-deprecated' will become # config_opts['system_yum_command'] = '/usr/bin/yum-deprecated' for bootstrap config diff --git a/mock/py/mockbuild/buildroot.py b/mock/py/mockbuild/buildroot.py index 646a47d28..bdc013260 100644 --- a/mock/py/mockbuild/buildroot.py +++ b/mock/py/mockbuild/buildroot.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # vim: noai:ts=4:sw=4:expandtab +from contextlib import contextmanager import errno import fcntl import glob @@ -19,7 +20,7 @@ from . import uid from . import util from .exception import (BuildRootLocked, Error, ResultDirNotAccessible, - BadCmdline) + BadCmdline, BootstrapError) from .package_manager import package_manager from .trace_decorator import getLog, traceLog from .podman import Podman @@ -211,6 +212,39 @@ def _init_locked(self): # Detect what package manager to use. self.set_package_manager() + @traceLog() + def _load_from_container_image(self): + if not self.uses_bootstrap_image or self.chroot_was_initialized: + return + + class _FallbackException(Exception): + pass + + @contextmanager + def _fallback(message): + try: + yield + except BootstrapError as exc: + if not self.config["image_fallback"]: + raise + raise _FallbackException( + f"{message}, falling back to bootstrap installation: {exc}" + ) from exc + + try: + with _fallback("Can't work with Podman"): + podman = Podman(self, self.bootstrap_image) + + with _fallback("Can't initialize from bootstrap image"): + podman.pull_image() + podman.cp(self.make_chroot_path(), self.config["tar_binary"]) + file_util.unlink_if_exists(os.path.join(self.make_chroot_path(), + "etc/rpm/macros.image-language-conf")) + except _FallbackException as exc: + getLog().warning("%s", exc) + self.use_bootstrap_image = False + + @traceLog() def _init(self, prebuild): @@ -229,13 +263,7 @@ def _init(self, prebuild): self.plugins.call_hooks('preinit') # intentionally we do not call bootstrap hook here - it does not have sense self.chroot_was_initialized = self.chroot_is_initialized() - if self.uses_bootstrap_image and not self.chroot_was_initialized: - podman = Podman(self, self.bootstrap_image) - podman.pull_image() - podman.cp(self.make_chroot_path(), self.config["tar_binary"]) - file_util.unlink_if_exists(os.path.join(self.make_chroot_path(), - "etc/rpm/macros.image-language-conf")) - + self._load_from_container_image() self._setup_dirs() # /dev is later overwritten by systemd-nspawn, but we need this for diff --git a/mock/py/mockbuild/config.py b/mock/py/mockbuild/config.py index 635bd7ed6..f4648d065 100644 --- a/mock/py/mockbuild/config.py +++ b/mock/py/mockbuild/config.py @@ -85,6 +85,7 @@ def setup_default_config_opts(): config_opts['use_bootstrap_image'] = True config_opts['bootstrap_image'] = 'fedora:latest' config_opts['bootstrap_image_ready'] = False + config_opts['bootstrap_image_fallback'] = True config_opts['internal_dev_setup'] = True diff --git a/mock/py/mockbuild/podman.py b/mock/py/mockbuild/podman.py index 20c46b085..b6979f757 100644 --- a/mock/py/mockbuild/podman.py +++ b/mock/py/mockbuild/podman.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # vim: noai:ts=4:sw=4:expandtab +import os import logging import subprocess from contextlib import contextmanager @@ -10,7 +11,7 @@ from mockbuild.exception import BootstrapError -def podman_check_native_image_architecture(image, logger=None): +def podman_check_native_image_architecture(image, logger=None, podman_binary=None): """ Return True if image's architecture is "native" for this host. Relates: @@ -19,10 +20,11 @@ def podman_check_native_image_architecture(image, logger=None): """ logger = logger or logging.getLogger() + podman = podman_binary or "/usr/bin/podman" logger.info("Checking that %s image matches host's architecture", image) - sys_check_cmd = ["podman", "version", "--format", "{{.OsArch}}"] - image_check_cmd = ["podman", "image", "inspect", - "--format", "{{.Os}}/{{.Architecture}}", image] + sys_check_cmd = [podman, "version", "--format", "{{.OsArch}}"] + image_check_cmd = [podman, "image", "inspect", + "--format", "{{.Os}}/{{.Architecture}}", image] def _podman_query(cmd): return subprocess.check_output(cmd, encoding="utf8").strip() @@ -46,6 +48,10 @@ class Podman: @traceLog() def __init__(self, buildroot, image): + self.podman_binary = "/usr/bin/podman" + if not os.path.exists(self.podman_binary): + raise BootstrapError(f"'{self.podman_binary}' not installed") + self.buildroot = buildroot self.image = image self.container_id = None @@ -56,7 +62,7 @@ def pull_image(self): """ pull the latest image """ logger = getLog() logger.info("Pulling image: %s", self.image) - cmd = ["podman", "pull", self.image] + cmd = [self.podman_binary, "pull", self.image] out, exit_status = util.do_with_status(cmd, env=self.buildroot.env, raiseExc=False, returnOutput=1) if exit_status: @@ -74,8 +80,8 @@ def mounted_image(self): bootstrap chroot directory. """ logger = getLog() - cmd_mount = ["podman", "image", "mount", self.image] - cmd_umount = ["podman", "image", "umount", self.image] + cmd_mount = [self.podman_binary, "image", "mount", self.image] + cmd_umount = [self.podman_binary, "image", "umount", self.image] result = subprocess.run(cmd_mount, capture_output=True, check=False, encoding="utf8") if result.returncode: message = "Podman mount failed: " + result.stderr From b3795126943dc74f6d17296425a475f87fb907f9 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Tue, 29 Aug 2023 09:49:30 +0200 Subject: [PATCH 2/3] podman: retry pulling the image Move podman_check_native_image_architecture() to the later stage, so we don't retry upon architecture mismatch. Relates: https://github.com/containers/podman/issues/19770 Closes: #1200 --- mock/docs/site-defaults.cfg | 6 ++++++ mock/mock.spec | 1 + mock/py/mockbuild/buildroot.py | 2 +- mock/py/mockbuild/config.py | 1 + mock/py/mockbuild/podman.py | 21 ++++++++++++++++----- tox.ini | 1 + 6 files changed, 26 insertions(+), 6 deletions(-) diff --git a/mock/docs/site-defaults.cfg b/mock/docs/site-defaults.cfg index f60456079..6d7fb2866 100644 --- a/mock/docs/site-defaults.cfg +++ b/mock/docs/site-defaults.cfg @@ -165,6 +165,12 @@ # package manager (e.g. using dnf --installroot). #config_opts['bootstrap_image_fallback'] = True +# When 'use_bootstrap_image' is True, bootstrap image must be downloaded and it +# may fail. Mock's logic is to retry downloads, using this option you can +# configure how long should Mock keep trying (using exponential algorithm with +# full jitter, see python-backoff docs for more info). +#config_opts['bootstrap_image_keep_getting'] = 120 # seconds + # anything you specify with 'bootstrap_*' will be copied to bootstrap config # e.g. config_opts['bootstrap_system_yum_command'] = '/usr/bin/yum-deprecated' will become # config_opts['system_yum_command'] = '/usr/bin/yum-deprecated' for bootstrap config diff --git a/mock/mock.spec b/mock/mock.spec index 681de6ddc..4a7f125fe 100644 --- a/mock/mock.spec +++ b/mock/mock.spec @@ -60,6 +60,7 @@ Requires: python%{python3_pkgversion}-rpm Requires: python%{python3_pkgversion}-pyroute2 Requires: python%{python3_pkgversion}-templated-dictionary Requires: python%{python3_pkgversion}-backoff +BuildRequires: python%{python3_pkgversion}-backoff BuildRequires: python%{python3_pkgversion}-devel %if %{with lint} BuildRequires: python%{python3_pkgversion}-pylint diff --git a/mock/py/mockbuild/buildroot.py b/mock/py/mockbuild/buildroot.py index bdc013260..900fa0fc6 100644 --- a/mock/py/mockbuild/buildroot.py +++ b/mock/py/mockbuild/buildroot.py @@ -236,7 +236,7 @@ def _fallback(message): podman = Podman(self, self.bootstrap_image) with _fallback("Can't initialize from bootstrap image"): - podman.pull_image() + podman.retry_image_pull(self.config["image_keep_getting"]) podman.cp(self.make_chroot_path(), self.config["tar_binary"]) file_util.unlink_if_exists(os.path.join(self.make_chroot_path(), "etc/rpm/macros.image-language-conf")) diff --git a/mock/py/mockbuild/config.py b/mock/py/mockbuild/config.py index f4648d065..fd3ce62dc 100644 --- a/mock/py/mockbuild/config.py +++ b/mock/py/mockbuild/config.py @@ -86,6 +86,7 @@ def setup_default_config_opts(): config_opts['bootstrap_image'] = 'fedora:latest' config_opts['bootstrap_image_ready'] = False config_opts['bootstrap_image_fallback'] = True + config_opts['bootstrap_image_keep_getting'] = 120 config_opts['internal_dev_setup'] = True diff --git a/mock/py/mockbuild/podman.py b/mock/py/mockbuild/podman.py index b6979f757..5556ee1e1 100644 --- a/mock/py/mockbuild/podman.py +++ b/mock/py/mockbuild/podman.py @@ -6,6 +6,7 @@ import subprocess from contextlib import contextmanager +import backoff from mockbuild.trace_decorator import getLog, traceLog from mockbuild import util from mockbuild.exception import BootstrapError @@ -59,7 +60,7 @@ def __init__(self, buildroot, image): @traceLog() def pull_image(self): - """ pull the latest image """ + """ pull the latest image, return True if successful """ logger = getLog() logger.info("Pulling image: %s", self.image) cmd = [self.podman_binary, "pull", self.image] @@ -67,10 +68,15 @@ def pull_image(self): raiseExc=False, returnOutput=1) if exit_status: logger.error(out) + return not exit_status - if not podman_check_native_image_architecture(self.image, logger): - raise BootstrapError("Pulled image has invalid architecture") - + def retry_image_pull(self, max_time): + """ Try pulling the image multiple times """ + @backoff.on_predicate(backoff.expo, lambda x: not x, + max_time=max_time, jitter=backoff.full_jitter) + def _keep_trying(): + return self.pull_image() + _keep_trying() @contextmanager def mounted_image(self): @@ -100,7 +106,12 @@ def mounted_image(self): @traceLog() def cp(self, destination, tar_cmd): """ copy content of container to destination directory """ - getLog().info("Copy content of container %s to %s", self.image, destination) + logger = getLog() + logger.info("Copy content of container %s to %s", self.image, destination) + + if not podman_check_native_image_architecture(self.image, logger): + raise BootstrapError("Pulled image has invalid architecture") + with self.mounted_image() as mount_path: # pipe-out the temporary mountpoint with the help of tar utility cmd_podman = [tar_cmd, "-C", mount_path, "-c", "."] diff --git a/tox.ini b/tox.ini index e325bda00..2d458e04f 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ deps = -rmock/requirements.txt pytest pytest-cov + backoff setenv = PYTHONPATH = ./mock/py commands = python -m pytest -v {posargs} --cov-report term-missing --cov mock/py mock/tests From a056d523723babdc808d38a4c5709b077ed54c16 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Tue, 29 Aug 2023 22:45:37 +0200 Subject: [PATCH 3/3] podman: clearer architecture check error message This message may be raised when /bin/podman fails for any reason, not just when that image arch != host arch. --- mock/py/mockbuild/podman.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mock/py/mockbuild/podman.py b/mock/py/mockbuild/podman.py index 5556ee1e1..278544bf3 100644 --- a/mock/py/mockbuild/podman.py +++ b/mock/py/mockbuild/podman.py @@ -110,7 +110,7 @@ def cp(self, destination, tar_cmd): logger.info("Copy content of container %s to %s", self.image, destination) if not podman_check_native_image_architecture(self.image, logger): - raise BootstrapError("Pulled image has invalid architecture") + raise BootstrapError("Container image architecture check failed") with self.mounted_image() as mount_path: # pipe-out the temporary mountpoint with the help of tar utility