podman: experiment with "fallback to normal bootstrap"

mock/docs/site-defaults.cfg

@@ -158,6 +158,19 @@
 # options invalidate the effect of this option.
 #config_opts['bootstrap_image_ready'] = False
 
+# If 'use_bootstrap_image' is True, Mock is instructed download the configured
+# container image from image registry.  This option controls the behavior when
+# the image can not be downloaded.  When set to False, Mock fails hard.  When
+# set to True, Mock falls-back to normal bootstrap chroot installation using
+# package manager (e.g. using dnf --installroot).
+#config_opts['bootstrap_image_fallback'] = True
+
+# When 'use_bootstrap_image' is True, bootstrap image must be downloaded and it
+# may fail.  Mock's logic is to retry downloads, using this option you can
+# configure how long should Mock keep trying (using exponential algorithm with
+# full jitter, see python-backoff docs for more info).
+#config_opts['bootstrap_image_keep_getting'] = 120  # seconds
+
 # anything you specify with 'bootstrap_*' will be copied to bootstrap config
 # e.g. config_opts['bootstrap_system_yum_command'] = '/usr/bin/yum-deprecated' will become
 # config_opts['system_yum_command'] = '/usr/bin/yum-deprecated' for bootstrap config

mock/mock.spec

@@ -60,6 +60,7 @@ Requires: python%{python3_pkgversion}-rpm
 Requires: python%{python3_pkgversion}-pyroute2
 Requires: python%{python3_pkgversion}-templated-dictionary
 Requires: python%{python3_pkgversion}-backoff
+BuildRequires: python%{python3_pkgversion}-backoff
 BuildRequires: python%{python3_pkgversion}-devel
 %if %{with lint}
 BuildRequires: python%{python3_pkgversion}-pylint

mock/py/mockbuild/buildroot.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # vim: noai:ts=4:sw=4:expandtab
 
+from contextlib import contextmanager
 import errno
 import fcntl
 import glob
@@ -19,7 +20,7 @@
 from . import uid
 from . import util
 from .exception import (BuildRootLocked, Error, ResultDirNotAccessible,
-                        BadCmdline)
+                        BadCmdline, BootstrapError)
 from .package_manager import package_manager
 from .trace_decorator import getLog, traceLog
 from .podman import Podman
@@ -211,6 +212,39 @@ def _init_locked(self):
         # Detect what package manager to use.
         self.set_package_manager()
 
+    @traceLog()
+    def _load_from_container_image(self):
+        if not self.uses_bootstrap_image or self.chroot_was_initialized:
+            return
+
+        class _FallbackException(Exception):
+            pass
+
+        @contextmanager
+        def _fallback(message):
+            try:
+                yield
+            except BootstrapError as exc:
+                if not self.config["image_fallback"]:
+                    raise
+                raise _FallbackException(
+                    f"{message}, falling back to bootstrap installation: {exc}"
+                ) from exc
+
+        try:
+            with _fallback("Can't work with Podman"):
+                podman = Podman(self, self.bootstrap_image)
+
+            with _fallback("Can't initialize from bootstrap image"):
+                podman.retry_image_pull(self.config["image_keep_getting"])
+                podman.cp(self.make_chroot_path(), self.config["tar_binary"])
+                file_util.unlink_if_exists(os.path.join(self.make_chroot_path(),
+                                                        "etc/rpm/macros.image-language-conf"))
+        except _FallbackException as exc:
+            getLog().warning("%s", exc)
+            self.use_bootstrap_image = False
+
+
     @traceLog()
     def _init(self, prebuild):
 
@@ -229,13 +263,7 @@ def _init(self, prebuild):
         self.plugins.call_hooks('preinit')
         # intentionally we do not call bootstrap hook here - it does not have sense
         self.chroot_was_initialized = self.chroot_is_initialized()
-        if self.uses_bootstrap_image and not self.chroot_was_initialized:
-            podman = Podman(self, self.bootstrap_image)
-            podman.pull_image()
-            podman.cp(self.make_chroot_path(), self.config["tar_binary"])
-            file_util.unlink_if_exists(os.path.join(self.make_chroot_path(),
-                                                    "etc/rpm/macros.image-language-conf"))
-
+        self._load_from_container_image()
         self._setup_dirs()
 
         # /dev is later overwritten by systemd-nspawn, but we need this for

mock/py/mockbuild/config.py

@@ -85,6 +85,8 @@ def setup_default_config_opts():
     config_opts['use_bootstrap_image'] = True
     config_opts['bootstrap_image'] = 'fedora:latest'
     config_opts['bootstrap_image_ready'] = False
+    config_opts['bootstrap_image_fallback'] = True
+    config_opts['bootstrap_image_keep_getting'] = 120
 
     config_opts['internal_dev_setup'] = True
 

mock/py/mockbuild/podman.py

@@ -1,16 +1,18 @@
 # -*- coding: utf-8 -*-
 # vim: noai:ts=4:sw=4:expandtab
 
+import os
 import logging
 import subprocess
 from contextlib import contextmanager
 
+import backoff
 from mockbuild.trace_decorator import getLog, traceLog
 from mockbuild import util
 from mockbuild.exception import BootstrapError
 
 
-def podman_check_native_image_architecture(image, logger=None):
+def podman_check_native_image_architecture(image, logger=None, podman_binary=None):
     """
     Return True if image's architecture is "native" for this host.
     Relates:
@@ -19,10 +21,11 @@ def podman_check_native_image_architecture(image, logger=None):
     """
 
     logger = logger or logging.getLogger()
+    podman = podman_binary or "/usr/bin/podman"
     logger.info("Checking that %s image matches host's architecture", image)
-    sys_check_cmd = ["podman", "version", "--format", "{{.OsArch}}"]
-    image_check_cmd = ["podman", "image", "inspect",
-                        "--format", "{{.Os}}/{{.Architecture}}", image]
+    sys_check_cmd = [podman, "version", "--format", "{{.OsArch}}"]
+    image_check_cmd = [podman, "image", "inspect",
+                       "--format", "{{.Os}}/{{.Architecture}}", image]
 
     def _podman_query(cmd):
         return subprocess.check_output(cmd, encoding="utf8").strip()
@@ -46,25 +49,34 @@ class Podman:
 
     @traceLog()
     def __init__(self, buildroot, image):
+        self.podman_binary = "/usr/bin/podman"
+        if not os.path.exists(self.podman_binary):
+            raise BootstrapError(f"'{self.podman_binary}' not installed")
+
         self.buildroot = buildroot
         self.image = image
         self.container_id = None
         getLog().info("Using bootstrap image: %s", image)
 
     @traceLog()
     def pull_image(self):
-        """ pull the latest image """
+        """ pull the latest image, return True if successful """
         logger = getLog()
         logger.info("Pulling image: %s", self.image)
-        cmd = ["podman", "pull", self.image]
+        cmd = [self.podman_binary, "pull", self.image]
         out, exit_status = util.do_with_status(cmd, env=self.buildroot.env,
                                                raiseExc=False, returnOutput=1)
         if exit_status:
             logger.error(out)
+        return not exit_status
 
-        if not podman_check_native_image_architecture(self.image, logger):
-            raise BootstrapError("Pulled image has invalid architecture")
-
+    def retry_image_pull(self, max_time):
+        """ Try pulling the image multiple times """
+        @backoff.on_predicate(backoff.expo, lambda x: not x,
+                              max_time=max_time, jitter=backoff.full_jitter)
+        def _keep_trying():
+            return self.pull_image()
+        _keep_trying()
 
     @contextmanager
     def mounted_image(self):
@@ -74,8 +86,8 @@ def mounted_image(self):
         bootstrap chroot directory.
         """
         logger = getLog()
-        cmd_mount = ["podman", "image", "mount", self.image]
-        cmd_umount = ["podman", "image", "umount", self.image]
+        cmd_mount = [self.podman_binary, "image", "mount", self.image]
+        cmd_umount = [self.podman_binary, "image", "umount", self.image]
         result = subprocess.run(cmd_mount, capture_output=True, check=False, encoding="utf8")
         if result.returncode:
             message = "Podman mount failed: " + result.stderr
@@ -94,7 +106,12 @@ def mounted_image(self):
     @traceLog()
     def cp(self, destination, tar_cmd):
         """ copy content of container to destination directory """
-        getLog().info("Copy content of container %s to %s", self.image, destination)
+        logger = getLog()
+        logger.info("Copy content of container %s to %s", self.image, destination)
+
+        if not podman_check_native_image_architecture(self.image, logger):
+            raise BootstrapError("Container image architecture check failed")
+
         with self.mounted_image() as mount_path:
             # pipe-out the temporary mountpoint with the help of tar utility
             cmd_podman = [tar_cmd, "-C", mount_path, "-c", "."]

tox.ini

@@ -7,6 +7,7 @@ deps =
     -rmock/requirements.txt
     pytest
     pytest-cov
+    backoff
 setenv =
     PYTHONPATH = ./mock/py
 commands = python -m pytest -v {posargs} --cov-report term-missing --cov mock/py mock/tests