podman: retry pulling the image

Move podman_check_native_image_architecture() to the later stage, so we
don't retry upon architecture mismatch.

Relates: containers/podman#19770
Closes: rpm-software-management#1200

mock/docs/site-defaults.cfg

@@ -165,6 +165,12 @@
 # package manager (e.g. using dnf --installroot).
 #config_opts['bootstrap_image_fallback'] = True
 
+# When 'use_bootstrap_image' is True, bootstrap image must be downloaded and it
+# may fail.  Mock's logic is to retry downloads, using this option you can
+# configure how long should Mock keep trying (using exponential algorithm with
+# full jitter, see python-backoff docs for more info).
+#config_opts['bootstrap_image_keep_getting'] = 120  # seconds
+
 # anything you specify with 'bootstrap_*' will be copied to bootstrap config
 # e.g. config_opts['bootstrap_system_yum_command'] = '/usr/bin/yum-deprecated' will become
 # config_opts['system_yum_command'] = '/usr/bin/yum-deprecated' for bootstrap config

mock/mock.spec

@@ -60,6 +60,7 @@ Requires: python%{python3_pkgversion}-rpm
 Requires: python%{python3_pkgversion}-pyroute2
 Requires: python%{python3_pkgversion}-templated-dictionary
 Requires: python%{python3_pkgversion}-backoff
+BuildRequires: python%{python3_pkgversion}-backoff
 BuildRequires: python%{python3_pkgversion}-devel
 %if %{with lint}
 BuildRequires: python%{python3_pkgversion}-pylint

mock/py/mockbuild/buildroot.py

@@ -234,7 +234,7 @@ def _fallback(message):
             return
 
         with _fallback("Can't initialize from bootstrap image"):
-            podman.pull_image()
+            podman.retry_image_pull(self.config["image_keep_getting"])
             podman.cp(self.make_chroot_path(), self.config["tar_binary"])
             file_util.unlink_if_exists(os.path.join(self.make_chroot_path(),
                                                     "etc/rpm/macros.image-language-conf"))

mock/py/mockbuild/config.py

@@ -86,6 +86,7 @@ def setup_default_config_opts():
     config_opts['bootstrap_image'] = 'fedora:latest'
     config_opts['bootstrap_image_ready'] = False
     config_opts['bootstrap_image_fallback'] = True
+    config_opts['bootstrap_image_keep_getting'] = 120
 
     config_opts['internal_dev_setup'] = True
 

mock/py/mockbuild/podman.py

@@ -6,6 +6,7 @@
 import subprocess
 from contextlib import contextmanager
 
+import backoff
 from mockbuild.trace_decorator import getLog, traceLog
 from mockbuild import util
 from mockbuild.exception import BootstrapError
@@ -59,18 +60,23 @@ def __init__(self, buildroot, image):
 
     @traceLog()
     def pull_image(self):
-        """ pull the latest image """
+        """ pull the latest image, return True if successful """
         logger = getLog()
         logger.info("Pulling image: %s", self.image)
         cmd = [self.podman_binary, "pull", self.image]
         out, exit_status = util.do_with_status(cmd, env=self.buildroot.env,
                                                raiseExc=False, returnOutput=1)
         if exit_status:
             logger.error(out)
+        return not exit_status
 
-        if not podman_check_native_image_architecture(self.image, logger):
-            raise BootstrapError("Pulled image has invalid architecture")
-
+    def retry_image_pull(self, max_time):
+        """ Try pulling the image multiple times """
+        @backoff.on_predicate(backoff.expo, lambda x: not x,
+                              max_time=max_time, jitter=backoff.full_jitter)
+        def _keep_trying():
+            return self.pull_image()
+        _keep_trying()
 
     @contextmanager
     def mounted_image(self):
@@ -100,7 +106,12 @@ def mounted_image(self):
     @traceLog()
     def cp(self, destination, tar_cmd):
         """ copy content of container to destination directory """
-        getLog().info("Copy content of container %s to %s", self.image, destination)
+        logger = getLog()
+        logger.info("Copy content of container %s to %s", self.image, destination)
+
+        if not podman_check_native_image_architecture(self.image, logger):
+            raise BootstrapError("Pulled image has invalid architecture")
+
         with self.mounted_image() as mount_path:
             # pipe-out the temporary mountpoint with the help of tar utility
             cmd_podman = [tar_cmd, "-C", mount_path, "-c", "."]

tox.ini

@@ -7,6 +7,7 @@ deps =
     -rmock/requirements.txt
     pytest
     pytest-cov
+    backoff
 setenv =
     PYTHONPATH = ./mock/py
 commands = python -m pytest -v {posargs} --cov-report term-missing --cov mock/py mock/tests