apt_dpkg: retry on HTTP 429 'Too Many Requests'

Running the system tests on different releases in parallel can cause
some tests of `test_packaging_apt_dpkg` fail:

```
__________________ test_install_packages_dependencies[deb822] __________________
[...]
apport/packaging_impl/apt_dpkg.py:1326: SystemExit
----------------------------- Captured stderr call -----------------------------
ERROR: Package download error, try again later: Failed to fetch http://ddebs.ubuntu.com/pool/main/p/pcre2/libpcre2-8-0-dbgsym_10.39-3build1_amd64.ddeb 429  Too Many Requests [IP: 185.125.190.18 80]
Failed to fetch http://ddebs.ubuntu.com/pool/main/libs/libselinux/libselinux1-dbgsym_3.3-1build2_amd64.ddeb 429  Too Many Requests [IP: 185.125.190.18 80]
```

Retry package download if HTTP 429 'Too Many Requests' errors can be
found in the `apt.cache.FetchFailedException` error. The download is
retries with an exponential backoff and giving up after around one hour.

Signed-off-by: Benjamin Drung <[email protected]>

apport/packaging_impl/apt_dpkg.py

@@ -1518,13 +1518,26 @@ def _get_primary_mirror_from_apt_sources(apt_dir: str) -> str:
     def _fetch_packages(
         apt_cache: apt.cache.Cache, fetcher: apt_pkg.Acquire | None = None
     ) -> None:
-        try:
-            apt_cache.fetch_archives(fetcher=fetcher)
-        except apt.cache.FetchFailedException as error:
-            apport.logging.error(
-                "Package download error, try again later: %s", str(error)
-            )
-            sys.exit(1)  # transient error
+        backoff = 1.0
+        while True:
+            try:
+                apt_cache.fetch_archives(fetcher=fetcher)
+                break
+            except apt.cache.FetchFailedException as error:
+                if backoff <= 3600 and "Too Many Requests" in str(error):
+                    apport.logging.warning(
+                        "Package download error, retrying in %i second(s): %s",
+                        backoff,
+                        str(error),
+                    )
+                else:
+                    apport.logging.error(
+                        "Package download error, try again later: %s", str(error)
+                    )
+                    sys.exit(1)  # transient error
+
+            time.sleep(backoff)
+            backoff *= 2
 
     def _get_mirror(self, arch: str) -> str:
         """Return the distribution mirror URL.

tests/unit/test_packaging_apt_dpkg.py

@@ -46,6 +46,29 @@ def test_fetch_packages_download_error(self) -> None:
 
         apt_cache.fetch_archives.assert_called_once_with(fetcher=None)
 
+    @unittest.mock.patch("time.sleep")
+    def test_fetch_packages_too_many_requests(self, sleep_mock: MagicMock) -> None:
+        """Test _fetch_packages() to retry on HTTP 429 'Too Many Requests'."""
+        too_many_requests_failure = apt.cache.FetchFailedException(
+            "Failed to fetch http://ddebs.ubuntu.com/pool/main"
+            "/p/pcre2/libpcre2-8-0-dbgsym_10.39-3build1_amd64.ddeb"
+            " 429  Too Many Requests [IP: 185.125.190.18 80]\n"
+            "Failed to fetch http://ddebs.ubuntu.com/pool/main"
+            "/libs/libselinux/libselinux1-dbgsym_3.3-1build2_amd64.ddeb"
+            " 429  Too Many Requests [IP: 185.125.190.18 80]\n"
+        )
+        apt_cache = MagicMock()
+        apt_cache.fetch_archives.side_effect = [
+            too_many_requests_failure,
+            too_many_requests_failure,
+            None,
+        ]
+
+        impl._fetch_packages(apt_cache)
+
+        apt_cache.fetch_archives.assert_called_with(fetcher=None)
+        self.assertEqual(sleep_mock.call_count, 2)
+
     @unittest.mock.patch("apt.Cache", spec=apt.Cache)
     def test_is_distro_package_no_candidate(self, apt_cache_mock: MagicMock) -> None:
         """is_distro_package() for package that has no candidate."""