diff --git a/pyproject.toml b/pyproject.toml index 57a29c6..cd2b373 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ "python-dateutil>=2.8.1,<3", "botocore>=1.23.50,<2", "lxml>=4.6.5,<5", + "zstd>=1.5.5.1", ] [project.optional-dependencies] diff --git a/rpm_s3_mirror.spec b/rpm_s3_mirror.spec index f934745..3064a01 100644 --- a/rpm_s3_mirror.spec +++ b/rpm_s3_mirror.spec @@ -13,6 +13,7 @@ Requires: python3-requests Requires: python3-dateutil Requires: python3-botocore Requires: python3-lxml +Requires: python3-zstd Requires: systemd Requires: zchunk diff --git a/rpm_s3_mirror/repository.py b/rpm_s3_mirror/repository.py index d81cbb9..43c175b 100644 --- a/rpm_s3_mirror/repository.py +++ b/rpm_s3_mirror/repository.py @@ -1,6 +1,7 @@ # Copyright (c) 2020 Aiven, Helsinki, Finland. https://aiven.io/ import dataclasses import lzma +import zstd import re import subprocess from abc import abstractmethod @@ -404,8 +405,15 @@ def _extract_package_list(self, primary: RepodataSection) -> PackageList: with self._req(self.session.get, path=primary.location, stream=True) as request: with TemporaryDirectory(prefix="/var/tmp/") as temp_dir: local_path = download_repodata_section(primary, request, temp_dir) - with gzip.open(local_path) as f: - return PackageList(base_url=self.base_url, packages_xml=f.read()) + + try: + with open(local_path, "rb") as f: + packages_xml = zstd.decompress(f.read()) + except zstd.Error: + with gzip.open(local_path) as f: + packages_xml = f.read() + + return PackageList(base_url=self.base_url, packages_xml=packages_xml) def parse_repomd(self, xml: Element) -> Dict[str, RepodataSection]: sections = {}