diff --git a/CHANGES/795.feature b/CHANGES/795.feature new file mode 100644 index 000000000..4c4042abf --- /dev/null +++ b/CHANGES/795.feature @@ -0,0 +1,2 @@ +Mitigating the Hash Sum Mismatch error in debian repos by generating additional metadata files named by checksum. +Wiki of the feature: https://wiki.ubuntu.com/AptByHash diff --git a/pulp_deb/app/settings.py b/pulp_deb/app/settings.py index 65260a130..c25f37825 100644 --- a/pulp_deb/app/settings.py +++ b/pulp_deb/app/settings.py @@ -7,3 +7,5 @@ FORBIDDEN_CHECKSUM_WARNINGS = True FORCE_IGNORE_MISSING_PACKAGE_INDICES = False + +APT_BY_HASH = True diff --git a/pulp_deb/app/tasks/publishing.py b/pulp_deb/app/tasks/publishing.py index 25d92d2de..95e65b030 100644 --- a/pulp_deb/app/tasks/publishing.py +++ b/pulp_deb/app/tasks/publishing.py @@ -2,6 +2,7 @@ import os import shutil from contextlib import suppress +from pathlib import Path from datetime import datetime, timezone from debian import deb822 @@ -34,11 +35,9 @@ from pulp_deb.app.serializers import Package822Serializer -from pulp_deb.app.constants import ( - NO_MD5_WARNING_MESSAGE, - CHECKSUM_TYPE_MAP, -) +from pulp_deb.app.constants import NO_MD5_WARNING_MESSAGE, CHECKSUM_TYPE_MAP +from pulp_deb.app.settings import APT_BY_HASH import logging from gettext import gettext as _ @@ -296,10 +295,28 @@ def finish(self): publication=self.parent.publication, file=File(open(package_index_path, "rb")) ) package_index.save() + gz_package_index = PublishedMetadata.create_from_file( publication=self.parent.publication, file=File(open(gz_package_index_path, "rb")) ) gz_package_index.save() + + # Generating metadata files using checksum + if APT_BY_HASH: + for path, index in ( + (package_index_path, package_index), + (gz_package_index_path, gz_package_index), + ): + for allowed_checksum in settings.ALLOWED_CONTENT_CHECKSUMS: + if allowed_checksum in CHECKSUM_TYPE_MAP: + hashed_index_path = _fetch_file_checksum(path, index, allowed_checksum) + hashed_index = PublishedMetadata.create_from_file( + publication=self.parent.publication, + file=File(open(path, "rb")), + relative_path=hashed_index_path, + ) + hashed_index.save() + self.parent.add_metadata(package_index) self.parent.add_metadata(gz_package_index) @@ -339,6 +356,7 @@ def __init__( self.release["Components"] = "" # Will be set later if release.description != NULL_VALUE: self.release["Description"] = release.description + self.release["Acquire-By-Hash"] = "yes" if APT_BY_HASH else "no" for checksum_type, deb_field in CHECKSUM_TYPE_MAP.items(): if checksum_type in settings.ALLOWED_CONTENT_CHECKSUMS: @@ -413,3 +431,10 @@ def _zip_file(file_path): with GzipFile(gz_file_path, "wb") as f_out: shutil.copyfileobj(f_in, f_out) return gz_file_path + + +def _fetch_file_checksum(file_path, index, allowed_checksum): + h = getattr(index.contentartifact_set.first().artifact, allowed_checksum) + checksum_type = CHECKSUM_TYPE_MAP[allowed_checksum] + hashed_path = Path(file_path).parents[0] / "by-hash" / checksum_type / h + return hashed_path