Generating checksum named metadata files for AptByHash (squashed)

closes pulp#795

CHANGES/795.feature

@@ -0,0 +1,2 @@
+Mitigating the Hash Sum Mismatch error in debian repos by generating additional metadata files named by checksum.
+Wiki of the feature: https://wiki.ubuntu.com/AptByHash

functest_requirements.txt

@@ -1,3 +1,4 @@
 pulp-smash @ git+https://github.com/pulp/pulp-smash.git
 pytest
 python-debian>=0.1.36
+epdb

pulp_deb/app/settings.py

@@ -10,3 +10,5 @@
 
 PUBLISH_RELEASE_FILE_LABEL = False
 PUBLISH_RELEASE_FILE_VERSION = False
+APT_BY_HASH = True
+APT_BY_HASH_CHECKSUM_TYPE = "sha256"

pulp_deb/app/tasks/publishing.py

@@ -2,6 +2,8 @@
 import os
 import shutil
 from contextlib import suppress
+from pathlib import Path
+import hashlib
 
 from datetime import datetime, timezone
 from debian import deb822
@@ -33,11 +35,9 @@
 
 from pulp_deb.app.serializers import Package822Serializer
 
-from pulp_deb.app.constants import (
-    NO_MD5_WARNING_MESSAGE,
-    CHECKSUM_TYPE_MAP,
-)
+from pulp_deb.app.constants import NO_MD5_WARNING_MESSAGE, CHECKSUM_TYPE_MAP
 
+from pulp_deb.app.settings import APT_BY_HASH, APT_BY_HASH_CHECKSUM_TYPE
 
 import logging
 from gettext import gettext as _
@@ -247,6 +247,7 @@ def add_package(self, package):
                 content_artifact=package.contentartifact_set.get(),
             )
             published_artifact.save()
+
         package_serializer = Package822Serializer(package, context={"request": None})
         package_serializer.to822(self.component).dump(
             self.package_index_files[package.architecture][0]
@@ -262,10 +263,23 @@ def finish(self):
                 publication=self.parent.publication, file=File(open(package_index_path, "rb"))
             )
             package_index.save()
+
             gz_package_index = PublishedMetadata.create_from_file(
                 publication=self.parent.publication, file=File(open(gz_package_index_path, "rb"))
             )
             gz_package_index.save()
+
+            # Generating metadata files using checksum
+            if APT_BY_HASH and APT_BY_HASH_CHECKSUM_TYPE in settings.ALLOWED_CONTENT_CHECKSUMS:
+                for path in (package_index_path, gz_package_index_path):
+                    hashed_index_path = _create_checksum_file(path)
+                    hashed_index = PublishedMetadata.create_from_file(
+                        publication=self.parent.publication,
+                        file=File(open(hashed_index_path, "rb")),
+                    )
+                    hashed_index.save()
+            # Done generating
+
             self.parent.add_metadata(package_index)
             self.parent.add_metadata(gz_package_index)
 
@@ -383,3 +397,19 @@ def _zip_file(file_path):
         with GzipFile(gz_file_path, "wb") as f_out:
             shutil.copyfileobj(f_in, f_out)
     return gz_file_path
+
+
+def _checksum_file(path):
+    h = hashlib.new(APT_BY_HASH_CHECKSUM_TYPE)
+    with open(path, "rb") as f:
+        for line in f:
+            h.update(line)
+    return h.hexdigest()
+
+
+def _create_checksum_file(file_path):
+    by_hash_path = Path(file_path).parents[0] / "byhash"
+    by_hash_path.mkdir(parents=True, exist_ok=True)
+    hashed_path = by_hash_path / _checksum_file(file_path)
+    shutil.copyfile(file_path, hashed_path)
+    return hashed_path