Resolving aptbyhash pr comments

closes pulp#795
adamsanaglo · Jul 18, 2023 · 5968ac0 · 5968ac0
1 parent 91281e3
commit 5968ac0
Show file tree

Hide file tree

Showing 84 changed files with 34 additions and 181 deletions.
diff --git a/CHANGES/795.feature b/CHANGES/795.feature
@@ -1 +1,2 @@
-Mitigating the Hash Sum Mismatch error by generating additional metadata files named by checksum
+Mitigating the Hash Sum Mismatch error in debian repos by generating additional metadata files named by checksum.
+Wiki of the feature: https://wiki.ubuntu.com/AptByHash
diff --git a/pulp_deb.egg-info/PKG-INFO b/pulp_deb.egg-info/PKG-INFO
diff --git a/pulp_deb.egg-info/SOURCES.txt b/pulp_deb.egg-info/SOURCES.txt
diff --git a/pulp_deb.egg-info/dependency_links.txt b/pulp_deb.egg-info/dependency_links.txt
diff --git a/pulp_deb.egg-info/entry_points.txt b/pulp_deb.egg-info/entry_points.txt
diff --git a/pulp_deb.egg-info/requires.txt b/pulp_deb.egg-info/requires.txt
diff --git a/pulp_deb.egg-info/top_level.txt b/pulp_deb.egg-info/top_level.txt
diff --git a/pulp_deb/__pycache__/__init__.cpython-38.pyc b/pulp_deb/__pycache__/__init__.cpython-38.pyc
diff --git a/pulp_deb/app/__pycache__/__init__.cpython-38.pyc b/pulp_deb/app/__pycache__/__init__.cpython-38.pyc
diff --git a/pulp_deb/app/__pycache__/constants.cpython-38.pyc b/pulp_deb/app/__pycache__/constants.cpython-38.pyc
diff --git a/pulp_deb/app/__pycache__/modelresource.cpython-38.pyc b/pulp_deb/app/__pycache__/modelresource.cpython-38.pyc
diff --git a/pulp_deb/app/__pycache__/settings.cpython-38.pyc b/pulp_deb/app/__pycache__/settings.cpython-38.pyc
diff --git a/pulp_deb/app/__pycache__/urls.cpython-38.pyc b/pulp_deb/app/__pycache__/urls.cpython-38.pyc
diff --git a/pulp_deb/app/constants.py b/pulp_deb/app/constants.py
@@ -17,4 +17,3 @@
 
 # Represents null values since nulls can't be used in unique indexes in postgres < 15
 NULL_VALUE = "__!!!NULL VALUE!!!__"
-APT_BY_HASH = True
diff --git a/pulp_deb/app/migrations/__pycache__/0001_initial.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0001_initial.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0002_auto_20190905_1000.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0002_auto_20190905_1000.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0003_rename_package_field.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0003_rename_package_field.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0004_auto_20191014_1722.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0004_auto_20191014_1722.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0005_rename_release_file.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0005_rename_release_file.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0006_debrepository.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0006_debrepository.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0007_create_metadata_models.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0007_create_metadata_models.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0008_debremote_gpgkey.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0008_debremote_gpgkey.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0009_apt_release_signing_service.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0009_apt_release_signing_service.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0010_debpublication_signing_service.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0010_debpublication_signing_service.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0011_rename_models_to_apt.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0011_rename_models_to_apt.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0012_auto_20200803_1337.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0012_auto_20200803_1337.cpython-38.pyc
diff --git a/...b/app/migrations/__pycache__/0013_aptremote_ignore_missing_package_indices.cpython-38.pyc b/...b/app/migrations/__pycache__/0013_aptremote_ignore_missing_package_indices.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0014_swap_distribution_model.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0014_swap_distribution_model.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0015_add_custom_fields_to_packages.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0015_add_custom_fields_to_packages.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0016_switch_to_newer_jsonfield_model.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0016_switch_to_newer_jsonfield_model.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0017_allow_longer_string_lists.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0017_allow_longer_string_lists.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0018_textfield_conversion.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0018_textfield_conversion.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/0019_immutable_metadata_constraints.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/0019_immutable_metadata_constraints.cpython-38.pyc
diff --git a/pulp_deb/app/migrations/__pycache__/__init__.cpython-38.pyc b/pulp_deb/app/migrations/__pycache__/__init__.cpython-38.pyc
diff --git a/pulp_deb/app/models/__pycache__/__init__.cpython-38.pyc b/pulp_deb/app/models/__pycache__/__init__.cpython-38.pyc
diff --git a/pulp_deb/app/models/__pycache__/content.cpython-38.pyc b/pulp_deb/app/models/__pycache__/content.cpython-38.pyc
diff --git a/pulp_deb/app/models/__pycache__/metadata_content.cpython-38.pyc b/pulp_deb/app/models/__pycache__/metadata_content.cpython-38.pyc
diff --git a/pulp_deb/app/models/__pycache__/publication.cpython-38.pyc b/pulp_deb/app/models/__pycache__/publication.cpython-38.pyc
diff --git a/pulp_deb/app/models/__pycache__/remote.cpython-38.pyc b/pulp_deb/app/models/__pycache__/remote.cpython-38.pyc
diff --git a/pulp_deb/app/models/__pycache__/repository.cpython-38.pyc b/pulp_deb/app/models/__pycache__/repository.cpython-38.pyc
diff --git a/pulp_deb/app/models/__pycache__/signing_service.cpython-38.pyc b/pulp_deb/app/models/__pycache__/signing_service.cpython-38.pyc
diff --git a/pulp_deb/app/models/__pycache__/structure_content.cpython-38.pyc b/pulp_deb/app/models/__pycache__/structure_content.cpython-38.pyc
diff --git a/pulp_deb/app/models/content/__pycache__/__init__.cpython-38.pyc b/pulp_deb/app/models/content/__pycache__/__init__.cpython-38.pyc
diff --git a/pulp_deb/app/models/content/__pycache__/content.cpython-38.pyc b/pulp_deb/app/models/content/__pycache__/content.cpython-38.pyc
diff --git a/pulp_deb/app/models/content/__pycache__/metadata.cpython-38.pyc b/pulp_deb/app/models/content/__pycache__/metadata.cpython-38.pyc
diff --git a/pulp_deb/app/models/content/__pycache__/structure_content.cpython-38.pyc b/pulp_deb/app/models/content/__pycache__/structure_content.cpython-38.pyc
diff --git a/pulp_deb/app/models/content/__pycache__/verbatim_metadata.cpython-38.pyc b/pulp_deb/app/models/content/__pycache__/verbatim_metadata.cpython-38.pyc
diff --git a/pulp_deb/app/schema/__pycache__/__init__.cpython-38.pyc b/pulp_deb/app/schema/__pycache__/__init__.cpython-38.pyc
diff --git a/pulp_deb/app/serializers/__pycache__/__init__.cpython-38.pyc b/pulp_deb/app/serializers/__pycache__/__init__.cpython-38.pyc
diff --git a/pulp_deb/app/serializers/__pycache__/content_serializers.cpython-38.pyc b/pulp_deb/app/serializers/__pycache__/content_serializers.cpython-38.pyc
diff --git a/pulp_deb/app/serializers/__pycache__/publication_serializers.cpython-38.pyc b/pulp_deb/app/serializers/__pycache__/publication_serializers.cpython-38.pyc
diff --git a/pulp_deb/app/serializers/__pycache__/remote_serializers.cpython-38.pyc b/pulp_deb/app/serializers/__pycache__/remote_serializers.cpython-38.pyc
diff --git a/pulp_deb/app/serializers/__pycache__/repository_serializers.cpython-38.pyc b/pulp_deb/app/serializers/__pycache__/repository_serializers.cpython-38.pyc
diff --git a/pulp_deb/app/settings.py b/pulp_deb/app/settings.py
@@ -7,3 +7,5 @@
 
 FORBIDDEN_CHECKSUM_WARNINGS = True
 FORCE_IGNORE_MISSING_PACKAGE_INDICES = False
+APT_BY_HASH = True
+APT_BY_HASH_CHECKSUM_TYPE = "sha256"
diff --git a/pulp_deb/app/tasks/__pycache__/__init__.cpython-38.pyc b/pulp_deb/app/tasks/__pycache__/__init__.cpython-38.pyc
diff --git a/pulp_deb/app/tasks/__pycache__/copy.cpython-38.pyc b/pulp_deb/app/tasks/__pycache__/copy.cpython-38.pyc
diff --git a/pulp_deb/app/tasks/__pycache__/exceptions.cpython-38.pyc b/pulp_deb/app/tasks/__pycache__/exceptions.cpython-38.pyc
diff --git a/pulp_deb/app/tasks/__pycache__/publishing.cpython-38.pyc b/pulp_deb/app/tasks/__pycache__/publishing.cpython-38.pyc
diff --git a/pulp_deb/app/tasks/__pycache__/synchronizing.cpython-38.pyc b/pulp_deb/app/tasks/__pycache__/synchronizing.cpython-38.pyc
diff --git a/pulp_deb/app/tasks/publishing.py b/pulp_deb/app/tasks/publishing.py
@@ -2,6 +2,7 @@
 import os
 import shutil
 from contextlib import suppress
+from pathlib import Path
 import hashlib
 
 from datetime import datetime, timezone
@@ -35,11 +36,9 @@
 
 from pulp_deb.app.serializers import Package822Serializer
 
-from pulp_deb.app.constants import (
-    NO_MD5_WARNING_MESSAGE,
-    CHECKSUM_TYPE_MAP,
-    APT_BY_HASH
-)
+from pulp_deb.app.constants import NO_MD5_WARNING_MESSAGE, CHECKSUM_TYPE_MAP
+
+from pulp_deb.app.settings import APT_BY_HASH, APT_BY_HASH_CHECKSUM_TYPE
 
 import logging
 from gettext import gettext as _
@@ -99,7 +98,7 @@ def publish(repository_version_pk, simple=False, structured=False, signing_servi
             structured=structured,
         )
     )
-    
+
     with tempfile.TemporaryDirectory("."):
         with AptPublication.create(repo_version, pass_through=False) as publication:
             publication.simple = simple
@@ -219,7 +218,7 @@ def publish(repository_version_pk, simple=False, structured=False, signing_servi
                 for release_helper in release_helpers:
                     release_helper.save_signed_metadata()
 
-    log.info(_("Publication: {publication} createddddd").format(publication=publication.pk))
+    log.info(_("Publication: {publication} created").format(publication=publication.pk))
 
 
 async def _concurrently_sign_metadata(release_helpers):
@@ -281,34 +280,26 @@ def finish(self):
             )
             package_index.save()
 
-            # Generating metadata files using checksum
-            if APT_BY_HASH:
-                by_hash_path = os.path.join(os.path.dirname(package_index_path), 'byhash')
-                if not os.path.exists(by_hash_path):
-                    os.makedirs(by_hash_path)
-                package_hashed_index_path = os.path.join(by_hash_path, _sha256sum(package_index_path))
-                shutil.copyfile(package_index_path, package_hashed_index_path)
-                package_hashed_index = PublishedMetadata.create_from_file(
-                    publication=self.parent.publication, file=File(open(package_hashed_index_path, "rb"))
-                )
-                package_hashed_index.save()
-            # Done generating
-
             gz_package_index = PublishedMetadata.create_from_file(
                 publication=self.parent.publication, file=File(open(gz_package_index_path, "rb"))
             )
             gz_package_index.save()
 
             # Generating metadata files using checksum
-            if APT_BY_HASH:
-                by_hash_path = os.path.join(os.path.dirname(gz_package_index_path), 'byhash')
-                if not os.path.exists(by_hash_path):
-                    os.makedirs(by_hash_path)
-                gz_package_hashed_index_path = os.path.join(by_hash_path, _sha256sum(gz_package_index_path))
-                shutil.copyfile(gz_package_index_path, gz_package_hashed_index_path)
+            if APT_BY_HASH and APT_BY_HASH_CHECKSUM_TYPE in ALLOWED_CONTENT_CHECKSUMS:
+                package_hashed_index_path = _create_checksum_file(package_index_path)
+                package_hashed_index = PublishedMetadata.create_from_file(
+                    publication=self.parent.publication,
+                    file=File(open(package_hashed_index_path, "rb")),
+                )
+                package_hashed_index.save()
+
+                gz_package_hashed_index_path = _create_checksum_file(gz_package_index_path)
                 gz_package_hashed_index = PublishedMetadata.create_from_file(
-                    publication=self.parent.publication, file=File(open(gz_package_hashed_index_path, "rb"))
+                    publication=self.parent.publication,
+                    file=File(open(gz_package_hashed_index_path, "rb")),
                 )
+
                 gz_package_hashed_index.save()
             # Done generating
 
@@ -426,9 +417,18 @@ def _zip_file(file_path):
             shutil.copyfileobj(f_in, f_out)
     return gz_file_path
 
-def _sha256sum(path):
-    h = hashlib.sha256()
+
+def _checksum_file(path):
+    h = hashlib.new(APT_BY_HASH_CHECKSUM_TYPE)
     with open(path, "rb") as f:
         for line in f:
             h.update(line)
-    return h.hexdigest()
+    return h.hexdigest()
+
+
+def _create_checksum_file(file_path):
+    by_hash_path = Path(file_path).parents[0] / "byhash"
+    by_hash_path.mkdir(parents=True, exists_ok=True)
+    hashed_path = by_hash_path / _checksum_file(file_path)
+    shutil.copyfile(file_path, hashed_path)
+    return hashed_path
diff --git a/pulp_deb/app/viewsets/__pycache__/__init__.cpython-38.pyc b/pulp_deb/app/viewsets/__pycache__/__init__.cpython-38.pyc
diff --git a/pulp_deb/app/viewsets/__pycache__/content.cpython-38.pyc b/pulp_deb/app/viewsets/__pycache__/content.cpython-38.pyc
diff --git a/pulp_deb/app/viewsets/__pycache__/publication.cpython-38.pyc b/pulp_deb/app/viewsets/__pycache__/publication.cpython-38.pyc
diff --git a/pulp_deb/app/viewsets/__pycache__/remote.cpython-38.pyc b/pulp_deb/app/viewsets/__pycache__/remote.cpython-38.pyc
diff --git a/pulp_deb/app/viewsets/__pycache__/repository.cpython-38.pyc b/pulp_deb/app/viewsets/__pycache__/repository.cpython-38.pyc
diff --git a/pulp_deb/tests/__pycache__/__init__.cpython-38.pyc b/pulp_deb/tests/__pycache__/__init__.cpython-38.pyc
diff --git a/pulp_deb/tests/functional/__pycache__/__init__.cpython-38.pyc b/pulp_deb/tests/functional/__pycache__/__init__.cpython-38.pyc
diff --git a/pulp_deb/tests/functional/__pycache__/conftest.cpython-38-pytest-7.4.0.pyc b/pulp_deb/tests/functional/__pycache__/conftest.cpython-38-pytest-7.4.0.pyc
diff --git a/pulp_deb/tests/functional/__pycache__/constants.cpython-38.pyc b/pulp_deb/tests/functional/__pycache__/constants.cpython-38.pyc
diff --git a/pulp_deb/tests/functional/__pycache__/utils.cpython-38.pyc b/pulp_deb/tests/functional/__pycache__/utils.cpython-38.pyc
diff --git a/pulp_deb/tests/functional/api/__pycache__/__init__.cpython-38.pyc b/pulp_deb/tests/functional/api/__pycache__/__init__.cpython-38.pyc
diff --git a/pulp_deb/tests/functional/api/__pycache__/test_crud_content_unit.cpython-38-pytest-7.4.0.pyc b/pulp_deb/tests/functional/api/__pycache__/test_crud_content_unit.cpython-38-pytest-7.4.0.pyc
diff --git a/pulp_deb/tests/functional/api/__pycache__/test_crud_remotes.cpython-38-pytest-7.4.0.pyc b/pulp_deb/tests/functional/api/__pycache__/test_crud_remotes.cpython-38-pytest-7.4.0.pyc
diff --git a/pulp_deb/tests/functional/api/__pycache__/test_download_content.cpython-38-pytest-7.4.0.pyc b/pulp_deb/tests/functional/api/__pycache__/test_download_content.cpython-38-pytest-7.4.0.pyc
diff --git a/pulp_deb/tests/functional/api/__pycache__/test_download_policies.cpython-38-pytest-7.4.0.pyc b/pulp_deb/tests/functional/api/__pycache__/test_download_policies.cpython-38-pytest-7.4.0.pyc
diff --git a/pulp_deb/tests/functional/api/__pycache__/test_publish.cpython-38-pytest-7.4.0.pyc b/pulp_deb/tests/functional/api/__pycache__/test_publish.cpython-38-pytest-7.4.0.pyc
diff --git a/...b/tests/functional/api/__pycache__/test_publish_complex_dists.cpython-38-pytest-7.4.0.pyc b/...b/tests/functional/api/__pycache__/test_publish_complex_dists.cpython-38-pytest-7.4.0.pyc
diff --git a/...ests/functional/api/__pycache__/test_publish_empty_repository.cpython-38-pytest-7.4.0.pyc b/...ests/functional/api/__pycache__/test_publish_empty_repository.cpython-38-pytest-7.4.0.pyc
diff --git a/...ests/functional/api/__pycache__/test_publish_flat_repo_format.cpython-38-pytest-7.4.0.pyc b/...ests/functional/api/__pycache__/test_publish_flat_repo_format.cpython-38-pytest-7.4.0.pyc
diff --git a/pulp_deb/tests/functional/api/__pycache__/test_pulpexport.cpython-38-pytest-7.4.0.pyc b/pulp_deb/tests/functional/api/__pycache__/test_pulpexport.cpython-38-pytest-7.4.0.pyc
diff --git a/pulp_deb/tests/functional/api/__pycache__/test_pulpimport.cpython-38-pytest-7.4.0.pyc b/pulp_deb/tests/functional/api/__pycache__/test_pulpimport.cpython-38-pytest-7.4.0.pyc
diff --git a/pulp_deb/tests/functional/api/__pycache__/test_sync.cpython-38-pytest-7.4.0.pyc b/pulp_deb/tests/functional/api/__pycache__/test_sync.cpython-38-pytest-7.4.0.pyc
diff --git a/...sts/functional/api/__pycache__/test_sync_missing_architecture.cpython-38-pytest-7.4.0.pyc b/...sts/functional/api/__pycache__/test_sync_missing_architecture.cpython-38-pytest-7.4.0.pyc
diff --git a/pulp_deb/tests/functional/api/__pycache__/test_sync_optimize.cpython-38-pytest-7.4.0.pyc b/pulp_deb/tests/functional/api/__pycache__/test_sync_optimize.cpython-38-pytest-7.4.0.pyc
Original file line number	Diff line number	Diff line change
Expand Up		@@ -17,4 +17,3 @@

		# Represents null values since nulls can't be used in unique indexes in postgres < 15
		NULL_VALUE = "__!!!NULL VALUE!!!__"
		APT_BY_HASH = True