Skip to content

Commit

Permalink
squashing previous commits
Browse files Browse the repository at this point in the history
restored one new line for improved readability
  • Loading branch information
Andrew Cheng committed Sep 1, 2023
1 parent 3264630 commit 012ca8d
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 4 deletions.
2 changes: 2 additions & 0 deletions CHANGES/795.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Mitigating the Hash Sum Mismatch error in debian repos by generating additional metadata files named by checksum.
Wiki of the feature: https://wiki.ubuntu.com/AptByHash
2 changes: 2 additions & 0 deletions pulp_deb/app/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,5 @@

FORBIDDEN_CHECKSUM_WARNINGS = True
FORCE_IGNORE_MISSING_PACKAGE_INDICES = False

APT_BY_HASH = True
33 changes: 29 additions & 4 deletions pulp_deb/app/tasks/publishing.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import os
import shutil
from contextlib import suppress
from pathlib import Path

from datetime import datetime, timezone
from debian import deb822
Expand Down Expand Up @@ -34,11 +35,9 @@

from pulp_deb.app.serializers import Package822Serializer

from pulp_deb.app.constants import (
NO_MD5_WARNING_MESSAGE,
CHECKSUM_TYPE_MAP,
)
from pulp_deb.app.constants import NO_MD5_WARNING_MESSAGE, CHECKSUM_TYPE_MAP

from pulp_deb.app.settings import APT_BY_HASH

import logging
from gettext import gettext as _
Expand Down Expand Up @@ -296,10 +295,28 @@ def finish(self):
publication=self.parent.publication, file=File(open(package_index_path, "rb"))
)
package_index.save()

gz_package_index = PublishedMetadata.create_from_file(
publication=self.parent.publication, file=File(open(gz_package_index_path, "rb"))
)
gz_package_index.save()

# Generating metadata files using checksum
if APT_BY_HASH:
for path, index in (
(package_index_path, package_index),
(gz_package_index_path, gz_package_index),
):
for allowed_checksum in settings.ALLOWED_CONTENT_CHECKSUMS:
if allowed_checksum in CHECKSUM_TYPE_MAP:
hashed_index_path = _fetch_file_checksum(path, index, allowed_checksum)
hashed_index = PublishedMetadata.create_from_file(
publication=self.parent.publication,
file=File(open(path, "rb")),
relative_path=hashed_index_path,
)
hashed_index.save()

self.parent.add_metadata(package_index)
self.parent.add_metadata(gz_package_index)

Expand Down Expand Up @@ -339,6 +356,7 @@ def __init__(
self.release["Components"] = "" # Will be set later
if release.description != NULL_VALUE:
self.release["Description"] = release.description
self.release["Acquire-By-Hash"] = "yes" if APT_BY_HASH else "no"

for checksum_type, deb_field in CHECKSUM_TYPE_MAP.items():
if checksum_type in settings.ALLOWED_CONTENT_CHECKSUMS:
Expand Down Expand Up @@ -413,3 +431,10 @@ def _zip_file(file_path):
with GzipFile(gz_file_path, "wb") as f_out:
shutil.copyfileobj(f_in, f_out)
return gz_file_path


def _fetch_file_checksum(file_path, index, allowed_checksum):
h = getattr(index.contentartifact_set.first().artifact, allowed_checksum)
checksum_type = CHECKSUM_TYPE_MAP[allowed_checksum]
hashed_path = Path(file_path).parents[0] / "by-hash" / checksum_type / h
return hashed_path

0 comments on commit 012ca8d

Please sign in to comment.