diff --git a/CHANGES/911.feature b/CHANGES/911.feature new file mode 100644 index 000000000..ede9546f2 --- /dev/null +++ b/CHANGES/911.feature @@ -0,0 +1,3 @@ +Added feature to serve published artifacts from previous publications for 3 days. +This fulfills the apt-by-hash/acquire-by-hash spec by allowing by-hash files to be cached for a +period of time. diff --git a/pulp_deb/app/migrations/0029_distributedpublication.py b/pulp_deb/app/migrations/0029_distributedpublication.py new file mode 100644 index 000000000..67072e815 --- /dev/null +++ b/pulp_deb/app/migrations/0029_distributedpublication.py @@ -0,0 +1,49 @@ +# Generated by Django 4.2.2 on 2023-11-06 21:00 + +from django.db import migrations, models +import django.db.models.deletion +import django_lifecycle.mixins +import pulpcore.app.models.base + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0114_remove_task_args_remove_task_kwargs"), + ("deb", "0028_sourcepackage_sourcepackagereleasecomponent_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="DistributedPublication", + fields=[ + ( + "pulp_id", + models.UUIDField( + default=pulpcore.app.models.base.pulp_uuid, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("pulp_created", models.DateTimeField(auto_now_add=True)), + ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "distribution", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="core.distribution" + ), + ), + ( + "publication", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="core.publication" + ), + ), + ], + options={ + "abstract": False, + }, + bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model), + ), + ] diff --git a/pulp_deb/app/models/publication.py b/pulp_deb/app/models/publication.py index 3c99d2709..53231b785 100644 --- a/pulp_deb/app/models/publication.py +++ b/pulp_deb/app/models/publication.py @@ -1,6 +1,10 @@ +from datetime import timedelta + from django.db import models +from django.utils import timezone +from django_lifecycle import hook, AFTER_CREATE, AFTER_SAVE -from pulpcore.plugin.models import Publication, Distribution +from pulpcore.plugin.models import BaseModel, Distribution, Publication, PublishedArtifact from pulp_deb.app.models.signing_service import AptReleaseSigningService @@ -17,6 +21,11 @@ class VerbatimPublication(Publication): TYPE = "verbatim-publication" + @hook(AFTER_CREATE) + def set_distributed_publication(self): + for distro in AptDistribution.objects.filter(repository__pk=self.repository.pk): + DistributedPublication(distribution=distro, publication=self).save() + class Meta: default_related_name = "%(app_label)s_%(model_name)s" @@ -36,6 +45,11 @@ class AptPublication(Publication): AptReleaseSigningService, on_delete=models.PROTECT, null=True ) + @hook(AFTER_CREATE) + def set_distributed_publication(self): + for distro in AptDistribution.objects.filter(repository__pk=self.repository.pk): + DistributedPublication(distribution=distro, publication=self).save() + class Meta: default_related_name = "%(app_label)s_%(model_name)s" @@ -46,6 +60,47 @@ class AptDistribution(Distribution): """ TYPE = "apt-distribution" + PUBLICATION_CACHE_DURATION = timedelta(days=3) + + @hook(AFTER_SAVE, when="publication", has_changed=True) + def set_distributed_publication(self): + if self.publication: + DistributedPublication(distribution=self, publication=self.publication).save() + + def content_handler(self, path): + recent_dp = self.distributedpublication_set.filter( + pulp_created__gte=timezone.now() - self.PUBLICATION_CACHE_DURATION + ).order_by("pulp_created") + pa = ( + PublishedArtifact.objects.filter( + relative_path=path, publication__distributedpublication__pk__in=recent_dp + ) + .order_by("-publication__distributedpublication__pulp_created") + .select_related( + "content_artifact", + "content_artifact__artifact", + ) + ).first() + + if pa: + return pa.content_artifact + return class Meta: default_related_name = "%(app_label)s_%(model_name)s" + + +class DistributedPublication(BaseModel): + """ + A history of previously distributed publications. + """ + + distribution = models.ForeignKey(Distribution, on_delete=models.CASCADE) + publication = models.ForeignKey(Publication, on_delete=models.CASCADE) + + @hook(AFTER_SAVE) + def cleanup(self): + """Clean up DistributedPublications older than PUBLICATION_CACHE_DURATION.""" + DistributedPublication.objects.filter( + pulp_created__lt=(timezone.now() - AptDistribution.PUBLICATION_CACHE_DURATION) + ).delete() diff --git a/pulp_deb/tests/functional/api/test_download_content.py b/pulp_deb/tests/functional/api/test_download_content.py index 6695ad82b..63efde071 100644 --- a/pulp_deb/tests/functional/api/test_download_content.py +++ b/pulp_deb/tests/functional/api/test_download_content.py @@ -168,4 +168,39 @@ def test_download_content( for unit_path in unit_paths: content = download_content_unit(distribution.base_path, unit_path[1]) pulp_hashes.append(hashlib.sha256(content).hexdigest()) + assert fixtures_hashes == pulp_hashes + + +@pytest.mark.parallel +def test_download_cached_content( + deb_init_and_sync, + deb_distribution_factory, + deb_publication_factory, + deb_fixture_server, + download_content_unit, + http_get, + deb_get_content_types, + deb_modify_repository, +): + """Verify that previously published content can still be downloaded.""" + # Create/sync a repo and then a distro + repo, _ = deb_init_and_sync() + distribution = deb_distribution_factory(repository=repo) + deb_publication_factory(repo, structured=True, simple=True) + + # Find a random package and get its hash digest + package_content = deb_get_content_types("apt_package_api", DEB_PACKAGE_NAME, repo) + package = choice(package_content) + url = deb_fixture_server.make_url(DEB_FIXTURE_STANDARD_REPOSITORY_NAME) + package_hash = hashlib.sha256(http_get(urljoin(url, package.relative_path))).hexdigest() + + # Remove content and republish + deb_modify_repository(repo, {"remove_content_units": ["*"]}) + deb_publication_factory(repo, structured=True, simple=True) + + # Download the package and check its checksum + content = download_content_unit(distribution.base_path, package.relative_path) + content_hash = hashlib.sha256(content).hexdigest() + + assert package_hash == content_hash diff --git a/pulp_deb/tests/functional/conftest.py b/pulp_deb/tests/functional/conftest.py index 284df1a95..6d4f85cbe 100644 --- a/pulp_deb/tests/functional/conftest.py +++ b/pulp_deb/tests/functional/conftest.py @@ -128,14 +128,17 @@ def apt_generic_content_api(apt_client): def deb_distribution_factory(apt_distribution_api, gen_object_with_cleanup): """Fixture that generates a deb distribution with cleanup from a given publication.""" - def _deb_distribution_factory(publication): + def _deb_distribution_factory(publication=None, repository=None): """Create a deb distribution. :param publication: The publication the distribution is based on. :returns: The created distribution. """ body = gen_distribution() - body["publication"] = publication.pulp_href + if publication: + body["publication"] = publication.pulp_href + if repository: + body["repository"] = repository.pulp_href return gen_object_with_cleanup(apt_distribution_api, body) return _deb_distribution_factory diff --git a/requirements.txt b/requirements.txt index fd7d8e464..4864d274f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # All things django and asyncio are deliberately left to pulpcore # Example transitive requirements: asgiref, asyncio, aiohttp -pulpcore>=3.40.1,<3.55 +pulpcore>=3.41.0,<3.55 python-debian>=0.1.44,<0.2.0 python-gnupg>=0.5,<0.6 jsonschema>=4.6,<5.0