Serve previously published artifacts for 3 days

fixes pulp#911

CHANGES/911.feature

@@ -0,0 +1,3 @@
+Added feature to serve published artifacts from previous publications for 3 days.
+This fulfills the apt-by-hash/acquire-by-hash spec by allowing by-hash files to be cached for a
+period of 3 days.

pulp_deb/app/migrations/0029_distributedpublication.py

@@ -0,0 +1,49 @@
+# Generated by Django 4.2.2 on 2023-11-06 21:00
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_lifecycle.mixins
+import pulpcore.app.models.base
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0114_remove_task_args_remove_task_kwargs"),
+        ("deb", "0028_sourcepackage_sourcepackagereleasecomponent_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DistributedPublication",
+            fields=[
+                (
+                    "pulp_id",
+                    models.UUIDField(
+                        default=pulpcore.app.models.base.pulp_uuid,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                ("pulp_created", models.DateTimeField(auto_now_add=True)),
+                ("pulp_last_updated", models.DateTimeField(auto_now=True, null=True)),
+                (
+                    "distribution",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="core.distribution"
+                    ),
+                ),
+                (
+                    "publication",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="core.publication"
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=(django_lifecycle.mixins.LifecycleModelMixin, models.Model),
+        ),
+    ]

pulp_deb/app/models/publication.py

@@ -1,6 +1,10 @@
+from datetime import timedelta
+
 from django.db import models
+from django.utils import timezone
+from django_lifecycle import hook, AFTER_CREATE, AFTER_SAVE
 
-from pulpcore.plugin.models import Publication, Distribution
+from pulpcore.plugin.models import BaseModel, Distribution, Publication, PublishedArtifact
 
 from pulp_deb.app.models.signing_service import AptReleaseSigningService
 
@@ -17,6 +21,11 @@ class VerbatimPublication(Publication):
 
     TYPE = "verbatim-publication"
 
+    @hook(AFTER_CREATE)
+    def set_distributed_publication(self):
+        for distro in AptDistribution.objects.filter(repository__pk=self.repository.pk):
+            DistributedPublication(distribution=distro, publication=self).save()
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
 
@@ -36,6 +45,11 @@ class AptPublication(Publication):
         AptReleaseSigningService, on_delete=models.PROTECT, null=True
     )
 
+    @hook(AFTER_CREATE)
+    def set_distributed_publication(self):
+        for distro in AptDistribution.objects.filter(repository__pk=self.repository.pk):
+            DistributedPublication(distribution=distro, publication=self).save()
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
 
@@ -46,6 +60,47 @@ class AptDistribution(Distribution):
     """
 
     TYPE = "apt-distribution"
+    PUBLICATION_CACHE_DURATION = timedelta(days=3)
+
+    @hook(AFTER_SAVE, when="publication", has_changed=True)
+    def set_distributed_publication(self):
+        if self.publication:
+            DistributedPublication(distribution=self, publication=self.publication).save()
+
+    def content_handler(self, path):
+        recent_dp = self.distributedpublication_set.filter(
+            pulp_created__gte=timezone.now() - self.PUBLICATION_CACHE_DURATION
+        ).order_by("pulp_created")
+        pa = (
+            PublishedArtifact.objects.filter(
+                relative_path=path, publication__distributedpublication__pk__in=recent_dp
+            )
+            .order_by("-publication__distributedpublication__pulp_created")
+            .select_related(
+                "content_artifact",
+                "content_artifact__artifact",
+            )
+        ).first()
+
+        if pa:
+            return pa.content_artifact
+        return
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
+
+
+class DistributedPublication(BaseModel):
+    """
+    A history of previously distributed publications.
+    """
+
+    distribution = models.ForeignKey(Distribution, on_delete=models.CASCADE)
+    publication = models.ForeignKey(Publication, on_delete=models.CASCADE)
+
+    @hook(AFTER_SAVE)
+    def cleanup(self):
+        """Clean up DistributedPublications older than PUBLICATION_CACHE_DURATION."""
+        DistributedPublication.objects.filter(
+            pulp_created__lt=(timezone.now() - AptDistribution.PUBLICATION_CACHE_DURATION)
+        ).delete()

pulp_deb/tests/functional/api/test_download_content.py

@@ -169,3 +169,37 @@ def test_download_content(
         content = download_content_unit(distribution.base_path, unit_path[1])
         pulp_hashes.append(hashlib.sha256(content).hexdigest())
     assert fixtures_hashes == pulp_hashes
+
+
+@pytest.mark.parallel
+def test_download_cached_content(
+    deb_init_and_sync,
+    deb_distribution_factory,
+    deb_publication_factory,
+    deb_fixture_server,
+    download_content_unit,
+    http_get,
+    deb_get_content_types,
+    deb_modify_repository,
+):
+    """Verify that previously published content can still be downloaded."""
+    # Create/sync a repo and then a distro
+    repo, _ = deb_init_and_sync()
+    distribution = deb_distribution_factory(repository=repo)
+    deb_publication_factory(repo, structured=True, simple=True)
+
+    # Find a random package and get its hash digest
+    package_content = deb_get_content_types("apt_package_api", DEB_PACKAGE_NAME, repo)
+    package = choice(package_content)
+    url = deb_fixture_server.make_url(DEB_FIXTURE_STANDARD_REPOSITORY_NAME)
+    package_hash = hashlib.sha256(http_get(urljoin(url, package.relative_path))).hexdigest()
+
+    # Remove content and republish
+    deb_modify_repository(repo, {"remove_content_units": ["*"]})
+    deb_publication_factory(repo, structured=True, simple=True)
+
+    # Download the package and check its checksum
+    content = download_content_unit(distribution.base_path, package.relative_path)
+    content_hash = hashlib.sha256(content).hexdigest()
+
+    assert package_hash == content_hash

pulp_deb/tests/functional/conftest.py

@@ -128,14 +128,17 @@ def apt_generic_content_api(apt_client):
 def deb_distribution_factory(apt_distribution_api, gen_object_with_cleanup):
     """Fixture that generates a deb distribution with cleanup from a given publication."""
 
-    def _deb_distribution_factory(publication):
+    def _deb_distribution_factory(publication=None, repository=None):
         """Create a deb distribution.
 
         :param publication: The publication the distribution is based on.
         :returns: The created distribution.
         """
         body = gen_distribution()
-        body["publication"] = publication.pulp_href
+        if publication:
+            body["publication"] = publication.pulp_href
+        if repository:
+            body["repository"] = repository.pulp_href
         return gen_object_with_cleanup(apt_distribution_api, body)
 
     return _deb_distribution_factory