Add signature support for CollectionVersions

Add new signature model CollectionVersionSignature
Add new content API to view CollectionVersionSignatures
Add new sign task to repository endpoint to create new
CollectionVersionSignatures
Add new signature metadata to CollectionVersion Galaxy V3 API to enable
syncing of signatures

fixes: pulp#748
fixes: pulp#757
fixes: pulp#758

.github/workflows/scripts/post_before_script.sh

@@ -1,8 +1,24 @@
 #!/usr/bin/env sh
 
+set -euv
+
 echo "machine pulp
 login admin
 password password
 " > ~/.netrc
 
-chmod og-rw ~/.netrc
+chmod og-rw ~/.netrc
+
+if [[ "$TEST" == "upgrade" ]]; then
+    exit
+fi
+
+cmd_stdin_prefix bash -c "cat > /var/lib/pulp/sign-metadata.sh" < "$GITHUB_WORKSPACE"/pulp_ansible/tests/assets/sign-metadata.sh
+
+cmd_prefix bash -c "curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-PRIVATE-KEY-pulp-qe | gpg --import"
+cmd_prefix bash -c "curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-KEY-pulp-qe | cat > /tmp/GPG-KEY-pulp-qe"
+cmd_prefix chmod a+x /var/lib/pulp/sign-metadata.sh
+
+KEY_FINGERPRINT="6EDF301256480B9B801EBA3D05A5E6DA269D9D98"
+TRUST_LEVEL="6"
+echo "$KEY_FINGERPRINT:$TRUST_LEVEL:" | cmd_stdin_prefix gpg --import-ownertrust

CHANGES/748.feature

@@ -0,0 +1 @@
+Added Collection Signatures to the Galaxy V3 API to allow for syncing of signatures during a collection sync.

CHANGES/757.feature

@@ -0,0 +1 @@
+Added ``CollectionVersionSignature`` content model to store signatures for Collections.

CHANGES/758.feature

@@ -0,0 +1 @@
+Added API to serve Collection Signatures at ``/pulp/api/v3/content/ansible/collection_signatures/``.

docs/settings.rst

@@ -26,6 +26,14 @@ ANSIBLE_CONTENT_HOSTNAME
    CONTENT_ORIGIN, this would default to "https://example.com/pulp/content".
 
 
+ANSIBLE_SIGNING_TASK_LIMITER
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+   This number determines the amount of concurrent signing processes that can be spawned at one time
+   during a signature task. Increasing this number will generally increase the speed of the task, but
+   will also consume more resources of the worker. Defaults to 10 concurrent processes.
+
+
 GALAXY_API_ROOT
 ^^^^^^^^^^^^^^^
 

pulp_ansible/app/galaxy/v3/serializers.py

@@ -171,6 +171,29 @@ class CollectionNamespaceSerializer(serializers.Serializer):
     name = serializers.CharField(source="namespace")
 
 
+class CollectionVersionSignatureSerializer(serializers.ModelSerializer):
+    """
+    A serializer for the signatures on a Collection Version.
+    """
+
+    signature = serializers.SerializerMethodField()
+    signing_service = serializers.SlugRelatedField(
+        slug_field="name",
+        allow_null=True,
+        read_only=True,
+    )
+
+    def get_signature(self, obj):
+        """
+        Get the signature data.
+        """
+        return bytes(obj.data).decode("utf-8")
+
+    class Meta:
+        model = models.CollectionVersionSignature
+        fields = ("signature", "pubkey_fingerprint", "signing_service", "pulp_created")
+
+
 class UnpaginatedCollectionVersionSerializer(CollectionVersionListSerializer):
     """
     A serializer for unpaginated CollectionVersion.
@@ -184,6 +207,7 @@ class UnpaginatedCollectionVersionSerializer(CollectionVersionListSerializer):
 
     metadata = CollectionMetadataSerializer(source="*", read_only=True)
     namespace = CollectionNamespaceSerializer(source="*", read_only=True)
+    signatures = serializers.SerializerMethodField()
 
     class Meta:
         model = models.CollectionVersion
@@ -193,6 +217,7 @@ class Meta:
             "download_url",
             "name",
             "namespace",
+            "signatures",
             "metadata",
             "git_url",
             "git_commit_sha",
@@ -235,6 +260,13 @@ def get_git_commit_sha(self, obj) -> str:
         if not content_artifact.get().artifact:
             return content_artifact.get().remoteartifact_set.all()[0].url[-40:]
 
+    def get_signatures(self, obj):
+        """
+        Get the signatures.
+        """
+        filtered_signatures = obj.signatures.filter(pk__in=self.context.get("sigs", []))
+        return CollectionVersionSignatureSerializer(filtered_signatures, many=True).data
+
 
 class CollectionVersionSerializer(UnpaginatedCollectionVersionSerializer):
     """

pulp_ansible/app/galaxy/v3/views.py

@@ -40,6 +40,7 @@
     AnsibleDistribution,
     Collection,
     CollectionVersion,
+    CollectionVersionSignature,
     CollectionImport,
 )
 from pulp_ansible.app.serializers import (
@@ -88,6 +89,8 @@ def get_serializer_context(self):
         context = super().get_serializer_context()
         if "path" in self.kwargs:
             context["path"] = self.kwargs["path"]
+        if "sigs" in self.kwargs:
+            context["sigs"] = self.kwargs["sigs"]
         return context
 
 
@@ -431,6 +434,14 @@ class CollectionVersionViewSet(
 
     lookup_field = "version"
 
+    def get_queryset(self):
+        """
+        Returns a CollectionVersions queryset for specified distribution.
+        """
+        distro_content = self._distro_content
+        self.kwargs["sigs"] = CollectionVersionSignature.objects.filter(pk__in=distro_content)
+        return CollectionVersion.objects.select_related().filter(pk__in=distro_content)
+
     def get_list_serializer(self, *args, **kwargs):
         """
         Return the list serializer instance.
@@ -463,14 +474,6 @@ class UnpaginatedCollectionVersionViewSet(CollectionVersionViewSet):
     serializer_class = UnpaginatedCollectionVersionSerializer
     pagination_class = None
 
-    def get_queryset(self):
-        """
-        Returns a CollectionVersions queryset for specified distribution.
-        """
-        distro_content = self._distro_content
-
-        return CollectionVersion.objects.select_related().filter(pk__in=distro_content)
-
     def list(self, request, *args, **kwargs):
         """
         Returns paginated CollectionVersions list.

pulp_ansible/app/migrations/0038_collectionversionsignature.py

@@ -0,0 +1,31 @@
+# Generated by Django 3.2.8 on 2021-11-29 21:17
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0077_move_remote_url_credentials'),
+        ('ansible', '0037_gitremote'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CollectionVersionSignature',
+            fields=[
+                ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='ansible_collectionversionsignature', serialize=False, to='core.content')),
+                ('data', models.BinaryField()),
+                ('digest', models.CharField(max_length=64)),
+                ('pubkey_fingerprint', models.CharField(max_length=64)),
+                ('signed_collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='signatures', to='ansible.collectionversion')),
+                ('signing_service', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='signatures', to='core.signingservice')),
+            ],
+            options={
+                'default_related_name': '%(app_label)s_%(model_name)s',
+                'unique_together': {('pubkey_fingerprint', 'signed_collection')},
+            },
+            bases=('core.content',),
+        ),
+    ]

pulp_ansible/app/migrations/0039_collectionremote_signed_only.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.11 on 2022-01-10 21:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ansible', '0038_collectionversionsignature'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='collectionremote',
+            name='signed_only',
+            field=models.BooleanField(default=False),
+        ),
+    ]

pulp_ansible/app/models.py

@@ -11,6 +11,7 @@
     Remote,
     Repository,
     Distribution,
+    SigningService,
     Task,
 )
 from .downloaders import AnsibleDownloaderFactory
@@ -178,6 +179,38 @@ class Meta:
         ]
 
 
+class CollectionVersionSignature(Content):
+    """
+    A content type representing a signature that is attached to a content unit.
+
+    Fields:
+        data (models.BinaryField): A signature, base64 encoded. # Not sure if it is base64 encoded
+        digest (models.CharField): A signature sha256 digest.
+        pubkey_fingerprint (models.CharField): A fingerprint of the public key used.
+
+    Relations:
+        signed_collection (models.ForeignKey): A collection version this signature is relevant to.
+        signing_service (models.ForeignKey): An optional signing service used for creation.
+    """
+
+    PROTECTED_FROM_RECLAIM = False
+    TYPE = "collection_signature"
+
+    signed_collection = models.ForeignKey(
+        CollectionVersion, on_delete=models.CASCADE, related_name="signatures"
+    )
+    data = models.BinaryField()  # This is what the pulp_container folks used
+    digest = models.CharField(max_length=64)  # does adding a min length improve db efficiency?
+    pubkey_fingerprint = models.CharField(max_length=64)
+    signing_service = models.ForeignKey(
+        SigningService, on_delete=models.SET_NULL, related_name="signatures", null=True
+    )
+
+    class Meta:
+        default_related_name = "%(app_label)s_%(model_name)s"
+        unique_together = ("pubkey_fingerprint", "signed_collection")
+
+
 class RoleRemote(Remote):
     """
     A Remote for Ansible content.
@@ -200,6 +233,7 @@ class CollectionRemote(Remote):
     auth_url = models.CharField(null=True, max_length=255)
     token = models.TextField(null=True, max_length=2000)
     sync_dependencies = models.BooleanField(default=True)
+    signed_only = models.BooleanField(default=False)
 
     @property
     def download_factory(self):
@@ -265,7 +299,12 @@ class AnsibleRepository(Repository):
     """
 
     TYPE = "ansible"
-    CONTENT_TYPES = [Role, CollectionVersion, AnsibleCollectionDeprecated]
+    CONTENT_TYPES = [
+        Role,
+        CollectionVersion,
+        AnsibleCollectionDeprecated,
+        CollectionVersionSignature,
+    ]
     REMOTE_TYPES = [RoleRemote, CollectionRemote]
 
     last_synced_metadata_time = models.DateTimeField(null=True)

pulp_ansible/app/serializers.py

@@ -4,10 +4,13 @@
 from jsonschema import Draft7Validator
 from rest_framework import serializers
 
-from pulpcore.plugin.models import Artifact, PulpTemporaryFile
+from pulpcore.plugin.models import Artifact, PulpTemporaryFile, SigningService
 from pulpcore.plugin.serializers import (
+    DetailRelatedField,
     ContentChecksumSerializer,
     ModelSerializer,
+    NoArtifactContentSerializer,
+    RelatedField,
     RemoteSerializer,
     RepositorySerializer,
     RepositorySyncURLSerializer,
@@ -28,6 +31,7 @@
     Collection,
     CollectionImport,
     CollectionVersion,
+    CollectionVersionSignature,
     CollectionRemote,
     Role,
     Tag,
@@ -171,6 +175,12 @@ class CollectionRemoteSerializer(RemoteSerializer):
         default=True,
     )
 
+    signed_only = serializers.BooleanField(
+        help_text=_("Sync only collections that have a signature"),
+        allow_null=True,
+        default=False,
+    )
+
     def validate(self, data):
         """
         Validate collection remote data.
@@ -221,6 +231,7 @@ class Meta:
             "auth_url",
             "token",
             "sync_dependencies",
+            "signed_only",
         )
         model = CollectionRemote
 
@@ -490,6 +501,58 @@ class Meta:
         model = CollectionVersion
 
 
+class CollectionVersionSignatureSerializer(NoArtifactContentSerializer):
+    """
+    A serializer for signature models.
+    """
+
+    signed_collection = DetailRelatedField(
+        help_text=_("The content this signature is pointing to."),
+        view_name_pattern=r"content(-.*/.*)-detail",
+        queryset=CollectionVersion.objects.all(),
+    )
+    pubkey_fingerprint = serializers.CharField(help_text=_("The fingerprint of the public key."))
+    signing_service = RelatedField(
+        help_text=_("The signing service used to create the signature."),
+        view_name="signing-services-detail",
+        queryset=SigningService.objects.all(),
+        allow_null=True,
+    )
+
+    class Meta:
+        model = CollectionVersionSignature
+        fields = NoArtifactContentSerializer.Meta.fields + (
+            "signed_collection",
+            "pubkey_fingerprint",
+            "signing_service",
+        )
+
+
+class AnsibleRepositorySignatureSerializer(serializers.Serializer):
+    """
+    A serializer for the signing action.
+    """
+
+    content_units = serializers.ListField(
+        required=True,
+        help_text=_(
+            "List of collection version hrefs to sign, use * to sign all content in repository"
+        ),
+    )
+    signing_service = RelatedField(
+        required=True,
+        view_name="signing-services-detail",
+        queryset=SigningService.objects.all(),
+        help_text=_("A signing service to use to sign the collections"),
+    )
+
+    def validate_content_units(self, value):
+        """Make sure the list is correctly formatted."""
+        if len(value) > 1 and "*" in value:
+            raise serializers.ValidationError("Cannot supply content units and '*'.")
+        return value
+
+
 class CollectionImportListSerializer(serializers.ModelSerializer):
     """
     A serializer for a CollectionImport list view.

pulp_ansible/app/settings.py

@@ -22,3 +22,4 @@
 
 ANSIBLE_API_HOSTNAME = "https://" + socket.getfqdn()
 ANSIBLE_CONTENT_HOSTNAME = settings.CONTENT_ORIGIN + "/pulp/content"
+ANSIBLE_SIGNING_TASK_LIMITER = 10