From 3fddd6fd4ed4c49ab01074c4f24ebf827757b129 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Thu, 18 Nov 2021 15:23:16 -0500 Subject: [PATCH] Add signature support for CollectionVersions Add new signature model CollectionVersionSignature Add new content API to view CollectionVersionSignatures Add new sign task to repository endpoint to create new CollectionVersionSignatures Add new signature metadata to CollectionVersion Galaxy V3 API to enable syncing of signatures fixes: #748 fixes: #757 fixes: #758 --- .../workflows/scripts/post_before_script.sh | 18 +- CHANGES/748.feature | 1 + CHANGES/757.feature | 1 + CHANGES/758.feature | 1 + pulp_ansible/app/galaxy/v3/serializers.py | 33 +++ pulp_ansible/app/galaxy/v3/views.py | 19 +- .../0038_collectionversionsignature.py | 31 +++ .../0039_collectionremote_signed_only.py | 18 ++ pulp_ansible/app/models.py | 41 +++- pulp_ansible/app/serializers.py | 65 +++++- pulp_ansible/app/tasks/collections.py | 19 ++ pulp_ansible/app/tasks/signature.py | 114 ++++++++++ pulp_ansible/app/viewsets.py | 69 ++++++ pulp_ansible/tests/assets/sign-metadata.sh | 18 ++ .../api/collection/test_signatures.py | 199 ++++++++++++++++++ pulp_ansible/tests/functional/utils.py | 43 +++- 16 files changed, 678 insertions(+), 12 deletions(-) create mode 100644 CHANGES/748.feature create mode 100644 CHANGES/757.feature create mode 100644 CHANGES/758.feature create mode 100644 pulp_ansible/app/migrations/0038_collectionversionsignature.py create mode 100644 pulp_ansible/app/migrations/0039_collectionremote_signed_only.py create mode 100644 pulp_ansible/app/tasks/signature.py create mode 100755 pulp_ansible/tests/assets/sign-metadata.sh create mode 100644 pulp_ansible/tests/functional/api/collection/test_signatures.py diff --git a/.github/workflows/scripts/post_before_script.sh b/.github/workflows/scripts/post_before_script.sh index 4101ef763..63d942e96 100755 --- a/.github/workflows/scripts/post_before_script.sh +++ b/.github/workflows/scripts/post_before_script.sh @@ -1,8 +1,24 @@ #!/usr/bin/env sh +set -euv + +if [[ "$TEST" == "upgrade" ]]; then + exit +fi + +cmd_stdin_prefix bash -c "cat > /var/lib/pulp/sign-metadata.sh" < "$GITHUB_WORKSPACE"/pulp_ansible/tests/assets/sign-metadata.sh + +cmd_prefix bash -c "curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-PRIVATE-KEY-pulp-qe | gpg --import" +cmd_prefix bash -c "curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-KEY-pulp-qe | cat > /tmp/GPG-KEY-pulp-qe" +cmd_prefix chmod a+x /var/lib/pulp/sign-metadata.sh + +KEY_FINGERPRINT="6EDF301256480B9B801EBA3D05A5E6DA269D9D98" +TRUST_LEVEL="6" +echo "$KEY_FINGERPRINT:$TRUST_LEVEL:" | cmd_stdin_prefix gpg --import-ownertrust + echo "machine pulp login admin password password " > ~/.netrc -chmod og-rw ~/.netrc \ No newline at end of file +chmod og-rw ~/.netrc diff --git a/CHANGES/748.feature b/CHANGES/748.feature new file mode 100644 index 000000000..bb7bb15be --- /dev/null +++ b/CHANGES/748.feature @@ -0,0 +1 @@ +Added Collection Signatures to the Galaxy V3 API to allow for syncing of signatures during a collection sync. diff --git a/CHANGES/757.feature b/CHANGES/757.feature new file mode 100644 index 000000000..b06b1f39b --- /dev/null +++ b/CHANGES/757.feature @@ -0,0 +1 @@ +Added ``CollectionVersionSignature`` content model to store signatures for Collections. diff --git a/CHANGES/758.feature b/CHANGES/758.feature new file mode 100644 index 000000000..b722ee4f7 --- /dev/null +++ b/CHANGES/758.feature @@ -0,0 +1 @@ +Added API to serve Collection Signatures at ``/pulp/api/v3/content/ansible/collection_signatures/``. diff --git a/pulp_ansible/app/galaxy/v3/serializers.py b/pulp_ansible/app/galaxy/v3/serializers.py index 907d6c664..ccee31710 100644 --- a/pulp_ansible/app/galaxy/v3/serializers.py +++ b/pulp_ansible/app/galaxy/v3/serializers.py @@ -171,6 +171,30 @@ class CollectionNamespaceSerializer(serializers.Serializer): name = serializers.CharField(source="namespace") +class CollectionVersionSignatureSerializer(serializers.ModelSerializer): + """ + A serializer for the signatures on a Collection Version. + """ + + signature = serializers.SerializerMethodField() + signing_service = serializers.SlugRelatedField( + slug_field="name", + allow_null=True, + read_only=True, + ) + + def get_signature(self, obj) -> str: + """ + Get the signature data. + """ + # Maybe I should decode into ascii? utf8 should be superset of ascii + return bytes(obj.data).decode("utf-8") + + class Meta: + model = models.CollectionVersionSignature + fields = ("signature", "pubkey_fingerprint", "signing_service", "pulp_created") + + class UnpaginatedCollectionVersionSerializer(CollectionVersionListSerializer): """ A serializer for unpaginated CollectionVersion. @@ -184,6 +208,7 @@ class UnpaginatedCollectionVersionSerializer(CollectionVersionListSerializer): metadata = CollectionMetadataSerializer(source="*", read_only=True) namespace = CollectionNamespaceSerializer(source="*", read_only=True) + signatures = serializers.SerializerMethodField() class Meta: model = models.CollectionVersion @@ -193,6 +218,7 @@ class Meta: "download_url", "name", "namespace", + "signatures", "metadata", "git_url", "git_commit_sha", @@ -235,6 +261,13 @@ def get_git_commit_sha(self, obj) -> str: if not content_artifact.get().artifact: return content_artifact.get().remoteartifact_set.all()[0].url[-40:] + def get_signatures(self, obj): + """ + Get the signatures. + """ + filtered_signatures = obj.signatures.filter(pk__in=self.context["sigs"]) + return CollectionVersionSignatureSerializer(filtered_signatures, many=True).data + class CollectionVersionSerializer(UnpaginatedCollectionVersionSerializer): """ diff --git a/pulp_ansible/app/galaxy/v3/views.py b/pulp_ansible/app/galaxy/v3/views.py index c919ad4f8..8898b8f58 100644 --- a/pulp_ansible/app/galaxy/v3/views.py +++ b/pulp_ansible/app/galaxy/v3/views.py @@ -40,6 +40,7 @@ AnsibleDistribution, Collection, CollectionVersion, + CollectionVersionSignature, CollectionImport, ) from pulp_ansible.app.serializers import ( @@ -88,6 +89,8 @@ def get_serializer_context(self): context = super().get_serializer_context() if "path" in self.kwargs: context["path"] = self.kwargs["path"] + if "sigs" in self.kwargs: + context["sigs"] = self.kwargs["sigs"] return context @@ -431,6 +434,14 @@ class CollectionVersionViewSet( lookup_field = "version" + def get_queryset(self): + """ + Returns a CollectionVersions queryset for specified distribution. + """ + distro_content = self._distro_content + self.kwargs["sigs"] = CollectionVersionSignature.objects.filter(pk__in=distro_content) + return CollectionVersion.objects.select_related().filter(pk__in=distro_content) + def get_list_serializer(self, *args, **kwargs): """ Return the list serializer instance. @@ -463,14 +474,6 @@ class UnpaginatedCollectionVersionViewSet(CollectionVersionViewSet): serializer_class = UnpaginatedCollectionVersionSerializer pagination_class = None - def get_queryset(self): - """ - Returns a CollectionVersions queryset for specified distribution. - """ - distro_content = self._distro_content - - return CollectionVersion.objects.select_related().filter(pk__in=distro_content) - def list(self, request, *args, **kwargs): """ Returns paginated CollectionVersions list. diff --git a/pulp_ansible/app/migrations/0038_collectionversionsignature.py b/pulp_ansible/app/migrations/0038_collectionversionsignature.py new file mode 100644 index 000000000..a5177cd4f --- /dev/null +++ b/pulp_ansible/app/migrations/0038_collectionversionsignature.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2.8 on 2021-11-29 21:17 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0077_move_remote_url_credentials'), + ('ansible', '0037_gitremote'), + ] + + operations = [ + migrations.CreateModel( + name='CollectionVersionSignature', + fields=[ + ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='ansible_collectionversionsignature', serialize=False, to='core.content')), + ('data', models.BinaryField()), + ('digest', models.CharField(max_length=64)), + ('pubkey_fingerprint', models.CharField(max_length=64)), + ('signed_collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='signatures', to='ansible.collectionversion')), + ('signing_service', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='signatures', to='core.signingservice')), + ], + options={ + 'default_related_name': '%(app_label)s_%(model_name)s', + 'unique_together': {('pubkey_fingerprint', 'signed_collection')}, + }, + bases=('core.content',), + ), + ] diff --git a/pulp_ansible/app/migrations/0039_collectionremote_signed_only.py b/pulp_ansible/app/migrations/0039_collectionremote_signed_only.py new file mode 100644 index 000000000..096650cbb --- /dev/null +++ b/pulp_ansible/app/migrations/0039_collectionremote_signed_only.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.11 on 2022-01-10 21:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ansible', '0038_collectionversionsignature'), + ] + + operations = [ + migrations.AddField( + model_name='collectionremote', + name='signed_only', + field=models.BooleanField(default=False), + ), + ] diff --git a/pulp_ansible/app/models.py b/pulp_ansible/app/models.py index a2747783e..cf83e22eb 100644 --- a/pulp_ansible/app/models.py +++ b/pulp_ansible/app/models.py @@ -11,6 +11,7 @@ Remote, Repository, Distribution, + SigningService, Task, ) from .downloaders import AnsibleDownloaderFactory @@ -178,6 +179,38 @@ class Meta: ] +class CollectionVersionSignature(Content): + """ + A content type representing a signature that is attached to a content unit. + + Fields: + data (models.BinaryField): A signature, base64 encoded. # Not sure if it is base64 encoded + digest (models.CharField): A signature sha256 digest. + pubkey_fingerprint (models.CharField): A fingerprint of the public key used. + + Relations: + signed_collection (models.ForeignKey): A collection version this signature is relevant to. + signing_service (models.ForeignKey): An optional signing service used for creation. + """ + + PROTECTED_FROM_RECLAIM = False + TYPE = "collection_signature" + + signed_collection = models.ForeignKey( + CollectionVersion, on_delete=models.CASCADE, related_name="signatures" + ) + data = models.BinaryField() # This is what the pulp_container folks used + digest = models.CharField(max_length=64) # does adding a min length improve db efficiency? + pubkey_fingerprint = models.CharField(max_length=64) + signing_service = models.ForeignKey( + SigningService, on_delete=models.SET_NULL, related_name="signatures", null=True + ) + + class Meta: + default_related_name = "%(app_label)s_%(model_name)s" + unique_together = ("pubkey_fingerprint", "signed_collection") + + class RoleRemote(Remote): """ A Remote for Ansible content. @@ -200,6 +233,7 @@ class CollectionRemote(Remote): auth_url = models.CharField(null=True, max_length=255) token = models.TextField(null=True, max_length=2000) sync_dependencies = models.BooleanField(default=True) + signed_only = models.BooleanField(default=False) @property def download_factory(self): @@ -265,7 +299,12 @@ class AnsibleRepository(Repository): """ TYPE = "ansible" - CONTENT_TYPES = [Role, CollectionVersion, AnsibleCollectionDeprecated] + CONTENT_TYPES = [ + Role, + CollectionVersion, + AnsibleCollectionDeprecated, + CollectionVersionSignature, + ] REMOTE_TYPES = [RoleRemote, CollectionRemote] last_synced_metadata_time = models.DateTimeField(null=True) diff --git a/pulp_ansible/app/serializers.py b/pulp_ansible/app/serializers.py index 51ce1859b..6ad3d0329 100644 --- a/pulp_ansible/app/serializers.py +++ b/pulp_ansible/app/serializers.py @@ -4,10 +4,13 @@ from jsonschema import Draft7Validator from rest_framework import serializers -from pulpcore.plugin.models import Artifact, PulpTemporaryFile +from pulpcore.plugin.models import Artifact, PulpTemporaryFile, SigningService from pulpcore.plugin.serializers import ( + DetailRelatedField, ContentChecksumSerializer, ModelSerializer, + NoArtifactContentSerializer, + RelatedField, RemoteSerializer, RepositorySerializer, RepositorySyncURLSerializer, @@ -28,6 +31,7 @@ Collection, CollectionImport, CollectionVersion, + CollectionVersionSignature, CollectionRemote, Role, Tag, @@ -171,6 +175,12 @@ class CollectionRemoteSerializer(RemoteSerializer): default=True, ) + signed_only = serializers.BooleanField( + help_text=_("Sync only collections that have a signature"), + allow_null=True, + default=False, + ) + def validate(self, data): """ Validate collection remote data. @@ -221,6 +231,7 @@ class Meta: "auth_url", "token", "sync_dependencies", + "signed_only", ) model = CollectionRemote @@ -490,6 +501,58 @@ class Meta: model = CollectionVersion +class CollectionVersionSignatureSerializer(NoArtifactContentSerializer): + """ + A serializer for signature models. + """ + + signed_collection = DetailRelatedField( + help_text=_("The content this signature is pointing to."), + view_name_pattern=r"content(-.*/.*)-detail", + queryset=CollectionVersion.objects.all(), + ) + pubkey_fingerprint = serializers.CharField(help_text=_("The fingerprint of the public key.")) + signing_service = RelatedField( + help_text=_("The signing service used to create the signature."), + view_name="signing-services-detail", + queryset=SigningService.objects.all(), + allow_null=True, + ) + + class Meta: + model = CollectionVersionSignature + fields = NoArtifactContentSerializer.Meta.fields + ( + "signed_collection", + "pubkey_fingerprint", + "signing_service", + ) + + +class AnsibleRepositorySignatureSerializer(serializers.Serializer): + """ + A serializer for the signing action. + """ + + content_units = serializers.ListField( + required=True, + help_text=_( + "List of collection version hrefs to sign, use * to sign all content in repository" + ), + ) + signing_service = RelatedField( + required=True, + view_name="signing-services-detail", + queryset=SigningService.objects.all(), + help_text=_("A signing service to use to sign the collections"), + ) + + def validate_content_units(self, value): + """Make sure the list is correctly formatted.""" + if len(value) > 1 and "*" in value: + raise serializers.ValidationError("Cannot supply content units and '*'.") + return value + + class CollectionImportListSerializer(serializers.ModelSerializer): """ A serializer for a CollectionImport list view. diff --git a/pulp_ansible/app/tasks/collections.py b/pulp_ansible/app/tasks/collections.py index 1b0a6874d..2dfbed07e 100644 --- a/pulp_ansible/app/tasks/collections.py +++ b/pulp_ansible/app/tasks/collections.py @@ -1,6 +1,7 @@ import asyncio from collections import defaultdict from gettext import gettext as _ +import hashlib import json import logging import tarfile @@ -54,6 +55,7 @@ CollectionImport, CollectionRemote, CollectionVersion, + CollectionVersionSignature, Tag, ) from pulp_ansible.app.serializers import CollectionVersionSerializer, CollectionRemoteSerializer @@ -484,6 +486,7 @@ def __init__(self, remote, repository, is_repo_remote, deprecation_before_sync, self.collection_info = parse_collections_requirements_file(remote.requirements_file) self.exclude_info = {} self.add_dependents = self.collection_info and self.remote.sync_dependencies + self.signed_only = self.remote.signed_only self.already_synced = set() self._unpaginated_collection_metadata = None self._unpaginated_collection_version_metadata = None @@ -565,6 +568,10 @@ async def _add_collection_version(self, api_version, collection_version_url, met self.already_synced.add(cv_unique) info = metadata["metadata"] + signatures = metadata.get("signatures") + + if self.signed_only and not signatures: + return if self.add_dependents: dependencies = info["dependencies"] @@ -606,6 +613,18 @@ async def _add_collection_version(self, api_version, collection_version_url, met await self.parsing_metadata_progress_bar.aincrement() await self.put(d_content) + if signatures: + collection_version = await d_content.resolution() + for signature in signatures: + sig = signature["signature"].encode("utf8") + cv_signature = CollectionVersionSignature( + signed_collection=collection_version, + data=sig, + digest=hashlib.sha256(sig), + pubkey_fingerprint=signature["pubkey_fingerprint"], + ) + await self.put(DeclarativeContent(content=cv_signature)) + async def _add_collection_version_from_git(self, url, gitref, metadata_only): d_content = await declarative_content_from_git_repo( self.remote, url, gitref, metadata_only=False diff --git a/pulp_ansible/app/tasks/signature.py b/pulp_ansible/app/tasks/signature.py new file mode 100644 index 000000000..ffdbcaf5c --- /dev/null +++ b/pulp_ansible/app/tasks/signature.py @@ -0,0 +1,114 @@ +import asyncio +import json +import tempfile +import hashlib +import gettext + +from pulpcore.plugin.stages import ( + ContentSaver, + DeclarativeContent, + DeclarativeVersion, + Stage, +) + +from pulp_ansible.app.models import ( + AnsibleRepository, + CollectionVersion, + CollectionVersionSignature, +) + +from pulpcore.plugin.models import SigningService, ProgressReport +from pulpcore.plugin.sync import sync_to_async_iterable, sync_to_async + +_ = gettext.gettext + + +def sign(repository_href, content_hrefs, signing_service_href): + """The signing task.""" + repository = AnsibleRepository.objects.get(pk=repository_href) + if content_hrefs == ["*"]: + filtered = repository.latest_version().content.filter( + pulp_type=CollectionVersion.get_pulp_type() + ) + content = CollectionVersion.objects.filter(pk__in=filtered) + else: + content = CollectionVersion.objects.filter(pk__in=content_hrefs) + signing_service = SigningService.objects.get(pk=signing_service_href) + first_stage = CollectionSigningFirstStage(content, signing_service) + SigningDeclarativeVersion(first_stage, repository).create() + + +class SigningDeclarativeVersion(DeclarativeVersion): + """Custom signature pipeline.""" + + def pipeline_stages(self, new_version): + """The stages for the signing process.""" + pipeline = [ + self.first_stage, # CollectionSigningFirstStage + ContentSaver(), + ] + return pipeline + + +class CollectionSigningFirstStage(Stage): + """ + This stage signs the content and creates CollectionVersionSignatures. + """ + + def __init__(self, content, signing_service): + """Initialize Signing first stage.""" + super().__init__() + self.content = content + self.signing_service = signing_service + # Maybe make this value a pulp_ansible constant setting? + self.semaphore = asyncio.Semaphore(10) + + async def sign_collection_version(self, collection_version): + """Signs the collection version.""" + # Limits the number of subprocesses spawned/running at one time + # Should everything be under the semaphore or just the signing part? + async with self.semaphore: + # We use the manifest to create the signature + with tempfile.TemporaryDirectory() as d: + # OpenPGP doesn't take filename into account for signatures, not sure about others + with open(f"{d}/MANIFEST.json", "w") as t: + json.dump(collection_version.manifest, t) + result = await self.signing_service.asign(t.name) + # Is it always a guarantee that the result will have a signature where the file is? + with open(result["signature"], "rb") as sig: + data = sig.read() + cv_signature = CollectionVersionSignature( + data=data, + digest=hashlib.sha256(data), + signed_collection=collection_version, + pubkey_fingerprint=self.signing_service.pubkey_fingerprint, + signing_service=self.signing_service, + ) + dc = DeclarativeContent(content=cv_signature) + await self.progress_report.aincrement() + await self.put(dc) + + async def run(self): + """Signs collections if they have not been signed with key.""" + tasks = [] + # Filter out any content that already has a signature with pubkey_fingerprint + current_signatures = CollectionVersionSignature.objects.filter( + pubkey_fingerprint=self.signing_service.pubkey_fingerprint + ) + new_content = self.content.exclude(signatures__in=current_signatures) + ntotal = await sync_to_async(new_content.count)() + nmsg = _("Signing new CollectionVersions") + async with ProgressReport(message=nmsg, code="sign.new.signature", total=ntotal) as p: + self.progress_report = p + async for collection_version in sync_to_async_iterable(new_content.iterator()): + tasks.append(asyncio.create_task(self.sign_collection_version(collection_version))) + await asyncio.gather(*tasks) + + # Add any signatures already present in Pulp if part of content list + present_content = current_signatures.filter(signed_collection__in=self.content) + ptotal = await sync_to_async(present_content.count)() + pmsg = _("Adding present CollectionVersionSignatures") + async with ProgressReport(message=pmsg, code="sign.present.signature", total=ptotal) as np: + async for signature in sync_to_async_iterable(present_content.iterator()): + await np.aincrement() + await self.put(DeclarativeContent(content=signature)) diff --git a/pulp_ansible/app/viewsets.py b/pulp_ansible/app/viewsets.py index 3bbf3710e..d61ed8398 100644 --- a/pulp_ansible/app/viewsets.py +++ b/pulp_ansible/app/viewsets.py @@ -24,6 +24,7 @@ ContentViewSet, NamedModelViewSet, OperationPostponedResponse, + ReadOnlyContentViewSet, RemoteViewSet, RepositoryViewSet, RepositoryVersionViewSet, @@ -38,6 +39,7 @@ AnsibleRepository, Collection, CollectionVersion, + CollectionVersionSignature, CollectionRemote, Role, Tag, @@ -48,8 +50,10 @@ RoleRemoteSerializer, AnsibleRepositorySerializer, AnsibleRepositorySyncURLSerializer, + AnsibleRepositorySignatureSerializer, CollectionSerializer, CollectionVersionSerializer, + CollectionVersionSignatureSerializer, CollectionVersionUploadSerializer, CollectionRemoteSerializer, CollectionOneShotSerializer, @@ -61,6 +65,7 @@ from .tasks.copy import copy_content from .tasks.roles import synchronize as role_sync from .tasks.git import synchronize as git_sync +from .tasks.signature import sign class RoleFilter(ContentFilter): @@ -208,6 +213,31 @@ def create(self, request): return OperationPostponedResponse(async_result, request) +class SignatureFilter(ContentFilter): + """ + A filter for signatures. + """ + + class Meta: + model = CollectionVersionSignature + fields = { + "signed_collection": ["exact"], + "pubkey_fingerprint": ["exact", "in"], + "signing_service": ["exact"], + } + + +class CollectionVersionSignatureViewSet(ReadOnlyContentViewSet): + """ + ViewSet for looking at signature objects for CollectionVersion content. + """ + + endpoint_name = "collection_signatures" + filterset_class = SignatureFilter + queryset = CollectionVersionSignature.objects.all() + serializer_class = CollectionVersionSignatureSerializer + + class CollectionDeprecatedViewSet(ContentViewSet): """ ViewSet for AnsibleCollectionDeprecated. @@ -290,6 +320,45 @@ def sync(self, request, pk): ) return OperationPostponedResponse(result, request) + @extend_schema( + description="Trigger an asynchronous task to sign Ansible content.", + responses={202: AsyncOperationResponseSerializer}, + ) + @action(detail=True, methods=["post"], serializer_class=AnsibleRepositorySignatureSerializer) + def sign(self, request, pk): + """ + Dispatches a sync task. + """ + content_units = {} + + repository = self.get_object() + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + signing_service = serializer.validated_data["signing_service"] + content = serializer.validated_data["content_units"] + + if "*" in content: + content_units = ["*"] + else: + for url in content: + content_units[NamedModelViewSet.extract_pk(url)] = url + content_units_pks = set(content_units.keys()) + existing_content_units = CollectionVersion.objects.filter(pk__in=content_units_pks) + self.verify_content_units(existing_content_units, content_units) + content_units = list(content_units.keys()) + + result = dispatch( + sign, + exclusive_resources=[repository], + kwargs={ + "repository_href": repository.pk, + "content_hrefs": content_units, + "signing_service_href": signing_service.pk, + }, + ) + return OperationPostponedResponse(result, request) + class AnsibleRepositoryVersionViewSet(RepositoryVersionViewSet): """ diff --git a/pulp_ansible/tests/assets/sign-metadata.sh b/pulp_ansible/tests/assets/sign-metadata.sh new file mode 100755 index 000000000..041e64917 --- /dev/null +++ b/pulp_ansible/tests/assets/sign-metadata.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +FILE_PATH=$1 +SIGNATURE_PATH="$1.asc" + +GPG_KEY_ID="Pulp QE" + +# Create a detached signature +gpg --quiet --batch --homedir ~/.gnupg/ --detach-sign --local-user "${GPG_KEY_ID}" \ + --armor --output ${SIGNATURE_PATH} ${FILE_PATH} + +# Check the exit status +STATUS=$? +if [[ ${STATUS} -eq 0 ]]; then + echo {\"file\": \"${FILE_PATH}\", \"signature\": \"${SIGNATURE_PATH}\"} +else + exit ${STATUS} +fi diff --git a/pulp_ansible/tests/functional/api/collection/test_signatures.py b/pulp_ansible/tests/functional/api/collection/test_signatures.py new file mode 100644 index 000000000..4a6c7d1b8 --- /dev/null +++ b/pulp_ansible/tests/functional/api/collection/test_signatures.py @@ -0,0 +1,199 @@ +"""Tests functionality around Collection-Version Signatures.""" +from pulp_smash.pulp3.bindings import delete_orphans, monitor_task +from pulp_ansible.tests.functional.utils import ( + create_signing_service, + delete_signing_service, + gen_repo, + gen_ansible_remote, + gen_distribution, + get_content, + SyncHelpersMixin, + TestCaseUsingBindings, + skip_if, +) +from pulp_ansible.tests.functional.constants import TEST_COLLECTION_CONFIGS +from orionutils.generator import build_collection +from pulpcore.client.pulp_ansible import AnsibleCollectionsApi, ContentCollectionSignaturesApi +from pulp_ansible.tests.functional.utils import set_up_module as setUpModule # noqa:F401 + + +class CRUDCollectionVersionSignatures(TestCaseUsingBindings, SyncHelpersMixin): + """ + CRUD CollectionVersionSignatures. + + This test targets the following issues: + + * `Pulp #757 `_ + * `Pulp #758 `_ + """ + + @classmethod + def setUpClass(cls): + """Sets up signing service used for creating signatures.""" + super().setUpClass() + delete_orphans() + cls.sign_service = create_signing_service() + cls.collections = [] + cls.signed_collections = [] + cls.repo = {} + cls.sig_api = ContentCollectionSignaturesApi(cls.client) + col_api = AnsibleCollectionsApi(cls.client) + for i in range(4): + collection = build_collection("skeleton", config=TEST_COLLECTION_CONFIGS[i]) + response = col_api.upload_collection(collection.filename) + task = monitor_task(response.task) + cls.collections.append(task.created_resources[0]) + + @classmethod + def tearDownClass(cls): + """Deletes repository and removes any content and signatures.""" + monitor_task(cls.repo_api.delete(cls.repo["pulp_href"]).task) + delete_signing_service(cls.sign_service.name) + delete_orphans() + + def test_01_create_signed_collections(self): + """Test collection signatures can be created through the sign task.""" + repo = self.repo_api.create(gen_repo()) + body = {"add_content_units": self.collections} + monitor_task(self.repo_api.modify(repo.pulp_href, body).task) + + body = {"content_units": self.collections, "signing_service": self.sign_service.pulp_href} + monitor_task(self.repo_api.sign(repo.pulp_href, body).task) + repo = self.repo_api.read(repo.pulp_href) + self.repo.update(repo.to_dict()) + + self.assertEqual(int(repo.latest_version_href[-2]), 2) + content_response = get_content(self.repo) + self.assertIn("ansible.collection_signature", content_response) + self.assertEqual(len(content_response["ansible.collection_signature"]), 4) + self.signed_collections.extend(content_response["ansible.collection_signature"]) + + @skip_if(bool, "signed_collections", False) + def test_02_read_signed_collection(self): + """Test that a collection's signature can be read.""" + signature = self.sig_api.read(self.signed_collections[0]["pulp_href"]) + self.assertIn(signature.signed_collection, self.collections) + self.assertEqual(signature.signing_service, self.sign_service.pulp_href) + + @skip_if(bool, "signed_collections", False) + def test_03_read_signed_collections(self): + """Test that collection signatures can be listed.""" + signatures = self.sig_api.list(repository_version=self.repo["latest_version_href"]) + self.assertEqual(signatures.count, len(self.signed_collections)) + signature_set = set([s.pulp_href for s in signatures.results]) + self.assertEqual(signature_set, {s["pulp_href"] for s in self.signed_collections}) + + @skip_if(bool, "signed_collections", False) + def test_04_partially_update(self): + """Attempt to update a content unit using HTTP PATCH. + + This HTTP method is not supported and a HTTP exception is expected. + """ + attrs = {"pubkey_fingerprint": "testing"} + with self.assertRaises(AttributeError) as exc: + self.sig_api.partial_update(self.signed_collections[0], attrs) + msg = "object has no attribute 'partial_update'" + self.assertIn(msg, exc.exception.args[0]) + + @skip_if(bool, "signed_collections", False) + def test_05_fully_update(self): + """Attempt to update a content unit using HTTP PUT. + + This HTTP method is not supported and a HTTP exception is expected. + """ + attrs = {"pubkey_fingerprint": "testing"} + with self.assertRaises(AttributeError) as exc: + self.sig_api.update(self.signed_collections[0]["pulp_href"], attrs) + msg = "object has no attribute 'update'" + self.assertIn(msg, exc.exception.args[0]) + + @skip_if(bool, "signed_collections", False) + def test_06_delete(self): + """Attempt to delete a content unit using HTTP DELETE. + + This HTTP method is not supported and a HTTP exception is expected. + """ + with self.assertRaises(AttributeError) as exc: + self.sig_api.delete(self.signed_collections[0]["pulp_href"]) + msg = "object has no attribute 'delete'" + self.assertIn(msg, exc.exception.args[0]) + + @skip_if(bool, "signed_collections", False) + def test_07_duplicate(self): + """Attempt to create a signature duplicate.""" + body = {"content_units": self.collections, "signing_service": self.sign_service.pulp_href} + result = monitor_task(self.repo_api.sign(self.repo["pulp_href"], body).task) + repo = self.repo_api.read(self.repo["pulp_href"]) + + self.assertEqual(repo.latest_version_href, self.repo["latest_version_href"]) + self.assertEqual(len(result.created_resources), 0) + + +class CollectionSignatureSyncing(TestCaseUsingBindings, SyncHelpersMixin): + """ + Tests for syncing Collections Signatures. + + This test targets the following issues: + + * `Pulp #748 `_ + """ + + @classmethod + def setUpClass(cls): + """Set up two distros for the sync tests.""" + super().setUpClass() + collections = [] + cls.signing_service = create_signing_service() + col_api = AnsibleCollectionsApi(cls.client) + for i in range(4): + collection = build_collection("skeleton", config=TEST_COLLECTION_CONFIGS[i]) + response = col_api.upload_collection(collection.filename) + task = monitor_task(response.task) + collections.append(task.created_resources[0]) + + cls.repo = cls.repo_api.create(gen_repo()) + body = {"add_content_units": collections} + monitor_task(cls.repo_api.modify(cls.repo.pulp_href, body).task) + + body = {"content_units": collections[:2], "signing_service": cls.signing_service.pulp_href} + monitor_task(cls.repo_api.sign(cls.repo.pulp_href, body).task) + body = {"content_units": collections[2:], "signing_service": cls.signing_service.pulp_href} + monitor_task(cls.repo_api.sign(cls.repo.pulp_href, body).task) + + body = gen_distribution(repository=cls.repo.pulp_href) + distro_href = monitor_task(cls.distributions_api.create(body).task).created_resources[0] + cls.distro1 = cls.distributions_api.read(distro_href) + body = gen_distribution(repository_version=f"{cls.repo.versions_href}2/") + distro_href = monitor_task(cls.distributions_api.create(body).task).created_resources[0] + cls.distro2 = cls.distributions_api.read(distro_href) + + @classmethod + def tearDownClass(cls): + """Destroys the distros and repos used for the tests.""" + monitor_task(cls.repo_api.delete(cls.repo.pulp_href).task) + monitor_task(cls.distributions_api.delete(cls.distro1.pulp_href).task) + monitor_task(cls.distributions_api.delete(cls.distro2.pulp_href).task) + delete_signing_service(cls.signing_service.name) + delete_orphans() + + def test_sync_signatures(self): + """Test that signatures are also synced.""" + body = gen_ansible_remote(self.distro1.client_url) + remote = self.remote_collection_api.create(body) + self.addCleanup(self.remote_collection_api.delete, remote.pulp_href) + repo = self._create_repo_and_sync_with_remote(remote) + + content_response = get_content(repo.to_dict()) + self.assertEqual(len(content_response["ansible.collection_version"]), 4) + self.assertEqual(len(content_response["ansible.collection_signature"]), 4) + + def test_sync_signatures_only(self): + """Test that only collections with a signatures are synced when specified.""" + body = gen_ansible_remote(self.distro2.client_url, signed_only=True) + remote = self.remote_collection_api.create(body) + self.addCleanup(self.remote_collection_api.delete, remote.pulp_href) + repo = self._create_repo_and_sync_with_remote(remote) + + content_response = get_content(repo.to_dict()) + self.assertEqual(len(content_response["ansible.collection_version"]), 2) + self.assertEqual(len(content_response["ansible.collection_signature"]), 2) diff --git a/pulp_ansible/tests/functional/utils.py b/pulp_ansible/tests/functional/utils.py index 71cce2e38..1acc5ae3c 100644 --- a/pulp_ansible/tests/functional/utils.py +++ b/pulp_ansible/tests/functional/utils.py @@ -2,9 +2,10 @@ from dynaconf import Dynaconf from functools import partial from time import sleep +import os import unittest -from pulp_smash import api, config, selectors +from pulp_smash import api, cli, config, selectors, utils from pulp_smash.pulp3.bindings import delete_orphans, monitor_task, PulpTestCase from pulp_smash.pulp3.utils import ( gen_distribution, @@ -28,6 +29,7 @@ from pulpcore.client.pulpcore import ( ApiClient as CoreApiClient, TasksApi, + SigningServicesApi, ) from pulpcore.client.pulp_ansible import ( ApiClient as AnsibleApiClient, @@ -130,6 +132,7 @@ def populate_pulp(cfg, url=ANSIBLE_FIXTURE_URL): core_client = CoreApiClient(configuration) tasks = TasksApi(core_client) +signing = SigningServicesApi(core_client) def wait_tasks(): @@ -268,3 +271,41 @@ def get_psql_smash_cmd(sql_statement): password = settings.DATABASES["default"]["PASSWORD"] dbname = settings.DATABASES["default"]["NAME"] return ("psql", "-c", sql_statement, f"postgresql://{user}:{password}@{host}/{dbname}") + + +def create_signing_service(): + """ + Generates an AsciiArmoredSigningService using the signing script in /assets. + + The signing script requires a GPG key and environment variables to be set in order to work. + TEST_PULP_SIGNING_SCRIPT - Where the signing script exists + TEST_PULP_SIGNING_KEY_ID - The signing key id used by the signing script + """ + cli_client = cli.Client(cfg) + name = utils.uuid4() + script = os.environ.get("TEST_PULP_SIGNING_SCRIPT", "/var/lib/pulp/sign-metadata.sh") + key_id = os.environ.get("TEST_PULP_SIGNING_KEY_ID", "Pulp QE") + cmd = ( + "pulpcore-manager", + "add-signing-service", + name, + script, + key_id, + ) + stdout = cli_client.run(cmd).stdout + if "Successfully added signing service" not in stdout: + raise Exception("Failed to create a signing service") + results = signing.list(name=name) + return results.results[0] + + +def delete_signing_service(name): + """ + Deletes a signing service on the test machine. + """ + python_cmd = ( + "from pulpcore.app.models import SigningService", + f"SigningService.objects.get(name='{name}').delete()", + ) + cli_client = cli.Client(cfg) + utils.execute_pulpcore_python(cli_client, "\n".join(python_cmd))