Fix improper signatures

fixes: pulp#841

.github/workflows/scripts/post_before_script.sh

@@ -17,8 +17,10 @@ cmd_stdin_prefix bash -c "cat > /var/lib/pulp/sign-metadata.sh" < "$GITHUB_WORKS
 
 cmd_prefix bash -c "curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-PRIVATE-KEY-pulp-qe | gpg --import"
 cmd_prefix bash -c "curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-KEY-pulp-qe | cat > /tmp/GPG-KEY-pulp-qe"
+curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-KEY-pulp-qe | cat > /tmp/GPG-KEY-pulp-qe
 cmd_prefix chmod a+x /var/lib/pulp/sign-metadata.sh
 
 KEY_FINGERPRINT="6EDF301256480B9B801EBA3D05A5E6DA269D9D98"
 TRUST_LEVEL="6"
 echo "$KEY_FINGERPRINT:$TRUST_LEVEL:" | cmd_stdin_prefix gpg --import-ownertrust
+echo "$KEY_FINGERPRINT:$TRUST_LEVEL:" | gpg --import-ownertrust

CHANGES/841.bugfix

@@ -0,0 +1 @@
+Signatures are now properly generated from a collection's MANIFEST.json file.

functest_requirements.txt

@@ -2,3 +2,4 @@ pulp-smash @ git+https://github.com/pulp/pulp-smash.git
 orionutils
 pytest
 dynaconf
+https://github.com/ansible/ansible/archive/devel.tar.gz

pulp_ansible/app/tasks/signature.py

@@ -1,7 +1,7 @@
 import aiofiles
 import asyncio
-import json
 import hashlib
+import tarfile
 from gettext import gettext as _
 
 from pulpcore.plugin.stages import (
@@ -18,8 +18,10 @@
 )
 
 from django.conf import settings
+from django.core.files.storage import default_storage as storage
 from pulpcore.plugin.models import SigningService, ProgressReport
 from pulpcore.plugin.sync import sync_to_async_iterable, sync_to_async
+from pulp_ansible.app.tasks.utils import get_file_obj_from_tarball
 
 
 def sign(repository_href, content_hrefs, signing_service_href):
@@ -71,13 +73,20 @@ async def sign_collection_version(self, collection_version):
         # Limits the number of subprocesses spawned/running at one time
         async with self.semaphore:
             # We use the manifest to create the signature
-            async with aiofiles.tempfile.TemporaryDirectory(dir=".") as d:
-                # OpenPGP doesn't take filename into account for signatures, not sure about others
-                async with aiofiles.open(f"{d}/MANIFEST.json", "w") as t:
-                    await t.write(json.dumps(collection_version.manifest))
-                result = await self.signing_service.asign(t.name)
-                async with aiofiles.open(result["signature"], "rb") as sig:
-                    data = await sig.read()
+            # OpenPGP doesn't take filename into account for signatures, not sure about others
+            async with aiofiles.tempfile.NamedTemporaryFile(dir=".", mode="wb") as m:
+                cartifact = await sync_to_async(
+                    collection_version.contentartifact_set.select_related("artifact").first
+                )()
+                artifact_name = cartifact.artifact.file.name
+                artifact_file = storage.open(artifact_name)
+                with tarfile.open(fileobj=artifact_file, mode="r") as tar:
+                    man = get_file_obj_from_tarball(tar, "MANIFEST.json", artifact_name)
+                    await m.write(man.read())
+                    await m.flush()
+                result = await self.signing_service.asign(m.name)
+            async with aiofiles.open(result["signature"], "rb") as sig:
+                data = await sig.read()
             cv_signature = CollectionVersionSignature(
                 data=data,
                 digest=hashlib.sha256(data),

pulp_ansible/tests/functional/cli/test_collection_install.py

@@ -9,15 +9,21 @@
     AnsibleRepositorySyncURL,
     RemotesCollectionApi,
 )
-from pulp_smash.pulp3.bindings import monitor_task, PulpTestCase
+from pulp_smash.pulp3.bindings import delete_orphans, monitor_task, PulpTestCase
 from pulp_smash.pulp3.utils import gen_distribution, gen_repo
 
 from pulp_ansible.tests.functional.constants import (
     ANSIBLE_DEMO_COLLECTION,
     ANSIBLE_DEMO_COLLECTION_REQUIREMENTS as DEMO_REQUIREMENTS,
     GALAXY_ANSIBLE_BASE_URL,
 )
-from pulp_ansible.tests.functional.utils import gen_ansible_client, gen_ansible_remote
+from pulp_ansible.tests.functional.utils import (
+    create_signing_service,
+    delete_signing_service,
+    gen_ansible_client,
+    gen_ansible_remote,
+    get_client_keyring,
+)
 from pulp_ansible.tests.functional.utils import set_up_module as setUpModule  # noqa:F401
 
 
@@ -27,12 +33,13 @@ class InstallCollectionTestCase(PulpTestCase):
     @classmethod
     def setUpClass(cls):
         """Create class-wide variables."""
+        delete_orphans()
         cls.client = gen_ansible_client()
         cls.repo_api = RepositoriesAnsibleApi(cls.client)
         cls.remote_collection_api = RemotesCollectionApi(cls.client)
         cls.distributions_api = DistributionsAnsibleApi(cls.client)
 
-    def create_install_scenario(self, body, collection_name):
+    def create_install_scenario(self, body):
         """Create Install scenario."""
         repo = self.repo_api.create(gen_repo())
         self.addCleanup(self.repo_api.delete, repo.pulp_href)
@@ -55,10 +62,13 @@ def create_install_scenario(self, body, collection_name):
         distribution = self.distributions_api.read(created_resources[0])
 
         self.addCleanup(self.distributions_api.delete, distribution.pulp_href)
+        return distribution
 
+    def perform_install_test(self, collection_name, distribution, extra_args=""):
+        """Test that the collection can be installed from Pulp."""
         with tempfile.TemporaryDirectory() as temp_dir:
-            cmd = "ansible-galaxy collection install {} -c -s {} -p {}".format(
-                collection_name, distribution.client_url, temp_dir
+            cmd = "ansible-galaxy collection install {} -c -s {} -p {} {}".format(
+                collection_name, distribution.client_url, temp_dir, extra_args
             )
 
             directory = "{}/ansible_collections/{}".format(
@@ -76,4 +86,22 @@ def create_install_scenario(self, body, collection_name):
     def test_install_collection(self):
         """Test whether ansible-galaxy can install a Collection hosted by Pulp."""
         body = gen_ansible_remote(url=GALAXY_ANSIBLE_BASE_URL, requirements_file=DEMO_REQUIREMENTS)
-        self.create_install_scenario(body, ANSIBLE_DEMO_COLLECTION)
+        distribution = self.create_install_scenario(body)
+        self.perform_install_test(ANSIBLE_DEMO_COLLECTION, distribution)
+
+    def test_signature_collection_install(self):
+        """Test whether ansible-galaxy can install a Collection w/ a signature hosted by Pulp."""
+        body = gen_ansible_remote(url=GALAXY_ANSIBLE_BASE_URL, requirements_file=DEMO_REQUIREMENTS)
+        distribution = self.create_install_scenario(body)
+        self.addCleanup(delete_orphans)
+        repository_href = distribution.repository
+        signing_service = create_signing_service()
+        self.addCleanup(delete_signing_service, signing_service.name)
+        # Switch this over to signature upload in the future
+        signing_body = {"signing_service": signing_service.pulp_href, "content_units": ["*"]}
+        monitor_task(self.repo_api.sign(repository_href, signing_body).task)
+        repo = self.repo_api.read(repository_href)
+        self.assertEqual(repo.latest_version_href[-2], "2")
+
+        keyring_arg = f"--keyring {get_client_keyring()}"
+        self.perform_install_test(ANSIBLE_DEMO_COLLECTION, distribution, keyring_arg)

pulp_ansible/tests/functional/utils.py

@@ -3,6 +3,7 @@
 from functools import partial
 from time import sleep
 import os
+import subprocess
 import unittest
 
 from pulp_smash import api, cli, config, selectors, utils
@@ -309,3 +310,23 @@ def delete_signing_service(name):
     )
     cli_client = cli.Client(cfg)
     utils.execute_pulpcore_python(cli_client, "\n".join(python_cmd))
+
+
+def get_client_keyring():
+    """
+    Finds the GPG keyring for the client used in the tests.
+
+    Uses the environment variables to change the default keyring
+    TEST_PULP_CLIENT_KEYRING - The location for the GPG keyring
+    TEST_PULP_SIGNING_KEY_ID - The signing key id used by the signing script
+    """
+    keyring = os.environ.get("TEST_PULP_CLIENT_KEYRING", "~/.gnupg/pubring.kbx")
+    key_id = os.environ.get("TEST_PULP_SIGNING_KEY_ID", "Pulp QE")
+
+    # Check that key_id is in the keyring
+    cmd = ("gpg", "--list-keys", "--keyring", keyring, "--no-default-keyring")
+    stdout = subprocess.run(cmd, capture_output=True, check=True).stdout
+    if key_id not in stdout.decode():
+        raise Exception(f"Key ID: {key_id} not found in keyring: {keyring}")
+
+    return keyring