Sync excludes collections found at /excludes/ endpoint

fixes: #8194

CHANGES/8194.feature

@@ -0,0 +1 @@
+Syncing now excludes collection versions found at ``/excludes/`` endpoint of remote.

pulp_ansible/app/tasks/collections.py

@@ -19,6 +19,7 @@
 from galaxy_importer.collection import CollectionFilename
 from galaxy_importer.exceptions import ImporterError
 from pkg_resources import Requirement
+from rest_framework.serializers import ValidationError
 
 from pulpcore.plugin.models import (
     Artifact,
@@ -163,6 +164,25 @@ def sync(remote_pk, repository_pk, mirror, optimize):
         ).all().update(version_removed=repo_version)
 
 
+def parse_requirements_entry(requirements_entry):
+    """Parses a `RequirementsFileEntry` and returns a `Requirement` object."""
+    if requirements_entry.version == "*":
+        requirement_version = Requirement.parse("collection")
+    else:
+        # We need specifiers to enforce Requirement object criteria
+        # https://setuptools.readthedocs.io/en/latest/pkg_resources.html#requirements-parsing
+        # https://setuptools.readthedocs.io/en/latest/pkg_resources.html#requirement-methods-and-attributes
+        # If requirements_entry.version is a valid version, adds == specifier to the requirement
+        try:
+            Version(requirements_entry.version)
+            req_to_parse = f"collection=={requirements_entry.version}"
+        except ValueError:
+            req_to_parse = f"collection{requirements_entry.version}"
+
+        requirement_version = Requirement.parse(req_to_parse)
+    return requirement_version
+
+
 def import_collection(
     temp_file_pk,
     repository_pk=None,
@@ -389,6 +409,7 @@ def __init__(self, remote, repository, is_repo_remote, deprecation_before_sync,
         self.deprecation_before_sync = deprecation_before_sync
         self.deprecation_after_sync = set()
         self.collection_info = parse_collections_requirements_file(remote.requirements_file)
+        self.exclude_info = {}
         self.add_dependents = self.collection_info and self.remote.sync_dependencies
         self.already_synced = set()
         self._unpaginated_collection_metadata = None
@@ -463,6 +484,9 @@ async def _add_collection_version(self, api_version, collection_version_url, met
             version=metadata["version"],
         )
         cv_unique = attrgetter("namespace", "name", "version")(collection_version)
+        fullname, version = f"{cv_unique[0]}.{cv_unique[1]}", cv_unique[2]
+        if fullname in self.exclude_info and version in self.exclude_info[fullname]:
+            return
         if cv_unique in self.already_synced:
             return
         self.already_synced.add(cv_unique)
@@ -599,20 +623,7 @@ async def _read_from_downloaded_metadata(self, name, namespace, requirement):
         await asyncio.gather(*tasks)
 
     async def _fetch_collection_metadata(self, requirements_entry):
-        if requirements_entry.version == "*":
-            requirement_version = Requirement.parse("collection")
-        else:
-            # We need specifiers to enforce Requirement object criteria
-            # https://setuptools.readthedocs.io/en/latest/pkg_resources.html#requirements-parsing
-            # https://setuptools.readthedocs.io/en/latest/pkg_resources.html#requirement-methods-and-attributes
-            # If requirements_entry.version is a valid version, adds == specifier to the requirement
-            try:
-                Version(requirements_entry.version)
-                req_to_parse = f"collection=={requirements_entry.version}"
-            except ValueError:
-                req_to_parse = f"collection{requirements_entry.version}"
-
-            requirement_version = Requirement.parse(req_to_parse)
+        requirement_version = parse_requirements_entry(requirements_entry)
 
         namespace, name = requirements_entry.name.split(".")
 
@@ -642,15 +653,32 @@ async def _download_unpaginated_metadata(self):
         root_endpoint, api_version = await self._get_root_api(self.remote.url)
         self._api_version = api_version
         if api_version > 2:
+            loop = asyncio.get_event_loop()
+
             collection_endpoint = f"{root_endpoint}/collections/all/"
-            downloader = self.remote.get_downloader(
+            excludes_endpoint = f"{root_endpoint}/excludes/"
+            col_downloader = self.remote.get_downloader(
                 url=collection_endpoint, silence_errors_for_response_status_codes={404}
             )
-            try:
-                collection_metadata_list = parse_metadata(await downloader.run())
-            except FileNotFoundError:
-                pass
-            else:
+            exc_downloader = self.remote.get_downloader(
+                url=excludes_endpoint, silence_errors_for_response_status_codes={404}
+            )
+            tasks = [loop.create_task(col_downloader.run()), loop.create_task(exc_downloader.run())]
+            col_results, exc_results = await asyncio.gather(*tasks, return_exceptions=True)
+
+            if not isinstance(exc_results, FileNotFoundError):
+                excludes_response = parse_metadata(exc_results)
+                if excludes_response:
+                    try:
+                        excludes_list = parse_collections_requirements_file(excludes_response)
+                    except ValidationError:
+                        pass
+                    else:
+                        excludes = {r.name: parse_requirements_entry(r) for r in excludes_list}
+                        self.exclude_info.update(excludes)
+
+            if not isinstance(col_results, FileNotFoundError):
+                collection_metadata_list = parse_metadata(col_results)
                 self._unpaginated_collection_metadata = defaultdict(dict)
                 for collection in collection_metadata_list:
                     namespace = collection["namespace"]

pulp_ansible/app/tasks/utils.py

@@ -73,17 +73,20 @@ def parse_collections_requirements_file(requirements_file_string):
     collection_info = []
 
     if requirements_file_string:
-        try:
-            requirements = yaml.safe_load(requirements_file_string)
-        except YAMLError as err:
-            raise ValidationError(
-                _(
-                    "Failed to parse the collection requirements yml: {file} "
-                    "with the following error: {error}".format(
-                        file=requirements_file_string, error=err
+        if isinstance(requirements_file_string, str):
+            try:
+                requirements = yaml.safe_load(requirements_file_string)
+            except YAMLError as err:
+                raise ValidationError(
+                    _(
+                        "Failed to parse the collection requirements yml: {file} "
+                        "with the following error: {error}".format(
+                            file=requirements_file_string, error=err
+                        )
                     )
                 )
-            )
+        else:
+            requirements = requirements_file_string
 
         if not isinstance(requirements, dict) or "collections" not in requirements:
             raise ValidationError(

pulp_ansible/tests/functional/api/collection/v3/test_sync.py

@@ -220,14 +220,17 @@ def setUpClass(cls):
         cls.distributions_api = DistributionsAnsibleApi(cls.client)
         cls.collections_api = PulpAnsibleApiV3CollectionsApi(cls.client)
         cls.cv_api = ContentCollectionVersionsApi(cls.client)
+        cls.url = "https://cloud.redhat.com/api/automation-hub/"
+        cls.aurl = (
+            "https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token"
+        )
 
     def test_sync_with_token_from_automation_hub(self):
         """Test whether we can sync with an auth token from Automation Hub."""
-        aurl = "https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token"
         body = gen_ansible_remote(
-            url="https://cloud.redhat.com/api/automation-hub/",
+            url=self.url,
             requirements_file="collections:\n  - ansible.posix",
-            auth_url=aurl,
+            auth_url=self.aurl,
             token=os.environ["AUTOMATION_HUB_TOKEN_AUTH"],
             rate_limit=10,
             tls_validation=False,
@@ -242,7 +245,45 @@ def test_sync_with_token_from_automation_hub(self):
         original_content = self.cv_api.list(repository_version=f"{repo.pulp_href}versions/1/")
         self.assertTrue(len(original_content.results) >= 3)  # check that we have at least 3 results
 
+    @unittest.skip("Skipping until synclist no longer creates new repository")
+    def test_syncing_with_excludes_list_from_automation_hub(self):
+        """Test if syncing collections know to be on the synclist will be mirrored."""
+        namespace = "autohubtest2"
+        # Collections are randomly generated, in case of failure, please change the names below:
+        name = "collection_dep_a_zzduuntr"
+        excluded = "collection_dep_a_tworzgsx"
+        excluded_version = "awcrosby.collection_test==2.1.0"
+        requirements = f"""
+        collections:
+          - {namespace}.{name}
+          - {namespace}.{excluded}
+          - {excluded_version}
+        """
+        body = gen_ansible_remote(
+            url=self.url,
+            requirements_file=requirements,
+            auth_url=self.aurl,
+            token=os.environ["AUTOMATION_HUB_TOKEN_AUTH"],
+            rate_limit=10,
+            tls_validation=False,
+            sync_dependencies=False,
+        )
+        remote = self.remote_collection_api.create(body)
+        self.addCleanup(self.remote_collection_api.delete, remote.pulp_href)
+
+        repo = self._create_repo_and_sync_with_remote(remote)
+
+        # Assert that at least one CollectionVersion was downloaded
+        repo_ver = f"{repo.pulp_href}versions/1/"
+        content = self.cv_api.list(repository_version=repo_ver)
+        self.assertTrue(content.count >= 1)
+
+        # Assert that excluded collection was not synced
+        exclude_content = self.cv_api.list(repository_version=repo_ver, name=excluded)
+        self.assertTrue(exclude_content.count == 0)
+
 
+# TODO QA-AH has been deprecated, remove/replace tests
 @unittest.skipUnless(
     "QA_AUTOMATION_HUB_TOKEN_AUTH" in os.environ,
     "'QA_AUTOMATION_HUB_TOKEN_AUTH' env var is not defined",