Fixed creating of repositories when importing content

closes #872

CHANGES/872.feature

@@ -0,0 +1 @@
+Fixed the ``create_repositories=True`` parameter for importing content.

pulp_deb/app/modelresource.py

@@ -1,5 +1,10 @@
+from import_export import fields
+from import_export.widgets import ForeignKeyWidget
+
 from pulpcore.plugin.importexport import BaseContentResource
+from pulpcore.plugin.modelresources import RepositoryResource
 from pulp_deb.app.models import (
+    AptRepository,
     GenericContent,
     InstallerFileIndex,
     Package,
@@ -98,6 +103,42 @@ class PackageReleaseComponentResource(BaseContentResource):
     Resource for import/export of apt_packagereleasecomponent entities.
     """
 
+    package = fields.Field(
+        column_name="package",
+        attribute="package",
+        widget=ForeignKeyWidget(Package, "sha256"),
+    )
+
+    def before_import_row(self, row, **kwargs):
+        """
+        Finds and sets release_component using upstream_id.
+
+        Args:
+            row (tablib.Dataset row): incoming import-row representing a single PRC.
+            kwargs: args passed along from the import() call.
+        """
+        super().before_import_row(row, **kwargs)
+
+        release_component = ReleaseComponent.objects.get(upstream_id=row["release_component"])
+        row["release_component"] = str(release_component.pk)
+
+    def set_up_queryset(self):
+        """
+        Set up a queryset for RepositoryVersion PackageReleaseComponents.
+
+        Returns:
+            django.db.models.QuerySet: The PackageReleaseComponents contained in the repo version.
+        """
+        release_component = ReleaseComponent.objects.filter(
+            pk__in=self.repo_version.content
+        ).first()
+        if release_component:
+            return PackageReleaseComponent.objects.filter(
+                release_component=release_component
+            ).order_by("pulp_id")
+        else:
+            return PackageReleaseComponent.objects.none()
+
     class Meta:
         model = PackageReleaseComponent
         import_id_fields = model.natural_key_fields()
@@ -113,15 +154,35 @@ class Meta:
         import_id_fields = model.natural_key_fields()
 
 
+class AptRepositoryResource(RepositoryResource):
+    """
+    A resource for import/export Deb repository entities.
+    """
+
+    def set_up_queryset(self):
+        """
+        Set up a queryset for DebRepositories.
+
+        Returns:
+            A queryset containing one repository that will be exported.
+        """
+        return AptRepository.objects.filter(pk=self.repo_version.repository)
+
+    class Meta:
+        model = AptRepository
+        exclude = RepositoryResource.Meta.exclude + ("most_recent_version",)
+
+
 IMPORT_ORDER = [
+    AptRepositoryResource,
+    PackageResource,
+    InstallerPackageResource,
+    ReleaseResource,
     InstallerFileIndexResource,
     ReleaseArchitectureResource,
     ReleaseComponentResource,
     ReleaseFileResource,
-    PackageReleaseComponentResource,
-    ReleaseResource,
-    PackageResource,
-    InstallerPackageResource,
     PackageIndexResource,
+    PackageReleaseComponentResource,
     GenericContentResource,
 ]

pulp_deb/tests/functional/api/test_pulpexport_pulpimport.py

@@ -103,7 +103,9 @@ def deb_importer_factory(
 ):
     """A fixture that creates a pulp importer."""
 
-    def _deb_importer_factory(import_repos=None, export_repos=None, name=None, mapping=None):
+    def _deb_importer_factory(
+        import_repos=None, export_repos=None, name=None, mapping=None, is_mapped=True
+    ):
         """Creates a pulp importer.
 
         :param import_repos: (Optional) List of already defined import repositories
@@ -112,24 +114,24 @@ def _deb_importer_factory(import_repos=None, export_repos=None, name=None, mappi
         :param mapping: (Optional) Mapped import repositories
         :returns: A pulp importer set up with name and mapped import repositories
         """
-        _import_repos, _export_repos = deb_gen_import_export_repos(import_repos, export_repos)
         if not name:
             name = str(uuid4())
 
-        if not mapping:
-            mapping = {}
-            if not import_repos:
-                import_repos = _import_repos
-            if not export_repos:
-                export_repos = _export_repos
+        body = {"name": name}
 
-            for idx, repo in enumerate(export_repos):
-                mapping[repo.name] = import_repos[idx].name
+        if is_mapped:
+            _import_repos, _export_repos = deb_gen_import_export_repos(import_repos, export_repos)
+            if not mapping:
+                mapping = {}
+                if not import_repos:
+                    import_repos = _import_repos
+                if not export_repos:
+                    export_repos = _export_repos
+
+                for idx, repo in enumerate(export_repos):
+                    mapping[repo.name] = import_repos[idx].name
+            body["repo_mapping"] = mapping
 
-        body = {
-            "name": name,
-            "repo_mapping": mapping,
-        }
         return gen_object_with_cleanup(importers_pulp_api_client, body)
 
     return _deb_importer_factory
@@ -145,7 +147,13 @@ def deb_perform_import(
     """A fixture that performs an import with a PulpImporter."""
 
     def _deb_perform_import(
-        importer, import_repos=None, export_repos=None, is_chunked=False, an_export=None, body=None
+        importer,
+        import_repos=None,
+        export_repos=None,
+        is_chunked=False,
+        an_export=None,
+        body=None,
+        generate_export=True,
     ):
         """Performs an import with a PulpImporter.
 
@@ -160,20 +168,25 @@ def _deb_perform_import(
         if body is None:
             body = {}
 
-        if not an_export:
-            if not (import_repos or export_repos):
-                import_repos, export_repos = deb_gen_import_export_repos
-
-            an_export = deb_create_export(import_repos, export_repos, is_chunked=is_chunked)
-
-        if is_chunked:
-            filenames = [f for f in list(an_export.output_file_info.keys()) if f.endswith("json")]
-            if "toc" not in body:
-                body["toc"] = filenames[0]
-        else:
-            filenames = [f for f in list(an_export.output_file_info.keys()) if f.endswith("tar.gz")]
-            if "path" not in body:
-                body["path"] = filenames[0]
+        if generate_export:
+            if not an_export:
+                if not (import_repos or export_repos):
+                    import_repos, export_repos = deb_gen_import_export_repos()
+
+                an_export = deb_create_export(import_repos, export_repos, is_chunked=is_chunked)
+
+            if is_chunked:
+                filenames = [
+                    f for f in list(an_export.output_file_info.keys()) if f.endswith("json")
+                ]
+                if "toc" not in body:
+                    body["toc"] = filenames[0]
+            else:
+                filenames = [
+                    f for f in list(an_export.output_file_info.keys()) if f.endswith("tar.gz")
+                ]
+                if "path" not in body:
+                    body["path"] = filenames[0]
 
         import_response = importers_pulp_imports_api_client.create(importer.pulp_href, body)
         task_group = monitor_task_group(import_response.task_group)
@@ -233,3 +246,63 @@ def test_export(deb_create_exporter, deb_create_export, deb_gen_import_export_re
     assert export.output_file_info is not None
     for an_export_filename in export.output_file_info.keys():
         assert "//" not in an_export_filename
+
+
+def test_import_create_repos(
+    apt_repository_api,
+    deb_cleanup_all_orphans,
+    deb_create_exporter,
+    deb_create_export,
+    deb_delete_remote,
+    deb_delete_repository,
+    deb_importer_factory,
+    deb_init_and_sync,
+    deb_perform_import,
+    exporters_pulp_api_client,
+    delete_orphans_pre,
+):
+    """Test whether PulpImporter can create repositories."""
+    entity_map = {}
+    repo, remote = deb_init_and_sync(remote_args={"policy": "immediate"})
+    entity_map["repo"] = repo
+    entity_map["remote"] = remote
+
+    # Create an exporter and remember the export path
+    exporter = deb_create_exporter(export_repos=[repo])
+    entity_map["exporter-path"] = exporter.path
+
+    # Export the repos and remember the export file name
+    export = deb_create_export(exporter=exporter)
+    filenames = [f for f in list(export.output_file_info.keys()) if f.endswith("tar.gz")]
+    entity_map["export-filename"] = filenames[0]
+
+    # Assure that the exported_resources matches the count of repos
+    assert len(exporter.repositories) == len(export.exported_resources)
+    assert export.output_file_info is not None
+
+    for an_export_filename in export.output_file_info.keys():
+        assert "//" not in an_export_filename
+
+    # Clean up exporter, repos and orphans
+    exporters_pulp_api_client.delete(exporter.pulp_href)
+    deb_delete_remote(remote)
+    deb_delete_repository(repo)
+    deb_cleanup_all_orphans()
+
+    # Remember the amount of repositories present before the import
+    existing_repos = apt_repository_api.list().count
+
+    # Create an importer and import the export files and create repositories
+    importer = deb_importer_factory(is_mapped=False)
+    body = {"path": entity_map["export-filename"], "create_repositories": True}
+    import_task_group = deb_perform_import(importer, body=body, generate_export=False)
+
+    # Verify that 1 import and 1 repository was created
+    assert import_task_group.completed == 2
+
+    # Find the repository
+    repo = apt_repository_api.list(name=entity_map["repo"].name).results[0]
+
+    # Inspect the results
+    assert repo.latest_version_href.endswith("/versions/1/")
+    assert apt_repository_api.list().count == existing_repos + 1

pulp_deb/tests/functional/conftest.py

@@ -569,6 +569,20 @@ def _deb_get_fixture_server_url(repo_name=DEB_FIXTURE_STANDARD_REPOSITORY_NAME):
     return _deb_get_fixture_server_url
 
 
+@pytest.fixture
+def deb_cleanup_all_orphans(monitor_task, orphans_cleanup_api_client):
+    """A fixture that will clean up all orphans by setting protection time to zero."""
+
+    def _deb_cleanup_all_orphans():
+        """Removes all orphans by setting protection time to zero.
+
+        :returns: Task of the operation.
+        """
+        return monitor_task(orphans_cleanup_api_client.cleanup({"orphan_protection_time": 0}).task)
+
+    return _deb_cleanup_all_orphans
+
+
 @pytest.fixture
 def deb_init_and_sync(
     apt_repository_api,