diff --git a/CHANGES/872.feature b/CHANGES/872.feature new file mode 100644 index 000000000..d453c4070 --- /dev/null +++ b/CHANGES/872.feature @@ -0,0 +1 @@ +Fixed the ``create_repositories=True`` parameter for importing content. diff --git a/pulp_deb/app/modelresource.py b/pulp_deb/app/modelresource.py index 4c5297603..db90aa98e 100644 --- a/pulp_deb/app/modelresource.py +++ b/pulp_deb/app/modelresource.py @@ -1,5 +1,10 @@ +from import_export import fields +from import_export.widgets import ForeignKeyWidget + from pulpcore.plugin.importexport import BaseContentResource +from pulpcore.plugin.modelresources import RepositoryResource from pulp_deb.app.models import ( + AptRepository, GenericContent, InstallerFileIndex, Package, @@ -98,6 +103,42 @@ class PackageReleaseComponentResource(BaseContentResource): Resource for import/export of apt_packagereleasecomponent entities. """ + package = fields.Field( + column_name="package", + attribute="package", + widget=ForeignKeyWidget(Package, "sha256"), + ) + + def before_import_row(self, row, **kwargs): + """ + Finds and sets release_component using upstream_id. + + Args: + row (tablib.Dataset row): incoming import-row representing a single PRC. + kwargs: args passed along from the import() call. + """ + super().before_import_row(row, **kwargs) + + release_component = ReleaseComponent.objects.get(upstream_id=row["release_component"]) + row["release_component"] = str(release_component.pk) + + def set_up_queryset(self): + """ + Set up a queryset for RepositoryVersion PackageReleaseComponents. + + Returns: + django.db.models.QuerySet: The PackageReleaseComponents contained in the repo version. + """ + release_component = ReleaseComponent.objects.filter( + pk__in=self.repo_version.content + ).first() + if release_component: + return PackageReleaseComponent.objects.filter( + release_component=release_component + ).order_by("pulp_id") + else: + return PackageReleaseComponent.objects.none() + class Meta: model = PackageReleaseComponent import_id_fields = model.natural_key_fields() @@ -113,15 +154,35 @@ class Meta: import_id_fields = model.natural_key_fields() +class AptRepositoryResource(RepositoryResource): + """ + A resource for import/export Deb repository entities. + """ + + def set_up_queryset(self): + """ + Set up a queryset for DebRepositories. + + Returns: + A queryset containing one repository that will be exported. + """ + return AptRepository.objects.filter(pk=self.repo_version.repository) + + class Meta: + model = AptRepository + exclude = RepositoryResource.Meta.exclude + ("most_recent_version",) + + IMPORT_ORDER = [ + AptRepositoryResource, + PackageResource, + InstallerPackageResource, + ReleaseResource, InstallerFileIndexResource, ReleaseArchitectureResource, ReleaseComponentResource, ReleaseFileResource, - PackageReleaseComponentResource, - ReleaseResource, - PackageResource, - InstallerPackageResource, PackageIndexResource, + PackageReleaseComponentResource, GenericContentResource, ] diff --git a/pulp_deb/tests/functional/api/test_pulpexport_pulpimport.py b/pulp_deb/tests/functional/api/test_pulpexport_pulpimport.py index ea96a9751..25535db27 100644 --- a/pulp_deb/tests/functional/api/test_pulpexport_pulpimport.py +++ b/pulp_deb/tests/functional/api/test_pulpexport_pulpimport.py @@ -103,7 +103,9 @@ def deb_importer_factory( ): """A fixture that creates a pulp importer.""" - def _deb_importer_factory(import_repos=None, export_repos=None, name=None, mapping=None): + def _deb_importer_factory( + import_repos=None, export_repos=None, name=None, mapping=None, is_mapped=True + ): """Creates a pulp importer. :param import_repos: (Optional) List of already defined import repositories @@ -112,24 +114,24 @@ def _deb_importer_factory(import_repos=None, export_repos=None, name=None, mappi :param mapping: (Optional) Mapped import repositories :returns: A pulp importer set up with name and mapped import repositories """ - _import_repos, _export_repos = deb_gen_import_export_repos(import_repos, export_repos) if not name: name = str(uuid4()) - if not mapping: - mapping = {} - if not import_repos: - import_repos = _import_repos - if not export_repos: - export_repos = _export_repos + body = {"name": name} - for idx, repo in enumerate(export_repos): - mapping[repo.name] = import_repos[idx].name + if is_mapped: + _import_repos, _export_repos = deb_gen_import_export_repos(import_repos, export_repos) + if not mapping: + mapping = {} + if not import_repos: + import_repos = _import_repos + if not export_repos: + export_repos = _export_repos + + for idx, repo in enumerate(export_repos): + mapping[repo.name] = import_repos[idx].name + body["repo_mapping"] = mapping - body = { - "name": name, - "repo_mapping": mapping, - } return gen_object_with_cleanup(importers_pulp_api_client, body) return _deb_importer_factory @@ -145,7 +147,13 @@ def deb_perform_import( """A fixture that performs an import with a PulpImporter.""" def _deb_perform_import( - importer, import_repos=None, export_repos=None, is_chunked=False, an_export=None, body=None + importer, + import_repos=None, + export_repos=None, + is_chunked=False, + an_export=None, + body=None, + generate_export=True, ): """Performs an import with a PulpImporter. @@ -160,20 +168,25 @@ def _deb_perform_import( if body is None: body = {} - if not an_export: - if not (import_repos or export_repos): - import_repos, export_repos = deb_gen_import_export_repos - - an_export = deb_create_export(import_repos, export_repos, is_chunked=is_chunked) - - if is_chunked: - filenames = [f for f in list(an_export.output_file_info.keys()) if f.endswith("json")] - if "toc" not in body: - body["toc"] = filenames[0] - else: - filenames = [f for f in list(an_export.output_file_info.keys()) if f.endswith("tar.gz")] - if "path" not in body: - body["path"] = filenames[0] + if generate_export: + if not an_export: + if not (import_repos or export_repos): + import_repos, export_repos = deb_gen_import_export_repos() + + an_export = deb_create_export(import_repos, export_repos, is_chunked=is_chunked) + + if is_chunked: + filenames = [ + f for f in list(an_export.output_file_info.keys()) if f.endswith("json") + ] + if "toc" not in body: + body["toc"] = filenames[0] + else: + filenames = [ + f for f in list(an_export.output_file_info.keys()) if f.endswith("tar.gz") + ] + if "path" not in body: + body["path"] = filenames[0] import_response = importers_pulp_imports_api_client.create(importer.pulp_href, body) task_group = monitor_task_group(import_response.task_group) @@ -233,3 +246,63 @@ def test_export(deb_create_exporter, deb_create_export, deb_gen_import_export_re assert export.output_file_info is not None for an_export_filename in export.output_file_info.keys(): assert "//" not in an_export_filename + + +def test_import_create_repos( + apt_repository_api, + deb_cleanup_all_orphans, + deb_create_exporter, + deb_create_export, + deb_delete_remote, + deb_delete_repository, + deb_importer_factory, + deb_init_and_sync, + deb_perform_import, + exporters_pulp_api_client, + delete_orphans_pre, +): + """Test whether PulpImporter can create repositories.""" + entity_map = {} + repo, remote = deb_init_and_sync(remote_args={"policy": "immediate"}) + entity_map["repo"] = repo + entity_map["remote"] = remote + + # Create an exporter and remember the export path + exporter = deb_create_exporter(export_repos=[repo]) + entity_map["exporter-path"] = exporter.path + + # Export the repos and remember the export file name + export = deb_create_export(exporter=exporter) + filenames = [f for f in list(export.output_file_info.keys()) if f.endswith("tar.gz")] + entity_map["export-filename"] = filenames[0] + + # Assure that the exported_resources matches the count of repos + assert len(exporter.repositories) == len(export.exported_resources) + assert export.output_file_info is not None + + for an_export_filename in export.output_file_info.keys(): + assert "//" not in an_export_filename + + # Clean up exporter, repos and orphans + exporters_pulp_api_client.delete(exporter.pulp_href) + deb_delete_remote(remote) + deb_delete_repository(repo) + deb_cleanup_all_orphans() + + # Remember the amount of repositories present before the import + existing_repos = apt_repository_api.list().count + + # Create an importer and import the export files and create repositories + importer = deb_importer_factory(is_mapped=False) + body = {"path": entity_map["export-filename"], "create_repositories": True} + import_task_group = deb_perform_import(importer, body=body, generate_export=False) + + # Verify that 1 import and 1 repository was created + assert import_task_group.completed == 2 + + # Find the repository + repo = apt_repository_api.list(name=entity_map["repo"].name).results[0] + + # Inspect the results + assert repo.latest_version_href.endswith("/versions/1/") + assert apt_repository_api.list().count == existing_repos + 1 diff --git a/pulp_deb/tests/functional/conftest.py b/pulp_deb/tests/functional/conftest.py index 90416168c..1b3701333 100644 --- a/pulp_deb/tests/functional/conftest.py +++ b/pulp_deb/tests/functional/conftest.py @@ -569,6 +569,20 @@ def _deb_get_fixture_server_url(repo_name=DEB_FIXTURE_STANDARD_REPOSITORY_NAME): return _deb_get_fixture_server_url +@pytest.fixture +def deb_cleanup_all_orphans(monitor_task, orphans_cleanup_api_client): + """A fixture that will clean up all orphans by setting protection time to zero.""" + + def _deb_cleanup_all_orphans(): + """Removes all orphans by setting protection time to zero. + + :returns: Task of the operation. + """ + return monitor_task(orphans_cleanup_api_client.cleanup({"orphan_protection_time": 0}).task) + + return _deb_cleanup_all_orphans + + @pytest.fixture def deb_init_and_sync( apt_repository_api,