From 79028cff0cf64627be92e4db93376cf8d983abb0 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Tue, 16 Nov 2021 18:48:18 -0500 Subject: [PATCH] Add container repository content management commands fixes: #422 --- CHANGES/422.feature | 1 + pulpcore/cli/common/generic.py | 26 ++-- pulpcore/cli/container/context.py | 45 +++++- pulpcore/cli/container/repository.py | 145 ++++++++++++++++++- tests/scripts/pulp_ansible/test_content.sh | 6 +- tests/scripts/pulp_container/test_content.sh | 14 ++ tests/scripts/pulp_container/test_copy.sh | 52 +++++++ tests/scripts/pulp_file/test_content.sh | 2 +- 8 files changed, 267 insertions(+), 24 deletions(-) create mode 100644 CHANGES/422.feature create mode 100755 tests/scripts/pulp_container/test_copy.sh diff --git a/CHANGES/422.feature b/CHANGES/422.feature new file mode 100644 index 000000000..18c331f6b --- /dev/null +++ b/CHANGES/422.feature @@ -0,0 +1 @@ +Added container repository content management commands. diff --git a/pulpcore/cli/common/generic.py b/pulpcore/cli/common/generic.py index 02ac3978a..a87971941 100644 --- a/pulpcore/cli/common/generic.py +++ b/pulpcore/cli/common/generic.py @@ -986,8 +986,6 @@ def repository_content_command(**kwargs: Any) -> click.Group: """A factory that creates a repository content command group.""" content_contexts = kwargs.pop("contexts", {}) - names = list(content_contexts.keys()) + ["all"] - content_contexts.update({"all": PulpContentContext}) def version_callback( ctx: click.Context, param: click.Parameter, value: Optional[int] @@ -1002,29 +1000,30 @@ def version_callback( ) return repo_ver_ctx - @click.command("list") - @click.option("-t", "--type", "type", type=click.Choice(names), default=names[0]) + @pulp_command("list") + @click.option("--all-types", is_flag=True) @limit_option @offset_option @repository_option @click.option("--version", type=int, callback=version_callback) @pass_pulp_context + @click.pass_context def content_list( + ctx: click.Context, pulp_ctx: PulpContext, version: PulpRepositoryVersionContext, offset: Optional[int], limit: Optional[int], - type: Optional[str], + all_types: Optional[bool], **params: Any, ) -> None: parameters = {k: v for k, v in params.items() if v is not None} parameters.update({"repository_version": version.pulp_href}) - result = content_contexts[type](pulp_ctx).list( - limit=limit, offset=offset, parameters=parameters - ) + ctx_obj = PulpContentContext(pulp_ctx) if all_types else ctx.obj + result = ctx_obj.list(limit=limit, offset=offset, parameters=parameters) pulp_ctx.output_result(result) - @click.command("add") + @pulp_command("add") @repository_option @click.option("--base-version", type=int, callback=version_callback) @pass_content_context @@ -1039,7 +1038,7 @@ def content_add( base_version=base_version.pulp_href, ) - @click.command("remove") + @pulp_command("remove") @click.option("--all", is_flag=True, help=_("Remove all content from repository version")) @repository_option @click.option("--base-version", type=int, callback=version_callback) @@ -1055,7 +1054,7 @@ def content_remove( repo_ctx.pulp_href, remove_content=remove_content, base_version=base_version.pulp_href ) - @click.command("modify") + @pulp_command("modify") @repository_option @click.option("--base-version", type=int, callback=version_callback) def content_modify( @@ -1077,11 +1076,12 @@ def content_modify( if not kwargs.get("name"): kwargs["name"] = "content" - @click.group(**kwargs) + @pulp_group(**kwargs) @pass_pulp_context @click.pass_context + @type_option(choices=content_contexts) def content_group(ctx: click.Context, pulp_ctx: PulpContext) -> None: - ctx.obj = PulpContentContext(pulp_ctx) + pass for command, options in command_decorators.items(): if options is not None: diff --git a/pulpcore/cli/container/context.py b/pulpcore/cli/container/context.py index 1691837c7..56bfdeb56 100644 --- a/pulpcore/cli/container/context.py +++ b/pulpcore/cli/container/context.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, List, Optional from pulpcore.cli.common.context import ( EntityDefinition, @@ -106,6 +106,49 @@ class PulpContainerRepositoryContext(PulpContainerBaseRepositoryContext): "tag": [PluginRequirement("container", "2.3.0")], } + def preprocess_body(self, body: EntityDefinition) -> EntityDefinition: + body = super().preprocess_body(body) + if "version" in body and "source_repository" in body: + body[ + "source_repository_version" + ] = f"{body.pop('source_repository')}versions/{body.pop('version')}/" + return body + + def modify( + self, + href: str, + add_content: Optional[List[str]] = None, + remove_content: Optional[List[str]] = None, + base_version: Optional[str] = None, + ) -> Any: + if remove_content: + self.call( + "remove", parameters={self.HREF: href}, body={"content_units": remove_content} + ) + if add_content: + self.call("add", parameters={self.HREF: href}, body={"content_units": add_content}) + + def copy_tag(self, source_href: str, version: Optional[str], tags: Optional[List[str]]) -> Any: + body = {"source_repository": source_href, "names": tags, "version": version} + body = self.preprocess_body(body) + return self.call("copy_tags", parameters={self.HREF: self.pulp_href}, body=body) + + def copy_manifest( + self, + source_href: str, + version: Optional[str], + digests: Optional[List[str]], + media_types: Optional[List[str]], + ) -> Any: + body = { + "source_repository": source_href, + "version": version, + "digests": digests, + "media_types": media_types, + } + body = self.preprocess_body(body) + return self.call("copy_manifests", parameters={self.HREF: self.pulp_href}, body=body) + class PulpContainerPushRepositoryContext(PulpContainerBaseRepositoryContext): HREF = "container_container_push_repository_href" diff --git a/pulpcore/cli/container/repository.py b/pulpcore/cli/container/repository.py index 498359024..8db64cd14 100644 --- a/pulpcore/cli/container/repository.py +++ b/pulpcore/cli/container/repository.py @@ -1,10 +1,12 @@ import re -from typing import Any, Dict +from typing import Any, Dict, List, Optional import click from pulpcore.cli.common.context import ( + EntityDefinition, EntityFieldDefinition, + PulpContext, PulpEntityContext, PulpRemoteContext, PulpRepositoryContext, @@ -19,6 +21,7 @@ list_command, name_option, pulp_group, + repository_content_command, repository_href_option, repository_option, resource_option, @@ -29,11 +32,15 @@ version_command, ) from pulpcore.cli.common.i18n import get_translation +from pulpcore.cli.container.content import show_options from pulpcore.cli.container.context import ( PulpContainerBaseRepositoryContext, + PulpContainerBlobContext, + PulpContainerManifestContext, PulpContainerPushRepositoryContext, PulpContainerRemoteContext, PulpContainerRepositoryContext, + PulpContainerTagContext, ) from pulpcore.cli.core.generic import task_command @@ -44,22 +51,62 @@ def _tag_callback(ctx: click.Context, param: click.Parameter, value: str) -> str: if len(value) == 0: - raise click.ClickException("Please pass a non empty tag name.") + raise click.ClickException(_("Please pass a non empty tag name.")) if re.match(VALID_TAG_REGEX, value) is None: - raise click.ClickException("Please pass a valid tag.") + raise click.ClickException(_("Please pass a valid tag.")) return value +def _source_callback(ctx: click.Context, param: click.Parameter, value: str) -> PulpEntityContext: + pulp_ctx = ctx.find_object(PulpContext) + assert pulp_ctx is not None + if len(value) == 0: + raise click.ClickException(_("Please pass in a non empty source repository.")) + + version: Optional[str] = None + pulp_href: Optional[str] = None + entity: Optional[EntityDefinition] = None + if value.startswith("/"): + pattern = rf"^{pulp_ctx.api_path}{PulpContainerRepositoryContext.HREF_PATTERN}" + if re.match(pattern, value) is None: + raise click.ClickException( + _("'{value}' is not a valid href for {option_name}.").format( + value=value, option_name=param.name + ) + ) + pulp_href = value + if "versions" in value: + pulp_href = value.partition("versions")[0] + version = value.split("/")[-2] + else: + split_value = value.split(":", maxsplit=1) + if len(split_value) == 2: + version = split_value[1] + entity = {"name": split_value[0]} + + repo_ctx = PulpContainerRepositoryContext(pulp_ctx, pulp_href=pulp_href, entity=entity) + if version: + # Check that the version makes sense + latest_version = int(repo_ctx.entity["latest_version_href"].split("/")[-2]) + if not (0 < int(version) <= latest_version): + raise click.ClickException( + _( + "Please specify a version that is less then the latest version {} and greater" + " than 0" + ).format(latest_version) + ) + repo_ctx.meta["version"] = version + return repo_ctx + + remote_option = resource_option( "--remote", default_plugin="container", default_type="container", context_table={"container:container": PulpContainerRemoteContext}, href_pattern=PulpRemoteContext.HREF_PATTERN, - help=_( - "Remote used for synching in the form '[[:]:]' or by href." - ), + help=_("Remote used for syncing in the form '[[:]:]' or by href."), ) @@ -83,6 +130,11 @@ def repository() -> None: retained_versions_option, ] create_options = update_options + [click.option("--name", required=True)] +contexts = { + "tag": PulpContainerTagContext, + "manifest": PulpContainerManifestContext, + "blob": PulpContainerBlobContext, +} container_context = (PulpContainerRepositoryContext,) repository.add_command(list_command(decorators=[label_select_option])) @@ -101,6 +153,14 @@ def repository() -> None: repository.add_command(task_command(decorators=nested_lookup_options)) repository.add_command(version_command(decorators=nested_lookup_options)) repository.add_command(label_command(decorators=nested_lookup_options)) +repository.add_command( + repository_content_command( + contexts=contexts, + add_decorators=show_options, + remove_decorators=show_options, + allowed_with_contexts=container_context, + ) +) @repository.command(allowed_with_contexts=container_context) @@ -163,3 +223,76 @@ def add_tag( @pass_repository_context def remove_tag(repository_ctx: PulpContainerBaseRepositoryContext, tag: str) -> None: repository_ctx.untag(tag) + + +@repository.command(allowed_with_contexts=container_context) +@name_option +@href_option +@click.option( + "--source", + help=_("Source repository to copy tags from. Specify in format of href or [:]"), + required=True, + callback=_source_callback, +) +@click.option( + "--tag", + "tags", + help=_("Multiple option of tag names to copy, leave blank to copy all"), + multiple=True, +) +@pass_repository_context +def copy_tag( + repository_ctx: PulpContainerRepositoryContext, + source: PulpContainerRepositoryContext, + tags: List[str], +) -> None: + version = source.meta.get("version") # Set in the source_callback + repository_ctx.copy_tag(source_href=source.pulp_href, version=version, tags=tags or None) + + +@repository.command(allowed_with_contexts=container_context) +@name_option +@href_option +@click.option( + "--source", + help=_( + "Source repository to copy manifests from. Specify in format of href or [:]" + ), + required=True, + callback=_source_callback, +) +@click.option( + "--digest", + "digests", + help=_("Multiple option of manifest digests to copy, leave blank to copy all"), + multiple=True, +) +@click.option( + "--media-type", + "media_types", + help=_("Multiple option of media-types to copy, leave blank to copy all types"), + type=click.Choice( + [ + "application/vnd.docker.distribution.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.oci.image.index.v1+json", + ] + ), + multiple=True, +) +@pass_repository_context +def copy_manifest( + repository_ctx: PulpContainerRepositoryContext, + source: PulpContainerRepositoryContext, + digests: List[str], + media_types: List[str], +) -> None: + version = source.meta.get("version") # Set in the source_callback + repository_ctx.copy_manifest( + source_href=source.pulp_href, + version=version, + digests=digests or None, + media_types=media_types or None, + ) diff --git a/tests/scripts/pulp_ansible/test_content.sh b/tests/scripts/pulp_ansible/test_content.sh index d8de4273d..46292a577 100755 --- a/tests/scripts/pulp_ansible/test_content.sh +++ b/tests/scripts/pulp_ansible/test_content.sh @@ -40,13 +40,13 @@ expect_succ pulp ansible repository create --name "cli_test_ansible_repository" expect_succ pulp ansible repository content add --repository "cli_test_ansible_repository" --name "posix" --namespace "ansible" --version "1.3.0" expect_succ pulp ansible repository content list --repository "cli_test_ansible_repository" --version 1 test "$(echo "$OUTPUT" | jq -r length)" -eq "1" -expect_succ pulp ansible repository content add --repository "cli_test_ansible_repository" --type "role" --name "kubernetes-modules" --namespace "ansible" --version "0.0.1" -expect_succ pulp ansible repository content list --repository "cli_test_ansible_repository" --version 2 --type "role" +expect_succ pulp ansible repository content --type "role" add --repository "cli_test_ansible_repository" --name "kubernetes-modules" --namespace "ansible" --version "0.0.1" +expect_succ pulp ansible repository content --type "role" list --repository "cli_test_ansible_repository" --version 2 test "$(echo "$OUTPUT" | jq -r length)" -eq "1" if pulp debug has-plugin --name "core" --min-version "3.11.0" then - expect_succ pulp ansible repository content list --repository "cli_test_ansible_repository" --version 2 --type "all" + expect_succ pulp ansible repository content list --repository "cli_test_ansible_repository" --version 2 --all-types test "$(echo "$OUTPUT" | jq -r length)" -eq "2" fi diff --git a/tests/scripts/pulp_container/test_content.sh b/tests/scripts/pulp_container/test_content.sh index e947601b2..cc441125f 100755 --- a/tests/scripts/pulp_container/test_content.sh +++ b/tests/scripts/pulp_container/test_content.sh @@ -45,3 +45,17 @@ tag_digest="$(echo "$OUTPUT" | jq -r .digest)" expect_succ pulp container content -t tag show --name "$tag_name" --digest "$tag_digest" test "$(echo "$OUTPUT" | jq -r .pulp_href)" = "$tag_href" + +# Test repository content commands +expect_succ pulp container repository content list --repository "cli_test_container_repository" --all-types +expect_succ pulp container repository content --type "tag" list --repository "cli_test_container_repository" +expect_succ pulp container repository content --type "manifest" list --repository "cli_test_container_repository" +expect_succ pulp container repository content --type "blob" list --repository "cli_test_container_repository" + +expect_succ pulp container repository content --type "blob" remove --repository "cli_test_container_repository" --digest "$blob_digest" +expect_succ pulp container repository content --type "manifest" remove --repository "cli_test_container_repository" --digest "$manifest_digest" +expect_succ pulp container repository content --type "tag" remove --repository "cli_test_container_repository" --name "$tag_name" --digest "$tag_digest" + +expect_succ pulp container repository content add --repository "cli_test_container_repository" --href "$blob_href" +expect_succ pulp container repository content add --repository "cli_test_container_repository" --href "$manifest_href" +expect_succ pulp container repository content add --repository "cli_test_container_repository" --href "$tag_href" diff --git a/tests/scripts/pulp_container/test_copy.sh b/tests/scripts/pulp_container/test_copy.sh new file mode 100755 index 000000000..8707f1849 --- /dev/null +++ b/tests/scripts/pulp_container/test_copy.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# shellcheck source=tests/scripts/config.source +. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source + +pulp debug has-plugin --name "container" || exit 3 + +cleanup() { + pulp container remote destroy --name "cli_test_container_remote" || true + pulp container repository destroy --name "cli_test_source_container_repository" || true + pulp container repository destroy --name "cli_test_dest_container_repository" || true + pulp orphan cleanup || true +} +trap cleanup EXIT + +# Prepare +pulp container remote create --name "cli_test_container_remote" --url "$CONTAINER_REMOTE_URL" --upstream-name "$CONTAINER_IMAGE" +source_href="$(pulp container repository create --name "cli_test_source_container_repository" | jq -r .pulp_href)" +pulp container repository create --name "cli_test_dest_container_repository" +pulp container repository sync --name "cli_test_source_container_repository" --remote "cli_test_container_remote" +tag="$(pulp container repository content list -t 'tag' --repository "cli_test_source_container_repository" | jq -r .[0].name)" +digest="$(pulp container repository content list -t 'manifest' --repository "cli_test_source_container_repository" | jq -r .[0].digest)" + +# Test copying manifests +expect_succ pulp container repository copy-manifest --name "cli_test_dest_container_repository" --source "cli_test_source_container_repository" --digest "$digest" +expect_succ pulp container repository content list -t 'manifest' --repository "cli_test_dest_container_repository" --version "1" +test "$(echo "$OUTPUT" | jq -r length)" -eq 1 +test "$(echo "$OUTPUT" | jq -r .[0].digest)" = "$digest" + +expect_succ pulp container repository copy-manifest --name "cli_test_dest_container_repository" --source "cli_test_source_container_repository:1" --media-type "application/vnd.docker.distribution.manifest.v2+json" +expect_succ pulp container repository content list -t 'manifest' --repository "cli_test_dest_container_repository" --version "2" +copied="$(echo "$OUTPUT" | jq -r length)" +test "$copied" -gt 1 + +expect_succ pulp container repository copy-manifest --name "cli_test_dest_container_repository" --source "$source_href" +expect_succ pulp container repository content list -t 'manifest' --repository "cli_test_dest_container_repository" --version "3" +test "$(echo "$OUTPUT" | jq -r length)" -gt "$copied" + +# Test copying tags +expect_succ pulp container repository copy-tag --name "cli_test_dest_container_repository" --source "cli_test_source_container_repository" --tag "$tag" +expect_succ pulp container repository content list -t 'tag' --repository "cli_test_dest_container_repository" --version "4" +test "$(echo "$OUTPUT" | jq -r length)" -eq 1 +test "$(echo "$OUTPUT" | jq -r .[0].name)" = "$tag" + +expect_succ pulp container repository copy-tag --name "cli_test_dest_container_repository" --source "$source_href""versions/1/" +expect_succ pulp container repository content list -t 'tag' --repository "cli_test_dest_container_repository" --version "5" +test "$(echo "$OUTPUT" | jq -r length)" -gt 1 + +# Test bad versions +expect_fail pulp container repository copy-tag --name "cli_test_source_container_repository" --source "cli_test_dest_container_repository:0" +expect_fail pulp container repository copy-tag --name "cli_test_source_container_repository" --source "cli_test_dest_container_repository:6" +test "$ERROUTPUT" = "Error: Please specify a version that is less then the latest version 5 and greater than 0" diff --git a/tests/scripts/pulp_file/test_content.sh b/tests/scripts/pulp_file/test_content.sh index 65667f44f..f9a3dde2b 100755 --- a/tests/scripts/pulp_file/test_content.sh +++ b/tests/scripts/pulp_file/test_content.sh @@ -42,7 +42,7 @@ expect_succ pulp file repository content add --repository "cli_test_file_reposit expect_succ pulp file repository content add --repository "cli_test_file_repository" --sha256 "$sha256" --relative-path upload_test/test.txt --base-version 0 expect_succ pulp file repository content list --repository "cli_test_file_repository" --version 1 test "$(echo "$OUTPUT" | jq -r length)" -eq "1" -expect_succ pulp file repository content list --repository "cli_test_file_repository" --type "all" +expect_succ pulp file repository content list --repository "cli_test_file_repository" --all-types expect_succ pulp file repository content remove --repository "cli_test_file_repository" --sha256 "$sha256" --relative-path upload_test/test.txt expect_succ pulp file repository content list --repository "cli_test_file_repository"