Add pulp file ACS refresh command

[noissue]

CHANGES/377.feature

@@ -0,0 +1 @@
+Added refresh command for pulp_file Alternate Content Sources.

pulpcore/cli/common/context.py

@@ -89,6 +89,7 @@ def __init__(
         self.format: str = format
         self.background_tasks: bool = background_tasks
         self.timeout: int = timeout
+        self.start_time: Optional[datetime.datetime] = None
 
     @property
     def api(self) -> OpenAPI:
@@ -152,49 +153,118 @@ def call(self, operation_id: str, non_blocking: bool = False, *args: Any, **kwar
             )
             if not non_blocking:
                 result = self.wait_for_task(result)
+        if isinstance(result, dict) and ["task_group"] == list(result.keys()):
+            task_group_href = result["task_group"]
+            result = self.api.call(
+                "task_groups_read", parameters={"task_group_href": task_group_href}
+            )
+            click.echo(
+                _("Started background task group {task_group_href}").format(
+                    task_group_href=task_group_href
+                ),
+                err=True,
+            )
+            if not non_blocking:
+                result = self.wait_for_task_group(result)
         return result
 
+    @staticmethod
+    def _check_task_finished(task: EntityDefinition) -> bool:
+        task_href = task["pulp_href"]
+
+        if task["state"] == "completed":
+            return True
+        elif task["state"] == "failed":
+            raise click.ClickException(
+                _("Task {task_href} failed: '{description}'").format(
+                    task_href=task_href, description=task["error"]["description"]
+                )
+            )
+        elif task["state"] == "canceled":
+            raise click.ClickException(_("Task {task_href} canceled").format(task_href=task_href))
+        elif task["state"] in ["waiting", "running", "canceling"]:
+            return False
+        else:
+            raise NotImplementedError(_("Unknown task state: {state}").format(state=task["state"]))
+
+    def _poll_task(self, task: EntityDefinition) -> EntityDefinition:
+        while True:
+            if self._check_task_finished(task):
+                click.echo("Done.", err=True)
+                return task
+            else:
+                if self.timeout:
+                    assert isinstance(self.start_time, datetime.datetime)
+                    if (datetime.datetime.now() - self.start_time).seconds > self.timeout:
+                        raise PulpNoWait(
+                            _("Waiting for task {task_href} timed out.").format(
+                                task_href=task["pulp_href"]
+                            )
+                        )
+                time.sleep(1)
+                click.echo(".", nl=False, err=True)
+                task = self.api.call("tasks_read", parameters={"task_href": task["pulp_href"]})
+
     def wait_for_task(self, task: EntityDefinition) -> Any:
         """
         Wait for a task to finish and return the finished task object.
+
         Raise `click.ClickException` on timeout, background, ctrl-c, if task failed or was canceled.
         """
-        timeout: int = self.timeout
+        self.start_time = datetime.datetime.now()
+
         if self.background_tasks:
             raise PulpNoWait(_("Not waiting for task because --background was specified."))
         task_href = task["pulp_href"]
+        try:
+            return self._poll_task(task)
+        except KeyboardInterrupt:
+            raise PulpNoWait(_("Task {task_href} sent to background.").format(task_href=task_href))
+
+    def wait_for_task_group(self, task_group: EntityDefinition) -> Any:
+        """
+        Wait for a task group to finish and return the finished task object.
+
+        Raise `click.ClickException` on timeout, background, ctrl-c, if tasks failed or were
+        canceled.
+        """
+        self.start_time = datetime.datetime.now()
+
+        if self.background_tasks:
+            raise PulpNoWait("Not waiting for task group because --background was specified.")
         try:
             while True:
-                if task["state"] == "completed":
-                    click.echo("Done.", err=True)
-                    return task
-                elif task["state"] == "failed":
-                    raise click.ClickException(
-                        _("Task {task_href} failed: '{description}'").format(
-                            task_href=task_href, description=task["error"]["description"]
+                if task_group["all_tasks_dispatched"] is True:
+                    for task in task_group["tasks"]:
+                        task = self.api.call(
+                            "tasks_read", parameters={"task_href": task["pulp_href"]}
                         )
-                    )
-                elif task["state"] == "canceled":
-                    raise click.ClickException(_("Task canceled"))
-                elif task["state"] in ["waiting", "running", "canceling"]:
+                        click.echo(
+                            _("Waiting for task {task_href}").format(task_href=task["pulp_href"]),
+                            err=True,
+                        )
+                        self._poll_task(task)
+                    return task_group
+                else:
                     if self.timeout:
-                        if timeout <= 0:
+                        assert isinstance(self.start_time, datetime.datetime)
+                        if (datetime.datetime.now() - self.start_time).seconds > self.timeout:
                             raise PulpNoWait(
-                                _("Waiting for task {task_href} timed out.").format(
-                                    task_href=task_href
+                                _("Waiting for task group {task_group_href} timed out.").format(
+                                    task_group_href=task_group["pulp_href"]
                                 )
                             )
-                        timeout -= 1
                     time.sleep(1)
                     click.echo(".", nl=False, err=True)
-                    task = self.api.call("tasks_read", parameters={"task_href": task_href})
-                else:
-                    raise NotImplementedError(
-                        _("Unknown task state: {state}").format(state=task["state"])
+                    task_group = self.api.call(
+                        "task_group_read", parameters={"task_group_href": task_group["pulp_href"]}
                     )
-            raise click.ClickException(_("Task timed out"))
         except KeyboardInterrupt:
-            raise PulpNoWait(_("Task {task_href} sent to background.").format(task_href=task_href))
+            raise PulpNoWait(
+                _("Task group {task_group_href} sent to background.").format(
+                    task_group_href=task_group["pulp_href"]
+                )
+            )
 
     def has_plugin(
         self,

pulpcore/cli/core/task.py

@@ -1,4 +1,5 @@
 import gettext
+import re
 from contextlib import suppress
 from typing import Optional
 
@@ -108,6 +109,6 @@ def cancel(
         try:
             pulp_ctx.wait_for_task(entity)
         except Exception as e:
-            if str(e) != "Task canceled":
+            if not re.match("Task /pulp/api/v3/tasks/[-0-9a-f]*/ canceled", str(e)):
                 raise e
         click.echo(_("Done."), err=True)

pulpcore/cli/file/acs.py

@@ -115,3 +115,12 @@ def remove(acs_ctx: PulpFileACSContext, paths: Iterable[str]) -> None:
 acs.add_command(create_command(decorators=create_options))
 acs.add_command(update_command(decorators=lookup_options + update_options))
 acs.add_command(destroy_command(decorators=lookup_options))
+
+
+@acs.command()
+@pass_entity_context
+@pass_pulp_context
+@href_option
+@name_option
+def refresh(pulp_ctx: PulpContext, acs_ctx: PulpFileACSContext) -> None:
+    acs_ctx.refresh(acs_ctx.pulp_href)

pulpcore/cli/file/context.py

@@ -1,4 +1,5 @@
 import gettext
+from typing import Any, ClassVar
 
 from pulpcore.cli.common.context import (
     EntityDefinition,
@@ -23,6 +24,13 @@ class PulpFileACSContext(PulpEntityContext):
     CREATE_ID = "acs_file_file_create"
     UPDATE_ID = "acs_file_file_partial_update"
     DELETE_ID = "acs_file_file_delete"
+    REFRESH_ID: ClassVar[str] = "acs_file_file_refresh"
+
+    def refresh(self, href: str) -> Any:
+        return self.pulp_ctx.call(
+            self.REFRESH_ID,
+            parameters={self.HREF: href},
+        )
 
 
 class PulpFileContentContext(PulpContentContext):

tests/scripts/pulp_file/test_acs.sh

@@ -3,33 +3,47 @@
 # shellcheck source=tests/scripts/config.source
 . "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
 
-pulp debug has-plugin --name "file" --min-version "1.9.0.dev" || exit 3
+pulp debug has-plugin --name "file" --min-version "1.10.0.dev" || exit 3
 
 acs_remote="cli_test_file_acs_remote"
 acs="cli_test_acs"
 
 cleanup() {
   pulp file acs destroy --name $acs || true
   pulp file remote destroy --name $acs_remote || true
+  pulp file repository destroy --name "cli-bad-repo" || true
+  pulp file remote destroy --name "cli-remote-manifest-only" || true
 }
 trap cleanup EXIT
 
 cleanup
 
-expect_succ pulp file remote create --name $acs_remote --url "http://example.com" --policy "on_demand"
+expect_succ pulp file remote create --name $acs_remote --url "$PULP_FIXTURES_URL" --policy "on_demand"
 
-expect_succ pulp file acs create --name $acs --remote $acs_remote --path "ab/" --path "cd/"
+expect_succ pulp file acs create --name $acs --remote $acs_remote --path "file/PULP_MANIFEST" --path "file2/PULP_MANIFEST"
 expect_succ pulp file acs list
 test "$(echo "$OUTPUT" | jq -r length)" -ge 1
 expect_succ pulp file acs show --name $acs
 test "$(echo "$OUTPUT" | jq ".paths | length")" -eq 2
 
-expect_succ pulp file acs path add --name $acs --path "ok/" --path "no/"
-expect_succ pulp file acs show --name $acs
-test "$(echo "$OUTPUT" | jq ".paths | length")" -eq 4
-
-expect_succ pulp file acs path remove --name $acs --path "ab/"
+# manipulate paths
+expect_succ pulp file acs path add --name $acs --path "file-invalid/PULP_MANIFEST"
 expect_succ pulp file acs show --name $acs
 test "$(echo "$OUTPUT" | jq ".paths | length")" -eq 3
+expect_succ pulp file acs path remove --name $acs --path "file-invalid/PULP_MANIFEST"
+expect_succ pulp file acs show --name $acs
+test "$(echo "$OUTPUT" | jq ".paths | length")" -eq 2
+
+# test refresh
+expect_succ pulp file acs refresh --name $acs
+task_group=$(echo "$ERROUTPUT" | grep -E -o "/pulp/api/v3/task-groups/[-[:xdigit:]]*/")
+expect_succ pulp task-group show --href "$task_group"
+test "$(echo "$OUTPUT" | jq ".tasks | length")" -eq 2
+
+# create a remote with manifest only and sync it
+expect_succ pulp file remote create --name "cli-remote-manifest-only" --url "$PULP_FIXTURES_URL/file-manifest/PULP_MANIFEST"
+remote_href="$(echo "$OUTPUT" | jq -r ".pulp_href")"
+expect_succ pulp file repository create --name "cli-bad-repo" --remote "$remote_href"
+expect_succ pulp file repository sync --name "cli-bad-repo"
 
 expect_succ pulp file acs destroy --name $acs