cli: job restart command (#16278)

Implement the new `nomad job restart` command that allows operators to
restart allocations tasks or reschedule then entire allocation.

Restarts can be batched to target multiple allocations in parallel.
Between each batch the command can stop and hold for a predefined time
or until the user confirms that the process should proceed.

This implements the "Stateless Restarts" alternative from the original
RFC
(https://gist.github.com/schmichael/e0b8b2ec1eb146301175fd87ddd46180).
The original concept is still worth implementing, as it allows this
functionality to be exposed over an API that can be consumed by the
Nomad UI and other clients. But the implementation turned out to be more
complex than we initially expected so we thought it would be better to
release a stateless CLI-based implementation first to gather feedback
and validate the restart behaviour.

Co-authored-by: Shishir Mahajan <[email protected]>

.changelog/16278.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+cli: Added new `nomad job restart` command to restart all allocations for a job
+```

api/tasks.go

@@ -976,6 +976,12 @@ func (t *Task) SetLogConfig(l *LogConfig) *Task {
 	return t
 }
 
+// SetLifecycle is used to set lifecycle config to a task.
+func (t *Task) SetLifecycle(l *TaskLifecycle) *Task {
+	t.Lifecycle = l
+	return t
+}
+
 // TaskState tracks the current state of a task and events that caused state
 // transitions.
 type TaskState struct {

command/commands.go

@@ -415,6 +415,14 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
+		"job restart": func() (cli.Command, error) {
+			// Use a *cli.ConcurrentUi because this command spawns several
+			// goroutines that write to the terminal concurrently.
+			meta.Ui = &cli.ConcurrentUi{Ui: meta.Ui}
+			return &JobRestartCommand{
+				Meta: meta,
+			}, nil
+		},
 		"job deployments": func() (cli.Command, error) {
 			return &JobDeploymentsCommand{
 				Meta: meta,

command/helpers.go

@@ -62,6 +62,13 @@ func limit(s string, length int) string {
 	return s[:length]
 }
 
+// indentString returns the string s padded with the given number of empty
+// spaces before each line except for the first one.
+func indentString(s string, pad int) string {
+	prefix := strings.Repeat(" ", pad)
+	return strings.Join(strings.Split(s, "\n"), fmt.Sprintf("\n%s", prefix))
+}
+
 // wrapAtLengthWithPadding wraps the given text at the maxLineLength, taking
 // into account any provided left padding.
 func wrapAtLengthWithPadding(s string, pad int) string {