container wait: support health states

Support two new wait conditions, "healthy" and "unhealthy".  This
further paves the way for integrating sdnotify with health checks which
is currently being tracked in containers#6160.

Fixes: containers#13627
Signed-off-by: Valentin Rothberg <[email protected]>

cmd/podman/common/completion.go

@@ -1474,7 +1474,9 @@ func AutocompleteImageSaveFormat(cmd *cobra.Command, args []string, toComplete s
 // AutocompleteWaitCondition - Autocomplete wait condition options.
 // -> "unknown", "configured", "created", "running", "stopped", "paused", "exited", "removing"
 func AutocompleteWaitCondition(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	states := []string{"unknown", "configured", "created", "running", "stopped", "paused", "exited", "removing"}
+	states := []string{"unknown", "configured", "created", "exited",
+		"healthy", "initialized", "paused", "removing", "running",
+		"stopped", "stopping", "unhealthy"}
 	return states, cobra.ShellCompDirectiveNoFileComp
 }
 

docs/source/markdown/podman-wait.1.md.in

@@ -24,7 +24,7 @@ with different exit codes, but `podman wait` can only display and detect one.
 ## OPTIONS
 
 #### **--condition**=*state*
-Container state or condition to wait for.  Can be specified multiple times where at least one condition must match for the command to return.  Supported values are "created", "exited", "initialized", "paused", "removing", "running", "stopped",  "stopping".  The default condition is "stopped".
+Container state or condition to wait for.  Can be specified multiple times where at least one condition must match for the command to return.  Supported values are "configured", "created", "exited", "healthy", "initialized", "paused", "removing", "running", "stopped",  "stopping", "unhealthy".  The default condition is "stopped".
 
 #### **--help**, **-h**
 

libpod/container_api.go

@@ -698,17 +698,26 @@ func (c *Container) WaitForConditionWithInterval(ctx context.Context, waitTimeou
 	resultChan := make(chan waitResult)
 	waitForExit := false
 	wantedStates := make(map[define.ContainerStatus]bool, len(conditions))
+	wantedHealthStates := make(map[string]bool)
 
 	for _, rawCondition := range conditions {
-		condition, err := define.StringToContainerStatus(rawCondition)
-		if err != nil {
-			return -1, err
-		}
-		switch condition {
-		case define.ContainerStateExited, define.ContainerStateStopped:
-			waitForExit = true
+		switch rawCondition {
+		case define.HealthCheckHealthy, define.HealthCheckUnhealthy:
+			if !c.HasHealthCheck() {
+				return -1, fmt.Errorf("cannot use condition %q: container %s has no healthcheck", rawCondition, c.ID())
+			}
+			wantedHealthStates[rawCondition] = true
 		default:
-			wantedStates[condition] = true
+			condition, err := define.StringToContainerStatus(rawCondition)
+			if err != nil {
+				return -1, err
+			}
+			switch condition {
+			case define.ContainerStateExited, define.ContainerStateStopped:
+				waitForExit = true
+			default:
+				wantedStates[condition] = true
+			}
 		}
 	}
 
@@ -731,20 +740,33 @@ func (c *Container) WaitForConditionWithInterval(ctx context.Context, waitTimeou
 		}()
 	}
 
-	if len(wantedStates) > 0 {
+	if len(wantedStates) > 0 || len(wantedHealthStates) > 0 {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
 
 			for {
-				state, err := c.State()
-				if err != nil {
-					trySend(-1, err)
-					return
+				if len(wantedStates) > 0 {
+					state, err := c.State()
+					if err != nil {
+						trySend(-1, err)
+						return
+					}
+					if _, found := wantedStates[state]; found {
+						trySend(-1, nil)
+						return
+					}
 				}
-				if _, found := wantedStates[state]; found {
-					trySend(-1, nil)
-					return
+				if len(wantedHealthStates) > 0 {
+					status, err := c.HealthCheckStatus()
+					if err != nil {
+						trySend(-1, err)
+						return
+					}
+					if _, found := wantedHealthStates[status]; found {
+						trySend(-1, nil)
+						return
+					}
 				}
 				select {
 				case <-ctx.Done():

pkg/api/handlers/utils/containers.go

@@ -28,8 +28,8 @@ type waitQueryDocker struct {
 }
 
 type waitQueryLibpod struct {
-	Interval  string                   `schema:"interval"`
-	Condition []define.ContainerStatus `schema:"condition"`
+	Interval   string   `schema:"interval"`
+	Conditions []string `schema:"condition"`
 }
 
 func WaitContainerDocker(w http.ResponseWriter, r *http.Request) {
@@ -118,19 +118,27 @@ func WaitContainerLibpod(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime)
+	containerEngine := &abi.ContainerEngine{Libpod: runtime}
+	opts := entities.WaitOptions{
+		Conditions: query.Conditions,
+		Interval:   interval,
+	}
 	name := GetName(r)
-
-	waitFn := createContainerWaitFn(r.Context(), name, interval)
-	exitCode, err := waitFn(query.Condition...)
+	reports, err := containerEngine.ContainerWait(r.Context(), []string{name}, opts)
 	if err != nil {
 		if errors.Is(err, define.ErrNoSuchCtr) {
 			ContainerNotFound(w, name, err)
 			return
 		}
 		InternalServerError(w, err)
+	}
+	if len(reports) != 1 {
+		Error(w, http.StatusInternalServerError, fmt.Errorf("the ContainerWait() function returned unexpected count of reports: %d", len(reports)))
 		return
 	}
-	WriteResponse(w, http.StatusOK, strconv.Itoa(int(exitCode)))
+
+	WriteResponse(w, http.StatusOK, strconv.Itoa(int(reports[0].ExitCode)))
 }
 
 type containerWaitFn func(conditions ...define.ContainerStatus) (int32, error)

pkg/api/server/register_containers.go

@@ -1229,12 +1229,15 @@ func (s *APIServer) registerContainersHandlers(r *mux.Router) error {
 	//      enum:
 	//       - configured
 	//       - created
-	//       - running
-	//       - stopped
-	//       - paused
 	//       - exited
+	//       - healthy
+	//       - initialized
+	//       - paused
 	//       - removing
+	//       - running
+	//       - stopped
 	//       - stopping
+	//       - unhealthy
 	//    description: "Conditions to wait for. If no condition provided the 'exited' condition is assumed."
 	//  - in: query
 	//    name: interval

pkg/bindings/containers/containers.go

@@ -341,6 +341,8 @@ func Unpause(ctx context.Context, nameOrID string, options *UnpauseOptions) erro
 func Wait(ctx context.Context, nameOrID string, options *WaitOptions) (int32, error) {
 	if options == nil {
 		options = new(WaitOptions)
+	} else if len(options.Condition) > 0 && len(options.Conditions) > 0 {
+		return -1, fmt.Errorf("%q field cannot be used with deprecated %q field", "Conditions", "Condition")
 	}
 	var exitCode int32
 	conn, err := bindings.GetClient(ctx)
@@ -351,6 +353,7 @@ func Wait(ctx context.Context, nameOrID string, options *WaitOptions) (int32, er
 	if err != nil {
 		return exitCode, err
 	}
+	delete(params, "conditions") // They're called "condition"
 	response, err := conn.DoRequest(ctx, nil, http.MethodPost, "/containers/%s/wait", params, nil, nameOrID)
 	if err != nil {
 		return exitCode, err

pkg/bindings/containers/types.go

@@ -229,8 +229,14 @@ type UnpauseOptions struct{}
 //
 //go:generate go run ../generator/generator.go WaitOptions
 type WaitOptions struct {
+	// Conditions to wait on.  Includes container statuses such as
+	// "running" or "stopped" and health-related values such "healthy".
+	Conditions []string `schema:"condition"`
+	// Time interval to wait before polling for completion.
+	Interval *string
+	// Container status to wait on.
+	// Deprecated: use Conditions instead.
 	Condition []define.ContainerStatus
-	Interval  *string
 }
 
 // StopOptions are optional options for stopping containers

pkg/domain/infra/tunnel/containers.go

@@ -38,17 +38,8 @@ func (ic *ContainerEngine) ContainerExists(ctx context.Context, nameOrID string,
 }
 
 func (ic *ContainerEngine) ContainerWait(ctx context.Context, namesOrIds []string, opts entities.WaitOptions) ([]entities.WaitReport, error) {
-	conditions := make([]define.ContainerStatus, 0, len(opts.Conditions))
-	for _, condition := range opts.Conditions {
-		cond, err := define.StringToContainerStatus(condition)
-		if err != nil {
-			return nil, err
-		}
-		conditions = append(conditions, cond)
-	}
-
 	responses := make([]entities.WaitReport, 0, len(namesOrIds))
-	options := new(containers.WaitOptions).WithCondition(conditions).WithInterval(opts.Interval.String())
+	options := new(containers.WaitOptions).WithConditions(opts.Conditions).WithInterval(opts.Interval.String())
 	for _, n := range namesOrIds {
 		response := entities.WaitReport{}
 		exitCode, err := containers.Wait(ic.ClientCtx, n, options)

test/system/220-healthcheck.bats

@@ -83,7 +83,7 @@ Log[-1].Output   | \"Uh-oh on stdout!\\\nUh-oh on stderr!\"
     _build_health_check_image $img cleanfile
     run_podman run -d --name $ctr      \
            --health-cmd /healthcheck   \
-           --health-retries=2          \
+           --health-retries=3          \
            --health-interval=disable   \
            $img
 
@@ -105,6 +105,46 @@ Log[-1].Output   | \"Uh-oh on stdout!\\\nUh-oh on stderr!\"
     run_podman rmi $img
 }
 
+@test "podman wait --condition={healthy,unhealthy}" {
+    ctr="healthcheck_c"
+    img="healthcheck_i"
+    wait_file="$PODMAN_TMPDIR/$(random_string).wait_for_me"
+    _build_health_check_image $img
+
+    for condition in healthy unhealthy;do
+        rm -f $wait_file
+        run_podman run -d --name $ctr      \
+               --health-cmd /healthcheck   \
+               --health-retries=1          \
+               --health-interval=disable   \
+               $img
+        if [[ $condition == "unhealthy" ]];then
+            # create the uh-oh file to let the health check fail
+            run_podman exec $ctr touch /uh-oh
+        fi
+
+        # Wait for the container in the background and create the $wait_file to
+        # signal the specified wait condition was met.
+        (timeout --foreground -v --kill=5 5 $PODMAN wait --condition=$condition $ctr && touch $wait_file) &
+
+        # Sleep 1 second to make sure above commands are running
+        sleep 1
+        if [[ -f $wait_file ]]; then
+            die "the wait file should only be created after the container turned healthy"
+        fi
+
+        if [[ $condition == "healthy" ]];then
+            run_podman healthcheck run $ctr
+        else
+            run_podman 1 healthcheck run $ctr
+        fi
+        wait_for_file $wait_file
+        run_podman rm -f -t0 $ctr
+    done
+
+    run_podman rmi $img
+}
+
 @test "podman healthcheck --health-on-failure" {
     run_podman 125 create --health-on-failure=kill $IMAGE
     is "$output" "Error: cannot set on-failure action to kill without a health check"