Merge pull request #2359 from tejal29/add_healthcheck

Add deployment health check implementation

cmd/skaffold/app/cmd/flags.go

@@ -226,7 +226,7 @@ var FlagRegistry = []Flag{
 		Value:         &opts.StatusCheck,
 		DefValue:      true,
 		FlagAddMethod: "BoolVar",
-		DefinedOn:     []string{"dev", "debug", "deploy"},
+		DefinedOn:     []string{"dev", "debug", "deploy", "run"},
 	},
 }
 

integration/deploy_test.go

@@ -67,7 +67,8 @@ func TestBuildDeploy(t *testing.T) {
 	dir.Write("build.out", string(outputBytes))
 
 	// Run Deploy using the build output
-	skaffold.Deploy("--build-artifacts", buildOutputFile).InDir("examples/microservices").InNs(ns.Name).RunOrFail(t)
+	// See https://github.com/GoogleContainerTools/skaffold/issues/2372 on why status-check=false
+	skaffold.Deploy("--build-artifacts", buildOutputFile, "--status-check=false").InDir("examples/microservices").InNs(ns.Name).RunOrFail(t)
 
 	depApp := client.GetDeployment("leeroy-app")
 	testutil.CheckDeepEqual(t, appTag, depApp.Spec.Template.Spec.Containers[0].Image)
@@ -96,3 +97,38 @@ func TestDeploy(t *testing.T) {
 
 	skaffold.Delete().InDir("examples/kustomize").InNs(ns.Name).RunOrFail(t)
 }
+
+func TestDeployWithInCorrectConfig(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+	if ShouldRunGCPOnlyTests() {
+		t.Skip("skipping test that is not gcp only")
+	}
+
+	ns, _, deleteNs := SetupNamespace(t)
+	defer deleteNs()
+
+	err := skaffold.Deploy().InDir("testdata/unstable-deployment").InNs(ns.Name).Run(t)
+	if err == nil {
+		t.Error("expected an error to see since the deployment is not stable. However deploy returned success")
+	}
+
+	skaffold.Delete().InDir("testdata/unstable-deployment").InNs(ns.Name).RunOrFail(t)
+}
+
+func TestDeployWithInCorrectConfigWithNoStatusCheck(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+	if ShouldRunGCPOnlyTests() {
+		t.Skip("skipping test that is not gcp only")
+	}
+
+	ns, _, deleteNs := SetupNamespace(t)
+	defer deleteNs()
+
+	skaffold.Deploy("--status-check=false").InDir("testdata/unstable-deployment").InNs(ns.Name).RunOrFailOutput(t)
+
+	skaffold.Delete().InDir("testdata/unstable-deployment").InNs(ns.Name).RunOrFail(t)
+}

integration/run_test.go

@@ -52,6 +52,8 @@ func TestRun(t *testing.T) {
 		}, {
 			description: "microservices",
 			dir:         "examples/microservices",
+			// See https://github.com/GoogleContainerTools/skaffold/issues/2372
+			args:        []string{"--status-check=false"},
 			deployments: []string{"leeroy-app", "leeroy-web"},
 		}, {
 			description: "envTagger",

integration/testdata/unstable-deployment/Dockerfile

@@ -0,0 +1,3 @@
+FROM SCRATCH
+COPY hello /
+CMD ["/hello"]

integration/testdata/unstable-deployment/README.adoc

@@ -0,0 +1,3 @@
+=== Example: This example is used in integration tests.
+
+This is an invalid deployment. Please do not use it.

integration/testdata/unstable-deployment/hello

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "Hello World"

integration/testdata/unstable-deployment/incorrect-deployment.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: unstable-deployment
+spec:
+  progressDeadlineSeconds: 10
+  replicas: 1
+  selector:
+    matchLabels:
+      app: unstable
+  template:
+    metadata:
+      labels:
+        app: unstable
+    spec:
+      containers:
+        - name: incorrect-example
+          image: gcr.io/k8s-skaffold/incorrect-example
+          readinessProbe:
+            exec:
+              command:
+                - cat
+                - /does-not-exist

integration/testdata/unstable-deployment/skaffold.yaml

@@ -0,0 +1,9 @@
+apiVersion: skaffold/v1beta12
+kind: Config
+build:
+  artifacts:
+    - image: gcr.io/k8s-skaffold/incorrect-example
+deploy:
+  kubectl:
+    manifests:
+      - incorrect-deployment.yaml

pkg/skaffold/deploy/kubectl/cli.go

@@ -118,3 +118,13 @@ func (c *CLI) args(command string, commandFlags []string, arg ...string) []strin
 
 	return args
 }
+
+// Run shells out kubectl CLI.
+func (c *CLI) RunOut(ctx context.Context, in io.Reader, command string, commandFlags []string, arg ...string) ([]byte, error) {
+	args := c.args(command, commandFlags, arg...)
+
+	cmd := exec.CommandContext(ctx, "kubectl", args...)
+	cmd.Stdin = in
+
+	return util.RunCmdOut(cmd)
+}

pkg/skaffold/deploy/status_check.go

@@ -0,0 +1,149 @@
+/*
+Copyright 2019 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package deploy
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy/kubectl"
+	kubernetesutil "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
+	runcontext "github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/context"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+var (
+	// TODO: Move this to a flag or global config.
+	// Default deadline set to 10 minutes. This is default value for progressDeadlineInSeconds
+	// See: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/api/apps/v1/types.go#L305
+	defaultStatusCheckDeadlineInSeconds int32 = 600
+	// Poll period for checking set to 100 milliseconds
+	defaultPollPeriodInMilliseconds = 100
+
+	// For testing
+	executeRolloutStatus = getRollOutStatus
+)
+
+func StatusCheck(ctx context.Context, defaultLabeller *DefaultLabeller, runCtx *runcontext.RunContext) error {
+
+	client, err := kubernetesutil.GetClientset()
+	if err != nil {
+		return err
+	}
+	dMap, err := getDeployments(client, runCtx.Opts.Namespace, defaultLabeller)
+	if err != nil {
+		return errors.Wrap(err, "could not fetch deployments")
+	}
+
+	wg := sync.WaitGroup{}
+	// Its safe to use sync.Map without locks here as each subroutine adds a different key to the map.
+	syncMap := &sync.Map{}
+	kubeCtl := &kubectl.CLI{
+		Namespace:   runCtx.Opts.Namespace,
+		KubeContext: runCtx.KubeContext,
+	}
+
+	for dName, deadline := range dMap {
+		deadlineDuration := time.Duration(deadline) * time.Second
+		wg.Add(1)
+		go func(dName string, deadlineDuration time.Duration) {
+			defer wg.Done()
+			pollDeploymentRolloutStatus(ctx, kubeCtl, dName, deadlineDuration, syncMap)
+		}(dName, deadlineDuration)
+	}
+
+	// Wait for all deployment status to be fetched
+	wg.Wait()
+	return getSkaffoldDeployStatus(syncMap)
+}
+
+func getDeployments(client kubernetes.Interface, ns string, l *DefaultLabeller) (map[string]int32, error) {
+
+	deps, err := client.AppsV1().Deployments(ns).List(metav1.ListOptions{
+		LabelSelector: l.K8sManagedByLabelKeyValueString(),
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "could not fetch deployments")
+	}
+
+	depMap := map[string]int32{}
+
+	for _, d := range deps.Items {
+		var deadline int32
+		if d.Spec.ProgressDeadlineSeconds == nil {
+			logrus.Debugf("no progressDeadlineSeconds config found for deployment %s. Setting deadline to %d seconds", d.Name, defaultStatusCheckDeadlineInSeconds)
+			deadline = defaultStatusCheckDeadlineInSeconds
+		} else {
+			deadline = *d.Spec.ProgressDeadlineSeconds
+		}
+		depMap[d.Name] = deadline
+	}
+
+	return depMap, nil
+}
+
+func pollDeploymentRolloutStatus(ctx context.Context, k *kubectl.CLI, dName string, deadline time.Duration, syncMap *sync.Map) {
+	pollDuration := time.Duration(defaultPollPeriodInMilliseconds) * time.Millisecond
+	// Add poll duration to account for one last attempt after progressDeadlineSeconds.
+	timeoutContext, cancel := context.WithTimeout(ctx, deadline+pollDuration)
+	logrus.Debugf("checking rollout status %s", dName)
+	defer cancel()
+	for {
+		select {
+		case <-timeoutContext.Done():
+			syncMap.Store(dName, errors.Wrap(timeoutContext.Err(), fmt.Sprintf("deployment rollout status could not be fetched within %v", deadline)))
+			return
+		case <-time.After(pollDuration):
+			status, err := executeRolloutStatus(timeoutContext, k, dName)
+			if err != nil {
+				syncMap.Store(dName, err)
+				return
+			}
+			if strings.Contains(status, "successfully rolled out") {
+				syncMap.Store(dName, nil)
+				return
+			}
+		}
+	}
+}
+
+func getSkaffoldDeployStatus(m *sync.Map) error {
+	errorStrings := []string{}
+	m.Range(func(k, v interface{}) bool {
+		if t, ok := v.(error); ok {
+			errorStrings = append(errorStrings, fmt.Sprintf("deployment %s failed due to %s", k, t.Error()))
+		}
+		return true
+	})
+
+	if len(errorStrings) == 0 {
+		return nil
+	}
+	return fmt.Errorf("following deployments are not stable:\n%s", strings.Join(errorStrings, "\n"))
+}
+
+func getRollOutStatus(ctx context.Context, k *kubectl.CLI, dName string) (string, error) {
+	b, err := k.RunOut(ctx, nil, "rollout", []string{"status", "deployment", dName},
+		"--watch=false")
+	return string(b), err
+}