Merge pull request #7460 from medyagh/wait-coredns

add new wait component apps_running
kubernetes · Apr 7, 2020 · a577ed0 · a577ed0
2 parents f99a546 + b880469
commit a577ed0
Show file tree

Hide file tree

Showing 3 changed files with 128 additions and 111 deletions.
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go
@@ -18,22 +18,7 @@ limitations under the License.
 package kverify
 
 import (
-	"fmt"
-	"os/exec"
-	"strings"
 	"time"
-
-	"github.com/docker/machine/libmachine/state"
-	"github.com/golang/glog"
-	core "k8s.io/api/core/v1"
-	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	"k8s.io/minikube/pkg/minikube/bootstrapper"
-	"k8s.io/minikube/pkg/minikube/command"
-	"k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/cruntime"
-	"k8s.io/minikube/pkg/minikube/logs"
 )
 
 // minLogCheckTime how long to wait before spamming error logs to console
@@ -46,120 +31,39 @@ const (
 	SystemPodsWaitKey = "system_pods"
 	// DefaultSAWaitKey is the name used in the flags for default service account
 	DefaultSAWaitKey = "default_sa"
+	// AppsRunning is the name used in the flags for waiting for k8s-apps to be running
+	AppsRunning = "apps_running"
 )
 
 //  vars related to the --wait flag
 var (
 	// DefaultComponents is map of the the default components to wait for
 	DefaultComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true}
 	// NoWaitComponents is map of componets to wait for if specified 'none' or 'false'
-	NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false}
+	NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunning: false}
 	// AllComponents is map for waiting for all components.
-	AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true}
+	AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunning: true}
 	// DefaultWaitList is list of all default components to wait for. only names to be used for start flags.
 	DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey}
 	// AllComponentsList list of all valid components keys to wait for. only names to be used used for start flags.
-	AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey}
-)
-
-// ShouldWait will return true if the config says need to wait
-func ShouldWait(wcs map[string]bool) bool {
-	for _, c := range AllComponentsList {
-		if wcs[c] {
-			return true
-		}
-	}
-	return false
-}
-
-// ExpectedComponentsRunning returns whether or not all expected components are running
-func ExpectedComponentsRunning(cs *kubernetes.Clientset) error {
-	expected := []string{
+	AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey, AppsRunning}
+	// AppsRunningList running list are valid k8s-app components to wait for them to be running
+	AppsRunningList = []string{
 		"kube-dns", // coredns
 		"etcd",
 		"kube-apiserver",
 		"kube-controller-manager",
 		"kube-proxy",
 		"kube-scheduler",
 	}
+)
 
-	found := map[string]bool{}
-
-	pods, err := cs.CoreV1().Pods("kube-system").List(meta.ListOptions{})
-	if err != nil {
-		return err
-	}
-
-	for _, pod := range pods.Items {
-		glog.Infof("found pod: %s", podStatusMsg(pod))
-		if pod.Status.Phase != core.PodRunning {
-			continue
-		}
-		for k, v := range pod.ObjectMeta.Labels {
-			if k == "component" || k == "k8s-app" {
-				found[v] = true
-			}
-		}
-	}
-
-	missing := []string{}
-	for _, e := range expected {
-		if !found[e] {
-			missing = append(missing, e)
-		}
-	}
-	if len(missing) > 0 {
-		return fmt.Errorf("missing components: %v", strings.Join(missing, ", "))
-	}
-	return nil
-}
-
-// podStatusMsg returns a human-readable pod status, for generating debug status
-func podStatusMsg(pod core.Pod) string {
-	var sb strings.Builder
-	sb.WriteString(fmt.Sprintf("%q [%s] %s", pod.ObjectMeta.GetName(), pod.ObjectMeta.GetUID(), pod.Status.Phase))
-	for i, c := range pod.Status.Conditions {
-		if c.Reason != "" {
-			if i == 0 {
-				sb.WriteString(": ")
-			} else {
-				sb.WriteString(" / ")
-			}
-			sb.WriteString(fmt.Sprintf("%s:%s", c.Type, c.Reason))
-		}
-		if c.Message != "" {
-			sb.WriteString(fmt.Sprintf(" (%s)", c.Message))
+// ShouldWait will return true if the config says need to wait
+func ShouldWait(wcs map[string]bool) bool {
+	for _, c := range AllComponentsList {
+		if wcs[c] {
+			return true
 		}
 	}
-	return sb.String()
-}
-
-// announceProblems checks for problems, and slows polling down if any are found
-func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner) {
-	problems := logs.FindProblems(r, bs, cfg, cr)
-	if len(problems) > 0 {
-		logs.OutputProblems(problems, 5)
-		time.Sleep(kconst.APICallRetryInterval * 15)
-	}
-}
-
-// KubeletStatus checks the kubelet status
-func KubeletStatus(cr command.Runner) (state.State, error) {
-	glog.Infof("Checking kubelet status ...")
-	rr, err := cr.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet"))
-	if err != nil {
-		// Do not return now, as we still have parsing to do!
-		glog.Warningf("%s returned error: %v", rr.Command(), err)
-	}
-	s := strings.TrimSpace(rr.Stdout.String())
-	glog.Infof("kubelet is-active: %s", s)
-	switch s {
-	case "active":
-		return state.Running, nil
-	case "inactive":
-		return state.Stopped, nil
-	case "activating":
-		return state.Starting, nil
-	}
-	return state.Error, nil
+	return false
 }
diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go
@@ -19,9 +19,14 @@ package kverify
 
 import (
 	"fmt"
+	"os/exec"
+	"strings"
 	"time"
 
+	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	core "k8s.io/api/core/v1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
@@ -30,6 +35,8 @@ import (
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/logs"
+	"k8s.io/minikube/pkg/util/retry"
 )
 
 // WaitForSystemPods verifies essential pods for running kurnetes is running
@@ -68,3 +75,98 @@ func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg con
 	glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart))
 	return nil
 }
+
+// ExpectAppsRunning returns whether or not all expected k8s-apps are running. (without waiting for them)
+func ExpectAppsRunning(cs *kubernetes.Clientset, expected []string) error {
+	found := map[string]bool{}
+
+	pods, err := cs.CoreV1().Pods("kube-system").List(meta.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	for _, pod := range pods.Items {
+		glog.Infof("found pod: %s", podStatusMsg(pod))
+		if pod.Status.Phase != core.PodRunning {
+			continue
+		}
+		for k, v := range pod.ObjectMeta.Labels {
+			if k == "component" || k == "k8s-app" {
+				found[v] = true
+			}
+		}
+	}
+
+	missing := []string{}
+	for _, e := range expected {
+		if !found[e] {
+			missing = append(missing, e)
+		}
+	}
+	if len(missing) > 0 {
+		return fmt.Errorf("missing components: %v", strings.Join(missing, ", "))
+	}
+	return nil
+}
+
+// WaitForAppsRunning waits for expected Apps To be running
+func WaitForAppsRunning(cs *kubernetes.Clientset, expected []string, timeout time.Duration) error {
+	glog.Info("waiting for k8s-apps to be running ...")
+	start := time.Now()
+	checkRunning := func() error { return ExpectAppsRunning(cs, expected) }
+	if err := retry.Expo(checkRunning, 500*time.Millisecond, timeout); err != nil {
+		return errors.Wrap(err, "waitings for k8s app running")
+	}
+	glog.Infof("duration metric: took %s to wait for k8s-apps to be running ...", time.Since(start))
+	return nil
+}
+
+// podStatusMsg returns a human-readable pod status, for generating debug status
+func podStatusMsg(pod core.Pod) string {
+	var sb strings.Builder
+	sb.WriteString(fmt.Sprintf("%q [%s] %s", pod.ObjectMeta.GetName(), pod.ObjectMeta.GetUID(), pod.Status.Phase))
+	for i, c := range pod.Status.Conditions {
+		if c.Reason != "" {
+			if i == 0 {
+				sb.WriteString(": ")
+			} else {
+				sb.WriteString(" / ")
+			}
+			sb.WriteString(fmt.Sprintf("%s:%s", c.Type, c.Reason))
+		}
+		if c.Message != "" {
+			sb.WriteString(fmt.Sprintf(" (%s)", c.Message))
+		}
+	}
+	return sb.String()
+}
+
+// announceProblems checks for problems, and slows polling down if any are found
+func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner) {
+	problems := logs.FindProblems(r, bs, cfg, cr)
+	if len(problems) > 0 {
+		logs.OutputProblems(problems, 5)
+		time.Sleep(kconst.APICallRetryInterval * 15)
+	}
+}
+
+// KubeletStatus checks the kubelet status
+func KubeletStatus(cr command.Runner) (state.State, error) {
+	glog.Infof("Checking kubelet status ...")
+	rr, err := cr.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet"))
+	if err != nil {
+		// Do not return now, as we still have parsing to do!
+		glog.Warningf("%s returned error: %v", rr.Command(), err)
+	}
+	s := strings.TrimSpace(rr.Stdout.String())
+	glog.Infof("kubelet is-active: %s", s)
+	switch s {
+	case "active":
+		return state.Running, nil
+	case "inactive":
+		return state.Stopped, nil
+	case "activating":
+		return state.Starting, nil
+	}
+	return state.Error, nil
+}
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -394,6 +394,17 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 			return errors.Wrap(err, "waiting for default service account")
 		}
 	}
+
+	if cfg.VerifyComponents[kverify.AppsRunning] {
+		client, err := k.client(hostname, port)
+		if err != nil {
+			return errors.Wrap(err, "get k8s client")
+		}
+		if err := kverify.WaitForAppsRunning(client, kverify.AppsRunningList, timeout); err != nil {
+			return errors.Wrap(err, "waiting for apps_running")
+		}
+	}
+
 	glog.Infof("duration metric: took %s to wait for : %+v ...", time.Since(start), cfg.VerifyComponents)
 	return nil
 }
@@ -416,7 +427,7 @@ func (k *Bootstrapper) needsReset(conf string, hostname string, port int, client
 		return true
 	}
 
-	if err := kverify.ExpectedComponentsRunning(client); err != nil {
+	if err := kverify.ExpectAppsRunning(client, kverify.AppsRunningList); err != nil {
 		glog.Infof("needs reset: %v", err)
 		return true
 	}