wip:Support multi control plane

cmd/minikube/cmd/node_add.go

@@ -26,8 +26,10 @@ import (
 	"k8s.io/minikube/pkg/minikube/mustload"
 	"k8s.io/minikube/pkg/minikube/node"
 	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/out/register"
 	"k8s.io/minikube/pkg/minikube/reason"
 	"k8s.io/minikube/pkg/minikube/style"
+	"k8s.io/minikube/pkg/util"
 )
 
 var (
@@ -42,21 +44,35 @@ var nodeAddCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		co := mustload.Healthy(ClusterFlagValue())
 		cc := co.Config
+		config.TagPrimaryControlPlane(cc)
 
 		if driver.BareMetal(cc.Driver) {
 			out.FailureT("none driver does not support multi-node clusters")
 		}
 
 		name := node.Name(len(cc.Nodes) + 1)
 
-		out.Step(style.Happy, "Adding node {{.name}} to cluster {{.cluster}}", out.V{"name": name, "cluster": cc.Name})
+		if cp {
+			out.Step(style.Happy, "Adding control plane node {{.name}} to cluster {{.cluster}}", out.V{"name": name, "cluster": cc.Name})
+		} else {
+			out.Step(style.Happy, "Adding node {{.name}} to cluster {{.cluster}}", out.V{"name": name, "cluster": cc.Name})
+		}
 
 		// TODO: Deal with parameters better. Ideally we should be able to acceot any node-specific minikube start params here.
 		n := config.Node{
-			Name:              name,
-			Worker:            worker,
-			ControlPlane:      cp,
-			KubernetesVersion: cc.KubernetesConfig.KubernetesVersion,
+			Name:                name,
+			Worker:              worker,
+			ControlPlane:        cp,
+			PrimaryControlPlane: false,
+			KubernetesVersion:   cc.KubernetesConfig.KubernetesVersion,
+		}
+
+		if n.ControlPlane {
+			err := util.CheckMultiControlPlaneVersion(cc.KubernetesConfig.KubernetesVersion)
+			if err != nil {
+				exit.Error(reason.KubernetesTooOld, "target kubernetes version too old", err)
+			}
+			n.Port = cc.KubernetesConfig.NodePort
 		}
 
 		// Make sure to decrease the default amount of memory we use per VM if this is the first worker node
@@ -70,6 +86,7 @@ var nodeAddCmd = &cobra.Command{
 			}
 		}
 
+		register.Reg.SetStep(register.InitialSetup)
 		if err := node.Add(cc, n, false); err != nil {
 			_, err := maybeDeleteAndRetry(cmd, *cc, n, nil, err)
 			if err != nil {

cmd/minikube/cmd/node_start.go

@@ -56,7 +56,7 @@ var nodeStartCmd = &cobra.Command{
 		}
 
 		register.Reg.SetStep(register.InitialSetup)
-		r, p, m, h, err := node.Provision(cc, n, n.ControlPlane, viper.GetBool(deleteOnFailure))
+		r, p, m, h, err := node.Provision(cc, n, viper.GetBool(deleteOnFailure))
 		if err != nil {
 			exit.Error(reason.GuestNodeProvision, "provisioning host for node", err)
 		}
@@ -71,7 +71,7 @@ var nodeStartCmd = &cobra.Command{
 			ExistingAddons: nil,
 		}
 
-		_, err = node.Start(s, n.ControlPlane)
+		_, err = node.Start(s)
 		if err != nil {
 			_, err := maybeDeleteAndRetry(cmd, *cc, *n, nil, err)
 			if err != nil {

cmd/minikube/cmd/node_stop.go

@@ -46,7 +46,7 @@ var nodeStopCmd = &cobra.Command{
 		}
 
 		machineName := config.MachineName(*cc, *n)
-
+		node.MustReset(*cc, *n, api, machineName)
 		err = machine.StopHost(api, machineName)
 		if err != nil {
 			out.FatalT("Failed to stop node {{.name}}", out.V{"name": name})

cmd/minikube/cmd/start.go

@@ -131,6 +131,20 @@ func platform() string {
 func runStart(cmd *cobra.Command, args []string) {
 	register.SetEventLogPath(localpath.EventLog(ClusterFlagValue()))
 
+	controlPlanesNum := viper.GetInt(controlPlanes)
+	nodesNum := viper.GetInt(nodes)
+	if controlPlanesNum > nodesNum {
+		out.WarningT(fmt.Sprintf("control planes number %v larger than nodes number %v, enlarge nodes to %v.", controlPlanesNum, nodesNum, controlPlanesNum))
+		viper.Set(nodes, controlPlanesNum)
+	}
+	k8sVersion := viper.GetString(kubernetesVersion)
+	if controlPlanesNum > 1 {
+		err := util.CheckMultiControlPlaneVersion(k8sVersion)
+		if err != nil {
+			exit.Error(reason.KubernetesTooOld, "target kubernetes version too old", err)
+		}
+	}
+
 	out.SetJSON(outputFormat == "json")
 	if err := pkgtrace.Initialize(viper.GetString(trace)); err != nil {
 		exit.Message(reason.Usage, "error initializing tracing: {{.Error}}", out.V{"Error": err.Error()})
@@ -328,7 +342,7 @@ func provisionWithDriver(cmd *cobra.Command, ds registry.DriverState, existing *
 		ssh.SetDefaultClient(ssh.External)
 	}
 
-	mRunner, preExists, mAPI, host, err := node.Provision(&cc, &n, true, viper.GetBool(deleteOnFailure))
+	mRunner, preExists, mAPI, host, err := node.Provision(&cc, &n, viper.GetBool(deleteOnFailure))
 	if err != nil {
 		return node.Starter{}, err
 	}
@@ -345,14 +359,21 @@ func provisionWithDriver(cmd *cobra.Command, ds registry.DriverState, existing *
 }
 
 func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config.ClusterConfig) (*kubeconfig.Settings, error) {
-	kubeconfig, err := node.Start(starter, true)
+	// TODO: Currently, we start the primary control plane first. If there are multiple control planes,
+	//  the kube-apiserver will keep crash to wait for other apiserver to respond, which blocks health checks.
+	//  As a temporary solution, we reset the stacked control planes before we stopped it.
+	//  To fix this, we could:
+	//  - Delay the health check.
+	//  - Start all control planes at the same time.
+	kubeconfig, err := node.Start(starter)
 	if err != nil {
 		kubeconfig, err = maybeDeleteAndRetry(cmd, *starter.Cfg, *starter.Node, starter.ExistingAddons, err)
 		if err != nil {
 			return nil, err
 		}
 	}
 
+	numControlPlanes := viper.GetInt(controlPlanes)
 	numNodes := viper.GetInt(nodes)
 	if existing != nil {
 		if numNodes > 1 {
@@ -369,10 +390,15 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config.
 				for i := 1; i < numNodes; i++ {
 					nodeName := node.Name(i + 1)
 					n := config.Node{
-						Name:              nodeName,
-						Worker:            true,
-						ControlPlane:      false,
-						KubernetesVersion: starter.Cfg.KubernetesConfig.KubernetesVersion,
+						Name:                nodeName,
+						Worker:              true,
+						ControlPlane:        false,
+						PrimaryControlPlane: false,
+						KubernetesVersion:   starter.Cfg.KubernetesConfig.KubernetesVersion,
+					}
+					if i < numControlPlanes {
+						n.ControlPlane = true
+						n.Port = starter.Cfg.KubernetesConfig.NodePort
 					}
 					out.Ln("") // extra newline for clarity on the command line
 					err := node.Add(starter.Cfg, n, viper.GetBool(deleteOnFailure))
@@ -382,7 +408,7 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config.
 				}
 			} else {
 				for _, n := range existing.Nodes {
-					if !n.ControlPlane {
+					if !n.PrimaryControlPlane {
 						err := node.Add(starter.Cfg, n, viper.GetBool(deleteOnFailure))
 						if err != nil {
 							return nil, errors.Wrap(err, "adding node")
@@ -489,7 +515,7 @@ func maybeDeleteAndRetry(cmd *cobra.Command, existing config.ClusterConfig, n co
 		cc := updateExistingConfigFromFlags(cmd, &existing)
 		var kubeconfig *kubeconfig.Settings
 		for _, n := range cc.Nodes {
-			r, p, m, h, err := node.Provision(&cc, &n, n.ControlPlane, false)
+			r, p, m, h, err := node.Provision(&cc, &n, false)
 			s := node.Starter{
 				Runner:         r,
 				PreExists:      p,
@@ -504,7 +530,7 @@ func maybeDeleteAndRetry(cmd *cobra.Command, existing config.ClusterConfig, n co
 				return nil, err
 			}
 
-			k, err := node.Start(s, n.ControlPlane)
+			k, err := node.Start(s)
 			if n.ControlPlane {
 				kubeconfig = k
 			}
@@ -1190,11 +1216,12 @@ func createNode(cc config.ClusterConfig, kubeNodeName string, existing *config.C
 	}
 
 	cp := config.Node{
-		Port:              cc.KubernetesConfig.NodePort,
-		KubernetesVersion: getKubernetesVersion(&cc),
-		Name:              kubeNodeName,
-		ControlPlane:      true,
-		Worker:            true,
+		Port:                cc.KubernetesConfig.NodePort,
+		KubernetesVersion:   getKubernetesVersion(&cc),
+		Name:                kubeNodeName,
+		ControlPlane:        true,
+		PrimaryControlPlane: true,
+		Worker:              true,
 	}
 	cc.Nodes = []config.Node{cp}
 	return cc, cp, nil

cmd/minikube/cmd/start_flags.go

@@ -102,6 +102,7 @@ const (
 	hostOnlyNicType         = "host-only-nic-type"
 	natNicType              = "nat-nic-type"
 	nodes                   = "nodes"
+	controlPlanes           = "control-planes"
 	preload                 = "preload"
 	deleteOnFailure         = "delete-on-failure"
 	forceSystemd            = "force-systemd"
@@ -150,6 +151,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().Bool(autoUpdate, true, "If set, automatically updates drivers to the latest version. Defaults to true.")
 	startCmd.Flags().Bool(installAddons, true, "If set, install addons. Defaults to true.")
 	startCmd.Flags().IntP(nodes, "n", 1, "The number of nodes to spin up. Defaults to 1.")
+	startCmd.Flags().Int(controlPlanes, 1, "The number of control planes to spin up. Defaults to 1.")
 	startCmd.Flags().Bool(preload, true, "If set, download tarball of preloaded images if available to improve start time. Defaults to true.")
 	startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.")
 	startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use sytemd as cgroup manager. Defaults to false.")
@@ -412,6 +414,8 @@ func upgradeExistingConfig(cc *config.ClusterConfig) {
 		cc.KicBaseImage = viper.GetString(kicBaseImage)
 		klog.Infof("config upgrade: KicBaseImage=%s", cc.KicBaseImage)
 	}
+
+	config.TagPrimaryControlPlane(cc)
 }
 
 // updateExistingConfigFromFlags will update the existing config from the flags - used on a second start
@@ -645,6 +649,8 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
 		cc.KicBaseImage = viper.GetString(kicBaseImage)
 	}
 
+	config.TagPrimaryControlPlane(&cc)
+
 	return cc
 }