chore(kuma-cp) deploy many instances using HELM (#1208)

Signed-off-by: Jakub Dyszkiewicz <[email protected]> (cherry picked from commit fd0bf9f) # Conflicts: # app/kumactl/pkg/install/k8s/control-plane/helmtemplates_vfsdata.go
kumahq · Nov 27, 2020 · 6e9d549 · 6e9d549
1 parent dd431a7
commit 6e9d549
Show file tree

Hide file tree

Showing 18 changed files with 233 additions and 119 deletions.
diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml
@@ -6308,6 +6308,7 @@ spec:
     rollingUpdate:
       maxSurge: 1
       maxUnavailable: 0
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: kuma

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml
@@ -6138,6 +6138,7 @@ spec:
     rollingUpdate:
       maxSurge: 1
       maxUnavailable: 0
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: kuma

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.global.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.global.golden.yaml
@@ -6147,6 +6147,7 @@ spec:
     rollingUpdate:
       maxSurge: 1
       maxUnavailable: 0
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: kuma

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml
@@ -6145,6 +6145,7 @@ spec:
     rollingUpdate:
       maxSurge: 1
       maxUnavailable: 0
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: kuma

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.remote.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.remote.golden.yaml
@@ -6145,6 +6145,7 @@ spec:
     rollingUpdate:
       maxSurge: 1
       maxUnavailable: 0
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: kuma

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml
@@ -6167,6 +6167,7 @@ spec:
     rollingUpdate:
       maxSurge: 1
       maxUnavailable: 0
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: kuma
@@ -6272,6 +6273,7 @@ spec:
     rollingUpdate:
       maxSurge: 1
       maxUnavailable: 0
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: kuma

diff --git a/app/kumactl/pkg/install/k8s/control-plane/helmtemplates_vfsdata.go b/app/kumactl/pkg/install/k8s/control-plane/helmtemplates_vfsdata.go
diff --git a/deployments/charts/kuma/README.md b/deployments/charts/kuma/README.md
@@ -17,6 +17,7 @@ The chart supports Helm v3+.
 | `controlPlane.zone`                                | Kuma zone name                                                                                      | nil                                                      |
 | `controlPlane.kdsGlobalAddress`                    | URL of Global Kuma CP                                                                               |                                                          |
 | `controlPlane.nodeSelector`                        | Node Selector for the Kuma Control Plane pods                                                       | `{ kubernetes.io/os: linux, kubernetes.io/arch: amd64 }` |
+| `controlPlane.replicas`                            | Number of replicas of the Kuma CP                                                                   | 1                                                        |
 | `controlPlane.injectorFailurePolicy`               | Failure policy of the mutating webhook implemented by the Kuma Injector component                   | `Ignore`                                                 |
 | `controlPlane.service.name`                        | Service name of the Kuma Control Plane                                                              | nil                                                      |
 | `controlPlane.service.type`                        | Service type of the Kuma Control Plane                                                              | ClusterIP                                                |
@@ -56,6 +57,7 @@ The chart supports Helm v3+.
 | `dataPlane.initImage.repository`                   | The Kuma DP init image repository                                                                   | `kuma-init`                                              |
 | `dataPlane.initImage.tag`                          | The Kuma DP init image tag                                                                          | nil, uses global                                         |
 | `ingress.enabled`                                  | If true, it deploys Ingress for cross cluster communication                                         | false                                                    |
+| `ingress.replicas`                                 | Number of replicas of the Ingress                                                                   | 1                                                        |
 | `ingress.drainTime`                                | Time for which old listener will still be active as draining                                        | 30s                                                      |
 | `ingress.service.name`                             | Service name of the Ingress                                                                         | nil                                                      |
 | `ingress.service.type`                             | Service type of the Ingress                                                                         | LoadBalancer                                             |

diff --git a/deployments/charts/kuma/templates/cp-deployment.yaml b/deployments/charts/kuma/templates/cp-deployment.yaml
@@ -7,6 +7,7 @@ metadata:
     {{- include "kuma.labels" . | nindent 4 }}
     app: kuma-control-plane
 spec:
+  replicas: {{ .Values.controlPlane.replicas }}
   strategy:
     rollingUpdate:
       maxSurge: 1

diff --git a/deployments/charts/kuma/templates/ingress-deployment.yaml b/deployments/charts/kuma/templates/ingress-deployment.yaml
@@ -12,6 +12,7 @@ spec:
     rollingUpdate:
       maxSurge: 1
       maxUnavailable: 0
+  replicas: {{ .Values.ingress.replicas }}
   selector:
     matchLabels:
       {{- include "kuma.selectorLabels" . | nindent 6 }}

diff --git a/deployments/charts/kuma/values.yaml b/deployments/charts/kuma/values.yaml
@@ -16,6 +16,8 @@ controlPlane:
 
   kdsGlobalAddress: ""
 
+  replicas: 1
+
   nodeSelector:
     kubernetes.io/os: linux
     kubernetes.io/arch: amd64
@@ -96,6 +98,7 @@ ingress:
   enabled: false
   mesh: default
   drainTime: 30s
+  replicas: 1
   service:
     # name: ""
     type: LoadBalancer

diff --git a/test/e2e/kuma_helm_deploy_multi_apps_test.go b/test/e2e/kuma_helm_deploy_multi_apps_test.go
@@ -56,6 +56,7 @@ metadata:
 			WithInstallationMode(HelmInstallationMode),
 			WithHelmReleaseName(releaseName),
 			WithSkipDefaultMesh(true), // it's common case for HELM deployments that Mesh is also managed by HELM therefore it's not created by default
+			WithCPReplicas(3),         // test HA capability
 			WithCNI(),
 		}
 

diff --git a/test/framework/interface.go b/test/framework/interface.go
@@ -28,6 +28,7 @@ type deployOptions struct {
 	env              map[string]string
 	ingress          bool
 	cni              bool
+	cpReplicas       int
 
 	// app specific
 	namespace string
@@ -44,6 +45,14 @@ func WithGlobalAddress(address string) DeployOptionsFunc {
 	}
 }
 
+// WithCPReplicas works only with HELM now
+func WithCPReplicas(cpReplicas int) DeployOptionsFunc {
+	return func(o *deployOptions) {
+		o.cpReplicas = cpReplicas
+	}
+}
+
+// WithSkipDefaultMesh works only with HELM now
 func WithSkipDefaultMesh(skip bool) DeployOptionsFunc {
 	return func(o *deployOptions) {
 		o.skipDefaultMesh = skip
@@ -149,7 +158,6 @@ type Cluster interface {
 	DeployKuma(mode string, opts ...DeployOptionsFunc) error
 	GetKuma() ControlPlane
 	VerifyKuma() error
-	RestartKuma() error
 	DeleteKuma(opts ...DeployOptionsFunc) error
 	InjectDNS(namespace ...string) error
 	GetKumactlOptions() *KumactlOptions

diff --git a/test/framework/k8s_cluster.go b/test/framework/k8s_cluster.go
@@ -24,7 +24,6 @@ import (
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/retry"
 	"github.com/gruntwork-io/terratest/modules/testing"
-	"github.com/onsi/gomega"
 	"go.uber.org/multierr"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -301,6 +300,9 @@ func (c *K8sCluster) deployKumaViaHelm(mode string, opts *deployOptions) error {
 		"dataPlane.initImage.repository":         KumaInitImageRepo,
 		"controlPlane.defaults.skipMeshCreation": strconv.FormatBool(opts.skipDefaultMesh),
 	}
+	if opts.cpReplicas != 0 {
+		values["controlPlane.replicas"] = strconv.Itoa(opts.cpReplicas)
+	}
 	for opt, value := range opts.helmOpts {
 		values[opt] = value
 	}
@@ -350,10 +352,13 @@ func (c *K8sCluster) deployKumaViaHelm(mode string, opts *deployOptions) error {
 }
 
 func (c *K8sCluster) DeployKuma(mode string, fs ...DeployOptionsFunc) error {
-	c.controlplane = NewK8sControlPlane(c.t, mode, c.name, c.kubeconfig,
-		c, c.loPort, c.hiPort, c.verbose)
-
 	opts := newDeployOpt(fs...)
+	replicas := 1
+	if opts.cpReplicas != 0 {
+		replicas = opts.cpReplicas
+	}
+	c.controlplane = NewK8sControlPlane(c.t, mode, c.name, c.kubeconfig, c, c.loPort, c.hiPort, c.verbose, replicas)
+
 	switch mode {
 	case core.Remote:
 		if opts.globalAddress == "" {
@@ -375,13 +380,13 @@ func (c *K8sCluster) DeployKuma(mode string, fs ...DeployOptionsFunc) error {
 		return err
 	}
 
-	err = c.WaitApp(KumaServiceName, KumaNamespace)
+	err = c.WaitApp(KumaServiceName, KumaNamespace, replicas)
 	if err != nil {
 		return err
 	}
 
 	if opts.cni {
-		err = c.WaitApp(cniApp, cniNamespace)
+		err = c.WaitApp(cniApp, cniNamespace, 1)
 		if err != nil {
 			return err
 		}
@@ -413,73 +418,6 @@ func (c *K8sCluster) GetKuma() ControlPlane {
 	return c.controlplane
 }
 
-func (c *K8sCluster) RestartKuma() error {
-	c.CleanupPortForwards()
-
-	kumacpPods := c.controlplane.GetKumaCPPods()
-	if len(kumacpPods) != 1 {
-		return errors.Errorf("Kuma CP pods: %d", len(kumacpPods))
-	}
-	oldPod := kumacpPods[0]
-
-	// creates the clientset
-	clientset, err := k8s.GetKubernetesClientFromOptionsE(c.t, c.GetKubectlOptions())
-	if err != nil {
-		return errors.Wrapf(err, "error in getting access to K8S")
-	}
-
-	// delete the pod
-	err = clientset.CoreV1().Pods(oldPod.Namespace).Delete(context.TODO(), oldPod.Name, metav1.DeleteOptions{})
-	if err != nil {
-		return err
-	}
-
-	// wait for pod to terminate
-	retry.DoWithRetry(c.t,
-		"Wait the Kuma CP pod to terminate.",
-		DefaultRetries,
-		DefaultTimeout,
-		func() (string, error) {
-			_, err := k8s.GetPodE(c.t,
-				c.GetKubectlOptions(oldPod.Namespace),
-				oldPod.Name)
-			if err != nil {
-				return "Pod " + oldPod.Name + " deleted", nil
-			}
-			return "Pod available " + oldPod.Name, fmt.Errorf("Pod %s still active", oldPod.Name)
-		})
-
-	k8s.WaitUntilNumPodsCreated(c.t,
-		c.GetKubectlOptions(KumaNamespace),
-		metav1.ListOptions{
-			LabelSelector: "app=" + KumaServiceName,
-		},
-		1,
-		DefaultRetries,
-		DefaultTimeout)
-
-	kumacpPods = c.controlplane.GetKumaCPPods()
-	if len(kumacpPods) != 1 {
-		return errors.Errorf("Kuma CP pods: %d", len(kumacpPods))
-	}
-
-	newPod := kumacpPods[0]
-	gomega.Expect(oldPod.Name).ToNot(gomega.Equal(newPod.Name))
-
-	k8s.WaitUntilPodAvailable(c.t,
-		c.GetKubectlOptions(KumaNamespace),
-		newPod.Name,
-		DefaultRetries,
-		DefaultTimeout)
-
-	err = c.controlplane.FinalizeAdd()
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (c *K8sCluster) VerifyKuma() error {
 	if err := c.controlplane.VerifyKumaGUI(); err != nil {
 		return err
@@ -674,13 +612,13 @@ func (c *K8sCluster) Deploy(deployment Deployment) error {
 	return deployment.Deploy(c)
 }
 
-func (c *K8sCluster) WaitApp(name, namespace string) error {
+func (c *K8sCluster) WaitApp(name, namespace string, replicas int) error {
 	k8s.WaitUntilNumPodsCreated(c.t,
 		c.GetKubectlOptions(namespace),
 		metav1.ListOptions{
 			LabelSelector: "app=" + name,
 		},
-		1,
+		replicas,
 		DefaultRetries,
 		DefaultTimeout)
 
@@ -690,14 +628,16 @@ func (c *K8sCluster) WaitApp(name, namespace string) error {
 			LabelSelector: "app=" + name,
 		},
 	)
-	if len(pods) < 1 {
-		return errors.Errorf("%s pods: %d", name, len(pods))
+	if len(pods) < replicas {
+		return errors.Errorf("%s pods: %d. expected %d", name, len(pods), replicas)
 	}
 
-	k8s.WaitUntilPodAvailable(c.t,
-		c.GetKubectlOptions(namespace),
-		pods[0].Name,
-		DefaultRetries,
-		DefaultTimeout)
+	for i := 0; i < replicas; i++ {
+		k8s.WaitUntilPodAvailable(c.t,
+			c.GetKubectlOptions(namespace),
+			pods[i].Name,
+			DefaultRetries,
+			DefaultTimeout)
+	}
 	return nil
 }
diff --git a/test/framework/k8s_clusters.go b/test/framework/k8s_clusters.go
@@ -96,16 +96,6 @@ func (cs *K8sClusters) GetKuma() ControlPlane {
 	panic("Not supported at this level.")
 }
 
-func (cs *K8sClusters) RestartKuma() error {
-	for name, c := range cs.clusters {
-		if err := c.RestartKuma(); err != nil {
-			return errors.Wrapf(err, "Restart Kuma on %s failed: %v", name, err)
-		}
-	}
-
-	return nil
-}
-
 func (cs *K8sClusters) VerifyKuma() error {
 	for name, c := range cs.clusters {
 		if err := c.VerifyKuma(); err != nil {

diff --git a/test/framework/k8s_controlplane.go b/test/framework/k8s_controlplane.go
@@ -33,12 +33,20 @@ type K8sControlPlane struct {
 	cluster    *K8sCluster
 	portFwd    PortFwd
 	verbose    bool
-}
-
-func NewK8sControlPlane(t testing.TestingT, mode core.CpMode, clusterName string,
-	kubeconfig string, cluster *K8sCluster,
-	loPort, hiPort uint32,
-	verbose bool) *K8sControlPlane {
+	replicas   int
+}
+
+func NewK8sControlPlane(
+	t testing.TestingT,
+	mode core.CpMode,
+	clusterName string,
+	kubeconfig string,
+	cluster *K8sCluster,
+	loPort uint32,
+	hiPort uint32,
+	verbose bool,
+	replicas int,
+) *K8sControlPlane {
 	name := clusterName + "-" + mode
 	kumactl, _ := NewKumactlOptions(t, name, verbose)
 	return &K8sControlPlane{
@@ -51,7 +59,8 @@ func NewK8sControlPlane(t testing.TestingT, mode core.CpMode, clusterName string
 		portFwd: PortFwd{
 			localAPIPort: loPort,
 		},
-		verbose: verbose,
+		verbose:  verbose,
+		replicas: replicas,
 	}
 }
 
@@ -73,7 +82,7 @@ func (c *K8sControlPlane) GetKubectlOptions(namespace ...string) *k8s.KubectlOpt
 
 func (c *K8sControlPlane) PortForwardKumaCP() error {
 	kumacpPods := c.GetKumaCPPods()
-	if len(kumacpPods) != 1 {
+	if len(kumacpPods) < 1 {
 		return errors.Errorf("Kuma CP pods: %d", len(kumacpPods))
 	}
 

diff --git a/test/framework/universal_cluster.go b/test/framework/universal_cluster.go
@@ -104,10 +104,6 @@ func (c *UniversalCluster) VerifyKuma() error {
 	return c.controlplane.kumactl.RunKumactl("get", "dataplanes")
 }
 
-func (c *UniversalCluster) RestartKuma() error {
-	return c.apps[AppModeCP].ReStart()
-}
-
 func (c *UniversalCluster) DeleteKuma(opts ...DeployOptionsFunc) error {
 	err := c.apps[AppModeCP].Stop()
 	delete(c.apps, AppModeCP)

diff --git a/test/framework/universal_clusters.go b/test/framework/universal_clusters.go
@@ -71,16 +71,6 @@ func (cs *UniversalClusters) GetKuma() ControlPlane {
 	panic("Not supported at this level.")
 }
 
-func (cs *UniversalClusters) RestartKuma() error {
-	for name, c := range cs.clusters {
-		if err := c.RestartKuma(); err != nil {
-			return errors.Wrapf(err, "Restart Kuma on %s failed: %v", name, err)
-		}
-	}
-
-	return nil
-}
-
 func (cs *UniversalClusters) VerifyKuma() error {
 	for name, c := range cs.clusters {
 		if err := c.VerifyKuma(); err != nil {