test(e2e): parallelize universal environment (#4164)

Signed-off-by: Jakub Dyszkiewicz <[email protected]>

.circleci/config.yml

@@ -422,6 +422,22 @@ workflows:
             - build
             - check
           parallelism: 1
+      - e2e:
+          name: e2e-universal
+          target: test/e2e-universal
+          requires:
+            - build
+            - check
+          parallelism: 1
+      - e2e:
+          <<: *work_branch_workflow_filter
+          name: e2e-universal-ipv6
+          target: test/e2e-universal
+          ipv6: true
+          requires:
+            - build
+            - check
+          parallelism: 1
       - e2e:
           # Avoids running expensive workflow on PRs
           <<: *work_branch_workflow_filter

mk/e2e.new.mk

@@ -28,6 +28,7 @@ TEST_NAMES = $(shell ls -1 ./test/e2e)
 ALL_TESTS = $(addprefix ./test/e2e/, $(addsuffix /..., $(TEST_NAMES)))
 E2E_PKG_LIST ?= $(ALL_TESTS)
 KUBE_E2E_PKG_LIST ?= ./test/e2e_env/kubernetes
+UNIVERSAL_E2E_PKG_LIST ?= ./test/e2e_env/universal
 GINKGO_E2E_FLAGS ?=
 
 ifdef GINKGO_TEST_RESULTS_DIR
@@ -144,3 +145,7 @@ test/e2e-kubernetes: $(E2E_DEPS_TARGETS)
 	$(MAKE) test/e2e/k8s/load/images/kuma-1
 	$(MAKE) GINKGO_E2E_FLAGS=-p E2E_PKG_LIST=$(KUBE_E2E_PKG_LIST) test/e2e/test || (ret=$$?; $(MAKE) test/e2e/k8s/stop/cluster/kuma-1 && exit $$ret)
 	$(MAKE) test/e2e/k8s/stop/cluster/kuma-1
+
+.PHONY: test/e2e-universal
+test/e2e-universal: build/kumactl images/test k3d/network/create
+	$(MAKE) GINKGO_E2E_FLAGS=-p E2E_PKG_LIST=$(UNIVERSAL_E2E_PKG_LIST) test/e2e/test

test/e2e/deploy/kuma_deploy_global_zone.go

@@ -79,12 +79,12 @@ spec:
 		Expect(zone).ToNot(BeNil())
 
 		// then
-		logs1, err := global.GetKumaCPLogs()
+		logs1, err := c1.GetKumaCPLogs()
 		Expect(err).ToNot(HaveOccurred())
 		Expect(logs1).To(ContainSubstring("\"mode\":\"global\""))
 
 		// and
-		logs2, err := zone.GetKumaCPLogs()
+		logs2, err := c2.GetKumaCPLogs()
 		Expect(err).ToNot(HaveOccurred())
 		Expect(logs2).To(ContainSubstring("\"mode\":\"zone\""))
 	})

test/e2e/helm/kuma_helm_deploy_global_zone.go

@@ -67,16 +67,6 @@ func ZoneAndGlobalWithHelmChart() {
 
 		zone = c2.GetKuma()
 		Expect(zone).ToNot(BeNil())
-
-		// then
-		logs1, err := global.GetKumaCPLogs()
-		Expect(err).ToNot(HaveOccurred())
-		Expect(logs1).To(ContainSubstring("\"mode\":\"global\""))
-
-		// and
-		logs2, err := zone.GetKumaCPLogs()
-		Expect(err).ToNot(HaveOccurred())
-		Expect(logs2).To(ContainSubstring("\"mode\":\"zone\""))
 	})
 
 	E2EAfterEach(func() {

test/e2e/membership/kubernetes/membership_kubernetes.go

@@ -74,7 +74,7 @@ spec:
 
 		// then the client is not allowed to do it
 		Eventually(func() (string, error) {
-			return cluster.GetKuma().GetKumaCPLogs()
+			return cluster.GetKumaCPLogs()
 		}, "30s", "1s").Should(ContainSubstring("dataplane cannot be a member of mesh"))
 		out, err := cluster.GetKumactlOptions().RunKumactlAndGetOutput("get", "dataplanes", "--mesh", "demo")
 		Expect(err).ToNot(HaveOccurred())

test/e2e/membership/universal/membership_universal.go

@@ -51,7 +51,7 @@ constraints:
 
 		// then it's not allowed
 		Eventually(func() (string, error) {
-			return cluster.GetKuma().GetKumaCPLogs()
+			return cluster.GetKumaCPLogs()
 		}, "30s", "1s").Should(ContainSubstring("dataplane cannot be a member of mesh"))
 		dataplanes, err := cluster.GetKumactlOptions().RunKumactlAndGetOutput("get", "dataplanes")
 		Expect(err).ToNot(HaveOccurred())

test/e2e/universal/auth/dp_auth.go → test/e2e_env/universal/auth/dp_auth.go

@@ -6,7 +6,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	"github.com/kumahq/kuma/test/e2e/universal/env"
+	"github.com/kumahq/kuma/test/e2e_env/universal/env"
 	. "github.com/kumahq/kuma/test/framework"
 )
 
@@ -17,7 +17,6 @@ func DpAuth() {
 		Expect(env.Cluster.Install(MeshUniversal(meshName))).To(Succeed())
 		E2EDeferCleanup(func() {
 			Expect(env.Cluster.DeleteMeshApps(meshName)).To(Succeed())
-			Expect(env.Cluster.DeleteMeshDataplaneProxies(meshName)).To(Succeed())
 			Expect(env.Cluster.DeleteMesh(meshName)).To(Succeed())
 		})
 	})
@@ -42,9 +41,10 @@ networking:
 		Expect(err).ToNot(HaveOccurred())
 
 		// then
-		Eventually(func() (string, error) {
-			return env.Cluster.GetKuma().GetKumaCPLogs()
-		}, "30s", "1s").Should(ContainSubstring("you are trying to override existing dataplane to which you don't have an access"))
+		// todo(jakubdyszkiewicz) uncomment once we can handle CP logs across all parallel executions
+		// Eventually(func() (string, error) {
+		//	return env.Cluster.GetKumaCPLogs()
+		// }, "30s", "1s").Should(ContainSubstring("you are trying to override existing dataplane to which you don't have an access"))
 	})
 
 	It("should be able to override old Dataplane of same service", func() {

test/e2e/universal/auth/user_auth.go → test/e2e_env/universal/auth/user_auth.go

@@ -4,7 +4,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	"github.com/kumahq/kuma/test/e2e/universal/env"
+	"github.com/kumahq/kuma/test/e2e_env/universal/env"
 	. "github.com/kumahq/kuma/test/framework"
 )
 
@@ -19,15 +19,14 @@ func UserAuth() {
 		Expect(err).ToNot(HaveOccurred())
 
 		// when kumactl is configured with new token
-		err = env.Cluster.GetKumactlOptions().KumactlConfigControlPlanesAdd(
+		kumactl := NewKumactlOptions(env.Cluster.GetTesting(), env.Cluster.GetKuma().GetName()+"test-admin", false)
+		err = kumactl.KumactlConfigControlPlanesAdd(
 			"test-admin",
 			env.Cluster.GetKuma().GetAPIServerAddress(),
 			token,
 		)
-		Expect(err).ToNot(HaveOccurred())
 
 		// then the new admin can access secrets
-		kumactl, err := NewKumactlOptions(env.Cluster.GetTesting(), env.Cluster.GetKuma().GetName(), false)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(kumactl.RunKumactl("get", "secrets")).To(Succeed())
 	})
@@ -42,15 +41,14 @@ func UserAuth() {
 		Expect(err).ToNot(HaveOccurred())
 
 		// when kumactl is configured with new token
-		err = env.Cluster.GetKumactlOptions().KumactlConfigControlPlanesAdd(
+		kumactl := NewKumactlOptions(env.Cluster.GetTesting(), env.Cluster.GetKuma().GetName()+"test-user", false)
+		err = kumactl.KumactlConfigControlPlanesAdd(
 			"test-user",
 			env.Cluster.GetKuma().GetAPIServerAddress(),
 			token,
 		)
-		Expect(err).ToNot(HaveOccurred())
 
 		// then the new member can access dataplanes but not secrets because they are not admin
-		kumactl, err := NewKumactlOptions(env.Cluster.GetTesting(), env.Cluster.GetKuma().GetName(), false)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(kumactl.RunKumactl("get", "dataplanes")).To(Succeed())
 		Expect(kumactl.RunKumactl("get", "secrets")).ToNot(Succeed())

test/e2e/universal/healthcheck/panic.go → test/e2e_env/universal/healthcheck/panic.go

@@ -7,7 +7,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	"github.com/kumahq/kuma/test/e2e/universal/env"
+	"github.com/kumahq/kuma/test/e2e_env/universal/env"
 	. "github.com/kumahq/kuma/test/framework"
 )
 
@@ -51,7 +51,6 @@ networking:
 	BeforeAll(func() {
 		E2EDeferCleanup(func() {
 			Expect(env.Cluster.DeleteMeshApps(meshName)).To(Succeed())
-			Expect(env.Cluster.DeleteMeshDataplaneProxies(meshName)).To(Succeed())
 			Expect(env.Cluster.DeleteMesh(meshName)).To(Succeed())
 		})
 

test/e2e_env/universal/universal_suite_test.go

@@ -0,0 +1,54 @@
+package auth_test
+
+import (
+	"encoding/json"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/kumahq/kuma/pkg/config/core"
+	"github.com/kumahq/kuma/pkg/test"
+	"github.com/kumahq/kuma/test/e2e_env/universal/auth"
+	"github.com/kumahq/kuma/test/e2e_env/universal/env"
+	"github.com/kumahq/kuma/test/e2e_env/universal/healthcheck"
+	. "github.com/kumahq/kuma/test/framework"
+)
+
+func TestE2E(t *testing.T) {
+	test.RunSpecs(t, "E2E Universal Suite")
+}
+
+var _ = SynchronizedBeforeSuite(
+	func() []byte {
+		env.Cluster = NewUniversalCluster(NewTestingT(), Kuma3, Silent)
+		E2EDeferCleanup(env.Cluster.DismissCluster)
+		Expect(env.Cluster.Install(Kuma(core.Standalone, WithEnv("KUMA_STORE_UNSAFE_DELETE", "true")))).To(Succeed())
+		pf := env.Cluster.GetKuma().(*UniversalControlPlane).Networking()
+		bytes, err := json.Marshal(pf)
+		Expect(err).ToNot(HaveOccurred())
+		return bytes
+	},
+	func(bytes []byte) {
+		if env.Cluster != nil {
+			return // cluster was already initiated with first function
+		}
+		networking := UniversalCPNetworking{}
+		Expect(json.Unmarshal(bytes, &networking)).To(Succeed())
+		env.Cluster = NewUniversalCluster(NewTestingT(), Kuma3, Silent)
+		E2EDeferCleanup(env.Cluster.DismissCluster) // clean up any containers if needed
+		cp, err := NewUniversalControlPlane(
+			env.Cluster.GetTesting(),
+			core.Standalone,
+			env.Cluster.Name(),
+			env.Cluster.Verbose(),
+			networking,
+		)
+		Expect(err).ToNot(HaveOccurred())
+		env.Cluster.SetCp(cp)
+	},
+)
+
+var _ = Describe("User Auth", auth.UserAuth)
+var _ = Describe("DP Auth", auth.DpAuth, Ordered)
+var _ = Describe("HealthCheck panic threshold", healthcheck.HealthCheckPanicThreshold, Ordered)

test/framework/deployments/docker.go

@@ -11,7 +11,7 @@ import (
 	"github.com/gruntwork-io/terratest/modules/testing"
 	"github.com/pkg/errors"
 
-	util_net "github.com/kumahq/kuma/pkg/util/net"
+	"github.com/kumahq/kuma/test/framework"
 )
 
 type DockerContainer struct {
@@ -91,11 +91,9 @@ func buildArgsForDocker(args map[string][]string, ports map[uint32]uint32) []str
 		}
 	}
 
-	for port, pubPort := range ports {
+	for port := range ports {
 		port := strconv.Itoa(int(port))
-		pubPort := strconv.Itoa(int(pubPort))
-
-		opts = append(opts, "--publish="+pubPort+":"+port)
+		opts = append(opts, "--publish="+port)
 	}
 
 	return opts
@@ -192,13 +190,8 @@ func (d *DockerContainer) GetName() (string, error) {
 
 func AllocatePublicPortsFor(ports ...uint32) DockerContainerOptFn {
 	return func(d *DockerContainer) error {
-		for i, port := range ports {
-			pubPortUInt32, err := util_net.PickTCPPort("", uint32(33204+i), uint32(34204+i))
-			if err != nil {
-				return err
-			}
-
-			d.ports[port] = pubPortUInt32
+		for _, port := range ports {
+			d.ports[port] = uint32(0)
 		}
 
 		return nil
@@ -212,10 +205,26 @@ func (d *DockerContainer) Run() error {
 	}
 
 	d.addID(container)
+	if err := d.updatePorts(); err != nil {
+		return err
+	}
 
 	return nil
 }
 
+func (d *DockerContainer) updatePorts() error {
+	var ports []uint32
+	for port := range d.ports {
+		ports = append(ports, port)
+	}
+	publishedPorts, err := framework.GetPublishedDockerPorts(d.t, d.id, ports)
+	if err != nil {
+		return err
+	}
+	d.ports = publishedPorts
+	return nil
+}
+
 func (d *DockerContainer) Stop() error {
 	retry.DoWithRetry(d.t, "stop "+d.id, 30, 3*time.Second,
 		func() (string, error) {

test/framework/deployments/externalservice/deployment.go

@@ -44,7 +44,7 @@ func Install(name string, commands ...Command) framework.InstallFunc {
 			deployment = &universalDeployment{
 				name:     name,
 				commands: commands,
-				ports:    map[string]string{},
+				ports:    map[uint32]uint32{},
 				verbose:  cluster.Verbose(),
 			}
 		default: