Fetch control plane IP at runtime (#376)

Set the CONTROL_PLANE_ENDPOINT_IP programmatically at runtime during E2E tests based on docker network settings. Co-authored-by: NilanjanDaw <[email protected]>
vmware-tanzu · Feb 10, 2022 · 0fdb123 · 0fdb123
1 parent d0eb65f
commit 0fdb123
Show file tree

Hide file tree

Showing 7 changed files with 27 additions and 2 deletions.
diff --git a/Makefile b/Makefile
@@ -126,7 +126,7 @@ prepare-byoh-docker-host-image-dev:
 	docker build test/e2e -f docs/BYOHDockerFileDev -t ${BYOH_BASE_IMG_DEV}
 
 test-e2e: take-user-input docker-build prepare-byoh-docker-host-image $(GINKGO) cluster-templates-e2e ## Run the end-to-end tests
-	CONTROL_PLANE_ENDPOINT_IP=172.18.10.151 $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" $(_SKIP_ARGS) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) test/e2e -- \
+	$(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" $(_SKIP_ARGS) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) test/e2e -- \
 	    -e2e.artifacts-folder="$(ARTIFACTS)" \
 	    -e2e.config="$(E2E_CONF_FILE)" \
 	    -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \

diff --git a/test/e2e/byohost_reuse_test.go b/test/e2e/byohost_reuse_test.go
@@ -95,6 +95,8 @@ var _ = Describe("When BYO Host rejoins the capacity pool", func() {
 		}()
 
 		By("Creating a cluster")
+
+		setControlPlaneIP(context.Background(), dockerClient)
 		clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
 			ClusterProxy: bootstrapClusterProxy,
 			ConfigCluster: clusterctl.ConfigClusterInput{
@@ -137,6 +139,7 @@ var _ = Describe("When BYO Host rejoins the capacity pool", func() {
 
 		By("Creating a new cluster")
 		clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6))
+
 		clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
 			ClusterProxy: bootstrapClusterProxy,
 			ConfigCluster: clusterctl.ConfigClusterInput{

diff --git a/test/e2e/config/provider.yaml b/test/e2e/config/provider.yaml
@@ -96,6 +96,7 @@ variables:
   # NOTE: INIT_WITH_BINARY is used only by the clusterctl upgrade test to initialize the management cluster to be upgraded
   INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.4/clusterctl-{OS}-{ARCH}"
   BUNDLE_LOOKUP_TAG: "v0.1.0_alpha.2"
+  CONTROL_PLANE_ENDPOINT_IP: ""
 
 intervals:
   default/wait-controllers: ["3m", "10s"]

diff --git a/test/e2e/e2e_debug_helper.go b/test/e2e/e2e_debug_helper.go
@@ -144,4 +144,4 @@ func ShowInfo(allAgentLogFiles []string) {
 	WriteShellScript(ReadByohControllerManagerLogShellFile, shellContent)
 	ShowFileContent(ReadByohControllerManagerLogShellFile)
 	ExecuteShellScript(ReadByohControllerManagerLogShellFile)
-}
+}
diff --git a/test/e2e/e2e_docker_helper.go b/test/e2e/e2e_docker_helper.go
@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/docker/api/types"
@@ -211,3 +212,20 @@ func setupByoDockerHost(ctx context.Context, clusterConName, byoHostName, namesp
 
 	return output, byohost.ID, err
 }
+
+func setControlPlaneIP(ctx context.Context, dockerClient *client.Client) {
+	_, ok := os.LookupEnv("CONTROL_PLANE_ENDPOINT_IP")
+	if ok {
+		return
+	}
+	inspect, _ := dockerClient.NetworkInspect(ctx, "kind", types.NetworkInspectOptions{})
+	ipOctets := strings.Split(inspect.IPAM.Config[0].Subnet, ".")
+
+	// The ControlPlaneEndpoint is a static IP that is in the hosts' 
+	// subnet but outside of its DHCP range. We believe 151 is a pretty 
+	// high number and we have < 10 containers being spun up, so we 
+	// can safely use this IP for the ControlPlaneEndpoint
+	ipOctets[3] = "151"
+	ip := strings.Join(ipOctets, ".")
+	os.Setenv("CONTROL_PLANE_ENDPOINT_IP", ip)
+}
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
@@ -97,6 +97,7 @@ var _ = Describe("When BYOH joins existing cluster [PR-Blocking]", func() {
 			}
 		}()
 
+		setControlPlaneIP(context.Background(), dockerClient)
 		clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
 			ClusterProxy: bootstrapClusterProxy,
 			ConfigCluster: clusterctl.ConfigClusterInput{

diff --git a/test/e2e/md_scale_test.go b/test/e2e/md_scale_test.go
@@ -84,6 +84,8 @@ var _ = Describe("When testing MachineDeployment scale out/in", func() {
 		// TODO: Write agent logs to files for better debugging
 
 		By("creating a workload cluster with one control plane node and one worker node")
+
+		setControlPlaneIP(context.Background(), dockerClient)
 		clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
 			ClusterProxy: bootstrapClusterProxy,
 			ConfigCluster: clusterctl.ConfigClusterInput{