Adding e2e tests for shared vpc installs

kubernetes-sigs · Jun 3, 2024 · 9c44f1a · 9c44f1a
1 parent cd4236f
commit 9c44f1a
Show file tree

Hide file tree

Showing 4 changed files with 214 additions and 35 deletions.
diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh
@@ -102,27 +102,38 @@ EOF
 
 # initialize a router and cloud NAT
 init_networks() {
-  if [[ ${GCP_NETWORK_NAME} != "default" ]]; then
-    gcloud compute networks create --project "$GCP_PROJECT" "${GCP_NETWORK_NAME}" --subnet-mode auto --quiet
-    gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-http --project "$GCP_PROJECT" \
-      --allow tcp:80 --network "${GCP_NETWORK_NAME}" --quiet
-    gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-https --project "$GCP_PROJECT" \
-      --allow tcp:443 --network "${GCP_NETWORK_NAME}" --quiet
-    gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-icmp --project "$GCP_PROJECT" \
-      --allow icmp --network "${GCP_NETWORK_NAME}" --priority 65534 --quiet
-    gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-internal --project "$GCP_PROJECT" \
-      --allow "tcp:0-65535,udp:0-65535,icmp" --network "${GCP_NETWORK_NAME}" --priority 65534 --quiet
-  fi
 
-  gcloud compute firewall-rules list --project "$GCP_PROJECT"
-  gcloud compute networks list --project="${GCP_PROJECT}"
-  gcloud compute networks describe "${GCP_NETWORK_NAME}" --project="${GCP_PROJECT}"
+  ARRAY=( "$GCP_PROJECT:$GCP_NETWORK_NAME"
+          "$GCP_HOST_PROJECT:${GCP_NETWORK_NAME}-shared-vpc" )
+
+  for data in "${ARRAY[@]}"; do
+      PROJECT=${data%%:*}
+      NETWORK=${data#*:}
+
+    echo "Creating network resources in project ${PROJECT}"
+
+    if [[ ${NETWORK} != "default" ]]; then
+      gcloud compute networks create --project "$PROJECT" "${NETWORK}" --subnet-mode auto --quiet
+      gcloud compute firewall-rules create "${NETWORK}"-allow-http --project "$PROJECT" \
+        --allow tcp:80 --network "${NETWORK}" --quiet
+      gcloud compute firewall-rules create "${NETWORK}"-allow-https --project "$PROJECT" \
+        --allow tcp:443 --network "${NETWORK}" --quiet
+      gcloud compute firewall-rules create "${NETWORK}"-allow-icmp --project "$PROJECT" \
+        --allow icmp --network "${NETWORK}" --priority 65534 --quiet
+      gcloud compute firewall-rules create "${NETWORK}"-allow-internal --project "$PROJECT" \
+        --allow "tcp:0-65535,udp:0-65535,icmp" --network "${NETWORK}" --priority 65534 --quiet
+    fi
+
+    gcloud compute firewall-rules list --project "$PROJECT"
+    gcloud compute networks list --project="${$PROJECT}"
+    gcloud compute networks describe "${NETWORK}" --project="${$PROJECT}"
 
-  gcloud compute routers create "${TEST_NAME}-myrouter" --project="${GCP_PROJECT}" \
-    --region="${GCP_REGION}" --network="${GCP_NETWORK_NAME}"
-  gcloud compute routers nats create "${TEST_NAME}-mynat" --project="${GCP_PROJECT}" \
-    --router-region="${GCP_REGION}" --router="${TEST_NAME}-myrouter" \
-    --nat-all-subnet-ip-ranges --auto-allocate-nat-external-ips
+    gcloud compute routers create "${TEST_NAME}-myrouter" --project="${$PROJECT}" \
+      --region="${GCP_REGION}" --network="${NETWORK}"
+    gcloud compute routers nats create "${TEST_NAME}-mynat" --project="${$PROJECT}" \
+      --router-region="${GCP_REGION}" --router="${TEST_NAME}-myrouter" \
+      --nat-all-subnet-ip-ranges --auto-allocate-nat-external-ips
+  done
 }
 
 
@@ -146,23 +157,38 @@ cleanup() {
   (gcloud compute instance-groups list --project "$GCP_PROJECT" | grep capg-e2e \
        | awk '{print "gcloud compute instance-groups unmanaged delete --project '"$GCP_PROJECT"' --quiet " $1 " --zone " $2 "\n"}' \
        | bash) || true
-  (gcloud compute firewall-rules list --project "$GCP_PROJECT" | grep capg-e2e \
-       | awk '{print "gcloud compute firewall-rules delete --project '"$GCP_PROJECT"' --quiet " $1 "\n"}' \
-       | bash) || true
 
-  # cleanup the networks
-  gcloud compute routers nats delete "${TEST_NAME}-mynat" --project="${GCP_PROJECT}" \
-    --router-region="${GCP_REGION}" --router="${TEST_NAME}-myrouter" --quiet || true
-  gcloud compute routers delete "${TEST_NAME}-myrouter" --project="${GCP_PROJECT}" \
-    --region="${GCP_REGION}" --quiet || true
-
-  if [[ ${GCP_NETWORK_NAME} != "default" ]]; then
-    (gcloud compute firewall-rules list --project "$GCP_PROJECT" | grep "$GCP_NETWORK_NAME" \
-         | awk '{print "gcloud compute firewall-rules delete --project '"$GCP_PROJECT"' --quiet " $1 "\n"}' \
-         | bash) || true
-    gcloud compute networks delete --project="${GCP_PROJECT}" \
-      --quiet "${GCP_NETWORK_NAME}" || true
-  fi
+
+  ARRAY=( "$GCP_PROJECT:$GCP_NETWORK_NAME"
+          "$GCP_HOST_PROJECT:${GCP_NETWORK_NAME}-shared-vpc" )
+
+  for data in "${ARRAY[@]}"; do
+      PROJECT=${data%%:*}
+      NETWORK=${data#*:}
+
+      echo "Cleaning up network resources from project $PROJECT"
+
+      (gcloud compute firewall-rules list --project "$PROJECT" | grep capg-e2e \
+           | awk '{print "gcloud compute firewall-rules delete --project '"$PROJECT"' --quiet " $1 "\n"}' \
+           | bash) || true
+
+      # cleanup the networks
+      gcloud compute routers nats delete "${TEST_NAME}-mynat" --project="${PROJECT}" \
+        --router-region="${GCP_REGION}" --router="${TEST_NAME}-myrouter" --quiet || true
+      gcloud compute routers delete "${TEST_NAME}-myrouter" --project="${PROJECT}" \
+        --region="${GCP_REGION}" --quiet || true
+
+      if [[ ${NETWORK} != "default" ]]; then
+        (gcloud compute firewall-rules list --project "$PROJECT" | grep "$NETWORK" \
+             | awk '{print "gcloud compute firewall-rules delete --project '"$PROJECT"' --quiet " $1 "\n"}' \
+             | bash) || true
+        gcloud compute networks delete --project="${PROJECT}" \
+          --quiet "${NETWORK}" || true
+      fi
+
+  done
+
+
 
   if [[ -n "${SKIP_INIT_IMAGE:-}" ]]; then
     echo "Skipping GCP image deletion..."

diff --git a/test/e2e/config/gcp-ci.yaml b/test/e2e/config/gcp-ci.yaml
@@ -72,6 +72,7 @@ providers:
       - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke.yaml"
       - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-autopilot.yaml"
       - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-custom-subnet.yaml"
+      - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-shared-vpc.yaml"
 
 variables:
   KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.29.0}"

diff --git a/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-shared-vpc.yaml b/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-shared-vpc.yaml
@@ -0,0 +1,127 @@
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: "${CLUSTER_NAME}"
+  labels:
+    cni: "${CLUSTER_NAME}-shared-vpc"
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks: ["192.168.0.0/16"]
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: GCPCluster
+    name: "${CLUSTER_NAME}"
+  controlPlaneRef:
+    kind: KubeadmControlPlane
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    name: "${CLUSTER_NAME}-control-plane"
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: GCPCluster
+metadata:
+  name: "${CLUSTER_NAME}"
+spec:
+  project: "${GCP_PROJECT}"
+  region: "${GCP_REGION}"
+  network:
+    name: "${GCP_NETWORK_NAME}"
+    hostProject: "${GCP_HOST_PROJECT}"
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: KubeadmControlPlane
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+spec:
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  machineTemplate:
+    infrastructureRef:
+      kind: GCPMachineTemplate
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      name: "${CLUSTER_NAME}-control-plane"
+  kubeadmConfigSpec:
+    useExperimentalRetryJoin: true
+    initConfiguration:
+      nodeRegistration:
+        name: '{{ ds.meta_data.local_hostname.split(".")[0] }}'
+        kubeletExtraArgs:
+          cloud-provider: gce
+    clusterConfiguration:
+      apiServer:
+        timeoutForControlPlane: 20m
+        extraArgs:
+          cloud-provider: gce
+      controllerManager:
+        extraArgs:
+          cloud-provider: gce
+          allocate-node-cidrs: "false"
+      kubernetesVersion: "${KUBERNETES_VERSION}"
+    joinConfiguration:
+      nodeRegistration:
+        name: '{{ ds.meta_data.local_hostname.split(".")[0] }}'
+        kubeletExtraArgs:
+          cloud-provider: gce
+  version: "${KUBERNETES_VERSION}"
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: GCPMachineTemplate
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+spec:
+  template:
+    spec:
+      instanceType: "${GCP_CONTROL_PLANE_MACHINE_TYPE}"
+      image: "${IMAGE_ID}"
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  clusterName: "${CLUSTER_NAME}"
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels:
+  template:
+    spec:
+      clusterName: "${CLUSTER_NAME}"
+      version: "${KUBERNETES_VERSION}"
+      bootstrap:
+        configRef:
+          name: "${CLUSTER_NAME}-md-0"
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+      infrastructureRef:
+        name: "${CLUSTER_NAME}-md-0"
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: GCPMachineTemplate
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: GCPMachineTemplate
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  template:
+    spec:
+      instanceType: "${GCP_NODE_MACHINE_TYPE}"
+      image: "${IMAGE_ID}"
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  template:
+    spec:
+      joinConfiguration:
+        nodeRegistration:
+          name: '{{ ds.meta_data.local_hostname.split(".")[0] }}'
+          kubeletExtraArgs:
+            cloud-provider: gce
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "${CLUSTER_NAME}-shared-vpc"
+data: ${CNI_RESOURCES}
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
@@ -182,4 +182,29 @@ var _ = Describe("Workload cluster creation", func() {
 			}, result)
 		})
 	})
+
+	Context("Creating a control-plane cluster with a shared vpc", func() {
+		It("Should create a cluster with 1 control-plane and 1 worker node where the network exists in a host project", func() {
+			By("Creating a cluster where the host project shares network resources with the service project")
+			clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
+				ClusterProxy: bootstrapClusterProxy,
+				ConfigCluster: clusterctl.ConfigClusterInput{
+					LogFolder:                clusterctlLogFolder,
+					ClusterctlConfigPath:     clusterctlConfigPath,
+					KubeconfigPath:           bootstrapClusterProxy.GetKubeconfigPath(),
+					InfrastructureProvider:   clusterctl.DefaultInfrastructureProvider,
+					Flavor:                   "ci-with-shared-vpc",
+					Namespace:                namespace.Name,
+					ClusterName:              clusterName,
+					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
+					ControlPlaneMachineCount: ptr.To[int64](1),
+					WorkerMachineCount:       ptr.To[int64](1),
+				},
+				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
+				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
+				WaitForMachineDeployments:    e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
+			}, result)
+		})
+	})
+
 })