Added e2e for cluster-class feature and CC docker templates

vmware-tanzu · Jun 26, 2022 · 1961cd8 · 1961cd8
1 parent f691bcf
commit 1961cd8
Show file tree

Hide file tree

Showing 7 changed files with 583 additions and 69 deletions.
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -25,5 +25,8 @@ jobs:
     - name: Set netfilter conntrack max
       run: sudo sysctl -w net.netfilter.nf_conntrack_max=131072
 
+    - name: Run Cluster Class e2e tests
+      run: yes | GINKGO_FOCUS="\[Cluster-Class\]" make test-e2e
+
     - name: Run PR-Blocking e2e tests
       run: yes | GINKGO_FOCUS="\[PR-Blocking\]" make test-e2e
diff --git a/test/e2e/config/provider.yaml b/test/e2e/config/provider.yaml
@@ -75,6 +75,8 @@ providers:
         files:
           # Add a cluster template
           - sourcePath: "../data/infrastructure-provider-byoh/v1beta1/templates/e2e/cluster-template.yaml"
+          - sourcePath: "../data/infrastructure-provider-byoh/v1beta1/templates/e2e/clusterclass-quickstart.yaml"
+          - sourcePath: "../data/infrastructure-provider-byoh/v1beta1/templates/e2e/cluster-template-topology.yaml"
           - sourcePath: "../../../metadata.yaml"
 variables:
   # default variables for the e2e test; those values could be overridden via env variables, thus
@@ -91,6 +93,7 @@ variables:
   CNI: "./data/cni/kindnet/kindnet.yaml"
   EXP_CLUSTER_RESOURCE_SET: "true"
   EXP_MACHINE_POOL: "true"
+  CLUSTER_TOPOLOGY: "true"
   KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml"
   NODE_DRAIN_TIMEOUT: "60s"
   # NOTE: INIT_WITH_BINARY is used only by the clusterctl upgrade test to initialize the management cluster to be upgraded

diff --git a/...data/infrastructure-provider-byoh/v1beta1/templates/docker/cluster-template-topology.yaml b/...data/infrastructure-provider-byoh/v1beta1/templates/docker/cluster-template-topology.yaml
@@ -0,0 +1,83 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: ${CLUSTER_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  clusterNetwork:
+    services:
+      cidrBlocks:
+        - 10.128.0.0/12
+    pods:
+      cidrBlocks:
+        - 192.168.0.0/16
+    serviceDomain: cluster.local
+  topology:
+    class: quickstart
+    version: ${KUBERNETES_VERSION}
+    controlPlane:
+      metadata: {}
+      replicas: 1
+    variables:
+      - name: bundleLookupBaseRegistry
+        value: "projects.registry.vmware.com/cluster_api_provider_bringyourownhost"
+      - name: controlPlaneIpAddr
+        value: ${CONTROL_PLANE_ENDPOINT_IP}
+      - name: kubeVipPodManifest
+        value: |
+          apiVersion: v1
+          kind: Pod
+          metadata:
+            creationTimestamp: null
+            name: kube-vip
+            namespace: kube-system
+          spec:
+            containers:
+            - args:
+              - manager
+              env:
+              - name: cp_enable
+                value: "true"
+              - name: vip_arp
+                value: "true"
+              - name: vip_leaderelection
+                value: "true"
+              - name: vip_address
+                value: ${CONTROL_PLANE_ENDPOINT_IP}
+              - name: vip_interface
+                value: "{{ .DefaultNetworkInterfaceName }}"
+              - name: vip_leaseduration
+                value: "15"
+              - name: vip_renewdeadline
+                value: "10"
+              - name: vip_retryperiod
+                value: "2"
+              image: ghcr.io/kube-vip/kube-vip:v0.4.1
+              imagePullPolicy: IfNotPresent
+              name: kube-vip
+              resources: {}
+              securityContext:
+                capabilities:
+                  add:
+                  - NET_ADMIN
+                  - NET_RAW
+              volumeMounts:
+              - mountPath: /etc/kubernetes/admin.conf
+                name: kubeconfig
+            hostNetwork: true
+            hostAliases:
+              - hostnames:
+                  - kubernetes
+                ip: 127.0.0.1
+            volumes:
+            - hostPath:
+                path: /etc/kubernetes/admin.conf
+                type: FileOrCreate
+              name: kubeconfig
+          status: {}
+    workers:
+      machineDeployments:
+        - class: quickstart-worker
+          metadata: {}
+          name: md-0
+          replicas: 1
diff --git a/...e2e/clusterclass-quickstart-template.yaml → ...lates/docker/clusterclass-quickstart.yaml b/...e2e/clusterclass-quickstart-template.yaml → ...lates/docker/clusterclass-quickstart.yaml
@@ -1,37 +1,38 @@
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: ClusterClass
 metadata:
-  name: ${CLUSTER_CLASS_NAME}
+  name: quickstart
+  namespace: ${NAMESPACE}
 spec:
   controlPlane:
     ref:
       apiVersion: controlplane.cluster.x-k8s.io/v1beta1
       kind: KubeadmControlPlaneTemplate
-      name: ${CLUSTER_CLASS_NAME}-control-plane
+      name: quickstart-control-plane
     machineInfrastructure:
       ref:
         kind: ByoMachineTemplate
         apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-        name: ${CLUSTER_CLASS_NAME}-control-plane-machine
+        name: quickstart-control-plane-machine
   infrastructure:
     ref:
       apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
       kind: ByoClusterTemplate
-      name: ${CLUSTER_CLASS_NAME}-cluster
+      name: quickstart-cluster
   workers:
     machineDeployments:
-      - class: ${CLUSTER_CLASS_NAME}-worker
+      - class: quickstart-worker
         template:
           bootstrap:
             ref:
               apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
               kind: KubeadmConfigTemplate
-              name: ${CLUSTER_CLASS_NAME}-worker-bootstrap-template
+              name: quickstart-worker-bootstrap-template
           infrastructure:
             ref:
               apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
               kind: ByoMachineTemplate
-              name: ${CLUSTER_CLASS_NAME}-worker-machinetemplate
+              name: quickstart-worker-machinetemplate
   variables:
     - name: bundleLookupBaseRegistry
       required: true
@@ -92,35 +93,13 @@ spec:
             matchResources:
               controlPlane: true
 ---
-apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-kind: ByoClusterTemplate
-metadata:
-  name: ${CLUSTER_CLASS_NAME}-cluster
-spec:
-  template:
-    spec: {}
----
-apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-kind: ByoMachineTemplate
-metadata:
-  name: ${CLUSTER_CLASS_NAME}-worker-machinetemplate
-  namespace: default
-spec:
-  template:
-    spec: {}
----
-apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-kind: ByoMachineTemplate
-metadata:
-  name: ${CLUSTER_CLASS_NAME}-control-plane-machine
-spec:
-  template:
-    spec: {}
----
 kind: KubeadmControlPlaneTemplate
 apiVersion: controlplane.cluster.x-k8s.io/v1beta1
 metadata:
-  name: ${CLUSTER_CLASS_NAME}-control-plane
+  labels:
+    nodepool: pool0
+  name: quickstart-control-plane
+  namespace: ${NAMESPACE}
 spec:
   template:
     spec:
@@ -142,53 +121,92 @@ spec:
           nodeRegistration:
             criSocket: /var/run/containerd/containerd.sock
             ignorePreflightErrors:
-            - Swap
-            - DirAvailable--etc-kubernetes-manifests
-            - FileAvailable--etc-kubernetes-kubelet.conf
+              - Swap
+              - DirAvailable--etc-kubernetes-manifests
+              - FileAvailable--etc-kubernetes-kubelet.conf
             kubeletExtraArgs:
               cgroup-driver: cgroupfs
               eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
         joinConfiguration:
           nodeRegistration:
             criSocket: /var/run/containerd/containerd.sock
             ignorePreflightErrors:
-            - Swap
-            - DirAvailable--etc-kubernetes-manifests
-            - FileAvailable--etc-kubernetes-kubelet.conf
+              - Swap
+              - DirAvailable--etc-kubernetes-manifests
+              - FileAvailable--etc-kubernetes-kubelet.conf
             kubeletExtraArgs:
               cgroup-driver: cgroupfs
               eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
 ---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: ByoMachineTemplate
+metadata:
+  name: quickstart-control-plane-machine
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      installerRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: K8sInstallerConfigTemplate
+        name: quickstart-control-plane-machine
+        namespace: "${NAMESPACE}"
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: K8sInstallerConfigTemplate
+metadata:
+  name: quickstart-control-plane-machine
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      bundleRepo: projects.registry.vmware.com/cluster_api_provider_bringyourownhost
+      bundleType: k8s
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: ByoClusterTemplate
+metadata:
+  name: quickstart-cluster
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec: {}
+
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: ByoMachineTemplate
+metadata:
+  name: quickstart-worker-machinetemplate
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      installerRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: K8sInstallerConfigTemplate
+        name: ${CLUSTER_NAME}-md-0
+        namespace: ${NAMESPACE}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: K8sInstallerConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      bundleRepo: projects.registry.vmware.com/cluster_api_provider_bringyourownhost
+      bundleType: k8s
+---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
 kind: KubeadmConfigTemplate
 metadata:
-  name: ${CLUSTER_CLASS_NAME}-worker-bootstrap-template
-  namespace: default
+  name: quickstart-worker-bootstrap-template
+  namespace: ${NAMESPACE}
 spec:
   template:
     spec:
       joinConfiguration:
         nodeRegistration:
           kubeletExtraArgs:
             cgroup-driver: cgroupfs
-            eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
----
-apiVersion: v1
-binaryData: null
-data: ${CNI_RESOURCES}
-kind: ConfigMap
-metadata:
-  name: cni-${CLUSTER_NAME}-crs-0
----
-apiVersion: addons.cluster.x-k8s.io/v1beta1
-kind: ClusterResourceSet
-metadata:
-  name: ${CLUSTER_NAME}-crs-0
-spec:
-  clusterSelector:
-    matchLabels:
-      cni: ${CLUSTER_NAME}-crs-0
-  resources:
-    - kind: ConfigMap
-      name: cni-${CLUSTER_NAME}-crs-0
-  strategy: ApplyOnce
+            eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%