Add separate cluster templates for vm hosts

- create new set of templates for vm based hosts - update GH action to include multiple cluster templates in release artifacts - update make tasks to generate multiple cluster templates
vmware-tanzu · Jan 17, 2022 · 0a79567 · 0a79567
1 parent 072890a
commit 0a79567
Show file tree

Hide file tree

Showing 12 changed files with 358 additions and 4 deletions.
diff --git a/.github/workflows/draft-release.yaml b/.github/workflows/draft-release.yaml
@@ -31,6 +31,7 @@ jobs:
           files: |
             _dist/byoh-hostagent-linux-amd64
             _dist/cluster-template.yaml
+            _dist/cluster-template-docker.yaml
             _dist/infrastructure-components.yaml
             _dist/metadata.yaml
         env:

diff --git a/Makefile b/Makefile
@@ -125,7 +125,8 @@ test-e2e: docker-build prepare-byoh-docker-host-image $(GINKGO) cluster-template
 cluster-templates: kustomize cluster-templates-v1beta1
 
 cluster-templates-v1beta1: kustomize ## Generate cluster templates for v1beta1
-	$(KUSTOMIZE) build $(BYOH_TEMPLATES)/v1beta1 --load_restrictor none > $(BYOH_TEMPLATES)/v1beta1/cluster-template.yaml
+	$(KUSTOMIZE) build $(BYOH_TEMPLATES)/v1beta1/templates/vm --load_restrictor none > $(BYOH_TEMPLATES)/v1beta1/templates/vm/cluster-template.yaml
+	$(KUSTOMIZE) build $(BYOH_TEMPLATES)/v1beta1/templates/docker --load_restrictor none > $(BYOH_TEMPLATES)/v1beta1/templates/docker/cluster-template.yaml
 
 $(GINKGO): # Build ginkgo from tools folder.
 	cd $(TOOLS_DIR) && go build -tags=tools -o $(BIN_DIR)/ginkgo github.com/onsi/ginkgo/ginkgo
@@ -182,8 +183,10 @@ $(RELEASE_DIR):
 build-release-artifacts: build-cluster-templates build-infra-yaml build-metadata-yaml build-host-agent-binary
 
 build-cluster-templates: $(RELEASE_DIR) cluster-templates
-	cp $(BYOH_TEMPLATES)/v1beta1/cluster-template.yaml $(RELEASE_DIR)/cluster-template.yaml
-	sed -i -e 1,20d $(RELEASE_DIR)/cluster-template.yaml
+	cp $(BYOH_TEMPLATES)/v1beta1/templates/docker/cluster-template.yaml $(RELEASE_DIR)/cluster-template-docker.yaml
+	sed -i -e 1,20d $(RELEASE_DIR)/cluster-template-docker.yaml
+	cp $(BYOH_TEMPLATES)/v1beta1/templates/vm/cluster-template.yaml $(RELEASE_DIR)/cluster-template.yaml
+
 
 build-infra-yaml:kustomize # Generate infrastructure-components.yaml for the provider
 	cd config/manager && $(KUSTOMIZE) edit set image gcr.io/k8s-staging-cluster-api/cluster-api-byoh-controller=${IMG}

diff --git a/test/e2e/config/provider.yaml b/test/e2e/config/provider.yaml
@@ -74,7 +74,7 @@ providers:
             new: "imagePullPolicy: IfNotPresent"
         files:
           # Add a cluster template
-          - sourcePath: "../data/infrastructure-provider-byoh/v1beta1/cluster-template.yaml"
+          - sourcePath: "../data/infrastructure-provider-byoh/v1beta1/templates/docker/cluster-template.yaml"
           - sourcePath: "../../../metadata.yaml"
 variables:
   # default variables for the e2e test; those values could be overridden via env variables, thus

diff --git a/...ovider-byoh/v1beta1/cluster-template.yaml → ...a1/templates/docker/cluster-template.yaml b/...ovider-byoh/v1beta1/cluster-template.yaml → ...a1/templates/docker/cluster-template.yaml
diff --git a/...ovider-byoh/v1beta1/cluster-with-kcp.yaml → ...a1/templates/docker/cluster-with-kcp.yaml b/...ovider-byoh/v1beta1/cluster-with-kcp.yaml → ...a1/templates/docker/cluster-with-kcp.yaml
diff --git a/...astructure-provider-byoh/v1beta1/crs.yaml → ...er-byoh/v1beta1/templates/docker/crs.yaml b/...astructure-provider-byoh/v1beta1/crs.yaml → ...er-byoh/v1beta1/templates/docker/crs.yaml
diff --git a/...-provider-byoh/v1beta1/kustomization.yaml → ...beta1/templates/docker/kustomization.yaml b/...-provider-byoh/v1beta1/kustomization.yaml → ...beta1/templates/docker/kustomization.yaml
diff --git a/...rastructure-provider-byoh/v1beta1/md.yaml → ...der-byoh/v1beta1/templates/docker/md.yaml b/...rastructure-provider-byoh/v1beta1/md.yaml → ...der-byoh/v1beta1/templates/docker/md.yaml
diff --git a/test/e2e/data/infrastructure-provider-byoh/v1beta1/templates/vm/cluster-template.yaml b/test/e2e/data/infrastructure-provider-byoh/v1beta1/templates/vm/cluster-template.yaml
@@ -0,0 +1,176 @@
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+  namespace: default
+spec:
+  template:
+    spec: {}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  labels:
+    cni: ${CLUSTER_NAME}-crs-0
+    crs: "true"
+  name: ${CLUSTER_NAME}
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+      - 192.168.0.0/16
+    serviceDomain: cluster.local
+    services:
+      cidrBlocks:
+      - 10.128.0.0/12
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    kind: KubeadmControlPlane
+    name: ${CLUSTER_NAME}-control-plane
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: ByoCluster
+    name: ${CLUSTER_NAME}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels: null
+  template:
+    metadata:
+      labels:
+        nodepool: pool1
+    spec:
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+          name: ${CLUSTER_NAME}-md-0
+      clusterName: ${CLUSTER_NAME}
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: ByoMachineTemplate
+        name: ${CLUSTER_NAME}-md-0
+      version: ${KUBERNETES_VERSION}
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: KubeadmControlPlane
+metadata:
+  labels:
+    nodepool: pool0
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  kubeadmConfigSpec:
+    clusterConfiguration:
+      apiServer:
+        certSANs:
+        - localhost
+        - 127.0.0.1
+        - 0.0.0.0
+        - host.docker.internal
+      controllerManager:
+        extraArgs:
+          enable-hostpath-provisioner: "true"
+    files:
+    - content: |
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          creationTimestamp: null
+          name: kube-vip
+          namespace: kube-system
+        spec:
+          containers:
+          - args:
+            - start
+            env:
+            - name: vip_arp
+              value: "true"
+            - name: vip_leaderelection
+              value: "true"
+            - name: vip_address
+              value: ${CONTROL_PLANE_ENDPOINT_IP}
+            - name: vip_interface
+              value: {{ .DefaultNetworkInterfaceName }}
+            - name: vip_leaseduration
+              value: "15"
+            - name: vip_renewdeadline
+              value: "10"
+            - name: vip_retryperiod
+              value: "2"
+            image: ghcr.io/kube-vip/kube-vip:v0.3.5
+            imagePullPolicy: IfNotPresent
+            name: kube-vip
+            resources: {}
+            securityContext:
+              capabilities:
+                add:
+                - NET_ADMIN
+                - SYS_TIME
+            volumeMounts:
+            - mountPath: /etc/kubernetes/admin.conf
+              name: kubeconfig
+          hostNetwork: true
+          volumes:
+          - hostPath:
+              path: /etc/kubernetes/admin.conf
+              type: FileOrCreate
+            name: kubeconfig
+        status: {}
+      owner: root:root
+      path: /etc/kubernetes/manifests/kube-vip.yaml
+    initConfiguration:
+      nodeRegistration:
+        criSocket: /var/run/containerd/containerd.sock
+        ignorePreflightErrors:
+        - Swap
+        - DirAvailable--etc-kubernetes-manifests
+        - FileAvailable--etc-kubernetes-kubelet.conf
+    joinConfiguration:
+      nodeRegistration:
+        criSocket: /var/run/containerd/containerd.sock
+        ignorePreflightErrors:
+        - Swap
+        - DirAvailable--etc-kubernetes-manifests
+        - FileAvailable--etc-kubernetes-kubelet.conf
+  machineTemplate:
+    infrastructureRef:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: ByoMachineTemplate
+      name: ${CLUSTER_NAME}-control-plane
+      namespace: ${NAMESPACE}
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  version: ${KUBERNETES_VERSION}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: ByoCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  bundleLookupBaseRegistry: projects.registry.vmware.com/cluster_api_provider_bringyourownhost
+  bundleLookupTag: ${BUNDLE_LOOKUP_TAG}
+  controlPlaneEndpoint:
+    host: ${CONTROL_PLANE_ENDPOINT_IP}
+    port: 6443
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: ByoMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+  namespace: default
+spec:
+  template:
+    spec: {}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: ByoMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  template:
+    spec: {}
diff --git a/test/e2e/data/infrastructure-provider-byoh/v1beta1/templates/vm/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-provider-byoh/v1beta1/templates/vm/cluster-with-kcp.yaml
@@ -0,0 +1,126 @@
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: ${CLUSTER_NAME}
+  labels:
+    cni: ${CLUSTER_NAME}-crs-0
+    crs: "true"
+spec:
+  clusterNetwork:
+    services:
+      cidrBlocks: ["10.128.0.0/12"]
+    pods:
+      cidrBlocks: ["192.168.0.0/16"]
+    serviceDomain: "cluster.local"
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: ByoCluster
+    name: ${CLUSTER_NAME}
+  controlPlaneRef:
+    kind: KubeadmControlPlane
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    name: "${CLUSTER_NAME}-control-plane"
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: ByoCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  bundleLookupBaseRegistry: projects.registry.vmware.com/cluster_api_provider_bringyourownhost
+  bundleLookupTag: ${BUNDLE_LOOKUP_TAG}
+  controlPlaneEndpoint:
+    host: ${CONTROL_PLANE_ENDPOINT_IP}
+    port: 6443
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: ByoMachineTemplate
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+spec:
+  template:
+    spec: {}
+---
+kind: KubeadmControlPlane
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+  labels:
+    nodepool: "pool0"
+spec:
+  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+  machineTemplate:
+    infrastructureRef:
+      kind: ByoMachineTemplate
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      name: "${CLUSTER_NAME}-control-plane"
+      namespace: "${NAMESPACE}"
+  kubeadmConfigSpec:
+    clusterConfiguration:
+      controllerManager:
+        extraArgs: {enable-hostpath-provisioner: 'true'}
+      apiServer:
+        certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
+    files:
+    - content: |
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          creationTimestamp: null
+          name: kube-vip
+          namespace: kube-system
+        spec:
+          containers:
+          - args:
+            - start
+            env:
+            - name: vip_arp
+              value: "true"
+            - name: vip_leaderelection
+              value: "true"
+            - name: vip_address
+              value: ${CONTROL_PLANE_ENDPOINT_IP}
+            - name: vip_interface
+              value: {{ .DefaultNetworkInterfaceName }}
+            - name: vip_leaseduration
+              value: "15"
+            - name: vip_renewdeadline
+              value: "10"
+            - name: vip_retryperiod
+              value: "2"
+            image: ghcr.io/kube-vip/kube-vip:v0.3.5
+            imagePullPolicy: IfNotPresent
+            name: kube-vip
+            resources: {}
+            securityContext:
+              capabilities:
+                add:
+                - NET_ADMIN
+                - SYS_TIME
+            volumeMounts:
+            - mountPath: /etc/kubernetes/admin.conf
+              name: kubeconfig
+          hostNetwork: true
+          volumes:
+          - hostPath:
+              path: /etc/kubernetes/admin.conf
+              type: FileOrCreate
+            name: kubeconfig
+        status: {}
+      owner: root:root
+      path: /etc/kubernetes/manifests/kube-vip.yaml
+    initConfiguration:
+      nodeRegistration:
+        ignorePreflightErrors:
+        - Swap
+        - DirAvailable--etc-kubernetes-manifests
+        - FileAvailable--etc-kubernetes-kubelet.conf
+        criSocket: /var/run/containerd/containerd.sock
+    joinConfiguration:
+      nodeRegistration:
+        ignorePreflightErrors:
+        - Swap
+        - DirAvailable--etc-kubernetes-manifests
+        - FileAvailable--etc-kubernetes-kubelet.conf
+        criSocket: /var/run/containerd/containerd.sock
+  version: ${KUBERNETES_VERSION}
diff --git a/test/e2e/data/infrastructure-provider-byoh/v1beta1/templates/vm/kustomization.yaml b/test/e2e/data/infrastructure-provider-byoh/v1beta1/templates/vm/kustomization.yaml
@@ -0,0 +1,3 @@
+bases:
+  - cluster-with-kcp.yaml
+  - md.yaml
diff --git a/test/e2e/data/infrastructure-provider-byoh/v1beta1/templates/vm/md.yaml b/test/e2e/data/infrastructure-provider-byoh/v1beta1/templates/vm/md.yaml
@@ -0,0 +1,45 @@
+
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: ByoMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+  namespace: default
+spec:
+  template:
+    spec: {}
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+  namespace: default
+spec:
+  template:
+    spec: {}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels:
+  template:
+    metadata:
+      labels:
+        nodepool: "pool1"
+    spec:
+      clusterName: ${CLUSTER_NAME}
+      version: ${KUBERNETES_VERSION}
+      bootstrap:
+        configRef:
+          name: ${CLUSTER_NAME}-md-0
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+      infrastructureRef:
+        name: ${CLUSTER_NAME}-md-0
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: ByoMachineTemplate