From 80a6ddb1e16c8d8d78d5333b89c455f026584999 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 16 May 2022 12:07:47 -0700 Subject: [PATCH] test: remove legacy Windows dockershim templates --- Makefile | 2 - Tiltfile | 3 - docs/book/src/topics/windows.md | 45 +- templates/addons/flannel-resource-set.yaml | 13 - templates/addons/windows/flannel/flannel.yaml | 767 --------- .../windows/flannel/kube-proxy-windows.yaml | 98 -- ...mplate-machinepool-windows-containerd.yaml | 311 ---- .../cluster-template-machinepool-windows.yaml | 98 +- .../cluster-template-windows-containerd.yaml | 317 ---- templates/cluster-template-windows.yaml | 103 +- .../flavors/base-windows/kustomization.yaml | 6 - .../flavors/base-windows/patches/cluster.yaml | 12 - .../patches/kubeadm-control-plane.yaml | 41 - .../kustomization.yaml | 8 - .../machine-pool-deployment-windows.yaml | 76 - .../machinepool-windows/kustomization.yaml | 7 +- .../machine-pool-deployment-windows.yaml | 17 +- .../machine-pool-deployment.yaml | 78 - .../windows-containerd/kustomization.yaml | 9 - .../machine-deployment-windows.yaml | 84 - templates/flavors/windows/kustomization.yaml | 8 +- .../windows/machine-deployment-windows.yaml | 23 +- .../flavors/windows/machine-deployment.yaml | 84 - ...ster-template-prow-ci-version-windows.yaml | 1429 ----------------- .../cluster-template-prow-windows-addons.yaml | 863 ---------- .../ci/cluster-template-prow-windows.yaml | 1249 -------------- .../test/ci/patches/cluster-cni-windows.yaml | 8 - .../kustomization.yaml | 32 - .../machine-deployment-ci-version.yaml | 34 - .../patches/windows-image-update.yaml | 16 - .../ci/prow-machine-pool/kustomization.yaml | 4 +- .../cni-resource-set-windows.yaml | 14 - .../test/ci/prow-windows/kustomization.yaml | 30 - .../machine-deployment-worker-counts.yaml | 14 - .../patches/windows-tmp-folder.yaml | 12 - templates/test/ci/prow/kustomization.yaml | 2 +- ...luster-template-custom-builds-windows.yaml | 1372 ---------------- .../custom-builds-windows/kustomization.yaml | 27 - .../patches/custom-builds-windows.yaml | 34 - .../patches/windows-image-update.yaml | 16 - test/e2e/azure_test.go | 49 - test/e2e/config/azure-dev.yaml | 6 - 42 files changed, 111 insertions(+), 7310 deletions(-) delete mode 100644 templates/addons/flannel-resource-set.yaml delete mode 100644 templates/addons/windows/flannel/flannel.yaml delete mode 100644 templates/addons/windows/flannel/kube-proxy-windows.yaml delete mode 100644 templates/cluster-template-machinepool-windows-containerd.yaml delete mode 100644 templates/cluster-template-windows-containerd.yaml delete mode 100644 templates/flavors/base-windows/kustomization.yaml delete mode 100644 templates/flavors/base-windows/patches/cluster.yaml delete mode 100644 templates/flavors/base-windows/patches/kubeadm-control-plane.yaml delete mode 100644 templates/flavors/machinepool-windows-containerd/kustomization.yaml delete mode 100644 templates/flavors/machinepool-windows-containerd/machine-pool-deployment-windows.yaml delete mode 100644 templates/flavors/machinepool-windows/machine-pool-deployment.yaml delete mode 100644 templates/flavors/windows-containerd/kustomization.yaml delete mode 100644 templates/flavors/windows-containerd/machine-deployment-windows.yaml delete mode 100644 templates/flavors/windows/machine-deployment.yaml delete mode 100644 templates/test/ci/cluster-template-prow-ci-version-windows.yaml delete mode 100644 templates/test/ci/cluster-template-prow-windows-addons.yaml delete mode 100644 templates/test/ci/cluster-template-prow-windows.yaml delete mode 100644 templates/test/ci/patches/cluster-cni-windows.yaml delete mode 100644 templates/test/ci/prow-ci-version-windows/kustomization.yaml delete mode 100644 templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml delete mode 100644 templates/test/ci/prow-ci-version-windows/patches/windows-image-update.yaml delete mode 100644 templates/test/ci/prow-windows/cni-resource-set-windows.yaml delete mode 100644 templates/test/ci/prow-windows/kustomization.yaml delete mode 100644 templates/test/ci/prow-windows/patches/machine-deployment-worker-counts.yaml delete mode 100644 templates/test/ci/prow-windows/patches/windows-tmp-folder.yaml delete mode 100644 templates/test/dev/cluster-template-custom-builds-windows.yaml delete mode 100644 templates/test/dev/custom-builds-windows/kustomization.yaml delete mode 100644 templates/test/dev/custom-builds-windows/patches/custom-builds-windows.yaml delete mode 100644 templates/test/dev/custom-builds-windows/patches/windows-image-update.yaml diff --git a/Makefile b/Makefile index 6f7859aaf4e..7766e9a5c1c 100644 --- a/Makefile +++ b/Makefile @@ -272,10 +272,8 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) ## Create a management clust kubectl create configmap calico-ipv6-addon --from-file=templates/addons/calico-ipv6.yaml kubectl create configmap calico-dual-stack-addon --from-file=templates/addons/calico-dual-stack.yaml kubectl create configmap calico-windows-addon --from-file=templates/addons/windows/calico - kubectl create configmap flannel-windows-addon --from-file=templates/addons/windows/flannel kubectl apply -f templates/addons/calico-resource-set.yaml - kubectl apply -f templates/addons/flannel-resource-set.yaml # Wait for CAPZ deployments kubectl wait --for=condition=Available --timeout=5m -n capz-system deployment -l cluster.x-k8s.io/provider=infrastructure-azure diff --git a/Tiltfile b/Tiltfile index 3fbd5925b5f..b6cfbbfc55b 100644 --- a/Tiltfile +++ b/Tiltfile @@ -234,18 +234,15 @@ def create_crs(): local(kubectl_cmd + " create configmap calico-addon --from-file=templates/addons/calico.yaml") local(kubectl_cmd + " delete configmaps calico-ipv6-addon --ignore-not-found=true") local(kubectl_cmd + " create configmap calico-ipv6-addon --from-file=templates/addons/calico-ipv6.yaml") - local(kubectl_cmd + " delete configmaps flannel-windows-addon --ignore-not-found=true") local(kubectl_cmd + " delete configmaps csi-proxy-addon --ignore-not-found=true") local(kubectl_cmd + " create configmap csi-proxy-addon --from-file=templates/addons/windows/csi-proxy/csi-proxy.yaml") # need to set version for kube-proxy on windows. os.putenv("KUBERNETES_VERSION", settings.get("kubernetes_version", {})) - local(kubectl_cmd + " create configmap flannel-windows-addon --from-file=templates/addons/windows/flannel/ --dry-run=client -o yaml | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -") local(kubectl_cmd + " create configmap calico-windows-addon --from-file=templates/addons/windows/calico/ --dry-run=client -o yaml | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -") # set up crs local(kubectl_cmd + " apply -f templates/addons/calico-resource-set.yaml") - local(kubectl_cmd + " apply -f templates/addons/flannel-resource-set.yaml") local(kubectl_cmd + " apply -f templates/addons/windows/csi-proxy/csi-proxy-resource-set.yaml") # create flavor resources from cluster-template files in the templates directory diff --git a/docs/book/src/topics/windows.md b/docs/book/src/topics/windows.md index 7bbf613aec2..7b0e0f5b2df 100644 --- a/docs/book/src/topics/windows.md +++ b/docs/book/src/topics/windows.md @@ -6,7 +6,7 @@ CAPZ enables you to create Windows Kubernetes clusters on Microsoft Azure. We re ### Using Containerd for Windows Clusters -To deploy a cluster using Windows, use the [Windows Containerd flavor template](../../../../templates/cluster-template-machinepool-windows-containerd.yaml). +To deploy a cluster using Windows, use the [Windows flavor template](../../../../templates/cluster-template-machinepool-windows.yaml). ## Deploy a workload @@ -82,7 +82,7 @@ The Windows HostProcess Container feature is Alpha for Kubernetes v1.22 and Beta Current requirements: - Kubernetes 1.22+ -- containerd 1.6+ +- containerd 1.6+ - `WindowsHostProcessContainers` feature-gate (Alpha for v1.22) turned on for kube-apiserver and kubelet if using Kubernetes 1.22 These requirements are satisfied by the Windows Containerd Template and Azure Marketplace reference image `cncf-upstream:capi-windows:k8s-1dot22dot1-windows-2019-containerd:2021.10.15` @@ -93,21 +93,21 @@ See the CAPI proposal for implementation details: https://github.com/kubernetes- ### VM and VMSS naming -Azure does not support creating Windows VM's with names longer than 15 characters ([see additional details historical restrictions](https://github.com/kubernetes-sigs/cluster-api/issues/2217#issuecomment-743336941)). +Azure does not support creating Windows VM's with names longer than 15 characters ([see additional details historical restrictions](https://github.com/kubernetes-sigs/cluster-api/issues/2217#issuecomment-743336941)). -When creating a cluster with `AzureMachine` if the AzureMachine is longer than 15 characters then the first 9 characters of the cluster name and appends the last 5 characters of the machine to create a unique machine name. +When creating a cluster with `AzureMachine` if the AzureMachine is longer than 15 characters then the first 9 characters of the cluster name and appends the last 5 characters of the machine to create a unique machine name. When creating a cluster with `Machinepool` if the Machine Pool name is longer than 9 characters then the Machine pool uses the prefix `win` and appends the last 5 characters of the machine pool name. ### VM password and access The VM password is [random generated](https://cloudbase-init.readthedocs.io/en/latest/plugins.html#setting-password-main) by Cloudbase-init during provisioning of the VM. For Access to the VM you can use ssh which will be configured with SSH -public key you provided during deployment. +public key you provided during deployment. To SSH: ``` -ssh -t -i .sshkey -o 'ProxyCommand ssh -i .sshkey -W %h:%p capi@' capi@ +ssh -t -i .sshkey -o 'ProxyCommand ssh -i .sshkey -W %h:%p capi@' capi@ ``` > There is also a [CAPZ kubectl plugin](https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/main/hack/debugging/Readme.md) that automates the ssh connection using the Management cluster @@ -121,12 +121,12 @@ ssh -L 5555::3389 capi@ And then open an RDP client on your local machine to `localhost:5555` ### Image creation -The images are built using [image-builder](https://github.com/kubernetes-sigs/image-builder) and published the the Azure Market place. They use [Cloudbase-init](https://cloudbase-init.readthedocs.io/en/latest/) to bootstrap the machines via Kubeadm. +The images are built using [image-builder](https://github.com/kubernetes-sigs/image-builder) and published the the Azure Market place. They use [Cloudbase-init](https://cloudbase-init.readthedocs.io/en/latest/) to bootstrap the machines via Kubeadm. -Find the latest published images: +Find the latest published images: ``` -az vm image list --publisher cncf-upstream --offer capi-windows -o table --all +az vm image list --publisher cncf-upstream --offer capi-windows -o table --all Offer Publisher Sku Urn Version ------------ ------------- ---------------------------- ------------------------------------------------------------------ ---------- capi-windows cncf-upstream k8s-1dot22dot1-windows-2019-containerd cncf-upstream:capi-windows:k8s-1dot22dot1-windows-2019-containerd:2021.10.15 2021.10.15 @@ -134,30 +134,3 @@ capi-windows cncf-upstream k8s-1dot22dot2-windows-2019-containerd cncf-upstre ``` If you would like customize your images please refer to the documentation on building your own [custom images](custom-images.md). - -### Using Docker EE and dockershim for Windows Clusters - - - -Windows nodes can either run [Containerd (recommended)](#using-containerd-for-windows-clusters) or Docker EE as the container runtime. -Docker EE requires the dockershim which will be [removed starting with Kubernetes 1.24](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed) and -will be [maintained by Mirantis](https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/) in the future. We do not plan to support dockershim -after its removal from upstream kubernetes in 1.24. - -To deploy a cluster using Windows using dockershim, use the [Windows flavor template](https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/main/templates/cluster-template-windows.yaml). - -#### Kube-proxy and CNIs for dockershim - -Kube-proxy and Windows CNIs are deployed via Cluster Resource Sets. Windows does not have a kube-proxy image due -to not having Privileged containers which would provide access to the host. The current solution is using wins.exe as -demonstrated in the [Kubeadm support for Windows](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/adding-windows-nodes/) guide. You may choose to run components as Windows services directly on the node but will require a [custom image](#image-creation) and modifications to the default Docker EE windows template. - -Flannel is being used as the default CNI with Docker EE and dockershim. An important note for Flannel vxlan deployments is that the MTU for the linux nodes must be set to 1400. -This is because [Azure's VNET MTU is 1400](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-tcpip-performance-tuning#azure-and-vm-mtu) which can cause fragmentation on packets sent from the Linux node to Windows node resulting in dropped packets. -To mitigate this we set the Linux eth0 port match 1400 and Flannel will automatically pick this up and [subtract 50](https://github.com/flannel-io/flannel/issues/1011) for the flannel network created. \ No newline at end of file diff --git a/templates/addons/flannel-resource-set.yaml b/templates/addons/flannel-resource-set.yaml deleted file mode 100644 index 67f80a39d35..00000000000 --- a/templates/addons/flannel-resource-set.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: flannel-windows - namespace: default -spec: - strategy: "ApplyOnce" - clusterSelector: - matchLabels: - cni: flannel-windows - resources: - - name: flannel-windows-addon - kind: ConfigMap diff --git a/templates/addons/windows/flannel/flannel.yaml b/templates/addons/windows/flannel/flannel.yaml deleted file mode 100644 index 6edb9f0e314..00000000000 --- a/templates/addons/windows/flannel/flannel.yaml +++ /dev/null @@ -1,767 +0,0 @@ ---- -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: psp.flannel.unprivileged - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default - seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default - apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default - apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default -spec: - privileged: false - volumes: - - configMap - - secret - - emptyDir - - hostPath - allowedHostPaths: - - pathPrefix: "/etc/cni/net.d" - - pathPrefix: "/etc/kube-flannel" - - pathPrefix: "/run/flannel" - readOnlyRootFilesystem: false - # Users and groups - runAsUser: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - fsGroup: - rule: RunAsAny - # Privilege Escalation - allowPrivilegeEscalation: false - defaultAllowPrivilegeEscalation: false - # Capabilities - allowedCapabilities: ['NET_ADMIN'] - defaultAddCapabilities: [] - requiredDropCapabilities: [] - # Host namespaces - hostPID: false - hostIPC: false - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - # SELinux - seLinux: - # SELinux is unused in CaaSP - rule: 'RunAsAny' ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: flannel -rules: - - apiGroups: ['extensions'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: ['psp.flannel.unprivileged'] - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - apiGroups: - - "" - resources: - - nodes - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: flannel -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flannel -subjects: -- kind: ServiceAccount - name: flannel - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flannel - namespace: kube-system ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: kube-flannel-cfg - namespace: kube-system - labels: - tier: node - app: flannel -data: - cni-conf.json: | - { - "name": "cbr0", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "flannel", - "delegate": { - "hairpinMode": true, - "isDefaultGateway": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - net-conf.json: | - { - "Network": "10.244.0.0/16", - "Backend": { - "Type": "vxlan", - "VNI" : 4096, - "Port": 4789 - } - } ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-flannel-ds-amd64 - namespace: kube-system - labels: - tier: node - app: flannel -spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-amd64 - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-amd64 - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-flannel-ds-arm64 - namespace: kube-system - labels: - tier: node - app: flannel -spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - arm64 - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-arm64 - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-arm64 - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-flannel-ds-arm - namespace: kube-system - labels: - tier: node - app: flannel -spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - arm - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-arm - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-arm - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-flannel-ds-ppc64le - namespace: kube-system - labels: - tier: node - app: flannel -spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-ppc64le - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-ppc64le - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-flannel-ds-s390x - namespace: kube-system - labels: - tier: node - app: flannel -spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-s390x - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-s390x - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: kube-flannel-windows-cfg - namespace: kube-system - labels: - tier: node - app: flannel -data: - run.ps1: | - $ErrorActionPreference = "Stop"; - - mkdir -force /host/etc/cni/net.d - mkdir -force /host/etc/kube-flannel - mkdir -force /host/opt/cni/bin - mkdir -force /host/k/flannel - mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount - - $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json - $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet - $podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet - $networkJson = wins cli net get | convertfrom-json - - $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet - $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet - Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100) - - cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel - cp -force -recurse /cni/* /host/opt/cni/bin - cp -force /k/flannel/* /host/k/flannel/ - cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml - cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/ - wins cli process run --path /k/flannel/setup.exe --args "--mode=overlay --interface=Ethernet 2" - wins cli route add --addresses 169.254.169.254 - wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE" - cni-conf.json: | - { - "name": "flannel.4096", - "cniVersion": "0.3.0", - "type": "flannel", - "capabilities": { - "dns": true - }, - "delegate": { - "type": "win-overlay", - "policies": [ - { - "Name": "EndpointPolicy", - "Value": { - "Type": "OutBoundNAT", - "ExceptionList": [] - } - }, - { - "Name": "EndpointPolicy", - "Value": { - "Type": "ROUTE", - "DestinationPrefix": "", - "NeedEncap": true - } - } - ] - } - } ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-flannel-ds-windows-amd64 - labels: - tier: node - app: flannel - namespace: kube-system -spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - windows - - key: kubernetes.io/arch - operator: In - values: - - amd64 - hostNetwork: true - serviceAccountName: flannel - tolerations: - - operator: Exists - effect: NoSchedule - containers: - - name: kube-flannel - image: sigwindowstools/flannel:v0.13.0-nanoserver - command: - - pwsh - args: - - -file - - /etc/kube-flannel-windows/run.ps1 - volumeMounts: - - name: wins - mountPath: \\.\pipe\rancher_wins - - name: host - mountPath: /host - - name: kube-proxy - mountPath: /kube-proxy - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - - name: flannel-windows-cfg - mountPath: /etc/kube-flannel-windows/ - - name: kubeadm-config - mountPath: /etc/kubeadm-config/ - env: - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - volumes: - - name: wins - hostPath: - path: \\.\pipe\rancher_wins - type: null - - name: opt - hostPath: - path: /opt - - name: host - hostPath: - path: / - - name: cni - hostPath: - path: /etc - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - - name: flannel-windows-cfg - configMap: - name: kube-flannel-windows-cfg - - name: kube-proxy - configMap: - name: kube-proxy - - name: kubeadm-config - configMap: - name: kubeadm-config - diff --git a/templates/addons/windows/flannel/kube-proxy-windows.yaml b/templates/addons/windows/flannel/kube-proxy-windows.yaml deleted file mode 100644 index acd27d78209..00000000000 --- a/templates/addons/windows/flannel/kube-proxy-windows.yaml +++ /dev/null @@ -1,98 +0,0 @@ -apiVersion: v1 -data: - run-script.ps1: |- - $ErrorActionPreference = "Stop"; - mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount - mkdir -force /host/k/kube-proxy - - $$CI_VERSION="${CI_VERSION:-}" - if($$CI_VERSION -ne "" -And (Test-Path -Path "/host/k/kube-proxy.exe")) - { - cp -force /host/k/kube-proxy.exe /k/kube-proxy/kube-proxy.exe - } - - cp -force /k/kube-proxy/* /host/k/kube-proxy - cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy - cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME? - - $networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name - $sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "." - yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip - yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName - yq w -i /host/var/lib/kube-proxy/config.conf featureGates.WinOverlay true - yq w -i /host/var/lib/kube-proxy/config.conf mode "kernelspace" - wins cli process run --path /k/kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true" -kind: ConfigMap -metadata: - labels: - app: kube-proxy - name: kube-proxy-windows - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - containers: - - command: - - pwsh - args: - - -file - - /var/lib/kube-proxy-windows/run-script.ps1 - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-nanoserver - name: kube-proxy - volumeMounts: - - name: wins - mountPath: \\.\pipe\rancher_wins - - name: host - mountPath: /host - - mountPath: /var/lib/kube-proxy - name: kube-proxy - - mountPath: /var/lib/kube-proxy-windows - name: kube-proxy-windows - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - name: wins - hostPath: - path: \\.\pipe\rancher_wins - type: null - - configMap: - defaultMode: 420 - name: kube-proxy-windows - name: kube-proxy-windows - - configMap: - name: kube-proxy - name: kube-proxy - - hostPath: - path: / - name: host - updateStrategy: - type: RollingUpdate diff --git a/templates/cluster-template-machinepool-windows-containerd.yaml b/templates/cluster-template-machinepool-windows-containerd.yaml deleted file mode 100644 index 2fe28351f62..00000000000 --- a/templates/cluster-template-machinepool-windows-containerd.yaml +++ /dev/null @@ -1,311 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: calico - csi-proxy: enabled - windows: enabled - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - feature-gates: WindowsHostProcessContainers=true - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - location: ${AZURE_LOCATION} - strategy: - rollingUpdate: - deletePolicy: Oldest - maxSurge: 25% - maxUnavailable: 1 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - annotations: - runtime: containerd - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - location: ${AZURE_LOCATION} - template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure - feature-gates: WindowsHostProcessContainers=true - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.4.1 - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell c:/create-external-network.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} diff --git a/templates/cluster-template-machinepool-windows.yaml b/templates/cluster-template-machinepool-windows.yaml index 8a75d813a0b..2fe28351f62 100644 --- a/templates/cluster-template-machinepool-windows.yaml +++ b/templates/cluster-template-machinepool-windows.yaml @@ -2,14 +2,16 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: - cni: flannel-windows + cni: calico + csi-proxy: enabled + windows: enabled name: ${CLUSTER_NAME} namespace: default spec: clusterNetwork: pods: cidrBlocks: - - 10.244.0.0/16 + - 192.168.0.0/16 controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane @@ -55,6 +57,7 @@ spec: extraArgs: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure + feature-gates: WindowsHostProcessContainers=true extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -63,11 +66,10 @@ spec: timeoutForControlPlane: 20m controllerManager: extraArgs: - allocate-node-cidrs: "true" + allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json cloud-provider: azure cluster-name: ${CLUSTER_NAME} - configure-cloud-routes: "false" extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -96,18 +98,6 @@ spec: overwrite: false tableType: gpt files: - - content: | - network: - version: 2 - ethernets: - eth0: - mtu: 1400 - match: - macaddress: MACADDRESS - set-name: eth0 - owner: root:root - path: /etc/netplan/60-eth0.yaml - permissions: "0644" - contentFrom: secret: key: control-plane-azure.json @@ -132,10 +122,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk - postKubeadmCommands: - - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') - - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml - - netplan apply + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: @@ -163,22 +150,6 @@ spec: sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal ---- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachinePool metadata: @@ -208,6 +179,12 @@ metadata: namespace: default spec: location: ${AZURE_LOCATION} + strategy: + rollingUpdate: + deletePolicy: Oldest + maxSurge: 25% + maxUnavailable: 1 + type: RollingUpdate template: osDisk: diskSizeGB: 30 @@ -231,18 +208,6 @@ spec: owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" - - content: | - network: - version: 2 - ethernets: - eth0: - mtu: 1400 - match: - macaddress: MACADDRESS - set-name: eth0 - owner: root:root - path: /etc/netplan/60-eth0.yaml - permissions: "0644" joinConfiguration: nodeRegistration: kubeletExtraArgs: @@ -250,11 +215,22 @@ spec: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') - - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml - - netplan apply - useExperimentalRetryJoin: true +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachinePool @@ -282,7 +258,7 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureMachinePool metadata: annotations: - runtime: dockershim + runtime: containerd name: ${CLUSTER_NAME}-mp-win namespace: default spec: @@ -310,24 +286,22 @@ spec: owner: root:root path: c:/k/azure.json permissions: "0644" - - content: |- - # required as a work around for Flannel and Wins bugs - # https://github.com/coreos/flannel/issues/1359 - # https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828 - ipmo C:\k\debug\hns.psm1; - New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; }) - path: C:/create-external-network.ps1 + - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + path: C:/defender-exclude-calico.ps1 permissions: "0744" joinConfiguration: nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json cloud-config: c:/k/azure.json cloud-provider: azure - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:1.4.1 + feature-gates: WindowsHostProcessContainers=true + pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.4.1 name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 preKubeadmCommands: - powershell c:/create-external-network.ps1 users: diff --git a/templates/cluster-template-windows-containerd.yaml b/templates/cluster-template-windows-containerd.yaml deleted file mode 100644 index 165c73a2c9d..00000000000 --- a/templates/cluster-template-windows-containerd.yaml +++ /dev/null @@ -1,317 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: calico - csi-proxy: enabled - windows: enabled - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - feature-gates: WindowsHostProcessContainers=true - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: null - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: [] ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: null - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: containerd - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: containerd - spec: - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure - feature-gates: WindowsHostProcessContainers=true - v: "2" - windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: [] - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index 7d5a173e446..165c73a2c9d 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -2,14 +2,16 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: - cni: flannel-windows + cni: calico + csi-proxy: enabled + windows: enabled name: ${CLUSTER_NAME} namespace: default spec: clusterNetwork: pods: cidrBlocks: - - 10.244.0.0/16 + - 192.168.0.0/16 controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane @@ -55,6 +57,7 @@ spec: extraArgs: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure + feature-gates: WindowsHostProcessContainers=true extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -63,11 +66,10 @@ spec: timeoutForControlPlane: 20m controllerManager: extraArgs: - allocate-node-cidrs: "true" + allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json cloud-provider: azure cluster-name: ${CLUSTER_NAME} - configure-cloud-routes: "false" extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -96,18 +98,6 @@ spec: overwrite: false tableType: gpt files: - - content: | - network: - version: 2 - ethernets: - eth0: - mtu: 1400 - match: - macaddress: MACADDRESS - set-name: eth0 - owner: root:root - path: /etc/netplan/60-eth0.yaml - permissions: "0644" - contentFrom: secret: key: control-plane-azure.json @@ -132,10 +122,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk - postKubeadmCommands: - - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') - - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml - - netplan apply + postKubeadmCommands: [] preKubeadmCommands: [] machineTemplate: infrastructureRef: @@ -170,7 +157,7 @@ metadata: namespace: default spec: clusterName: ${CLUSTER_NAME} - replicas: ${LINUX_WORKER_MACHINE_COUNT:-1} + replicas: ${WORKER_MACHINE_COUNT} selector: matchLabels: null template: @@ -197,8 +184,6 @@ spec: spec: osDisk: diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} vmSize: ${AZURE_NODE_MACHINE_TYPE} @@ -219,18 +204,6 @@ spec: owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" - - content: | - network: - version: 2 - ethernets: - eth0: - mtu: 1400 - match: - macaddress: MACADDRESS - set-name: eth0 - owner: root:root - path: /etc/netplan/60-eth0.yaml - permissions: "0644" joinConfiguration: nodeRegistration: kubeletExtraArgs: @@ -238,12 +211,23 @@ spec: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') - - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml - - netplan apply preKubeadmCommands: [] - useExperimentalRetryJoin: true +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -273,14 +257,14 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureMachineTemplate metadata: annotations: - runtime: dockershim + runtime: containerd name: ${CLUSTER_NAME}-md-win namespace: default spec: template: metadata: annotations: - runtime: dockershim + runtime: containerd spec: osDisk: diskSizeGB: 128 @@ -306,43 +290,28 @@ spec: owner: root:root path: c:/k/azure.json permissions: "0644" - - content: | - # required as a work around for Flannel and Wins bugs - # https://github.com/coreos/flannel/issues/1359 - # https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828 - ipmo C:\k\debug\hns.psm1; - New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; }) - path: C:/create-external-network.ps1 + - content: |- + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe + path: C:/defender-exclude-calico.ps1 permissions: "0744" joinConfiguration: nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json cloud-config: c:/k/azure.json cloud-provider: azure + feature-gates: WindowsHostProcessContainers=true + v: "2" + windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - nssm set kubelet start SERVICE_AUTO_START - preKubeadmCommands: - - powershell c:/create-external-network.ps1 + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: [] users: - groups: Administrators name: capi sshAuthorizedKeys: - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal diff --git a/templates/flavors/base-windows/kustomization.yaml b/templates/flavors/base-windows/kustomization.yaml deleted file mode 100644 index 2cf3489d21a..00000000000 --- a/templates/flavors/base-windows/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -namespace: default -resources: - - ../base -patchesStrategicMerge: - - patches/kubeadm-control-plane.yaml - - patches/cluster.yaml diff --git a/templates/flavors/base-windows/patches/cluster.yaml b/templates/flavors/base-windows/patches/cluster.yaml deleted file mode 100644 index 21d98864c75..00000000000 --- a/templates/flavors/base-windows/patches/cluster.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - labels: - cni: "flannel-windows" -spec: - clusterNetwork: - pods: - cidrBlocks: - - 10.244.0.0/16 \ No newline at end of file diff --git a/templates/flavors/base-windows/patches/kubeadm-control-plane.yaml b/templates/flavors/base-windows/patches/kubeadm-control-plane.yaml deleted file mode 100644 index 261b1f48167..00000000000 --- a/templates/flavors/base-windows/patches/kubeadm-control-plane.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - kubeadmConfigSpec: - postKubeadmCommands: - # Azures vnet MTU is 1400. - # When using Flannel VXLAN to avoid packet fragmentation - # that results dropped packets on Windows we need to match. - # Flannel will automatically choose eth0 - 50 - # a bug in netplan requires matching on macaddress - # https://bugs.launchpad.net/netplan/+bug/1807273 - - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') - - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml - - netplan apply - clusterConfiguration: - controllerManager: - extraArgs: - allocate-node-cidrs: "true" - configure-cloud-routes: "false" - files: - - path: /etc/netplan/60-eth0.yaml - permissions: "0644" - owner: root:root - content: | - network: - version: 2 - ethernets: - eth0: - mtu: 1400 - match: - macaddress: MACADDRESS - set-name: eth0 - - contentFrom: - secret: - name: ${CLUSTER_NAME}-control-plane-azure-json - key: control-plane-azure.json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" diff --git a/templates/flavors/machinepool-windows-containerd/kustomization.yaml b/templates/flavors/machinepool-windows-containerd/kustomization.yaml deleted file mode 100644 index 83a53584e22..00000000000 --- a/templates/flavors/machinepool-windows-containerd/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -namespace: default -resources: - - ../machinepool - - machine-pool-deployment-windows.yaml - -patchesStrategicMerge: - - ../base-windows-containerd/kubeadm-control-plane.yaml - - ../base-windows-containerd/cluster.yaml diff --git a/templates/flavors/machinepool-windows-containerd/machine-pool-deployment-windows.yaml b/templates/flavors/machinepool-windows-containerd/machine-pool-deployment-windows.yaml deleted file mode 100644 index 7f90b786d32..00000000000 --- a/templates/flavors/machinepool-windows-containerd/machine-pool-deployment-windows.yaml +++ /dev/null @@ -1,76 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: "${CLUSTER_NAME}-mp-win" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-mp-win" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - infrastructureRef: - name: "${CLUSTER_NAME}-mp-win" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: "${CLUSTER_NAME}-mp-win" - annotations: - runtime: containerd -spec: - location: ${AZURE_LOCATION} - template: - vmSize: ${AZURE_NODE_MACHINE_TYPE} - osDisk: - osType: "Windows" - diskSizeGB: 30 - managedDisk: - storageAccountType: "Premium_LRS" - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: "${CLUSTER_NAME}-mp-win" -spec: - users: - - name: capi - groups: Administrators - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} - preKubeadmCommands: - - powershell c:/create-external-network.ps1 - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - joinConfiguration: - nodeRegistration: - name: '{{ ds.meta_data["local_hostname"] }}' - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: azure - cloud-config: 'c:/k/azure.json' - azure-container-registry-config: 'c:/k/azure.json' - pod-infra-container-image: "mcr.microsoft.com/oss/kubernetes/pause:3.4.1" - feature-gates: "WindowsHostProcessContainers=true" - files: - - contentFrom: - secret: - name: ${CLUSTER_NAME}-mp-win-azure-json - key: worker-node-azure.json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe diff --git a/templates/flavors/machinepool-windows/kustomization.yaml b/templates/flavors/machinepool-windows/kustomization.yaml index 586f3a62e83..83a53584e22 100644 --- a/templates/flavors/machinepool-windows/kustomization.yaml +++ b/templates/flavors/machinepool-windows/kustomization.yaml @@ -1,9 +1,8 @@ namespace: default resources: - - ../base-windows - - ../../azure-cluster-identity - - machine-pool-deployment.yaml + - ../machinepool - machine-pool-deployment-windows.yaml patchesStrategicMerge: - - ../../azure-cluster-identity/azurecluster-identity-ref.yaml + - ../base-windows-containerd/kubeadm-control-plane.yaml + - ../base-windows-containerd/cluster.yaml diff --git a/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml b/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml index fdd224f7b94..7f90b786d32 100644 --- a/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml +++ b/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml @@ -25,7 +25,7 @@ kind: AzureMachinePool metadata: name: "${CLUSTER_NAME}-mp-win" annotations: - runtime: dockershim + runtime: containerd spec: location: ${AZURE_LOCATION} template: @@ -51,14 +51,17 @@ spec: - powershell c:/create-external-network.ps1 postKubeadmCommands: - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 joinConfiguration: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' + criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: cloud-provider: azure cloud-config: 'c:/k/azure.json' azure-container-registry-config: 'c:/k/azure.json' - pod-infra-container-image: "mcr.microsoft.com/oss/kubernetes/pause:1.4.1" + pod-infra-container-image: "mcr.microsoft.com/oss/kubernetes/pause:3.4.1" + feature-gates: "WindowsHostProcessContainers=true" files: - contentFrom: secret: @@ -67,11 +70,7 @@ spec: owner: root:root path: c:/k/azure.json permissions: "0644" - - path: C:/create-external-network.ps1 + - path: C:/defender-exclude-calico.ps1 permissions: "0744" - content: | - # required as a work around for Flannel and Wins bugs - # https://github.com/coreos/flannel/issues/1359 - # https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828 - ipmo C:\k\debug\hns.psm1; - New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; }) \ No newline at end of file + content: |- + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe diff --git a/templates/flavors/machinepool-windows/machine-pool-deployment.yaml b/templates/flavors/machinepool-windows/machine-pool-deployment.yaml deleted file mode 100644 index dab4b271937..00000000000 --- a/templates/flavors/machinepool-windows/machine-pool-deployment.yaml +++ /dev/null @@ -1,78 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-mp-0" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - infrastructureRef: - name: "${CLUSTER_NAME}-mp-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - location: ${AZURE_LOCATION} - template: - vmSize: ${AZURE_NODE_MACHINE_TYPE} - osDisk: - osType: "Linux" - diskSizeGB: 30 - managedDisk: - storageAccountType: "Premium_LRS" - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - postKubeadmCommands: - # Azures vnet MTU is 1400. - # When using Flannel VXLAN to avoid packet fragmentation - # that results dropped packets on Windows we need to match. - # Flannel will automatically choose eth0 - 50 - - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') - - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml - - netplan apply - useExperimentalRetryJoin: true - joinConfiguration: - nodeRegistration: - name: '{{ ds.meta_data["local_hostname"] }}' - kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json - azure-container-registry-config: /etc/kubernetes/azure.json - files: - - contentFrom: - secret: - name: ${CLUSTER_NAME}-mp-0-azure-json - key: worker-node-azure.json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - path: /etc/netplan/60-eth0.yaml - permissions: "0644" - owner: root:root - content: | - network: - version: 2 - ethernets: - eth0: - mtu: 1400 - match: - macaddress: MACADDRESS - set-name: eth0 diff --git a/templates/flavors/windows-containerd/kustomization.yaml b/templates/flavors/windows-containerd/kustomization.yaml deleted file mode 100644 index f119c9653dc..00000000000 --- a/templates/flavors/windows-containerd/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -namespace: default -resources: - - ../default - - machine-deployment-windows.yaml - -patchesStrategicMerge: - - ../base-windows-containerd/kubeadm-control-plane.yaml - - ../base-windows-containerd/cluster.yaml - diff --git a/templates/flavors/windows-containerd/machine-deployment-windows.yaml b/templates/flavors/windows-containerd/machine-deployment-windows.yaml deleted file mode 100644 index 98ed536ff79..00000000000 --- a/templates/flavors/windows-containerd/machine-deployment-windows.yaml +++ /dev/null @@ -1,84 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: "${CLUSTER_NAME}-md-win" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-md-win" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - infrastructureRef: - name: "${CLUSTER_NAME}-md-win" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: "${CLUSTER_NAME}-md-win" - annotations: - runtime: containerd -spec: - template: - metadata: - annotations: - runtime: containerd - spec: - vmSize: ${AZURE_NODE_MACHINE_TYPE} - osDisk: - osType: "Windows" - diskSizeGB: 128 - managedDisk: - storageAccountType: "Premium_LRS" - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: "${CLUSTER_NAME}-md-win" -spec: - template: - spec: - users: - - name: capi - groups: Administrators - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} - preKubeadmCommands: [] - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - joinConfiguration: - nodeRegistration: - name: '{{ ds.meta_data["local_hostname"] }}' - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - cloud-provider: azure - cloud-config: 'c:/k/azure.json' - azure-container-registry-config: 'c:/k/azure.json' - feature-gates: "WindowsHostProcessContainers=true" - v: "2" - windows-priorityclass: "ABOVE_NORMAL_PRIORITY_CLASS" - files: - - contentFrom: - secret: - name: ${CLUSTER_NAME}-md-win-azure-json - key: worker-node-azure.json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe diff --git a/templates/flavors/windows/kustomization.yaml b/templates/flavors/windows/kustomization.yaml index a5902fd9ea8..f119c9653dc 100644 --- a/templates/flavors/windows/kustomization.yaml +++ b/templates/flavors/windows/kustomization.yaml @@ -1,9 +1,9 @@ namespace: default resources: - - ../base-windows - - machine-deployment.yaml + - ../default - machine-deployment-windows.yaml - - ../../azure-cluster-identity patchesStrategicMerge: - - ../../azure-cluster-identity/azurecluster-identity-ref.yaml + - ../base-windows-containerd/kubeadm-control-plane.yaml + - ../base-windows-containerd/cluster.yaml + diff --git a/templates/flavors/windows/machine-deployment-windows.yaml b/templates/flavors/windows/machine-deployment-windows.yaml index 9cbe3e96b7e..98ed536ff79 100644 --- a/templates/flavors/windows/machine-deployment-windows.yaml +++ b/templates/flavors/windows/machine-deployment-windows.yaml @@ -27,12 +27,12 @@ kind: AzureMachineTemplate metadata: name: "${CLUSTER_NAME}-md-win" annotations: - runtime: dockershim + runtime: containerd spec: template: metadata: annotations: - runtime: dockershim + runtime: containerd spec: vmSize: ${AZURE_NODE_MACHINE_TYPE} osDisk: @@ -54,17 +54,21 @@ spec: groups: Administrators sshAuthorizedKeys: - ${AZURE_SSH_PUBLIC_KEY:=""} - preKubeadmCommands: - - powershell c:/create-external-network.ps1 + preKubeadmCommands: [] postKubeadmCommands: - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 joinConfiguration: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' + criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: cloud-provider: azure cloud-config: 'c:/k/azure.json' azure-container-registry-config: 'c:/k/azure.json' + feature-gates: "WindowsHostProcessContainers=true" + v: "2" + windows-priorityclass: "ABOVE_NORMAL_PRIORITY_CLASS" files: - contentFrom: secret: @@ -73,11 +77,8 @@ spec: owner: root:root path: c:/k/azure.json permissions: "0644" - - path: C:/create-external-network.ps1 + - path: C:/defender-exclude-calico.ps1 permissions: "0744" - content: | - # required as a work around for Flannel and Wins bugs - # https://github.com/coreos/flannel/issues/1359 - # https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828 - ipmo C:\k\debug\hns.psm1; - New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; }) + content: |- + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe diff --git a/templates/flavors/windows/machine-deployment.yaml b/templates/flavors/windows/machine-deployment.yaml deleted file mode 100644 index c4fd7dea64b..00000000000 --- a/templates/flavors/windows/machine-deployment.yaml +++ /dev/null @@ -1,84 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: "${CLUSTER_NAME}-md-0" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${LINUX_WORKER_MACHINE_COUNT:-1} - selector: - matchLabels: - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-md-0" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - infrastructureRef: - name: "${CLUSTER_NAME}-md-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: "${CLUSTER_NAME}-md-0" -spec: - template: - spec: - vmSize: ${AZURE_NODE_MACHINE_TYPE} - osDisk: - osType: "Linux" - diskSizeGB: 128 - managedDisk: - storageAccountType: "Premium_LRS" - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: "${CLUSTER_NAME}-md-0" -spec: - template: - spec: - preKubeadmCommands: [] - postKubeadmCommands: - # Azures vnet MTU is 1400. - # When using Flannel VXLAN to avoid packet fragmentation - # that results dropped packets on Windows we need to match. - # Flannel will automatically choose eth0 - 50 - - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') - - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml - - netplan apply - useExperimentalRetryJoin: true - joinConfiguration: - nodeRegistration: - name: '{{ ds.meta_data["local_hostname"] }}' - kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json - azure-container-registry-config: /etc/kubernetes/azure.json - files: - - contentFrom: - secret: - name: ${CLUSTER_NAME}-md-0-azure-json - key: worker-node-azure.json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - path: /etc/netplan/60-eth0.yaml - permissions: "0644" - owner: root:root - content: | - network: - version: 2 - ethernets: - eth0: - mtu: 1400 - match: - macaddress: MACADDRESS - set-name: eth0 - \ No newline at end of file diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows.yaml deleted file mode 100644 index ca3e5e3785c..00000000000 --- a/templates/test/ci/cluster-template-prow-ci-version-windows.yaml +++ /dev/null @@ -1,1429 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: ${CLUSTER_NAME}-flannel - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 10.244.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "true" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - configure-cloud-routes: "false" - v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - kubernetesVersion: ci/${CI_VERSION} - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - content: | - network: - version: 2 - ethernets: - eth0: - mtu: 1400 - match: - macaddress: MACADDRESS - set-name: eth0 - owner: root:root - path: /etc/netplan/60-eth0.yaml - permissions: "0644" - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - - # This test installs release packages or binaries that are a result of the CI and release builds. - # It runs '... --version' commands to verify that the binaries are correctly installed - # and finally uninstalls the packages. - # For the release packages it tests all versions in the support skew. - LINE_SEPARATOR="*************************************************" - echo "$$LINE_SEPARATOR" - CI_VERSION=${CI_VERSION} - if [[ "$${CI_VERSION}" != "" ]]; then - CI_DIR=/tmp/k8s-ci - mkdir -p $$CI_DIR - declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") - declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") - CONTAINER_EXT="tar" - echo "* testing CI version $$CI_VERSION" - # Check for semver - if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" - VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list - apt-get update - # replace . with \. - VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" - PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" - DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION - done - else - CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" - fi - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" - wget "$$CI_URL/$$CI_PACKAGE" -O "$$CI_DIR/$$CI_PACKAGE" - chmod +x "$$CI_DIR/$$CI_PACKAGE" - mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" - done - systemctl restart kubelet - fi - for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do - echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" - wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" - $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" - $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - done - fi - echo "* checking binary versions" - echo "ctr version: " $(ctr version) - echo "kubeadm version: " $(kubeadm version -o=short) - echo "kubectl version: " $(kubectl version --client=true --short=true) - echo "kubelet version: " $(kubelet --version) - echo "$$LINE_SEPARATOR" - owner: root:root - path: /tmp/kubeadm-bootstrap.sh - permissions: "0744" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: - - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') - - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml - - netplan apply - preKubeadmCommands: - - bash -c /tmp/kubeadm-bootstrap.sh - useExperimentalRetryJoin: true - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - image: - marketplace: - offer: capi - publisher: cncf-upstream - sku: k8s-1dot18dot8-ubuntu-1804 - version: 2020.08.17 - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT:-1} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - image: - marketplace: - offer: capi - publisher: cncf-upstream - sku: k8s-1dot18dot8-ubuntu-1804 - version: 2020.08.17 - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - network: - version: 2 - ethernets: - eth0: - mtu: 1400 - match: - macaddress: MACADDRESS - set-name: eth0 - owner: root:root - path: /etc/netplan/60-eth0.yaml - permissions: "0644" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - - # This test installs release packages or binaries that are a result of the CI and release builds. - # It runs '... --version' commands to verify that the binaries are correctly installed - # and finally uninstalls the packages. - # For the release packages it tests all versions in the support skew. - LINE_SEPARATOR="*************************************************" - echo "$$LINE_SEPARATOR" - CI_VERSION=${CI_VERSION} - if [[ "$${CI_VERSION}" != "" ]]; then - CI_DIR=/tmp/k8s-ci - mkdir -p $$CI_DIR - declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") - declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") - CONTAINER_EXT="tar" - echo "* testing CI version $$CI_VERSION" - # Check for semver - if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" - VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list - apt-get update - # replace . with \. - VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" - PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" - DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION - done - else - CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" - fi - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" - wget "$$CI_URL/$$CI_PACKAGE" -O "$$CI_DIR/$$CI_PACKAGE" - chmod +x "$$CI_DIR/$$CI_PACKAGE" - mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" - done - systemctl restart kubelet - fi - for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do - echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" - wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" - $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" - $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - done - fi - echo "* checking binary versions" - echo "ctr version: " $(ctr version) - echo "kubeadm version: " $(kubeadm version -o=short) - echo "kubectl version: " $(kubectl version --client=true --short=true) - echo "kubelet version: " $(kubelet --version) - echo "$$LINE_SEPARATOR" - owner: root:root - path: /tmp/kubeadm-bootstrap.sh - permissions: "0744" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') - - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml - - netplan apply - preKubeadmCommands: - - bash -c /tmp/kubeadm-bootstrap.sh - useExperimentalRetryJoin: true ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-2} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: dockershim - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: dockershim - spec: - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: k8s-1dot18dot19-windows-2019 - version: 2021.05.17 - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: | - # required as a work around for Flannel and Wins bugs - # https://github.com/coreos/flannel/issues/1359 - # https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828 - ipmo C:\k\debug\hns.psm1; - New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; }) - path: C:/create-external-network.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$CI_VERSION="${CI_VERSION}" - if($$CI_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # We are using a VHD that maps to v1.18.19 so the kubeproxy image is already pulled. (pull it just in case) - # Tag it to the ci version. The image knows how to use the copy locally. - docker pull sigwindowstools/kube-proxy:v1.18.19-nanoserver - docker tag sigwindowstools/kube-proxy:v1.18.19-nanoserver "sigwindowstools/kube-proxy:${CI_VERSION/+/_}-nanoserver" - - kubeadm.exe version -o=short - kubectl.exe version --client=true --short=true - kubelet.exe --version - path: C:/replace-ci-binaries.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - preKubeadmCommands: - - powershell c:/create-external-network.ps1 - - powershell C:/create-temp-folder.ps1 - - powershell C:/replace-ci-binaries.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-flannel - namespace: default -spec: - clusterSelector: - matchLabels: - cni: ${CLUSTER_NAME}-flannel - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-flannel - strategy: ApplyOnce ---- -apiVersion: v1 -data: - cni: |+ - --- - apiVersion: policy/v1beta1 - kind: PodSecurityPolicy - metadata: - name: psp.flannel.unprivileged - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default - seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default - apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default - apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default - spec: - privileged: false - volumes: - - configMap - - secret - - emptyDir - - hostPath - allowedHostPaths: - - pathPrefix: "/etc/cni/net.d" - - pathPrefix: "/etc/kube-flannel" - - pathPrefix: "/run/flannel" - readOnlyRootFilesystem: false - # Users and groups - runAsUser: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - fsGroup: - rule: RunAsAny - # Privilege Escalation - allowPrivilegeEscalation: false - defaultAllowPrivilegeEscalation: false - # Capabilities - allowedCapabilities: ['NET_ADMIN'] - defaultAddCapabilities: [] - requiredDropCapabilities: [] - # Host namespaces - hostPID: false - hostIPC: false - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - # SELinux - seLinux: - # SELinux is unused in CaaSP - rule: 'RunAsAny' - --- - kind: ClusterRole - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: flannel - rules: - - apiGroups: ['extensions'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: ['psp.flannel.unprivileged'] - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - apiGroups: - - "" - resources: - - nodes - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: flannel - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flannel - subjects: - - kind: ServiceAccount - name: flannel - namespace: kube-system - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: flannel - namespace: kube-system - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: kube-flannel-cfg - namespace: kube-system - labels: - tier: node - app: flannel - data: - cni-conf.json: | - { - "name": "cbr0", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "flannel", - "delegate": { - "hairpinMode": true, - "isDefaultGateway": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - net-conf.json: | - { - "Network": "10.244.0.0/16", - "Backend": { - "Type": "vxlan", - "VNI" : 4096, - "Port": 4789 - } - } - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-amd64 - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-amd64 - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-amd64 - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-arm64 - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - arm64 - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-arm64 - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-arm64 - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-arm - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - arm - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-arm - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-arm - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-ppc64le - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-ppc64le - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-ppc64le - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-s390x - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-s390x - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-s390x - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: kube-flannel-windows-cfg - namespace: kube-system - labels: - tier: node - app: flannel - data: - run.ps1: | - $ErrorActionPreference = "Stop"; - - mkdir -force /host/etc/cni/net.d - mkdir -force /host/etc/kube-flannel - mkdir -force /host/opt/cni/bin - mkdir -force /host/k/flannel - mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount - - $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json - $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet - $podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet - $networkJson = wins cli net get | convertfrom-json - - $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet - $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet - Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100) - - cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel - cp -force -recurse /cni/* /host/opt/cni/bin - cp -force /k/flannel/* /host/k/flannel/ - cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml - cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/ - wins cli process run --path /k/flannel/setup.exe --args "--mode=overlay --interface=Ethernet 2" - wins cli route add --addresses 169.254.169.254 - wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE" - cni-conf.json: | - { - "name": "flannel.4096", - "cniVersion": "0.3.0", - "type": "flannel", - "capabilities": { - "dns": true - }, - "delegate": { - "type": "win-overlay", - "policies": [ - { - "Name": "EndpointPolicy", - "Value": { - "Type": "OutBoundNAT", - "ExceptionList": [] - } - }, - { - "Name": "EndpointPolicy", - "Value": { - "Type": "ROUTE", - "DestinationPrefix": "", - "NeedEncap": true - } - } - ] - } - } - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-windows-amd64 - labels: - tier: node - app: flannel - namespace: kube-system - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - windows - - key: kubernetes.io/arch - operator: In - values: - - amd64 - hostNetwork: true - serviceAccountName: flannel - tolerations: - - operator: Exists - effect: NoSchedule - containers: - - name: kube-flannel - image: sigwindowstools/flannel:v0.13.0-nanoserver - command: - - pwsh - args: - - -file - - /etc/kube-flannel-windows/run.ps1 - volumeMounts: - - name: wins - mountPath: \\.\pipe\rancher_wins - - name: host - mountPath: /host - - name: kube-proxy - mountPath: /kube-proxy - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - - name: flannel-windows-cfg - mountPath: /etc/kube-flannel-windows/ - - name: kubeadm-config - mountPath: /etc/kubeadm-config/ - env: - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - volumes: - - name: wins - hostPath: - path: \\.\pipe\rancher_wins - type: null - - name: opt - hostPath: - path: /opt - - name: host - hostPath: - path: / - - name: cni - hostPath: - path: /etc - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - - name: flannel-windows-cfg - configMap: - name: kube-flannel-windows-cfg - - name: kube-proxy - configMap: - name: kube-proxy - - name: kubeadm-config - configMap: - name: kubeadm-config - - proxy: | - apiVersion: v1 - data: - run-script.ps1: |- - $ErrorActionPreference = "Stop"; - mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount - mkdir -force /host/k/kube-proxy - - $$CI_VERSION="${CI_VERSION:-}" - if($$CI_VERSION -ne "" -And (Test-Path -Path "/host/k/kube-proxy.exe")) - { - cp -force /host/k/kube-proxy.exe /k/kube-proxy/kube-proxy.exe - } - - cp -force /k/kube-proxy/* /host/k/kube-proxy - cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy - cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME? - - $networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name - $sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "." - yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip - yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName - yq w -i /host/var/lib/kube-proxy/config.conf featureGates.WinOverlay true - yq w -i /host/var/lib/kube-proxy/config.conf mode "kernelspace" - wins cli process run --path /k/kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true" - kind: ConfigMap - metadata: - labels: - app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - containers: - - command: - - pwsh - args: - - -file - - /var/lib/kube-proxy-windows/run-script.ps1 - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-nanoserver - name: kube-proxy - volumeMounts: - - name: wins - mountPath: \\.\pipe\rancher_wins - - name: host - mountPath: /host - - mountPath: /var/lib/kube-proxy - name: kube-proxy - - mountPath: /var/lib/kube-proxy-windows - name: kube-proxy-windows - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - name: wins - hostPath: - path: \\.\pipe\rancher_wins - type: null - - configMap: - defaultMode: 420 - name: kube-proxy-windows - name: kube-proxy-windows - - configMap: - name: kube-proxy - name: kube-proxy - - hostPath: - path: / - name: host - updateStrategy: - type: RollingUpdate -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-flannel - namespace: default diff --git a/templates/test/ci/cluster-template-prow-windows-addons.yaml b/templates/test/ci/cluster-template-prow-windows-addons.yaml deleted file mode 100644 index 645b1f2975e..00000000000 --- a/templates/test/ci/cluster-template-prow-windows-addons.yaml +++ /dev/null @@ -1,863 +0,0 @@ -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - annotations: - apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default - apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default - seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default - seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default - name: psp.flannel.unprivileged -spec: - allowPrivilegeEscalation: false - allowedCapabilities: - - NET_ADMIN - allowedHostPaths: - - pathPrefix: /etc/cni/net.d - - pathPrefix: /etc/kube-flannel - - pathPrefix: /run/flannel - defaultAddCapabilities: [] - defaultAllowPrivilegeEscalation: false - fsGroup: - rule: RunAsAny - hostIPC: false - hostNetwork: true - hostPID: false - hostPorts: - - max: 65535 - min: 0 - privileged: false - readOnlyRootFilesystem: false - requiredDropCapabilities: [] - runAsUser: - rule: RunAsAny - seLinux: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - volumes: - - configMap - - secret - - emptyDir - - hostPath ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: flannel -rules: -- apiGroups: - - extensions - resourceNames: - - psp.flannel.unprivileged - resources: - - podsecuritypolicies - verbs: - - use -- apiGroups: - - "" - resources: - - pods - verbs: - - get -- apiGroups: - - "" - resources: - - nodes - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: flannel -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flannel -subjects: -- kind: ServiceAccount - name: flannel - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flannel - namespace: kube-system ---- -apiVersion: v1 -data: - cni-conf.json: | - { - "name": "cbr0", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "flannel", - "delegate": { - "hairpinMode": true, - "isDefaultGateway": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - net-conf.json: | - { - "Network": "10.244.0.0/16", - "Backend": { - "Type": "vxlan", - "VNI" : 4096, - "Port": 4789 - } - } -kind: ConfigMap -metadata: - labels: - app: flannel - tier: node - name: kube-flannel-cfg - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: flannel - tier: node - name: kube-flannel-ds-amd64 - namespace: kube-system -spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - app: flannel - tier: node - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - containers: - - args: - - --ip-masq - - --kube-subnet-mgr - command: - - /opt/bin/flanneld - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: quay.io/coreos/flannel:v0.12.0-amd64 - name: kube-flannel - resources: - limits: - cpu: 100m - memory: 50Mi - requests: - cpu: 100m - memory: 50Mi - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: false - volumeMounts: - - mountPath: /run/flannel - name: run - - mountPath: /etc/kube-flannel/ - name: flannel-cfg - hostNetwork: true - initContainers: - - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - command: - - cp - image: quay.io/coreos/flannel:v0.12.0-amd64 - name: install-cni - volumeMounts: - - mountPath: /etc/cni/net.d - name: cni - - mountPath: /etc/kube-flannel/ - name: flannel-cfg - serviceAccountName: flannel - tolerations: - - effect: NoSchedule - operator: Exists - volumes: - - hostPath: - path: /run/flannel - name: run - - hostPath: - path: /etc/cni/net.d - name: cni - - configMap: - name: kube-flannel-cfg - name: flannel-cfg ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: flannel - tier: node - name: kube-flannel-ds-arm64 - namespace: kube-system -spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - app: flannel - tier: node - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - arm64 - containers: - - args: - - --ip-masq - - --kube-subnet-mgr - command: - - /opt/bin/flanneld - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: quay.io/coreos/flannel:v0.12.0-arm64 - name: kube-flannel - resources: - limits: - cpu: 100m - memory: 50Mi - requests: - cpu: 100m - memory: 50Mi - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: false - volumeMounts: - - mountPath: /run/flannel - name: run - - mountPath: /etc/kube-flannel/ - name: flannel-cfg - hostNetwork: true - initContainers: - - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - command: - - cp - image: quay.io/coreos/flannel:v0.12.0-arm64 - name: install-cni - volumeMounts: - - mountPath: /etc/cni/net.d - name: cni - - mountPath: /etc/kube-flannel/ - name: flannel-cfg - serviceAccountName: flannel - tolerations: - - effect: NoSchedule - operator: Exists - volumes: - - hostPath: - path: /run/flannel - name: run - - hostPath: - path: /etc/cni/net.d - name: cni - - configMap: - name: kube-flannel-cfg - name: flannel-cfg ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: flannel - tier: node - name: kube-flannel-ds-arm - namespace: kube-system -spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - app: flannel - tier: node - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - arm - containers: - - args: - - --ip-masq - - --kube-subnet-mgr - command: - - /opt/bin/flanneld - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: quay.io/coreos/flannel:v0.12.0-arm - name: kube-flannel - resources: - limits: - cpu: 100m - memory: 50Mi - requests: - cpu: 100m - memory: 50Mi - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: false - volumeMounts: - - mountPath: /run/flannel - name: run - - mountPath: /etc/kube-flannel/ - name: flannel-cfg - hostNetwork: true - initContainers: - - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - command: - - cp - image: quay.io/coreos/flannel:v0.12.0-arm - name: install-cni - volumeMounts: - - mountPath: /etc/cni/net.d - name: cni - - mountPath: /etc/kube-flannel/ - name: flannel-cfg - serviceAccountName: flannel - tolerations: - - effect: NoSchedule - operator: Exists - volumes: - - hostPath: - path: /run/flannel - name: run - - hostPath: - path: /etc/cni/net.d - name: cni - - configMap: - name: kube-flannel-cfg - name: flannel-cfg ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: flannel - tier: node - name: kube-flannel-ds-ppc64le - namespace: kube-system -spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - app: flannel - tier: node - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - containers: - - args: - - --ip-masq - - --kube-subnet-mgr - command: - - /opt/bin/flanneld - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: quay.io/coreos/flannel:v0.12.0-ppc64le - name: kube-flannel - resources: - limits: - cpu: 100m - memory: 50Mi - requests: - cpu: 100m - memory: 50Mi - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: false - volumeMounts: - - mountPath: /run/flannel - name: run - - mountPath: /etc/kube-flannel/ - name: flannel-cfg - hostNetwork: true - initContainers: - - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - command: - - cp - image: quay.io/coreos/flannel:v0.12.0-ppc64le - name: install-cni - volumeMounts: - - mountPath: /etc/cni/net.d - name: cni - - mountPath: /etc/kube-flannel/ - name: flannel-cfg - serviceAccountName: flannel - tolerations: - - effect: NoSchedule - operator: Exists - volumes: - - hostPath: - path: /run/flannel - name: run - - hostPath: - path: /etc/cni/net.d - name: cni - - configMap: - name: kube-flannel-cfg - name: flannel-cfg ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: flannel - tier: node - name: kube-flannel-ds-s390x - namespace: kube-system -spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - app: flannel - tier: node - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - containers: - - args: - - --ip-masq - - --kube-subnet-mgr - command: - - /opt/bin/flanneld - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: quay.io/coreos/flannel:v0.12.0-s390x - name: kube-flannel - resources: - limits: - cpu: 100m - memory: 50Mi - requests: - cpu: 100m - memory: 50Mi - securityContext: - capabilities: - add: - - NET_ADMIN - privileged: false - volumeMounts: - - mountPath: /run/flannel - name: run - - mountPath: /etc/kube-flannel/ - name: flannel-cfg - hostNetwork: true - initContainers: - - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - command: - - cp - image: quay.io/coreos/flannel:v0.12.0-s390x - name: install-cni - volumeMounts: - - mountPath: /etc/cni/net.d - name: cni - - mountPath: /etc/kube-flannel/ - name: flannel-cfg - serviceAccountName: flannel - tolerations: - - effect: NoSchedule - operator: Exists - volumes: - - hostPath: - path: /run/flannel - name: run - - hostPath: - path: /etc/cni/net.d - name: cni - - configMap: - name: kube-flannel-cfg - name: flannel-cfg ---- -apiVersion: v1 -data: - cni-conf.json: | - { - "name": "flannel.4096", - "cniVersion": "0.3.0", - "type": "flannel", - "capabilities": { - "dns": true - }, - "delegate": { - "type": "win-overlay", - "policies": [ - { - "Name": "EndpointPolicy", - "Value": { - "Type": "OutBoundNAT", - "ExceptionList": [] - } - }, - { - "Name": "EndpointPolicy", - "Value": { - "Type": "ROUTE", - "DestinationPrefix": "", - "NeedEncap": true - } - } - ] - } - } - run.ps1: | - $ErrorActionPreference = "Stop"; - - mkdir -force /host/etc/cni/net.d - mkdir -force /host/etc/kube-flannel - mkdir -force /host/opt/cni/bin - mkdir -force /host/k/flannel - mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount - - $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json - $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet - $podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet - $networkJson = wins cli net get | convertfrom-json - - $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet - $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet - Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100) - - cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel - cp -force -recurse /cni/* /host/opt/cni/bin - cp -force /k/flannel/* /host/k/flannel/ - cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml - cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/ - wins cli process run --path /k/flannel/setup.exe --args "--mode=overlay --interface=Ethernet 2" - wins cli route add --addresses 169.254.169.254 - wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE" -kind: ConfigMap -metadata: - labels: - app: flannel - tier: node - name: kube-flannel-windows-cfg - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: flannel - tier: node - name: kube-flannel-ds-windows-amd64 - namespace: kube-system -spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - app: flannel - tier: node - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - windows - - key: kubernetes.io/arch - operator: In - values: - - amd64 - containers: - - args: - - -file - - /etc/kube-flannel-windows/run.ps1 - command: - - pwsh - env: - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - image: sigwindowstools/flannel:v0.13.0-nanoserver - name: kube-flannel - volumeMounts: - - mountPath: \\.\pipe\rancher_wins - name: wins - - mountPath: /host - name: host - - mountPath: /kube-proxy - name: kube-proxy - - mountPath: /etc/cni/net.d - name: cni - - mountPath: /etc/kube-flannel/ - name: flannel-cfg - - mountPath: /etc/kube-flannel-windows/ - name: flannel-windows-cfg - - mountPath: /etc/kubeadm-config/ - name: kubeadm-config - hostNetwork: true - serviceAccountName: flannel - tolerations: - - effect: NoSchedule - operator: Exists - volumes: - - hostPath: - path: \\.\pipe\rancher_wins - type: null - name: wins - - hostPath: - path: /opt - name: opt - - hostPath: - path: / - name: host - - hostPath: - path: /etc - name: cni - - configMap: - name: kube-flannel-cfg - name: flannel-cfg - - configMap: - name: kube-flannel-windows-cfg - name: flannel-windows-cfg - - configMap: - name: kube-proxy - name: kube-proxy - - configMap: - name: kubeadm-config - name: kubeadm-config ---- -apiVersion: v1 -data: - run-script.ps1: |- - $ErrorActionPreference = "Stop"; - mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount - mkdir -force /host/k/kube-proxy - - cp -force /k/kube-proxy/* /host/k/kube-proxy - cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy - cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME? - - $networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name - $sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "." - yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip - yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName - yq w -i /host/var/lib/kube-proxy/config.conf featureGates.WinOverlay true - yq w -i /host/var/lib/kube-proxy/config.conf featureGates.IPv6DualStack false - yq w -i /host/var/lib/kube-proxy/config.conf mode "kernelspace" - wins cli process run --path /k/kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true" -kind: ConfigMap -metadata: - labels: - app: kube-proxy - name: kube-proxy-windows - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - containers: - - args: - - -file - - /var/lib/kube-proxy-windows/run-script.ps1 - command: - - pwsh - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION}-nanoserver - name: kube-proxy - volumeMounts: - - mountPath: \\.\pipe\rancher_wins - name: wins - - mountPath: /host - name: host - - mountPath: /var/lib/kube-proxy - name: kube-proxy - - mountPath: /var/lib/kube-proxy-windows - name: kube-proxy-windows - nodeSelector: - kubernetes.io/os: windows - serviceAccountName: kube-proxy - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - hostPath: - path: \\.\pipe\rancher_wins - type: null - name: wins - - configMap: - defaultMode: 420 - name: kube-proxy-windows - name: kube-proxy-windows - - configMap: - name: kube-proxy - name: kube-proxy - - hostPath: - path: / - name: host - updateStrategy: - type: RollingUpdate diff --git a/templates/test/ci/cluster-template-prow-windows.yaml b/templates/test/ci/cluster-template-prow-windows.yaml deleted file mode 100644 index c35e2cad242..00000000000 --- a/templates/test/ci/cluster-template-prow-windows.yaml +++ /dev/null @@ -1,1249 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: ${CLUSTER_NAME}-flannel - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 10.244.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "true" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - configure-cloud-routes: "false" - v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - content: | - network: - version: 2 - ethernets: - eth0: - mtu: 1400 - match: - macaddress: MACADDRESS - set-name: eth0 - owner: root:root - path: /etc/netplan/60-eth0.yaml - permissions: "0644" - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: - - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') - - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml - - netplan apply - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT:-1} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - network: - version: 2 - ethernets: - eth0: - mtu: 1400 - match: - macaddress: MACADDRESS - set-name: eth0 - owner: root:root - path: /etc/netplan/60-eth0.yaml - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') - - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml - - netplan apply - preKubeadmCommands: [] - useExperimentalRetryJoin: true ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-2} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: dockershim - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: dockershim - spec: - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: | - # required as a work around for Flannel and Wins bugs - # https://github.com/coreos/flannel/issues/1359 - # https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828 - ipmo C:\k\debug\hns.psm1; - New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; }) - path: C:/create-external-network.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - preKubeadmCommands: - - powershell c:/create-external-network.ps1 - - powershell C:/create-temp-folder.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-flannel - namespace: default -spec: - clusterSelector: - matchLabels: - cni: ${CLUSTER_NAME}-flannel - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-flannel - strategy: ApplyOnce ---- -apiVersion: v1 -data: - cni: |+ - --- - apiVersion: policy/v1beta1 - kind: PodSecurityPolicy - metadata: - name: psp.flannel.unprivileged - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default - seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default - apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default - apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default - spec: - privileged: false - volumes: - - configMap - - secret - - emptyDir - - hostPath - allowedHostPaths: - - pathPrefix: "/etc/cni/net.d" - - pathPrefix: "/etc/kube-flannel" - - pathPrefix: "/run/flannel" - readOnlyRootFilesystem: false - # Users and groups - runAsUser: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - fsGroup: - rule: RunAsAny - # Privilege Escalation - allowPrivilegeEscalation: false - defaultAllowPrivilegeEscalation: false - # Capabilities - allowedCapabilities: ['NET_ADMIN'] - defaultAddCapabilities: [] - requiredDropCapabilities: [] - # Host namespaces - hostPID: false - hostIPC: false - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - # SELinux - seLinux: - # SELinux is unused in CaaSP - rule: 'RunAsAny' - --- - kind: ClusterRole - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: flannel - rules: - - apiGroups: ['extensions'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: ['psp.flannel.unprivileged'] - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - apiGroups: - - "" - resources: - - nodes - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: flannel - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flannel - subjects: - - kind: ServiceAccount - name: flannel - namespace: kube-system - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: flannel - namespace: kube-system - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: kube-flannel-cfg - namespace: kube-system - labels: - tier: node - app: flannel - data: - cni-conf.json: | - { - "name": "cbr0", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "flannel", - "delegate": { - "hairpinMode": true, - "isDefaultGateway": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - net-conf.json: | - { - "Network": "10.244.0.0/16", - "Backend": { - "Type": "vxlan", - "VNI" : 4096, - "Port": 4789 - } - } - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-amd64 - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-amd64 - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-amd64 - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-arm64 - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - arm64 - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-arm64 - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-arm64 - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-arm - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - arm - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-arm - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-arm - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-ppc64le - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-ppc64le - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-ppc64le - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-s390x - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-s390x - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-s390x - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: kube-flannel-windows-cfg - namespace: kube-system - labels: - tier: node - app: flannel - data: - run.ps1: | - $ErrorActionPreference = "Stop"; - - mkdir -force /host/etc/cni/net.d - mkdir -force /host/etc/kube-flannel - mkdir -force /host/opt/cni/bin - mkdir -force /host/k/flannel - mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount - - $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json - $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet - $podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet - $networkJson = wins cli net get | convertfrom-json - - $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet - $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet - Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100) - - cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel - cp -force -recurse /cni/* /host/opt/cni/bin - cp -force /k/flannel/* /host/k/flannel/ - cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml - cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/ - wins cli process run --path /k/flannel/setup.exe --args "--mode=overlay --interface=Ethernet 2" - wins cli route add --addresses 169.254.169.254 - wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE" - cni-conf.json: | - { - "name": "flannel.4096", - "cniVersion": "0.3.0", - "type": "flannel", - "capabilities": { - "dns": true - }, - "delegate": { - "type": "win-overlay", - "policies": [ - { - "Name": "EndpointPolicy", - "Value": { - "Type": "OutBoundNAT", - "ExceptionList": [] - } - }, - { - "Name": "EndpointPolicy", - "Value": { - "Type": "ROUTE", - "DestinationPrefix": "", - "NeedEncap": true - } - } - ] - } - } - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-windows-amd64 - labels: - tier: node - app: flannel - namespace: kube-system - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - windows - - key: kubernetes.io/arch - operator: In - values: - - amd64 - hostNetwork: true - serviceAccountName: flannel - tolerations: - - operator: Exists - effect: NoSchedule - containers: - - name: kube-flannel - image: sigwindowstools/flannel:v0.13.0-nanoserver - command: - - pwsh - args: - - -file - - /etc/kube-flannel-windows/run.ps1 - volumeMounts: - - name: wins - mountPath: \\.\pipe\rancher_wins - - name: host - mountPath: /host - - name: kube-proxy - mountPath: /kube-proxy - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - - name: flannel-windows-cfg - mountPath: /etc/kube-flannel-windows/ - - name: kubeadm-config - mountPath: /etc/kubeadm-config/ - env: - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - volumes: - - name: wins - hostPath: - path: \\.\pipe\rancher_wins - type: null - - name: opt - hostPath: - path: /opt - - name: host - hostPath: - path: / - - name: cni - hostPath: - path: /etc - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - - name: flannel-windows-cfg - configMap: - name: kube-flannel-windows-cfg - - name: kube-proxy - configMap: - name: kube-proxy - - name: kubeadm-config - configMap: - name: kubeadm-config - - proxy: | - apiVersion: v1 - data: - run-script.ps1: |- - $ErrorActionPreference = "Stop"; - mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount - mkdir -force /host/k/kube-proxy - - $$CI_VERSION="${CI_VERSION:-}" - if($$CI_VERSION -ne "" -And (Test-Path -Path "/host/k/kube-proxy.exe")) - { - cp -force /host/k/kube-proxy.exe /k/kube-proxy/kube-proxy.exe - } - - cp -force /k/kube-proxy/* /host/k/kube-proxy - cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy - cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME? - - $networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name - $sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "." - yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip - yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName - yq w -i /host/var/lib/kube-proxy/config.conf featureGates.WinOverlay true - yq w -i /host/var/lib/kube-proxy/config.conf mode "kernelspace" - wins cli process run --path /k/kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true" - kind: ConfigMap - metadata: - labels: - app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - containers: - - command: - - pwsh - args: - - -file - - /var/lib/kube-proxy-windows/run-script.ps1 - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-nanoserver - name: kube-proxy - volumeMounts: - - name: wins - mountPath: \\.\pipe\rancher_wins - - name: host - mountPath: /host - - mountPath: /var/lib/kube-proxy - name: kube-proxy - - mountPath: /var/lib/kube-proxy-windows - name: kube-proxy-windows - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - name: wins - hostPath: - path: \\.\pipe\rancher_wins - type: null - - configMap: - defaultMode: 420 - name: kube-proxy-windows - name: kube-proxy-windows - - configMap: - name: kube-proxy - name: kube-proxy - - hostPath: - path: / - name: host - updateStrategy: - type: RollingUpdate -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-flannel - namespace: default diff --git a/templates/test/ci/patches/cluster-cni-windows.yaml b/templates/test/ci/patches/cluster-cni-windows.yaml deleted file mode 100644 index ab86656e8c5..00000000000 --- a/templates/test/ci/patches/cluster-cni-windows.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: default - labels: - cni: "${CLUSTER_NAME}-flannel" diff --git a/templates/test/ci/prow-ci-version-windows/kustomization.yaml b/templates/test/ci/prow-ci-version-windows/kustomization.yaml deleted file mode 100644 index a849d0e7a7c..00000000000 --- a/templates/test/ci/prow-ci-version-windows/kustomization.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: default -resources: - - ../prow-windows -patchesStrategicMerge: - - patches/windows-image-update.yaml - - ../patches/control-plane-image-ci-version.yaml - - ../patches/controller-manager.yaml - - ../prow-ci-version/patches/machine-deployment-ci-version.yaml -patches: -- target: - group: bootstrap.cluster.x-k8s.io - version: v1beta1 - kind: KubeadmConfigTemplate - name: .*-md-win - namespace: default - path: patches/machine-deployment-ci-version.yaml -- target: - group: bootstrap.cluster.x-k8s.io - version: v1beta1 - kind: KubeadmConfigTemplate - name: .*-md-0 - namespace: default - path: ../prow-ci-version/patches/kubeadm-bootstrap.yaml -- target: - group: controlplane.cluster.x-k8s.io - version: v1beta1 - kind: KubeadmControlPlane - name: .*-control-plane - namespace: default - path: ../patches/control-plane-kubeadm-boostrap-ci-version.yaml diff --git a/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml b/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml deleted file mode 100644 index 28e7280aed2..00000000000 --- a/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml +++ /dev/null @@ -1,34 +0,0 @@ -- op: add - path: /spec/template/spec/files/- - value: - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$CI_VERSION="${CI_VERSION}" - if($$CI_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # We are using a VHD that maps to v1.18.19 so the kubeproxy image is already pulled. (pull it just in case) - # Tag it to the ci version. The image knows how to use the copy locally. - docker pull sigwindowstools/kube-proxy:v1.18.19-nanoserver - docker tag sigwindowstools/kube-proxy:v1.18.19-nanoserver "sigwindowstools/kube-proxy:${CI_VERSION/+/_}-nanoserver" - - kubeadm.exe version -o=short - kubectl.exe version --client=true --short=true - kubelet.exe --version - path: C:/replace-ci-binaries.ps1 - permissions: "0744" -- op: add - path: /spec/template/spec/preKubeadmCommands/- - value: - powershell C:/replace-ci-binaries.ps1 diff --git a/templates/test/ci/prow-ci-version-windows/patches/windows-image-update.yaml b/templates/test/ci/prow-ci-version-windows/patches/windows-image-update.yaml deleted file mode 100644 index f2b860dde9c..00000000000 --- a/templates/test/ci/prow-ci-version-windows/patches/windows-image-update.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: "${CLUSTER_NAME}-md-win" -spec: - template: - spec: - image: - # we use the 1.18.19 image as a workaround there is no published marketplace image for k8s CI versions. - # 1.18.19 binaries and images will get replaced to the desired version by the script above. - marketplace: - publisher: cncf-upstream - offer: capi-windows - sku: k8s-1dot18dot19-windows-2019 - version: "2021.05.17" diff --git a/templates/test/ci/prow-machine-pool/kustomization.yaml b/templates/test/ci/prow-machine-pool/kustomization.yaml index 95af1e91f47..0663d7e8a39 100644 --- a/templates/test/ci/prow-machine-pool/kustomization.yaml +++ b/templates/test/ci/prow-machine-pool/kustomization.yaml @@ -2,12 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: - - ../../../flavors/machinepool-windows-containerd + - ../../../flavors/machinepool-windows - ../prow/cni-resource-set.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/cluster-cni.yaml - - ../patches/controller-manager.yaml + - ../patches/controller-manager.yaml - ../patches/machine-pool-worker-counts.yaml - ../patches/windows-containerd-patch.yaml configMapGenerator: diff --git a/templates/test/ci/prow-windows/cni-resource-set-windows.yaml b/templates/test/ci/prow-windows/cni-resource-set-windows.yaml deleted file mode 100644 index 26ddda7e6cb..00000000000 --- a/templates/test/ci/prow-windows/cni-resource-set-windows.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-flannel - namespace: default -spec: - clusterSelector: - matchLabels: - cni: ${CLUSTER_NAME}-flannel - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-flannel - strategy: ApplyOnce diff --git a/templates/test/ci/prow-windows/kustomization.yaml b/templates/test/ci/prow-windows/kustomization.yaml deleted file mode 100644 index 0fb97eb0fbc..00000000000 --- a/templates/test/ci/prow-windows/kustomization.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: default -resources: - - ../../../flavors/windows - - cni-resource-set-windows.yaml -patchesStrategicMerge: - - ../patches/tags.yaml - - ../patches/cluster-cni-windows.yaml - - ../patches/controller-manager.yaml - - patches/machine-deployment-worker-counts.yaml -patches: -- target: - group: bootstrap.cluster.x-k8s.io - version: v1beta1 - kind: KubeadmConfigTemplate - name: .*-md-win - namespace: default - path: patches/windows-tmp-folder.yaml -configMapGenerator: - - name: cni-${CLUSTER_NAME}-flannel - files: - - cni=../../../addons/windows/flannel/flannel.yaml - - proxy=../../../addons/windows/flannel/kube-proxy-windows.yaml -generatorOptions: - disableNameSuffixHash: true - labels: - type: generated - annotations: - note: generated diff --git a/templates/test/ci/prow-windows/patches/machine-deployment-worker-counts.yaml b/templates/test/ci/prow-windows/patches/machine-deployment-worker-counts.yaml deleted file mode 100644 index a6cc1fa7849..00000000000 --- a/templates/test/ci/prow-windows/patches/machine-deployment-worker-counts.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: "${CLUSTER_NAME}-md-win" -spec: - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-2} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: "${CLUSTER_NAME}-md-0" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT:-1} diff --git a/templates/test/ci/prow-windows/patches/windows-tmp-folder.yaml b/templates/test/ci/prow-windows/patches/windows-tmp-folder.yaml deleted file mode 100644 index 38ec14d1d15..00000000000 --- a/templates/test/ci/prow-windows/patches/windows-tmp-folder.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- op: add - path: /spec/template/spec/files/- - value: - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" -- op: add - path: /spec/template/spec/preKubeadmCommands/- - value: - powershell C:/create-temp-folder.ps1 diff --git a/templates/test/ci/prow/kustomization.yaml b/templates/test/ci/prow/kustomization.yaml index 554c66a1445..c1af4732be8 100644 --- a/templates/test/ci/prow/kustomization.yaml +++ b/templates/test/ci/prow/kustomization.yaml @@ -4,7 +4,7 @@ namespace: default resources: - ../../../flavors/base - ../../../flavors/default/machine-deployment.yaml - - ../../../flavors/windows-containerd/machine-deployment-windows.yaml + - ../../../flavors/windows/machine-deployment-windows.yaml - mhc.yaml - mhc-windows.yaml - cni-resource-set.yaml diff --git a/templates/test/dev/cluster-template-custom-builds-windows.yaml b/templates/test/dev/cluster-template-custom-builds-windows.yaml deleted file mode 100644 index 696da355156..00000000000 --- a/templates/test/dev/cluster-template-custom-builds-windows.yaml +++ /dev/null @@ -1,1372 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: ${CLUSTER_NAME}-flannel - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 10.244.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - annotations: - controlplane.cluster.x-k8s.io/skip-kube-proxy: "true" - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "true" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - configure-cloud-routes: "false" - v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - kubernetesVersion: ci/${CI_VERSION} - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - content: | - network: - version: 2 - ethernets: - eth0: - mtu: 1400 - match: - macaddress: MACADDRESS - set-name: eth0 - owner: root:root - path: /etc/netplan/60-eth0.yaml - permissions: "0644" - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - - systemctl stop kubelet - declare -a BINARIES=("kubeadm" "kubectl" "kubelet") - for BINARY in "$${BINARIES[@]}"; do - echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" - done - systemctl restart kubelet - - # prepull images from gcr.io/k8s-staging-ci-images and retag it to - # k8s.gcr.io so kubeadm can fetch correct images no matter what - declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - for IMAGE in "$${IMAGES[@]}"; do - $${SUDO} ctr -n k8s.io images pull "gcr.io/k8s-staging-ci-images/$${IMAGE}:${CI_VERSION/+/_}" - $${SUDO} ctr -n k8s.io images tag "gcr.io/k8s-staging-ci-images/$${IMAGE}:${CI_VERSION/+/_}" "k8s.gcr.io/$${IMAGE}:${CI_VERSION/+/_}" - done - - echo "kubeadm version: $(kubeadm version -o=short)" - echo "kubectl version: $(kubectl version --client=true --short=true)" - echo "kubelet version: $(kubelet --version)" - owner: root:root - path: /tmp/replace-k8s-binaries.sh - permissions: "0744" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - - curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz - tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq - rm /tmp/yq_linux_amd64.tar.gz - - export KUBECONFIG=/etc/kubernetes/admin.conf - kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${IMAGE_TAG}" - yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml - yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml - yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml - owner: root:root - path: /tmp/replace-k8s-components.sh - permissions: "0744" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: - - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') - - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml - - netplan apply - - bash -c /tmp/replace-k8s-components.sh - preKubeadmCommands: - - bash -c /tmp/replace-k8s-binaries.sh - useExperimentalRetryJoin: true - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - image: - marketplace: - offer: capi - publisher: cncf-upstream - sku: k8s-1dot18dot8-ubuntu-1804 - version: 2020.08.17 - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT:-1} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - image: - marketplace: - offer: capi - publisher: cncf-upstream - sku: k8s-1dot18dot8-ubuntu-1804 - version: 2020.08.17 - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - network: - version: 2 - ethernets: - eth0: - mtu: 1400 - match: - macaddress: MACADDRESS - set-name: eth0 - owner: root:root - path: /etc/netplan/60-eth0.yaml - permissions: "0644" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - - systemctl stop kubelet - declare -a BINARIES=("kubeadm" "kubectl" "kubelet") - for BINARY in "$${BINARIES[@]}"; do - echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" - done - systemctl restart kubelet - - echo "kubeadm version: $(kubeadm version -o=short)" - echo "kubectl version: $(kubectl version --client=true --short=true)" - echo "kubelet version: $(kubelet --version)" - owner: root:root - path: /tmp/replace-k8s-binaries.sh - permissions: "0744" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }') - - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml - - netplan apply - preKubeadmCommands: - - bash -c /tmp/replace-k8s-binaries.sh - useExperimentalRetryJoin: true ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-2} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: dockershim - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: dockershim - spec: - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: k8s-1dot18dot19-windows-2019 - version: 2021.05.17 - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: | - # required as a work around for Flannel and Wins bugs - # https://github.com/coreos/flannel/issues/1359 - # https://github.com/kubernetes-sigs/sig-windows-tools/issues/103#issuecomment-709426828 - ipmo C:\k\debug\hns.psm1; - New-HnsNetwork -Type Overlay -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -AdapterName "Ethernet 2" -SubnetPolicies @(@{Type = "VSID"; VSID = 9999; }) - path: C:/create-external-network.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - - content: | - Stop-Service kubelet -Force - - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "installing package: $$binary ${KUBE_GIT_VERSION}" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - - # We are using a VHD that maps to v1.18.19 so the kubeproxy image is already pulled. (pull it just in case) - # Tag it to the ci_version which is the version set when kicking off the CI builds and doesn't match the KUBE_GIT_VERSION - # but matches the kubeproxy image tag when it gets generated. The image configuration knows how to use the binary locally. - # This does mean the image tag will not match the version of the binary running. - # See: - # https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/529dbb507962a52ee9fd5a56f3d3856b9bcc53c1/templates/addons/windows/kube-proxy-windows.yaml#L60 - # https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/529dbb507962a52ee9fd5a56f3d3856b9bcc53c1/scripts/ci-build-kubernetes.sh#L54-L59 - docker pull sigwindowstools/kube-proxy:v1.18.19-nanoserver - docker tag sigwindowstools/kube-proxy:v1.18.19-nanoserver "sigwindowstools/kube-proxy:${CI_VERSION/+/_}-nanoserver" - - kubeadm.exe version -o=short - kubectl.exe version --client=true --short=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-k8s-binaries.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - preKubeadmCommands: - - powershell c:/create-external-network.ps1 - - powershell C:/create-temp-folder.ps1 - - powershell C:/replace-k8s-binaries.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-flannel - namespace: default -spec: - clusterSelector: - matchLabels: - cni: ${CLUSTER_NAME}-flannel - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-flannel - strategy: ApplyOnce ---- -apiVersion: v1 -data: - cni: |+ - --- - apiVersion: policy/v1beta1 - kind: PodSecurityPolicy - metadata: - name: psp.flannel.unprivileged - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default - seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default - apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default - apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default - spec: - privileged: false - volumes: - - configMap - - secret - - emptyDir - - hostPath - allowedHostPaths: - - pathPrefix: "/etc/cni/net.d" - - pathPrefix: "/etc/kube-flannel" - - pathPrefix: "/run/flannel" - readOnlyRootFilesystem: false - # Users and groups - runAsUser: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - fsGroup: - rule: RunAsAny - # Privilege Escalation - allowPrivilegeEscalation: false - defaultAllowPrivilegeEscalation: false - # Capabilities - allowedCapabilities: ['NET_ADMIN'] - defaultAddCapabilities: [] - requiredDropCapabilities: [] - # Host namespaces - hostPID: false - hostIPC: false - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - # SELinux - seLinux: - # SELinux is unused in CaaSP - rule: 'RunAsAny' - --- - kind: ClusterRole - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: flannel - rules: - - apiGroups: ['extensions'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: ['psp.flannel.unprivileged'] - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - apiGroups: - - "" - resources: - - nodes - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: flannel - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flannel - subjects: - - kind: ServiceAccount - name: flannel - namespace: kube-system - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: flannel - namespace: kube-system - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: kube-flannel-cfg - namespace: kube-system - labels: - tier: node - app: flannel - data: - cni-conf.json: | - { - "name": "cbr0", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "flannel", - "delegate": { - "hairpinMode": true, - "isDefaultGateway": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - net-conf.json: | - { - "Network": "10.244.0.0/16", - "Backend": { - "Type": "vxlan", - "VNI" : 4096, - "Port": 4789 - } - } - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-amd64 - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-amd64 - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-amd64 - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-arm64 - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - arm64 - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-arm64 - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-arm64 - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-arm - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - arm - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-arm - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-arm - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-ppc64le - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-ppc64le - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-ppc64le - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-s390x - namespace: kube-system - labels: - tier: node - app: flannel - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/os - operator: In - values: - - linux - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - serviceAccountName: flannel - initContainers: - - name: install-cni - image: quay.io/coreos/flannel:v0.12.0-s390x - command: - - cp - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - volumeMounts: - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - containers: - - name: kube-flannel - image: quay.io/coreos/flannel:v0.12.0-s390x - command: - - /opt/bin/flanneld - args: - - --ip-masq - - --kube-subnet-mgr - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: false - capabilities: - add: ["NET_ADMIN"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: run - mountPath: /run/flannel - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - volumes: - - name: run - hostPath: - path: /run/flannel - - name: cni - hostPath: - path: /etc/cni/net.d - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: kube-flannel-windows-cfg - namespace: kube-system - labels: - tier: node - app: flannel - data: - run.ps1: | - $ErrorActionPreference = "Stop"; - - mkdir -force /host/etc/cni/net.d - mkdir -force /host/etc/kube-flannel - mkdir -force /host/opt/cni/bin - mkdir -force /host/k/flannel - mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount - - $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json - $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet - $podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet - $networkJson = wins cli net get | convertfrom-json - - $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet - $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet - Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100) - - cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel - cp -force -recurse /cni/* /host/opt/cni/bin - cp -force /k/flannel/* /host/k/flannel/ - cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml - cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/ - wins cli process run --path /k/flannel/setup.exe --args "--mode=overlay --interface=Ethernet 2" - wins cli route add --addresses 169.254.169.254 - wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE" - cni-conf.json: | - { - "name": "flannel.4096", - "cniVersion": "0.3.0", - "type": "flannel", - "capabilities": { - "dns": true - }, - "delegate": { - "type": "win-overlay", - "policies": [ - { - "Name": "EndpointPolicy", - "Value": { - "Type": "OutBoundNAT", - "ExceptionList": [] - } - }, - { - "Name": "EndpointPolicy", - "Value": { - "Type": "ROUTE", - "DestinationPrefix": "", - "NeedEncap": true - } - } - ] - } - } - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kube-flannel-ds-windows-amd64 - labels: - tier: node - app: flannel - namespace: kube-system - spec: - selector: - matchLabels: - app: flannel - template: - metadata: - labels: - tier: node - app: flannel - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - windows - - key: kubernetes.io/arch - operator: In - values: - - amd64 - hostNetwork: true - serviceAccountName: flannel - tolerations: - - operator: Exists - effect: NoSchedule - containers: - - name: kube-flannel - image: sigwindowstools/flannel:v0.13.0-nanoserver - command: - - pwsh - args: - - -file - - /etc/kube-flannel-windows/run.ps1 - volumeMounts: - - name: wins - mountPath: \\.\pipe\rancher_wins - - name: host - mountPath: /host - - name: kube-proxy - mountPath: /kube-proxy - - name: cni - mountPath: /etc/cni/net.d - - name: flannel-cfg - mountPath: /etc/kube-flannel/ - - name: flannel-windows-cfg - mountPath: /etc/kube-flannel-windows/ - - name: kubeadm-config - mountPath: /etc/kubeadm-config/ - env: - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - volumes: - - name: wins - hostPath: - path: \\.\pipe\rancher_wins - type: null - - name: opt - hostPath: - path: /opt - - name: host - hostPath: - path: / - - name: cni - hostPath: - path: /etc - - name: flannel-cfg - configMap: - name: kube-flannel-cfg - - name: flannel-windows-cfg - configMap: - name: kube-flannel-windows-cfg - - name: kube-proxy - configMap: - name: kube-proxy - - name: kubeadm-config - configMap: - name: kubeadm-config - - proxy: | - apiVersion: v1 - data: - run-script.ps1: |- - $ErrorActionPreference = "Stop"; - mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount - mkdir -force /host/k/kube-proxy - - $$CI_VERSION="${CI_VERSION:-}" - if($$CI_VERSION -ne "" -And (Test-Path -Path "/host/k/kube-proxy.exe")) - { - cp -force /host/k/kube-proxy.exe /k/kube-proxy/kube-proxy.exe - } - - cp -force /k/kube-proxy/* /host/k/kube-proxy - cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy - cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME? - - $networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name - $sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "." - yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip - yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName - yq w -i /host/var/lib/kube-proxy/config.conf featureGates.WinOverlay true - yq w -i /host/var/lib/kube-proxy/config.conf mode "kernelspace" - wins cli process run --path /k/kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true" - kind: ConfigMap - metadata: - labels: - app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - containers: - - command: - - pwsh - args: - - -file - - /var/lib/kube-proxy-windows/run-script.ps1 - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-nanoserver - name: kube-proxy - volumeMounts: - - name: wins - mountPath: \\.\pipe\rancher_wins - - name: host - mountPath: /host - - mountPath: /var/lib/kube-proxy - name: kube-proxy - - mountPath: /var/lib/kube-proxy-windows - name: kube-proxy-windows - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - name: wins - hostPath: - path: \\.\pipe\rancher_wins - type: null - - configMap: - defaultMode: 420 - name: kube-proxy-windows - name: kube-proxy-windows - - configMap: - name: kube-proxy - name: kube-proxy - - hostPath: - path: / - name: host - updateStrategy: - type: RollingUpdate -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-flannel - namespace: default diff --git a/templates/test/dev/custom-builds-windows/kustomization.yaml b/templates/test/dev/custom-builds-windows/kustomization.yaml deleted file mode 100644 index f5cfb1e51d1..00000000000 --- a/templates/test/dev/custom-builds-windows/kustomization.yaml +++ /dev/null @@ -1,27 +0,0 @@ -namespace: default -resources: - - ../../../test/ci/prow-windows -patchesStrategicMerge: - - patches/windows-image-update.yaml - - ../custom-builds/patches/machine-deployment-pr-version.yaml -patchesJson6902: -- target: - group: bootstrap.cluster.x-k8s.io - version: v1beta1 - kind: KubeadmConfigTemplate - name: .*-md-win - namespace: default - path: patches/custom-builds-windows.yaml -- target: - group: bootstrap.cluster.x-k8s.io - version: v1beta1 - kind: KubeadmConfigTemplate - name: .*-md-0 - namespace: default - path: ../custom-builds/patches/kubeadm-bootstrap.yaml -- target: - group: controlplane.cluster.x-k8s.io - version: v1beta1 - kind: KubeadmControlPlane - name: .*-control-plane - path: ../custom-builds/patches/kubeadm-controlplane-bootstrap.yaml diff --git a/templates/test/dev/custom-builds-windows/patches/custom-builds-windows.yaml b/templates/test/dev/custom-builds-windows/patches/custom-builds-windows.yaml deleted file mode 100644 index 750f3862b2e..00000000000 --- a/templates/test/dev/custom-builds-windows/patches/custom-builds-windows.yaml +++ /dev/null @@ -1,34 +0,0 @@ -- op: add - path: /spec/template/spec/files/- - value: - content: | - Stop-Service kubelet -Force - - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "installing package: $$binary ${KUBE_GIT_VERSION}" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - - # We are using a VHD that maps to v1.18.19 so the kubeproxy image is already pulled. (pull it just in case) - # Tag it to the ci_version which is the version set when kicking off the CI builds and doesn't match the KUBE_GIT_VERSION - # but matches the kubeproxy image tag when it gets generated. The image configuration knows how to use the binary locally. - # This does mean the image tag will not match the version of the binary running. - # See: - # https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/529dbb507962a52ee9fd5a56f3d3856b9bcc53c1/templates/addons/windows/kube-proxy-windows.yaml#L60 - # https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/529dbb507962a52ee9fd5a56f3d3856b9bcc53c1/scripts/ci-build-kubernetes.sh#L54-L59 - docker pull sigwindowstools/kube-proxy:v1.18.19-nanoserver - docker tag sigwindowstools/kube-proxy:v1.18.19-nanoserver "sigwindowstools/kube-proxy:${CI_VERSION/+/_}-nanoserver" - - kubeadm.exe version -o=short - kubectl.exe version --client=true --short=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-k8s-binaries.ps1 - permissions: "0744" -- op: add - path: /spec/template/spec/preKubeadmCommands/- - value: - powershell C:/replace-k8s-binaries.ps1 diff --git a/templates/test/dev/custom-builds-windows/patches/windows-image-update.yaml b/templates/test/dev/custom-builds-windows/patches/windows-image-update.yaml deleted file mode 100644 index f2b860dde9c..00000000000 --- a/templates/test/dev/custom-builds-windows/patches/windows-image-update.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: "${CLUSTER_NAME}-md-win" -spec: - template: - spec: - image: - # we use the 1.18.19 image as a workaround there is no published marketplace image for k8s CI versions. - # 1.18.19 binaries and images will get replaced to the desired version by the script above. - marketplace: - publisher: cncf-upstream - offer: capi-windows - sku: k8s-1dot18dot19-windows-2019 - version: "2021.05.17" diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 33c2d18f12c..0f5b49e5816 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -501,55 +501,6 @@ var _ = Describe("Workload cluster creation", func() { }) }) - // ci-e2e.sh and Prow CI skip this test by default. To include this test, set `GINKGO_SKIP=""`. - Context("Creating a Windows Enabled cluster with dockershim [OPTIONAL]", func() { - // Requires 3 control planes due to https://github.com/kubernetes-sigs/cluster-api-provider-azure/issues/857 - It("With 3 control-plane nodes and 1 Linux worker node and 1 Windows worker node", func() { - clusterName = getClusterName(clusterNamePrefix, "win-ha") - clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ - ClusterProxy: bootstrapClusterProxy, - ConfigCluster: clusterctl.ConfigClusterInput{ - LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), - ClusterctlConfigPath: clusterctlConfigPath, - KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), - InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, - Flavor: "windows", - Namespace: namespace.Name, - ClusterName: clusterName, - KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64Ptr(3), - WorkerMachineCount: pointer.Int64Ptr(1), - }, - WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), - WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), - WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), - }, result) - - Context("Creating an accessible load balancer", func() { - AzureLBSpec(ctx, func() AzureLBSpecInput { - return AzureLBSpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Namespace: namespace, - ClusterName: clusterName, - SkipCleanup: skipCleanup, - } - }) - }) - - Context("Creating an accessible load balancer for windows", func() { - AzureLBSpec(ctx, func() AzureLBSpecInput { - return AzureLBSpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Namespace: namespace, - ClusterName: clusterName, - SkipCleanup: skipCleanup, - Windows: true, - } - }) - }) - }) - }) - // ci-e2e.sh and Prow CI skip this test by default. To include this test, set `GINKGO_SKIP=""`. // This spec expects a user-assigned identity named "cloud-provider-user-identity" in a "capz-ci" // resource group. Override these defaults by setting the USER_IDENTITY and CI_RG environment variables. diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 0e1014f034c..21ac58f1880 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -155,8 +155,6 @@ providers: targetName: "cluster-template-private.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version.yaml" targetName: "cluster-template-conformance-ci-artifacts.yaml" - - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-windows.yaml" - targetName: "cluster-template-conformance-ci-artifacts-windows.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml" targetName: "cluster-template-conformance-ci-artifacts-windows-containerd-2022.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version.yaml" @@ -165,10 +163,6 @@ providers: targetName: "cluster-template-conformance-presubmit-artifacts.yaml" - sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds.yaml" targetName: "cluster-template-conformance-presubmit-artifacts-windows-containerd.yaml" - - sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds-windows.yaml" - targetName: "cluster-template-conformance-presubmit-artifacts-windows.yaml" - - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-windows.yaml" - targetName: "cluster-template-windows.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml" targetName: "cluster-template-external-cloud-provider.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks.yaml"