Merge pull request #1371 from jsturtevant/fix-windows-cni-ci

Clean up the CNI CRS which is done via generators
kubernetes-sigs · May 7, 2021 · 9d34960 · 9d34960
2 parents b246fde + eddb72d
commit 9d34960
Show file tree

Hide file tree

Showing 10 changed files with 194 additions and 37 deletions.
diff --git a/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml
@@ -324,7 +324,7 @@ spec:
 ---
 apiVersion: v1
 data:
-  resources: |+
+  cni: |+
     ---
     apiVersion: policy/v1beta1
     kind: PodSecurityPolicy
@@ -1092,6 +1092,100 @@ data:
             configMap:
               name: kubeadm-config
 
+  proxy: |
+    apiVersion: v1
+    data:
+      run-script.ps1: |-
+        $ErrorActionPreference = "Stop";
+        mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount
+        mkdir -force /host/k/kube-proxy
+
+        cp -force /k/kube-proxy/* /host/k/kube-proxy
+        cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy
+        cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME?
+
+        $networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name
+        $sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "."
+        yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip
+        yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName
+        yq w -i /host/var/lib/kube-proxy/config.conf featureGates.WinOverlay true
+        yq w -i /host/var/lib/kube-proxy/config.conf featureGates.IPv6DualStack false
+        yq w -i /host/var/lib/kube-proxy/config.conf mode "kernelspace"
+        wins cli process run --path /k/kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true"
+    kind: ConfigMap
+    metadata:
+      labels:
+        app: kube-proxy
+      name: kube-proxy-windows
+      namespace: kube-system
+    ---
+    apiVersion: apps/v1
+    kind: DaemonSet
+    metadata:
+      labels:
+        k8s-app: kube-proxy
+      name: kube-proxy-windows
+      namespace: kube-system
+    spec:
+      selector:
+        matchLabels:
+          k8s-app: kube-proxy-windows
+      template:
+        metadata:
+          labels:
+            k8s-app: kube-proxy-windows
+        spec:
+          serviceAccountName: kube-proxy
+          containers:
+          - command:
+            - pwsh
+            args:
+            - -file
+            - /var/lib/kube-proxy-windows/run-script.ps1
+            env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION}-nanoserver
+            name: kube-proxy
+            volumeMounts:
+            - name: wins
+              mountPath: \\.\pipe\rancher_wins
+            - name: host
+              mountPath: /host
+            - mountPath: /var/lib/kube-proxy
+              name: kube-proxy
+            - mountPath: /var/lib/kube-proxy-windows
+              name: kube-proxy-windows
+          nodeSelector:
+            kubernetes.io/os: windows
+          tolerations:
+          - key: CriticalAddonsOnly
+            operator: Exists
+          - operator: Exists
+          volumes:
+          - name: wins
+            hostPath:
+              path: \\.\pipe\rancher_wins
+              type: null
+          - configMap:
+              defaultMode: 420
+              name: kube-proxy-windows
+            name: kube-proxy-windows
+          - configMap:
+              name: kube-proxy
+            name: kube-proxy
+          - hostPath:
+              path: /
+            name: host
+      updateStrategy:
+        type: RollingUpdate
 kind: ConfigMap
 metadata:
   annotations:

diff --git a/templates/test/ci/cluster-template-prow-windows.yaml b/templates/test/ci/cluster-template-prow-windows.yaml
@@ -332,7 +332,7 @@ spec:
 ---
 apiVersion: v1
 data:
-  resources: |+
+  cni: |+
     ---
     apiVersion: policy/v1beta1
     kind: PodSecurityPolicy
@@ -1100,6 +1100,100 @@ data:
             configMap:
               name: kubeadm-config
 
+  proxy: |
+    apiVersion: v1
+    data:
+      run-script.ps1: |-
+        $ErrorActionPreference = "Stop";
+        mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount
+        mkdir -force /host/k/kube-proxy
+
+        cp -force /k/kube-proxy/* /host/k/kube-proxy
+        cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy
+        cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME?
+
+        $networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name
+        $sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "."
+        yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip
+        yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName
+        yq w -i /host/var/lib/kube-proxy/config.conf featureGates.WinOverlay true
+        yq w -i /host/var/lib/kube-proxy/config.conf featureGates.IPv6DualStack false
+        yq w -i /host/var/lib/kube-proxy/config.conf mode "kernelspace"
+        wins cli process run --path /k/kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true"
+    kind: ConfigMap
+    metadata:
+      labels:
+        app: kube-proxy
+      name: kube-proxy-windows
+      namespace: kube-system
+    ---
+    apiVersion: apps/v1
+    kind: DaemonSet
+    metadata:
+      labels:
+        k8s-app: kube-proxy
+      name: kube-proxy-windows
+      namespace: kube-system
+    spec:
+      selector:
+        matchLabels:
+          k8s-app: kube-proxy-windows
+      template:
+        metadata:
+          labels:
+            k8s-app: kube-proxy-windows
+        spec:
+          serviceAccountName: kube-proxy
+          containers:
+          - command:
+            - pwsh
+            args:
+            - -file
+            - /var/lib/kube-proxy-windows/run-script.ps1
+            env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION}-nanoserver
+            name: kube-proxy
+            volumeMounts:
+            - name: wins
+              mountPath: \\.\pipe\rancher_wins
+            - name: host
+              mountPath: /host
+            - mountPath: /var/lib/kube-proxy
+              name: kube-proxy
+            - mountPath: /var/lib/kube-proxy-windows
+              name: kube-proxy-windows
+          nodeSelector:
+            kubernetes.io/os: windows
+          tolerations:
+          - key: CriticalAddonsOnly
+            operator: Exists
+          - operator: Exists
+          volumes:
+          - name: wins
+            hostPath:
+              path: \\.\pipe\rancher_wins
+              type: null
+          - configMap:
+              defaultMode: 420
+              name: kube-proxy-windows
+            name: kube-proxy-windows
+          - configMap:
+              name: kube-proxy
+            name: kube-proxy
+          - hostPath:
+              path: /
+            name: host
+      updateStrategy:
+        type: RollingUpdate
 kind: ConfigMap
 metadata:
   annotations:

diff --git a/templates/test/ci/prow-machine-pool-windows/kustomization.yaml b/templates/test/ci/prow-machine-pool-windows/kustomization.yaml
@@ -11,7 +11,8 @@ patchesStrategicMerge:
 configMapGenerator:
   - name: cni-${CLUSTER_NAME}-flannel
     files:
-      - resources=../../../addons/windows/flannel.yaml
+      - cni=../../../addons/windows/flannel.yaml
+      - proxy=../../../addons/windows/kube-proxy-windows.yaml
 generatorOptions:
   disableNameSuffixHash: true
   labels:

diff --git a/templates/test/ci/prow-windows-addons/kustomization.yaml b/templates/test/ci/prow-windows-addons/kustomization.yaml
diff --git a/templates/test/ci/prow-windows/kustomization.yaml b/templates/test/ci/prow-windows/kustomization.yaml
@@ -11,7 +11,8 @@ patchesStrategicMerge:
 configMapGenerator:
   - name: cni-${CLUSTER_NAME}-flannel
     files:
-      - resources=../../../addons/windows/flannel.yaml
+      - cni=../../../addons/windows/flannel.yaml
+      - proxy=../../../addons/windows/kube-proxy-windows.yaml
 generatorOptions:
   disableNameSuffixHash: true
   labels:

diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go
@@ -57,7 +57,6 @@ var _ = Describe("Workload cluster creation", func() {
 		Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid argument. artifactFolder can't be created for %s spec", specName)
 
 		Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.KubernetesVersion))
-		Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.CNIPath))
 
 		clusterName = os.Getenv("CLUSTER_NAME")
 		if clusterName == "" {

diff --git a/test/e2e/common.go b/test/e2e/common.go
@@ -50,10 +50,6 @@ const (
 	AzureInternalLBIP        = "AZURE_INTERNAL_LB_IP"
 	AzureCPSubnetCidr        = "AZURE_CP_SUBNET_CIDR"
 	AzureNodeSubnetCidr      = "AZURE_NODE_SUBNET_CIDR"
-	CNIPathIPv6              = "CNI_IPV6"
-	CNIResourcesIPv6         = "CNI_RESOURCES_IPV6"
-	CNIPathWindows           = "CNI_WINDOWS"
-	CNIResourcesWindows      = "CNI_RESOURCES_WINDOWS"
 	MultiTenancyIdentityName = "MULTI_TENANCY_IDENTITY_NAME"
 	VMSSHPort                = "VM_SSH_PORT"
 	JobName                  = "JOB_NAME"

diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml
@@ -91,9 +91,6 @@ variables:
   COREDNS_VERSION_UPGRADE_TO: "1.6.7"
   KUBERNETES_VERSION_UPGRADE_TO: "${KUBERNETES_VERSION_UPGRADE_TO:-v1.19.7}"
   KUBERNETES_VERSION_UPGRADE_FROM: "${KUBERNETES_VERSION_UPGRADE_FROM:-v1.18.15}"
-  CNI: "${PWD}/templates/addons/calico.yaml"
-  CNI_IPV6: "${PWD}/templates/addons/calico-ipv6.yaml"
-  CNI_WINDOWS: "${PWD}/templates/test/ci/cluster-template-prow-windows-addons.yaml"
   REDACT_LOG_SCRIPT: "${PWD}/hack/log/redact.sh"
   EXP_AKS: "true"
   EXP_MACHINE_POOL: "true"

diff --git a/test/e2e/conformance_test.go b/test/e2e/conformance_test.go
@@ -54,7 +54,6 @@ var _ = Describe("Conformance Tests", func() {
 		Expect(clusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. clusterctlConfigPath must be an existing file when calling %s spec", specName)
 
 		Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.KubernetesVersion))
-		Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.CNIPath))
 
 		clusterName = os.Getenv("CLUSTER_NAME")
 		if clusterName == "" {

diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
@@ -46,7 +46,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4"
-	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/bootstrap"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
@@ -346,24 +345,6 @@ func createClusterctlLocalRepository(config *clusterctl.E2EConfig, repositoryFol
 		RepositoryFolder: repositoryFolder,
 	}
 
-	// Ensuring a CNI file is defined in the config and register a FileTransformation to inject the referenced file as in place of the CNI_RESOURCES envSubst variable.
-	Expect(config.Variables).To(HaveKey(capi_e2e.CNIPath), "Missing %s variable in the config", capi_e2e.CNIPath)
-	cniPath := config.GetVariable(capi_e2e.CNIPath)
-	Expect(cniPath).To(BeAnExistingFile(), "The %s variable should resolve to an existing file", capi_e2e.CNIPath)
-	createRepositoryInput.RegisterClusterResourceSetConfigMapTransformation(cniPath, capi_e2e.CNIResources)
-
-	// Do the same for CNI_RESOURCES_IPV6.
-	Expect(config.Variables).To(HaveKey(CNIPathIPv6), "Missing %s variable in the config", CNIPathIPv6)
-	cniPathIPv6 := config.GetVariable(CNIPathIPv6)
-	Expect(cniPathIPv6).To(BeAnExistingFile(), "The %s variable should resolve to an existing file", CNIPathIPv6)
-	createRepositoryInput.RegisterClusterResourceSetConfigMapTransformation(cniPathIPv6, CNIResourcesIPv6)
-
-	// Read CNI_WINDOWS file and set CNI_RESOURCES_WINDOWS environmental variable
-	Expect(config.Variables).To(HaveKey(CNIPathWindows), "Missing %s variable in the config", CNIPathWindows)
-	cniPathWindows := config.GetVariable(CNIPathWindows)
-	Expect(cniPathWindows).To(BeAnExistingFile(), "The %s variable should resolve to an existing file", CNIPathWindows)
-	createRepositoryInput.RegisterClusterResourceSetConfigMapTransformation(cniPathWindows, CNIResourcesWindows)
-
 	clusterctlConfig := clusterctl.CreateRepository(context.TODO(), createRepositoryInput)
 	Expect(clusterctlConfig).To(BeAnExistingFile(), "The clusterctl config file does not exists in the local repository %s", repositoryFolder)
 	return clusterctlConfig