From eddb72daba7e0483fd832704565d8574503ca990 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 6 May 2021 16:00:55 -0700 Subject: [PATCH] clean up the CNI CRS which is done via generators --- ...er-template-prow-machine-pool-windows.yaml | 96 ++++++++++++++++++- .../ci/cluster-template-prow-windows.yaml | 96 ++++++++++++++++++- .../kustomization.yaml | 3 +- .../ci/prow-windows-addons/kustomization.yaml | 5 - .../test/ci/prow-windows/kustomization.yaml | 3 +- test/e2e/azure_test.go | 1 - test/e2e/common.go | 4 - test/e2e/config/azure-dev.yaml | 3 - test/e2e/conformance_test.go | 1 - test/e2e/e2e_suite_test.go | 19 ---- 10 files changed, 194 insertions(+), 37 deletions(-) delete mode 100644 templates/test/ci/prow-windows-addons/kustomization.yaml diff --git a/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml index 3c69dfbea6d..2b5534ba28f 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml @@ -324,7 +324,7 @@ spec: --- apiVersion: v1 data: - resources: |+ + cni: |+ --- apiVersion: policy/v1beta1 kind: PodSecurityPolicy @@ -1092,6 +1092,100 @@ data: configMap: name: kubeadm-config + proxy: | + apiVersion: v1 + data: + run-script.ps1: |- + $ErrorActionPreference = "Stop"; + mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount + mkdir -force /host/k/kube-proxy + + cp -force /k/kube-proxy/* /host/k/kube-proxy + cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy + cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME? + + $networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name + $sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "." + yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip + yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName + yq w -i /host/var/lib/kube-proxy/config.conf featureGates.WinOverlay true + yq w -i /host/var/lib/kube-proxy/config.conf featureGates.IPv6DualStack false + yq w -i /host/var/lib/kube-proxy/config.conf mode "kernelspace" + wins cli process run --path /k/kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true" + kind: ConfigMap + metadata: + labels: + app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + containers: + - command: + - pwsh + args: + - -file + - /var/lib/kube-proxy-windows/run-script.ps1 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION}-nanoserver + name: kube-proxy + volumeMounts: + - name: wins + mountPath: \\.\pipe\rancher_wins + - name: host + mountPath: /host + - mountPath: /var/lib/kube-proxy + name: kube-proxy + - mountPath: /var/lib/kube-proxy-windows + name: kube-proxy-windows + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - name: wins + hostPath: + path: \\.\pipe\rancher_wins + type: null + - configMap: + defaultMode: 420 + name: kube-proxy-windows + name: kube-proxy-windows + - configMap: + name: kube-proxy + name: kube-proxy + - hostPath: + path: / + name: host + updateStrategy: + type: RollingUpdate kind: ConfigMap metadata: annotations: diff --git a/templates/test/ci/cluster-template-prow-windows.yaml b/templates/test/ci/cluster-template-prow-windows.yaml index 3eb7021f360..10a8a17c0c8 100644 --- a/templates/test/ci/cluster-template-prow-windows.yaml +++ b/templates/test/ci/cluster-template-prow-windows.yaml @@ -332,7 +332,7 @@ spec: --- apiVersion: v1 data: - resources: |+ + cni: |+ --- apiVersion: policy/v1beta1 kind: PodSecurityPolicy @@ -1100,6 +1100,100 @@ data: configMap: name: kubeadm-config + proxy: | + apiVersion: v1 + data: + run-script.ps1: |- + $ErrorActionPreference = "Stop"; + mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount + mkdir -force /host/k/kube-proxy + + cp -force /k/kube-proxy/* /host/k/kube-proxy + cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy + cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME? + + $networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name + $sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "." + yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip + yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName + yq w -i /host/var/lib/kube-proxy/config.conf featureGates.WinOverlay true + yq w -i /host/var/lib/kube-proxy/config.conf featureGates.IPv6DualStack false + yq w -i /host/var/lib/kube-proxy/config.conf mode "kernelspace" + wins cli process run --path /k/kube-proxy/kube-proxy.exe --args "--v=6 --config=/var/lib/kube-proxy/config.conf --hostname-override=$env:NODE_NAME --feature-gates=WinOverlay=true" + kind: ConfigMap + metadata: + labels: + app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + containers: + - command: + - pwsh + args: + - -file + - /var/lib/kube-proxy-windows/run-script.ps1 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION}-nanoserver + name: kube-proxy + volumeMounts: + - name: wins + mountPath: \\.\pipe\rancher_wins + - name: host + mountPath: /host + - mountPath: /var/lib/kube-proxy + name: kube-proxy + - mountPath: /var/lib/kube-proxy-windows + name: kube-proxy-windows + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - name: wins + hostPath: + path: \\.\pipe\rancher_wins + type: null + - configMap: + defaultMode: 420 + name: kube-proxy-windows + name: kube-proxy-windows + - configMap: + name: kube-proxy + name: kube-proxy + - hostPath: + path: / + name: host + updateStrategy: + type: RollingUpdate kind: ConfigMap metadata: annotations: diff --git a/templates/test/ci/prow-machine-pool-windows/kustomization.yaml b/templates/test/ci/prow-machine-pool-windows/kustomization.yaml index 916e7ba9d3a..93d30b90f12 100644 --- a/templates/test/ci/prow-machine-pool-windows/kustomization.yaml +++ b/templates/test/ci/prow-machine-pool-windows/kustomization.yaml @@ -11,7 +11,8 @@ patchesStrategicMerge: configMapGenerator: - name: cni-${CLUSTER_NAME}-flannel files: - - resources=../../../addons/windows/flannel.yaml + - cni=../../../addons/windows/flannel.yaml + - proxy=../../../addons/windows/kube-proxy-windows.yaml generatorOptions: disableNameSuffixHash: true labels: diff --git a/templates/test/ci/prow-windows-addons/kustomization.yaml b/templates/test/ci/prow-windows-addons/kustomization.yaml deleted file mode 100644 index e5a5d1d9ff9..00000000000 --- a/templates/test/ci/prow-windows-addons/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: kube-system -resources: - - ../../../addons/windows diff --git a/templates/test/ci/prow-windows/kustomization.yaml b/templates/test/ci/prow-windows/kustomization.yaml index 5d147956a67..6093305e9cd 100644 --- a/templates/test/ci/prow-windows/kustomization.yaml +++ b/templates/test/ci/prow-windows/kustomization.yaml @@ -11,7 +11,8 @@ patchesStrategicMerge: configMapGenerator: - name: cni-${CLUSTER_NAME}-flannel files: - - resources=../../../addons/windows/flannel.yaml + - cni=../../../addons/windows/flannel.yaml + - proxy=../../../addons/windows/kube-proxy-windows.yaml generatorOptions: disableNameSuffixHash: true labels: diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index ac6422a6e32..79eef40046b 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -57,7 +57,6 @@ var _ = Describe("Workload cluster creation", func() { Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid argument. artifactFolder can't be created for %s spec", specName) Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.KubernetesVersion)) - Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.CNIPath)) clusterName = os.Getenv("CLUSTER_NAME") if clusterName == "" { diff --git a/test/e2e/common.go b/test/e2e/common.go index bec993ca9e6..3e0809dc479 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -50,10 +50,6 @@ const ( AzureInternalLBIP = "AZURE_INTERNAL_LB_IP" AzureCPSubnetCidr = "AZURE_CP_SUBNET_CIDR" AzureNodeSubnetCidr = "AZURE_NODE_SUBNET_CIDR" - CNIPathIPv6 = "CNI_IPV6" - CNIResourcesIPv6 = "CNI_RESOURCES_IPV6" - CNIPathWindows = "CNI_WINDOWS" - CNIResourcesWindows = "CNI_RESOURCES_WINDOWS" MultiTenancyIdentityName = "MULTI_TENANCY_IDENTITY_NAME" VMSSHPort = "VM_SSH_PORT" JobName = "JOB_NAME" diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 8f38dd37ae4..4824069ec3e 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -91,9 +91,6 @@ variables: COREDNS_VERSION_UPGRADE_TO: "1.6.7" KUBERNETES_VERSION_UPGRADE_TO: "${KUBERNETES_VERSION_UPGRADE_TO:-v1.19.7}" KUBERNETES_VERSION_UPGRADE_FROM: "${KUBERNETES_VERSION_UPGRADE_FROM:-v1.18.15}" - CNI: "${PWD}/templates/addons/calico.yaml" - CNI_IPV6: "${PWD}/templates/addons/calico-ipv6.yaml" - CNI_WINDOWS: "${PWD}/templates/test/ci/cluster-template-prow-windows-addons.yaml" REDACT_LOG_SCRIPT: "${PWD}/hack/log/redact.sh" EXP_AKS: "true" EXP_MACHINE_POOL: "true" diff --git a/test/e2e/conformance_test.go b/test/e2e/conformance_test.go index 0f23abc3ed8..af63752b415 100644 --- a/test/e2e/conformance_test.go +++ b/test/e2e/conformance_test.go @@ -54,7 +54,6 @@ var _ = Describe("Conformance Tests", func() { Expect(clusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. clusterctlConfigPath must be an existing file when calling %s spec", specName) Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.KubernetesVersion)) - Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.CNIPath)) clusterName = os.Getenv("CLUSTER_NAME") if clusterName == "" { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0d00daa7c70..94a8935b41c 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -46,7 +46,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4" - capi_e2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/bootstrap" "sigs.k8s.io/cluster-api/test/framework/clusterctl" @@ -346,24 +345,6 @@ func createClusterctlLocalRepository(config *clusterctl.E2EConfig, repositoryFol RepositoryFolder: repositoryFolder, } - // Ensuring a CNI file is defined in the config and register a FileTransformation to inject the referenced file as in place of the CNI_RESOURCES envSubst variable. - Expect(config.Variables).To(HaveKey(capi_e2e.CNIPath), "Missing %s variable in the config", capi_e2e.CNIPath) - cniPath := config.GetVariable(capi_e2e.CNIPath) - Expect(cniPath).To(BeAnExistingFile(), "The %s variable should resolve to an existing file", capi_e2e.CNIPath) - createRepositoryInput.RegisterClusterResourceSetConfigMapTransformation(cniPath, capi_e2e.CNIResources) - - // Do the same for CNI_RESOURCES_IPV6. - Expect(config.Variables).To(HaveKey(CNIPathIPv6), "Missing %s variable in the config", CNIPathIPv6) - cniPathIPv6 := config.GetVariable(CNIPathIPv6) - Expect(cniPathIPv6).To(BeAnExistingFile(), "The %s variable should resolve to an existing file", CNIPathIPv6) - createRepositoryInput.RegisterClusterResourceSetConfigMapTransformation(cniPathIPv6, CNIResourcesIPv6) - - // Read CNI_WINDOWS file and set CNI_RESOURCES_WINDOWS environmental variable - Expect(config.Variables).To(HaveKey(CNIPathWindows), "Missing %s variable in the config", CNIPathWindows) - cniPathWindows := config.GetVariable(CNIPathWindows) - Expect(cniPathWindows).To(BeAnExistingFile(), "The %s variable should resolve to an existing file", CNIPathWindows) - createRepositoryInput.RegisterClusterResourceSetConfigMapTransformation(cniPathWindows, CNIResourcesWindows) - clusterctlConfig := clusterctl.CreateRepository(context.TODO(), createRepositoryInput) Expect(clusterctlConfig).To(BeAnExistingFile(), "The clusterctl config file does not exists in the local repository %s", repositoryFolder) return clusterctlConfig