From 580565ec7dc2511fac46b91c42e4f1a03edfb70e Mon Sep 17 00:00:00 2001 From: Jont828 Date: Mon, 15 May 2023 18:59:32 -0400 Subject: [PATCH 1/8] Install cloud-provider-azure Helm chart in e2e tests using CAAPH --- Makefile | 5 + go.mod | 9 +- go.sum | 14 +- internal/test/env/env.go | 2 + main.go | 2 + templates/caaph/cloud-provider-azure-ci.yaml | 25 + templates/caaph/cloud-provider-azure.yaml | 17 + templates/cluster-template-aad.yaml | 2 + templates/cluster-template-azure-bastion.yaml | 2 + templates/cluster-template-dual-stack.yaml | 1 + templates/cluster-template-edgezone.yaml | 2 + templates/cluster-template-ephemeral.yaml | 2 + templates/cluster-template-flatcar.yaml | 2 + templates/cluster-template-ipv6.yaml | 3 + .../cluster-template-machinepool-flex.yaml | 699 ++++++++++ .../cluster-template-machinepool-windows.yaml | 1 + templates/cluster-template-machinepool.yaml | 2 + templates/cluster-template-nvidia-gpu.yaml | 2 + templates/cluster-template-private.yaml | 2 + templates/cluster-template-windows.yaml | 1 + templates/cluster-template.yaml | 2 + templates/flavors/base/cluster-template.yaml | 2 + templates/flavors/ipv6/patches/ipv6.yaml | 2 + ...ow-ci-version-windows-containerd-2022.yaml | 1177 +++++++++++++++++ .../ci/cluster-template-prow-ci-version.yaml | 1 + .../ci/cluster-template-prow-custom-vnet.yaml | 2 + .../ci/cluster-template-prow-dual-stack.yaml | 1 + .../ci/cluster-template-prow-edgezone.yaml | 2 + .../ci/cluster-template-prow-flatcar.yaml | 2 + ...ow-intree-cloud-provider-machine-pool.yaml | 1 + ...r-template-prow-intree-cloud-provider.yaml | 1 + .../test/ci/cluster-template-prow-ipv6.yaml | 3 + ...template-prow-machine-pool-ci-version.yaml | 1 + ...uster-template-prow-machine-pool-flex.yaml | 1 + .../cluster-template-prow-machine-pool.yaml | 1 + .../ci/cluster-template-prow-nvidia-gpu.yaml | 2 + .../ci/cluster-template-prow-private.yaml | 1 + templates/test/ci/cluster-template-prow.yaml | 1 + ...r-template-custom-builds-machine-pool.yaml | 1 + .../dev/cluster-template-custom-builds.yaml | 1 + test/e2e/azure_clusterproxy.go | 2 + test/e2e/azure_test.go | 39 + test/e2e/cloud-provider-azure.go | 41 - test/e2e/cni.go | 2 +- test/e2e/common.go | 8 +- test/e2e/config/azure-dev.yaml | 17 + .../v1beta1_addon_provider/metadata.yaml | 5 + test/e2e/e2e_suite_test.go | 4 + test/e2e/e2e_suite_vars.go | 6 + 49 files changed, 2067 insertions(+), 57 deletions(-) create mode 100644 templates/caaph/cloud-provider-azure-ci.yaml create mode 100644 templates/caaph/cloud-provider-azure.yaml create mode 100644 templates/cluster-template-machinepool-flex.yaml create mode 100644 templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml create mode 100644 test/e2e/data/shared/v1beta1_addon_provider/metadata.yaml diff --git a/Makefile b/Makefile index 98a998c3bfc..92195bfefc5 100644 --- a/Makefile +++ b/Makefile @@ -164,6 +164,9 @@ GINKGO_ARGS ?= ARTIFACTS ?= $(ROOT_DIR)/_artifacts E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml +E2E_CLOUD_PROVIDER_AZURE_PATH ?= $(ROOT_DIR)/templates/caaph/cloud-provider-azure.yaml +E2E_CLOUD_PROVIDER_AZURE_CI_PATH ?= $(ROOT_DIR)/templates/caaph/cloud-provider-azure-ci.yaml + SKIP_CLEANUP ?= false SKIP_LOG_COLLECTION ?= false SKIP_CREATE_MGMT_CLUSTER ?= false @@ -649,6 +652,8 @@ test-e2e-run: generate-e2e-templates install-tools ## Run e2e tests. $(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \ -e2e.artifacts-folder="$(ARTIFACTS)" \ -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ + -e2e.cloud-provider-azure="$(E2E_CLOUD_PROVIDER_AZURE_PATH)" \ + -e2e.cloud-provider-azure-ci="$(E2E_CLOUD_PROVIDER_AZURE_CI_PATH)" \ -e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \ -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS) diff --git a/go.mod b/go.mod index 05025fc0cef..70e346ab234 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d github.com/blang/semver v3.5.1+incompatible + github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 github.com/go-logr/logr v1.2.4 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 @@ -47,8 +48,9 @@ require ( k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 sigs.k8s.io/cloud-provider-azure v1.26.7 sigs.k8s.io/cluster-api v1.4.2 + sigs.k8s.io/cluster-api-addon-provider-helm v0.1.0-alpha.5 sigs.k8s.io/cluster-api/test v1.4.2 - sigs.k8s.io/controller-runtime v0.14.5 + sigs.k8s.io/controller-runtime v0.14.6 sigs.k8s.io/kind v0.18.0 ) @@ -92,7 +94,6 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect @@ -225,4 +226,6 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.4.2 +replace sigs.k8s.io/cluster-api => github.com/Jont828/cluster-api v0.4.3-0.20230412015901-405900d8bb69 + +replace sigs.k8s.io/cluster-api/test => github.com/Jont828/cluster-api/test v0.4.3-0.20230412015901-405900d8bb69 diff --git a/go.sum b/go.sum index ec7ad016057..ef07763130b 100644 --- a/go.sum +++ b/go.sum @@ -92,6 +92,10 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/Jont828/cluster-api v0.4.3-0.20230412015901-405900d8bb69 h1:OF4FuySy7PRvHDBezR9KTzgM/lQ2I4L961oh3ByHZt0= +github.com/Jont828/cluster-api v0.4.3-0.20230412015901-405900d8bb69/go.mod h1:3NCXYKcbsAB5gl1CmLaiXiz1yaYIVhQiB48IxwqGErI= +github.com/Jont828/cluster-api/test v0.4.3-0.20230412015901-405900d8bb69 h1:G8yGtS6b8BjdFf+NX8G9SOtFIZuzbbLv4lhpujNyEMI= +github.com/Jont828/cluster-api/test v0.4.3-0.20230412015901-405900d8bb69/go.mod h1:ff2gsEFgyCoglpTVxf1DCR+4itUBJ7MpiaQhZyri2Eg= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -1300,12 +1304,10 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/cloud-provider-azure v1.26.7 h1:LSVRPyxeTQZPOF7z42zNZGzL1S1N5tg28RBZVy7gea4= sigs.k8s.io/cloud-provider-azure v1.26.7/go.mod h1:UIwr0Bk4wQb77wNL9cdT4zZw6DP2AtOQ9EKRt9c5g7Q= -sigs.k8s.io/cluster-api v1.4.2 h1:hdIz0Ms2j7YaU1qBK5yF2R8ii0GcGb3jQ7EO6i3tAN8= -sigs.k8s.io/cluster-api v1.4.2/go.mod h1:IIebZTsqyXU8CHbINV2zuMh0/wykqdr+vEXxQNeteEU= -sigs.k8s.io/cluster-api/test v1.4.2 h1:uHFtn0SFOFOxIbdahLoYo4kz84yLqCmhbVLV4vsk1gQ= -sigs.k8s.io/cluster-api/test v1.4.2/go.mod h1:/64ycj3YFMW1BGVtCtfwmlVAXGN0DFTZEkIClh68Svo= -sigs.k8s.io/controller-runtime v0.14.5 h1:6xaWFqzT5KuAQ9ufgUaj1G/+C4Y1GRkhrxl+BJ9i+5s= -sigs.k8s.io/controller-runtime v0.14.5/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= +sigs.k8s.io/cluster-api-addon-provider-helm v0.1.0-alpha.5 h1:F4phJjl9TrdbuRKIQorQo2jQg3AQTgyXkN+PY5E1So4= +sigs.k8s.io/cluster-api-addon-provider-helm v0.1.0-alpha.5/go.mod h1:s3klPverZSGHRlvJxA35TwtM2+6NeRrcljd3cDsD+Jk= +sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= +sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kind v0.18.0 h1:ahgZdVV1pdhXlYe1f+ztISakT23KdrBl/NFY9JMygzs= diff --git a/internal/test/env/env.go b/internal/test/env/env.go index 3c0deb26798..b9804e2ce66 100644 --- a/internal/test/env/env.go +++ b/internal/test/env/env.go @@ -35,6 +35,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/klog/v2" "k8s.io/utils/pointer" + addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/internal/test/record" @@ -59,6 +60,7 @@ func init() { utilruntime.Must(expv1.AddToScheme(scheme)) utilruntime.Must(infrav1.AddToScheme(scheme)) utilruntime.Must(infrav1exp.AddToScheme(scheme)) + utilruntime.Must(addonsv1alpha1.AddToScheme(scheme)) // Get the root of the current file to use in CRD paths. _, filename, _, _ := goruntime.Caller(0) //nolint:dogsled // Ignore "declaration has 3 blank identifiers" check. diff --git a/main.go b/main.go index 4cd66fbc26c..13b8b75b0ca 100644 --- a/main.go +++ b/main.go @@ -36,6 +36,7 @@ import ( cgrecord "k8s.io/client-go/tools/record" "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" + addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/controllers" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" @@ -68,6 +69,7 @@ func init() { _ = infrav1exp.AddToScheme(scheme) _ = clusterv1.AddToScheme(scheme) _ = expv1.AddToScheme(scheme) + _ = addonsv1alpha1.AddToScheme(scheme) _ = kubeadmv1.AddToScheme(scheme) // +kubebuilder:scaffold:scheme diff --git a/templates/caaph/cloud-provider-azure-ci.yaml b/templates/caaph/cloud-provider-azure-ci.yaml new file mode 100644 index 00000000000..6d71cf0155b --- /dev/null +++ b/templates/caaph/cloud-provider-azure-ci.yaml @@ -0,0 +1,25 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci +spec: + clusterSelector: + matchLabels: + installCloudProviderAzureChart: "true" + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + chartName: cloud-provider-azure + releaseName: cloud-provider-azure + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME} + imageRepository: ${CCM_IMAGE_REGISTRY} + imageTag: ${IMAGE_TAG_CCM} + logVerbosity: 4 + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME} + imageRepository: ${CNM_IMAGE_REGISTRY} + imageTag: ${IMAGE_TAG_CCM} + diff --git a/templates/caaph/cloud-provider-azure.yaml b/templates/caaph/cloud-provider-azure.yaml new file mode 100644 index 00000000000..de480814739 --- /dev/null +++ b/templates/caaph/cloud-provider-azure.yaml @@ -0,0 +1,17 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart +spec: + clusterSelector: + matchLabels: + installCloudProviderAzureChart: "true" + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + chartName: cloud-provider-azure + releaseName: cloud-provider-azure + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml index 07f530abb2a..4f42a4ae070 100644 --- a/templates/cluster-template-aad.yaml +++ b/templates/cluster-template-aad.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index 98ffb2f540c..1658d2cfe61 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-dual-stack.yaml b/templates/cluster-template-dual-stack.yaml index 1a72e1a0eca..f1eba4a7af0 100644 --- a/templates/cluster-template-dual-stack.yaml +++ b/templates/cluster-template-dual-stack.yaml @@ -3,6 +3,7 @@ kind: Cluster metadata: labels: cni: calico-dual-stack + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-edgezone.yaml b/templates/cluster-template-edgezone.yaml index 54b93177496..c7e8f67671f 100644 --- a/templates/cluster-template-edgezone.yaml +++ b/templates/cluster-template-edgezone.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index 55ec76cf701..196388d4968 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-flatcar.yaml b/templates/cluster-template-flatcar.yaml index 2978263dab6..f270ed1be10 100644 --- a/templates/cluster-template-flatcar.yaml +++ b/templates/cluster-template-flatcar.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-ipv6.yaml b/templates/cluster-template-ipv6.yaml index caa7a12f41b..e0d22cbf6df 100644 --- a/templates/cluster-template-ipv6.yaml +++ b/templates/cluster-template-ipv6.yaml @@ -1,6 +1,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + cni: calico-ipv6 + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-machinepool-flex.yaml b/templates/cluster-template-machinepool-flex.yaml new file mode 100644 index 00000000000..9c13ab8a578 --- /dev/null +++ b/templates/cluster-template-machinepool-flex.yaml @@ -0,0 +1,699 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + windows: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: [] + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + location: ${AZURE_LOCATION} + orchestrationMode: Flexible + strategy: + rollingUpdate: + maxSurge: 0% + maxUnavailable: 0 + type: RollingUpdate + template: + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-mp-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + annotations: + runtime: containerd + windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + location: ${AZURE_LOCATION} + orchestrationMode: Flexible + strategy: + rollingUpdate: + maxSurge: 0% + maxUnavailable: 0 + type: RollingUpdate + template: + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-mp-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + azure-container-registry-config: c:/k/azure.json + cloud-provider: external + pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell c:/create-external-network.ps1 + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico-windows + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: containerd-logger-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + containerd-logger: enabled + resources: + - kind: ConfigMap + name: containerd-logger-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: v1 +data: + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: + v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n + \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: + |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": + [\n {\n \"windows_use_single_network\": true,\n \"type\": + \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n + \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n + \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n + \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": + {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n + \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": + \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": + \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n + \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n + \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": + \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": + \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": + \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n + \ },\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": + \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n + \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: + DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n + \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n + \ app: calico\n template:\n metadata:\n labels:\n tier: node\n + \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n + \ nodeSelectorTerms:\n - matchExpressions:\n - + key: kubernetes.io/os\n operator: In\n values:\n + \ - windows\n - key: kubernetes.io/arch\n + \ operator: In\n values:\n - + amd64\n securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: + true\n serviceAccountName: calico-node\n tolerations:\n - operator: + Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on + for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n + \ - effect: NoExecute\n operator: Exists\n initContainers:\n # + This container installs the CNI binaries\n # and CNI network config file + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.25.0-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n + \ imagePullPolicy: Always\n env:\n # Name of the CNI + config file to create.\n - name: CNI_CONF_NAME\n value: + \"10-calico.conflist\"\n # The CNI network config to install on each + node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n + \ name: calico-config-windows\n key: cni_network_config\n + \ # Set the hostname based on the k8s node name.\n - name: + KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: + spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n + \ valueFrom:\n configMapKeyRef:\n name: + calico-config-windows\n key: veth_mtu\n # Prevents + the container from sleeping forever.\n - name: SLEEP\n value: + \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n + \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: + cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: + cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n + \ securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.25.0-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n + \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: + Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: + /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n + \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n + \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n + \ image: sigwindowstools/calico-node:v3.25.0-hostprocess\n args: + [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: + Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n + \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n + \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n + \ subPath: static-rules.json\n env:\n - name: POD_NAME\n + \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: + \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n + \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n + \ configMap:\n name: calico-static-rules\n # Used to install + CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n + \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n + \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n---\napiVersion: + apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n + \ group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: + IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: + false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n + \ properties:\n apiVersion:\n description: 'APIVersion + defines the versioned schema of this representation\n of an object. + Servers should convert recognized schemas to the latest\n internal + value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n + \ type: string\n kind:\n description: 'Kind is a + string value representing the REST resource this\n object represents. + Servers may infer this from the endpoint the client\n submits requests + to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n + \ type: string\n metadata:\n type: object\n spec:\n + \ description: IPAMConfigSpec contains the specification for an IPAMConfig\n + \ resource.\n properties:\n autoAllocateBlocks:\n + \ type: boolean\n maxBlocksPerHost:\n description: + MaxBlocksPerHost, if non-zero, is the max number of blocks\n that + can be affine to each host.\n maximum: 2147483647\n minimum: + 0\n type: integer\n strictAffinity:\n type: + boolean\n required:\n - autoAllocateBlocks\n - + strictAffinity\n type: object\n type: object\n served: true\n + \ storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n + \ conditions: []\n storedVersions: []\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default +--- +apiVersion: v1 +data: + containerd-windows-logger: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: containerd-logger + name: containerd-logger + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: containerd-logger + template: + metadata: + labels: + k8s-app: containerd-logger + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 + args: [ "config.json" ] + name: containerd-logger + imagePullPolicy: Always + volumeMounts: + - name: containerd-logger-config + mountPath: /config.json + subPath: config.json + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: containerd-logger-config + name: containerd-logger-config + updateStrategy: + type: RollingUpdate + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: containerd-logger-config + namespace: kube-system + data: + config.json: | + { + "inputs": [ + { + "type": "ETW", + "sessionNamePrefix": "containerd", + "cleanupOldSessions": true, + "reuseExistingSession": true, + "providers": [ + { + "providerName": "Microsoft.Virtualization.RunHCS", + "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", + "level": "Verbose" + }, + { + "providerName": "ContainerD", + "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", + "level": "Verbose" + } + ] + } + ], + "filters": [ + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" + } + ], + "outputs": [ + { + "type": "StdOutput" + } + ], + "schemaVersion": "2016-08-11" + } +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: containerd-logger-${CLUSTER_NAME} + namespace: default diff --git a/templates/cluster-template-machinepool-windows.yaml b/templates/cluster-template-machinepool-windows.yaml index d691f0bdf97..22671e03038 100644 --- a/templates/cluster-template-machinepool-windows.yaml +++ b/templates/cluster-template-machinepool-windows.yaml @@ -4,6 +4,7 @@ metadata: labels: cni-windows: calico csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template-machinepool.yaml b/templates/cluster-template-machinepool.yaml index b4f9bd97f1e..347d22480ff 100644 --- a/templates/cluster-template-machinepool.yaml +++ b/templates/cluster-template-machinepool.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml index bb758313b47..1320d0a3653 100644 --- a/templates/cluster-template-nvidia-gpu.yaml +++ b/templates/cluster-template-nvidia-gpu.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml index 252b03eab1a..9d340b00200 100644 --- a/templates/cluster-template-private.yaml +++ b/templates/cluster-template-private.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index c84d2a1b6af..bd234c01bd9 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -4,6 +4,7 @@ metadata: labels: cni-windows: calico csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 270dd7b3758..b4c961cab77 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/flavors/base/cluster-template.yaml b/templates/flavors/base/cluster-template.yaml index 95f645b7153..120f4cc27a9 100644 --- a/templates/flavors/base/cluster-template.yaml +++ b/templates/flavors/base/cluster-template.yaml @@ -3,6 +3,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: name: ${CLUSTER_NAME} + labels: + installCloudProviderAzureChart: "true" spec: clusterNetwork: pods: diff --git a/templates/flavors/ipv6/patches/ipv6.yaml b/templates/flavors/ipv6/patches/ipv6.yaml index ea0eff7286d..1731bfa559a 100644 --- a/templates/flavors/ipv6/patches/ipv6.yaml +++ b/templates/flavors/ipv6/patches/ipv6.yaml @@ -3,6 +3,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: name: ${CLUSTER_NAME} + labels: + cni: "calico-ipv6" spec: clusterNetwork: pods: diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml new file mode 100644 index 00000000000..e422c99539d --- /dev/null +++ b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml @@ -0,0 +1,1177 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + enabledCloudProviderAzureChart: true + metrics-server: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + natGateway: + name: node-natgateway + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + feature-gates: ${K8S_FEATURE_GATES:-""} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + feature-gates: HPAContainerMetrics=true + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + kubernetesVersion: ci/${CI_VERSION} + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + wget "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true --short=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" + owner: root:root + path: /tmp/kubeadm-bootstrap.sh + permissions: "0744" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: + - bash -c /tmp/kubeadm-bootstrap.sh + verbosity: 5 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-1804-gen1 + version: latest + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: {} + template: + metadata: + labels: + nodepool: pool1 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-1804-gen1 + version: latest + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + wget "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true --short=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" + owner: root:root + path: /tmp/kubeadm-bootstrap.sh + permissions: "0744" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - bash -c /tmp/kubeadm-bootstrap.sh + verbosity: 5 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} + selector: {} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + annotations: + runtime: containerd + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + metadata: + annotations: + runtime: containerd + windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} + spec: + image: + marketplace: + offer: capi-windows + publisher: cncf-upstream + sku: windows-2022-containerd-gen1 + version: latest + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: |- + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + - content: | + # /tmp is assumed created and required for upstream e2e tests to pass + New-Item -ItemType Directory -Force -Path C:\tmp\ + path: C:/create-temp-folder.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" + if($$CONTAINERD_URL -ne ""){ + # Kubelet service depends on contianerd service so make a best effort attempt to stop it + Stop-Service kubelet -Force -ErrorAction SilentlyContinue + Stop-Service containerd -Force + echo "downloading containerd: $$CONTAINERD_URL" + curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" + tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 + + Start-Service containerd + } + + containerd.exe --version + containerd-shim-runhcs-v1.exe --version + path: C:/replace-containerd.ps1 + permissions: "0744" + - content: | + mkdir -Force c:/localdumps + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f + # Enable sftp so we can copy crash dump files during log collection of stfp + $sshd_config = "$env:ProgramData\ssh\sshd_config" + if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } + Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" + sc.exe stop sshd + sc.exe start sshd + path: C:/collect-hns-crashes.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + Stop-Service kubelet -Force + + $$CI_VERSION="${CI_VERSION}" + if($$CI_VERSION -ne "") + { + $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") + $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" + foreach ( $$binary in $$binaries ) + { + echo "downloading binary: $$ci_url/$$binary.exe" + curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" + } + } + + # Tag it to the ci version. The image knows how to use the copy locally with the configmap + # that is applied at at this stage (windows-kubeproxy-ci.yaml) + ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess + ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" + + kubeadm.exe version -o=short + kubectl.exe version --client=true --short=true + kubelet.exe --version + kube-proxy.exe --version + path: C:/replace-ci-binaries.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + azure-container-registry-config: c:/k/azure.json + cloud-provider: external + v: "2" + windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell C:/create-temp-folder.ps1 + - powershell C:/replace-containerd.ps1 + - powershell C:/collect-hns-crashes.ps1 + - powershell C:/replace-ci-binaries.ps1 + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-mhc-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + nodepool: pool1 + unhealthyConditions: + - status: "True" + timeout: 30s + type: E2ENodeUnhealthy +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico-windows + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: containerd-logger-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + containerd-logger: enabled + resources: + - kind: ConfigMap + name: containerd-logger-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: v1 +data: + kube-proxy-patch: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: windows-kubeproxy-ci + namespace: kube-system + data: + KUBEPROXY_PATH: "c:/k/kube-proxy.exe" + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: + v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n + \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: + |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": + [\n {\n \"windows_use_single_network\": true,\n \"type\": + \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n + \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n + \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n + \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": + {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n + \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": + \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": + \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n + \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n + \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": + \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": + \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": + \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n + \ },\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": + \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n + \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: + DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n + \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n + \ app: calico\n template:\n metadata:\n labels:\n tier: node\n + \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n + \ nodeSelectorTerms:\n - matchExpressions:\n - + key: kubernetes.io/os\n operator: In\n values:\n + \ - windows\n - key: kubernetes.io/arch\n + \ operator: In\n values:\n - + amd64\n securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: + true\n serviceAccountName: calico-node\n tolerations:\n - operator: + Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on + for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n + \ - effect: NoExecute\n operator: Exists\n initContainers:\n # + This container installs the CNI binaries\n # and CNI network config file + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.25.0-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n + \ imagePullPolicy: Always\n env:\n # Name of the CNI + config file to create.\n - name: CNI_CONF_NAME\n value: + \"10-calico.conflist\"\n # The CNI network config to install on each + node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n + \ name: calico-config-windows\n key: cni_network_config\n + \ # Set the hostname based on the k8s node name.\n - name: + KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: + spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n + \ valueFrom:\n configMapKeyRef:\n name: + calico-config-windows\n key: veth_mtu\n # Prevents + the container from sleeping forever.\n - name: SLEEP\n value: + \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n + \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: + cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: + cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n + \ securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.25.0-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n + \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: + Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: + /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n + \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n + \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n + \ image: sigwindowstools/calico-node:v3.25.0-hostprocess\n args: + [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: + Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n + \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n + \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n + \ subPath: static-rules.json\n env:\n - name: POD_NAME\n + \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: + \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n + \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n + \ configMap:\n name: calico-static-rules\n # Used to install + CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n + \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n + \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n---\napiVersion: + apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n + \ group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: + IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: + false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n + \ properties:\n apiVersion:\n description: 'APIVersion + defines the versioned schema of this representation\n of an object. + Servers should convert recognized schemas to the latest\n internal + value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n + \ type: string\n kind:\n description: 'Kind is a + string value representing the REST resource this\n object represents. + Servers may infer this from the endpoint the client\n submits requests + to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n + \ type: string\n metadata:\n type: object\n spec:\n + \ description: IPAMConfigSpec contains the specification for an IPAMConfig\n + \ resource.\n properties:\n autoAllocateBlocks:\n + \ type: boolean\n maxBlocksPerHost:\n description: + MaxBlocksPerHost, if non-zero, is the max number of blocks\n that + can be affine to each host.\n maximum: 2147483647\n minimum: + 0\n type: integer\n strictAffinity:\n type: + boolean\n required:\n - autoAllocateBlocks\n - + strictAffinity\n type: object\n type: object\n served: true\n + \ storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n + \ conditions: []\n storedVersions: []\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default +--- +apiVersion: v1 +data: + containerd-windows-logger: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: containerd-logger + name: containerd-logger + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: containerd-logger + template: + metadata: + labels: + k8s-app: containerd-logger + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 + args: [ "config.json" ] + name: containerd-logger + imagePullPolicy: Always + volumeMounts: + - name: containerd-logger-config + mountPath: /config.json + subPath: config.json + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: containerd-logger-config + name: containerd-logger-config + updateStrategy: + type: RollingUpdate + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: containerd-logger-config + namespace: kube-system + data: + config.json: | + { + "inputs": [ + { + "type": "ETW", + "sessionNamePrefix": "containerd", + "cleanupOldSessions": true, + "reuseExistingSession": true, + "providers": [ + { + "providerName": "Microsoft.Virtualization.RunHCS", + "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", + "level": "Verbose" + }, + { + "providerName": "ContainerD", + "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", + "level": "Verbose" + } + ] + } + ], + "filters": [ + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" + } + ], + "outputs": [ + { + "type": "StdOutput" + } + ], + "schemaVersion": "2016-08-11" + } +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: containerd-logger-${CLUSTER_NAME} + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: metrics-server-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + metrics-server: enabled + resources: + - kind: ConfigMap + name: metrics-server-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: v1 +data: + metrics-server: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader + rules: + - apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + rules: + - apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get + - apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: v1 + kind: Service + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --kubelet-insecure-tls + image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - emptyDir: {} + name: tmp-dir + --- + apiVersion: apiregistration.k8s.io/v1 + kind: APIService + metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io + spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: metrics-server-${CLUSTER_NAME} + namespace: default diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 13a69579c8f..a8901abaf58 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" metrics-server: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 877f8c4521e..6a2cc05f1c4 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index cd735e156ed..9c105e52945 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -3,6 +3,7 @@ kind: Cluster metadata: labels: cni: calico-dual-stack + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index c15c46bab89..d61aa39c228 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index 38a0c5ddf31..00fe8060c9f 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml index db32250a6c9..11ec314986b 100644 --- a/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml index 9f662623fd8..5810b203bef 100644 --- a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index 378c835f9b8..9c2aeedcf43 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -1,6 +1,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + cni: calico-ipv6 + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 5d7f8dd7954..529476dd626 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index 9c13ab8a578..cbf34ecbf2e 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 3424c731f5f..5f43005ffac 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 939e2427765..1e0f67dd9d4 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index 6c443b81688..13696e7739d 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -3,6 +3,7 @@ kind: Cluster metadata: labels: cni: ${CLUSTER_NAME}-calico + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index b03005e8edf..eafe10a3f4f 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index a6a90eb3257..4c497b50c73 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index 51b4cd5d8d2..b69c3115cec 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" metrics-server: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/test/e2e/azure_clusterproxy.go b/test/e2e/azure_clusterproxy.go index ec17cd5b49f..2af4a415a25 100644 --- a/test/e2e/azure_clusterproxy.go +++ b/test/e2e/azure_clusterproxy.go @@ -44,6 +44,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kubectl/pkg/describe" "k8s.io/utils/pointer" + addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" @@ -73,6 +74,7 @@ func initScheme() *runtime.Scheme { Expect(infrav1.AddToScheme(scheme)).To(Succeed()) Expect(infrav1exp.AddToScheme(scheme)).To(Succeed()) Expect(expv1.AddToScheme(scheme)).To(Succeed()) + Expect(addonsv1alpha1.AddToScheme(scheme)).To(Succeed()) // Add aadpodidentity v1 to the scheme. aadPodIdentityGroupVersion := schema.GroupVersion{Group: aadpodv1.GroupName, Version: "v1"} scheme.AddKnownTypes(aadPodIdentityGroupVersion, diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 8158b11821a..6b107815726 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -25,16 +25,20 @@ import ( "os" "time" + "github.com/drone/envsubst/v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/yaml" + addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/controller-runtime/pkg/client" ) var _ = Describe("Workload cluster creation", func() { @@ -76,6 +80,17 @@ var _ = Describe("Workload cluster creation", func() { namespace, cancelWatches, err = setupSpecNamespace(ctx, clusterNamePrefix, bootstrapClusterProxy, artifactFolder) Expect(err).NotTo(HaveOccurred()) + By("Initialize bootstrap client to install add-ons") + bootstrapClient := bootstrapClusterProxy.GetClient() + Expect(bootstrapClient).NotTo(BeNil()) + + By("Creating cloud-provider-azure HelmChartProxy") + path := cloudProviderAzurePath + if useCIArtifacts { + path = cloudProviderAzureCIPath + } + createHelmChartProxyFromFilePath(bootstrapClient, path, namespace.Name) + result = new(clusterctl.ApplyClusterTemplateAndWaitResult) spClientSecret := os.Getenv(AzureClientSecret) @@ -903,3 +918,27 @@ var _ = Describe("Workload cluster creation", func() { }) }) }) + +func createHelmChartProxyFromFilePath(client client.Client, path string, namespace string) { + content, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + + result, err := envsubst.EvalEnv(string(content)) + Expect(err).NotTo(HaveOccurred()) + content = []byte(result) + + Logf("HelmChartProxy content:\n %s", string(content)) + + var helmChartProxy addonsv1alpha1.HelmChartProxy + err = yaml.Unmarshal(content, &helmChartProxy) + Expect(err).NotTo(HaveOccurred()) + + helmChartProxy.Namespace = namespace + + Logf("HelmChartProxy object is %+v", helmChartProxy) + + if err := client.Create(context.Background(), &helmChartProxy); err != nil { + Logf("Failed to create HelmChartProxy: %v", err) + Expect(err).NotTo(HaveOccurred()) + } +} diff --git a/test/e2e/cloud-provider-azure.go b/test/e2e/cloud-provider-azure.go index 5f7ef53e2cf..0b1ff2d7b5b 100644 --- a/test/e2e/cloud-provider-azure.go +++ b/test/e2e/cloud-provider-azure.go @@ -21,9 +21,6 @@ package e2e import ( "context" - "fmt" - "os" - "strings" . "github.com/onsi/ginkgo/v2" helmVals "helm.sh/helm/v3/pkg/cli/values" @@ -31,49 +28,11 @@ import ( ) const ( - cloudProviderAzureHelmRepoURL = "https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo" - cloudProviderAzureChartName = "cloud-provider-azure" - cloudProviderAzureHelmReleaseName = "cloud-provider-azure-oot" azureDiskCSIDriverHelmRepoURL = "https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts" azureDiskCSIDriverChartName = "azuredisk-csi-driver" azureDiskCSIDriverHelmReleaseName = "azuredisk-csi-driver-oot" ) -// InstallCalicoAndCloudProviderAzureHelmChart installs the official cloud-provider-azure helm chart -// and validates that expected pods exist and are Ready. -func InstallCalicoAndCloudProviderAzureHelmChart(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool) { - specName := "cloud-provider-azure-install" - By("Installing cloud-provider-azure components via helm") - options := &helmVals.Options{ - Values: []string{ - fmt.Sprintf("infra.clusterName=%s", input.ConfigCluster.ClusterName), - "cloudControllerManager.logVerbosity=4", - }, - StringValues: []string{fmt.Sprintf("cloudControllerManager.clusterCIDR=%s", strings.Join(cidrBlocks, `\,`))}, - } - // If testing a CI version of Kubernetes, use CCM and CNM images built from source. - if useCIArtifacts || usePRArtifacts { - options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageName=%s", os.Getenv("CCM_IMAGE_NAME"))) - options.Values = append(options.Values, fmt.Sprintf("cloudNodeManager.imageName=%s", os.Getenv("CNM_IMAGE_NAME"))) - options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageRepository=%s", os.Getenv("IMAGE_REGISTRY"))) - options.Values = append(options.Values, fmt.Sprintf("cloudNodeManager.imageRepository=%s", os.Getenv("IMAGE_REGISTRY"))) - options.StringValues = append(options.StringValues, fmt.Sprintf("cloudControllerManager.imageTag=%s", os.Getenv("IMAGE_TAG_CCM"))) - options.StringValues = append(options.StringValues, fmt.Sprintf("cloudNodeManager.imageTag=%s", os.Getenv("IMAGE_TAG_CNM"))) - } - - clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName) - InstallHelmChart(ctx, clusterProxy, defaultNamespace, cloudProviderAzureHelmRepoURL, cloudProviderAzureChartName, cloudProviderAzureHelmReleaseName, options) - - // Install Calico CNI Helm Chart. We do this before waiting for the pods to be ready because there is a co-dependency between CNI (nodes ready) and cloud-provider being initialized. - InstallCalicoHelmChart(ctx, input, cidrBlocks, hasWindows) - - By("Waiting for Ready cloud-controller-manager deployment pods") - for _, d := range []string{"cloud-controller-manager"} { - waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName) - WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...) - } -} - // InstallAzureDiskCSIDriverHelmChart installs the official azure-disk CSI driver helm chart func InstallAzureDiskCSIDriverHelmChart(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, hasWindows bool) { specName := "azuredisk-csi-drivers-install" diff --git a/test/e2e/cni.go b/test/e2e/cni.go index 4cc41302588..244a189734c 100644 --- a/test/e2e/cni.go +++ b/test/e2e/cni.go @@ -46,7 +46,7 @@ const ( func InstallCalicoHelmChart(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool) { specName := "calico-install" - By("Installing Calico CNI via helm") + By("Installing Calico via Helm") values := getCalicoValues(cidrBlocks) clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName) InstallHelmChart(ctx, clusterProxy, calicoOperatorNamespace, calicoHelmChartRepoURL, calicoHelmChartName, calicoHelmReleaseName, values) diff --git a/test/e2e/common.go b/test/e2e/common.go index d6376958cc9..1a8d957d9bd 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -275,12 +275,8 @@ func EnsureControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCl }, input.WaitForControlPlaneIntervals...).Should(Succeed(), "API Server was not reachable in time") _, hasWindows := cluster.Labels["cni-windows"] - if kubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.ControllerManager.ExtraArgs["cloud-provider"] != "azure" { - // There is a co-dependency between cloud-provider and CNI so we install both together if cloud-provider is external. - InstallCalicoAndCloudProviderAzureHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) - } else { - InstallCalicoHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) - } + InstallCalicoHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) + controlPlane := discoveryAndWaitForControlPlaneInitialized(ctx, input, result) v, err := semver.ParseTolerant(input.ConfigCluster.KubernetesVersion) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 7a44c9efb50..ceff8149849 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -9,6 +9,8 @@ images: loadBehavior: tryLoad - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.4.2 loadBehavior: tryLoad + - name: gcr.io/k8s-staging-cluster-api-helm/cluster-api-helm-controller:v0.1.0-alpha.5 + loadBehavior: tryLoad providers: - name: cluster-api @@ -140,6 +142,21 @@ providers: - old: "--v=0" new: "--v=2" + - name: helm + type: AddonProvider + versions: + - name: v0.1.0-alpha.5 + value: https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/releases/download/v0.1.0-alpha.5/add-on-components.yaml + type: url + contract: v1beta1 + files: + - sourcePath: "../data/shared/v1beta1_addon_provider/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "image: registry.k8s.io/cluster-api-helm/cluster-api-helm-controller:v0.1.0-alpha.5" + new: "image: gcr.io/k8s-staging-cluster-api-helm/cluster-api-helm-controller:v0.1.0-alpha.5" + variables: AKS_KUBERNETES_VERSION: "latest" AKS_KUBERNETES_VERSION_UPGRADE_FROM: "latest-1" diff --git a/test/e2e/data/shared/v1beta1_addon_provider/metadata.yaml b/test/e2e/data/shared/v1beta1_addon_provider/metadata.yaml new file mode 100644 index 00000000000..a087e998608 --- /dev/null +++ b/test/e2e/data/shared/v1beta1_addon_provider/metadata.yaml @@ -0,0 +1,5 @@ +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +releaseSeries: + - major: 0 + minor: 1 + contract: v1beta1 diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 51e61ed48ba..e84ea38b173 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -39,6 +39,8 @@ import ( func init() { flag.StringVar(&configPath, "e2e.config", "", "path to the e2e config file") + flag.StringVar(&cloudProviderAzurePath, "e2e.cloud-provider-azure", "", "path to the cloud-provider-azure HelmChartProxy") + flag.StringVar(&cloudProviderAzureCIPath, "e2e.cloud-provider-azure-ci", "", "path to the cloud-provider-azure HelmChartProxy with CI artifacts") flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored") flag.BoolVar(&useCIArtifacts, "kubetest.use-ci-artifacts", false, "use the latest build from the main branch of the Kubernetes repository. Set KUBERNETES_VERSION environment variable to latest-1.xx to use the build from 1.xx release branch.") flag.BoolVar(&usePRArtifacts, "kubetest.use-pr-artifacts", false, "use the build from a PR of the Kubernetes repository") @@ -74,6 +76,8 @@ var _ = SynchronizedBeforeSuite(func() []byte { By("Initializing the bootstrap cluster") initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) + // TODO: set this helm stuff up + // encode the e2e config into the byte array. var configBuf bytes.Buffer enc := gob.NewEncoder(&configBuf) diff --git a/test/e2e/e2e_suite_vars.go b/test/e2e/e2e_suite_vars.go index c39ce2376b1..221ff3e513b 100644 --- a/test/e2e/e2e_suite_vars.go +++ b/test/e2e/e2e_suite_vars.go @@ -76,4 +76,10 @@ var ( // usePRArtifacts specifies whether or not to use the build from a PR of the Kubernetes repository usePRArtifacts bool + + // cloudProviderAzurePath specifies the path to the cloud-provider-azure HelmChartProxy + cloudProviderAzurePath string + + // cloudProviderAzureCIPath specifies the path to the cloud-provider-azure HelmChartProxy with CI artifacts enabled + cloudProviderAzureCIPath string ) From 4bf54262129f96b791c5c02f0fce5fbc9a5cf73f Mon Sep 17 00:00:00 2001 From: Jont828 Date: Tue, 16 May 2023 22:17:21 -0400 Subject: [PATCH 2/8] Install calico chart in e2e tests using CAAPH --- Makefile | 6 + templates/caaph/calico-dual-stack.yaml | 32 + templates/caaph/calico-ipv6.yaml | 27 + templates/caaph/calico.yaml | 23 + templates/cluster-template-aad.yaml | 1 + templates/cluster-template-azure-bastion.yaml | 1 + templates/cluster-template-edgezone.yaml | 1 + templates/cluster-template-ephemeral.yaml | 1 + templates/cluster-template-flatcar.yaml | 1 + .../cluster-template-machinepool-flex.yaml | 699 ------------------ .../cluster-template-machinepool-windows.yaml | 1 + templates/cluster-template-machinepool.yaml | 1 + templates/cluster-template-nvidia-gpu.yaml | 1 + templates/cluster-template-private.yaml | 1 + templates/cluster-template-windows.yaml | 1 + templates/cluster-template.yaml | 1 + templates/flavors/base/cluster-template.yaml | 1 + .../ci/cluster-template-prow-ci-version.yaml | 1 + .../ci/cluster-template-prow-custom-vnet.yaml | 1 + .../ci/cluster-template-prow-edgezone.yaml | 1 + .../ci/cluster-template-prow-flatcar.yaml | 1 + ...ow-intree-cloud-provider-machine-pool.yaml | 1 + ...r-template-prow-intree-cloud-provider.yaml | 1 + ...template-prow-machine-pool-ci-version.yaml | 1 + ...uster-template-prow-machine-pool-flex.yaml | 1 + .../cluster-template-prow-machine-pool.yaml | 1 + .../ci/cluster-template-prow-nvidia-gpu.yaml | 1 + templates/test/ci/cluster-template-prow.yaml | 1 + ...r-template-custom-builds-machine-pool.yaml | 1 + .../dev/cluster-template-custom-builds.yaml | 1 + test/e2e/azure_test.go | 4 + test/e2e/cni.go | 8 +- test/e2e/common.go | 2 +- test/e2e/e2e_suite_test.go | 3 + test/e2e/e2e_suite_vars.go | 9 + 35 files changed, 133 insertions(+), 705 deletions(-) create mode 100644 templates/caaph/calico-dual-stack.yaml create mode 100644 templates/caaph/calico-ipv6.yaml create mode 100644 templates/caaph/calico.yaml delete mode 100644 templates/cluster-template-machinepool-flex.yaml diff --git a/Makefile b/Makefile index 92195bfefc5..dbff24a5a75 100644 --- a/Makefile +++ b/Makefile @@ -166,6 +166,9 @@ E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml E2E_CLOUD_PROVIDER_AZURE_PATH ?= $(ROOT_DIR)/templates/caaph/cloud-provider-azure.yaml E2E_CLOUD_PROVIDER_AZURE_CI_PATH ?= $(ROOT_DIR)/templates/caaph/cloud-provider-azure-ci.yaml +E2E_CALICO_PATH ?= $(ROOT_DIR)/templates/caaph/calico.yaml +E2E_CALICO_IPV6_PATH ?= $(ROOT_DIR)/templates/caaph/calico-ipv6.yaml +E2E_CALICO_DUAL_STACK_PATH ?= $(ROOT_DIR)/templates/caaph/calico-dual-stack.yaml SKIP_CLEANUP ?= false SKIP_LOG_COLLECTION ?= false @@ -654,6 +657,9 @@ test-e2e-run: generate-e2e-templates install-tools ## Run e2e tests. -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ -e2e.cloud-provider-azure="$(E2E_CLOUD_PROVIDER_AZURE_PATH)" \ -e2e.cloud-provider-azure-ci="$(E2E_CLOUD_PROVIDER_AZURE_CI_PATH)" \ + -e2e.calico="$(E2E_CALICO_PATH)" \ + -e2e.calico-ipv6="$(E2E_CALICO_IPV6_PATH)" \ + -e2e.calico-dual-stack="$(E2E_CALICO_DUAL_STACK_PATH)" \ -e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \ -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS) diff --git a/templates/caaph/calico-dual-stack.yaml b/templates/caaph/calico-dual-stack.yaml new file mode 100644 index 00000000000..91ea3fd7e62 --- /dev/null +++ b/templates/caaph/calico-dual-stack.yaml @@ -0,0 +1,32 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico-dual-stack-chart +spec: + clusterSelector: + matchLabels: + cni: calico-dual-stack + repoURL: https://docs.tigera.io/calico/charts + chartName: tigera-operator + releaseName: projectcalico + namespace: tigera-operator + valuesTemplate: | + installation: + cni: + type: Calico + ipam: + type: HostLocal + calicoNetwork: + bgp: Disabled + mtu: 1350 + ipPools: + - blockSize: 26 + cidr: {{ index .Cluster.spec.clusterNetwork.pods.cidrBlocks 0 }} + encapsulation: None + natOutgoing: Enabled + nodeSelector: all() + - blockSize: 122 + cidr: {{ index .Cluster.spec.clusterNetwork.pods.cidrBlocks 1 }} + encapsulation: None + natOutgoing: Enabled + nodeSelector: all() diff --git a/templates/caaph/calico-ipv6.yaml b/templates/caaph/calico-ipv6.yaml new file mode 100644 index 00000000000..7c20aa90c4a --- /dev/null +++ b/templates/caaph/calico-ipv6.yaml @@ -0,0 +1,27 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico-ipv6-chart +spec: + clusterSelector: + matchLabels: + cni: calico-ipv6 + repoURL: https://docs.tigera.io/calico/charts + chartName: tigera-operator + releaseName: projectcalico + namespace: tigera-operator + valuesTemplate: | + installation: + cni: + type: Calico + ipam: + type: HostLocal + calicoNetwork: + bgp: Disabled + mtu: 1350 + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - blockSize: 122 + cidr: {{ $cidr }} + encapsulation: None + natOutgoing: Enabled + nodeSelector: all(){{end}} diff --git a/templates/caaph/calico.yaml b/templates/caaph/calico.yaml new file mode 100644 index 00000000000..bbed8beb246 --- /dev/null +++ b/templates/caaph/calico.yaml @@ -0,0 +1,23 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico-chart +spec: + clusterSelector: + matchLabels: + cni: calico + repoURL: https://docs.tigera.io/calico/charts + chartName: tigera-operator + releaseName: projectcalico + namespace: tigera-operator + valuesTemplate: | + installation: + cni: + type: Calico + calicoNetwork: + bgp: Disabled + mtu: 1350 + ipPools: + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml index 4f42a4ae070..4a90b2b36b6 100644 --- a/templates/cluster-template-aad.yaml +++ b/templates/cluster-template-aad.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index 1658d2cfe61..296585b8aaf 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template-edgezone.yaml b/templates/cluster-template-edgezone.yaml index c7e8f67671f..cdf62599644 100644 --- a/templates/cluster-template-edgezone.yaml +++ b/templates/cluster-template-edgezone.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index 196388d4968..801b9c88e21 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template-flatcar.yaml b/templates/cluster-template-flatcar.yaml index f270ed1be10..71bd485d0c3 100644 --- a/templates/cluster-template-flatcar.yaml +++ b/templates/cluster-template-flatcar.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template-machinepool-flex.yaml b/templates/cluster-template-machinepool-flex.yaml deleted file mode 100644 index 9c13ab8a578..00000000000 --- a/templates/cluster-template-machinepool-flex.yaml +++ /dev/null @@ -1,699 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled - windows: enabled - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-provider: external - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-provider: external - cluster-name: ${CLUSTER_NAME} - v: "4" - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-provider: external - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-provider: external - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - location: ${AZURE_LOCATION} - orchestrationMode: Flexible - strategy: - rollingUpdate: - maxSurge: 0% - maxUnavailable: 0 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmExtensions: - - name: CustomScript - protectedSettings: - commandToExecute: | - #!/bin/sh - echo "This script is a no-op used for extension testing purposes ..." - touch test_file - publisher: Microsoft.Azure.Extensions - version: "2.1" - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-provider: external - name: '{{ ds.meta_data["local_hostname"] }}' ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - location: ${AZURE_LOCATION} - orchestrationMode: Flexible - strategy: - rollingUpdate: - maxSurge: 0% - maxUnavailable: 0 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - azure-container-registry-config: c:/k/azure.json - cloud-provider: external - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell c:/create-external-network.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- -apiVersion: v1 -data: - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: - v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n - \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: - |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": - [\n {\n \"windows_use_single_network\": true,\n \"type\": - \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n - \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n - \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n - \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": - {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n - \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": - \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": - \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n - \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n - \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": - \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": - \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": - \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n - \ },\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": - \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n - \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: - DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n - \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n - \ app: calico\n template:\n metadata:\n labels:\n tier: node\n - \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - \ nodeSelectorTerms:\n - matchExpressions:\n - - key: kubernetes.io/os\n operator: In\n values:\n - \ - windows\n - key: kubernetes.io/arch\n - \ operator: In\n values:\n - - amd64\n securityContext:\n windowsOptions:\n hostProcess: - true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: - true\n serviceAccountName: calico-node\n tolerations:\n - operator: - Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on - for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n - \ - effect: NoExecute\n operator: Exists\n initContainers:\n # - This container installs the CNI binaries\n # and CNI network config file - on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.25.0-hostprocess\n - \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n - \ imagePullPolicy: Always\n env:\n # Name of the CNI - config file to create.\n - name: CNI_CONF_NAME\n value: - \"10-calico.conflist\"\n # The CNI network config to install on each - node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n - \ name: calico-config-windows\n key: cni_network_config\n - \ # Set the hostname based on the k8s node name.\n - name: - KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: - spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n - \ valueFrom:\n configMapKeyRef:\n name: - calico-config-windows\n key: veth_mtu\n # Prevents - the container from sleeping forever.\n - name: SLEEP\n value: - \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n - \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: - cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: - cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n - \ securityContext:\n windowsOptions:\n hostProcess: - true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n - \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.25.0-hostprocess\n - \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n - \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: - Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: - /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n - \ fieldRef:\n apiVersion: v1\n fieldPath: - metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n - \ apiVersion: v1\n fieldPath: metadata.namespace\n - - name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n - \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n - \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n - \ image: sigwindowstools/calico-node:v3.25.0-hostprocess\n args: - [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: - Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n - \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n - \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n - \ subPath: static-rules.json\n env:\n - name: POD_NAME\n - \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: - metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n - \ apiVersion: v1\n fieldPath: metadata.namespace\n - - name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: - \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n - \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n - \ configMap:\n name: calico-static-rules\n # Used to install - CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n - \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n - \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n---\napiVersion: - apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n - \ group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: - IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: - false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n - \ properties:\n apiVersion:\n description: 'APIVersion - defines the versioned schema of this representation\n of an object. - Servers should convert recognized schemas to the latest\n internal - value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n - \ type: string\n kind:\n description: 'Kind is a - string value representing the REST resource this\n object represents. - Servers may infer this from the endpoint the client\n submits requests - to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n - \ type: string\n metadata:\n type: object\n spec:\n - \ description: IPAMConfigSpec contains the specification for an IPAMConfig\n - \ resource.\n properties:\n autoAllocateBlocks:\n - \ type: boolean\n maxBlocksPerHost:\n description: - MaxBlocksPerHost, if non-zero, is the max number of blocks\n that - can be affine to each host.\n maximum: 2147483647\n minimum: - 0\n type: integer\n strictAffinity:\n type: - boolean\n required:\n - autoAllocateBlocks\n - - strictAffinity\n type: object\n type: object\n served: true\n - \ storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n - \ conditions: []\n storedVersions: []\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default diff --git a/templates/cluster-template-machinepool-windows.yaml b/templates/cluster-template-machinepool-windows.yaml index 22671e03038..236c005d78c 100644 --- a/templates/cluster-template-machinepool-windows.yaml +++ b/templates/cluster-template-machinepool-windows.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico cni-windows: calico csi-proxy: enabled installCloudProviderAzureChart: "true" diff --git a/templates/cluster-template-machinepool.yaml b/templates/cluster-template-machinepool.yaml index 347d22480ff..2a032c5db06 100644 --- a/templates/cluster-template-machinepool.yaml +++ b/templates/cluster-template-machinepool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml index 1320d0a3653..03c6f5fa605 100644 --- a/templates/cluster-template-nvidia-gpu.yaml +++ b/templates/cluster-template-nvidia-gpu.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml index 9d340b00200..5f3efd695d3 100644 --- a/templates/cluster-template-private.yaml +++ b/templates/cluster-template-private.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index bd234c01bd9..9dace46ac78 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico cni-windows: calico csi-proxy: enabled installCloudProviderAzureChart: "true" diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index b4c961cab77..ffad56d8f51 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default diff --git a/templates/flavors/base/cluster-template.yaml b/templates/flavors/base/cluster-template.yaml index 120f4cc27a9..a578e471538 100644 --- a/templates/flavors/base/cluster-template.yaml +++ b/templates/flavors/base/cluster-template.yaml @@ -5,6 +5,7 @@ metadata: name: ${CLUSTER_NAME} labels: installCloudProviderAzureChart: "true" + cni: calico spec: clusterNetwork: pods: diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index a8901abaf58..4919b854b80 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 6a2cc05f1c4..a83377fbdc5 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index d61aa39c228..8b81004d696 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index 00fe8060c9f..3849798bad8 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml index 11ec314986b..c1cb42f510b 100644 --- a/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml index 5810b203bef..c6b04e8809f 100644 --- a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 529476dd626..f8c7fb9cded 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index cbf34ecbf2e..b3e0ba86a00 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 5f43005ffac..fad6af32039 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 1e0f67dd9d4..ec99d9e5ceb 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index eafe10a3f4f..8eb7538238e 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index 4c497b50c73..2a597e952e6 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index b69c3115cec..4f9f588541b 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 6b107815726..f8c8a9fcc84 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -91,6 +91,10 @@ var _ = Describe("Workload cluster creation", func() { } createHelmChartProxyFromFilePath(bootstrapClient, path, namespace.Name) + createHelmChartProxyFromFilePath(bootstrapClient, calicoPath, namespace.Name) + createHelmChartProxyFromFilePath(bootstrapClient, calicoIPv6Path, namespace.Name) + createHelmChartProxyFromFilePath(bootstrapClient, calicoDualStackPath, namespace.Name) + result = new(clusterctl.ApplyClusterTemplateAndWaitResult) spClientSecret := os.Getenv(AzureClientSecret) diff --git a/test/e2e/cni.go b/test/e2e/cni.go index 244a189734c..75a5149c295 100644 --- a/test/e2e/cni.go +++ b/test/e2e/cni.go @@ -41,15 +41,13 @@ const ( kubeadmConfigMapName string = "kubeadm-config" ) -// InstallCalicoHelmChart installs the official calico helm chart +// EnsureCalicoHelmChart installs the official calico helm chart // and validates that expected pods exist and are Ready. -func InstallCalicoHelmChart(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool) { +func EnsureCalicoHelmChart(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool) { specName := "calico-install" - By("Installing Calico via Helm") - values := getCalicoValues(cidrBlocks) + By("Waiting for Calico to be installed via CAAPH") clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName) - InstallHelmChart(ctx, clusterProxy, calicoOperatorNamespace, calicoHelmChartRepoURL, calicoHelmChartName, calicoHelmReleaseName, values) workloadClusterClient := clusterProxy.GetClient() // Copy the kubeadm configmap to the calico-system namespace. This is a workaround needed for the calico-node-windows daemonset to be able to run in the calico-system namespace. diff --git a/test/e2e/common.go b/test/e2e/common.go index 1a8d957d9bd..ee0cec10453 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -275,7 +275,7 @@ func EnsureControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCl }, input.WaitForControlPlaneIntervals...).Should(Succeed(), "API Server was not reachable in time") _, hasWindows := cluster.Labels["cni-windows"] - InstallCalicoHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) + EnsureCalicoHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) controlPlane := discoveryAndWaitForControlPlaneInitialized(ctx, input, result) v, err := semver.ParseTolerant(input.ConfigCluster.KubernetesVersion) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index e84ea38b173..6402fb6de70 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -41,6 +41,9 @@ func init() { flag.StringVar(&configPath, "e2e.config", "", "path to the e2e config file") flag.StringVar(&cloudProviderAzurePath, "e2e.cloud-provider-azure", "", "path to the cloud-provider-azure HelmChartProxy") flag.StringVar(&cloudProviderAzureCIPath, "e2e.cloud-provider-azure-ci", "", "path to the cloud-provider-azure HelmChartProxy with CI artifacts") + flag.StringVar(&calicoPath, "e2e.calico", "", "path to the calico HelmChartProxy") + flag.StringVar(&calicoIPv6Path, "e2e.calico-ipv6", "", "path to the calico HelmChartProxy with IPv6") + flag.StringVar(&calicoDualStackPath, "e2e.calico-dual-stack", "", "path to the calico HelmChartProxy with dual stack") flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored") flag.BoolVar(&useCIArtifacts, "kubetest.use-ci-artifacts", false, "use the latest build from the main branch of the Kubernetes repository. Set KUBERNETES_VERSION environment variable to latest-1.xx to use the build from 1.xx release branch.") flag.BoolVar(&usePRArtifacts, "kubetest.use-pr-artifacts", false, "use the build from a PR of the Kubernetes repository") diff --git a/test/e2e/e2e_suite_vars.go b/test/e2e/e2e_suite_vars.go index 221ff3e513b..f6facc8efd5 100644 --- a/test/e2e/e2e_suite_vars.go +++ b/test/e2e/e2e_suite_vars.go @@ -82,4 +82,13 @@ var ( // cloudProviderAzureCIPath specifies the path to the cloud-provider-azure HelmChartProxy with CI artifacts enabled cloudProviderAzureCIPath string + + // calicoPath specifies the path to the calico HelmChartProxy + calicoPath string + + // calicoIPv6Path specifies the path to the calico HelmChartProxy with IPv6 + calicoIPv6Path string + + // calicoDualStackPath specifies the path to the calico HelmChartProxy with dual stack + calicoDualStackPath string ) From bd05917ffb1319cdc464c37e97323fd7e06be5e7 Mon Sep 17 00:00:00 2001 From: Jont828 Date: Fri, 19 May 2023 17:11:14 -0400 Subject: [PATCH 3/8] Remove unused --- test/e2e/cni.go | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/test/e2e/cni.go b/test/e2e/cni.go index 75a5149c295..bf8db16b06f 100644 --- a/test/e2e/cni.go +++ b/test/e2e/cni.go @@ -21,13 +21,8 @@ package e2e import ( "context" - "fmt" - "path/filepath" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - helmVals "helm.sh/helm/v3/pkg/cli/values" - k8snet "k8s.io/utils/net" "sigs.k8s.io/cluster-api/test/framework/clusterctl" ) @@ -70,38 +65,3 @@ func EnsureCalicoHelmChart(ctx context.Context, input clusterctl.ApplyClusterTem WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...) } } - -func getCalicoValues(cidrBlocks []string) *helmVals.Options { - var ipv6CidrBlock, ipv4CidrBlock string - var values *helmVals.Options - for _, cidr := range cidrBlocks { - if k8snet.IsIPv6CIDRString(cidr) { - ipv6CidrBlock = cidr - } else { - Expect(k8snet.IsIPv4CIDRString(cidr)).To(BeTrue(), "CIDR %s is not a valid IPv4 or IPv6 CIDR", cidr) - ipv4CidrBlock = cidr - } - } - addonsPath := e2eConfig.GetVariable(AddonsPath) - switch { - case ipv6CidrBlock != "" && ipv4CidrBlock != "": - By("Configuring calico CNI helm chart for dual-stack configuration") - values = &helmVals.Options{ - StringValues: []string{fmt.Sprintf("installation.calicoNetwork.ipPools[0].cidr=%s", ipv4CidrBlock), fmt.Sprintf("installation.calicoNetwork.ipPools[1].cidr=%s", ipv6CidrBlock)}, - ValueFiles: []string{filepath.Join(addonsPath, "calico-dual-stack", "values.yaml")}, - } - case ipv6CidrBlock != "": - By("Configuring calico CNI helm chart for IPv6 configuration") - values = &helmVals.Options{ - StringValues: []string{fmt.Sprintf("installation.calicoNetwork.ipPools[0].cidr=%s", ipv6CidrBlock)}, - ValueFiles: []string{filepath.Join(addonsPath, "calico-ipv6", "values.yaml")}, - } - default: - By("Configuring calico CNI helm chart for IPv4 configuration") - values = &helmVals.Options{ - StringValues: []string{fmt.Sprintf("installation.calicoNetwork.ipPools[0].cidr=%s", ipv4CidrBlock)}, - ValueFiles: []string{filepath.Join(addonsPath, "calico", "values.yaml")}, - } - } - return values -} From a794293a53c6c3fc872f4acd49a33a16cc00bfd5 Mon Sep 17 00:00:00 2001 From: Jont828 Date: Wed, 17 May 2023 16:02:48 -0400 Subject: [PATCH 4/8] Install azure disk CSI driver Helm chart with CAAPH --- Makefile | 3 +++ templates/caaph/azuredisk-csi-driver.yaml | 18 ++++++++++++++++++ test/e2e/azure_test.go | 2 ++ test/e2e/cloud-provider-azure.go | 15 ++++----------- test/e2e/common.go | 2 +- test/e2e/e2e_suite_test.go | 1 + test/e2e/e2e_suite_vars.go | 3 +++ 7 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 templates/caaph/azuredisk-csi-driver.yaml diff --git a/Makefile b/Makefile index dbff24a5a75..6cf905b08ab 100644 --- a/Makefile +++ b/Makefile @@ -169,6 +169,8 @@ E2E_CLOUD_PROVIDER_AZURE_CI_PATH ?= $(ROOT_DIR)/templates/caaph/cloud-provider-a E2E_CALICO_PATH ?= $(ROOT_DIR)/templates/caaph/calico.yaml E2E_CALICO_IPV6_PATH ?= $(ROOT_DIR)/templates/caaph/calico-ipv6.yaml E2E_CALICO_DUAL_STACK_PATH ?= $(ROOT_DIR)/templates/caaph/calico-dual-stack.yaml +E2E_CALICO_DUAL_STACK_PATH ?= $(ROOT_DIR)/templates/caaph/calico-dual-stack.yaml +E2E_AZUREDISK_CSI_DRIVER_PATH ?= $(ROOT_DIR)/templates/caaph/azure-disk-csi-driver.yaml SKIP_CLEANUP ?= false SKIP_LOG_COLLECTION ?= false @@ -660,6 +662,7 @@ test-e2e-run: generate-e2e-templates install-tools ## Run e2e tests. -e2e.calico="$(E2E_CALICO_PATH)" \ -e2e.calico-ipv6="$(E2E_CALICO_IPV6_PATH)" \ -e2e.calico-dual-stack="$(E2E_CALICO_DUAL_STACK_PATH)" \ + -e2e.azuredisk-csi-driver="$(E2E_AZUREDISK_CSI_DRIVER_PATH)" \ -e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \ -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS) diff --git a/templates/caaph/azuredisk-csi-driver.yaml b/templates/caaph/azuredisk-csi-driver.yaml new file mode 100644 index 00000000000..59134d6ebc8 --- /dev/null +++ b/templates/caaph/azuredisk-csi-driver.yaml @@ -0,0 +1,18 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart +spec: + clusterSelector: + matchLabels: + installAzureDiskCSIDriverChart: "true" + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + chartName: azuredisk-csi-driver + releaseName: azuredisk-csi-driver-oot + namespace: kube-system + valuesTemplate: | + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} \ No newline at end of file diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index f8c8a9fcc84..57268912f51 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -95,6 +95,8 @@ var _ = Describe("Workload cluster creation", func() { createHelmChartProxyFromFilePath(bootstrapClient, calicoIPv6Path, namespace.Name) createHelmChartProxyFromFilePath(bootstrapClient, calicoDualStackPath, namespace.Name) + createHelmChartProxyFromFilePath(bootstrapClient, azureDiskCSIDriverPath, namespace.Name) + result = new(clusterctl.ApplyClusterTemplateAndWaitResult) spClientSecret := os.Getenv(AzureClientSecret) diff --git a/test/e2e/cloud-provider-azure.go b/test/e2e/cloud-provider-azure.go index 0b1ff2d7b5b..9c60d1b5b1c 100644 --- a/test/e2e/cloud-provider-azure.go +++ b/test/e2e/cloud-provider-azure.go @@ -23,7 +23,6 @@ import ( "context" . "github.com/onsi/ginkgo/v2" - helmVals "helm.sh/helm/v3/pkg/cli/values" "sigs.k8s.io/cluster-api/test/framework/clusterctl" ) @@ -33,19 +32,13 @@ const ( azureDiskCSIDriverHelmReleaseName = "azuredisk-csi-driver-oot" ) -// InstallAzureDiskCSIDriverHelmChart installs the official azure-disk CSI driver helm chart -func InstallAzureDiskCSIDriverHelmChart(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, hasWindows bool) { +// EnsureAzureDiskCSIDriverHelmChart installs the official azure-disk CSI driver helm chart +func EnsureAzureDiskCSIDriverHelmChart(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, hasWindows bool) { specName := "azuredisk-csi-drivers-install" - By("Installing azure-disk CSI driver components via helm") - options := &helmVals.Options{ - Values: []string{"controller.replicas=1", "controller.runOnControlPlane=true"}, - } + By("Waiting for azure-disk CSI driver components via CAAPH") + // TODO: make this always true once HostProcessContainers are on for all supported k8s versions. - if hasWindows { - options.Values = append(options.Values, "windows.useHostProcessContainers=true") - } clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName) - InstallHelmChart(ctx, clusterProxy, kubesystem, azureDiskCSIDriverHelmRepoURL, azureDiskCSIDriverChartName, azureDiskCSIDriverHelmReleaseName, options) By("Waiting for Ready csi-azuredisk-controller deployment pods") for _, d := range []string{"csi-azuredisk-controller"} { waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName) diff --git a/test/e2e/common.go b/test/e2e/common.go index ee0cec10453..527fb8e2731 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -281,7 +281,7 @@ func EnsureControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCl v, err := semver.ParseTolerant(input.ConfigCluster.KubernetesVersion) Expect(err).NotTo(HaveOccurred()) if v.GTE(semver.MustParse("1.23.0")) { - InstallAzureDiskCSIDriverHelmChart(ctx, input, hasWindows) + EnsureAzureDiskCSIDriverHelmChart(ctx, input, hasWindows) } else { Logf("Skipping Azure Disk CSI Driver installation for Kubernetes version %s", input.ConfigCluster.KubernetesVersion) } diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 6402fb6de70..2e2fb33fcc2 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -44,6 +44,7 @@ func init() { flag.StringVar(&calicoPath, "e2e.calico", "", "path to the calico HelmChartProxy") flag.StringVar(&calicoIPv6Path, "e2e.calico-ipv6", "", "path to the calico HelmChartProxy with IPv6") flag.StringVar(&calicoDualStackPath, "e2e.calico-dual-stack", "", "path to the calico HelmChartProxy with dual stack") + flag.StringVar(&azureDiskCSIDriverPath, "e2e.azuredisk-csi-driver", "", "path to the azure disk CSI driver HelmChartProxy") flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored") flag.BoolVar(&useCIArtifacts, "kubetest.use-ci-artifacts", false, "use the latest build from the main branch of the Kubernetes repository. Set KUBERNETES_VERSION environment variable to latest-1.xx to use the build from 1.xx release branch.") flag.BoolVar(&usePRArtifacts, "kubetest.use-pr-artifacts", false, "use the build from a PR of the Kubernetes repository") diff --git a/test/e2e/e2e_suite_vars.go b/test/e2e/e2e_suite_vars.go index f6facc8efd5..f08bb715e13 100644 --- a/test/e2e/e2e_suite_vars.go +++ b/test/e2e/e2e_suite_vars.go @@ -91,4 +91,7 @@ var ( // calicoDualStackPath specifies the path to the calico HelmChartProxy with dual stack calicoDualStackPath string + + // azureDiskCSIDriverPath specifies the path to the azure disk CSI driver HelmChartProxy + azureDiskCSIDriverPath string ) From df028a0203a2d1246d434e88800ed8651ee88239 Mon Sep 17 00:00:00 2001 From: Jont828 Date: Wed, 17 May 2023 17:45:10 -0400 Subject: [PATCH 5/8] Fix makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6cf905b08ab..dbddcbc7ccc 100644 --- a/Makefile +++ b/Makefile @@ -170,7 +170,7 @@ E2E_CALICO_PATH ?= $(ROOT_DIR)/templates/caaph/calico.yaml E2E_CALICO_IPV6_PATH ?= $(ROOT_DIR)/templates/caaph/calico-ipv6.yaml E2E_CALICO_DUAL_STACK_PATH ?= $(ROOT_DIR)/templates/caaph/calico-dual-stack.yaml E2E_CALICO_DUAL_STACK_PATH ?= $(ROOT_DIR)/templates/caaph/calico-dual-stack.yaml -E2E_AZUREDISK_CSI_DRIVER_PATH ?= $(ROOT_DIR)/templates/caaph/azure-disk-csi-driver.yaml +E2E_AZUREDISK_CSI_DRIVER_PATH ?= $(ROOT_DIR)/templates/caaph/azuredisk-csi-driver.yaml SKIP_CLEANUP ?= false SKIP_LOG_COLLECTION ?= false From 8d5b9609ba924b04c385272e9eff66d6727e28c5 Mon Sep 17 00:00:00 2001 From: Jont828 Date: Fri, 19 May 2023 15:18:18 -0400 Subject: [PATCH 6/8] Regenerate --- config/capz/manager_image_patch.yaml | 2 +- config/capz/manager_pull_policy.yaml | 2 +- templates/caaph/azuredisk-csi-driver.yaml | 2 +- templates/caaph/cloud-provider-azure-ci.yaml | 2 +- templates/caaph/cloud-provider-azure.yaml | 2 +- templates/cluster-template-aad.yaml | 3 ++- templates/cluster-template-azure-bastion.yaml | 3 ++- templates/cluster-template-dual-stack.yaml | 3 ++- templates/cluster-template-edgezone.yaml | 3 ++- templates/cluster-template-ephemeral.yaml | 3 ++- templates/cluster-template-flatcar.yaml | 3 ++- templates/cluster-template-ipv6.yaml | 3 ++- templates/cluster-template-machinepool-windows.yaml | 3 ++- templates/cluster-template-machinepool.yaml | 3 ++- templates/cluster-template-nvidia-gpu.yaml | 3 ++- templates/cluster-template-private.yaml | 3 ++- templates/cluster-template-windows.yaml | 3 ++- templates/cluster-template.yaml | 3 ++- templates/flavors/base/cluster-template.yaml | 3 ++- templates/test/ci/cluster-template-prow-ci-version.yaml | 3 ++- templates/test/ci/cluster-template-prow-custom-vnet.yaml | 3 ++- templates/test/ci/cluster-template-prow-dual-stack.yaml | 3 ++- templates/test/ci/cluster-template-prow-edgezone.yaml | 3 ++- templates/test/ci/cluster-template-prow-flatcar.yaml | 3 ++- ...uster-template-prow-intree-cloud-provider-machine-pool.yaml | 3 ++- .../test/ci/cluster-template-prow-intree-cloud-provider.yaml | 3 ++- templates/test/ci/cluster-template-prow-ipv6.yaml | 3 ++- .../test/ci/cluster-template-prow-machine-pool-ci-version.yaml | 3 ++- templates/test/ci/cluster-template-prow-machine-pool-flex.yaml | 3 ++- templates/test/ci/cluster-template-prow-machine-pool.yaml | 3 ++- templates/test/ci/cluster-template-prow-nvidia-gpu.yaml | 3 ++- templates/test/ci/cluster-template-prow-private.yaml | 3 ++- templates/test/ci/cluster-template-prow.yaml | 3 ++- .../test/dev/cluster-template-custom-builds-machine-pool.yaml | 3 ++- templates/test/dev/cluster-template-custom-builds.yaml | 3 ++- 35 files changed, 65 insertions(+), 35 deletions(-) diff --git a/config/capz/manager_image_patch.yaml b/config/capz/manager_image_patch.yaml index 0876a1db40d..c3660d784d2 100644 --- a/config/capz/manager_image_patch.yaml +++ b/config/capz/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller:latest + - image: docker.io/jont828/cluster-api-azure-controller-amd64:20230518222332 name: manager diff --git a/config/capz/manager_pull_policy.yaml b/config/capz/manager_pull_policy.yaml index 74a0879c604..cd7ae12c01e 100644 --- a/config/capz/manager_pull_policy.yaml +++ b/config/capz/manager_pull_policy.yaml @@ -8,4 +8,4 @@ spec: spec: containers: - name: manager - imagePullPolicy: Always + imagePullPolicy: IfNotPresent diff --git a/templates/caaph/azuredisk-csi-driver.yaml b/templates/caaph/azuredisk-csi-driver.yaml index 59134d6ebc8..45af920b988 100644 --- a/templates/caaph/azuredisk-csi-driver.yaml +++ b/templates/caaph/azuredisk-csi-driver.yaml @@ -5,7 +5,7 @@ metadata: spec: clusterSelector: matchLabels: - installAzureDiskCSIDriverChart: "true" + installAzureDiskCSIDriver: "true" repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts chartName: azuredisk-csi-driver releaseName: azuredisk-csi-driver-oot diff --git a/templates/caaph/cloud-provider-azure-ci.yaml b/templates/caaph/cloud-provider-azure-ci.yaml index 6d71cf0155b..dad9cc91feb 100644 --- a/templates/caaph/cloud-provider-azure-ci.yaml +++ b/templates/caaph/cloud-provider-azure-ci.yaml @@ -5,7 +5,7 @@ metadata: spec: clusterSelector: matchLabels: - installCloudProviderAzureChart: "true" + installCloudProviderAzure: "true" repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo chartName: cloud-provider-azure releaseName: cloud-provider-azure diff --git a/templates/caaph/cloud-provider-azure.yaml b/templates/caaph/cloud-provider-azure.yaml index de480814739..4a722054ae8 100644 --- a/templates/caaph/cloud-provider-azure.yaml +++ b/templates/caaph/cloud-provider-azure.yaml @@ -5,7 +5,7 @@ metadata: spec: clusterSelector: matchLabels: - installCloudProviderAzureChart: "true" + installCloudProviderAzure: "true" repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo chartName: cloud-provider-azure releaseName: cloud-provider-azure diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml index 4a90b2b36b6..79442ee877c 100644 --- a/templates/cluster-template-aad.yaml +++ b/templates/cluster-template-aad.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index 296585b8aaf..01ec5dfe79e 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-dual-stack.yaml b/templates/cluster-template-dual-stack.yaml index f1eba4a7af0..aaee67ca1a3 100644 --- a/templates/cluster-template-dual-stack.yaml +++ b/templates/cluster-template-dual-stack.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico-dual-stack - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-edgezone.yaml b/templates/cluster-template-edgezone.yaml index cdf62599644..df5cc850ace 100644 --- a/templates/cluster-template-edgezone.yaml +++ b/templates/cluster-template-edgezone.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index 801b9c88e21..76131952fa0 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-flatcar.yaml b/templates/cluster-template-flatcar.yaml index 71bd485d0c3..8c9cfdb3dbf 100644 --- a/templates/cluster-template-flatcar.yaml +++ b/templates/cluster-template-flatcar.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-ipv6.yaml b/templates/cluster-template-ipv6.yaml index e0d22cbf6df..46104027b7d 100644 --- a/templates/cluster-template-ipv6.yaml +++ b/templates/cluster-template-ipv6.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico-ipv6 - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-machinepool-windows.yaml b/templates/cluster-template-machinepool-windows.yaml index 236c005d78c..204bb6fe060 100644 --- a/templates/cluster-template-machinepool-windows.yaml +++ b/templates/cluster-template-machinepool-windows.yaml @@ -5,7 +5,8 @@ metadata: cni: calico cni-windows: calico csi-proxy: enabled - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template-machinepool.yaml b/templates/cluster-template-machinepool.yaml index 2a032c5db06..81773e8cb8e 100644 --- a/templates/cluster-template-machinepool.yaml +++ b/templates/cluster-template-machinepool.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml index 03c6f5fa605..31153a668d0 100644 --- a/templates/cluster-template-nvidia-gpu.yaml +++ b/templates/cluster-template-nvidia-gpu.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml index 5f3efd695d3..bbc2873dd56 100644 --- a/templates/cluster-template-private.yaml +++ b/templates/cluster-template-private.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index 9dace46ac78..852f5a21a14 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -5,7 +5,8 @@ metadata: cni: calico cni-windows: calico csi-proxy: enabled - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index ffad56d8f51..cad130ba8eb 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/flavors/base/cluster-template.yaml b/templates/flavors/base/cluster-template.yaml index a578e471538..1a7a99b4819 100644 --- a/templates/flavors/base/cluster-template.yaml +++ b/templates/flavors/base/cluster-template.yaml @@ -4,7 +4,8 @@ kind: Cluster metadata: name: ${CLUSTER_NAME} labels: - installCloudProviderAzureChart: "true" + installCloudProviderAzure: "true" + installAzureDiskCSIDriver: "true" cni: calico spec: clusterNetwork: diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 4919b854b80..188c19ccd17 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -6,7 +6,8 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" metrics-server: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index a83377fbdc5..86c9eb64d5a 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index 9c105e52945..fe57ecb3635 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico-dual-stack - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index 8b81004d696..eb4dd9317ef 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index 3849798bad8..354f061fc4f 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml index c1cb42f510b..ce881603cdd 100644 --- a/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml @@ -6,7 +6,8 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml index c6b04e8809f..93c62df2b6e 100644 --- a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml @@ -6,7 +6,8 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index 9c2aeedcf43..99786bd3b80 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico-ipv6 - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index f8c7fb9cded..c213c5f7bab 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -6,7 +6,8 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index b3e0ba86a00..7cee8e5b955 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -6,7 +6,8 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index fad6af32039..88fcb9ccd2d 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -6,7 +6,8 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index ec99d9e5ceb..550c8b1a9f7 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index 13696e7739d..5452f39c7d2 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -3,7 +3,8 @@ kind: Cluster metadata: labels: cni: ${CLUSTER_NAME}-calico - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 8eb7538238e..2e147b9a715 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -6,7 +6,8 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index 2a597e952e6..e3b94e5c593 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -6,7 +6,8 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index 4f9f588541b..310fd83978b 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -6,7 +6,8 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled - installCloudProviderAzureChart: "true" + installAzureDiskCSIDriver: "true" + installCloudProviderAzure: "true" metrics-server: enabled name: ${CLUSTER_NAME} namespace: default From 28ea88d1c5f7219c34533a4b2e0caa8ae64b550d Mon Sep 17 00:00:00 2001 From: Jont828 Date: Fri, 19 May 2023 16:03:59 -0400 Subject: [PATCH 7/8] Fix indentation --- templates/caaph/azuredisk-csi-driver.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/caaph/azuredisk-csi-driver.yaml b/templates/caaph/azuredisk-csi-driver.yaml index 45af920b988..4673a4f447e 100644 --- a/templates/caaph/azuredisk-csi-driver.yaml +++ b/templates/caaph/azuredisk-csi-driver.yaml @@ -13,6 +13,6 @@ spec: valuesTemplate: | controller: replicas: 1 - runOnControlPlane: true + runOnControlPlane: true windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} \ No newline at end of file From 94c9ba0ef0042448735a6e4e8f0d69f3429d4c0e Mon Sep 17 00:00:00 2001 From: Jont828 Date: Tue, 23 May 2023 12:38:13 -0400 Subject: [PATCH 8/8] Remove unused vars --- test/e2e/cloud-provider-azure.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/e2e/cloud-provider-azure.go b/test/e2e/cloud-provider-azure.go index 9c60d1b5b1c..1423566351c 100644 --- a/test/e2e/cloud-provider-azure.go +++ b/test/e2e/cloud-provider-azure.go @@ -26,12 +26,6 @@ import ( "sigs.k8s.io/cluster-api/test/framework/clusterctl" ) -const ( - azureDiskCSIDriverHelmRepoURL = "https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts" - azureDiskCSIDriverChartName = "azuredisk-csi-driver" - azureDiskCSIDriverHelmReleaseName = "azuredisk-csi-driver-oot" -) - // EnsureAzureDiskCSIDriverHelmChart installs the official azure-disk CSI driver helm chart func EnsureAzureDiskCSIDriverHelmChart(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, hasWindows bool) { specName := "azuredisk-csi-drivers-install"