diff --git a/Makefile b/Makefile index d89d12b4892..c66b5aa961e 100644 --- a/Makefile +++ b/Makefile @@ -164,6 +164,9 @@ GINKGO_ARGS ?= ARTIFACTS ?= $(ROOT_DIR)/_artifacts E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml +E2E_CLOUD_PROVIDER_AZURE_PATH ?= $(ROOT_DIR)/templates/caaph/cloud-provider-azure.yaml +E2E_CLOUD_PROVIDER_AZURE_CI_PATH ?= $(ROOT_DIR)/templates/caaph/cloud-provider-azure-ci.yaml + SKIP_CLEANUP ?= false SKIP_LOG_COLLECTION ?= false SKIP_CREATE_MGMT_CLUSTER ?= false @@ -669,6 +672,8 @@ test-e2e-run: generate-e2e-templates install-tools ## Run e2e tests. $(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \ -e2e.artifacts-folder="$(ARTIFACTS)" \ -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ + -e2e.cloud-provider-azure="$(E2E_CLOUD_PROVIDER_AZURE_PATH)" \ + -e2e.cloud-provider-azure-ci="$(E2E_CLOUD_PROVIDER_AZURE_CI_PATH)" \ -e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \ -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS) diff --git a/go.mod b/go.mod index 0abacbf977b..33670a59383 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d github.com/blang/semver v3.5.1+incompatible + github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 github.com/go-logr/logr v1.2.4 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 @@ -47,8 +48,9 @@ require ( k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 sigs.k8s.io/cloud-provider-azure v1.26.7 sigs.k8s.io/cluster-api v1.4.1 + sigs.k8s.io/cluster-api-addon-provider-helm v0.1.0-alpha.5 sigs.k8s.io/cluster-api/test v1.4.1 - sigs.k8s.io/controller-runtime v0.14.5 + sigs.k8s.io/controller-runtime v0.14.6 sigs.k8s.io/kind v0.18.0 ) @@ -91,7 +93,6 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect @@ -221,4 +222,6 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.4.1 +replace sigs.k8s.io/cluster-api => github.com/Jont828/cluster-api v0.4.3-0.20230412015901-405900d8bb69 + +replace sigs.k8s.io/cluster-api/test => github.com/Jont828/cluster-api/test v0.4.3-0.20230412015901-405900d8bb69 diff --git a/go.sum b/go.sum index 35a82eb22cc..3d98acae689 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,10 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/Jont828/cluster-api v0.4.3-0.20230412015901-405900d8bb69 h1:OF4FuySy7PRvHDBezR9KTzgM/lQ2I4L961oh3ByHZt0= +github.com/Jont828/cluster-api v0.4.3-0.20230412015901-405900d8bb69/go.mod h1:3NCXYKcbsAB5gl1CmLaiXiz1yaYIVhQiB48IxwqGErI= +github.com/Jont828/cluster-api/test v0.4.3-0.20230412015901-405900d8bb69 h1:G8yGtS6b8BjdFf+NX8G9SOtFIZuzbbLv4lhpujNyEMI= +github.com/Jont828/cluster-api/test v0.4.3-0.20230412015901-405900d8bb69/go.mod h1:ff2gsEFgyCoglpTVxf1DCR+4itUBJ7MpiaQhZyri2Eg= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -1323,12 +1327,10 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/cloud-provider-azure v1.26.7 h1:LSVRPyxeTQZPOF7z42zNZGzL1S1N5tg28RBZVy7gea4= sigs.k8s.io/cloud-provider-azure v1.26.7/go.mod h1:UIwr0Bk4wQb77wNL9cdT4zZw6DP2AtOQ9EKRt9c5g7Q= -sigs.k8s.io/cluster-api v1.4.1 h1:GtA7OJGhLvgJMgEIxKIoGLxXezM3THI/Yi10QpQ0EN4= -sigs.k8s.io/cluster-api v1.4.1/go.mod h1:IIebZTsqyXU8CHbINV2zuMh0/wykqdr+vEXxQNeteEU= -sigs.k8s.io/cluster-api/test v1.4.1 h1:4ezS3IVGqL+GmsPx4gzo2e4hs2GOrMBsgch+RQPBEWs= -sigs.k8s.io/cluster-api/test v1.4.1/go.mod h1:RHazilXiXNuRYYh/aiX6ZvpUaXOzwNjmaEou/TltSOs= -sigs.k8s.io/controller-runtime v0.14.5 h1:6xaWFqzT5KuAQ9ufgUaj1G/+C4Y1GRkhrxl+BJ9i+5s= -sigs.k8s.io/controller-runtime v0.14.5/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= +sigs.k8s.io/cluster-api-addon-provider-helm v0.1.0-alpha.5 h1:F4phJjl9TrdbuRKIQorQo2jQg3AQTgyXkN+PY5E1So4= +sigs.k8s.io/cluster-api-addon-provider-helm v0.1.0-alpha.5/go.mod h1:s3klPverZSGHRlvJxA35TwtM2+6NeRrcljd3cDsD+Jk= +sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= +sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kind v0.18.0 h1:ahgZdVV1pdhXlYe1f+ztISakT23KdrBl/NFY9JMygzs= diff --git a/internal/test/env/env.go b/internal/test/env/env.go index 3986e70b331..83381f26839 100644 --- a/internal/test/env/env.go +++ b/internal/test/env/env.go @@ -35,6 +35,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/klog/v2" "k8s.io/utils/pointer" + addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" infrav1alpha4 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" @@ -63,6 +64,7 @@ func init() { utilruntime.Must(infrav1exp.AddToScheme(scheme)) utilruntime.Must(infrav1alpha3.AddToScheme(scheme)) utilruntime.Must(infrav1alpha4.AddToScheme(scheme)) + utilruntime.Must(addonsv1alpha1.AddToScheme(scheme)) // Get the root of the current file to use in CRD paths. _, filename, _, _ := goruntime.Caller(0) //nolint:dogsled // Ignore "declaration has 3 blank identifiers" check. diff --git a/main.go b/main.go index ee81c911f1d..9d650eb3d93 100644 --- a/main.go +++ b/main.go @@ -37,6 +37,7 @@ import ( cgrecord "k8s.io/client-go/tools/record" "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" + addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" infrav1alpha4 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" @@ -81,6 +82,7 @@ func init() { _ = expv1alpha4.AddToScheme(scheme) _ = clusterv1.AddToScheme(scheme) _ = expv1.AddToScheme(scheme) + _ = addonsv1alpha1.AddToScheme(scheme) _ = kubeadmv1.AddToScheme(scheme) // +kubebuilder:scaffold:scheme diff --git a/templates/caaph/cloud-provider-azure-ci.yaml b/templates/caaph/cloud-provider-azure-ci.yaml new file mode 100644 index 00000000000..d27de949f59 --- /dev/null +++ b/templates/caaph/cloud-provider-azure-ci.yaml @@ -0,0 +1,24 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci +spec: + clusterSelector: + matchLabels: + installCloudProviderAzureChart: "true" + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + chartName: cloud-provider-azure + releaseName: cloud-provider-azure + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME} + imageRepository: ${CCM_IMAGE_REGISTRY} + imageTag: ${IMAGE_TAG_CCM} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME} + imageRepository: ${CNM_IMAGE_REGISTRY} + imageTag: ${IMAGE_TAG_CCM} + diff --git a/templates/caaph/cloud-provider-azure.yaml b/templates/caaph/cloud-provider-azure.yaml new file mode 100644 index 00000000000..2636de5b39e --- /dev/null +++ b/templates/caaph/cloud-provider-azure.yaml @@ -0,0 +1,16 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart +spec: + clusterSelector: + matchLabels: + installCloudProviderAzureChart: "true" + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + chartName: cloud-provider-azure + releaseName: cloud-provider-azure + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml index 07f530abb2a..4f42a4ae070 100644 --- a/templates/cluster-template-aad.yaml +++ b/templates/cluster-template-aad.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index 98ffb2f540c..1658d2cfe61 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-dual-stack.yaml b/templates/cluster-template-dual-stack.yaml index 1a72e1a0eca..f1eba4a7af0 100644 --- a/templates/cluster-template-dual-stack.yaml +++ b/templates/cluster-template-dual-stack.yaml @@ -3,6 +3,7 @@ kind: Cluster metadata: labels: cni: calico-dual-stack + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-edgezone.yaml b/templates/cluster-template-edgezone.yaml index 54b93177496..c7e8f67671f 100644 --- a/templates/cluster-template-edgezone.yaml +++ b/templates/cluster-template-edgezone.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index 55ec76cf701..196388d4968 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-flatcar.yaml b/templates/cluster-template-flatcar.yaml index 2978263dab6..f270ed1be10 100644 --- a/templates/cluster-template-flatcar.yaml +++ b/templates/cluster-template-flatcar.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-ipv6.yaml b/templates/cluster-template-ipv6.yaml index caa7a12f41b..e0d22cbf6df 100644 --- a/templates/cluster-template-ipv6.yaml +++ b/templates/cluster-template-ipv6.yaml @@ -1,6 +1,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + cni: calico-ipv6 + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-machinepool-flex.yaml b/templates/cluster-template-machinepool-flex.yaml new file mode 100644 index 00000000000..9c13ab8a578 --- /dev/null +++ b/templates/cluster-template-machinepool-flex.yaml @@ -0,0 +1,699 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + windows: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: [] + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + location: ${AZURE_LOCATION} + orchestrationMode: Flexible + strategy: + rollingUpdate: + maxSurge: 0% + maxUnavailable: 0 + type: RollingUpdate + template: + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-mp-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + annotations: + runtime: containerd + windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + location: ${AZURE_LOCATION} + orchestrationMode: Flexible + strategy: + rollingUpdate: + maxSurge: 0% + maxUnavailable: 0 + type: RollingUpdate + template: + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-mp-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + azure-container-registry-config: c:/k/azure.json + cloud-provider: external + pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell c:/create-external-network.ps1 + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico-windows + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: containerd-logger-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + containerd-logger: enabled + resources: + - kind: ConfigMap + name: containerd-logger-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: v1 +data: + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: + v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n + \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: + |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": + [\n {\n \"windows_use_single_network\": true,\n \"type\": + \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n + \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n + \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n + \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": + {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n + \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": + \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": + \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n + \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n + \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": + \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": + \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": + \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n + \ },\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": + \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n + \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: + DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n + \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n + \ app: calico\n template:\n metadata:\n labels:\n tier: node\n + \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n + \ nodeSelectorTerms:\n - matchExpressions:\n - + key: kubernetes.io/os\n operator: In\n values:\n + \ - windows\n - key: kubernetes.io/arch\n + \ operator: In\n values:\n - + amd64\n securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: + true\n serviceAccountName: calico-node\n tolerations:\n - operator: + Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on + for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n + \ - effect: NoExecute\n operator: Exists\n initContainers:\n # + This container installs the CNI binaries\n # and CNI network config file + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.25.0-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n + \ imagePullPolicy: Always\n env:\n # Name of the CNI + config file to create.\n - name: CNI_CONF_NAME\n value: + \"10-calico.conflist\"\n # The CNI network config to install on each + node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n + \ name: calico-config-windows\n key: cni_network_config\n + \ # Set the hostname based on the k8s node name.\n - name: + KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: + spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n + \ valueFrom:\n configMapKeyRef:\n name: + calico-config-windows\n key: veth_mtu\n # Prevents + the container from sleeping forever.\n - name: SLEEP\n value: + \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n + \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: + cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: + cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n + \ securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.25.0-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n + \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: + Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: + /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n + \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n + \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n + \ image: sigwindowstools/calico-node:v3.25.0-hostprocess\n args: + [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: + Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n + \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n + \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n + \ subPath: static-rules.json\n env:\n - name: POD_NAME\n + \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: + \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n + \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n + \ configMap:\n name: calico-static-rules\n # Used to install + CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n + \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n + \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n---\napiVersion: + apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n + \ group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: + IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: + false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n + \ properties:\n apiVersion:\n description: 'APIVersion + defines the versioned schema of this representation\n of an object. + Servers should convert recognized schemas to the latest\n internal + value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n + \ type: string\n kind:\n description: 'Kind is a + string value representing the REST resource this\n object represents. + Servers may infer this from the endpoint the client\n submits requests + to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n + \ type: string\n metadata:\n type: object\n spec:\n + \ description: IPAMConfigSpec contains the specification for an IPAMConfig\n + \ resource.\n properties:\n autoAllocateBlocks:\n + \ type: boolean\n maxBlocksPerHost:\n description: + MaxBlocksPerHost, if non-zero, is the max number of blocks\n that + can be affine to each host.\n maximum: 2147483647\n minimum: + 0\n type: integer\n strictAffinity:\n type: + boolean\n required:\n - autoAllocateBlocks\n - + strictAffinity\n type: object\n type: object\n served: true\n + \ storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n + \ conditions: []\n storedVersions: []\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default +--- +apiVersion: v1 +data: + containerd-windows-logger: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: containerd-logger + name: containerd-logger + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: containerd-logger + template: + metadata: + labels: + k8s-app: containerd-logger + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 + args: [ "config.json" ] + name: containerd-logger + imagePullPolicy: Always + volumeMounts: + - name: containerd-logger-config + mountPath: /config.json + subPath: config.json + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: containerd-logger-config + name: containerd-logger-config + updateStrategy: + type: RollingUpdate + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: containerd-logger-config + namespace: kube-system + data: + config.json: | + { + "inputs": [ + { + "type": "ETW", + "sessionNamePrefix": "containerd", + "cleanupOldSessions": true, + "reuseExistingSession": true, + "providers": [ + { + "providerName": "Microsoft.Virtualization.RunHCS", + "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", + "level": "Verbose" + }, + { + "providerName": "ContainerD", + "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", + "level": "Verbose" + } + ] + } + ], + "filters": [ + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" + } + ], + "outputs": [ + { + "type": "StdOutput" + } + ], + "schemaVersion": "2016-08-11" + } +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: containerd-logger-${CLUSTER_NAME} + namespace: default diff --git a/templates/cluster-template-machinepool-windows.yaml b/templates/cluster-template-machinepool-windows.yaml index d691f0bdf97..22671e03038 100644 --- a/templates/cluster-template-machinepool-windows.yaml +++ b/templates/cluster-template-machinepool-windows.yaml @@ -4,6 +4,7 @@ metadata: labels: cni-windows: calico csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template-machinepool.yaml b/templates/cluster-template-machinepool.yaml index b4f9bd97f1e..347d22480ff 100644 --- a/templates/cluster-template-machinepool.yaml +++ b/templates/cluster-template-machinepool.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml index bb758313b47..1320d0a3653 100644 --- a/templates/cluster-template-nvidia-gpu.yaml +++ b/templates/cluster-template-nvidia-gpu.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml index 252b03eab1a..9d340b00200 100644 --- a/templates/cluster-template-private.yaml +++ b/templates/cluster-template-private.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index c84d2a1b6af..bd234c01bd9 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -4,6 +4,7 @@ metadata: labels: cni-windows: calico csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 270dd7b3758..b4c961cab77 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/flavors/base/cluster-template.yaml b/templates/flavors/base/cluster-template.yaml index 95f645b7153..120f4cc27a9 100644 --- a/templates/flavors/base/cluster-template.yaml +++ b/templates/flavors/base/cluster-template.yaml @@ -3,6 +3,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: name: ${CLUSTER_NAME} + labels: + installCloudProviderAzureChart: "true" spec: clusterNetwork: pods: diff --git a/templates/flavors/ipv6/patches/ipv6.yaml b/templates/flavors/ipv6/patches/ipv6.yaml index ea0eff7286d..1731bfa559a 100644 --- a/templates/flavors/ipv6/patches/ipv6.yaml +++ b/templates/flavors/ipv6/patches/ipv6.yaml @@ -3,6 +3,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: name: ${CLUSTER_NAME} + labels: + cni: "calico-ipv6" spec: clusterNetwork: pods: diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml new file mode 100644 index 00000000000..e422c99539d --- /dev/null +++ b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml @@ -0,0 +1,1177 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + enabledCloudProviderAzureChart: true + metrics-server: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + natGateway: + name: node-natgateway + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + feature-gates: ${K8S_FEATURE_GATES:-""} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + feature-gates: HPAContainerMetrics=true + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + kubernetesVersion: ci/${CI_VERSION} + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + wget "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true --short=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" + owner: root:root + path: /tmp/kubeadm-bootstrap.sh + permissions: "0744" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: + - bash -c /tmp/kubeadm-bootstrap.sh + verbosity: 5 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-1804-gen1 + version: latest + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: {} + template: + metadata: + labels: + nodepool: pool1 + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + image: + marketplace: + offer: capi + publisher: cncf-upstream + sku: ubuntu-1804-gen1 + version: latest + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + wget "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + IMAGE_REGISTRY_PREFIX=registry.k8s.io + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true --short=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" + owner: root:root + path: /tmp/kubeadm-bootstrap.sh + permissions: "0744" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - bash -c /tmp/kubeadm-bootstrap.sh + verbosity: 5 +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} + selector: {} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + annotations: + runtime: containerd + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + metadata: + annotations: + runtime: containerd + windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} + spec: + image: + marketplace: + offer: capi-windows + publisher: cncf-upstream + sku: windows-2022-containerd-gen1 + version: latest + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: |- + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + - content: | + # /tmp is assumed created and required for upstream e2e tests to pass + New-Item -ItemType Directory -Force -Path C:\tmp\ + path: C:/create-temp-folder.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" + if($$CONTAINERD_URL -ne ""){ + # Kubelet service depends on contianerd service so make a best effort attempt to stop it + Stop-Service kubelet -Force -ErrorAction SilentlyContinue + Stop-Service containerd -Force + echo "downloading containerd: $$CONTAINERD_URL" + curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" + tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 + + Start-Service containerd + } + + containerd.exe --version + containerd-shim-runhcs-v1.exe --version + path: C:/replace-containerd.ps1 + permissions: "0744" + - content: | + mkdir -Force c:/localdumps + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f + reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f + # Enable sftp so we can copy crash dump files during log collection of stfp + $sshd_config = "$env:ProgramData\ssh\sshd_config" + if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } + Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" + sc.exe stop sshd + sc.exe start sshd + path: C:/collect-hns-crashes.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + Stop-Service kubelet -Force + + $$CI_VERSION="${CI_VERSION}" + if($$CI_VERSION -ne "") + { + $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") + $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" + foreach ( $$binary in $$binaries ) + { + echo "downloading binary: $$ci_url/$$binary.exe" + curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" + } + } + + # Tag it to the ci version. The image knows how to use the copy locally with the configmap + # that is applied at at this stage (windows-kubeproxy-ci.yaml) + ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess + ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" + + kubeadm.exe version -o=short + kubectl.exe version --client=true --short=true + kubelet.exe --version + kube-proxy.exe --version + path: C:/replace-ci-binaries.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + azure-container-registry-config: c:/k/azure.json + cloud-provider: external + v: "2" + windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell C:/create-temp-folder.ps1 + - powershell C:/replace-containerd.ps1 + - powershell C:/collect-hns-crashes.ps1 + - powershell C:/replace-ci-binaries.ps1 + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: ${CLUSTER_NAME}-mhc-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + maxUnhealthy: 100% + selector: + matchLabels: + nodepool: pool1 + unhealthyConditions: + - status: "True" + timeout: 30s + type: E2ENodeUnhealthy +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico-windows + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: containerd-logger-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + containerd-logger: enabled + resources: + - kind: ConfigMap + name: containerd-logger-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: v1 +data: + kube-proxy-patch: |- + apiVersion: v1 + kind: ConfigMap + metadata: + name: windows-kubeproxy-ci + namespace: kube-system + data: + KUBEPROXY_PATH: "c:/k/kube-proxy.exe" + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: + v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n + \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: + |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": + [\n {\n \"windows_use_single_network\": true,\n \"type\": + \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n + \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n + \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n + \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": + {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n + \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": + \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": + \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n + \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n + \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": + \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": + \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": + \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n + \ },\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": + \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n + \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: + DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n + \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n + \ app: calico\n template:\n metadata:\n labels:\n tier: node\n + \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n + \ nodeSelectorTerms:\n - matchExpressions:\n - + key: kubernetes.io/os\n operator: In\n values:\n + \ - windows\n - key: kubernetes.io/arch\n + \ operator: In\n values:\n - + amd64\n securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: + true\n serviceAccountName: calico-node\n tolerations:\n - operator: + Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on + for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n + \ - effect: NoExecute\n operator: Exists\n initContainers:\n # + This container installs the CNI binaries\n # and CNI network config file + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.25.0-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n + \ imagePullPolicy: Always\n env:\n # Name of the CNI + config file to create.\n - name: CNI_CONF_NAME\n value: + \"10-calico.conflist\"\n # The CNI network config to install on each + node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n + \ name: calico-config-windows\n key: cni_network_config\n + \ # Set the hostname based on the k8s node name.\n - name: + KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: + spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n + \ valueFrom:\n configMapKeyRef:\n name: + calico-config-windows\n key: veth_mtu\n # Prevents + the container from sleeping forever.\n - name: SLEEP\n value: + \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n + \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: + cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: + cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n + \ securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.25.0-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n + \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: + Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: + /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n + \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n + \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n + \ image: sigwindowstools/calico-node:v3.25.0-hostprocess\n args: + [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: + Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n + \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n + \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n + \ subPath: static-rules.json\n env:\n - name: POD_NAME\n + \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: + \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n + \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n + \ configMap:\n name: calico-static-rules\n # Used to install + CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n + \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n + \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n---\napiVersion: + apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n + \ group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: + IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: + false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n + \ properties:\n apiVersion:\n description: 'APIVersion + defines the versioned schema of this representation\n of an object. + Servers should convert recognized schemas to the latest\n internal + value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n + \ type: string\n kind:\n description: 'Kind is a + string value representing the REST resource this\n object represents. + Servers may infer this from the endpoint the client\n submits requests + to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n + \ type: string\n metadata:\n type: object\n spec:\n + \ description: IPAMConfigSpec contains the specification for an IPAMConfig\n + \ resource.\n properties:\n autoAllocateBlocks:\n + \ type: boolean\n maxBlocksPerHost:\n description: + MaxBlocksPerHost, if non-zero, is the max number of blocks\n that + can be affine to each host.\n maximum: 2147483647\n minimum: + 0\n type: integer\n strictAffinity:\n type: + boolean\n required:\n - autoAllocateBlocks\n - + strictAffinity\n type: object\n type: object\n served: true\n + \ storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n + \ conditions: []\n storedVersions: []\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default +--- +apiVersion: v1 +data: + containerd-windows-logger: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: containerd-logger + name: containerd-logger + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: containerd-logger + template: + metadata: + labels: + k8s-app: containerd-logger + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 + args: [ "config.json" ] + name: containerd-logger + imagePullPolicy: Always + volumeMounts: + - name: containerd-logger-config + mountPath: /config.json + subPath: config.json + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: containerd-logger-config + name: containerd-logger-config + updateStrategy: + type: RollingUpdate + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: containerd-logger-config + namespace: kube-system + data: + config.json: | + { + "inputs": [ + { + "type": "ETW", + "sessionNamePrefix": "containerd", + "cleanupOldSessions": true, + "reuseExistingSession": true, + "providers": [ + { + "providerName": "Microsoft.Virtualization.RunHCS", + "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", + "level": "Verbose" + }, + { + "providerName": "ContainerD", + "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", + "level": "Verbose" + } + ] + } + ], + "filters": [ + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" + } + ], + "outputs": [ + { + "type": "StdOutput" + } + ], + "schemaVersion": "2016-08-11" + } +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: containerd-logger-${CLUSTER_NAME} + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: metrics-server-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + metrics-server: enabled + resources: + - kind: ConfigMap + name: metrics-server-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: v1 +data: + metrics-server: | + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader + rules: + - apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + rules: + - apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get + - apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: metrics-server:system:auth-delegator + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + k8s-app: metrics-server + name: system:metrics-server + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server + subjects: + - kind: ServiceAccount + name: metrics-server + namespace: kube-system + --- + apiVersion: v1 + kind: Service + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 + template: + metadata: + labels: + k8s-app: metrics-server + spec: + containers: + - args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --kubelet-insecure-tls + image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - emptyDir: {} + name: tmp-dir + --- + apiVersion: apiregistration.k8s.io/v1 + kind: APIService + metadata: + labels: + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io + spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: metrics-server-${CLUSTER_NAME} + namespace: default diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 65b5c425c8d..742d2cab8b6 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" metrics-server: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 877f8c4521e..6a2cc05f1c4 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index 809444c80a3..a73f13ec783 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -3,6 +3,7 @@ kind: Cluster metadata: labels: cni: calico-dual-stack + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index c15c46bab89..d61aa39c228 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index 38a0c5ddf31..00fe8060c9f 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml index db32250a6c9..11ec314986b 100644 --- a/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml index 9f662623fd8..5810b203bef 100644 --- a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index 8a82e1f972e..d3dc98a871f 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -1,6 +1,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + cni: calico-ipv6 + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index c113bae1eb1..12b6ee2dffb 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index 9c13ab8a578..cbf34ecbf2e 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 3424c731f5f..5f43005ffac 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 939e2427765..1e0f67dd9d4 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index 6c443b81688..13696e7739d 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -3,6 +3,7 @@ kind: Cluster metadata: labels: cni: ${CLUSTER_NAME}-calico + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index b03005e8edf..eafe10a3f4f 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index 1b42982b1f7..8e4804dd53f 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" windows: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index 574e7c2dd2b..6c6f1bf03fa 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -5,6 +5,7 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled + installCloudProviderAzureChart: "true" metrics-server: enabled name: ${CLUSTER_NAME} namespace: default diff --git a/test/e2e/azure_clusterproxy.go b/test/e2e/azure_clusterproxy.go index ec17cd5b49f..2af4a415a25 100644 --- a/test/e2e/azure_clusterproxy.go +++ b/test/e2e/azure_clusterproxy.go @@ -44,6 +44,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kubectl/pkg/describe" "k8s.io/utils/pointer" + addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" @@ -73,6 +74,7 @@ func initScheme() *runtime.Scheme { Expect(infrav1.AddToScheme(scheme)).To(Succeed()) Expect(infrav1exp.AddToScheme(scheme)).To(Succeed()) Expect(expv1.AddToScheme(scheme)).To(Succeed()) + Expect(addonsv1alpha1.AddToScheme(scheme)).To(Succeed()) // Add aadpodidentity v1 to the scheme. aadPodIdentityGroupVersion := schema.GroupVersion{Group: aadpodv1.GroupName, Version: "v1"} scheme.AddKnownTypes(aadPodIdentityGroupVersion, diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 8dcc8c7d6d7..fd22f5204fd 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -25,16 +25,20 @@ import ( "os" "time" + "github.com/drone/envsubst/v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/yaml" + addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/controller-runtime/pkg/client" ) var _ = Describe("Workload cluster creation", func() { @@ -76,6 +80,17 @@ var _ = Describe("Workload cluster creation", func() { namespace, cancelWatches, err = setupSpecNamespace(ctx, clusterNamePrefix, bootstrapClusterProxy, artifactFolder) Expect(err).NotTo(HaveOccurred()) + By("Initialize bootstrap client to install add-ons") + bootstrapClient := bootstrapClusterProxy.GetClient() + Expect(bootstrapClient).NotTo(BeNil()) + + By("Creating cloud-provider-azure HelmChartProxy") + path := cloudProviderAzurePath + if useCIArtifacts { + path = cloudProviderAzureCIPath + } + createHelmChartProxyFromFilePath(bootstrapClient, path, namespace.Name) + result = new(clusterctl.ApplyClusterTemplateAndWaitResult) spClientSecret := os.Getenv(AzureClientSecret) @@ -904,3 +919,27 @@ var _ = Describe("Workload cluster creation", func() { }) }) }) + +func createHelmChartProxyFromFilePath(client client.Client, path string, namespace string) { + content, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + + result, err := envsubst.EvalEnv(string(content)) + Expect(err).NotTo(HaveOccurred()) + content = []byte(result) + + Logf("HelmChartProxy content:\n %s", string(content)) + + var helmChartProxy addonsv1alpha1.HelmChartProxy + err = yaml.Unmarshal(content, &helmChartProxy) + Expect(err).NotTo(HaveOccurred()) + + helmChartProxy.Namespace = namespace + + Logf("HelmChartProxy object is %+v", helmChartProxy) + + if err := client.Create(context.Background(), &helmChartProxy); err != nil { + Logf("Failed to create HelmChartProxy: %v", err) + Expect(err).NotTo(HaveOccurred()) + } +} diff --git a/test/e2e/cloud-provider-azure.go b/test/e2e/cloud-provider-azure.go index 809c304ab1e..0b1ff2d7b5b 100644 --- a/test/e2e/cloud-provider-azure.go +++ b/test/e2e/cloud-provider-azure.go @@ -21,9 +21,6 @@ package e2e import ( "context" - "fmt" - "os" - "strings" . "github.com/onsi/ginkgo/v2" helmVals "helm.sh/helm/v3/pkg/cli/values" @@ -31,49 +28,11 @@ import ( ) const ( - cloudProviderAzureHelmRepoURL = "https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo" - cloudProviderAzureChartName = "cloud-provider-azure" - cloudProviderAzureHelmReleaseName = "cloud-provider-azure-oot" azureDiskCSIDriverHelmRepoURL = "https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts" azureDiskCSIDriverChartName = "azuredisk-csi-driver" azureDiskCSIDriverHelmReleaseName = "azuredisk-csi-driver-oot" ) -// InstallCalicoAndCloudProviderAzureHelmChart installs the official cloud-provider-azure helm chart -// and validates that expected pods exist and are Ready. -func InstallCalicoAndCloudProviderAzureHelmChart(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool) { - specName := "cloud-provider-azure-install" - By("Installing cloud-provider-azure components via helm") - options := &helmVals.Options{ - Values: []string{ - fmt.Sprintf("infra.clusterName=%s", input.ConfigCluster.ClusterName), - "cloudControllerManager.logVerbosity=4", - }, - StringValues: []string{fmt.Sprintf("cloudControllerManager.clusterCIDR=%s", strings.Join(cidrBlocks, `\,`))}, - } - // If testing a CI version of Kubernetes, use CCM and CNM images built from source. - if useCIArtifacts { - options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageName=%s", os.Getenv("CCM_IMAGE_NAME"))) - options.Values = append(options.Values, fmt.Sprintf("cloudNodeManager.imageName=%s", os.Getenv("CNM_IMAGE_NAME"))) - options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageRepository=%s", os.Getenv("IMAGE_REGISTRY"))) - options.Values = append(options.Values, fmt.Sprintf("cloudNodeManager.imageRepository=%s", os.Getenv("IMAGE_REGISTRY"))) - options.StringValues = append(options.StringValues, fmt.Sprintf("cloudControllerManager.imageTag=%s", os.Getenv("IMAGE_TAG_CCM"))) - options.StringValues = append(options.StringValues, fmt.Sprintf("cloudNodeManager.imageTag=%s", os.Getenv("IMAGE_TAG_CNM"))) - } - - clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName) - InstallHelmChart(ctx, clusterProxy, defaultNamespace, cloudProviderAzureHelmRepoURL, cloudProviderAzureChartName, cloudProviderAzureHelmReleaseName, options) - - // Install Calico CNI Helm Chart. We do this before waiting for the pods to be ready because there is a co-dependency between CNI (nodes ready) and cloud-provider being initialized. - InstallCalicoHelmChart(ctx, input, cidrBlocks, hasWindows) - - By("Waiting for Ready cloud-controller-manager deployment pods") - for _, d := range []string{"cloud-controller-manager"} { - waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName) - WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...) - } -} - // InstallAzureDiskCSIDriverHelmChart installs the official azure-disk CSI driver helm chart func InstallAzureDiskCSIDriverHelmChart(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, hasWindows bool) { specName := "azuredisk-csi-drivers-install" diff --git a/test/e2e/cni.go b/test/e2e/cni.go index 4cc41302588..244a189734c 100644 --- a/test/e2e/cni.go +++ b/test/e2e/cni.go @@ -46,7 +46,7 @@ const ( func InstallCalicoHelmChart(ctx context.Context, input clusterctl.ApplyClusterTemplateAndWaitInput, cidrBlocks []string, hasWindows bool) { specName := "calico-install" - By("Installing Calico CNI via helm") + By("Installing Calico via Helm") values := getCalicoValues(cidrBlocks) clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName) InstallHelmChart(ctx, clusterProxy, calicoOperatorNamespace, calicoHelmChartRepoURL, calicoHelmChartName, calicoHelmReleaseName, values) diff --git a/test/e2e/common.go b/test/e2e/common.go index fe9fb874fbc..9aaadcea74d 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -274,12 +274,8 @@ func EnsureControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCl }, input.WaitForControlPlaneIntervals...).Should(Succeed(), "API Server was not reachable in time") _, hasWindows := cluster.Labels["cni-windows"] - if kubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.ControllerManager.ExtraArgs["cloud-provider"] != "azure" { - // There is a co-dependency between cloud-provider and CNI so we install both together if cloud-provider is external. - InstallCalicoAndCloudProviderAzureHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) - } else { - InstallCalicoHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) - } + InstallCalicoHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) + controlPlane := discoveryAndWaitForControlPlaneInitialized(ctx, input, result) v, err := semver.ParseTolerant(input.ConfigCluster.KubernetesVersion) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 8039f8b630c..d71edee469c 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -9,6 +9,8 @@ images: loadBehavior: tryLoad - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.4.1 loadBehavior: tryLoad + - name: gcr.io/k8s-staging-cluster-api-helm/cluster-api-helm-controller:v0.1.0-alpha.5 + loadBehavior: tryLoad providers: - name: cluster-api @@ -140,6 +142,21 @@ providers: - old: "--v=0" new: "--v=2" + - name: helm + type: AddonProvider + versions: + - name: v0.1.0-alpha.5 + value: https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/releases/download/v0.1.0-alpha.5/add-on-components.yaml + type: url + contract: v1beta1 + files: + - sourcePath: "../data/shared/v1beta1_addon_provider/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "image: registry.k8s.io/cluster-api-helm/cluster-api-helm-controller:v0.1.0-alpha.5" + new: "image: gcr.io/k8s-staging-cluster-api-helm/cluster-api-helm-controller:v0.1.0-alpha.5" + variables: AKS_KUBERNETES_VERSION: "latest" AKS_KUBERNETES_VERSION_UPGRADE_FROM: "latest-1" diff --git a/test/e2e/data/shared/v1beta1_addon_provider/metadata.yaml b/test/e2e/data/shared/v1beta1_addon_provider/metadata.yaml new file mode 100644 index 00000000000..a087e998608 --- /dev/null +++ b/test/e2e/data/shared/v1beta1_addon_provider/metadata.yaml @@ -0,0 +1,5 @@ +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +releaseSeries: + - major: 0 + minor: 1 + contract: v1beta1 diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 51e61ed48ba..e84ea38b173 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -39,6 +39,8 @@ import ( func init() { flag.StringVar(&configPath, "e2e.config", "", "path to the e2e config file") + flag.StringVar(&cloudProviderAzurePath, "e2e.cloud-provider-azure", "", "path to the cloud-provider-azure HelmChartProxy") + flag.StringVar(&cloudProviderAzureCIPath, "e2e.cloud-provider-azure-ci", "", "path to the cloud-provider-azure HelmChartProxy with CI artifacts") flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored") flag.BoolVar(&useCIArtifacts, "kubetest.use-ci-artifacts", false, "use the latest build from the main branch of the Kubernetes repository. Set KUBERNETES_VERSION environment variable to latest-1.xx to use the build from 1.xx release branch.") flag.BoolVar(&usePRArtifacts, "kubetest.use-pr-artifacts", false, "use the build from a PR of the Kubernetes repository") @@ -74,6 +76,8 @@ var _ = SynchronizedBeforeSuite(func() []byte { By("Initializing the bootstrap cluster") initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) + // TODO: set this helm stuff up + // encode the e2e config into the byte array. var configBuf bytes.Buffer enc := gob.NewEncoder(&configBuf) diff --git a/test/e2e/e2e_suite_vars.go b/test/e2e/e2e_suite_vars.go index c39ce2376b1..221ff3e513b 100644 --- a/test/e2e/e2e_suite_vars.go +++ b/test/e2e/e2e_suite_vars.go @@ -76,4 +76,10 @@ var ( // usePRArtifacts specifies whether or not to use the build from a PR of the Kubernetes repository usePRArtifacts bool + + // cloudProviderAzurePath specifies the path to the cloud-provider-azure HelmChartProxy + cloudProviderAzurePath string + + // cloudProviderAzureCIPath specifies the path to the cloud-provider-azure HelmChartProxy with CI artifacts enabled + cloudProviderAzureCIPath string )