From 88f288c2fe1ad72ed0bccc1efca727349814d4b2 Mon Sep 17 00:00:00 2001 From: Jont828 Date: Mon, 23 Oct 2023 17:13:15 -0400 Subject: [PATCH] Install azure disk CSI driver Helm chart with CAAPH --- hack/create-dev-cluster.sh | 2 +- .../azuredisk-csi-driver.yaml | 18 + .../cluster-template-prow-azure-cni-v1.yaml | 22 + ...r-template-prow-ci-version-dual-stack.yaml | 21 + ...cluster-template-prow-ci-version-ipv6.yaml | 21 + ...ow-ci-version-windows-containerd-2022.yaml | 1179 ----------------- .../ci/cluster-template-prow-ci-version.yaml | 21 + .../ci/cluster-template-prow-custom-vnet.yaml | 21 + .../ci/cluster-template-prow-dual-stack.yaml | 21 + .../ci/cluster-template-prow-edgezone.yaml | 21 + .../ci/cluster-template-prow-flatcar.yaml | 21 + ...ow-intree-cloud-provider-machine-pool.yaml | 21 + ...r-template-prow-intree-cloud-provider.yaml | 21 + .../test/ci/cluster-template-prow-ipv6.yaml | 21 + ...template-prow-machine-pool-ci-version.yaml | 21 + ...uster-template-prow-machine-pool-flex.yaml | 21 + .../cluster-template-prow-machine-pool.yaml | 21 + .../ci/cluster-template-prow-nvidia-gpu.yaml | 21 + .../ci/cluster-template-prow-private.yaml | 21 + .../ci/cluster-template-prow-topology.yaml | 21 + ...uster-template-prow-workload-identity.yaml | 21 + templates/test/ci/cluster-template-prow.yaml | 21 + .../cluster-label-azuredisk-csi-driver.yaml | 6 + .../ci/prow-azure-cni-v1/kustomization.yaml | 2 + .../ci/prow-custom-vnet/kustomization.yaml | 4 +- .../ci/prow-dual-stack/kustomization.yaml | 2 + .../test/ci/prow-edgezone/kustomization.yaml | 4 +- .../test/ci/prow-flatcar/kustomization.yaml | 2 + .../test/ci/prow-ipv6/kustomization.yaml | 3 + .../ci/prow-machine-pool/kustomization.yaml | 2 + .../ci/prow-nvidia-gpu/kustomization.yaml | 2 + .../test/ci/prow-private/kustomization.yaml | 2 + .../test/ci/prow-topology/kustomization.yaml | 2 + .../prow-workload-identity/kustomization.yaml | 2 + templates/test/ci/prow/kustomization.yaml | 2 + ...r-template-custom-builds-machine-pool.yaml | 21 + .../dev/cluster-template-custom-builds.yaml | 21 + test/e2e/cloud-provider-azure.go | 30 +- test/e2e/common.go | 2 +- test/e2e/config/azure-dev.yaml | 5 +- .../v1beta1/bases/azuredisk-csi-driver.yaml | 18 + .../cluster-template-kcp-remediation.yaml | 20 + .../kustomization.yaml | 2 + .../cluster-template-kcp-scale-in.yaml | 20 + .../kustomization.yaml | 2 + .../cluster-template-machine-pool.yaml | 20 + .../kustomization.yaml | 2 + .../cluster-template-md-remediation.yaml | 20 + .../kustomization.yaml | 2 + .../v1beta1/cluster-template-node-drain.yaml | 20 + .../kustomization.yaml | 2 + .../v1beta1/cluster-template.yaml | 20 + .../cluster-template/kustomization.yaml | 2 + .../cluster-label-azuredisk-csi-driver.yaml | 6 + 54 files changed, 673 insertions(+), 1196 deletions(-) create mode 100644 templates/addons/cluster-api-helm/azuredisk-csi-driver.yaml delete mode 100644 templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml create mode 100644 templates/test/ci/patches/cluster-label-azuredisk-csi-driver.yaml create mode 100644 test/e2e/data/infrastructure-azure/v1beta1/bases/azuredisk-csi-driver.yaml create mode 100644 test/e2e/data/infrastructure-azure/v1beta1/patches/cluster-label-azuredisk-csi-driver.yaml diff --git a/hack/create-dev-cluster.sh b/hack/create-dev-cluster.sh index eb52a266bc8f..328c515e90c3 100755 --- a/hack/create-dev-cluster.sh +++ b/hack/create-dev-cluster.sh @@ -48,7 +48,7 @@ export CONTROL_PLANE_MACHINE_COUNT=${CONTROL_PLANE_MACHINE_COUNT:-3} export AZURE_CONTROL_PLANE_MACHINE_TYPE="${CONTROL_PLANE_MACHINE_TYPE:-Standard_B2s}" export AZURE_NODE_MACHINE_TYPE="${NODE_MACHINE_TYPE:-Standard_B2s}" export WORKER_MACHINE_COUNT=${WORKER_MACHINE_COUNT:-2} -export KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.25.6}" +export KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.26.6}" export CLUSTER_TEMPLATE="${CLUSTER_TEMPLATE:-cluster-template.yaml}" # identity secret settings. diff --git a/templates/addons/cluster-api-helm/azuredisk-csi-driver.yaml b/templates/addons/cluster-api-helm/azuredisk-csi-driver.yaml new file mode 100644 index 000000000000..07afde0faf6b --- /dev/null +++ b/templates/addons/cluster-api-helm/azuredisk-csi-driver.yaml @@ -0,0 +1,18 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart +spec: + clusterSelector: + matchLabels: + azuredisk-csi: "true" + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + chartName: azuredisk-csi-driver + releaseName: azuredisk-csi-driver-oot + namespace: kube-system + valuesTemplate: | + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} \ No newline at end of file diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index b6f02f0153cf..a6b97a786c5b 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + azuredisk-csi: "true" name: ${CLUSTER_NAME} namespace: default spec: @@ -220,3 +222,23 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index 9f5693d2fbc3..01dd67922e42 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico-dual-stack cni-windows: ${CLUSTER_NAME}-calico containerd-logger: disabled @@ -452,6 +453,26 @@ spec: name: containerd-logger-${CLUSTER_NAME} strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- apiVersion: v1 data: csi-proxy: | diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index 8fe0fd46532e..275e8187721c 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico-ipv6 cni-windows: ${CLUSTER_NAME}-calico containerd-logger: disabled @@ -470,6 +471,26 @@ spec: name: containerd-logger-${CLUSTER_NAME} strategy: ApplyOnce --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- apiVersion: v1 data: csi-proxy: | diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml deleted file mode 100644 index c7ab81cb73e7..000000000000 --- a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml +++ /dev/null @@ -1,1179 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled - metrics-server: enabled - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-provider: external - feature-gates: ${K8S_FEATURE_GATES:-""} - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-provider: external - cluster-name: ${CLUSTER_NAME} - feature-gates: HPAContainerMetrics=true - v: "4" - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - kubernetesVersion: ci/${CI_VERSION} - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - - # This test installs release packages or binaries that are a result of the CI and release builds. - # It runs '... --version' commands to verify that the binaries are correctly installed - # and finally uninstalls the packages. - # For the release packages it tests all versions in the support skew. - LINE_SEPARATOR="*************************************************" - echo "$$LINE_SEPARATOR" - CI_VERSION=${CI_VERSION} - if [[ "$${CI_VERSION}" != "" ]]; then - CI_DIR=/tmp/k8s-ci - mkdir -p $$CI_DIR - declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") - declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") - CONTAINER_EXT="tar" - echo "* testing CI version $$CI_VERSION" - # Check for semver - if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl - curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg - echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list - apt-get update - # replace . with \. - VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" - PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" - DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION - done - else - CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" - wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" - chmod +x "$$CI_DIR/$$CI_PACKAGE" - mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" - done - IMAGE_REGISTRY_PREFIX=registry.k8s.io - for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do - echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" - wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" - $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - done - fi - systemctl restart kubelet - fi - echo "* checking binary versions" - echo "ctr version: " $(ctr version) - echo "kubeadm version: " $(kubeadm version -o=short) - echo "kubectl version: " $(kubectl version --client=true --short=true) - echo "kubelet version: " $(kubelet --version) - echo "$$LINE_SEPARATOR" - owner: root:root - path: /tmp/kubeadm-bootstrap.sh - permissions: "0744" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-provider: external - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-provider: external - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: - - bash -c /tmp/kubeadm-bootstrap.sh - verbosity: 5 - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - image: - marketplace: - offer: capi - publisher: cncf-upstream - sku: ubuntu-1804-gen1 - version: latest - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: {} - template: - metadata: - labels: - nodepool: pool1 - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - image: - marketplace: - offer: capi - publisher: cncf-upstream - sku: ubuntu-1804-gen1 - version: latest - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmExtensions: - - name: CustomScript - protectedSettings: - commandToExecute: | - #!/bin/sh - echo "This script is a no-op used for extension testing purposes ..." - touch test_file - publisher: Microsoft.Azure.Extensions - version: "2.1" - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - - # This test installs release packages or binaries that are a result of the CI and release builds. - # It runs '... --version' commands to verify that the binaries are correctly installed - # and finally uninstalls the packages. - # For the release packages it tests all versions in the support skew. - LINE_SEPARATOR="*************************************************" - echo "$$LINE_SEPARATOR" - CI_VERSION=${CI_VERSION} - if [[ "$${CI_VERSION}" != "" ]]; then - CI_DIR=/tmp/k8s-ci - mkdir -p $$CI_DIR - declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") - declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") - CONTAINER_EXT="tar" - echo "* testing CI version $$CI_VERSION" - # Check for semver - if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl - curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg - echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list - apt-get update - # replace . with \. - VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" - PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" - DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION - done - else - CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" - wget --inet4-only "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" - chmod +x "$$CI_DIR/$$CI_PACKAGE" - mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" - done - IMAGE_REGISTRY_PREFIX=registry.k8s.io - for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do - echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" - wget --inet4-only "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" - $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - done - fi - systemctl restart kubelet - fi - echo "* checking binary versions" - echo "ctr version: " $(ctr version) - echo "kubeadm version: " $(kubeadm version -o=short) - echo "kubectl version: " $(kubectl version --client=true --short=true) - echo "kubelet version: " $(kubelet --version) - echo "$$LINE_SEPARATOR" - owner: root:root - path: /tmp/kubeadm-bootstrap.sh - permissions: "0744" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-provider: external - name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - bash -c /tmp/kubeadm-bootstrap.sh - verbosity: 5 ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: containerd - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - spec: - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: windows-2022-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" - if($$CONTAINERD_URL -ne ""){ - # Kubelet service depends on contianerd service so make a best effort attempt to stop it - Stop-Service kubelet -Force -ErrorAction SilentlyContinue - Stop-Service containerd -Force - echo "downloading containerd: $$CONTAINERD_URL" - curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" - tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 - - Start-Service containerd - } - - containerd.exe --version - containerd-shim-runhcs-v1.exe --version - path: C:/replace-containerd.ps1 - permissions: "0744" - - content: | - mkdir -Force c:/localdumps - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f - # Enable sftp so we can copy crash dump files during log collection of stfp - $sshd_config = "$env:ProgramData\ssh\sshd_config" - if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } - Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" - sc.exe stop sshd - sc.exe start sshd - path: C:/collect-hns-crashes.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$CI_VERSION="${CI_VERSION}" - if($$CI_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true --short=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-ci-binaries.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - azure-container-registry-config: c:/k/azure.json - cloud-provider: external - feature-gates: ${NODE_FEATURE_GATES:-""} - v: "2" - windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell C:/create-temp-folder.ps1 - - powershell C:/replace-containerd.ps1 - - powershell C:/collect-hns-crashes.ps1 - - powershell C:/replace-ci-binaries.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineHealthCheck -metadata: - name: ${CLUSTER_NAME}-mhc-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - maxUnhealthy: 100% - selector: - matchLabels: - nodepool: pool1 - unhealthyConditions: - - status: "True" - timeout: 30s - type: E2ENodeUnhealthy ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: - v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n - \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: - |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": - [\n {\n \"windows_use_single_network\": true,\n \"type\": - \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n - \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n - \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n - \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": - {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n - \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": - \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": - \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n - \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n - \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": - \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": - \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": - \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n - \ },\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": - \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n - \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: - DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n - \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n - \ app: calico\n template:\n metadata:\n labels:\n tier: node\n - \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - \ nodeSelectorTerms:\n - matchExpressions:\n - - key: kubernetes.io/os\n operator: In\n values:\n - \ - windows\n - key: kubernetes.io/arch\n - \ operator: In\n values:\n - - amd64\n securityContext:\n windowsOptions:\n hostProcess: - true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: - true\n serviceAccountName: calico-node\n tolerations:\n - operator: - Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on - for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n - \ - effect: NoExecute\n operator: Exists\n initContainers:\n # - This container installs the CNI binaries\n # and CNI network config file - on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.26.1-hostprocess\n - \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n - \ imagePullPolicy: Always\n env:\n # Name of the CNI - config file to create.\n - name: CNI_CONF_NAME\n value: - \"10-calico.conflist\"\n # The CNI network config to install on each - node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n - \ name: calico-config-windows\n key: cni_network_config\n - \ # Set the hostname based on the k8s node name.\n - name: - KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: - spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n - \ valueFrom:\n configMapKeyRef:\n name: - calico-config-windows\n key: veth_mtu\n # Prevents - the container from sleeping forever.\n - name: SLEEP\n value: - \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n - \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: - cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: - cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n - \ securityContext:\n windowsOptions:\n hostProcess: - true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n - \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.26.1-hostprocess\n - \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n - \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: - Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: - /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n - \ fieldRef:\n apiVersion: v1\n fieldPath: - metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n - \ apiVersion: v1\n fieldPath: metadata.namespace\n - - name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n - \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n - \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n - \ image: sigwindowstools/calico-node:v3.26.1-hostprocess\n args: - [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: - Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n - \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n - \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n - \ subPath: static-rules.json\n env:\n - name: POD_NAME\n - \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: - metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n - \ apiVersion: v1\n fieldPath: metadata.namespace\n - - name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: - \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n - \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n - \ configMap:\n name: calico-static-rules\n # Used to install - CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n - \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n - \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n---\napiVersion: - apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n - \ group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: - IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: - false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n - \ properties:\n apiVersion:\n description: 'APIVersion - defines the versioned schema of this representation\n of an object. - Servers should convert recognized schemas to the latest\n internal - value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n - \ type: string\n kind:\n description: 'Kind is a - string value representing the REST resource this\n object represents. - Servers may infer this from the endpoint the client\n submits requests - to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n - \ type: string\n metadata:\n type: object\n spec:\n - \ description: IPAMConfigSpec contains the specification for an IPAMConfig\n - \ resource.\n properties:\n autoAllocateBlocks:\n - \ type: boolean\n maxBlocksPerHost:\n description: - MaxBlocksPerHost, if non-zero, is the max number of blocks\n that - can be affine to each host.\n maximum: 2147483647\n minimum: - 0\n type: integer\n strictAffinity:\n type: - boolean\n required:\n - autoAllocateBlocks\n - - strictAffinity\n type: object\n type: object\n served: true\n - \ storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n - \ conditions: []\n storedVersions: []\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: metrics-server-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - metrics-server: enabled - resources: - - kind: ConfigMap - name: metrics-server-${CLUSTER_NAME} - strategy: ApplyOnce ---- -apiVersion: v1 -data: - metrics-server: | - apiVersion: v1 - kind: ServiceAccount - metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - k8s-app: metrics-server - rbac.authorization.k8s.io/aggregate-to-admin: "true" - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rbac.authorization.k8s.io/aggregate-to-view: "true" - name: system:aggregated-metrics-reader - rules: - - apiGroups: - - metrics.k8s.io - resources: - - pods - - nodes - verbs: - - get - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server - rules: - - apiGroups: - - "" - resources: - - nodes/metrics - verbs: - - get - - apiGroups: - - "" - resources: - - pods - - nodes - verbs: - - get - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - labels: - k8s-app: metrics-server - name: metrics-server-auth-reader - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - k8s-app: metrics-server - name: metrics-server:system:auth-delegator - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:metrics-server - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - --- - apiVersion: v1 - kind: Service - metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system - spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - k8s-app: metrics-server - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: metrics-server - strategy: - rollingUpdate: - maxUnavailable: 0 - template: - metadata: - labels: - k8s-app: metrics-server - spec: - containers: - - args: - - --cert-dir=/tmp - - --secure-port=4443 - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --kubelet-use-node-status-port - - --metric-resolution=15s - - --kubelet-insecure-tls - image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /livez - port: https - scheme: HTTPS - periodSeconds: 10 - name: metrics-server - ports: - - containerPort: 4443 - name: https - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readyz - port: https - scheme: HTTPS - initialDelaySeconds: 20 - periodSeconds: 10 - resources: - requests: - cpu: 100m - memory: 200Mi - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - mountPath: /tmp - name: tmp-dir - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-cluster-critical - serviceAccountName: metrics-server - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - volumes: - - emptyDir: {} - name: tmp-dir - --- - apiVersion: apiregistration.k8s.io/v1 - kind: APIService - metadata: - labels: - k8s-app: metrics-server - name: v1beta1.metrics.k8s.io - spec: - group: metrics.k8s.io - groupPriorityMinimum: 100 - insecureSkipTLSVerify: true - service: - name: metrics-server - namespace: kube-system - version: v1beta1 - versionPriority: 100 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: metrics-server-${CLUSTER_NAME} - namespace: default diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index acd162ffb95d..cef3a2b4e909 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -628,6 +629,26 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- apiVersion: v1 data: kube-proxy-patch: |- diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 368e2c611d39..0c48396e5345 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico name: ${CLUSTER_NAME} namespace: default @@ -277,3 +278,23 @@ spec: calicoctl: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index 8c44b51b9098..9191bb6846f2 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico-dual-stack name: ${CLUSTER_NAME} namespace: default @@ -366,3 +367,23 @@ spec: calicoctl: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index f5a657ce6e40..c5bf852d13e1 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico name: ${CLUSTER_NAME} namespace: default @@ -261,3 +262,23 @@ spec: calicoctl: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index 2717aaacde9f..2254726d5699 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico name: ${CLUSTER_NAME} namespace: default @@ -288,3 +289,23 @@ spec: calicoctl: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml index c3a43b4cdb23..1b2776209aa4 100644 --- a/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -398,6 +399,26 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- apiVersion: v1 data: proxy: | diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml index 7b65f75175c2..12a90dee52b3 100644 --- a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -462,6 +463,26 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- apiVersion: v1 data: proxy: | diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index d41391ba463c..6421d369d20a 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico-ipv6 name: ${CLUSTER_NAME} namespace: default @@ -383,3 +384,23 @@ spec: calicoctl: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 96eea451785e..bb6058f5b8a7 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -562,6 +563,26 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- apiVersion: v1 data: kube-proxy-patch: |- diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index 3579753e1b75..9105375b4a9a 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -388,6 +389,26 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- apiVersion: v1 data: proxy: | diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index fb55151d07da..0947e9a2c30e 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -382,6 +383,26 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- apiVersion: v1 data: proxy: | diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 6d863ca8ce07..23062dd4383d 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico name: ${CLUSTER_NAME} namespace: default @@ -247,3 +248,23 @@ spec: calicoctl: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index 439830865b6c..8c31d600b2f3 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico name: ${CLUSTER_NAME} namespace: default @@ -316,6 +317,26 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- apiVersion: v1 data: resources: | diff --git a/templates/test/ci/cluster-template-prow-topology.yaml b/templates/test/ci/cluster-template-prow-topology.yaml index b0201abf9d99..c6e880ded370 100644 --- a/templates/test/ci/cluster-template-prow-topology.yaml +++ b/templates/test/ci/cluster-template-prow-topology.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -113,6 +114,26 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- apiVersion: v1 data: proxy: | diff --git a/templates/test/ci/cluster-template-prow-workload-identity.yaml b/templates/test/ci/cluster-template-prow-workload-identity.yaml index cbb61315b469..d46601ecb242 100644 --- a/templates/test/ci/cluster-template-prow-workload-identity.yaml +++ b/templates/test/ci/cluster-template-prow-workload-identity.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico name: ${CLUSTER_NAME} namespace: default @@ -253,3 +254,23 @@ spec: calicoctl: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index f31e475dbaae..e3dcbd88459e 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -446,6 +447,26 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- apiVersion: v1 data: proxy: | diff --git a/templates/test/ci/patches/cluster-label-azuredisk-csi-driver.yaml b/templates/test/ci/patches/cluster-label-azuredisk-csi-driver.yaml new file mode 100644 index 000000000000..080f57937ff4 --- /dev/null +++ b/templates/test/ci/patches/cluster-label-azuredisk-csi-driver.yaml @@ -0,0 +1,6 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + labels: + azuredisk-csi: "true" \ No newline at end of file diff --git a/templates/test/ci/prow-azure-cni-v1/kustomization.yaml b/templates/test/ci/prow-azure-cni-v1/kustomization.yaml index 459a03a481b6..fe00ee0a1413 100644 --- a/templates/test/ci/prow-azure-cni-v1/kustomization.yaml +++ b/templates/test/ci/prow-azure-cni-v1/kustomization.yaml @@ -3,6 +3,8 @@ kind: Kustomization namespace: default resources: - ../../../flavors/azure-cni-v1/ + - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml diff --git a/templates/test/ci/prow-custom-vnet/kustomization.yaml b/templates/test/ci/prow-custom-vnet/kustomization.yaml index 1ec37da5fade..0565fdf4edf5 100644 --- a/templates/test/ci/prow-custom-vnet/kustomization.yaml +++ b/templates/test/ci/prow-custom-vnet/kustomization.yaml @@ -5,6 +5,7 @@ resources: - ../../../flavors/default - ../prow/mhc.yaml - ../../../addons/cluster-api-helm/calico.yaml + - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/mhc.yaml @@ -12,4 +13,5 @@ patchesStrategicMerge: - patches/custom-vnet.yaml - ../patches/uami-md-0.yaml - ../patches/uami-control-plane.yaml - - ../patches/cluster-label-calico.yaml \ No newline at end of file + - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml diff --git a/templates/test/ci/prow-dual-stack/kustomization.yaml b/templates/test/ci/prow-dual-stack/kustomization.yaml index 459b8c4c9071..d123be7c0dfd 100644 --- a/templates/test/ci/prow-dual-stack/kustomization.yaml +++ b/templates/test/ci/prow-dual-stack/kustomization.yaml @@ -5,9 +5,11 @@ resources: - ../../../flavors/dual-stack - machine-pool-dualstack.yaml - ../../../addons/cluster-api-helm/calico-dual-stack.yaml + - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml - patches/azure-machine-template-control-plane.yaml - patches/azure-machine-template.yaml - patches/cluster-label-calico-dual-stack.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml diff --git a/templates/test/ci/prow-edgezone/kustomization.yaml b/templates/test/ci/prow-edgezone/kustomization.yaml index 6bb2a8f70df3..e8a0aa31b58e 100644 --- a/templates/test/ci/prow-edgezone/kustomization.yaml +++ b/templates/test/ci/prow-edgezone/kustomization.yaml @@ -4,6 +4,7 @@ namespace: default resources: - ../../../flavors/edgezone - ../../../addons/cluster-api-helm/calico.yaml + - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml @@ -14,4 +15,5 @@ patchesStrategicMerge: - patches/standardssd-disk.yaml - patches/machine-type.yaml - patches/kubernetes-version.yaml - - ../patches/cluster-label-calico.yaml \ No newline at end of file + - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml diff --git a/templates/test/ci/prow-flatcar/kustomization.yaml b/templates/test/ci/prow-flatcar/kustomization.yaml index c341146cab43..60af393772e4 100644 --- a/templates/test/ci/prow-flatcar/kustomization.yaml +++ b/templates/test/ci/prow-flatcar/kustomization.yaml @@ -4,7 +4,9 @@ namespace: default resources: - ../../../flavors/flatcar/ - ../../../addons/cluster-api-helm/calico.yaml + - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml diff --git a/templates/test/ci/prow-ipv6/kustomization.yaml b/templates/test/ci/prow-ipv6/kustomization.yaml index b72de16056c4..93cb5cf349bc 100644 --- a/templates/test/ci/prow-ipv6/kustomization.yaml +++ b/templates/test/ci/prow-ipv6/kustomization.yaml @@ -5,7 +5,10 @@ resources: - ../../../flavors/ipv6 - machine-pool-ipv6.yaml - ../../../addons/cluster-api-helm/calico-ipv6.yaml + - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml - patches/cluster-label-calico-ipv6.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml + diff --git a/templates/test/ci/prow-machine-pool/kustomization.yaml b/templates/test/ci/prow-machine-pool/kustomization.yaml index c189fde424e9..59822b47c4a1 100644 --- a/templates/test/ci/prow-machine-pool/kustomization.yaml +++ b/templates/test/ci/prow-machine-pool/kustomization.yaml @@ -7,6 +7,7 @@ resources: - ../../../addons/windows/csi-proxy/csi-proxy-resource-set.yaml - ../../../addons/windows/containerd-logging/containerd-logger-resource-set.yaml - ../../../addons/cluster-api-helm/calico.yaml + - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/azuremachinepool-vmextension.yaml - ../patches/tags.yaml @@ -14,6 +15,7 @@ patchesStrategicMerge: - ../patches/machine-pool-worker-counts.yaml - ../patches/windows-containerd-labels.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml configMapGenerator: - name: cni-${CLUSTER_NAME}-calico-windows files: diff --git a/templates/test/ci/prow-nvidia-gpu/kustomization.yaml b/templates/test/ci/prow-nvidia-gpu/kustomization.yaml index 13c1085fc3c1..d96de0180bab 100644 --- a/templates/test/ci/prow-nvidia-gpu/kustomization.yaml +++ b/templates/test/ci/prow-nvidia-gpu/kustomization.yaml @@ -4,11 +4,13 @@ namespace: default resources: - ../../../flavors/nvidia-gpu - ../../../addons/cluster-api-helm/calico.yaml + - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml - ../patches/azurecluster-gpu.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml patches: - path: patches/node-storage-type.yaml target: diff --git a/templates/test/ci/prow-private/kustomization.yaml b/templates/test/ci/prow-private/kustomization.yaml index a097dc10699d..a624b79ed552 100644 --- a/templates/test/ci/prow-private/kustomization.yaml +++ b/templates/test/ci/prow-private/kustomization.yaml @@ -5,6 +5,7 @@ resources: - ../../../flavors/private - cni-resource-set.yaml - ../../../addons/cluster-api-helm/calico.yaml + - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml @@ -15,6 +16,7 @@ patchesStrategicMerge: - ../prow-intree-cloud-provider/patches/intree-cp.yaml # TODO: remove once CAPI supports Helm addons - ../prow-intree-cloud-provider/patches/intree-md-0.yaml # TODO: remove once CAPI supports Helm addons - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml patches: - path: patches/user-assigned.yaml target: diff --git a/templates/test/ci/prow-topology/kustomization.yaml b/templates/test/ci/prow-topology/kustomization.yaml index 654099474e2f..da93db782a5a 100644 --- a/templates/test/ci/prow-topology/kustomization.yaml +++ b/templates/test/ci/prow-topology/kustomization.yaml @@ -6,10 +6,12 @@ resources: - cni-resource-set.yaml - ../../../addons/windows/csi-proxy/csi-proxy-resource-set.yaml - ../../../addons/cluster-api-helm/calico.yaml + - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/windows-containerd-labels.yaml - cluster.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml configMapGenerator: - name: cni-${CLUSTER_NAME}-calico-windows files: diff --git a/templates/test/ci/prow-workload-identity/kustomization.yaml b/templates/test/ci/prow-workload-identity/kustomization.yaml index 454911624606..642b67c258aa 100644 --- a/templates/test/ci/prow-workload-identity/kustomization.yaml +++ b/templates/test/ci/prow-workload-identity/kustomization.yaml @@ -4,6 +4,7 @@ namespace: default resources: - ../../../flavors/default - ../../../addons/cluster-api-helm/calico.yaml + - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/azureclusteridentity-azwi.yaml - ../patches/tags.yaml @@ -12,3 +13,4 @@ patchesStrategicMerge: - ../patches/uami-md-0.yaml - ../patches/uami-control-plane.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml diff --git a/templates/test/ci/prow/kustomization.yaml b/templates/test/ci/prow/kustomization.yaml index 22bc09935044..9eb601928264 100644 --- a/templates/test/ci/prow/kustomization.yaml +++ b/templates/test/ci/prow/kustomization.yaml @@ -11,6 +11,7 @@ resources: - ../../../addons/windows/csi-proxy/csi-proxy-resource-set.yaml - ../../../addons/windows/containerd-logging/containerd-logger-resource-set.yaml - ../../../addons/cluster-api-helm/calico.yaml + - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/mhc.yaml @@ -22,6 +23,7 @@ patchesStrategicMerge: - ../patches/windows-containerd-labels.yaml - ../patches/windows-server-version.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml patches: - target: group: bootstrap.cluster.x-k8s.io diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index 7ebfabbde8ca..9a361afa4195 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -510,6 +511,26 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- apiVersion: v1 data: kube-proxy-patch: |- diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index ef7704a608a5..b1537305b445 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -575,6 +576,26 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart + namespace: default +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- apiVersion: v1 data: kube-proxy-patch: |- diff --git a/test/e2e/cloud-provider-azure.go b/test/e2e/cloud-provider-azure.go index a976a239d374..eee1ddc4b85c 100644 --- a/test/e2e/cloud-provider-azure.go +++ b/test/e2e/cloud-provider-azure.go @@ -77,19 +77,25 @@ func InstallCNIAndCloudProviderAzureHelmChart(ctx context.Context, input cluster } } -// InstallAzureDiskCSIDriverHelmChart installs the official azure-disk CSI driver helm chart -func InstallAzureDiskCSIDriverHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, hasWindows bool) { - specName := "azuredisk-csi-drivers-install" - By("Installing azure-disk CSI driver components via helm") - options := &HelmOptions{ - Values: []string{"controller.replicas=1", "controller.runOnControlPlane=true"}, - } - // TODO: make this always true once HostProcessContainers are on for all supported k8s versions. - if hasWindows { - options.Values = append(options.Values, "windows.useHostProcessContainers=true") - } +// EnsureAzureDiskCSIDriverHelmChart installs the official azure-disk CSI driver helm chart +func EnsureAzureDiskCSIDriverHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, installHelmChart bool, hasWindows bool) { + specName := "ensure-azuredisk-csi-drivers" clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.Namespace, input.ClusterName) - InstallHelmChart(ctx, clusterProxy, kubesystem, azureDiskCSIDriverHelmRepoURL, azureDiskCSIDriverChartName, azureDiskCSIDriverHelmReleaseName, options, "") + + if installHelmChart { + By("Installing azure-disk CSI driver components via helm") + options := &HelmOptions{ + Values: []string{"controller.replicas=1", "controller.runOnControlPlane=true"}, + } + // TODO: make this always true once HostProcessContainers are on for all supported k8s versions. + if hasWindows { + options.Values = append(options.Values, "windows.useHostProcessContainers=true") + } + InstallHelmChart(ctx, clusterProxy, kubesystem, azureDiskCSIDriverHelmRepoURL, azureDiskCSIDriverChartName, azureDiskCSIDriverHelmReleaseName, options, "") + } else { + By("Ensuring azure-disk CSI driver is installed via CAAPH") + } + By("Waiting for Ready csi-azuredisk-controller deployment pods") for _, d := range []string{"csi-azuredisk-controller"} { waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, kubesystem, specName) diff --git a/test/e2e/common.go b/test/e2e/common.go index a21ea10a57f5..afaca2482259 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -294,7 +294,7 @@ func ensureControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCu EnsureCNI(ctx, input, installHelmCharts, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) } controlPlane := discoveryAndWaitForControlPlaneInitialized(ctx, input, result) - InstallAzureDiskCSIDriverHelmChart(ctx, input, hasWindows) + EnsureAzureDiskCSIDriverHelmChart(ctx, input, installHelmCharts, hasWindows) result.ControlPlane = controlPlane } diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 6a6fa1ec4b38..b69edbb8f0d3 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -9,7 +9,8 @@ images: loadBehavior: tryLoad - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.5.2 loadBehavior: tryLoad - - name: registry.k8s.io/cluster-api-helm/cluster-api-helm-controller:v0.1.0-alpha.9 + # TODO: Switch off of staging registry once image promotion is done. + - name: gcr.io/k8s-staging-cluster-api-helm/cluster-api-helm-controller:v0.1.0-alpha.10 loadBehavior: tryLoad providers: @@ -161,6 +162,8 @@ providers: replacements: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" + - old: "registry.k8s.io/cluster-api-helm/cluster-api-helm-controller:v0.1.0-alpha.10" + new: "gcr.io/k8s-staging-cluster-api-helm/cluster-api-helm-controller:v0.1.0-alpha.10" variables: AKS_KUBERNETES_VERSION: "latest" diff --git a/test/e2e/data/infrastructure-azure/v1beta1/bases/azuredisk-csi-driver.yaml b/test/e2e/data/infrastructure-azure/v1beta1/bases/azuredisk-csi-driver.yaml new file mode 100644 index 000000000000..07afde0faf6b --- /dev/null +++ b/test/e2e/data/infrastructure-azure/v1beta1/bases/azuredisk-csi-driver.yaml @@ -0,0 +1,18 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart +spec: + clusterSelector: + matchLabels: + azuredisk-csi: "true" + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + chartName: azuredisk-csi-driver + releaseName: azuredisk-csi-driver-oot + namespace: kube-system + valuesTemplate: | + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} \ No newline at end of file diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml index 0dc99e10b742..3675868068ce 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml @@ -1,5 +1,24 @@ apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy metadata: name: calico spec: @@ -56,6 +75,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico name: ${CLUSTER_NAME} namespace: default diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation/kustomization.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation/kustomization.yaml index 52e45be1f138..9e1f19137fe8 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation/kustomization.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation/kustomization.yaml @@ -4,7 +4,9 @@ bases: - mhc.yaml - ../bases/azure-cluster-identity.yaml - ../bases/calico.yaml + - ../bases/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/azurecluster-identity-ref.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml index 30eda5aae7c3..2634076849f3 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml @@ -1,5 +1,24 @@ apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy metadata: name: calico spec: @@ -56,6 +75,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico name: ${CLUSTER_NAME} namespace: default diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in/kustomization.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in/kustomization.yaml index 546624556839..52dd66fa0933 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in/kustomization.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in/kustomization.yaml @@ -3,8 +3,10 @@ bases: - ../bases/md.yaml - ../bases/azure-cluster-identity.yaml - ../bases/calico.yaml +- ../bases/azuredisk-csi-driver.yaml patchesStrategicMerge: - ./cluster-with-kcp.yaml - ../patches/azurecluster-identity-ref.yaml - ../patches/cluster-label-calico.yaml +- ../patches/cluster-label-azuredisk-csi-driver.yaml diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml index 594948ca9417..44ad8083fcac 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml @@ -1,5 +1,24 @@ apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy metadata: name: calico spec: @@ -55,6 +74,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico name: ${CLUSTER_NAME} namespace: default diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool/kustomization.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool/kustomization.yaml index 566770ccf026..dce55fb18ada 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool/kustomization.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool/kustomization.yaml @@ -3,7 +3,9 @@ resources: - ../bases/mp.yaml - ../bases/azure-cluster-identity.yaml - ../bases/calico.yaml + - ../bases/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/azurecluster-identity-ref.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-azuredisk-csi-driver.yaml diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml index 1c29569aa7d5..bc22c3e24ab9 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml @@ -1,5 +1,24 @@ apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy metadata: name: calico spec: @@ -56,6 +75,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico name: ${CLUSTER_NAME} namespace: default diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation/kustomization.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation/kustomization.yaml index 52219917bf39..ed195fc0bd6b 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation/kustomization.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation/kustomization.yaml @@ -4,8 +4,10 @@ bases: - mhc.yaml - ../bases/azure-cluster-identity.yaml - ../bases/calico.yaml + - ../bases/azuredisk-csi-driver.yaml patchesStrategicMerge: - ./md.yaml - ../patches/azurecluster-identity-ref.yaml - ../patches/cluster-label-calico.yaml +- ../patches/cluster-label-azuredisk-csi-driver.yaml diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml index 63e173f160b7..621d053fa6a0 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml @@ -1,5 +1,24 @@ apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy metadata: name: calico spec: @@ -56,6 +75,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico name: ${CLUSTER_NAME} namespace: default diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain/kustomization.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain/kustomization.yaml index 23f1031e120c..c292b89295f7 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain/kustomization.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain/kustomization.yaml @@ -3,9 +3,11 @@ bases: - ../bases/md.yaml - ../bases/azure-cluster-identity.yaml - ../bases/calico.yaml +- ../bases/azuredisk-csi-driver.yaml patchesStrategicMerge: - ./md.yaml - ./cluster-with-kcp.yaml - ../patches/azurecluster-identity-ref.yaml - ../patches/cluster-label-calico.yaml +- ../patches/cluster-label-azuredisk-csi-driver.yaml diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml index 6e25b06c1b06..1d3a8a0703f4 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml @@ -1,5 +1,24 @@ apiVersion: addons.cluster.x-k8s.io/v1alpha1 kind: HelmChartProxy +metadata: + name: azuredisk-csi-driver-chart +spec: + chartName: azuredisk-csi-driver + clusterSelector: + matchLabels: + azuredisk-csi: "true" + namespace: kube-system + releaseName: azuredisk-csi-driver-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts + valuesTemplate: |- + controller: + replicas: 1 + runOnControlPlane: true + windows: + useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy metadata: name: calico spec: @@ -56,6 +75,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + azuredisk-csi: "true" cni: calico name: ${CLUSTER_NAME} namespace: default diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template/kustomization.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template/kustomization.yaml index a5808c06bf38..313469333407 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template/kustomization.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template/kustomization.yaml @@ -3,7 +3,9 @@ bases: - ../bases/md.yaml - ../bases/azure-cluster-identity.yaml - ../bases/calico.yaml +- ../bases/azuredisk-csi-driver.yaml patchesStrategicMerge: - ../patches/azurecluster-identity-ref.yaml - ../patches/cluster-label-calico.yaml +- ../patches/cluster-label-azuredisk-csi-driver.yaml diff --git a/test/e2e/data/infrastructure-azure/v1beta1/patches/cluster-label-azuredisk-csi-driver.yaml b/test/e2e/data/infrastructure-azure/v1beta1/patches/cluster-label-azuredisk-csi-driver.yaml new file mode 100644 index 000000000000..080f57937ff4 --- /dev/null +++ b/test/e2e/data/infrastructure-azure/v1beta1/patches/cluster-label-azuredisk-csi-driver.yaml @@ -0,0 +1,6 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + labels: + azuredisk-csi: "true" \ No newline at end of file