From a879c48740b379da74176f8e907f73a1f00ef5a9 Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Tue, 24 Oct 2023 19:07:51 +0000 Subject: [PATCH] Add support for OOT cred provider in ci templates --- scripts/ci-build-azure-ccm.sh | 40 +- ...ow-ci-version-windows-containerd-2022.yaml | 1179 ----------------- .../ci/cluster-template-prow-ci-version.yaml | 55 + ...template-prow-machine-pool-ci-version.yaml | 55 + .../ci/prow-ci-version/kustomization.yaml | 20 + .../patches/oot-credential-provider-kcp.yaml | 40 + .../patches/oot-credential-provider-win.yaml | 25 + .../patches/oot-credential-provider.yaml | 32 + .../kustomization.yaml | 6 + ...adm-bootstrap-windows-k8s-ci-binaries.yaml | 25 + .../patches/machine-pool-ci-version.yaml | 25 + ...r-template-custom-builds-machine-pool.yaml | 55 + .../dev/cluster-template-custom-builds.yaml | 55 + .../kustomization.yaml | 6 + .../patches/custom-builds.yaml | 25 + ...-machine-pool-windows-k8s-pr-binaries.yaml | 25 + .../test/dev/custom-builds/kustomization.yaml | 20 + 17 files changed, 486 insertions(+), 1202 deletions(-) delete mode 100644 templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml create mode 100644 templates/test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml create mode 100644 templates/test/ci/prow-ci-version/patches/oot-credential-provider-win.yaml create mode 100644 templates/test/ci/prow-ci-version/patches/oot-credential-provider.yaml diff --git a/scripts/ci-build-azure-ccm.sh b/scripts/ci-build-azure-ccm.sh index 9a2b9e9dcba..7e0d6725fbb 100755 --- a/scripts/ci-build-azure-ccm.sh +++ b/scripts/ci-build-azure-ccm.sh @@ -56,11 +56,9 @@ setup() { echo "Image registry is ${REGISTRY}" echo "Image Tag CCM is ${IMAGE_TAG_CCM}" echo "Image Tag CNM is ${IMAGE_TAG_CNM}" - if [[ "${TEST_ACR_CREDENTIAL_PROVIDER:-}" =~ "true" ]]; then - IMAGE_TAG_ACR_CREDENTIAL_PROVIDER="${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER:-${IMAGE_TAG}}" - export IMAGE_TAG_ACR_CREDENTIAL_PROVIDER - echo "Image Tag ACR credential provider is ${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}" - fi + IMAGE_TAG_ACR_CREDENTIAL_PROVIDER="${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER:-${IMAGE_TAG}}" + export IMAGE_TAG_ACR_CREDENTIAL_PROVIDER + echo "Image Tag ACR credential provider is ${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}" if [[ -n "${WINDOWS_SERVER_VERSION:-}" ]]; then if [[ "${WINDOWS_SERVER_VERSION}" == "windows-2019" ]]; then @@ -80,19 +78,17 @@ main() { echo "Building Linux amd64 and Windows ${WINDOWS_IMAGE_VERSION} amd64 cloud node managers" make -C "${AZURE_CLOUD_PROVIDER_ROOT}" build-node-image-linux-amd64 push-node-image-linux-amd64 push-node-image-windows-"${WINDOWS_IMAGE_VERSION}"-amd64 manifest-node-manager-image-windows-"${WINDOWS_IMAGE_VERSION}"-amd64 - if [[ "${TEST_ACR_CREDENTIAL_PROVIDER:-}" =~ "true" ]]; then - echo "Building and pushing Linux and Windows amd64 Azure ACR credential provider" - make -C "${AZURE_CLOUD_PROVIDER_ROOT}" bin/azure-acr-credential-provider bin/azure-acr-credential-provider.exe - - if [[ "$(az storage container exists --name "${AZURE_BLOB_CONTAINER_NAME}" --query exists --output tsv)" == "false" ]]; then - echo "Creating ${AZURE_BLOB_CONTAINER_NAME} storage container" - az storage container create --name "${AZURE_BLOB_CONTAINER_NAME}" > /dev/null - az storage container set-permission --name "${AZURE_BLOB_CONTAINER_NAME}" --public-access container > /dev/null - fi + echo "Building and pushing Linux and Windows amd64 Azure ACR credential provider" + make -C "${AZURE_CLOUD_PROVIDER_ROOT}" bin/azure-acr-credential-provider bin/azure-acr-credential-provider.exe - az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/bin/azure-acr-credential-provider" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" - az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/bin/azure-acr-credential-provider.exe" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" + if [[ "$(az storage container exists --name "${AZURE_BLOB_CONTAINER_NAME}" --query exists --output tsv)" == "false" ]]; then + echo "Creating ${AZURE_BLOB_CONTAINER_NAME} storage container" + az storage container create --name "${AZURE_BLOB_CONTAINER_NAME}" > /dev/null + az storage container set-permission --name "${AZURE_BLOB_CONTAINER_NAME}" --public-access container > /dev/null fi + + az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/bin/azure-acr-credential-provider" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/bin/azure-acr-credential-provider.exe" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" fi } @@ -110,13 +106,11 @@ can_reuse_artifacts() { echo "false" && return fi - if [[ "${TEST_ACR_CREDENTIAL_PROVIDER:-}" =~ "true" ]]; then - for BINARY in azure-acr-credential-provider azure-acr-credential-provider.exe; do - if [[ "$(az storage blob exists --container-name "${AZURE_BLOB_CONTAINER_NAME}" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/${BINARY}" --query exists --output tsv)" == "false" ]]; then - echo "false" && return - fi - done - fi + for BINARY in azure-acr-credential-provider azure-acr-credential-provider.exe; do + if [[ "$(az storage blob exists --container-name "${AZURE_BLOB_CONTAINER_NAME}" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/${BINARY}" --query exists --output tsv)" == "false" ]]; then + echo "false" && return + fi + done echo "true" } diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml deleted file mode 100644 index 98a029b0438..00000000000 --- a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml +++ /dev/null @@ -1,1179 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled - metrics-server: enabled - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-provider: external - feature-gates: ${K8S_FEATURE_GATES:-""} - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-provider: external - cluster-name: ${CLUSTER_NAME} - feature-gates: HPAContainerMetrics=true - v: "4" - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - kubernetesVersion: ci/${CI_VERSION} - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - - # This test installs release packages or binaries that are a result of the CI and release builds. - # It runs '... --version' commands to verify that the binaries are correctly installed - # and finally uninstalls the packages. - # For the release packages it tests all versions in the support skew. - LINE_SEPARATOR="*************************************************" - echo "$$LINE_SEPARATOR" - CI_VERSION=${CI_VERSION} - if [[ "$${CI_VERSION}" != "" ]]; then - CI_DIR=/tmp/k8s-ci - mkdir -p $$CI_DIR - declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") - declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") - CONTAINER_EXT="tar" - echo "* testing CI version $$CI_VERSION" - # Check for semver - if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list - apt-get update - # replace . with \. - VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" - PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" - DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION - done - else - CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" - wget "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" - chmod +x "$$CI_DIR/$$CI_PACKAGE" - mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" - done - IMAGE_REGISTRY_PREFIX=registry.k8s.io - for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do - echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" - wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" - $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - done - fi - systemctl restart kubelet - fi - echo "* checking binary versions" - echo "ctr version: " $(ctr version) - echo "kubeadm version: " $(kubeadm version -o=short) - echo "kubectl version: " $(kubectl version --client=true --short=true) - echo "kubelet version: " $(kubelet --version) - echo "$$LINE_SEPARATOR" - owner: root:root - path: /tmp/kubeadm-bootstrap.sh - permissions: "0744" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-provider: external - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-provider: external - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: - - bash -c /tmp/kubeadm-bootstrap.sh - verbosity: 5 - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - image: - marketplace: - offer: capi - publisher: cncf-upstream - sku: ubuntu-1804-gen1 - version: latest - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: {} - template: - metadata: - labels: - nodepool: pool1 - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - image: - marketplace: - offer: capi - publisher: cncf-upstream - sku: ubuntu-1804-gen1 - version: latest - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmExtensions: - - name: CustomScript - protectedSettings: - commandToExecute: | - #!/bin/sh - echo "This script is a no-op used for extension testing purposes ..." - touch test_file - publisher: Microsoft.Azure.Extensions - version: "2.1" - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - - # This test installs release packages or binaries that are a result of the CI and release builds. - # It runs '... --version' commands to verify that the binaries are correctly installed - # and finally uninstalls the packages. - # For the release packages it tests all versions in the support skew. - LINE_SEPARATOR="*************************************************" - echo "$$LINE_SEPARATOR" - CI_VERSION=${CI_VERSION} - if [[ "$${CI_VERSION}" != "" ]]; then - CI_DIR=/tmp/k8s-ci - mkdir -p $$CI_DIR - declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") - declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") - CONTAINER_EXT="tar" - echo "* testing CI version $$CI_VERSION" - # Check for semver - if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list - apt-get update - # replace . with \. - VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" - PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" - DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION - done - else - CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" - wget "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" - chmod +x "$$CI_DIR/$$CI_PACKAGE" - mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" - done - IMAGE_REGISTRY_PREFIX=registry.k8s.io - for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do - echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" - wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" - $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - done - fi - systemctl restart kubelet - fi - echo "* checking binary versions" - echo "ctr version: " $(ctr version) - echo "kubeadm version: " $(kubeadm version -o=short) - echo "kubectl version: " $(kubectl version --client=true --short=true) - echo "kubelet version: " $(kubelet --version) - echo "$$LINE_SEPARATOR" - owner: root:root - path: /tmp/kubeadm-bootstrap.sh - permissions: "0744" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-provider: external - name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - bash -c /tmp/kubeadm-bootstrap.sh - verbosity: 5 ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: containerd - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - spec: - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: windows-2022-containerd-gen1 - version: latest - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" - if($$CONTAINERD_URL -ne ""){ - # Kubelet service depends on contianerd service so make a best effort attempt to stop it - Stop-Service kubelet -Force -ErrorAction SilentlyContinue - Stop-Service containerd -Force - echo "downloading containerd: $$CONTAINERD_URL" - curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" - tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 - - Start-Service containerd - } - - containerd.exe --version - containerd-shim-runhcs-v1.exe --version - path: C:/replace-containerd.ps1 - permissions: "0744" - - content: | - mkdir -Force c:/localdumps - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f - # Enable sftp so we can copy crash dump files during log collection of stfp - $sshd_config = "$env:ProgramData\ssh\sshd_config" - if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } - Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" - sc.exe stop sshd - sc.exe start sshd - path: C:/collect-hns-crashes.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$CI_VERSION="${CI_VERSION}" - if($$CI_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true --short=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-ci-binaries.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - azure-container-registry-config: c:/k/azure.json - cloud-provider: external - feature-gates: ${NODE_FEATURE_GATES:-""} - v: "2" - windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell C:/create-temp-folder.ps1 - - powershell C:/replace-containerd.ps1 - - powershell C:/collect-hns-crashes.ps1 - - powershell C:/replace-ci-binaries.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineHealthCheck -metadata: - name: ${CLUSTER_NAME}-mhc-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - maxUnhealthy: 100% - selector: - matchLabels: - nodepool: pool1 - unhealthyConditions: - - status: "True" - timeout: 30s - type: E2ENodeUnhealthy ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- -apiVersion: v1 -data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: - v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n - \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: - |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": - [\n {\n \"windows_use_single_network\": true,\n \"type\": - \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n - \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n - \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n - \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": - {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n - \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": - \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": - \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n - \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n - \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": - \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": - \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": - \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n - \ },\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": - \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n - \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: - DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n - \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n - \ app: calico\n template:\n metadata:\n labels:\n tier: node\n - \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - \ nodeSelectorTerms:\n - matchExpressions:\n - - key: kubernetes.io/os\n operator: In\n values:\n - \ - windows\n - key: kubernetes.io/arch\n - \ operator: In\n values:\n - - amd64\n securityContext:\n windowsOptions:\n hostProcess: - true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: - true\n serviceAccountName: calico-node\n tolerations:\n - operator: - Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on - for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n - \ - effect: NoExecute\n operator: Exists\n initContainers:\n # - This container installs the CNI binaries\n # and CNI network config file - on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.26.1-hostprocess\n - \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n - \ imagePullPolicy: Always\n env:\n # Name of the CNI - config file to create.\n - name: CNI_CONF_NAME\n value: - \"10-calico.conflist\"\n # The CNI network config to install on each - node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n - \ name: calico-config-windows\n key: cni_network_config\n - \ # Set the hostname based on the k8s node name.\n - name: - KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: - spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n - \ valueFrom:\n configMapKeyRef:\n name: - calico-config-windows\n key: veth_mtu\n # Prevents - the container from sleeping forever.\n - name: SLEEP\n value: - \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n - \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: - cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: - cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n - \ securityContext:\n windowsOptions:\n hostProcess: - true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n - \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.26.1-hostprocess\n - \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n - \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: - Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: - /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n - \ fieldRef:\n apiVersion: v1\n fieldPath: - metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n - \ apiVersion: v1\n fieldPath: metadata.namespace\n - - name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n - \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n - \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n - \ image: sigwindowstools/calico-node:v3.26.1-hostprocess\n args: - [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: - Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n - \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n - \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n - \ subPath: static-rules.json\n env:\n - name: POD_NAME\n - \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: - metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n - \ apiVersion: v1\n fieldPath: metadata.namespace\n - - name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: - \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n - \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n - \ configMap:\n name: calico-static-rules\n # Used to install - CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n - \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n - \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n---\napiVersion: - apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n - \ group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: - IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: - false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n - \ properties:\n apiVersion:\n description: 'APIVersion - defines the versioned schema of this representation\n of an object. - Servers should convert recognized schemas to the latest\n internal - value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n - \ type: string\n kind:\n description: 'Kind is a - string value representing the REST resource this\n object represents. - Servers may infer this from the endpoint the client\n submits requests - to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n - \ type: string\n metadata:\n type: object\n spec:\n - \ description: IPAMConfigSpec contains the specification for an IPAMConfig\n - \ resource.\n properties:\n autoAllocateBlocks:\n - \ type: boolean\n maxBlocksPerHost:\n description: - MaxBlocksPerHost, if non-zero, is the max number of blocks\n that - can be affine to each host.\n maximum: 2147483647\n minimum: - 0\n type: integer\n strictAffinity:\n type: - boolean\n required:\n - autoAllocateBlocks\n - - strictAffinity\n type: object\n type: object\n served: true\n - \ storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n - \ conditions: []\n storedVersions: []\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: metrics-server-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - metrics-server: enabled - resources: - - kind: ConfigMap - name: metrics-server-${CLUSTER_NAME} - strategy: ApplyOnce ---- -apiVersion: v1 -data: - metrics-server: | - apiVersion: v1 - kind: ServiceAccount - metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - k8s-app: metrics-server - rbac.authorization.k8s.io/aggregate-to-admin: "true" - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rbac.authorization.k8s.io/aggregate-to-view: "true" - name: system:aggregated-metrics-reader - rules: - - apiGroups: - - metrics.k8s.io - resources: - - pods - - nodes - verbs: - - get - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server - rules: - - apiGroups: - - "" - resources: - - nodes/metrics - verbs: - - get - - apiGroups: - - "" - resources: - - pods - - nodes - verbs: - - get - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - labels: - k8s-app: metrics-server - name: metrics-server-auth-reader - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - k8s-app: metrics-server - name: metrics-server:system:auth-delegator - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:metrics-server - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - --- - apiVersion: v1 - kind: Service - metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system - spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - k8s-app: metrics-server - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: metrics-server - strategy: - rollingUpdate: - maxUnavailable: 0 - template: - metadata: - labels: - k8s-app: metrics-server - spec: - containers: - - args: - - --cert-dir=/tmp - - --secure-port=4443 - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --kubelet-use-node-status-port - - --metric-resolution=15s - - --kubelet-insecure-tls - image: registry.k8s.io/metrics-server/metrics-server:v0.6.3 - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /livez - port: https - scheme: HTTPS - periodSeconds: 10 - name: metrics-server - ports: - - containerPort: 4443 - name: https - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readyz - port: https - scheme: HTTPS - initialDelaySeconds: 20 - periodSeconds: 10 - resources: - requests: - cpu: 100m - memory: 200Mi - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - mountPath: /tmp - name: tmp-dir - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-cluster-critical - serviceAccountName: metrics-server - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - volumes: - - emptyDir: {} - name: tmp-dir - --- - apiVersion: apiregistration.k8s.io/v1 - kind: APIService - metadata: - labels: - k8s-app: metrics-server - name: v1beta1.metrics.k8s.io - spec: - group: metrics.k8s.io - groupPriorityMinimum: 100 - insecureSkipTLSVerify: true - service: - name: metrics-server - namespace: kube-system - version: v1beta1 - versionPriority: 100 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: metrics-server-${CLUSTER_NAME} - namespace: default diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 94159b73bc9..b1c1d2c26bb 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -107,6 +107,23 @@ spec: set -o errexit [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + # This test installs release packages or binaries that are a result of the CI and release builds. # It runs '... --version' commands to verify that the binaries are correctly installed # and finally uninstalls the packages. @@ -168,18 +185,23 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: @@ -293,6 +315,23 @@ spec: set -o errexit [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + # This test installs release packages or binaries that are a result of the CI and release builds. # It runs '... --version' commands to verify that the binaries are correctly installed # and finally uninstalls the packages. @@ -354,8 +393,11 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 --- @@ -471,6 +513,16 @@ spec: - content: | $ErrorActionPreference = 'Stop' + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + Stop-Service kubelet -Force $$CI_VERSION="${CI_VERSION}" @@ -503,6 +555,8 @@ spec: azure-container-registry-config: c:/k/azure.json cloud-provider: external feature-gates: ${NODE_FEATURE_GATES:-""} + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml v: "2" windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS name: '{{ ds.meta_data["local_hostname"] }}' @@ -513,6 +567,7 @@ spec: - powershell C:/create-temp-folder.ps1 - powershell C:/replace-containerd.ps1 - powershell C:/collect-hns-crashes.ps1 + - powershell C:/oot-cred-provider.ps1 - powershell C:/replace-ci-binaries.ps1 users: - groups: Administrators diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 7390123c805..951ead90e58 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -105,6 +105,23 @@ spec: set -o errexit [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + # This test installs release packages or binaries that are a result of the CI and release builds. # It runs '... --version' commands to verify that the binaries are correctly installed # and finally uninstalls the packages. @@ -166,18 +183,23 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: @@ -286,6 +308,23 @@ spec: set -o errexit [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + # This test installs release packages or binaries that are a result of the CI and release builds. # It runs '... --version' commands to verify that the binaries are correctly installed # and finally uninstalls the packages. @@ -355,8 +394,11 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -466,12 +508,24 @@ spec: kubelet.exe --version path: C:/replace-k8s-binaries.ps1 permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" joinConfiguration: nodeRegistration: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: @@ -480,6 +534,7 @@ spec: preKubeadmCommands: - powershell c:/create-external-network.ps1 - powershell C:/replace-k8s-binaries.ps1 + - powershell C:/oot-cred-provider.ps1 users: - groups: Administrators name: capi diff --git a/templates/test/ci/prow-ci-version/kustomization.yaml b/templates/test/ci/prow-ci-version/kustomization.yaml index f18899c8284..d5b18eb1dc9 100644 --- a/templates/test/ci/prow-ci-version/kustomization.yaml +++ b/templates/test/ci/prow-ci-version/kustomization.yaml @@ -15,6 +15,26 @@ patchesStrategicMerge: - ../patches/metrics-server-enabled-cluster.yaml - ../patches/controller-manager-featuregates.yaml patches: +- target: + group: bootstrap.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + path: patches/oot-credential-provider.yaml +- target: + group: bootstrap.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmConfigTemplate + name: .*-md-win + namespace: default + path: patches/oot-credential-provider-win.yaml +- target: + group: controlplane.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmControlPlane + name: .*-control-plane + path: patches/oot-credential-provider-kcp.yaml - target: group: bootstrap.cluster.x-k8s.io version: v1beta1 diff --git a/templates/test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml b/templates/test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml new file mode 100644 index 00000000000..c33ab3c0433 --- /dev/null +++ b/templates/test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml @@ -0,0 +1,40 @@ +- op: add + path: /spec/kubeadmConfigSpec/files/- + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + path: /tmp/oot-cred-provider.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/kubeadmConfigSpec/preKubeadmCommands/- + value: + bash -c /tmp/oot-cred-provider.sh +- op: add + path: /spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml +- op: add + path: /spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml diff --git a/templates/test/ci/prow-ci-version/patches/oot-credential-provider-win.yaml b/templates/test/ci/prow-ci-version/patches/oot-credential-provider-win.yaml new file mode 100644 index 00000000000..298e9f67023 --- /dev/null +++ b/templates/test/ci/prow-ci-version/patches/oot-credential-provider-win.yaml @@ -0,0 +1,25 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + $ErrorActionPreference = 'Stop' + + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + powershell C:/oot-cred-provider.ps1 +- op: add + path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml \ No newline at end of file diff --git a/templates/test/ci/prow-ci-version/patches/oot-credential-provider.yaml b/templates/test/ci/prow-ci-version/patches/oot-credential-provider.yaml new file mode 100644 index 00000000000..27ad27e8a89 --- /dev/null +++ b/templates/test/ci/prow-ci-version/patches/oot-credential-provider.yaml @@ -0,0 +1,32 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + path: /tmp/oot-cred-provider.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + bash -c /tmp/oot-cred-provider.sh +- op: add + path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml diff --git a/templates/test/ci/prow-machine-pool-ci-version/kustomization.yaml b/templates/test/ci/prow-machine-pool-ci-version/kustomization.yaml index 625e476b142..64190027e68 100644 --- a/templates/test/ci/prow-machine-pool-ci-version/kustomization.yaml +++ b/templates/test/ci/prow-machine-pool-ci-version/kustomization.yaml @@ -10,6 +10,12 @@ patchesStrategicMerge: - ../patches/machine-pool-worker-counts.yaml - patches/machine-pool-ci-version-windows.yaml patches: +- target: + group: controlplane.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmControlPlane + name: .*-control-plane + path: ../prow-ci-version/patches/oot-credential-provider-kcp.yaml - target: group: controlplane.cluster.x-k8s.io version: v1beta1 diff --git a/templates/test/ci/prow-machine-pool-ci-version/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml b/templates/test/ci/prow-machine-pool-ci-version/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml index ca29361c8f9..ea1268c5817 100644 --- a/templates/test/ci/prow-machine-pool-ci-version/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml +++ b/templates/test/ci/prow-machine-pool-ci-version/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml @@ -28,7 +28,32 @@ kubelet.exe --version path: C:/replace-k8s-binaries.ps1 permissions: "0744" +- op: add + path: /spec/files/- + value: + content: | + $ErrorActionPreference = 'Stop' + + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" - op: add path: /spec/preKubeadmCommands/- value: powershell C:/replace-k8s-binaries.ps1 +- op: add + path: /spec/preKubeadmCommands/- + value: + powershell C:/oot-cred-provider.ps1 +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml \ No newline at end of file diff --git a/templates/test/ci/prow-machine-pool-ci-version/patches/machine-pool-ci-version.yaml b/templates/test/ci/prow-machine-pool-ci-version/patches/machine-pool-ci-version.yaml index da1aa793bbd..8536f83d473 100644 --- a/templates/test/ci/prow-machine-pool-ci-version/patches/machine-pool-ci-version.yaml +++ b/templates/test/ci/prow-machine-pool-ci-version/patches/machine-pool-ci-version.yaml @@ -3,9 +3,34 @@ kind: KubeadmConfig metadata: name: ${CLUSTER_NAME}-mp-0 spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh files: + - path: /tmp/oot-cred-provider.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml - path: /tmp/kubeadm-bootstrap.sh owner: "root:root" permissions: "0744" diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index 72f78221a29..18bbb36151f 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -151,17 +151,38 @@ spec: owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" initConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -170,6 +191,7 @@ spec: - bash -c /tmp/replace-k8s-components.sh preKubeadmCommands: - bash -c /tmp/replace-k8s-binaries.sh + - bash -c /tmp/oot-cred-provider.sh verbosity: 5 machineTemplate: infrastructureRef: @@ -272,6 +294,23 @@ spec: - content: | #!/bin/bash + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + set -o nounset set -o pipefail set -o errexit @@ -302,8 +341,11 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -414,12 +456,24 @@ spec: kube-proxy.exe --version path: C:/replace-pr-binaries.ps1 permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" joinConfiguration: nodeRegistration: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: @@ -428,6 +482,7 @@ spec: preKubeadmCommands: - powershell c:/create-external-network.ps1 - powershell C:/replace-pr-binaries.ps1 + - powershell C:/oot-cred-provider.ps1 users: - groups: Administrators name: capi diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index c1e154af278..3769053698e 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -104,6 +104,23 @@ spec: - content: | #!/bin/bash + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + set -o nounset set -o pipefail set -o errexit @@ -159,12 +176,16 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -172,6 +193,7 @@ spec: postKubeadmCommands: - bash -c /tmp/replace-k8s-components.sh preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh verbosity: 5 machineTemplate: @@ -286,6 +308,23 @@ spec: - content: | #!/bin/bash + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + set -o nounset set -o pipefail set -o errexit @@ -310,8 +349,11 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -426,6 +468,16 @@ spec: sc.exe start sshd path: C:/collect-hns-crashes.ps1 permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" - content: | Write-Host "Installing Azure CLI" $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi @@ -477,6 +529,8 @@ spec: azure-container-registry-config: c:/k/azure.json cloud-provider: external feature-gates: ${NODE_FEATURE_GATES:-""} + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml v: "2" windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS name: '{{ ds.meta_data["local_hostname"] }}' @@ -487,6 +541,7 @@ spec: - powershell C:/create-temp-folder.ps1 - powershell C:/replace-containerd.ps1 - powershell C:/collect-hns-crashes.ps1 + - powershell C:/oot-cred-provider.ps1 - powershell C:/install-az-cli.ps1 - powershell C:/replace-pr-binaries.ps1 users: diff --git a/templates/test/dev/custom-builds-machine-pool/kustomization.yaml b/templates/test/dev/custom-builds-machine-pool/kustomization.yaml index d13059993e6..f499bad3e86 100644 --- a/templates/test/dev/custom-builds-machine-pool/kustomization.yaml +++ b/templates/test/dev/custom-builds-machine-pool/kustomization.yaml @@ -6,6 +6,12 @@ patchesStrategicMerge: - patches/machine-pool-deployment-pr-version-windows.yaml - patches/custom-builds.yaml patches: + - target: + group: controlplane.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmControlPlane + name: .*-control-plane + path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml - target: group: bootstrap.cluster.x-k8s.io version: v1beta1 diff --git a/templates/test/dev/custom-builds-machine-pool/patches/custom-builds.yaml b/templates/test/dev/custom-builds-machine-pool/patches/custom-builds.yaml index 3a9816b5779..c01c7e69219 100644 --- a/templates/test/dev/custom-builds-machine-pool/patches/custom-builds.yaml +++ b/templates/test/dev/custom-builds-machine-pool/patches/custom-builds.yaml @@ -3,9 +3,34 @@ kind: KubeadmConfig metadata: name: ${CLUSTER_NAME}-mp-0 spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh files: + - path: /tmp/oot-cred-provider.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml - path: /tmp/replace-k8s-binaries.sh owner: "root:root" permissions: "0744" diff --git a/templates/test/dev/custom-builds-machine-pool/patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml b/templates/test/dev/custom-builds-machine-pool/patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml index 117d7196fbf..b6ac616e896 100644 --- a/templates/test/dev/custom-builds-machine-pool/patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml +++ b/templates/test/dev/custom-builds-machine-pool/patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml @@ -29,7 +29,32 @@ kube-proxy.exe --version path: C:/replace-pr-binaries.ps1 permissions: "0744" +- op: add + path: /spec/files/- + value: + content: | + $ErrorActionPreference = 'Stop' + + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" - op: add path: /spec/preKubeadmCommands/- value: powershell C:/replace-pr-binaries.ps1 +- op: add + path: /spec/preKubeadmCommands/- + value: + powershell C:/oot-cred-provider.ps1 +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml \ No newline at end of file diff --git a/templates/test/dev/custom-builds/kustomization.yaml b/templates/test/dev/custom-builds/kustomization.yaml index f45bff7e326..55d225af765 100644 --- a/templates/test/dev/custom-builds/kustomization.yaml +++ b/templates/test/dev/custom-builds/kustomization.yaml @@ -11,6 +11,26 @@ patchesStrategicMerge: - ../../../test/ci/patches/metrics-server-enabled-cluster.yaml - ../../../test/ci/patches/controller-manager-featuregates.yaml patches: +- target: + group: bootstrap.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider.yaml +- target: + group: bootstrap.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmConfigTemplate + name: .*-md-win + namespace: default + path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider-win.yaml +- target: + group: controlplane.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmControlPlane + name: .*-control-plane + path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml - target: group: bootstrap.cluster.x-k8s.io version: v1beta1