From b972f37f8ba7729d989eb48c71f8bc1569793013 Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Tue, 24 Oct 2023 19:07:51 +0000 Subject: [PATCH] Add support for OOT cred provider in ci templates --- scripts/ci-build-azure-ccm.sh | 40 ++++++-------- ...r-template-prow-ci-version-dual-stack.yaml | 42 ++++++++++++++ ...cluster-template-prow-ci-version-ipv6.yaml | 42 ++++++++++++++ .../ci/cluster-template-prow-ci-version.yaml | 55 +++++++++++++++++++ ...template-prow-machine-pool-ci-version.yaml | 55 +++++++++++++++++++ .../ci/prow-ci-version/kustomization.yaml | 20 +++++++ .../patches/oot-credential-provider-kcp.yaml | 40 ++++++++++++++ .../patches/oot-credential-provider-win.yaml | 25 +++++++++ .../patches/oot-credential-provider.yaml | 32 +++++++++++ .../kustomization.yaml | 6 ++ ...adm-bootstrap-windows-k8s-ci-binaries.yaml | 25 +++++++++ .../patches/machine-pool-ci-version.yaml | 25 +++++++++ ...r-template-custom-builds-machine-pool.yaml | 55 +++++++++++++++++++ .../dev/cluster-template-custom-builds.yaml | 55 +++++++++++++++++++ .../kustomization.yaml | 6 ++ .../patches/custom-builds.yaml | 25 +++++++++ ...-machine-pool-windows-k8s-pr-binaries.yaml | 25 +++++++++ .../test/dev/custom-builds/kustomization.yaml | 20 +++++++ 18 files changed, 570 insertions(+), 23 deletions(-) create mode 100644 templates/test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml create mode 100644 templates/test/ci/prow-ci-version/patches/oot-credential-provider-win.yaml create mode 100644 templates/test/ci/prow-ci-version/patches/oot-credential-provider.yaml diff --git a/scripts/ci-build-azure-ccm.sh b/scripts/ci-build-azure-ccm.sh index 9a2b9e9dcba..7e0d6725fbb 100755 --- a/scripts/ci-build-azure-ccm.sh +++ b/scripts/ci-build-azure-ccm.sh @@ -56,11 +56,9 @@ setup() { echo "Image registry is ${REGISTRY}" echo "Image Tag CCM is ${IMAGE_TAG_CCM}" echo "Image Tag CNM is ${IMAGE_TAG_CNM}" - if [[ "${TEST_ACR_CREDENTIAL_PROVIDER:-}" =~ "true" ]]; then - IMAGE_TAG_ACR_CREDENTIAL_PROVIDER="${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER:-${IMAGE_TAG}}" - export IMAGE_TAG_ACR_CREDENTIAL_PROVIDER - echo "Image Tag ACR credential provider is ${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}" - fi + IMAGE_TAG_ACR_CREDENTIAL_PROVIDER="${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER:-${IMAGE_TAG}}" + export IMAGE_TAG_ACR_CREDENTIAL_PROVIDER + echo "Image Tag ACR credential provider is ${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}" if [[ -n "${WINDOWS_SERVER_VERSION:-}" ]]; then if [[ "${WINDOWS_SERVER_VERSION}" == "windows-2019" ]]; then @@ -80,19 +78,17 @@ main() { echo "Building Linux amd64 and Windows ${WINDOWS_IMAGE_VERSION} amd64 cloud node managers" make -C "${AZURE_CLOUD_PROVIDER_ROOT}" build-node-image-linux-amd64 push-node-image-linux-amd64 push-node-image-windows-"${WINDOWS_IMAGE_VERSION}"-amd64 manifest-node-manager-image-windows-"${WINDOWS_IMAGE_VERSION}"-amd64 - if [[ "${TEST_ACR_CREDENTIAL_PROVIDER:-}" =~ "true" ]]; then - echo "Building and pushing Linux and Windows amd64 Azure ACR credential provider" - make -C "${AZURE_CLOUD_PROVIDER_ROOT}" bin/azure-acr-credential-provider bin/azure-acr-credential-provider.exe - - if [[ "$(az storage container exists --name "${AZURE_BLOB_CONTAINER_NAME}" --query exists --output tsv)" == "false" ]]; then - echo "Creating ${AZURE_BLOB_CONTAINER_NAME} storage container" - az storage container create --name "${AZURE_BLOB_CONTAINER_NAME}" > /dev/null - az storage container set-permission --name "${AZURE_BLOB_CONTAINER_NAME}" --public-access container > /dev/null - fi + echo "Building and pushing Linux and Windows amd64 Azure ACR credential provider" + make -C "${AZURE_CLOUD_PROVIDER_ROOT}" bin/azure-acr-credential-provider bin/azure-acr-credential-provider.exe - az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/bin/azure-acr-credential-provider" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" - az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/bin/azure-acr-credential-provider.exe" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" + if [[ "$(az storage container exists --name "${AZURE_BLOB_CONTAINER_NAME}" --query exists --output tsv)" == "false" ]]; then + echo "Creating ${AZURE_BLOB_CONTAINER_NAME} storage container" + az storage container create --name "${AZURE_BLOB_CONTAINER_NAME}" > /dev/null + az storage container set-permission --name "${AZURE_BLOB_CONTAINER_NAME}" --public-access container > /dev/null fi + + az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/bin/azure-acr-credential-provider" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/bin/azure-acr-credential-provider.exe" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" fi } @@ -110,13 +106,11 @@ can_reuse_artifacts() { echo "false" && return fi - if [[ "${TEST_ACR_CREDENTIAL_PROVIDER:-}" =~ "true" ]]; then - for BINARY in azure-acr-credential-provider azure-acr-credential-provider.exe; do - if [[ "$(az storage blob exists --container-name "${AZURE_BLOB_CONTAINER_NAME}" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/${BINARY}" --query exists --output tsv)" == "false" ]]; then - echo "false" && return - fi - done - fi + for BINARY in azure-acr-credential-provider azure-acr-credential-provider.exe; do + if [[ "$(az storage blob exists --container-name "${AZURE_BLOB_CONTAINER_NAME}" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/${BINARY}" --query exists --output tsv)" == "false" ]]; then + echo "false" && return + fi + done echo "true" } diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index 05b6f26632b..34a7937dfcf 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -125,6 +125,23 @@ spec: set -o errexit [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + # This test installs release packages or binaries that are a result of the CI and release builds. # It runs '... --version' commands to verify that the binaries are correctly installed # and finally uninstalls the packages. @@ -188,6 +205,8 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: controlPlane: @@ -197,6 +216,8 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -207,6 +228,7 @@ spec: /etc/resolv.conf - systemctl restart systemd-resolved containerd preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: @@ -322,6 +344,23 @@ spec: set -o errexit [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + # This test installs release packages or binaries that are a result of the CI and release builds. # It runs '... --version' commands to verify that the binaries are correctly installed # and finally uninstalls the packages. @@ -383,6 +422,8 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - echo "DNSStubListener=no" >> /etc/systemd/resolved.conf @@ -390,6 +431,7 @@ spec: /etc/resolv.conf - systemctl restart systemd-resolved containerd preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 --- diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index 13a3934bf05..3331958192f 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -128,6 +128,23 @@ spec: set -o errexit [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + # This test installs release packages or binaries that are a result of the CI and release builds. # It runs '... --version' commands to verify that the binaries are correctly installed # and finally uninstalls the packages. @@ -193,6 +210,8 @@ spec: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external cluster-dns: fd00::10 + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: controlPlane: @@ -204,6 +223,8 @@ spec: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external cluster-dns: fd00::10 + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -214,6 +235,7 @@ spec: /etc/resolv.conf - systemctl restart systemd-resolved containerd preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: @@ -339,6 +361,23 @@ spec: set -o errexit [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + # This test installs release packages or binaries that are a result of the CI and release builds. # It runs '... --version' commands to verify that the binaries are correctly installed # and finally uninstalls the packages. @@ -401,6 +440,8 @@ spec: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external cluster-dns: '[fd00::10]' + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - echo "DNSStubListener=no" >> /etc/systemd/resolved.conf @@ -408,6 +449,7 @@ spec: /etc/resolv.conf - systemctl restart systemd-resolved containerd preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 --- diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index ab842110b54..5c72f16249a 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -109,6 +109,23 @@ spec: set -o errexit [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + # This test installs release packages or binaries that are a result of the CI and release builds. # It runs '... --version' commands to verify that the binaries are correctly installed # and finally uninstalls the packages. @@ -170,18 +187,23 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: @@ -295,6 +317,23 @@ spec: set -o errexit [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + # This test installs release packages or binaries that are a result of the CI and release builds. # It runs '... --version' commands to verify that the binaries are correctly installed # and finally uninstalls the packages. @@ -356,8 +395,11 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 --- @@ -473,6 +515,16 @@ spec: - content: | $ErrorActionPreference = 'Stop' + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + Stop-Service kubelet -Force $$CI_VERSION="${CI_VERSION}" @@ -505,6 +557,8 @@ spec: azure-container-registry-config: c:/k/azure.json cloud-provider: external feature-gates: ${NODE_FEATURE_GATES:-""} + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml v: "2" windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS name: '{{ ds.meta_data["local_hostname"] }}' @@ -515,6 +569,7 @@ spec: - powershell C:/create-temp-folder.ps1 - powershell C:/replace-containerd.ps1 - powershell C:/collect-hns-crashes.ps1 + - powershell C:/oot-cred-provider.ps1 - powershell C:/replace-ci-binaries.ps1 users: - groups: Administrators diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index b1312f90adc..47fad868a9d 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -107,6 +107,23 @@ spec: set -o errexit [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + # This test installs release packages or binaries that are a result of the CI and release builds. # It runs '... --version' commands to verify that the binaries are correctly installed # and finally uninstalls the packages. @@ -168,18 +185,23 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh verbosity: 5 machineTemplate: @@ -288,6 +310,23 @@ spec: set -o errexit [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + # This test installs release packages or binaries that are a result of the CI and release builds. # It runs '... --version' commands to verify that the binaries are correctly installed # and finally uninstalls the packages. @@ -357,8 +396,11 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -468,12 +510,24 @@ spec: kubelet.exe --version path: C:/replace-k8s-binaries.ps1 permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" joinConfiguration: nodeRegistration: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: @@ -482,6 +536,7 @@ spec: preKubeadmCommands: - powershell c:/create-external-network.ps1 - powershell C:/replace-k8s-binaries.ps1 + - powershell C:/oot-cred-provider.ps1 users: - groups: Administrators name: capi diff --git a/templates/test/ci/prow-ci-version/kustomization.yaml b/templates/test/ci/prow-ci-version/kustomization.yaml index f18899c8284..d5b18eb1dc9 100644 --- a/templates/test/ci/prow-ci-version/kustomization.yaml +++ b/templates/test/ci/prow-ci-version/kustomization.yaml @@ -15,6 +15,26 @@ patchesStrategicMerge: - ../patches/metrics-server-enabled-cluster.yaml - ../patches/controller-manager-featuregates.yaml patches: +- target: + group: bootstrap.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + path: patches/oot-credential-provider.yaml +- target: + group: bootstrap.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmConfigTemplate + name: .*-md-win + namespace: default + path: patches/oot-credential-provider-win.yaml +- target: + group: controlplane.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmControlPlane + name: .*-control-plane + path: patches/oot-credential-provider-kcp.yaml - target: group: bootstrap.cluster.x-k8s.io version: v1beta1 diff --git a/templates/test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml b/templates/test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml new file mode 100644 index 00000000000..c33ab3c0433 --- /dev/null +++ b/templates/test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml @@ -0,0 +1,40 @@ +- op: add + path: /spec/kubeadmConfigSpec/files/- + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + path: /tmp/oot-cred-provider.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/kubeadmConfigSpec/preKubeadmCommands/- + value: + bash -c /tmp/oot-cred-provider.sh +- op: add + path: /spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/kubeadmConfigSpec/initConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml +- op: add + path: /spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/kubeadmConfigSpec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml diff --git a/templates/test/ci/prow-ci-version/patches/oot-credential-provider-win.yaml b/templates/test/ci/prow-ci-version/patches/oot-credential-provider-win.yaml new file mode 100644 index 00000000000..298e9f67023 --- /dev/null +++ b/templates/test/ci/prow-ci-version/patches/oot-credential-provider-win.yaml @@ -0,0 +1,25 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + $ErrorActionPreference = 'Stop' + + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + powershell C:/oot-cred-provider.ps1 +- op: add + path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml \ No newline at end of file diff --git a/templates/test/ci/prow-ci-version/patches/oot-credential-provider.yaml b/templates/test/ci/prow-ci-version/patches/oot-credential-provider.yaml new file mode 100644 index 00000000000..27ad27e8a89 --- /dev/null +++ b/templates/test/ci/prow-ci-version/patches/oot-credential-provider.yaml @@ -0,0 +1,32 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + path: /tmp/oot-cred-provider.sh + owner: "root:root" + permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + bash -c /tmp/oot-cred-provider.sh +- op: add + path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/template/spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml diff --git a/templates/test/ci/prow-machine-pool-ci-version/kustomization.yaml b/templates/test/ci/prow-machine-pool-ci-version/kustomization.yaml index 625e476b142..64190027e68 100644 --- a/templates/test/ci/prow-machine-pool-ci-version/kustomization.yaml +++ b/templates/test/ci/prow-machine-pool-ci-version/kustomization.yaml @@ -10,6 +10,12 @@ patchesStrategicMerge: - ../patches/machine-pool-worker-counts.yaml - patches/machine-pool-ci-version-windows.yaml patches: +- target: + group: controlplane.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmControlPlane + name: .*-control-plane + path: ../prow-ci-version/patches/oot-credential-provider-kcp.yaml - target: group: controlplane.cluster.x-k8s.io version: v1beta1 diff --git a/templates/test/ci/prow-machine-pool-ci-version/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml b/templates/test/ci/prow-machine-pool-ci-version/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml index 3dbc56c83af..62f4f779848 100644 --- a/templates/test/ci/prow-machine-pool-ci-version/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml +++ b/templates/test/ci/prow-machine-pool-ci-version/patches/kubeadm-bootstrap-windows-k8s-ci-binaries.yaml @@ -28,7 +28,32 @@ kubelet.exe --version path: C:/replace-k8s-binaries.ps1 permissions: "0744" +- op: add + path: /spec/files/- + value: + content: | + $ErrorActionPreference = 'Stop' + + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" - op: add path: /spec/preKubeadmCommands/- value: powershell C:/replace-k8s-binaries.ps1 +- op: add + path: /spec/preKubeadmCommands/- + value: + powershell C:/oot-cred-provider.ps1 +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml \ No newline at end of file diff --git a/templates/test/ci/prow-machine-pool-ci-version/patches/machine-pool-ci-version.yaml b/templates/test/ci/prow-machine-pool-ci-version/patches/machine-pool-ci-version.yaml index acf883ac800..fa40c3f8de0 100644 --- a/templates/test/ci/prow-machine-pool-ci-version/patches/machine-pool-ci-version.yaml +++ b/templates/test/ci/prow-machine-pool-ci-version/patches/machine-pool-ci-version.yaml @@ -3,9 +3,34 @@ kind: KubeadmConfig metadata: name: ${CLUSTER_NAME}-mp-0 spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh files: + - path: /tmp/oot-cred-provider.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml - path: /tmp/kubeadm-bootstrap.sh owner: "root:root" permissions: "0744" diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index c72a102b7c9..a0e9e815c70 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -153,17 +153,38 @@ spec: owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" + - content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" initConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -172,6 +193,7 @@ spec: - bash -c /tmp/replace-k8s-components.sh preKubeadmCommands: - bash -c /tmp/replace-k8s-binaries.sh + - bash -c /tmp/oot-cred-provider.sh verbosity: 5 machineTemplate: infrastructureRef: @@ -274,6 +296,23 @@ spec: - content: | #!/bin/bash + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + set -o nounset set -o pipefail set -o errexit @@ -304,8 +343,11 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -416,12 +458,24 @@ spec: kube-proxy.exe --version path: C:/replace-pr-binaries.ps1 permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" joinConfiguration: nodeRegistration: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.9 name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: @@ -430,6 +484,7 @@ spec: preKubeadmCommands: - powershell c:/create-external-network.ps1 - powershell C:/replace-pr-binaries.ps1 + - powershell C:/oot-cred-provider.ps1 users: - groups: Administrators name: capi diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index 042c973f8dd..24d17685b52 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -106,6 +106,23 @@ spec: - content: | #!/bin/bash + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + set -o nounset set -o pipefail set -o errexit @@ -161,12 +178,16 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -174,6 +195,7 @@ spec: postKubeadmCommands: - bash -c /tmp/replace-k8s-components.sh preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh verbosity: 5 machineTemplate: @@ -288,6 +310,23 @@ spec: - content: | #!/bin/bash + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml + owner: root:root + path: /tmp/oot-cred-provider.sh + permissions: "0744" + - content: | + #!/bin/bash + set -o nounset set -o pipefail set -o errexit @@ -312,8 +351,11 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -428,6 +470,16 @@ spec: sc.exe start sshd path: C:/collect-hns-crashes.ps1 permissions: "0744" + - content: | + $ErrorActionPreference = 'Stop' + + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" - content: | Write-Host "Installing Azure CLI" $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi @@ -479,6 +531,8 @@ spec: azure-container-registry-config: c:/k/azure.json cloud-provider: external feature-gates: ${NODE_FEATURE_GATES:-""} + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml v: "2" windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS name: '{{ ds.meta_data["local_hostname"] }}' @@ -489,6 +543,7 @@ spec: - powershell C:/create-temp-folder.ps1 - powershell C:/replace-containerd.ps1 - powershell C:/collect-hns-crashes.ps1 + - powershell C:/oot-cred-provider.ps1 - powershell C:/install-az-cli.ps1 - powershell C:/replace-pr-binaries.ps1 users: diff --git a/templates/test/dev/custom-builds-machine-pool/kustomization.yaml b/templates/test/dev/custom-builds-machine-pool/kustomization.yaml index d13059993e6..f499bad3e86 100644 --- a/templates/test/dev/custom-builds-machine-pool/kustomization.yaml +++ b/templates/test/dev/custom-builds-machine-pool/kustomization.yaml @@ -6,6 +6,12 @@ patchesStrategicMerge: - patches/machine-pool-deployment-pr-version-windows.yaml - patches/custom-builds.yaml patches: + - target: + group: controlplane.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmControlPlane + name: .*-control-plane + path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml - target: group: bootstrap.cluster.x-k8s.io version: v1beta1 diff --git a/templates/test/dev/custom-builds-machine-pool/patches/custom-builds.yaml b/templates/test/dev/custom-builds-machine-pool/patches/custom-builds.yaml index dd7b0cc1536..6d6b29c4eea 100644 --- a/templates/test/dev/custom-builds-machine-pool/patches/custom-builds.yaml +++ b/templates/test/dev/custom-builds-machine-pool/patches/custom-builds.yaml @@ -3,9 +3,34 @@ kind: KubeadmConfig metadata: name: ${CLUSTER_NAME}-mp-0 spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + image-credential-provider-bin-dir: /var/lib/kubelet/credential-provider + image-credential-provider-config: /var/lib/kubelet/credential-provider-config.yaml preKubeadmCommands: + - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/replace-k8s-binaries.sh files: + - path: /tmp/oot-cred-provider.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + echo "Use OOT credential provider" + mkdir -p /var/lib/kubelet/credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider/acr-credential-provider "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider" + chmod 755 /var/lib/kubelet/credential-provider/acr-credential-provider + curl --retry 10 --retry-delay 5 -Lo /var/lib/kubelet/credential-provider-config.yaml https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config.yaml + chmod 644 /var/lib/kubelet/credential-provider-config.yaml - path: /tmp/replace-k8s-binaries.sh owner: "root:root" permissions: "0744" diff --git a/templates/test/dev/custom-builds-machine-pool/patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml b/templates/test/dev/custom-builds-machine-pool/patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml index e52570e7103..7d954f1deda 100644 --- a/templates/test/dev/custom-builds-machine-pool/patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml +++ b/templates/test/dev/custom-builds-machine-pool/patches/kubeadm-bootstrap-machine-pool-windows-k8s-pr-binaries.yaml @@ -29,7 +29,32 @@ kube-proxy.exe --version path: C:/replace-pr-binaries.ps1 permissions: "0744" +- op: add + path: /spec/files/- + value: + content: | + $ErrorActionPreference = 'Stop' + + echo "Use OOT credential provider" + mkdir C:\var\lib\kubelet\credential-provider + curl.exe --retry 10 --retry-delay 5 -L "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_BLOB_CONTAINER_NAME}/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe" --output C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe + cp C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe C:\var\lib\kubelet\credential-provider\acr-credential-provider + curl.exe --retry 10 --retry-delay 5 -L https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/examples/out-of-tree/credential-provider-config-win.yaml --output C:\var\lib\kubelet\credential-provider-config.yaml + path: C:/oot-cred-provider.ps1 + permissions: "0744" - op: add path: /spec/preKubeadmCommands/- value: powershell C:/replace-pr-binaries.ps1 +- op: add + path: /spec/preKubeadmCommands/- + value: + powershell C:/oot-cred-provider.ps1 +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-bin-dir + value: + /var/lib/kubelet/credential-provider +- op: add + path: /spec/joinConfiguration/nodeRegistration/kubeletExtraArgs/image-credential-provider-config + value: + /var/lib/kubelet/credential-provider-config.yaml \ No newline at end of file diff --git a/templates/test/dev/custom-builds/kustomization.yaml b/templates/test/dev/custom-builds/kustomization.yaml index f45bff7e326..55d225af765 100644 --- a/templates/test/dev/custom-builds/kustomization.yaml +++ b/templates/test/dev/custom-builds/kustomization.yaml @@ -11,6 +11,26 @@ patchesStrategicMerge: - ../../../test/ci/patches/metrics-server-enabled-cluster.yaml - ../../../test/ci/patches/controller-manager-featuregates.yaml patches: +- target: + group: bootstrap.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmConfigTemplate + name: .*-md-0 + namespace: default + path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider.yaml +- target: + group: bootstrap.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmConfigTemplate + name: .*-md-win + namespace: default + path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider-win.yaml +- target: + group: controlplane.cluster.x-k8s.io + version: v1beta1 + kind: KubeadmControlPlane + name: .*-control-plane + path: ../../../test/ci/prow-ci-version/patches/oot-credential-provider-kcp.yaml - target: group: bootstrap.cluster.x-k8s.io version: v1beta1