From a0702cd2edce988676b71b56a0b26f6abfe9a767 Mon Sep 17 00:00:00 2001 From: Jeremy Rickard Date: Thu, 21 Sep 2023 13:20:00 -0600 Subject: [PATCH] Update test/dev/cluster-template-custom-builds.yaml to use managed id and az storage blob download Signed-off-by: Jeremy Rickard Co-authored-by: Danny Brito --- .../dev/cluster-template-custom-builds.yaml | 34 +++++++++++++++++-- ...adm-bootstrap-windows-k8s-pr-binaries.yaml | 25 +++++++++++++- .../patches/kubeadm-bootstrap.yaml | 3 +- .../kubeadm-controlplane-bootstrap.yaml | 3 +- ...machine-deployment-pr-version-windows.yaml | 3 ++ .../machine-deployment-pr-version.yaml | 6 ++++ 6 files changed, 68 insertions(+), 6 deletions(-) diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index 44299c7ab59..a20cfcd5f89 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -111,9 +111,10 @@ spec: systemctl stop kubelet declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity for BINARY in "$${BINARIES[@]}"; do echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login done systemctl restart kubelet @@ -194,6 +195,7 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk + identity: UserAssigned image: marketplace: offer: capi @@ -204,6 +206,8 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -240,6 +244,7 @@ metadata: spec: template: spec: + identity: UserAssigned image: marketplace: offer: capi @@ -250,6 +255,8 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} vmExtensions: - name: CustomScript protectedSettings: @@ -286,9 +293,10 @@ spec: systemctl stop kubelet declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity for BINARY in "$${BINARIES[@]}"; do echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login done systemctl restart kubelet @@ -344,6 +352,7 @@ spec: runtime: containerd windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} spec: + identity: UserAssigned image: marketplace: offer: capi-windows @@ -356,6 +365,8 @@ spec: storageAccountType: Premium_LRS osType: Windows sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 @@ -416,6 +427,16 @@ spec: sc.exe start sshd path: C:/collect-hns-crashes.ps1 permissions: "0744" + - content: | + Write-Host "Installing Azure CLI" + $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + Write-Host "Version Azure CLI:" + az version + path: C:/install-az-cli.ps1 + permissions: "0744" - content: | $ErrorActionPreference = 'Stop' @@ -424,12 +445,18 @@ spec: $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" if($$KUBE_GIT_VERSION -ne "") { + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + # Install Binaries $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" + echo "running az login via identity" + az login --identity foreach ( $$binary in $$binaries ) { echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" + az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login } } @@ -461,6 +488,7 @@ spec: - powershell C:/create-temp-folder.ps1 - powershell C:/replace-containerd.ps1 - powershell C:/collect-hns-crashes.ps1 + - powershell C:/install-az-cli.ps1 - powershell C:/replace-pr-binaries.ps1 users: - groups: Administrators diff --git a/templates/test/dev/custom-builds/patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml b/templates/test/dev/custom-builds/patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml index 0bd59793f47..4ceacf8311c 100644 --- a/templates/test/dev/custom-builds/patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml +++ b/templates/test/dev/custom-builds/patches/kubeadm-bootstrap-windows-k8s-pr-binaries.yaml @@ -1,3 +1,16 @@ +- op: add + path: /spec/template/spec/files/- + value: + content: | + Write-Host "Installing Azure CLI" + $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.53.0.msi -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; Remove-Item .\AzureCLI.msi + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + Write-Host "Version Azure CLI:" + az version + path: C:/install-az-cli.ps1 + permissions: "0744" - op: add path: /spec/template/spec/files/- value: @@ -9,12 +22,18 @@ $$KUBE_GIT_VERSION="${KUBE_GIT_VERSION}" if($$KUBE_GIT_VERSION -ne "") { + # Need to add manually AZ to PATH as it is not added without a reset + Write-Host "Adding AZ to path:" + $env:PATH +=";C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + # Install Binaries $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") $$ci_url="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/windows/amd64" + echo "running az login via identity" + az login --identity foreach ( $$binary in $$binaries ) { echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" + az storage blob download --blob-url "$$ci_url/$$binary.exe" --file "c:/k/$$binary.exe" --auth-mode login } } @@ -29,6 +48,10 @@ kube-proxy.exe --version path: C:/replace-pr-binaries.ps1 permissions: "0744" +- op: add + path: /spec/template/spec/preKubeadmCommands/- + value: + powershell C:/install-az-cli.ps1 - op: add path: /spec/template/spec/preKubeadmCommands/- value: diff --git a/templates/test/dev/custom-builds/patches/kubeadm-bootstrap.yaml b/templates/test/dev/custom-builds/patches/kubeadm-bootstrap.yaml index 42be3a83049..fa7eb66b773 100644 --- a/templates/test/dev/custom-builds/patches/kubeadm-bootstrap.yaml +++ b/templates/test/dev/custom-builds/patches/kubeadm-bootstrap.yaml @@ -10,9 +10,10 @@ systemctl stop kubelet declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity for BINARY in "$${BINARIES[@]}"; do echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login done systemctl restart kubelet diff --git a/templates/test/dev/custom-builds/patches/kubeadm-controlplane-bootstrap.yaml b/templates/test/dev/custom-builds/patches/kubeadm-controlplane-bootstrap.yaml index 85d67a7f03b..37734acfc23 100644 --- a/templates/test/dev/custom-builds/patches/kubeadm-controlplane-bootstrap.yaml +++ b/templates/test/dev/custom-builds/patches/kubeadm-controlplane-bootstrap.yaml @@ -10,9 +10,10 @@ systemctl stop kubelet declare -a BINARIES=("kubeadm" "kubectl" "kubelet") + az login --identity for BINARY in "$${BINARIES[@]}"; do echo "* installing package: $${BINARY} ${KUBE_GIT_VERSION}" - curl --retry 10 --retry-delay 5 "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" --output "/usr/bin/$${BINARY}" + az storage blob download --blob-url "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${JOB_NAME}/${KUBE_GIT_VERSION}/bin/linux/amd64/$${BINARY}" -f "/usr/bin/$${BINARY}" --auth-mode login done systemctl restart kubelet diff --git a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml index 19e44aee327..31e1df753bf 100644 --- a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml +++ b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version-windows.yaml @@ -6,6 +6,9 @@ metadata: spec: template: spec: + identity: UserAssigned + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} image: # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. # latest binaries and images will get replaced to the desired version by the script above. diff --git a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml index 8e9b0bc0233..40cf4c70840 100644 --- a/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml +++ b/templates/test/dev/custom-builds/patches/machine-deployment-pr-version.yaml @@ -5,6 +5,9 @@ metadata: spec: template: spec: + identity: UserAssigned + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} image: # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. # latest binaries and images will get replaced to the desired version by the script above. @@ -21,6 +24,9 @@ metadata: spec: template: spec: + identity: UserAssigned + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} image: # we use the latest image as a workaround there is no published marketplace image for k8s CI versions. # latest binaries and images will get replaced to the desired version by the script above.