From 3497425bfac474cb8aa7446a3c068950a775eed4 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 11 Feb 2021 17:01:30 -0800 Subject: [PATCH] chore: ensure containerd has image cache on VHDs (#4249) --- parts/k8s/cloud-init/artifacts/cse_config.sh | 23 ++-- parts/k8s/cloud-init/artifacts/cse_install.sh | 14 ++- pkg/engine/templates_generated.go | 37 +++--- vhd/packer/install-dependencies.sh | 109 +++++++----------- 4 files changed, 84 insertions(+), 99 deletions(-) diff --git a/parts/k8s/cloud-init/artifacts/cse_config.sh b/parts/k8s/cloud-init/artifacts/cse_config.sh index 5e4df19f30..4cae039b42 100755 --- a/parts/k8s/cloud-init/artifacts/cse_config.sh +++ b/parts/k8s/cloud-init/artifacts/cse_config.sh @@ -304,9 +304,10 @@ enableCRISystemdMonitor() { } {{- if NeedsContainerd}} installContainerd() { + removeMoby local v v=$(containerd -version | cut -d " " -f 3 | sed 's|v||') - if [[ $v != "${CONTAINERD_VERSION}" ]]; then + if [[ $v != "${CONTAINERD_VERSION}"* ]]; then os_lower=$(echo ${OS} | tr '[:upper:]' '[:lower:]') if [[ ${OS} == "${UBUNTU_OS_NAME}" ]]; then url_path="${os_lower}/${UBUNTU_RELEASE}/multiarch/prod" @@ -315,12 +316,7 @@ installContainerd() { else exit 25 fi - removeMoby removeContainerd - retrycmd_no_stats 120 5 25 curl ${MS_APT_REPO}/config/ubuntu/${UBUNTU_RELEASE}/prod.list >/tmp/microsoft-prod.list || exit 25 - retrycmd 10 5 10 cp /tmp/microsoft-prod.list /etc/apt/sources.list.d/ || exit 25 - retrycmd_no_stats 120 5 25 curl ${MS_APT_REPO}/keys/microsoft.asc | gpg --dearmor >/tmp/microsoft.gpg || exit 26 - retrycmd 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit 26 apt_get_update || exit 99 apt_get_install 20 30 120 moby-runc moby-containerd=${CONTAINERD_VERSION}* --allow-downgrades || exit 27 fi @@ -676,14 +672,17 @@ installSGXDrivers() { {{end}} {{- if HasVHDDistroNodes}} cleanUpContainerImages() { - docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${KUBERNETES_VERSION}$|${KUBERNETES_VERSION}-|${KUBERNETES_VERSION}_" | grep 'hyperkube') & - docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${KUBERNETES_VERSION}$|${KUBERNETES_VERSION}-|${KUBERNETES_VERSION}_" | grep 'cloud-controller-manager') & + {{- if NeedsContainerd}} + docker rmi -f $(docker images -a -q) & + {{else}} docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${ETCD_VERSION}$|${ETCD_VERSION}-|${ETCD_VERSION}_" | grep 'etcd') & - docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep 'hcp-tunnel-front') & - docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep 'kube-svc-redirect') & - docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep 'nginx') & - + docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${KUBERNETES_VERSION}$|${KUBERNETES_VERSION}-|${KUBERNETES_VERSION}_" | grep 'kube-proxy') & + docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${KUBERNETES_VERSION}$|${KUBERNETES_VERSION}-|${KUBERNETES_VERSION}_" | grep 'kube-controller-manager') & + docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${KUBERNETES_VERSION}$|${KUBERNETES_VERSION}-|${KUBERNETES_VERSION}_" | grep 'kube-apiserver') & + docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${KUBERNETES_VERSION}$|${KUBERNETES_VERSION}-|${KUBERNETES_VERSION}_" | grep 'kube-scheduler') & docker rmi registry:2.7.1 & + ctr -n=k8s.io image rm $(ctr -n=k8s.io images ls -q) & + {{- end}} } cleanUpGPUDrivers() { rm -Rf $GPU_DEST diff --git a/parts/k8s/cloud-init/artifacts/cse_install.sh b/parts/k8s/cloud-init/artifacts/cse_install.sh index 676a9b2038..d6c5de4890 100755 --- a/parts/k8s/cloud-init/artifacts/cse_install.sh +++ b/parts/k8s/cloud-init/artifacts/cse_install.sh @@ -37,6 +37,10 @@ installDeps() { if [[ ${OS} == "${UBUNTU_OS_NAME}" ]]; then retrycmd_no_stats 120 5 25 curl -fsSL ${MS_APT_REPO}/config/ubuntu/${UBUNTU_RELEASE}/packages-microsoft-prod.deb >/tmp/packages-microsoft-prod.deb || exit 42 retrycmd 60 5 10 dpkg -i /tmp/packages-microsoft-prod.deb || exit 43 + retrycmd_no_stats 120 5 25 curl ${MS_APT_REPO}/config/ubuntu/${UBUNTU_RELEASE}/prod.list >/tmp/microsoft-prod.list || exit 25 + retrycmd 10 5 10 cp /tmp/microsoft-prod.list /etc/apt/sources.list.d/ || exit 25 + retrycmd_no_stats 120 5 25 curl ${MS_APT_REPO}/keys/microsoft.asc | gpg --dearmor >/tmp/microsoft.gpg || exit 26 + retrycmd 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit 26 aptmarkWALinuxAgent hold packages+=" cgroup-lite ceph-common glusterfs-client" if [[ $UBUNTU_RELEASE == "18.04" ]]; then @@ -110,11 +114,6 @@ installMoby() { removeMoby fi if [ -n "${install_pkgs}" ]; then - retrycmd_no_stats 120 5 25 curl ${MS_APT_REPO}/config/ubuntu/${UBUNTU_RELEASE}/prod.list >/tmp/microsoft-prod.list || exit 25 - retrycmd 10 5 10 cp /tmp/microsoft-prod.list /etc/apt/sources.list.d/ || exit 25 - retrycmd_no_stats 120 5 25 curl ${MS_APT_REPO}/keys/microsoft.asc | gpg --dearmor >/tmp/microsoft.gpg || exit 26 - retrycmd 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit 26 - apt_get_update || exit 99 apt_get_install 20 30 120 ${install_pkgs} --allow-downgrades || exit 27 fi } @@ -233,6 +232,11 @@ pullContainerImage() { local cli_tool=$1 url=$2 retrycmd 60 1 1200 $cli_tool pull $url || exit 35 } +loadContainerImage() { + docker pull $1 || exit 35 + docker save $1 | ctr -n=k8s.io images import - || exit 35 + +} overrideNetworkConfig() { CONFIG_FILEPATH="/etc/cloud/cloud.cfg.d/80_azure_net_config.cfg" touch ${CONFIG_FILEPATH} diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index b3ae74ef30..4574f286f2 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -12300,9 +12300,10 @@ enableCRISystemdMonitor() { } {{- if NeedsContainerd}} installContainerd() { + removeMoby local v v=$(containerd -version | cut -d " " -f 3 | sed 's|v||') - if [[ $v != "${CONTAINERD_VERSION}" ]]; then + if [[ $v != "${CONTAINERD_VERSION}"* ]]; then os_lower=$(echo ${OS} | tr '[:upper:]' '[:lower:]') if [[ ${OS} == "${UBUNTU_OS_NAME}" ]]; then url_path="${os_lower}/${UBUNTU_RELEASE}/multiarch/prod" @@ -12311,12 +12312,7 @@ installContainerd() { else exit 25 fi - removeMoby removeContainerd - retrycmd_no_stats 120 5 25 curl ${MS_APT_REPO}/config/ubuntu/${UBUNTU_RELEASE}/prod.list >/tmp/microsoft-prod.list || exit 25 - retrycmd 10 5 10 cp /tmp/microsoft-prod.list /etc/apt/sources.list.d/ || exit 25 - retrycmd_no_stats 120 5 25 curl ${MS_APT_REPO}/keys/microsoft.asc | gpg --dearmor >/tmp/microsoft.gpg || exit 26 - retrycmd 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit 26 apt_get_update || exit 99 apt_get_install 20 30 120 moby-runc moby-containerd=${CONTAINERD_VERSION}* --allow-downgrades || exit 27 fi @@ -12672,14 +12668,17 @@ installSGXDrivers() { {{end}} {{- if HasVHDDistroNodes}} cleanUpContainerImages() { - docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${KUBERNETES_VERSION}$|${KUBERNETES_VERSION}-|${KUBERNETES_VERSION}_" | grep 'hyperkube') & - docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${KUBERNETES_VERSION}$|${KUBERNETES_VERSION}-|${KUBERNETES_VERSION}_" | grep 'cloud-controller-manager') & + {{- if NeedsContainerd}} + docker rmi -f $(docker images -a -q) & + {{else}} docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${ETCD_VERSION}$|${ETCD_VERSION}-|${ETCD_VERSION}_" | grep 'etcd') & - docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep 'hcp-tunnel-front') & - docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep 'kube-svc-redirect') & - docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep 'nginx') & - + docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${KUBERNETES_VERSION}$|${KUBERNETES_VERSION}-|${KUBERNETES_VERSION}_" | grep 'kube-proxy') & + docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${KUBERNETES_VERSION}$|${KUBERNETES_VERSION}-|${KUBERNETES_VERSION}_" | grep 'kube-controller-manager') & + docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${KUBERNETES_VERSION}$|${KUBERNETES_VERSION}-|${KUBERNETES_VERSION}_" | grep 'kube-apiserver') & + docker rmi $(docker images --format '{{OpenBraces}}.Repository{{CloseBraces}}:{{OpenBraces}}.Tag{{CloseBraces}}' | grep -vE "${KUBERNETES_VERSION}$|${KUBERNETES_VERSION}-|${KUBERNETES_VERSION}_" | grep 'kube-scheduler') & docker rmi registry:2.7.1 & + ctr -n=k8s.io image rm $(ctr -n=k8s.io images ls -q) & + {{- end}} } cleanUpGPUDrivers() { rm -Rf $GPU_DEST @@ -13269,6 +13268,10 @@ installDeps() { if [[ ${OS} == "${UBUNTU_OS_NAME}" ]]; then retrycmd_no_stats 120 5 25 curl -fsSL ${MS_APT_REPO}/config/ubuntu/${UBUNTU_RELEASE}/packages-microsoft-prod.deb >/tmp/packages-microsoft-prod.deb || exit 42 retrycmd 60 5 10 dpkg -i /tmp/packages-microsoft-prod.deb || exit 43 + retrycmd_no_stats 120 5 25 curl ${MS_APT_REPO}/config/ubuntu/${UBUNTU_RELEASE}/prod.list >/tmp/microsoft-prod.list || exit 25 + retrycmd 10 5 10 cp /tmp/microsoft-prod.list /etc/apt/sources.list.d/ || exit 25 + retrycmd_no_stats 120 5 25 curl ${MS_APT_REPO}/keys/microsoft.asc | gpg --dearmor >/tmp/microsoft.gpg || exit 26 + retrycmd 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit 26 aptmarkWALinuxAgent hold packages+=" cgroup-lite ceph-common glusterfs-client" if [[ $UBUNTU_RELEASE == "18.04" ]]; then @@ -13342,11 +13345,6 @@ installMoby() { removeMoby fi if [ -n "${install_pkgs}" ]; then - retrycmd_no_stats 120 5 25 curl ${MS_APT_REPO}/config/ubuntu/${UBUNTU_RELEASE}/prod.list >/tmp/microsoft-prod.list || exit 25 - retrycmd 10 5 10 cp /tmp/microsoft-prod.list /etc/apt/sources.list.d/ || exit 25 - retrycmd_no_stats 120 5 25 curl ${MS_APT_REPO}/keys/microsoft.asc | gpg --dearmor >/tmp/microsoft.gpg || exit 26 - retrycmd 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit 26 - apt_get_update || exit 99 apt_get_install 20 30 120 ${install_pkgs} --allow-downgrades || exit 27 fi } @@ -13465,6 +13463,11 @@ pullContainerImage() { local cli_tool=$1 url=$2 retrycmd 60 1 1200 $cli_tool pull $url || exit 35 } +loadContainerImage() { + docker pull $1 || exit 35 + docker save $1 | ctr -n=k8s.io images import - || exit 35 + +} overrideNetworkConfig() { CONFIG_FILEPATH="/etc/cloud/cloud.cfg.d/80_azure_net_config.cfg" touch ${CONFIG_FILEPATH} diff --git a/vhd/packer/install-dependencies.sh b/vhd/packer/install-dependencies.sh index 7b7da25ed5..ab116f2275 100644 --- a/vhd/packer/install-dependencies.sh +++ b/vhd/packer/install-dependencies.sh @@ -85,7 +85,7 @@ echo " - bpftrace" >> ${VHD_LOGS_FILEPATH} MOBY_VERSION="19.03.14" CONTAINERD_VERSION="1.3.9" installMoby -systemctl start docker +systemctl_restart 100 5 30 docker || exit 1 echo " - moby v${MOBY_VERSION}" >> ${VHD_LOGS_FILEPATH} downloadGPUDrivers echo " - nvidia-docker2 nvidia-container-runtime" >> ${VHD_LOGS_FILEPATH} @@ -126,6 +126,7 @@ done installImg echo " - img" >> ${VHD_LOGS_FILEPATH} +systemctl status docker --no-pager || exit 1 echo "Docker images pre-pulled:" >> ${VHD_LOGS_FILEPATH} DASHBOARD_VERSIONS=" @@ -133,7 +134,7 @@ DASHBOARD_VERSIONS=" " for DASHBOARD_VERSION in ${DASHBOARD_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/dashboard:v${DASHBOARD_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -142,7 +143,7 @@ DASHBOARD_METRICS_SCRAPER_VERSIONS=" " for DASHBOARD_METRICS_SCRAPER_VERSION in ${DASHBOARD_METRICS_SCRAPER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/metrics-scraper:v${DASHBOARD_METRICS_SCRAPER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -151,7 +152,7 @@ ADDON_RESIZER_VERSIONS=" " for ADDON_RESIZER_VERSION in ${ADDON_RESIZER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/autoscaler/addon-resizer:${ADDON_RESIZER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -160,7 +161,7 @@ METRICS_SERVER_VERSIONS=" " for METRICS_SERVER_VERSION in ${METRICS_SERVER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/metrics-server:v${METRICS_SERVER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -169,7 +170,7 @@ KUBE_ADDON_MANAGER_VERSIONS=" " for KUBE_ADDON_MANAGER_VERSION in ${KUBE_ADDON_MANAGER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/kube-addon-manager:v${KUBE_ADDON_MANAGER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -177,7 +178,7 @@ MCR_PAUSE_VERSIONS="1.4.0" for PAUSE_VERSION in ${MCR_PAUSE_VERSIONS}; do # Pull the arch independent MCR pause image which is built for Linux and Windows CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/pause:${PAUSE_VERSION}" - pullContainerImage "docker" "${CONTAINER_IMAGE}" + loadContainerImage "${CONTAINER_IMAGE}" echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -189,7 +190,7 @@ CLUSTER_AUTOSCALER_VERSIONS=" " for CLUSTER_AUTOSCALER_VERSION in ${CLUSTER_AUTOSCALER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/autoscaler/cluster-autoscaler:v${CLUSTER_AUTOSCALER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -198,7 +199,7 @@ CORE_DNS_VERSIONS=" " for CORE_DNS_VERSION in ${CORE_DNS_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/coredns:${CORE_DNS_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -207,14 +208,14 @@ RESCHEDULER_VERSIONS=" " for RESCHEDULER_VERSION in ${RESCHEDULER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/rescheduler:v${RESCHEDULER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done VIRTUAL_KUBELET_VERSIONS="1.2.1.2" for VIRTUAL_KUBELET_VERSION in ${VIRTUAL_KUBELET_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/virtual-kubelet/virtual-kubelet:${VIRTUAL_KUBELET_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -224,7 +225,7 @@ AZURE_CNI_NETWORKMONITOR_VERSIONS=" " for AZURE_CNI_NETWORKMONITOR_VERSION in ${AZURE_CNI_NETWORKMONITOR_VERSIONS}; do CONTAINER_IMAGE="${AZURE_CNIIMAGEBASE}/networkmonitor:v${AZURE_CNI_NETWORKMONITOR_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -233,7 +234,7 @@ AZURE_NPM_VERSIONS=" " for AZURE_NPM_VERSION in ${AZURE_NPM_VERSIONS}; do CONTAINER_IMAGE="${AZURE_CNIIMAGEBASE}/azure-npm:v${AZURE_NPM_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -242,21 +243,21 @@ NVIDIA_DEVICE_PLUGIN_VERSIONS=" " for NVIDIA_DEVICE_PLUGIN_VERSION in ${NVIDIA_DEVICE_PLUGIN_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/nvidia/k8s-device-plugin:${NVIDIA_DEVICE_PLUGIN_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done KV_FLEXVOLUME_VERSIONS="0.0.16" for KV_FLEXVOLUME_VERSION in ${KV_FLEXVOLUME_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/k8s/flexvolume/keyvault-flexvolume:v${KV_FLEXVOLUME_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done BLOBFUSE_FLEXVOLUME_VERSIONS="1.0.8" for BLOBFUSE_FLEXVOLUME_VERSION in ${BLOBFUSE_FLEXVOLUME_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/k8s/flexvolume/blobfuse-flexvolume:${BLOBFUSE_FLEXVOLUME_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -265,18 +266,18 @@ IP_MASQ_AGENT_VERSIONS=" " for IP_MASQ_AGENT_VERSION in ${IP_MASQ_AGENT_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/ip-masq-agent:v${IP_MASQ_AGENT_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done KMS_PLUGIN_VERSIONS="0.0.10" for KMS_PLUGIN_VERSION in ${KMS_PLUGIN_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/k8s/kms/keyvault:v${KMS_PLUGIN_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done -pullContainerImage "docker" "busybox" +loadContainerImage "busybox" echo " - busybox" >> ${VHD_LOGS_FILEPATH} K8S_VERSIONS=" @@ -293,35 +294,13 @@ K8S_VERSIONS=" 1.17.16 " for KUBERNETES_VERSION in ${K8S_VERSIONS}; do - if (( $(echo ${KUBERNETES_VERSION} | cut -d"." -f2) < 17 )); then - HYPERKUBE_URL="mcr.microsoft.com/oss/kubernetes/hyperkube:v${KUBERNETES_VERSION}" - extractHyperkube "docker" - echo " - ${HYPERKUBE_URL}" >> ${VHD_LOGS_FILEPATH} - else - for component in kube-apiserver kube-controller-manager kube-proxy kube-scheduler; do - CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/${component}:v${KUBERNETES_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} - echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} - done - KUBE_BINARY_URL="https://kubernetesartifacts.azureedge.net/kubernetes/v${KUBERNETES_VERSION}/binaries/kubernetes-node-linux-amd64.tar.gz" - extractKubeBinaries - fi - if (( $(echo ${KUBERNETES_VERSION} | cut -d"." -f2) < 16 )) && [[ $KUBERNETES_VERSION != *"azs"* ]]; then - CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/cloud-controller-manager:v${KUBERNETES_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + for component in kube-apiserver kube-controller-manager kube-proxy kube-scheduler; do + CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/${component}:v${KUBERNETES_VERSION}" + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} - fi -done - -# Use kube-proxy image instead of hyperkube for kube-proxy container. Fixes #3529. -KUBE_PROXY_VERSIONS=" -1.16.15 -1.16.14 -" -for KUBE_PROXY_VERSION in ${KUBE_PROXY_VERSIONS}; do - CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/kube-proxy:v${KUBE_PROXY_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} - echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} + done + KUBE_BINARY_URL="https://kubernetesartifacts.azureedge.net/kubernetes/v${KUBERNETES_VERSION}/binaries/kubernetes-node-linux-amd64.tar.gz" + extractKubeBinaries done # Starting with 1.16 we pull cloud-controller-manager and cloud-node-manager @@ -331,7 +310,7 @@ CLOUD_MANAGER_VERSIONS=" for CLOUD_MANAGER_VERSION in ${CLOUD_MANAGER_VERSIONS}; do for COMPONENT in azure-cloud-controller-manager azure-cloud-node-manager; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/${COMPONENT}:v${CLOUD_MANAGER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done done @@ -341,7 +320,7 @@ AZUREDISK_CSI_VERSIONS=" " for AZUREDISK_CSI_VERSION in ${AZUREDISK_CSI_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/k8s/csi/azuredisk-csi:v${AZUREDISK_CSI_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -350,7 +329,7 @@ AZUREFILE_CSI_VERSIONS=" " for AZUREFILE_CSI_VERSION in ${AZUREFILE_CSI_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/k8s/csi/azurefile-csi:v${AZUREFILE_CSI_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -359,7 +338,7 @@ CSI_ATTACHER_VERSIONS=" " for CSI_ATTACHER_VERSION in ${CSI_ATTACHER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes-csi/csi-attacher:v${CSI_ATTACHER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -368,7 +347,7 @@ CSI_NODE_DRIVER_REGISTRAR_VERSIONS=" " for CSI_NODE_DRIVER_REGISTRAR_VERSION in ${CSI_NODE_DRIVER_REGISTRAR_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v${CSI_NODE_DRIVER_REGISTRAR_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -378,7 +357,7 @@ CSI_PROVISIONER_VERSIONS=" " for CSI_PROVISIONER_VERSION in ${CSI_PROVISIONER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes-csi/csi-provisioner:v${CSI_PROVISIONER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -387,7 +366,7 @@ LIVENESSPROBE_VERSIONS=" " for LIVENESSPROBE_VERSION in ${LIVENESSPROBE_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v${LIVENESSPROBE_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -396,7 +375,7 @@ CSI_RESIZER_VERSIONS=" " for CSI_RESIZER_VERSION in ${CSI_RESIZER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes-csi/csi-resizer:v${CSI_RESIZER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -406,7 +385,7 @@ CSI_SNAPSHOTTER_VERSIONS=" " for CSI_SNAPSHOTTER_VERSION in ${CSI_SNAPSHOTTER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes-csi/csi-snapshotter:v${CSI_SNAPSHOTTER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -415,7 +394,7 @@ SNAPSHOT_CONTROLLER_VERSIONS=" " for SNAPSHOT_CONTROLLER_VERSION in ${SNAPSHOT_CONTROLLER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes-csi/snapshot-controller:v${SNAPSHOT_CONTROLLER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -424,7 +403,7 @@ CSI_SECRETS_STORE_PROVIDER_AZURE_VERSIONS=" " for CSI_SECRETS_STORE_PROVIDER_AZURE_VERSION in ${CSI_SECRETS_STORE_PROVIDER_AZURE_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/azure/secrets-store/provider-azure:${CSI_SECRETS_STORE_PROVIDER_AZURE_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -433,7 +412,7 @@ CSI_SECRETS_STORE_DRIVER_VERSIONS=" " for CSI_SECRETS_STORE_DRIVER_VERSION in ${CSI_SECRETS_STORE_DRIVER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes-csi/secrets-store/driver:v${CSI_SECRETS_STORE_DRIVER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -442,7 +421,7 @@ AAD_POD_IDENTITY_MIC_VERSIONS=" " for AAD_POD_IDENTITY_MIC_VERSION in ${AAD_POD_IDENTITY_MIC_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/k8s/aad-pod-identity/mic:${AAD_POD_IDENTITY_MIC_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -451,7 +430,7 @@ AAD_POD_IDENTITY_NMI_VERSIONS=" " for AAD_POD_IDENTITY_NMI_VERSION in ${AAD_POD_IDENTITY_NMI_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/k8s/aad-pod-identity/nmi:${AAD_POD_IDENTITY_NMI_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done @@ -460,21 +439,21 @@ CLUSTER_PROPORTIONAL_AUTOSCALER_VERSIONS=" " for CLUSTER_PROPORTIONAL_AUTOSCALER_VERSION in ${CLUSTER_PROPORTIONAL_AUTOSCALER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/autoscaler/cluster-proportional-autoscaler:${CLUSTER_PROPORTIONAL_AUTOSCALER_VERSION}" - pullContainerImage "docker" ${CONTAINER_IMAGE} + loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done # This is to accommodate air-gapped environments, e.g., Azure Stack CONTAINER_IMAGE="registry:2.7.1" -pullContainerImage "docker" ${CONTAINER_IMAGE} +loadContainerImage ${CONTAINER_IMAGE} echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} df -h # warn at 75% space taken [ -s $(df -P | grep '/dev/sda1' | awk '0+$5 >= 75 {print}') ] || echo "WARNING: 75% of /dev/sda1 is used" >> ${VHD_LOGS_FILEPATH} -# error at 90% space taken -[ -s $(df -P | grep '/dev/sda1' | awk '0+$5 >= 90 {print}') ] || exit 1 +# error at 95% space taken +[ -s $(df -P | grep '/dev/sda1' | awk '0+$5 >= 95 {print}') ] || exit 1 echo "Using kernel:" >> ${VHD_LOGS_FILEPATH} tee -a ${VHD_LOGS_FILEPATH} < /proc/version