Merge branch 'master' into chore/cache-new-cni-images

Azure · Apr 11, 2024 · 53bc0b6 · 53bc0b6
2 parents f17c19f + 2647291
commit 53bc0b6
Show file tree

Hide file tree

Showing 194 changed files with 722 additions and 1,029 deletions.
diff --git a/.pipelines/.vsts-vhd-builder-release-windows.yaml b/.pipelines/.vsts-vhd-builder-release-windows.yaml
@@ -40,11 +40,11 @@ parameters:
 - name: build23H2
   displayName: Build 23H2
   type: boolean
-  default: False
+  default: True
 - name: build23H2gen2
   displayName: Build 23H2 Gen 2
   type: boolean
-  default: False
+  default: True
 - name: dryrun
   displayName: Dry run
   type: boolean

diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml
@@ -141,7 +141,7 @@ parameters:
 
 variables:
 - name: CONTAINER_IMAGE
-  value: 'mcr.microsoft.com/oss/azcu/go-dev:v1.34.7'
+  value: 'mcr.microsoft.com/oss/azcu/go-dev:v1.38.1'
 
 pool:
   name: $(POOL_NAME)

diff --git a/.pipelines/.vsts-vhd-builder.yaml b/.pipelines/.vsts-vhd-builder.yaml
@@ -27,7 +27,7 @@ pool:
 
 variables:
 - name: CONTAINER_IMAGE
-  value: 'mcr.microsoft.com/oss/azcu/go-dev:v1.34.7'
+  value: 'mcr.microsoft.com/oss/azcu/go-dev:v1.38.1'
 
 stages:
   - stage: build_vhd

diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml
@@ -73,9 +73,6 @@ steps:
         -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
         -w /go/src/github.com/Azure/AgentBaker \
         -e POOL_NAME=$(POOL_NAME) \
-        -e CLIENT_ID=${CLIENT_ID} \
-        -e CLIENT_SECRET="$(CLIENT_SECRET)" \
-        -e TENANT_ID=${TENANT_ID} \
         -e SUBSCRIPTION_ID="${SUBSCRIPTION_ID}" \
         -e AZURE_VM_SIZE=$(AZURE_VM_SIZE) \
         -e AZURE_RESOURCE_GROUP_NAME=${AZURE_RESOURCE_GROUP_NAME} \
@@ -117,9 +114,6 @@ steps:
         docker run --rm \
         -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
         -w /go/src/github.com/Azure/AgentBaker \
-        -e CLIENT_ID=${CLIENT_ID} \
-        -e CLIENT_SECRET="$(CLIENT_SECRET)" \
-        -e TENANT_ID=${TENANT_ID} \
         -e SUBSCRIPTION_ID="${SUBSCRIPTION_ID}" \
         -e AZURE_RESOURCE_GROUP_NAME=${AZURE_RESOURCE_GROUP_NAME} \
         -e OS_DISK_URI=${OS_DISK_URI} \
@@ -170,9 +164,6 @@ steps:
       docker run --rm \
       -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
       -w /go/src/github.com/Azure/AgentBaker \
-      -e CLIENT_ID=${CLIENT_ID} \
-      -e CLIENT_SECRET="$(CLIENT_SECRET)" \
-      -e TENANT_ID=${TENANT_ID} \
       -e SUBSCRIPTION_ID="${SUBSCRIPTION_ID}" \
       -e PKR_RG_NAME=${PKR_RG_NAME} \
       -e MODE=${MODE} \
@@ -201,9 +192,6 @@ steps:
         docker run --rm \
         -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
         -w /go/src/github.com/Azure/AgentBaker \
-        -e CLIENT_ID=${CLIENT_ID} \
-        -e CLIENT_SECRET="$(CLIENT_SECRET)" \
-        -e TENANT_ID=${TENANT_ID} \
         -e SUBSCRIPTION_ID="${SUBSCRIPTION_ID}" \
         -e RESOURCE_GROUP_NAME="${AZURE_RESOURCE_GROUP_NAME}" \
         -e LOCATION="${AZURE_LOCATION}" \
@@ -229,9 +217,6 @@ steps:
         docker run --rm \
         -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
         -w /go/src/github.com/Azure/AgentBaker \
-        -e CLIENT_ID=${CLIENT_ID} \
-        -e CLIENT_SECRET="$(CLIENT_SECRET)" \
-        -e TENANT_ID=${TENANT_ID} \
         -e SUBSCRIPTION_ID="${SUBSCRIPTION_ID}" \
         -e RESOURCE_GROUP_NAME="${AZURE_RESOURCE_GROUP_NAME}" \
         -e SIG_IMAGE_NAME=${SIG_IMAGE_NAME} \

diff --git a/e2e/scenario/base_vhd_catalog.json b/e2e/scenario/base_vhd_catalog.json
@@ -2,17 +2,17 @@
     "ubuntu1804": {
         "gen2containerd": {
             "artifactName": "1804-gen2-containerd",
-            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/1804Gen2/versions/1.1711058371.9154"
+            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/1804Gen2/versions/1.1712787825.26381"
         }
     },
     "ubuntu2204": {
         "gen2arm64containerd": {
             "artifactName": "2204-arm64-gen2-containerd",
-            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/2204Gen2Arm64/versions/1.1711050975.24443"
+            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/2204Gen2Arm64/versions/1.1712787796.17656"
         },
         "gen2containerd": {
             "artifactName": "2204-gen2-containerd",
-            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/2204Gen2/versions/1.1711058378.5173"
+            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/2204Gen2/versions/1.1712787789.1362"
         },
         "gen2containerdprivatekubepkg": {
             "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/2204Gen2/versions/1.1704411049.2812"
@@ -21,21 +21,21 @@
     "azurelinuxv2": {
         "gen2arm64": {
             "artifactName": "azurelinuxv2-gen2-arm64",
-            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/AzureLinuxV2Gen2Arm64/versions/1.1711050992.12578"
+            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/AzureLinuxV2Gen2Arm64/versions/1.1712787785.1412"
         },
         "gen2": {
             "artifactName": "azurelinuxv2-gen2",
-            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/AzureLinuxV2Gen2/versions/1.1711058377.30895"
+            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/AzureLinuxV2Gen2/versions/1.1712787807.28137"
         }
     },
     "cblmarinerv2": {
         "gen2arm64": {
             "artifactName": "marinerv2-gen2-arm64",
-            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/CBLMarinerV2Gen2Arm64/versions/1.1711051003.16083"
+            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/CBLMarinerV2Gen2Arm64/versions/1.1712787820.21615"
         },
         "gen2": {
             "artifactName": "marinerv2-gen2",
-            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/CBLMarinerV2Gen2/versions/1.1711051012.17839"
+            "resourceId": "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/CBLMarinerV2Gen2/versions/1.1712787804.29363"
         }
     }
 }
diff --git a/parts/linux/cloud-init/artifacts/manifest.json b/parts/linux/cloud-init/artifacts/manifest.json
@@ -38,13 +38,8 @@
         "downloadLocation": "",
         "downloadURL": "https://acs-mirror.azureedge.net/kubernetes/v${PATCHED_KUBE_BINARY_VERSION}/binaries/kubernetes-node-linux-${CPU_ARCH}.tar.gz",
         "versions": [
-            "1.26.6",
-            "1.26.10",
-            "1.26.12",
-            "1.27.3",
             "1.27.7",
             "1.27.9",
-            "1.28.1",
             "1.28.3",
             "1.28.5",
             "1.29.0",

diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go
@@ -108,6 +108,8 @@ var AvailableUbuntu2204Distros = []Distro{
 	AKSUbuntuEdgeZoneContainerd2204Gen2,
 	AKSUbuntuMinimalContainerd2204,
 	AKSUbuntuMinimalContainerd2204Gen2,
+	AKSUbuntuFipsContainerd2204,
+	AKSUbuntuFipsContainerd2204Gen2,
 }
 
 //nolint:gochecknoglobals
@@ -120,6 +122,8 @@ var AvailableContainerdDistros = []Distro{
 	AKSUbuntuFipsContainerd1804Gen2,
 	AKSUbuntuFipsContainerd2004,
 	AKSUbuntuFipsContainerd2004Gen2,
+	AKSUbuntuFipsContainerd2204,
+	AKSUbuntuFipsContainerd2204Gen2,
 	AKSUbuntuEdgeZoneContainerd1804,
 	AKSUbuntuEdgeZoneContainerd1804Gen2,
 	AKSCBLMarinerV1,
@@ -165,6 +169,7 @@ var AvailableGen2Distros = []Distro{
 	AKSUbuntuGPUContainerd1804Gen2,
 	AKSUbuntuFipsContainerd1804Gen2,
 	AKSUbuntuFipsContainerd2004Gen2,
+	AKSUbuntuFipsContainerd2204Gen2,
 	AKSUbuntuEdgeZoneContainerd1804Gen2,
 	AKSUbuntuArm64Containerd2204Gen2,
 	AKSUbuntuContainerd2204Gen2,
@@ -477,6 +482,20 @@ var (
 		Version:       LinuxSIGImageVersion,
 	}
 
+	SIGUbuntuFipsContainerd2204ImageConfigTemplate = SigImageConfigTemplate{
+		ResourceGroup: AKSUbuntuResourceGroup,
+		Gallery:       AKSUbuntuGalleryName,
+		Definition:    "2204fipscontainerd",
+		Version:       "202404.09.0", // TODO(artunduman): Update version when the image is ready
+	}
+
+	SIGUbuntuFipsContainerd2204Gen2ImageConfigTemplate = SigImageConfigTemplate{
+		ResourceGroup: AKSUbuntuResourceGroup,
+		Gallery:       AKSUbuntuGalleryName,
+		Definition:    "2204gen2fipscontainerd",
+		Version:       "202404.09.0", // TODO(artunduman): Update version when the image is ready
+	}
+
 	SIGUbuntuArm64Containerd2204Gen2ImageConfigTemplate = SigImageConfigTemplate{
 		ResourceGroup: AKSUbuntuResourceGroup,
 		Gallery:       AKSUbuntuGalleryName,
@@ -703,6 +722,8 @@ func getSigUbuntuImageConfigMapWithOpts(opts ...SigImageConfigOpt) map[Distro]Si
 		AKSUbuntuFipsContainerd1804Gen2:    SIGUbuntuFipsContainerd1804Gen2ImageConfigTemplate.WithOptions(opts...),
 		AKSUbuntuFipsContainerd2004:        SIGUbuntuFipsContainerd2004ImageConfigTemplate.WithOptions(opts...),
 		AKSUbuntuFipsContainerd2004Gen2:    SIGUbuntuFipsContainerd2004Gen2ImageConfigTemplate.WithOptions(opts...),
+		AKSUbuntuFipsContainerd2204:        SIGUbuntuFipsContainerd2204ImageConfigTemplate.WithOptions(opts...),
+		AKSUbuntuFipsContainerd2204Gen2:    SIGUbuntuFipsContainerd2204Gen2ImageConfigTemplate.WithOptions(opts...),
 		AKSUbuntuContainerd2204:            SIGUbuntuContainerd2204ImageConfigTemplate.WithOptions(opts...),
 		AKSUbuntuContainerd2204Gen2:        SIGUbuntuContainerd2204Gen2ImageConfigTemplate.WithOptions(opts...),
 		AKSUbuntuContainerd2004CVMGen2:     SIGUbuntuContainerd2004CVMGen2ImageConfigTemplate.WithOptions(opts...),

diff --git a/pkg/agent/datamodel/sig_config_test.go b/pkg/agent/datamodel/sig_config_test.go
@@ -47,7 +47,7 @@ var _ = Describe("GetSIGAzureCloudSpecConfig", func() {
 		Expect(sigConfig.SigTenantID).To(Equal("sometenantid"))
 		Expect(sigConfig.SubscriptionID).To(Equal("somesubid"))
 
-		Expect(len(sigConfig.SigUbuntuImageConfig)).To(Equal(21))
+		Expect(len(sigConfig.SigUbuntuImageConfig)).To(Equal(23))
 
 		aksUbuntuGPU1804Gen2 := sigConfig.SigUbuntuImageConfig[AKSUbuntuGPU1804Gen2]
 		Expect(aksUbuntuGPU1804Gen2.ResourceGroup).To(Equal("resourcegroup"))

diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go
@@ -172,6 +172,8 @@ const (
 	AKSUbuntuFipsContainerd1804Gen2     Distro = "aks-ubuntu-fips-containerd-18.04-gen2"
 	AKSUbuntuFipsContainerd2004         Distro = "aks-ubuntu-fips-containerd-20.04"
 	AKSUbuntuFipsContainerd2004Gen2     Distro = "aks-ubuntu-fips-containerd-20.04-gen2"
+	AKSUbuntuFipsContainerd2204         Distro = "aks-ubuntu-fips-containerd-22.04"
+	AKSUbuntuFipsContainerd2204Gen2     Distro = "aks-ubuntu-fips-containerd-22.04-gen2"
 	AKSUbuntuEdgeZoneContainerd1804     Distro = "aks-ubuntu-edgezone-containerd-18.04"
 	AKSUbuntuEdgeZoneContainerd1804Gen2 Distro = "aks-ubuntu-edgezone-containerd-18.04-gen2"
 	AKSUbuntuEdgeZoneContainerd2204     Distro = "aks-ubuntu-edgezone-containerd-22.04"
@@ -246,6 +248,8 @@ var AKSDistrosAvailableOnVHD = []Distro{
 	AKSUbuntuFipsContainerd1804Gen2,
 	AKSUbuntuFipsContainerd2004,
 	AKSUbuntuFipsContainerd2004Gen2,
+	AKSUbuntuFipsContainerd2204,
+	AKSUbuntuFipsContainerd2204Gen2,
 	AKSUbuntuEdgeZoneContainerd1804,
 	AKSUbuntuEdgeZoneContainerd1804Gen2,
 	AKSUbuntuEdgeZoneContainerd2204,

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData
@@ -74,7 +74,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/8STTW/TMBzG7/0UUegNYi+FrS+3tDVaRUmnrEFCCEVO7G0WqV05Tgur8t2RDW3S4mYIJu36+Pm//7zrOI7juJngCjNOJXFHzi/N6HcspyFeUXfkuCuR/vBqY9LdTRbhMpiFKJomn1B0O1uE1Wv8WErqHb3dBMvJ9cEBCE3dN3UJIrY8F5jMRYYVE1yXgmKtYF0K7j2FLTCO5jrmQal1MYJQtwnSXKQgE5KCLeNEbAvAqTJP8GQM2DIG7O7icRwu42SymKIw+IgqmDNeftez38RJEE2uq9OErXsp05Kr8pA2QnMU3KKqbNlXs9bp7jZUFkzwwh05X7429DXjnB6f0uj+4OKd3pUP+sD3fPfwWjWCKbmnB9Pl3vXb4cqSZ08woi1JdxfF4eTc/Obt7+c8y4gu9fx0mKzWAWDKBGfZOQqef/J/v7AP/J79wIwXCue5LZzQO1zmyprhCAO+YYThGns9umIGgjNoPHXPluOdXYitJSKZ9rxgI9/KlEpOFW1povZ4XBDqGaA8LLMHoLAE94//0eWedZwV3opJKSQwDOqPbUivi8NNd2dgRNPkQzxGyXgWBtHnA5owZRxLRgtob7hJ7p99N1d0ginoXYGrhrdW/Qu73LPIffDWqvat6tCiDoBvVW15B+DSog6BreEhaHydYzwSRVfrHKsX+SudqvMKLd53fgYAAP//bupWl/gHAAA=
+    H4sIAAAAAAAA/8STUW/TMBSF3/srotA3iL0Mpm59S1ujVZR0yhokhFDkxN5mkdqV47SwKv8d2dAkLU6GYNJej49977n3837gOI7jZoIrzDiVxB07vzSj37GchnhN3bHjrkX6w2uMyXA/XYarYB6iaJZ8QtHtfBlWr/FjKal3dHYTrKbXtQMQmrpvmhJE7HguMFmIDCsmuC4FxUbBphQ8eArbxTha6DsPSm2KMYS6TZDmIgWZkBTsGCdiVwBOlTmCJzFgTww43MeTOFzFyXQ5Q2HwEVUwZ7z8rrPfxEkQTa+r0wd751KmJVdl/WyEFii4RVXZM692rdPZbaksmOCFO3a+fG3pG8Y5PV6l0f3Ls3d6Vj4YAd/z3fq0al2m5J7WpouD67fDlSXPnmBEW5LhPorDaVd+c/b3OTsZ0aWenw7zqjUATJngLOui4PmT//uGfeCf2xfMeKFwntuuE3qHy1xZXzjCgG8ZYbjBXkdXzEDQgcZT++xZXudAbC0RybTnBRv5VqZUcqpoTxONx+OCUM8A5WGZPQCFJbh//I8uD6zjrPDWTEohgWFQf2xDelMcbod7AyOaJR/iCUom8zCIPtdowpRxLBktoL3hNrl/9t0e0Qmm4HwERi1vo15Z1Evw1qpeWNQrcGZVWygfrytRdL3JsXoRdgfV4BVavh/8DAAA//+ilP/giAcAAA==
 
 - path: /opt/azure/containers/init-aks-custom-cloud.sh
   permissions: "0744"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/line77.sh b/pkg/agent/testdata/AKSUbuntu1604+Containerd/line77.sh
@@ -38,13 +38,8 @@
         "downloadLocation": "",
         "downloadURL": "https://acs-mirror.azureedge.net/kubernetes/v${PATCHED_KUBE_BINARY_VERSION}/binaries/kubernetes-node-linux-${CPU_ARCH}.tar.gz",
         "versions": [
-            "1.26.6",
-            "1.26.10",
-            "1.26.12",
-            "1.27.3",
             "1.27.7",
             "1.27.9",
-            "1.28.1",
             "1.28.3",
             "1.28.5",
             "1.29.0",

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData
@@ -74,7 +74,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/8STTW/TMBzG7/0UUegNYi+FrS+3tDVaRUmnrEFCCEVO7G0WqV05Tgur8t2RDW3S4mYIJu36+Pm//7zrOI7juJngCjNOJXFHzi/N6HcspyFeUXfkuCuR/vBqY9LdTRbhMpiFKJomn1B0O1uE1Wv8WErqHb3dBMvJ9cEBCE3dN3UJIrY8F5jMRYYVE1yXgmKtYF0K7j2FLTCO5jrmQal1MYJQtwnSXKQgE5KCLeNEbAvAqTJP8GQM2DIG7O7icRwu42SymKIw+IgqmDNeftez38RJEE2uq9OErXsp05Kr8pA2QnMU3KKqbNlXs9bp7jZUFkzwwh05X7429DXjnB6f0uj+4OKd3pUP+sD3fPfwWjWCKbmnB9Pl3vXb4cqSZ08woi1JdxfF4eTc/Obt7+c8y4gu9fx0mKzWAWDKBGfZOQqef/J/v7AP/J79wIwXCue5LZzQO1zmyprhCAO+YYThGns9umIGgjNoPHXPluOdXYitJSKZ9rxgI9/KlEpOFW1povZ4XBDqGaA8LLMHoLAE94//0eWedZwV3opJKSQwDOqPbUivi8NNd2dgRNPkQzxGyXgWBtHnA5owZRxLRgtob7hJ7p99N1d0ginoXYGrhrdW/Qu73LPIffDWqvat6tCiDoBvVW15B+DSog6BreEhaHydYzwSRVfrHKsX+SudqvMKLd53fgYAAP//bupWl/gHAAA=
+    H4sIAAAAAAAA/8STUW/TMBSF3/srotA3iL0Mpm59S1ujVZR0yhokhFDkxN5mkdqV47SwKv8d2dAkLU6GYNJej49977n3837gOI7jZoIrzDiVxB07vzSj37GchnhN3bHjrkX6w2uMyXA/XYarYB6iaJZ8QtHtfBlWr/FjKal3dHYTrKbXtQMQmrpvmhJE7HguMFmIDCsmuC4FxUbBphQ8eArbxTha6DsPSm2KMYS6TZDmIgWZkBTsGCdiVwBOlTmCJzFgTww43MeTOFzFyXQ5Q2HwEVUwZ7z8rrPfxEkQTa+r0wd751KmJVdl/WyEFii4RVXZM692rdPZbaksmOCFO3a+fG3pG8Y5PV6l0f3Ls3d6Vj4YAd/z3fq0al2m5J7WpouD67fDlSXPnmBEW5LhPorDaVd+c/b3OTsZ0aWenw7zqjUATJngLOui4PmT//uGfeCf2xfMeKFwntuuE3qHy1xZXzjCgG8ZYbjBXkdXzEDQgcZT++xZXudAbC0RybTnBRv5VqZUcqpoTxONx+OCUM8A5WGZPQCFJbh//I8uD6zjrPDWTEohgWFQf2xDelMcbod7AyOaJR/iCUom8zCIPtdowpRxLBktoL3hNrl/9t0e0Qmm4HwERi1vo15Z1Evw1qpeWNQrcGZVWygfrytRdL3JsXoRdgfV4BVavh/8DAAA//+ilP/giAcAAA==
 
 - path: /opt/azure/containers/init-aks-custom-cloud.sh
   permissions: "0744"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line77.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line77.sh
@@ -38,13 +38,8 @@
         "downloadLocation": "",
         "downloadURL": "https://acs-mirror.azureedge.net/kubernetes/v${PATCHED_KUBE_BINARY_VERSION}/binaries/kubernetes-node-linux-${CPU_ARCH}.tar.gz",
         "versions": [
-            "1.26.6",
-            "1.26.10",
-            "1.26.12",
-            "1.27.3",
             "1.27.7",
             "1.27.9",
-            "1.28.1",
             "1.28.3",
             "1.28.5",
             "1.29.0",

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData
@@ -74,7 +74,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/8STTW/TMBzG7/0UUegNYi+FrS+3tDVaRUmnrEFCCEVO7G0WqV05Tgur8t2RDW3S4mYIJu36+Pm//7zrOI7juJngCjNOJXFHzi/N6HcspyFeUXfkuCuR/vBqY9LdTRbhMpiFKJomn1B0O1uE1Wv8WErqHb3dBMvJ9cEBCE3dN3UJIrY8F5jMRYYVE1yXgmKtYF0K7j2FLTCO5jrmQal1MYJQtwnSXKQgE5KCLeNEbAvAqTJP8GQM2DIG7O7icRwu42SymKIw+IgqmDNeftez38RJEE2uq9OErXsp05Kr8pA2QnMU3KKqbNlXs9bp7jZUFkzwwh05X7429DXjnB6f0uj+4OKd3pUP+sD3fPfwWjWCKbmnB9Pl3vXb4cqSZ08woi1JdxfF4eTc/Obt7+c8y4gu9fx0mKzWAWDKBGfZOQqef/J/v7AP/J79wIwXCue5LZzQO1zmyprhCAO+YYThGns9umIGgjNoPHXPluOdXYitJSKZ9rxgI9/KlEpOFW1povZ4XBDqGaA8LLMHoLAE94//0eWedZwV3opJKSQwDOqPbUivi8NNd2dgRNPkQzxGyXgWBtHnA5owZRxLRgtob7hJ7p99N1d0ginoXYGrhrdW/Qu73LPIffDWqvat6tCiDoBvVW15B+DSog6BreEhaHydYzwSRVfrHKsX+SudqvMKLd53fgYAAP//bupWl/gHAAA=
+    H4sIAAAAAAAA/8STUW/TMBSF3/srotA3iL0Mpm59S1ujVZR0yhokhFDkxN5mkdqV47SwKv8d2dAkLU6GYNJej49977n3837gOI7jZoIrzDiVxB07vzSj37GchnhN3bHjrkX6w2uMyXA/XYarYB6iaJZ8QtHtfBlWr/FjKal3dHYTrKbXtQMQmrpvmhJE7HguMFmIDCsmuC4FxUbBphQ8eArbxTha6DsPSm2KMYS6TZDmIgWZkBTsGCdiVwBOlTmCJzFgTww43MeTOFzFyXQ5Q2HwEVUwZ7z8rrPfxEkQTa+r0wd751KmJVdl/WyEFii4RVXZM692rdPZbaksmOCFO3a+fG3pG8Y5PV6l0f3Ls3d6Vj4YAd/z3fq0al2m5J7WpouD67fDlSXPnmBEW5LhPorDaVd+c/b3OTsZ0aWenw7zqjUATJngLOui4PmT//uGfeCf2xfMeKFwntuuE3qHy1xZXzjCgG8ZYbjBXkdXzEDQgcZT++xZXudAbC0RybTnBRv5VqZUcqpoTxONx+OCUM8A5WGZPQCFJbh//I8uD6zjrPDWTEohgWFQf2xDelMcbod7AyOaJR/iCUom8zCIPtdowpRxLBktoL3hNrl/9t0e0Qmm4HwERi1vo15Z1Evw1qpeWNQrcGZVWygfrytRdL3JsXoRdgfV4BVavh/8DAAA//+ilP/giAcAAA==
 
 - path: /opt/azure/containers/init-aks-custom-cloud.sh
   permissions: "0744"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line77.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line77.sh
@@ -38,13 +38,8 @@
         "downloadLocation": "",
         "downloadURL": "https://acs-mirror.azureedge.net/kubernetes/v${PATCHED_KUBE_BINARY_VERSION}/binaries/kubernetes-node-linux-${CPU_ARCH}.tar.gz",
         "versions": [
-            "1.26.6",
-            "1.26.10",
-            "1.26.12",
-            "1.27.3",
             "1.27.7",
             "1.27.9",
-            "1.28.1",
             "1.28.3",
             "1.28.5",
             "1.29.0",

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData
@@ -74,7 +74,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/8STTW/TMBzG7/0UUegNYi+FrS+3tDVaRUmnrEFCCEVO7G0WqV05Tgur8t2RDW3S4mYIJu36+Pm//7zrOI7juJngCjNOJXFHzi/N6HcspyFeUXfkuCuR/vBqY9LdTRbhMpiFKJomn1B0O1uE1Wv8WErqHb3dBMvJ9cEBCE3dN3UJIrY8F5jMRYYVE1yXgmKtYF0K7j2FLTCO5jrmQal1MYJQtwnSXKQgE5KCLeNEbAvAqTJP8GQM2DIG7O7icRwu42SymKIw+IgqmDNeftez38RJEE2uq9OErXsp05Kr8pA2QnMU3KKqbNlXs9bp7jZUFkzwwh05X7429DXjnB6f0uj+4OKd3pUP+sD3fPfwWjWCKbmnB9Pl3vXb4cqSZ08woi1JdxfF4eTc/Obt7+c8y4gu9fx0mKzWAWDKBGfZOQqef/J/v7AP/J79wIwXCue5LZzQO1zmyprhCAO+YYThGns9umIGgjNoPHXPluOdXYitJSKZ9rxgI9/KlEpOFW1povZ4XBDqGaA8LLMHoLAE94//0eWedZwV3opJKSQwDOqPbUivi8NNd2dgRNPkQzxGyXgWBtHnA5owZRxLRgtob7hJ7p99N1d0ginoXYGrhrdW/Qu73LPIffDWqvat6tCiDoBvVW15B+DSog6BreEhaHydYzwSRVfrHKsX+SudqvMKLd53fgYAAP//bupWl/gHAAA=
+    H4sIAAAAAAAA/8STUW/TMBSF3/srotA3iL0Mpm59S1ujVZR0yhokhFDkxN5mkdqV47SwKv8d2dAkLU6GYNJej49977n3837gOI7jZoIrzDiVxB07vzSj37GchnhN3bHjrkX6w2uMyXA/XYarYB6iaJZ8QtHtfBlWr/FjKal3dHYTrKbXtQMQmrpvmhJE7HguMFmIDCsmuC4FxUbBphQ8eArbxTha6DsPSm2KMYS6TZDmIgWZkBTsGCdiVwBOlTmCJzFgTww43MeTOFzFyXQ5Q2HwEVUwZ7z8rrPfxEkQTa+r0wd751KmJVdl/WyEFii4RVXZM692rdPZbaksmOCFO3a+fG3pG8Y5PV6l0f3Ls3d6Vj4YAd/z3fq0al2m5J7WpouD67fDlSXPnmBEW5LhPorDaVd+c/b3OTsZ0aWenw7zqjUATJngLOui4PmT//uGfeCf2xfMeKFwntuuE3qHy1xZXzjCgG8ZYbjBXkdXzEDQgcZT++xZXudAbC0RybTnBRv5VqZUcqpoTxONx+OCUM8A5WGZPQCFJbh//I8uD6zjrPDWTEohgWFQf2xDelMcbod7AyOaJR/iCUom8zCIPtdowpRxLBktoL3hNrl/9t0e0Qmm4HwERi1vo15Z1Evw1qpeWNQrcGZVWygfrytRdL3JsXoRdgfV4BVavh/8DAAA//+ilP/giAcAAA==
 
 - path: /opt/azure/containers/init-aks-custom-cloud.sh
   permissions: "0744"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line77.sh b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line77.sh
@@ -38,13 +38,8 @@
         "downloadLocation": "",
         "downloadURL": "https://acs-mirror.azureedge.net/kubernetes/v${PATCHED_KUBE_BINARY_VERSION}/binaries/kubernetes-node-linux-${CPU_ARCH}.tar.gz",
         "versions": [
-            "1.26.6",
-            "1.26.10",
-            "1.26.12",
-            "1.27.3",
             "1.27.7",
             "1.27.9",
-            "1.28.1",
             "1.28.3",
             "1.28.5",
             "1.29.0",