From a2784937af8d92c2a02061b2edad9940531fc17f Mon Sep 17 00:00:00 2001
From: Hongkai Liu <hongkailiu@users.noreply.github.com>
Date: Fri, 29 Sep 2023 20:10:53 -0400
Subject: [PATCH 1/6] Add logs upon ignoring disabled clusters

---
 prow/kube/config.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/prow/kube/config.go b/prow/kube/config.go
index 7e3da5d52868..9e98f6a8e1c4 100644
--- a/prow/kube/config.go
+++ b/prow/kube/config.go
@@ -143,6 +143,7 @@ func LoadClusterConfigs(opts *Options) (map[string]rest.Config, error) {
 
 	for _, disabledCluster := range opts.disabledClusters.UnsortedList() {
 		delete(allKubeCfgs, disabledCluster)
+		logrus.WithField("disabledCluster", disabledCluster).Info("Removed kubeconfig for disabled cluster")
 	}
 
 	if opts.noInClusterConfig {

From b36e53472b352291626f0dc3513916881b236497 Mon Sep 17 00:00:00 2001
From: Arnaud Meukam <ameukam@gmail.com>
Date: Sat, 30 Sep 2023 16:38:49 +0200
Subject: [PATCH 2/6] k8s-infra: Add secret for kOps build cluster

Related to:
 - https://github.com/kubernetes/k8s.io/issues/5127

Add an ExternelSecret that will sync the kubeconfig for the cluster
k8s-infra-kops-prow-build to the k8s-prow cluster.

Signed-off-by: Arnaud Meukam <ameukam@gmail.com>
---
 .../prow/cluster/kubernetes_external_secrets.yaml   | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/config/prow/cluster/kubernetes_external_secrets.yaml b/config/prow/cluster/kubernetes_external_secrets.yaml
index 04a471e36a53..9a63c3f1d2ec 100644
--- a/config/prow/cluster/kubernetes_external_secrets.yaml
+++ b/config/prow/cluster/kubernetes_external_secrets.yaml
@@ -143,3 +143,16 @@ spec:
   - key: eks-prow-build-cluster-kubeconfig
     name: kubeconfig
     version: latest
+---
+apiVersion: kubernetes-client.io/v1
+kind: ExternalSecret
+metadata:
+  name: kubeconfig-k8s-infra-kops-prow-build
+  namespace: default
+spec:
+  backendType: gcpSecretsManager
+  projectId: k8s-infra-prow-build-trusted
+  data:
+  - key: k8s-infra-kops-prow-build-kubeconfig
+    name: kubeconfig
+    version: latest

From 9aa814e33f3bb8d7ff811b01d49279b4fb0561f4 Mon Sep 17 00:00:00 2001
From: rjsadow <richard.j.sadowski@gmail.com>
Date: Wed, 27 Sep 2023 15:55:05 -0400
Subject: [PATCH 3/6] migrate `sig-autoscaling` jobs to community cluster

Signed-off-by: rjsadow <richard.j.sadowski@gmail.com>
---
 .../sig-autoscaling-config.yaml               | 82 ++++++++++++++++++-
 .../sig-autoscaling-presubmits.yaml           | 25 +++++-
 2 files changed, 102 insertions(+), 5 deletions(-)

diff --git a/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-config.yaml b/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-config.yaml
index 5204fc053676..1c6490f7eab9 100644
--- a/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-config.yaml
+++ b/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-config.yaml
@@ -1,5 +1,6 @@
 periodics:
 - name: ci-kubernetes-e2e-autoscaling-vpa-actuation
+  cluster: k8s-infra-prow-build
   interval: 2h
   labels:
     preset-service-account: "true"
@@ -33,11 +34,19 @@ periodics:
       # docker-in-docker needs privileged mode
       securityContext:
         privileged: true
+      resources:
+        limits:
+          cpu: 2
+          memory: 6Gi
+        requests:
+          cpu: 2
+          memory: 6Gi
 
   annotations:
     testgrid-dashboards: sig-autoscaling-vpa
     testgrid-tab-name: autoscaling-vpa-actuation
 - name: ci-kubernetes-e2e-autoscaling-vpa-admission-controller
+  cluster: k8s-infra-prow-build
   interval: 2h
   labels:
     preset-service-account: "true"
@@ -71,11 +80,19 @@ periodics:
       # docker-in-docker needs privileged mode
       securityContext:
         privileged: true
+      resources:
+        limits:
+          cpu: 2
+          memory: 6Gi
+        requests:
+          cpu: 2
+          memory: 6Gi
 
   annotations:
     testgrid-dashboards: sig-autoscaling-vpa
     testgrid-tab-name: autoscaling-vpa-admission-controller
 - name: ci-kubernetes-e2e-autoscaling-vpa-full
+  cluster: k8s-infra-prow-build
   interval: 2h
   labels:
     preset-service-account: "true"
@@ -109,11 +126,19 @@ periodics:
       # docker-in-docker needs privileged mode
       securityContext:
         privileged: true
+      resources:
+        limits:
+          cpu: 2
+          memory: 6Gi
+        requests:
+          cpu: 2
+          memory: 6Gi
 
   annotations:
     testgrid-dashboards: sig-autoscaling-vpa
     testgrid-tab-name: autoscaling-vpa-full
 - name: ci-kubernetes-e2e-autoscaling-vpa-recommender
+  cluster: k8s-infra-prow-build
   interval: 3h
   labels:
     preset-service-account: "true"
@@ -147,11 +172,19 @@ periodics:
       # docker-in-docker needs privileged mode
       securityContext:
         privileged: true
+      resources:
+        limits:
+          cpu: 2
+          memory: 6Gi
+        requests:
+          cpu: 2
+          memory: 6Gi
 
   annotations:
     testgrid-dashboards: sig-autoscaling-vpa
     testgrid-tab-name: autoscaling-vpa-recommender
 - name: ci-kubernetes-e2e-autoscaling-vpa-updater
+  cluster: k8s-infra-prow-build
   interval: 2h
   labels:
     preset-service-account: "true"
@@ -185,6 +218,13 @@ periodics:
       # docker-in-docker needs privileged mode
       securityContext:
         privileged: true
+      resources:
+        limits:
+          cpu: 2
+          memory: 6Gi
+        requests:
+          cpu: 2
+          memory: 6Gi
 
   annotations:
     testgrid-dashboards: sig-autoscaling-vpa
@@ -215,19 +255,26 @@ periodics:
       - --extract=ci/latest
       - --gcp-node-image=gci
       - --gcp-nodes=3
-      - --gcp-project=k8s-jkns-gci-autoscaling
       - --gcp-zone=us-west1-b
       - --provider=gce
       - --runtime-config=scheduling.k8s.io/v1alpha1=true
       - --test_args=--ginkgo.focus=\[Feature:ClusterSizeAutoscalingScaleUp\]|\[Feature:ClusterSizeAutoscalingScaleDown\]|\[Feature:InitialResources\] --ginkgo.skip=\[Flaky\] --minStartupPods=8
       - --timeout=300m
       image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20230727-ea685f8747-master
+      resources:
+        limits:
+          cpu: 2
+          memory: 6Gi
+        requests:
+          cpu: 2
+          memory: 6Gi
 
   annotations:
     testgrid-dashboards: sig-autoscaling-cluster-autoscaler
     testgrid-tab-name: gci-gce-autoscaling
 - interval: 30m
   name: ci-kubernetes-e2e-gci-gce-autoscaling-hpa-cm
+  cluster: k8s-infra-prow-build
   labels:
     preset-service-account: "true"
     preset-k8s-ssh: "true"
@@ -243,7 +290,6 @@ periodics:
       - --cluster=hpa
       - --extract=ci/latest
       - --gcp-node-image=gci
-      - --gcp-project=k8s-jkns-gci-autoscaling
       - --gcp-zone=us-west1-b
       - --provider=gce
       # Enable HPAContainerMetrics and HPAScaleToZero. Required for container metrics and scale to zero tests.
@@ -251,6 +297,13 @@ periodics:
       - --test_args=--ginkgo.focus=\[Feature:CustomMetricsAutoscaling\] --ginkgo.skip=\[Flaky\] --minStartupPods=8
       - --timeout=300m
       image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20230727-ea685f8747-master
+      resources:
+        limits:
+          cpu: 2
+          memory: 6Gi
+        requests:
+          cpu: 2
+          memory: 6Gi
 
   annotations:
     testgrid-dashboards: sig-autoscaling-hpa
@@ -281,19 +334,26 @@ periodics:
       - --extract=ci/latest
       - --gcp-node-image=gci
       - --gcp-nodes=3
-      - --gcp-project=k8s-jkns-gci-autoscaling-migs
       - --gcp-zone=us-west1-b
       - --provider=gce
       - --runtime-config=scheduling.k8s.io/v1alpha1=true
       - --test_args=--ginkgo.focus=\[Feature:ClusterSizeAutoscalingScaleUp\]|\[Feature:ClusterSizeAutoscalingScaleDown\] --ginkgo.skip=\[Flaky\] --minStartupPods=8
       - --timeout=300m
       image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20230727-ea685f8747-master
+      resources:
+        limits:
+          cpu: 2
+          memory: 6Gi
+        requests:
+          cpu: 2
+          memory: 6Gi
 
   annotations:
     testgrid-dashboards: sig-autoscaling-cluster-autoscaler
     testgrid-tab-name: gci-gce-autoscaling-migs
 - interval: 30m
   name: ci-kubernetes-e2e-gci-gce-autoscaling-migs-hpa
+  cluster: k8s-infra-prow-build
   labels:
     preset-service-account: "true"
     preset-k8s-ssh: "true"
@@ -309,7 +369,6 @@ periodics:
       - --cluster=hpa
       - --extract=ci/latest
       - --gcp-node-image=gci
-      - --gcp-project=k8s-jkns-gci-autoscaling-migs
       - --gcp-zone=us-west1-b
       - --provider=gce
       # Enable HPAContainerMetrics and HPAScaleToZero. Required for container metrics and scale to zero tests.
@@ -317,12 +376,20 @@ periodics:
       - --test_args=--ginkgo.focus=\[Feature:CustomMetricsAutoscaling\] --ginkgo.skip=\[Flaky\] --minStartupPods=8
       - --timeout=300m
       image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20230727-ea685f8747-master
+      resources:
+        limits:
+          cpu: 2
+          memory: 6Gi
+        requests:
+          cpu: 2
+          memory: 6Gi
 
   annotations:
     testgrid-dashboards: sig-autoscaling-hpa
     testgrid-tab-name: gci-gce-autoscaling-migs-hpa
 - interval: 30m
   name: ci-kubernetes-e2e-autoscaling-hpa-cpu
+  cluster: k8s-infra-prow-build
   labels:
     preset-k8s-ssh: "true"
     preset-service-account: "true"
@@ -347,6 +414,13 @@ periodics:
         --minStartupPods=8
       - --ginkgo-parallel=1
       image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20230727-ea685f8747-master
+      resources:
+        limits:
+          cpu: 2
+          memory: 6Gi
+        requests:
+          cpu: 2
+          memory: 6Gi
 
   annotations:
     # TODO: add to release blocking dashboards once run is successful
diff --git a/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-presubmits.yaml b/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-presubmits.yaml
index dcedaed45ee4..5b7f9f70f439 100644
--- a/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-presubmits.yaml
+++ b/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-presubmits.yaml
@@ -1,6 +1,7 @@
 presubmits:
   kubernetes/kubernetes:
   - name: pull-kubernetes-e2e-gci-gce-autoscaling
+    cluster: k8s-infra-prow-build
     annotations:
       testgrid-dashboards: sig-autoscaling-cluster-autoscaler
       testgrid-tab-name: gci-gce-autoscaling-pull
@@ -40,7 +41,6 @@ presubmits:
         - --extract=local
         - --gcp-node-image=gci
         - --gcp-nodes=3
-        - --gcp-project=k8s-jkns-gci-autoscaling
         - --gcp-zone=us-west1-b
         - --provider=gce
         - --stage=gs://kubernetes-release-pull/ci/pull-kubernetes-e2e-gci-gce-autoscaling
@@ -50,8 +50,16 @@ presubmits:
         image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20230727-ea685f8747-master
         securityContext:
           privileged: true
+        resources:
+          limits:
+            cpu: 2
+            memory: 6Gi
+          requests:
+            cpu: 2
+            memory: 6Gi
 
   - name: pull-kubernetes-e2e-autoscaling-hpa-cpu
+    cluster: k8s-infra-prow-build
     annotations:
       testgrid-dashboards: sig-autoscaling-hpa
       testgrid-tab-name: gci-gce-autoscaling-hpa-cpu-pull
@@ -92,8 +100,16 @@ presubmits:
         image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20230727-ea685f8747-master
         securityContext:
           privileged: true
+        resources:
+          limits:
+            cpu: 2
+            memory: 6Gi
+          requests:
+            cpu: 2
+            memory: 6Gi
 
   - name: pull-kubernetes-e2e-autoscaling-hpa-cm
+    cluster: k8s-infra-prow-build
     annotations:
       testgrid-dashboards: sig-autoscaling-hpa
       testgrid-tab-name: gci-gce-autoscaling-hpa-cm-pull
@@ -134,3 +150,10 @@ presubmits:
         image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20230727-ea685f8747-master
         securityContext:
           privileged: true
+        resources:
+          limits:
+            cpu: 2
+            memory: 6Gi
+          requests:
+            cpu: 2
+            memory: 6Gi

From 37409a818134f6005f4ce2f7f39348986a8fe7cc Mon Sep 17 00:00:00 2001
From: Priyanka Saggu <priyankasaggu11929@gmail.com>
Date: Sun, 1 Oct 2023 09:53:21 +0530
Subject: [PATCH 4/6] peribolos: bump ci jobs timeout to 150m

---
 config/jobs/kubernetes/test-infra/test-infra-trusted.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/config/jobs/kubernetes/test-infra/test-infra-trusted.yaml b/config/jobs/kubernetes/test-infra/test-infra-trusted.yaml
index 31ab24b64704..c1c95af0f4c4 100644
--- a/config/jobs/kubernetes/test-infra/test-infra-trusted.yaml
+++ b/config/jobs/kubernetes/test-infra/test-infra-trusted.yaml
@@ -3,6 +3,8 @@ postsubmits:
   - name: post-org-peribolos
     cluster: test-infra-trusted
     decorate: true
+    decoration_config:
+      timeout: 150m
     branches:
     - ^main$
     max_concurrency: 1
@@ -548,6 +550,8 @@ periodics:
     testgrid-num-failures-to-alert: '1'
   cluster: test-infra-trusted
   decorate: true
+  decoration_config:
+    timeout: 150m
   max_concurrency: 1
   extra_refs:
   - org: kubernetes

From 44d581fd12b2ca7b48d486bf856bf65fcb28f30f Mon Sep 17 00:00:00 2001
From: upodroid <upodroid@users.noreply.github.com>
Date: Sun, 1 Oct 2023 17:08:14 +0100
Subject: [PATCH 5/6] split disruptive and serials tests

---
 config/jobs/kubernetes/kops/build_jobs.py     |  21 ++-
 .../kops/kops-periodics-distros.yaml          |   2 +-
 .../kubernetes/kops/kops-periodics-grid.yaml  | 166 +++++++++---------
 .../kubernetes/kops/kops-periodics-misc2.yaml |  27 ++-
 .../kops/kops-presubmits-distros.yaml         |   2 +-
 .../kubernetes/kops/kops-presubmits-e2e.yaml  |  10 +-
 6 files changed, 113 insertions(+), 115 deletions(-)

diff --git a/config/jobs/kubernetes/kops/build_jobs.py b/config/jobs/kubernetes/kops/build_jobs.py
index 66f642cddd10..16c45c7a11d5 100644
--- a/config/jobs/kubernetes/kops/build_jobs.py
+++ b/config/jobs/kubernetes/kops/build_jobs.py
@@ -907,7 +907,7 @@ def generate_misc():
                    extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops"],
                    runs_per_day=6),
 
-        build_test(name_override="ci-kubernetes-e2e-cos-gce-conformance-concurrrency-canary",
+        build_test(name_override="ci-kubernetes-e2e-cos-gce-disruptive-canary",
                    cloud="gce",
                    distro="cos105",
                    networking="kubenet",
@@ -918,14 +918,13 @@ def generate_misc():
                    extra_flags=[
                        "--image=cos-cloud/cos-105-17412-156-49",
                    ],
-                   cluster_name="ci-kubernetes-e2e-cos-gce-conformance-concurrrency.k8s.local",
-                   focus_regex=r'\[Conformance\]',
-                   skip_regex=r'\[FOOBAR\]', # leaving it empty will allow kops to add extra skips
-                   test_timeout_minutes=100,
-                   test_parallelism=8,
+                   focus_regex=r'\[Disruptive\]',
+                   skip_regex=r'\[Driver:.gcepd\]|\[Flaky\]|\[Feature:.+\]', # pylint: disable=line-too-long
+                   test_timeout_minutes=300,
+                   test_parallelism=1, # serial tests
                    test_args="-num-nodes=3 --master-os-distro=gci --node-os-distro=gci",
                    extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops"],
-                   runs_per_day=9),
+                   runs_per_day=6),
 
         build_test(name_override="ci-kubernetes-e2e-cos-gce-serial-canary",
                    cloud="gce",
@@ -939,13 +938,13 @@ def generate_misc():
                        "--image=cos-cloud/cos-105-17412-156-49",
                        "--node-volume-size=100",
                    ],
-                   focus_regex=r'\[Serial\]|\[Disruptive\]',
+                   focus_regex=r'\[Serial\]',
                    skip_regex=r'\[Driver:.gcepd\]|\[Flaky\]|\[Feature:.+\]', # pylint: disable=line-too-long
                    test_timeout_minutes=500,
                    test_parallelism=1, # serial tests
                    test_args="-num-nodes=3 --master-os-distro=gci --node-os-distro=gci",
                    extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops"],
-                   runs_per_day=4),
+                   runs_per_day=6),
 
         build_test(name_override="ci-kubernetes-e2e-cos-gce-alpha-features",
                    cloud="gce",
@@ -1837,7 +1836,8 @@ def generate_presubmits_e2e():
                 "--image=cos-cloud/cos-105-17412-156-49",
                 "--node-volume-size=100",
             ],
-            focus_regex=r'\[Serial\]|\[Disruptive\]',
+            build_cluster="k8s-infra-prow-build",
+            focus_regex=r'\[Serial\]',
             skip_regex=r'\[Driver:.gcepd\]|\[Flaky\]|\[Feature:.+\]', # pylint: disable=line-too-long
             test_timeout_minutes=500,
             always_run=False,
@@ -1854,7 +1854,6 @@ def generate_presubmits_e2e():
             kops_channel="alpha",
             extra_flags=[
                 "--image=cos-cloud/cos-105-17412-156-49",
-                "--set=spec.nodeProblemDetector.enabled=true",
                 "--node-volume-size=100",
             ],
             skip_regex=r'\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]', # pylint: disable=line-too-long
diff --git a/config/jobs/kubernetes/kops/kops-periodics-distros.yaml b/config/jobs/kubernetes/kops/kops-periodics-distros.yaml
index d53b261822b8..5cf4aa4ebe1c 100644
--- a/config/jobs/kubernetes/kops/kops-periodics-distros.yaml
+++ b/config/jobs/kubernetes/kops/kops-periodics-distros.yaml
@@ -480,7 +480,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium --discovery-store=s3://k8s-kops-prow/discovery" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium --discovery-store=s3://k8s-kops-prow/discovery" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable.txt \
           --test=kops \
diff --git a/config/jobs/kubernetes/kops/kops-periodics-grid.yaml b/config/jobs/kubernetes/kops/kops-periodics-grid.yaml
index 9ddee9c3193d..d1228efc8c30 100644
--- a/config/jobs/kubernetes/kops/kops-periodics-grid.yaml
+++ b/config/jobs/kubernetes/kops/kops-periodics-grid.yaml
@@ -32,7 +32,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --test=kops \
@@ -95,7 +95,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -159,7 +159,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -223,7 +223,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --test=kops \
@@ -286,7 +286,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -350,7 +350,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -414,7 +414,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --test=kops \
@@ -477,7 +477,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -541,7 +541,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -605,7 +605,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --test=kops \
@@ -668,7 +668,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --validation-wait=20m \
@@ -732,7 +732,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kubenet" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.28.txt \
           --test=kops \
@@ -4594,7 +4594,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=calico" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=calico" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --test=kops \
@@ -4657,7 +4657,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=calico" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=calico" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -4721,7 +4721,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=calico" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=calico" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -4785,7 +4785,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=calico" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=calico" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --test=kops \
@@ -4848,7 +4848,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=calico" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=calico" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -4912,7 +4912,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=calico" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=calico" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -4976,7 +4976,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=calico" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=calico" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --test=kops \
@@ -5039,7 +5039,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=calico" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=calico" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -5103,7 +5103,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=calico" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=calico" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -5167,7 +5167,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=calico" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=calico" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --test=kops \
@@ -5230,7 +5230,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=calico" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=calico" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --validation-wait=20m \
@@ -5294,7 +5294,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=calico" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=calico" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.28.txt \
           --test=kops \
@@ -9156,7 +9156,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --test=kops \
@@ -9219,7 +9219,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -9283,7 +9283,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -9347,7 +9347,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --test=kops \
@@ -9410,7 +9410,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -9474,7 +9474,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -9538,7 +9538,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --test=kops \
@@ -9601,7 +9601,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -9665,7 +9665,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -9729,7 +9729,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --test=kops \
@@ -9792,7 +9792,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --validation-wait=20m \
@@ -9856,7 +9856,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.28.txt \
           --test=kops \
@@ -13718,7 +13718,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --test=kops \
@@ -13781,7 +13781,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -13845,7 +13845,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -13909,7 +13909,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --test=kops \
@@ -13972,7 +13972,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -14036,7 +14036,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -14100,7 +14100,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --test=kops \
@@ -14163,7 +14163,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -14227,7 +14227,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -14291,7 +14291,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --test=kops \
@@ -14354,7 +14354,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --validation-wait=20m \
@@ -14418,7 +14418,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-etcd" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.28.txt \
           --test=kops \
@@ -18280,7 +18280,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --test=kops \
@@ -18344,7 +18344,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -18409,7 +18409,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -18474,7 +18474,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --test=kops \
@@ -18538,7 +18538,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -18603,7 +18603,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -18668,7 +18668,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --test=kops \
@@ -18732,7 +18732,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -18797,7 +18797,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -18862,7 +18862,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --test=kops \
@@ -18926,7 +18926,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --validation-wait=20m \
@@ -18991,7 +18991,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=cilium-eni --node-size=t3.large" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.28.txt \
           --test=kops \
@@ -22914,7 +22914,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=flannel" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=flannel" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --test=kops \
@@ -22977,7 +22977,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=flannel" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=flannel" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -23041,7 +23041,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=flannel" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=flannel" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -23105,7 +23105,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=flannel" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=flannel" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --test=kops \
@@ -23168,7 +23168,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=flannel" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=flannel" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -23232,7 +23232,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=flannel" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=flannel" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -23296,7 +23296,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=flannel" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=flannel" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --test=kops \
@@ -23359,7 +23359,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=flannel" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=flannel" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -23423,7 +23423,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=flannel" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=flannel" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -23487,7 +23487,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=flannel" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=flannel" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --test=kops \
@@ -23550,7 +23550,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=flannel" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=flannel" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --validation-wait=20m \
@@ -27097,7 +27097,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --test=kops \
@@ -27160,7 +27160,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -27224,7 +27224,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.24.txt \
           --validation-wait=20m \
@@ -27288,7 +27288,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --test=kops \
@@ -27351,7 +27351,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -27415,7 +27415,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.25.txt \
           --validation-wait=20m \
@@ -27479,7 +27479,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --test=kops \
@@ -27542,7 +27542,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.26/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -27606,7 +27606,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.26.txt \
           --validation-wait=20m \
@@ -27670,7 +27670,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --test=kops \
@@ -27733,7 +27733,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/markers/release-1.27/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.27.txt \
           --validation-wait=20m \
@@ -27797,7 +27797,7 @@ periodics:
           -v 2 \
           --up --down \
           --cloud-provider=aws \
-          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
+          --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=kopeio" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
           --kubernetes-version=https://dl.k8s.io/release/stable-1.28.txt \
           --test=kops \
diff --git a/config/jobs/kubernetes/kops/kops-periodics-misc2.yaml b/config/jobs/kubernetes/kops/kops-periodics-misc2.yaml
index 7b918f65f8ff..6fefdedad2de 100644
--- a/config/jobs/kubernetes/kops/kops-periodics-misc2.yaml
+++ b/config/jobs/kubernetes/kops/kops-periodics-misc2.yaml
@@ -1,5 +1,5 @@
 # Test jobs generated by build_jobs.py (do not manually edit)
-# 38 jobs, total of 1288 runs per week
+# 38 jobs, total of 1281 runs per week
 periodics:
 
 # {"cloud": "aws", "distro": "u2204", "extra_flags": "--discovery-store=s3://k8s-kops-prow/discovery", "k8s_version": "1.28", "kops_channel": "alpha", "kops_version": "latest", "networking": "cilium"}
@@ -2265,14 +2265,14 @@ periodics:
     testgrid-tab-name: ci-kubernetes-e2e-cos-gce-conformance-canary
 
 # {"cloud": "gce", "distro": "cos105", "extra_flags": "--image=cos-cloud/cos-105-17412-156-49", "k8s_version": "ci", "kops_channel": "alpha", "kops_version": "latest", "networking": "kubenet"}
-- name: e2e-ci-kubernetes-e2e-cos-gce-conformance-concurrrency-canary
-  cron: '56 2-23/2 * * *'
+- name: e2e-ci-kubernetes-e2e-cos-gce-disruptive-canary
+  cron: '20 2-23/4 * * *'
   labels:
     preset-k8s-ssh: "true"
   cluster: k8s-infra-prow-build
   decorate: true
   decoration_config:
-    timeout: 130m
+    timeout: 330m
   extra_refs:
   - org: kubernetes
     repo: kops
@@ -2295,18 +2295,17 @@ periodics:
           --cloud-provider=gce \
           --admin-access=0.0.0.0/0 \
           --create-args="--channel=alpha --networking=kubenet --image=cos-cloud/cos-105-17412-156-49" \
-          --cluster-name="ci-kubernetes-e2e-cos-gce-conformance-concurrrency.k8s.local" \
           --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci.txt \
           --kubernetes-version=https://storage.googleapis.com/k8s-release-dev/ci/latest.txt \
           --test=kops \
           -- \
-          --test-args="-test.timeout=100m -num-nodes=3 --master-os-distro=gci --node-os-distro=gci" \
+          --test-args="-test.timeout=300m -num-nodes=3 --master-os-distro=gci --node-os-distro=gci" \
           --test-package-bucket=k8s-release-dev \
           --test-package-dir=ci \
           --test-package-marker=latest.txt \
-          --focus-regex="\[Conformance\]" \
-          --skip-regex="\[FOOBAR\]" \
-          --parallel=8
+          --focus-regex="\[Disruptive\]" \
+          --skip-regex="\[Driver:.gcepd\]|\[Flaky\]|\[Feature:.+\]" \
+          --parallel=1
       env:
       - name: KUBE_SSH_KEY_PATH
         value: /etc/ssh-key-secret/ssh-private
@@ -2330,12 +2329,12 @@ periodics:
     test.kops.k8s.io/kops_version: latest
     test.kops.k8s.io/networking: kubenet
     testgrid-dashboards: kops-distro-cos105, kops-gce, kops-k8s-ci, kops-latest, sig-cluster-lifecycle-kops, sig-cluster-lifecycle-kubeup-to-kops
-    testgrid-days-of-results: '31'
-    testgrid-tab-name: ci-kubernetes-e2e-cos-gce-conformance-concurrrency-canary
+    testgrid-days-of-results: '47'
+    testgrid-tab-name: ci-kubernetes-e2e-cos-gce-disruptive-canary
 
 # {"cloud": "gce", "distro": "cos105", "extra_flags": "--image=cos-cloud/cos-105-17412-156-49 --node-volume-size=100", "k8s_version": "ci", "kops_channel": "alpha", "kops_version": "latest", "networking": "kubenet"}
 - name: e2e-ci-kubernetes-e2e-cos-gce-serial-canary
-  cron: '31 4-23/6 * * *'
+  cron: '31 0-23/4 * * *'
   labels:
     preset-k8s-ssh: "true"
   cluster: k8s-infra-prow-build
@@ -2372,7 +2371,7 @@ periodics:
           --test-package-bucket=k8s-release-dev \
           --test-package-dir=ci \
           --test-package-marker=latest.txt \
-          --focus-regex="\[Serial\]|\[Disruptive\]" \
+          --focus-regex="\[Serial\]" \
           --skip-regex="\[Driver:.gcepd\]|\[Flaky\]|\[Feature:.+\]" \
           --parallel=1
       env:
@@ -2398,7 +2397,7 @@ periodics:
     test.kops.k8s.io/kops_version: latest
     test.kops.k8s.io/networking: kubenet
     testgrid-dashboards: kops-distro-cos105, kops-gce, kops-k8s-ci, kops-latest, sig-cluster-lifecycle-kops, sig-cluster-lifecycle-kubeup-to-kops
-    testgrid-days-of-results: '71'
+    testgrid-days-of-results: '47'
     testgrid-tab-name: ci-kubernetes-e2e-cos-gce-serial-canary
 
 # {"cloud": "gce", "distro": "cos105", "extra_flags": "--image=cos-cloud/cos-105-17412-156-49", "k8s_version": "ci", "kops_channel": "alpha", "kops_version": "latest", "networking": "kubenet"}
diff --git a/config/jobs/kubernetes/kops/kops-presubmits-distros.yaml b/config/jobs/kubernetes/kops/kops-presubmits-distros.yaml
index 9c62abd4f170..9dc51fa3e12e 100644
--- a/config/jobs/kubernetes/kops/kops-presubmits-distros.yaml
+++ b/config/jobs/kubernetes/kops/kops-presubmits-distros.yaml
@@ -504,7 +504,7 @@ presubmits:
             -v 2 \
             --up --build --down \
             --cloud-provider=aws \
-            --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230912.0-x86_64-gp2' --channel=alpha --networking=calico --discovery-store=s3://k8s-kops-prow/discovery" \
+            --create-args="--image='137112412989/amzn2-ami-kernel-5.10-hvm-2.0.20230926.0-x86_64-gp2' --channel=alpha --networking=calico --discovery-store=s3://k8s-kops-prow/discovery" \
             --kubernetes-version=https://dl.k8s.io/release/stable.txt \
             --kops-binary-path=/home/prow/go/src/k8s.io/kops/.build/dist/linux/amd64/kops \
             --test=kops \
diff --git a/config/jobs/kubernetes/kops/kops-presubmits-e2e.yaml b/config/jobs/kubernetes/kops/kops-presubmits-e2e.yaml
index 665aafee8f76..8bdc9da6e6e9 100644
--- a/config/jobs/kubernetes/kops/kops-presubmits-e2e.yaml
+++ b/config/jobs/kubernetes/kops/kops-presubmits-e2e.yaml
@@ -2528,7 +2528,7 @@ presubmits:
 
 # {"cloud": "gce", "distro": "cos105", "extra_flags": "--image=cos-cloud/cos-105-17412-156-49 --node-volume-size=100", "k8s_version": "ci", "kops_channel": "alpha", "networking": "kubenet"}
   - name: pull-kops-kubernetes-e2e-cos-gce-serial
-    cluster: default
+    cluster: k8s-infra-prow-build
     branches:
     - master
     always_run: false
@@ -2566,7 +2566,7 @@ presubmits:
             --test-package-bucket=k8s-release-dev \
             --test-package-dir=ci \
             --test-package-marker=latest.txt \
-            --focus-regex="\[Serial\]|\[Disruptive\]" \
+            --focus-regex="\[Serial\]" \
             --skip-regex="\[Driver:.gcepd\]|\[Flaky\]|\[Feature:.+\]" \
             --parallel=1
         securityContext:
@@ -2596,7 +2596,7 @@ presubmits:
       testgrid-days-of-results: '90'
       testgrid-tab-name: pull-kops-kubernetes-e2e-cos-gce-serial
 
-# {"cloud": "gce", "distro": "cos105", "extra_flags": "--image=cos-cloud/cos-105-17412-156-49 --set=spec.nodeProblemDetector.enabled=true --node-volume-size=100", "k8s_version": "ci", "kops_channel": "alpha", "networking": "kubenet"}
+# {"cloud": "gce", "distro": "cos105", "extra_flags": "--image=cos-cloud/cos-105-17412-156-49 --node-volume-size=100", "k8s_version": "ci", "kops_channel": "alpha", "networking": "kubenet"}
   - name: pull-kops-kubernetes-e2e-cos-gce
     cluster: default
     branches:
@@ -2627,7 +2627,7 @@ presubmits:
             --up --build --down \
             --cloud-provider=gce \
             --admin-access=0.0.0.0/0 \
-            --create-args="--channel=alpha --networking=kubenet --image=cos-cloud/cos-105-17412-156-49 --set=spec.nodeProblemDetector.enabled=true --node-volume-size=100" \
+            --create-args="--channel=alpha --networking=kubenet --image=cos-cloud/cos-105-17412-156-49 --node-volume-size=100" \
             --kubernetes-version=https://storage.googleapis.com/k8s-release-dev/ci/latest.txt \
             --kops-binary-path=/home/prow/go/src/k8s.io/kops/.build/dist/linux/amd64/kops \
             --test=kops \
@@ -2657,7 +2657,7 @@ presubmits:
     annotations:
       test.kops.k8s.io/cloud: gce
       test.kops.k8s.io/distro: cos105
-      test.kops.k8s.io/extra_flags: --image=cos-cloud/cos-105-17412-156-49 --set=spec.nodeProblemDetector.enabled=true --node-volume-size=100
+      test.kops.k8s.io/extra_flags: --image=cos-cloud/cos-105-17412-156-49 --node-volume-size=100
       test.kops.k8s.io/k8s_version: ci
       test.kops.k8s.io/kops_channel: alpha
       test.kops.k8s.io/networking: kubenet

From ffc4ccbcd3f221e8adfcb0da531d931eb80e15da Mon Sep 17 00:00:00 2001
From: Umesh Roy <umesh.roysqr@gmail.com>
Date: Sun, 1 Oct 2023 19:08:11 -0400
Subject: [PATCH 6/6] Allow running e2e tests as presubmits on GitHub #5918

---
 .../sig-autoscaling-presubmits.yaml           | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-presubmits.yaml b/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-presubmits.yaml
index 5b7f9f70f439..eb96c4c12218 100644
--- a/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-presubmits.yaml
+++ b/config/jobs/kubernetes/sig-autoscaling/sig-autoscaling-presubmits.yaml
@@ -157,3 +157,42 @@ presubmits:
           requests:
             cpu: 2
             memory: 6Gi
+
+  - name: pull-kubernetes-e2e-autoscaling-vpa-full
+    annotations:
+      testgrid-dashboards: sig-autoscaling-vpa
+      testgrid-tab-name: autoscaling-vpa-full-presubmits
+    optional: true
+    decorate: true
+    decoration_config:
+      timeout: 120m
+    run_if_changed: '^vertical-pod-autoscaler\/'
+    path_alias: k8s.io/kubernetes
+    extra_refs:
+    - org: kubernetes
+      repo: autoscaler
+      base_ref: master
+      path_alias: k8s.io/autoscaler
+      workdir: true
+    labels:
+      preset-service-account: "true"
+      preset-k8s-ssh: "true"
+      preset-dind-enabled: "true"
+    spec:
+      containers:
+      - command:
+        - runner.sh
+        - /workspace/scenarios/kubernetes_e2e.py
+        args:
+        - --check-leaked-resources
+        - --extract=ci/latest
+        - --gcp-node-image=gci
+        - --gcp-zone=us-west1-b
+        - --provider=gce
+        - --test=false
+        - --test-cmd=../vertical-pod-autoscaler/hack/run-e2e.sh
+        - --test-cmd-args=full-vpa
+        - --timeout=100m
+        image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20230727-ea685f8747-master
+        securityContext:
+          privileged: true