Merge pull request kubernetes#1914 from mborsz/latency

Use the same image in daemonset and pod startup latency
ameukam · Oct 18, 2021 · e1d5f0e · e1d5f0e
2 parents a81c576 + 81568de
commit e1d5f0e
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 4 deletions.
diff --git a/clusterloader2/testing/load/config.yaml b/clusterloader2/testing/load/config.yaml
@@ -81,6 +81,8 @@
 {{$EXIT_AFTER_EXEC := DefaultParam .CL2_EXIT_AFTER_EXEC false}}
 {{$SLEEP_AFTER_EXEC_DURATION := DefaultParam .CL2_SLEEP_AFTER_EXEC_DURATION "0s"}}
 
+{{$latencyPodImage := DefaultParam .CL2_LATENCY_POD_IMAGE "k8s.gcr.io/pause:3.1"}}
+
 name: load
 namespace:
   number: {{AddInt $namespaces $schedulerThroughputNamespaces}}
@@ -170,7 +172,13 @@ steps:
       namespaces: {{$namespaces}}
       tuningSet: RandomizedSaturationTimeLimited
       testMaxReplicaFactor: {{$RANDOM_SCALE_FACTOR}}
-      daemonSetImage: k8s.gcr.io/pause:3.0
+      # We rely on the fact that daemonset is using the same image as the 'pod-startup-latency' module.
+      # The goal is to cache the image to all nodes before we start any latency pod,
+      # so that when we measure pod startup latency, the image is already present on all nodes.
+      # This way, the pod startup latency we measure excludes (or limits impact) of image pulling,
+      # whuch matches our SLO definition: https://github.com/kubernetes/community/blob/master/sig-scalability/slos/pod_startup_latency.md.
+      daemonSetImage: {{$latencyPodImage}}
+      daemonSetEnv: "before update"
       daemonSetReplicas: 1
       bigDeploymentSize: {{$BIG_GROUP_SIZE}}
       bigDeploymentsPerNamespace: {{$bigDeploymentsPerNamespace}}
@@ -282,6 +290,7 @@ steps:
     params:
       namespaces: {{$namespaces}}
       minPodsInSmallCluster: {{$MIN_PODS_IN_SMALL_CLUSTERS}}
+      image: {{$latencyPodImage}}
 {{end}}
 
 - module:
@@ -292,7 +301,8 @@ steps:
       tuningSet: RandomizedScalingTimeLimited
       randomScaleFactor: {{$RANDOM_SCALE_FACTOR}}
       testMaxReplicaFactor: {{$RANDOM_SCALE_FACTOR}}
-      daemonSetImage: k8s.gcr.io/pause:3.1
+      daemonSetImage: {{$latencyPodImage}}
+      daemonSetEnv: "after update"
       daemonSetReplicas: 1
       bigDeploymentSize: {{$BIG_GROUP_SIZE}}
       bigDeploymentsPerNamespace: {{$bigDeploymentsPerNamespace}}
@@ -318,7 +328,6 @@ steps:
       namespaces: {{$namespaces}}
       tuningSet: RandomizedDeletionTimeLimited
       testMaxReplicaFactor: {{$RANDOM_SCALE_FACTOR}}
-      daemonSetImage: k8s.gcr.io/pause:3.1
       daemonSetReplicas: 0
       bigDeploymentSize: {{$BIG_GROUP_SIZE}}
       bigDeploymentsPerNamespace: 0

diff --git a/clusterloader2/testing/load/daemonset.yaml b/clusterloader2/testing/load/daemonset.yaml
@@ -1,5 +1,6 @@
 {{$HostNetworkMode := DefaultParam .CL2_USE_HOST_NETWORK_PODS false}}
 {{$Image := DefaultParam .Image "k8s.gcr.io/pause:3.1"}}
+{{$Env := DefaultParam .Env ""}}
 
 apiVersion: apps/v1
 kind: DaemonSet
@@ -24,6 +25,9 @@ spec:
       containers:
       - name: {{.Name}}
         image: {{$Image}}
+        env:
+          - name: TEST_ENV
+            value: {{$Env}}
         resources:
           requests:
             cpu: 10m

diff --git a/clusterloader2/testing/load/modules/pod-startup-latency.yaml b/clusterloader2/testing/load/modules/pod-startup-latency.yaml
@@ -3,6 +3,7 @@
 ## Input Params
 {{$namespaces := .namespaces}}
 {{$minPodsInSmallCluster := .minPodsInSmallCluster}}
+{{$image := .image}}
 
 ## CL2 Params
 # LATENCY_POD_MEMORY and LATENCY_POD_CPU are calculated for 1-core 4GB node.
@@ -50,6 +51,7 @@ steps:
             Group: latency
             CpuRequest: {{$LATENCY_POD_CPU}}m
             MemoryRequest: {{$LATENCY_POD_MEMORY}}M
+            Image: {{$image}}
 - name: Waiting for latency pods to be running
   measurements:
     - Identifier: WaitForRunningLatencyDeployments

diff --git a/clusterloader2/testing/load/modules/reconcile-objects.yaml b/clusterloader2/testing/load/modules/reconcile-objects.yaml
@@ -14,6 +14,7 @@
 # DaemonSets
 {{$daemonSetImage := DefaultParam .daemonSetImage "k8s.gcr.io/pause:3.0"}}
 {{$daemonSetReplicas := .daemonSetReplicas}}
+{{$daemonSetEnv := .daemonSetEnv}}
 
 # Deployments
 {{$bigDeploymentSize := .bigDeploymentSize}}
@@ -89,6 +90,7 @@ steps:
       objectTemplatePath: daemonset.yaml
       templateFillMap:
         Image: {{$daemonSetImage}}
+        Env: {{$daemonSetEnv}}
 {{end}}
   - namespaceRange:
       min: 1

diff --git a/clusterloader2/testing/load/simple-deployment.yaml b/clusterloader2/testing/load/simple-deployment.yaml
@@ -1,4 +1,5 @@
 {{$HostNetworkMode := DefaultParam .CL2_USE_HOST_NETWORK_PODS false}}
+{{$Image := DefaultParam .Image "k8s.gcr.io/pause:3.1"}}
 
 apiVersion: apps/v1
 kind: Deployment
@@ -22,7 +23,7 @@ spec:
     spec:
       hostNetwork: {{$HostNetworkMode}}
       containers:
-      - image: k8s.gcr.io/pause:3.1
+      - image: {{$Image}}
         imagePullPolicy: IfNotPresent
         name: {{.Name}}
         ports: