From e914fd1137c171e9a28bb16a54835cb8280b7e90 Mon Sep 17 00:00:00 2001 From: Mucahit Kurt Date: Sat, 29 Jun 2019 18:02:33 +0300 Subject: [PATCH 1/5] pod with ephemeral volumes test cases Signed-off-by: Mucahit Kurt --- .../pkg/config/template_functions.go | 5 + .../ephemeral-volumes/ConfigMap.yaml | 6 ++ .../pod-startup/ephemeral-volumes/README.md | 8 ++ .../pod-startup/ephemeral-volumes/Secret.yaml | 8 ++ .../pod-startup/ephemeral-volumes/config.yaml | 97 +++++++++++++++++++ .../ephemeral-volumes/configmap/override.yaml | 2 + .../downwardapi/override.yaml | 1 + .../pod-startup/ephemeral-volumes/pod.yaml | 49 ++++++++++ .../ephemeral-volumes/secret/override.yaml | 2 + 9 files changed, 178 insertions(+) create mode 100644 clusterloader2/testing/storage/pod-startup/ephemeral-volumes/ConfigMap.yaml create mode 100644 clusterloader2/testing/storage/pod-startup/ephemeral-volumes/README.md create mode 100644 clusterloader2/testing/storage/pod-startup/ephemeral-volumes/Secret.yaml create mode 100644 clusterloader2/testing/storage/pod-startup/ephemeral-volumes/config.yaml create mode 100644 clusterloader2/testing/storage/pod-startup/ephemeral-volumes/configmap/override.yaml create mode 100644 clusterloader2/testing/storage/pod-startup/ephemeral-volumes/downwardapi/override.yaml create mode 100644 clusterloader2/testing/storage/pod-startup/ephemeral-volumes/pod.yaml create mode 100644 clusterloader2/testing/storage/pod-startup/ephemeral-volumes/secret/override.yaml diff --git a/clusterloader2/pkg/config/template_functions.go b/clusterloader2/pkg/config/template_functions.go index 8be65f4a89..8b81b7f71c 100644 --- a/clusterloader2/pkg/config/template_functions.go +++ b/clusterloader2/pkg/config/template_functions.go @@ -57,9 +57,14 @@ func GetFuncs() template.FuncMap { "DefaultParam": defaultParam, "IncludeFile": includeFile, "YamlQuote": yamlQuote, + "Seq": seq, } } +func seq(size interface{}) []int { + return make([]int, int(toFloat64(size))) +} + func toFloat64(val interface{}) float64 { switch i := val.(type) { case float64: diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/ConfigMap.yaml b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/ConfigMap.yaml new file mode 100644 index 0000000000..05f13f0952 --- /dev/null +++ b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/ConfigMap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{.Name}} +data: + TEST_KEY1: testData1 \ No newline at end of file diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/README.md b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/README.md new file mode 100644 index 0000000000..74abcd0f39 --- /dev/null +++ b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/README.md @@ -0,0 +1,8 @@ +Below test cases can be run with the parameters like; + +- 1 pod with X volumes and 1 node +-- `.Nodes := 1`, `$NODES_PER_NAMESPACE := 1`, `$PODS_PER_NODE := 1`, `$VOLUMES_PER_POD := ` +- X pods with 1 volume each on 1 node in parallel +-- `.Nodes := 1`, `$NODES_PER_NAMESPACE := 1`, `$PODS_PER_NODE := `, `$VOLUMES_PER_POD := 1` +- X pods with 1 volume each in parallel +-- `.Nodes := `, `$NODES_PER_NAMESPACE := `, `$PODS_PER_NODE := `, `$VOLUMES_PER_POD := 1` \ No newline at end of file diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/Secret.yaml b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/Secret.yaml new file mode 100644 index 0000000000..91b732b371 --- /dev/null +++ b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/Secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{.Name}} +type: Opaque +data: + username: dXNlcm5hbWVfCg== + password: cGFzc3dvcmRfCg== \ No newline at end of file diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/config.yaml b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/config.yaml new file mode 100644 index 0000000000..6c4d01e8f1 --- /dev/null +++ b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/config.yaml @@ -0,0 +1,97 @@ +#Constants +{{$NODES_PER_NAMESPACE := DefaultParam .NODES_PER_NAMESPACE 1}} +{{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 1}} +{{$LOAD_TEST_THROUGHPUT := DefaultParam .LOAD_TEST_THROUGHPUT 10}} + +{{$VOLUMES_PER_POD := 4}} +{{$VOLUME_TYPE := DefaultParam .VOLUME_TYPE "EmptyDir"}} +{{$PROVISION_VOLUME := DefaultParam .PROVISION_VOLUME false}} + +{{$APP_NAME := "pod-load"}} +{{$GROUP := "pod-with-ephemeral-volume-startup-latency"}} + +#Variables +# TODO think about this! +{{$podStartupTimeout := $VOLUMES_PER_POD}} +{{$namespaces := DivideInt .Nodes $NODES_PER_NAMESPACE}} +{{$podsPerNamespace := MultiplyInt $NODES_PER_NAMESPACE $PODS_PER_NODE}} +{{$totalPods := MultiplyInt $namespaces $NODES_PER_NAMESPACE $PODS_PER_NODE}} +{{$volumesPerNamespace := MultiplyInt $podsPerNamespace $VOLUMES_PER_POD}} + +name: pod-with-ephemeral-volume-startup-latency +automanagedNamespaces: {{$namespaces}} +tuningSets: +# TODO different options? +- name: UniformQPS + qpsLoad: + qps: {{$LOAD_TEST_THROUGHPUT}} +steps: +- measurements: + - Identifier: APIResponsiveness + Method: APIResponsiveness + Params: + action: reset +{{ if $PROVISION_VOLUME }} +# Provision volumes +- phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$volumesPerNamespace}} + tuningSet: UniformQPS + objectBundle: + - basename: vol-{{$APP_NAME}} + objectTemplatePath: {{$VOLUME_TYPE}}.yaml +{{ end }} +# Start measurements +- measurements: + - Identifier: PodWithMultiVolumeStartupLatency + Method: PodStartupLatency + Params: + action: start + labelSelector: group = {{$GROUP}} + threshold: {{$podStartupTimeout}}s +# Create pods +- phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$podsPerNamespace}} + tuningSet: UniformQPS + objectBundle: + - basename: {{$APP_NAME}} + objectTemplatePath: pod.yaml + templateFillMap: + Group: {{$GROUP}} + VolumesPerPod: {{$VOLUMES_PER_POD}} + VolumeType: {{$VOLUME_TYPE}} + AppName: {{$APP_NAME}} +- measurements: + - Identifier: WaitForRunningPodsWithStorage + Method: WaitForRunningPods + Params: + desiredPodCount: {{$totalPods}} + labelSelector: group = {{$GROUP}} + # TODO how can i determine this? + timeout: 15m +- measurements: + - Identifier: PodWithMultiVolumeStartupLatency + Method: PodStartupLatency + Params: + action: gather +# Delete pods +- phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: 0 + tuningSet: UniformQPS + objectBundle: + - basename: {{$APP_NAME}} + objectTemplatePath: pod.yaml +# Collect measurements +- measurements: + - Identifier: APIResponsiveness + Method: APIResponsiveness + Params: + action: gather \ No newline at end of file diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/configmap/override.yaml b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/configmap/override.yaml new file mode 100644 index 0000000000..9a5300016e --- /dev/null +++ b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/configmap/override.yaml @@ -0,0 +1,2 @@ +VOLUME_TYPE: ConfigMap +PROVISION_VOLUME: true \ No newline at end of file diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/downwardapi/override.yaml b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/downwardapi/override.yaml new file mode 100644 index 0000000000..56b6748ad8 --- /dev/null +++ b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/downwardapi/override.yaml @@ -0,0 +1 @@ +VOLUME_TYPE: DownwardAPI \ No newline at end of file diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/pod.yaml b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/pod.yaml new file mode 100644 index 0000000000..fdd12656b2 --- /dev/null +++ b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/pod.yaml @@ -0,0 +1,49 @@ +{{$volumeType := .VolumeType}} +{{$index := .Index}} +{{$appName := .AppName}} +{{$volumesPerPod := .VolumesPerPod}} +apiVersion: v1 +kind: Pod +metadata: + name: {{.Name}} + labels: + app: {{.Name}} + group: {{.Group}} +spec: + containers: + - name: {{.Name}} + image: k8s.gcr.io/pause:3.1 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 80 + name: {{.Name}} + volumeMounts: + {{ range $volumeIndex, $vol := Seq .VolumesPerPod }} + - name: vol-{{$volumeIndex}} + mountPath: /usr/share/{{$volumeIndex}} + {{ end }} + volumes: + {{ range $volumeIndex, $vol := Seq .VolumesPerPod }} + - name: vol-{{$volumeIndex}} + {{ if eq $volumeType "ConfigMap" }} + configMap: + name: vol-{{$appName}}-{{AddInt $volumeIndex (MultiplyInt $index $volumesPerPod)}} + {{ end }} + {{ if eq $volumeType "EmptyDir" }} + emptyDir: {} + {{ end }} + {{ if eq $volumeType "Secret" }} + secret: + secretName: vol-{{$appName}}-{{AddInt $volumeIndex (MultiplyInt $index $volumesPerPod)}} + {{ end }} + {{ if eq $volumeType "DownwardAPI" }} + downwardAPI: + items: + - path: "pod_name" + fieldRef: + fieldPath: metadata.name + - path: "pod_namespace" + fieldRef: + fieldPath: metadata.namespace + {{ end }} + {{ end }} \ No newline at end of file diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/secret/override.yaml b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/secret/override.yaml new file mode 100644 index 0000000000..48a79ff5f9 --- /dev/null +++ b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/secret/override.yaml @@ -0,0 +1,2 @@ +VOLUME_TYPE: Secret +PROVISION_VOLUME: true \ No newline at end of file From 185997a7f36113bab3c0c2ccb943dc12e18e62ca Mon Sep 17 00:00:00 2001 From: Mucahit Kurt Date: Tue, 9 Jul 2019 00:13:47 +0300 Subject: [PATCH 2/5] simplify test cases only 1-node test cases and separate pod configs for different volume types Signed-off-by: Mucahit Kurt --- .../ephemeral-volumes/1_node/README.md | 8 ++++ .../ephemeral-volumes/1_node}/config.yaml | 46 +++++++++---------- .../1_node/configmap}/ConfigMap.yaml | 2 +- .../1_node/configmap/override.yaml | 3 ++ .../1_node/configmap/pod_with_ConfigMaps.yaml | 26 +++++++++++ .../1_node/downwardapi/override.yaml | 1 + .../downwardapi/pod_with_DownwardAPIs.yaml} | 17 ------- .../1_node/pod_with_EmptyDirs.yaml | 25 ++++++++++ .../1_node/secret}/Secret.yaml | 2 +- .../1_node/secret/override.yaml | 3 ++ .../1_node/secret/pod_with_Secrets.yaml | 26 +++++++++++ .../pod-startup/ephemeral-volumes/README.md | 8 ---- .../ephemeral-volumes/configmap/override.yaml | 2 - .../downwardapi/override.yaml | 1 - .../ephemeral-volumes/secret/override.yaml | 2 - 15 files changed, 116 insertions(+), 56 deletions(-) create mode 100644 clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/README.md rename clusterloader2/testing/{storage/pod-startup/ephemeral-volumes => experimental/storage/pod-startup/ephemeral-volumes/1_node}/config.yaml (62%) rename clusterloader2/testing/{storage/pod-startup/ephemeral-volumes => experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap}/ConfigMap.yaml (73%) create mode 100644 clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/override.yaml create mode 100644 clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/pod_with_ConfigMaps.yaml create mode 100644 clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/override.yaml rename clusterloader2/testing/{storage/pod-startup/ephemeral-volumes/pod.yaml => experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/pod_with_DownwardAPIs.yaml} (59%) create mode 100644 clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/pod_with_EmptyDirs.yaml rename clusterloader2/testing/{storage/pod-startup/ephemeral-volumes => experimental/storage/pod-startup/ephemeral-volumes/1_node/secret}/Secret.yaml (78%) create mode 100644 clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/override.yaml create mode 100644 clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/pod_with_Secrets.yaml delete mode 100644 clusterloader2/testing/storage/pod-startup/ephemeral-volumes/README.md delete mode 100644 clusterloader2/testing/storage/pod-startup/ephemeral-volumes/configmap/override.yaml delete mode 100644 clusterloader2/testing/storage/pod-startup/ephemeral-volumes/downwardapi/override.yaml delete mode 100644 clusterloader2/testing/storage/pod-startup/ephemeral-volumes/secret/override.yaml diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/README.md b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/README.md new file mode 100644 index 0000000000..ed4734817c --- /dev/null +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/README.md @@ -0,0 +1,8 @@ +Below test cases can be run with the parameters like; + +- 1 pod with X volumes and 1 node +-- `.Nodes := 1`, `$TOTAL_PODS := 1`, `$VOLUMES_PER_POD := ` +- X pods with 1 volume each on 1 node in parallel +-- `.Nodes := 1`, `$TOTAL_PODS := `, `$VOLUMES_PER_POD := 1` + +To test with different volume types please use the override file for the specific volume type, default volume type is `EmptyDir` diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/config.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml similarity index 62% rename from clusterloader2/testing/storage/pod-startup/ephemeral-volumes/config.yaml rename to clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml index 6c4d01e8f1..c3aac4b35b 100644 --- a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/config.yaml +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml @@ -1,27 +1,26 @@ +# ASSUMPTIONS: +# - this test cases should be run on only 1 node + #Constants -{{$NODES_PER_NAMESPACE := DefaultParam .NODES_PER_NAMESPACE 1}} -{{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 1}} +{{$TOTAL_PODS := DefaultParam .TOTAL_PODS 1}} +{{$POD_TEMPLATE_PATH := DefaultParam .POD_TEMPLATE_PATH "pod_with_EmptyDirs.yaml"}} {{$LOAD_TEST_THROUGHPUT := DefaultParam .LOAD_TEST_THROUGHPUT 10}} -{{$VOLUMES_PER_POD := 4}} -{{$VOLUME_TYPE := DefaultParam .VOLUME_TYPE "EmptyDir"}} +{{$VOLUMES_PER_POD := DefaultParam .VOLUMES_PER_POD 4}} +{{$VOLUME_TEMPLATE_PATH := .VOLUME_TEMPLATE_PATH}} {{$PROVISION_VOLUME := DefaultParam .PROVISION_VOLUME false}} {{$APP_NAME := "pod-load"}} {{$GROUP := "pod-with-ephemeral-volume-startup-latency"}} #Variables -# TODO think about this! -{{$podStartupTimeout := $VOLUMES_PER_POD}} -{{$namespaces := DivideInt .Nodes $NODES_PER_NAMESPACE}} -{{$podsPerNamespace := MultiplyInt $NODES_PER_NAMESPACE $PODS_PER_NODE}} -{{$totalPods := MultiplyInt $namespaces $NODES_PER_NAMESPACE $PODS_PER_NODE}} -{{$volumesPerNamespace := MultiplyInt $podsPerNamespace $VOLUMES_PER_POD}} +# used stateless pod SLO for initial run +{{$podStartupTimeout := 5}} +{{$totalVolumes := MultiplyInt $TOTAL_PODS $VOLUMES_PER_POD}} name: pod-with-ephemeral-volume-startup-latency -automanagedNamespaces: {{$namespaces}} +automanagedNamespaces: 1 tuningSets: -# TODO different options? - name: UniformQPS qpsLoad: qps: {{$LOAD_TEST_THROUGHPUT}} @@ -36,12 +35,12 @@ steps: - phases: - namespaceRange: min: 1 - max: {{$namespaces}} - replicasPerNamespace: {{$volumesPerNamespace}} + max: 1 + replicasPerNamespace: {{$totalVolumes}} tuningSet: UniformQPS objectBundle: - basename: vol-{{$APP_NAME}} - objectTemplatePath: {{$VOLUME_TYPE}}.yaml + objectTemplatePath: {{$VOLUME_TEMPLATE_PATH}} {{ end }} # Start measurements - measurements: @@ -55,24 +54,23 @@ steps: - phases: - namespaceRange: min: 1 - max: {{$namespaces}} - replicasPerNamespace: {{$podsPerNamespace}} + max: 1 + replicasPerNamespace: {{$TOTAL_PODS}} tuningSet: UniformQPS objectBundle: - basename: {{$APP_NAME}} - objectTemplatePath: pod.yaml + objectTemplatePath: {{$POD_TEMPLATE_PATH}} templateFillMap: Group: {{$GROUP}} VolumesPerPod: {{$VOLUMES_PER_POD}} - VolumeType: {{$VOLUME_TYPE}} AppName: {{$APP_NAME}} - measurements: - Identifier: WaitForRunningPodsWithStorage Method: WaitForRunningPods Params: - desiredPodCount: {{$totalPods}} + desiredPodCount: {{$TOTAL_PODS}} labelSelector: group = {{$GROUP}} - # TODO how can i determine this? + # TODO decide this after test roll-out phase timeout: 15m - measurements: - Identifier: PodWithMultiVolumeStartupLatency @@ -83,15 +81,15 @@ steps: - phases: - namespaceRange: min: 1 - max: {{$namespaces}} + max: 1 replicasPerNamespace: 0 tuningSet: UniformQPS objectBundle: - basename: {{$APP_NAME}} - objectTemplatePath: pod.yaml + objectTemplatePath: {{$POD_TEMPLATE_PATH}} # Collect measurements - measurements: - Identifier: APIResponsiveness Method: APIResponsiveness Params: - action: gather \ No newline at end of file + action: gather diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/ConfigMap.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/ConfigMap.yaml similarity index 73% rename from clusterloader2/testing/storage/pod-startup/ephemeral-volumes/ConfigMap.yaml rename to clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/ConfigMap.yaml index 05f13f0952..d18983a809 100644 --- a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/ConfigMap.yaml +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/ConfigMap.yaml @@ -3,4 +3,4 @@ kind: ConfigMap metadata: name: {{.Name}} data: - TEST_KEY1: testData1 \ No newline at end of file + TEST_KEY1: testData1 diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/override.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/override.yaml new file mode 100644 index 0000000000..fb14d6a7a6 --- /dev/null +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/override.yaml @@ -0,0 +1,3 @@ +PROVISION_VOLUME: true +POD_TEMPLATE_PATH: "configmap/pod_with_ConfigMaps.yaml" +VOLUME_TEMPLATE_PATH: "configmap/ConfigMap.yaml" diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/pod_with_ConfigMaps.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/pod_with_ConfigMaps.yaml new file mode 100644 index 0000000000..08d201a859 --- /dev/null +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/pod_with_ConfigMaps.yaml @@ -0,0 +1,26 @@ +{{$index := .Index}} +{{$appName := .AppName}} +{{$volumesPerPod := .VolumesPerPod}} +apiVersion: v1 +kind: Pod +metadata: + name: {{.Name}} + labels: + app: {{.Name}} + group: {{.Group}} +spec: + containers: + - name: {{.Name}} + image: k8s.gcr.io/pause:3.1 + imagePullPolicy: IfNotPresent + volumeMounts: + {{ range $volumeIndex, $vol := Seq .VolumesPerPod }} + - name: vol-{{$volumeIndex}} + mountPath: /usr/share/{{$volumeIndex}} + {{ end }} + volumes: + {{ range $volumeIndex, $vol := Seq .VolumesPerPod }} + - name: vol-{{$volumeIndex}} + configMap: + name: vol-{{$appName}}-{{AddInt $volumeIndex (MultiplyInt $index $volumesPerPod)}} + {{ end }} diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/override.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/override.yaml new file mode 100644 index 0000000000..ecf6234f84 --- /dev/null +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/override.yaml @@ -0,0 +1 @@ +POD_TEMPLATE_PATH: "downwardapi/pod_with_DownwardAPIs.yaml" diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/pod.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/pod_with_DownwardAPIs.yaml similarity index 59% rename from clusterloader2/testing/storage/pod-startup/ephemeral-volumes/pod.yaml rename to clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/pod_with_DownwardAPIs.yaml index fdd12656b2..03ce130315 100644 --- a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/pod.yaml +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/pod_with_DownwardAPIs.yaml @@ -1,4 +1,3 @@ -{{$volumeType := .VolumeType}} {{$index := .Index}} {{$appName := .AppName}} {{$volumesPerPod := .VolumesPerPod}} @@ -14,9 +13,6 @@ spec: - name: {{.Name}} image: k8s.gcr.io/pause:3.1 imagePullPolicy: IfNotPresent - ports: - - containerPort: 80 - name: {{.Name}} volumeMounts: {{ range $volumeIndex, $vol := Seq .VolumesPerPod }} - name: vol-{{$volumeIndex}} @@ -25,18 +21,6 @@ spec: volumes: {{ range $volumeIndex, $vol := Seq .VolumesPerPod }} - name: vol-{{$volumeIndex}} - {{ if eq $volumeType "ConfigMap" }} - configMap: - name: vol-{{$appName}}-{{AddInt $volumeIndex (MultiplyInt $index $volumesPerPod)}} - {{ end }} - {{ if eq $volumeType "EmptyDir" }} - emptyDir: {} - {{ end }} - {{ if eq $volumeType "Secret" }} - secret: - secretName: vol-{{$appName}}-{{AddInt $volumeIndex (MultiplyInt $index $volumesPerPod)}} - {{ end }} - {{ if eq $volumeType "DownwardAPI" }} downwardAPI: items: - path: "pod_name" @@ -46,4 +30,3 @@ spec: fieldRef: fieldPath: metadata.namespace {{ end }} - {{ end }} \ No newline at end of file diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/pod_with_EmptyDirs.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/pod_with_EmptyDirs.yaml new file mode 100644 index 0000000000..ea58fdeaed --- /dev/null +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/pod_with_EmptyDirs.yaml @@ -0,0 +1,25 @@ +{{$index := .Index}} +{{$appName := .AppName}} +{{$volumesPerPod := .VolumesPerPod}} +apiVersion: v1 +kind: Pod +metadata: + name: {{.Name}} + labels: + app: {{.Name}} + group: {{.Group}} +spec: + containers: + - name: {{.Name}} + image: k8s.gcr.io/pause:3.1 + imagePullPolicy: IfNotPresent + volumeMounts: + {{ range $volumeIndex, $vol := Seq .VolumesPerPod }} + - name: vol-{{$volumeIndex}} + mountPath: /usr/share/{{$volumeIndex}} + {{ end }} + volumes: + {{ range $volumeIndex, $vol := Seq .VolumesPerPod }} + - name: vol-{{$volumeIndex}} + emptyDir: {} + {{ end }} diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/Secret.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/Secret.yaml similarity index 78% rename from clusterloader2/testing/storage/pod-startup/ephemeral-volumes/Secret.yaml rename to clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/Secret.yaml index 91b732b371..fc9b8a4049 100644 --- a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/Secret.yaml +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/Secret.yaml @@ -5,4 +5,4 @@ metadata: type: Opaque data: username: dXNlcm5hbWVfCg== - password: cGFzc3dvcmRfCg== \ No newline at end of file + password: cGFzc3dvcmRfCg== diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/override.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/override.yaml new file mode 100644 index 0000000000..cb72e58ee5 --- /dev/null +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/override.yaml @@ -0,0 +1,3 @@ +PROVISION_VOLUME: true +POD_TEMPLATE_PATH: "secret/pod_with_Secrets.yaml" +VOLUME_TEMPLATE_PATH: "secret/Secret.yaml" diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/pod_with_Secrets.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/pod_with_Secrets.yaml new file mode 100644 index 0000000000..981d144408 --- /dev/null +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/pod_with_Secrets.yaml @@ -0,0 +1,26 @@ +{{$index := .Index}} +{{$appName := .AppName}} +{{$volumesPerPod := .VolumesPerPod}} +apiVersion: v1 +kind: Pod +metadata: + name: {{.Name}} + labels: + app: {{.Name}} + group: {{.Group}} +spec: + containers: + - name: {{.Name}} + image: k8s.gcr.io/pause:3.1 + imagePullPolicy: IfNotPresent + volumeMounts: + {{ range $volumeIndex, $vol := Seq .VolumesPerPod }} + - name: vol-{{$volumeIndex}} + mountPath: /usr/share/{{$volumeIndex}} + {{ end }} + volumes: + {{ range $volumeIndex, $vol := Seq .VolumesPerPod }} + - name: vol-{{$volumeIndex}} + secret: + secretName: vol-{{$appName}}-{{AddInt $volumeIndex (MultiplyInt $index $volumesPerPod)}} + {{ end }} diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/README.md b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/README.md deleted file mode 100644 index 74abcd0f39..0000000000 --- a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/README.md +++ /dev/null @@ -1,8 +0,0 @@ -Below test cases can be run with the parameters like; - -- 1 pod with X volumes and 1 node --- `.Nodes := 1`, `$NODES_PER_NAMESPACE := 1`, `$PODS_PER_NODE := 1`, `$VOLUMES_PER_POD := ` -- X pods with 1 volume each on 1 node in parallel --- `.Nodes := 1`, `$NODES_PER_NAMESPACE := 1`, `$PODS_PER_NODE := `, `$VOLUMES_PER_POD := 1` -- X pods with 1 volume each in parallel --- `.Nodes := `, `$NODES_PER_NAMESPACE := `, `$PODS_PER_NODE := `, `$VOLUMES_PER_POD := 1` \ No newline at end of file diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/configmap/override.yaml b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/configmap/override.yaml deleted file mode 100644 index 9a5300016e..0000000000 --- a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/configmap/override.yaml +++ /dev/null @@ -1,2 +0,0 @@ -VOLUME_TYPE: ConfigMap -PROVISION_VOLUME: true \ No newline at end of file diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/downwardapi/override.yaml b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/downwardapi/override.yaml deleted file mode 100644 index 56b6748ad8..0000000000 --- a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/downwardapi/override.yaml +++ /dev/null @@ -1 +0,0 @@ -VOLUME_TYPE: DownwardAPI \ No newline at end of file diff --git a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/secret/override.yaml b/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/secret/override.yaml deleted file mode 100644 index 48a79ff5f9..0000000000 --- a/clusterloader2/testing/storage/pod-startup/ephemeral-volumes/secret/override.yaml +++ /dev/null @@ -1,2 +0,0 @@ -VOLUME_TYPE: Secret -PROVISION_VOLUME: true \ No newline at end of file From bf3143507087859e22f00f8b61fa34195dfacb7e Mon Sep 17 00:00:00 2001 From: Mucahit Kurt Date: Wed, 10 Jul 2019 07:49:46 +0300 Subject: [PATCH 3/5] move podstartupmeasurements to up Signed-off-by: Mucahit Kurt --- .../ephemeral-volumes/1_node/config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml index c3aac4b35b..30b006fd9b 100644 --- a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml @@ -25,11 +25,19 @@ tuningSets: qpsLoad: qps: {{$LOAD_TEST_THROUGHPUT}} steps: +# Start measurements - measurements: - Identifier: APIResponsiveness Method: APIResponsiveness Params: action: reset +- measurements: + - Identifier: PodWithMultiVolumeStartupLatency + Method: PodStartupLatency + Params: + action: start + labelSelector: group = {{$GROUP}} + threshold: {{$podStartupTimeout}}s {{ if $PROVISION_VOLUME }} # Provision volumes - phases: @@ -42,14 +50,6 @@ steps: - basename: vol-{{$APP_NAME}} objectTemplatePath: {{$VOLUME_TEMPLATE_PATH}} {{ end }} -# Start measurements -- measurements: - - Identifier: PodWithMultiVolumeStartupLatency - Method: PodStartupLatency - Params: - action: start - labelSelector: group = {{$GROUP}} - threshold: {{$podStartupTimeout}}s # Create pods - phases: - namespaceRange: From 30effd8675c821092b6edd86cf0ae2f1cc1bd129 Mon Sep 17 00:00:00 2001 From: Mucahit Kurt Date: Wed, 10 Jul 2019 23:43:06 +0300 Subject: [PATCH 4/5] move podstartupmeasurements gather to bottom, rename volume config files Signed-off-by: Mucahit Kurt --- .../pod-startup/ephemeral-volumes/1_node/config.yaml | 12 +++++------- .../configmap/{ConfigMap.yaml => configmap.yaml} | 0 .../ephemeral-volumes/1_node/configmap/override.yaml | 4 ++-- ..._with_ConfigMaps.yaml => pod_with_configmap.yaml} | 0 .../1_node/downwardapi/override.yaml | 2 +- ...h_DownwardAPIs.yaml => pod_with_downwardapi.yaml} | 0 ...od_with_EmptyDirs.yaml => pod_with_emptydir.yaml} | 0 .../ephemeral-volumes/1_node/secret/override.yaml | 4 ++-- .../{pod_with_Secrets.yaml => pod_with_secret.yaml} | 0 .../1_node/secret/{Secret.yaml => secret.yaml} | 0 10 files changed, 10 insertions(+), 12 deletions(-) rename clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/{ConfigMap.yaml => configmap.yaml} (100%) rename clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/{pod_with_ConfigMaps.yaml => pod_with_configmap.yaml} (100%) rename clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/{pod_with_DownwardAPIs.yaml => pod_with_downwardapi.yaml} (100%) rename clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/{pod_with_EmptyDirs.yaml => pod_with_emptydir.yaml} (100%) rename clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/{pod_with_Secrets.yaml => pod_with_secret.yaml} (100%) rename clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/{Secret.yaml => secret.yaml} (100%) diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml index 30b006fd9b..1f70fb4188 100644 --- a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml @@ -3,7 +3,7 @@ #Constants {{$TOTAL_PODS := DefaultParam .TOTAL_PODS 1}} -{{$POD_TEMPLATE_PATH := DefaultParam .POD_TEMPLATE_PATH "pod_with_EmptyDirs.yaml"}} +{{$POD_TEMPLATE_PATH := DefaultParam .POD_TEMPLATE_PATH "pod_with_emptydir.yaml"}} {{$LOAD_TEST_THROUGHPUT := DefaultParam .LOAD_TEST_THROUGHPUT 10}} {{$VOLUMES_PER_POD := DefaultParam .VOLUMES_PER_POD 4}} @@ -31,7 +31,6 @@ steps: Method: APIResponsiveness Params: action: reset -- measurements: - Identifier: PodWithMultiVolumeStartupLatency Method: PodStartupLatency Params: @@ -72,11 +71,6 @@ steps: labelSelector: group = {{$GROUP}} # TODO decide this after test roll-out phase timeout: 15m -- measurements: - - Identifier: PodWithMultiVolumeStartupLatency - Method: PodStartupLatency - Params: - action: gather # Delete pods - phases: - namespaceRange: @@ -89,6 +83,10 @@ steps: objectTemplatePath: {{$POD_TEMPLATE_PATH}} # Collect measurements - measurements: + - Identifier: PodWithMultiVolumeStartupLatency + Method: PodStartupLatency + Params: + action: gather - Identifier: APIResponsiveness Method: APIResponsiveness Params: diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/ConfigMap.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/configmap.yaml similarity index 100% rename from clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/ConfigMap.yaml rename to clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/configmap.yaml diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/override.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/override.yaml index fb14d6a7a6..c8602d2600 100644 --- a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/override.yaml +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/override.yaml @@ -1,3 +1,3 @@ PROVISION_VOLUME: true -POD_TEMPLATE_PATH: "configmap/pod_with_ConfigMaps.yaml" -VOLUME_TEMPLATE_PATH: "configmap/ConfigMap.yaml" +POD_TEMPLATE_PATH: "configmap/pod_with_configmap.yaml" +VOLUME_TEMPLATE_PATH: "configmap/configmap.yaml" diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/pod_with_ConfigMaps.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/pod_with_configmap.yaml similarity index 100% rename from clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/pod_with_ConfigMaps.yaml rename to clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/pod_with_configmap.yaml diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/override.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/override.yaml index ecf6234f84..835d890a36 100644 --- a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/override.yaml +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/override.yaml @@ -1 +1 @@ -POD_TEMPLATE_PATH: "downwardapi/pod_with_DownwardAPIs.yaml" +POD_TEMPLATE_PATH: "downwardapi/pod_with_downwardapi.yaml" diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/pod_with_DownwardAPIs.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/pod_with_downwardapi.yaml similarity index 100% rename from clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/pod_with_DownwardAPIs.yaml rename to clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/pod_with_downwardapi.yaml diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/pod_with_EmptyDirs.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/pod_with_emptydir.yaml similarity index 100% rename from clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/pod_with_EmptyDirs.yaml rename to clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/pod_with_emptydir.yaml diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/override.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/override.yaml index cb72e58ee5..b33a957772 100644 --- a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/override.yaml +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/override.yaml @@ -1,3 +1,3 @@ PROVISION_VOLUME: true -POD_TEMPLATE_PATH: "secret/pod_with_Secrets.yaml" -VOLUME_TEMPLATE_PATH: "secret/Secret.yaml" +POD_TEMPLATE_PATH: "secret/pod_with_secret.yaml" +VOLUME_TEMPLATE_PATH: "secret/secret.yaml" diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/pod_with_Secrets.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/pod_with_secret.yaml similarity index 100% rename from clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/pod_with_Secrets.yaml rename to clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/pod_with_secret.yaml diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/Secret.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/secret.yaml similarity index 100% rename from clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/Secret.yaml rename to clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/secret.yaml From 377ab09489d2cc9a1602f3824bffa28166e10b70 Mon Sep 17 00:00:00 2001 From: Mucahit Kurt Date: Tue, 16 Jul 2019 22:26:52 +0300 Subject: [PATCH 5/5] override file is added for max volumes per node test case, update default value for volumes per pod Signed-off-by: Mucahit Kurt --- .../storage/pod-startup/ephemeral-volumes/1_node/README.md | 2 ++ .../storage/pod-startup/ephemeral-volumes/1_node/config.yaml | 2 +- .../ephemeral-volumes/1_node/max_volumes_per_node/override.yaml | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/max_volumes_per_node/override.yaml diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/README.md b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/README.md index ed4734817c..df1cca74e3 100644 --- a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/README.md +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/README.md @@ -6,3 +6,5 @@ Below test cases can be run with the parameters like; -- `.Nodes := 1`, `$TOTAL_PODS := `, `$VOLUMES_PER_POD := 1` To test with different volume types please use the override file for the specific volume type, default volume type is `EmptyDir` + +To test for the max volume per node test case please use override file under `max_volumes_per_node`, default test case is max volume per node. diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml index 1f70fb4188..d336ceb71b 100644 --- a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml @@ -6,7 +6,7 @@ {{$POD_TEMPLATE_PATH := DefaultParam .POD_TEMPLATE_PATH "pod_with_emptydir.yaml"}} {{$LOAD_TEST_THROUGHPUT := DefaultParam .LOAD_TEST_THROUGHPUT 10}} -{{$VOLUMES_PER_POD := DefaultParam .VOLUMES_PER_POD 4}} +{{$VOLUMES_PER_POD := DefaultParam .VOLUMES_PER_POD 100}} {{$VOLUME_TEMPLATE_PATH := .VOLUME_TEMPLATE_PATH}} {{$PROVISION_VOLUME := DefaultParam .PROVISION_VOLUME false}} diff --git a/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/max_volumes_per_node/override.yaml b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/max_volumes_per_node/override.yaml new file mode 100644 index 0000000000..ff85916cad --- /dev/null +++ b/clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/max_volumes_per_node/override.yaml @@ -0,0 +1,2 @@ +TOTAL_PODS: 100 +VOLUMES_PER_POD: 1