Merge pull request #601 from mucahitkurt/storage/scalability-performa…

…nce-tests-pod-startup

Scalability and Performance Test Cases for Ephemeral Volumes

clusterloader2/pkg/config/template_functions.go

@@ -57,9 +57,14 @@ func GetFuncs() template.FuncMap {
 		"DefaultParam":  defaultParam,
 		"IncludeFile":   includeFile,
 		"YamlQuote":     yamlQuote,
+		"Seq":           seq,
 	}
 }
 
+func seq(size interface{}) []int {
+	return make([]int, int(toFloat64(size)))
+}
+
 func toFloat64(val interface{}) float64 {
 	switch i := val.(type) {
 	case float64:

clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/README.md

@@ -0,0 +1,10 @@
+Below test cases can be run with the parameters like;
+
+- 1 pod with X volumes and 1 node
+-- `.Nodes := 1`, `$TOTAL_PODS := 1`, `$VOLUMES_PER_POD := <X>`
+- X pods with 1 volume each on 1 node in parallel
+-- `.Nodes := 1`, `$TOTAL_PODS := <X>`, `$VOLUMES_PER_POD := 1`
+
+To test with different volume types please use the override file for the specific volume type, default volume type is `EmptyDir`
+
+To test for the max volume per node test case please use override file under `max_volumes_per_node`, default test case is max volume per node.

clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/config.yaml

@@ -0,0 +1,93 @@
+# ASSUMPTIONS:
+# - this test cases should be run on only 1 node
+
+#Constants
+{{$TOTAL_PODS := DefaultParam .TOTAL_PODS 1}}
+{{$POD_TEMPLATE_PATH := DefaultParam .POD_TEMPLATE_PATH "pod_with_emptydir.yaml"}}
+{{$LOAD_TEST_THROUGHPUT := DefaultParam .LOAD_TEST_THROUGHPUT 10}}
+
+{{$VOLUMES_PER_POD := DefaultParam .VOLUMES_PER_POD 100}}
+{{$VOLUME_TEMPLATE_PATH := .VOLUME_TEMPLATE_PATH}}
+{{$PROVISION_VOLUME := DefaultParam .PROVISION_VOLUME false}}
+
+{{$APP_NAME := "pod-load"}}
+{{$GROUP := "pod-with-ephemeral-volume-startup-latency"}}
+
+#Variables
+# used stateless pod SLO for initial run
+{{$podStartupTimeout := 5}}
+{{$totalVolumes := MultiplyInt $TOTAL_PODS $VOLUMES_PER_POD}}
+
+name: pod-with-ephemeral-volume-startup-latency
+automanagedNamespaces: 1
+tuningSets:
+- name: UniformQPS
+  qpsLoad:
+    qps: {{$LOAD_TEST_THROUGHPUT}}
+steps:
+# Start measurements
+- measurements:
+  - Identifier: APIResponsiveness
+    Method: APIResponsiveness
+    Params:
+      action: reset
+  - Identifier: PodWithMultiVolumeStartupLatency
+    Method: PodStartupLatency
+    Params:
+      action: start
+      labelSelector: group = {{$GROUP}}
+      threshold: {{$podStartupTimeout}}s
+{{ if $PROVISION_VOLUME }}
+# Provision volumes
+- phases:
+  - namespaceRange:
+      min: 1
+      max: 1
+    replicasPerNamespace: {{$totalVolumes}}
+    tuningSet: UniformQPS
+    objectBundle:
+    - basename: vol-{{$APP_NAME}}
+      objectTemplatePath: {{$VOLUME_TEMPLATE_PATH}}
+{{ end }}
+# Create pods
+- phases:
+  - namespaceRange:
+      min: 1
+      max: 1
+    replicasPerNamespace: {{$TOTAL_PODS}}
+    tuningSet: UniformQPS
+    objectBundle:
+    - basename: {{$APP_NAME}}
+      objectTemplatePath: {{$POD_TEMPLATE_PATH}}
+      templateFillMap:
+        Group: {{$GROUP}}
+        VolumesPerPod: {{$VOLUMES_PER_POD}}
+        AppName: {{$APP_NAME}}
+- measurements:
+  - Identifier: WaitForRunningPodsWithStorage
+    Method: WaitForRunningPods
+    Params:
+      desiredPodCount: {{$TOTAL_PODS}}
+      labelSelector: group = {{$GROUP}}
+      # TODO decide this after test roll-out phase
+      timeout: 15m
+# Delete pods
+- phases:
+  - namespaceRange:
+      min: 1
+      max: 1
+    replicasPerNamespace: 0
+    tuningSet: UniformQPS
+    objectBundle:
+    - basename: {{$APP_NAME}}
+      objectTemplatePath: {{$POD_TEMPLATE_PATH}}
+# Collect measurements
+- measurements:
+  - Identifier: PodWithMultiVolumeStartupLatency
+    Method: PodStartupLatency
+    Params:
+      action: gather
+  - Identifier: APIResponsiveness
+    Method: APIResponsiveness
+    Params:
+      action: gather

clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/configmap.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{.Name}}
+data:
+  TEST_KEY1: testData1

clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/override.yaml

@@ -0,0 +1,3 @@
+PROVISION_VOLUME: true
+POD_TEMPLATE_PATH: "configmap/pod_with_configmap.yaml"
+VOLUME_TEMPLATE_PATH: "configmap/configmap.yaml"

clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/configmap/pod_with_configmap.yaml

@@ -0,0 +1,26 @@
+{{$index := .Index}}
+{{$appName := .AppName}}
+{{$volumesPerPod := .VolumesPerPod}}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{.Name}}
+  labels:
+    app: {{.Name}}
+    group: {{.Group}}
+spec:
+  containers:
+  - name: {{.Name}}
+    image: k8s.gcr.io/pause:3.1
+    imagePullPolicy: IfNotPresent
+    volumeMounts:
+    {{ range $volumeIndex, $vol := Seq .VolumesPerPod }}
+    - name: vol-{{$volumeIndex}}
+      mountPath: /usr/share/{{$volumeIndex}}
+    {{ end }}
+  volumes:
+  {{ range $volumeIndex, $vol := Seq .VolumesPerPod }}
+  - name: vol-{{$volumeIndex}}
+    configMap:
+      name: vol-{{$appName}}-{{AddInt $volumeIndex (MultiplyInt $index $volumesPerPod)}}
+  {{ end }}

clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/override.yaml

@@ -0,0 +1 @@
+POD_TEMPLATE_PATH: "downwardapi/pod_with_downwardapi.yaml"

clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/downwardapi/pod_with_downwardapi.yaml

@@ -0,0 +1,32 @@
+{{$index := .Index}}
+{{$appName := .AppName}}
+{{$volumesPerPod := .VolumesPerPod}}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{.Name}}
+  labels:
+    app: {{.Name}}
+    group: {{.Group}}
+spec:
+  containers:
+  - name: {{.Name}}
+    image: k8s.gcr.io/pause:3.1
+    imagePullPolicy: IfNotPresent
+    volumeMounts:
+    {{ range $volumeIndex, $vol := Seq .VolumesPerPod }}
+    - name: vol-{{$volumeIndex}}
+      mountPath: /usr/share/{{$volumeIndex}}
+    {{ end }}
+  volumes:
+  {{ range $volumeIndex, $vol := Seq .VolumesPerPod }}
+  - name: vol-{{$volumeIndex}}
+    downwardAPI:
+      items:
+      - path: "pod_name"
+        fieldRef:
+          fieldPath: metadata.name
+      - path: "pod_namespace"
+        fieldRef:
+          fieldPath: metadata.namespace
+  {{ end }}

clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/max_volumes_per_node/override.yaml

@@ -0,0 +1,2 @@
+TOTAL_PODS: 100
+VOLUMES_PER_POD: 1

clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/pod_with_emptydir.yaml

@@ -0,0 +1,25 @@
+{{$index := .Index}}
+{{$appName := .AppName}}
+{{$volumesPerPod := .VolumesPerPod}}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{.Name}}
+  labels:
+    app: {{.Name}}
+    group: {{.Group}}
+spec:
+  containers:
+  - name: {{.Name}}
+    image: k8s.gcr.io/pause:3.1
+    imagePullPolicy: IfNotPresent
+    volumeMounts:
+    {{ range $volumeIndex, $vol := Seq .VolumesPerPod }}
+    - name: vol-{{$volumeIndex}}
+      mountPath: /usr/share/{{$volumeIndex}}
+    {{ end }}
+  volumes:
+  {{ range $volumeIndex, $vol := Seq .VolumesPerPod }}
+  - name: vol-{{$volumeIndex}}
+    emptyDir: {}
+  {{ end }}

clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/override.yaml

@@ -0,0 +1,3 @@
+PROVISION_VOLUME: true
+POD_TEMPLATE_PATH: "secret/pod_with_secret.yaml"
+VOLUME_TEMPLATE_PATH: "secret/secret.yaml"

clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/pod_with_secret.yaml

@@ -0,0 +1,26 @@
+{{$index := .Index}}
+{{$appName := .AppName}}
+{{$volumesPerPod := .VolumesPerPod}}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{.Name}}
+  labels:
+    app: {{.Name}}
+    group: {{.Group}}
+spec:
+  containers:
+  - name: {{.Name}}
+    image: k8s.gcr.io/pause:3.1
+    imagePullPolicy: IfNotPresent
+    volumeMounts:
+    {{ range $volumeIndex, $vol := Seq .VolumesPerPod }}
+    - name: vol-{{$volumeIndex}}
+      mountPath: /usr/share/{{$volumeIndex}}
+    {{ end }}
+  volumes:
+  {{ range $volumeIndex, $vol := Seq .VolumesPerPod }}
+  - name: vol-{{$volumeIndex}}
+    secret:
+      secretName: vol-{{$appName}}-{{AddInt $volumeIndex (MultiplyInt $index $volumesPerPod)}}
+  {{ end }}

clusterloader2/testing/experimental/storage/pod-startup/ephemeral-volumes/1_node/secret/secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{.Name}}
+type: Opaque
+data:
+  username: dXNlcm5hbWVfCg==
+  password: cGFzc3dvcmRfCg==