add persistent volume scale tests

clusterloader2/testing/experimental/storage/pod-startup/README.md

@@ -10,10 +10,9 @@
     - tries to stress the max number of volumes a single pod can have
 - X pods with 1 volume each on cluster in parallel
     - run on a cluster with num nodes that is a multiple of `NODES_PER_NAMESPACE` (default 100)
-    - no config file is necessary (this is the default)
+    - use override file in `tests/cluster_load_scale_by_nodes` 
 
 **Volume Type**
 
-Each test must use a type of volume. The empty-dir volume
-is used by default. Use a override in `volume-types` to set
+Each test must use a type of volume. Use a override in `volume-types` to set
 a specific type of volume to test.

clusterloader2/testing/experimental/storage/pod-startup/cluster_load_scale_by_nodes/override.yaml

@@ -0,0 +1,3 @@
+TEST_NAME: cluster-load-scale-by-node
+PODS_PER_NODE: 10
+VOLUMES_PER_POD: 1

clusterloader2/testing/experimental/storage/pod-startup/config.yaml

@@ -1,100 +1,142 @@
 # ASSUMPTIONS:
 # - Number of nodes should be divisible by NODES_PER_NAMESPACE (default 100).
 
-#Constants
-# TODO(mucahitkurt) copied from load tests, open for discussion.
-{{$NODE_MODE := DefaultParam .NODE_MODE "allnodes"}}
-{{$NODES_PER_NAMESPACE := DefaultParam .NODES_PER_NAMESPACE 100}}
-{{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 30}}
+# Cluster Variables
+  {{$NODE_MODE := DefaultParam .NODE_MODE "allnodes"}}
+  {{$NODES_PER_NAMESPACE := DefaultParam .NODES_PER_NAMESPACE 100}}
 
-{{$POD_TEMPLATE_PATH := DefaultParam .POD_TEMPLATE_PATH "../volume-types/emptydir/pod_with_emptydir.yaml"}}
-{{$LOAD_TEST_THROUGHPUT := DefaultParam .LOAD_TEST_THROUGHPUT 10}}
+# Test Variales
+  {{$TEST_NAME := DefaultParam .TEST_NAME "generic-test"}}
+  {{$PODS_PER_NODE := .PODS_PER_NODE}}
+  {{$POD_TEMPLATE_PATH := .POD_TEMPLATE_PATH }}
+  {{$VOLUMES_PER_POD := .VOLUMES_PER_POD}}
+  {{$VOLUME_TEMPLATE_PATH := .VOLUME_TEMPLATE_PATH}}
+  {{$PROVISION_VOLUME := DefaultParam .PROVISION_VOLUME false}}
+  {{$VOL_SIZE := DefaultParam .VOL_SIZE "8Gi"}}
+  {{$WAIT_FOR_PVC := DefaultParam .WAIT_FOR_PVC false}}
+  # TODO(hantaowang): remove knob after deciding on right values
+  {{$POD_STARTUP_TIMEOUT := DefaultParam .POD_STARTUP_TIMEOUT "15m"}}
+  {{$POD_STARTUP_SLO := DefaultParam .POD_STARTUP_SLO 300}}
 
-{{$VOLUMES_PER_POD := DefaultParam .VOLUMES_PER_POD 1}}
-{{$VOLUME_TEMPLATE_PATH := .VOLUME_TEMPLATE_PATH}}
-{{$PROVISION_VOLUME := DefaultParam .PROVISION_VOLUME false}}
+# Computed Variables
+  {{$namespaces := DivideInt .Nodes $NODES_PER_NAMESPACE | MaxInt 1}}
+  {{$totalPods := MultiplyInt $namespaces $NODES_PER_NAMESPACE $PODS_PER_NODE}}
+  {{$podsPerNamespace := MultiplyInt $NODES_PER_NAMESPACE $PODS_PER_NODE}}
+  {{$volumesPerNamespace := MultiplyInt $podsPerNamespace $VOLUMES_PER_POD}}
+  {{$totalVols := MultiplyInt $volumesPerNamespace $namespaces}}
+  {{$PVCBoundTime := MultiplyInt $totalVols 2 | MaxInt 60}}
 
-{{$APP_NAME := "pod-load"}}
-{{$GROUP := "pod-with-ephemeral-volume-startup-latency"}}
 
-#Variables
-# used stateless pod SLO for initial run
-{{$namespaces := DivideInt .Nodes $NODES_PER_NAMESPACE | MaxInt 1}}
-{{$podStartupTimeout := 5}}
-{{$totalPods := MultiplyInt $namespaces $NODES_PER_NAMESPACE $PODS_PER_NODE}}
-{{$podsPerNamespace := MultiplyInt $NODES_PER_NAMESPACE $PODS_PER_NODE}}
-{{$volumesPerNamespace := MultiplyInt $podsPerNamespace $VOLUMES_PER_POD}}
-
-name: pod-with-ephemeral-volume-startup-latency
+name: storage-{{$TEST_NAME}}
 automanagedNamespaces: {{$namespaces}}
 tuningSets:
-- name: UniformQPS
-  qpsLoad:
-    qps: {{$LOAD_TEST_THROUGHPUT}}
+  - name: Sequence
+    parallelismLimitedLoad:
+      parallelismLimit: 1
 steps:
 # Start measurements
 - measurements:
   - Identifier: APIResponsiveness
     Method: APIResponsiveness
     Params:
       action: reset
-  - Identifier: PodWithMultiVolumeStartupLatency
+  - Identifier: APIResponsivenessPrometheus
+    Method: APIResponsivenessPrometheus
+    Params:
+      action: start
+  - Identifier: TestMetrics
+    Method: TestMetrics
+    Params:
+      action: start
+      nodeMode: {{$NODE_MODE}}
+  - Identifier: PodWithVolumesStartupLatency
     Method: PodStartupLatency
     Params:
       action: start
-      labelSelector: group = {{$GROUP}}
-      threshold: {{$podStartupTimeout}}s
+      labelSelector: group = volume-test
+      threshold: {{$POD_STARTUP_SLO}}s
 {{ if $PROVISION_VOLUME }}
 # Provision volumes
 - phases:
   - namespaceRange:
       min: 1
       max: {{$namespaces}}
     replicasPerNamespace: {{$volumesPerNamespace}}
-    tuningSet: UniformQPS
+    tuningSet: Sequence
     objectBundle:
-    - basename: vol-{{$APP_NAME}}
+    - basename: vol-{{$TEST_NAME}}
       objectTemplatePath: {{$VOLUME_TEMPLATE_PATH}}
+      templateFillMap:
+        Group: volume-test
+        VolSize: {{$VOL_SIZE}}
+{{ end }}
+{{ if $WAIT_FOR_PVC }}
+- measurements:
+  - Identifier: WaitForPVCsToBeBound
+    Method: WaitForBoundPVCs
+    Params:
+      desiredPVCCount: {{$totalVols}}
+      apiVersion: v1
+      labelSelector: group = volume-test
+      timeout: {{$PVCBoundTime}}s
 {{ end }}
 # Create pods
 - phases:
   - namespaceRange:
       min: 1
       max: {{$namespaces}}
     replicasPerNamespace: {{$podsPerNamespace}}
-    tuningSet: UniformQPS
+    tuningSet: Sequence
     objectBundle:
-    - basename: {{$APP_NAME}}
+    - basename: pod-{{$TEST_NAME}}
       objectTemplatePath: {{$POD_TEMPLATE_PATH}}
       templateFillMap:
-        Group: {{$GROUP}}
+        Group: volume-test
         VolumesPerPod: {{$VOLUMES_PER_POD}}
-        AppName: {{$APP_NAME}}
 - measurements:
-  - Identifier: WaitForRunningPodsWithStorage
+  - Identifier: WaitForRunningPodsWithVolumes
     Method: WaitForRunningPods
     Params:
       desiredPodCount: {{$totalPods}}
-      labelSelector: group = {{$GROUP}}
-      # TODO(mucahitkurt) decide this after test roll-out phase
-      timeout: 15m
+      labelSelector: group = volume-test
+      timeout: {{$POD_STARTUP_TIMEOUT}}
 # Delete pods
 - phases:
   - namespaceRange:
       min: 1
       max: {{$namespaces}}
     replicasPerNamespace: 0
-    tuningSet: UniformQPS
+    tuningSet: Sequence
     objectBundle:
-    - basename: {{$APP_NAME}}
+    - basename: pod-{{$TEST_NAME}}
       objectTemplatePath: {{$POD_TEMPLATE_PATH}}
+{{ if $PROVISION_VOLUME }}
+# Delete volumes
+- phases:
+  - namespaceRange:
+      min: 1
+      max: {{$namespaces}}
+    replicasPerNamespace: 0
+    tuningSet: Sequence
+    objectBundle:
+      - basename: vol-{{$TEST_NAME}}
+        objectTemplatePath: {{$VOLUME_TEMPLATE_PATH}}
+{{ end }}
 # Collect measurements
 - measurements:
-  - Identifier: PodWithMultiVolumeStartupLatency
-    Method: PodStartupLatency
-    Params:
-      action: gather
   - Identifier: APIResponsiveness
     Method: APIResponsiveness
     Params:
       action: gather
+  - Identifier: APIResponsivenessPrometheus
+    Method: APIResponsivenessPrometheus
+    Params:
+      action: gather
+  - Identifier: TestMetrics
+    Method: TestMetrics
+    Params:
+      action: gather
+  - Identifier: PodWithVolumesStartupLatency
+    Method: PodStartupLatency
+    Params:
+      action: gather

clusterloader2/testing/experimental/storage/pod-startup/max_volumes_per_node/override.yaml

@@ -1,3 +1,4 @@
+TEST_NAME: max-volumes-per-node
 PODS_PER_NODE: 100
 VOLUMES_PER_POD: 1
 NODES_PER_NAMESPACE: 1

clusterloader2/testing/experimental/storage/pod-startup/max_volumes_per_pod/override.yaml

@@ -1,3 +1,4 @@
+TEST_NAME: max-volumes-per-pod
 PODS_PER_NODE: 1
 VOLUMES_PER_POD: 100
 NODES_PER_NAMESPACE: 1

clusterloader2/testing/experimental/storage/pod-startup/volume-types/configmap/configmap.yaml

@@ -2,5 +2,8 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{.Name}}
+  labels:
+    app: {{.Name}}
+    group: {{.Group}}
 data:
   TEST_KEY1: testData1

clusterloader2/testing/experimental/storage/pod-startup/volume-types/persistentvolume/override.yaml

@@ -0,0 +1,4 @@
+PROVISION_VOLUME: true
+WAIT_FOR_PVC: true
+POD_TEMPLATE_PATH: "volume-types/persistentvolume/pod_with_pvc.yaml"
+VOLUME_TEMPLATE_PATH: "volume-types/persistentvolume/pvc.yaml"

clusterloader2/testing/experimental/storage/pod-startup/volume-types/persistentvolume/pod_with_pvc.yaml

@@ -0,0 +1,27 @@
+{{$index := .Index}}
+{{$appName := .AppName}}
+{{$volumesPerPod := .VolumesPerPod}}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{.Name}}
+  labels:
+    app: {{.Name}}
+    group: {{.Group}}
+spec:
+  containers:
+    - name: {{.Name}}
+      image: k8s.gcr.io/pause:3.1
+      imagePullPolicy: IfNotPresent
+      volumeMounts:
+      {{ range $volumeIndex, $vol := Seq .VolumesPerPod }}
+      - name: vol-{{$volumeIndex}}
+        mountPath: /usr/share/{{$volumeIndex}}
+      {{ end }}
+  volumes:
+    {{ range $volumeIndex, $vol := Seq .VolumesPerPod }}
+    - name: vol-{{$volumeIndex}}
+      persistentVolumeClaim:
+        claimName: vol-{{$appName}}-{{AddInt $volumeIndex (MultiplyInt $index $volumesPerPod)}}
+        readOnly: false
+    {{ end }}

clusterloader2/testing/experimental/storage/pod-startup/volume-types/persistentvolume/pvc.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{.Name}}
+  labels:
+    app: {{.Name}}
+    group: {{.Group}}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{.VolSize}}

clusterloader2/testing/experimental/storage/pod-startup/volume-types/secret/secret.yaml

@@ -2,6 +2,9 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: {{.Name}}
+  labels:
+    app: {{.Name}}
+    group: {{.Group}}
 type: Opaque
 data:
   username: dXNlcm5hbWVfCg==