Add TEST_TYPE parameter to each testcase

solana-labs · Mar 17, 2020 · 3a3bddf · 3a3bddf
1 parent e6cc249
commit 3a3bddf
Show file tree

Hide file tree

Showing 30 changed files with 31 additions and 1 deletion.
diff --git a/system-test/deprecated-testcases/colo-cpu-only-perf.yml b/system-test/deprecated-testcases/colo-cpu-only-perf.yml
@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 2
       CLIENT_OPTIONS: "bench-tps=2=--tx_count 20000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/deprecated-testcases/colo-gpu-perf-high-txcount.yml b/system-test/deprecated-testcases/colo-gpu-perf-high-txcount.yml
@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 2
       CLIENT_OPTIONS: "bench-tps=2=--tx_count 30000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/deprecated-testcases/colo-gpu-perf.yml b/system-test/deprecated-testcases/colo-gpu-perf.yml
@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 2
       CLIENT_OPTIONS: "bench-tps=2=--tx_count 20000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/deprecated-testcases/gce-gpu-perf-100-node.yml b/system-test/deprecated-testcases/gce-gpu-perf-100-node.yml
@@ -15,5 +15,6 @@ steps:
       ALLOW_BOOT_FAILURES: "true"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"
diff --git a/system-test/partition-testcases/colo-3-partition.yml b/system-test/partition-testcases/colo-3-partition.yml
@@ -15,5 +15,6 @@ steps:
       PARTITION_ACTIVE_DURATION: 30
       PARTITION_INACTIVE_DURATION: 30
       PARTITION_ITERATION_COUNT: 5
+      TEST_TYPE: "partition"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/partition-testcases/colo-partition-long-sanity-test.yml b/system-test/partition-testcases/colo-partition-long-sanity-test.yml
@@ -15,5 +15,6 @@ steps:
       PARTITION_ACTIVE_DURATION: 60
       PARTITION_INACTIVE_DURATION: 60
       PARTITION_ITERATION_COUNT: 10
+      TEST_TYPE: "partition"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/partition-testcases/colo-partition-once-then-stabilize.yml b/system-test/partition-testcases/colo-partition-once-then-stabilize.yml
@@ -15,5 +15,6 @@ steps:
       PARTITION_ACTIVE_DURATION: 60
       PARTITION_INACTIVE_DURATION: 300
       PARTITION_ITERATION_COUNT: 1
+      TEST_TYPE: "partition"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/partition-testcases/gce-5-node-3-partition.yml b/system-test/partition-testcases/gce-5-node-3-partition.yml
@@ -18,5 +18,6 @@ steps:
       PARTITION_ACTIVE_DURATION: 30
       PARTITION_INACTIVE_DURATION: 30
       PARTITION_ITERATION_COUNT: 5
+      TEST_TYPE: "partition"
     agents:
       - "queue=testnet-deploy"
diff --git a/system-test/partition-testcases/gce-5-node-single-region-2-partitions.yml b/system-test/partition-testcases/gce-5-node-single-region-2-partitions.yml
@@ -18,5 +18,6 @@ steps:
       PARTITION_ACTIVE_DURATION: 60
       PARTITION_INACTIVE_DURATION: 300
       PARTITION_ITERATION_COUNT: 5
+      TEST_TYPE: "partition"
     agents:
       - "queue=gce-deploy"
diff --git a/system-test/partition-testcases/gce-partition-once-then-stabilize.yml b/system-test/partition-testcases/gce-partition-once-then-stabilize.yml
@@ -18,5 +18,6 @@ steps:
       PARTITION_ACTIVE_DURATION: 60
       PARTITION_INACTIVE_DURATION: 300
       PARTITION_ITERATION_COUNT: 1
+      TEST_TYPE: "partition"
     agents:
       - "queue=gce-deploy"
diff --git a/system-test/performance-testcases/aws-cpu-only-perf-10-node.yml b/system-test/performance-testcases/aws-cpu-only-perf-10-node.yml
@@ -15,5 +15,6 @@ steps:
       TESTNET_ZONES: "us-west-1a,us-west-1c,us-east-1a,eu-west-1a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=aws-deploy"
diff --git a/system-test/performance-testcases/aws-cpu-only-perf-5-node.yml b/system-test/performance-testcases/aws-cpu-only-perf-5-node.yml
@@ -15,5 +15,6 @@ steps:
       TESTNET_ZONES: "us-west-1a,us-west-1c,us-east-1a,eu-west-1a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=aws-deploy"
diff --git a/system-test/performance-testcases/azure-cpu-only-perf-5-node.yml b/system-test/performance-testcases/azure-cpu-only-perf-5-node.yml
@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "westus"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=azure-deploy"
diff --git a/system-test/performance-testcases/colo-cpu-only-perf-4-val-1-client.yml b/system-test/performance-testcases/colo-cpu-only-perf-4-val-1-client.yml
@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 40000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/performance-testcases/colo-gpu-perf-4-val-1-client.yml b/system-test/performance-testcases/colo-gpu-perf-4-val-1-client.yml
@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 40000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/performance-testcases/colo-gpu-perf-high-txcount-4-val-1-client.yml b/system-test/performance-testcases/colo-gpu-perf-high-txcount-4-val-1-client.yml
@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 60000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/performance-testcases/gce-cpu-only-perf-10-node.yml b/system-test/performance-testcases/gce-cpu-only-perf-10-node.yml
@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"
diff --git a/system-test/performance-testcases/gce-cpu-only-perf-5-node.yml b/system-test/performance-testcases/gce-cpu-only-perf-5-node.yml
@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"
diff --git a/system-test/performance-testcases/gce-gpu-perf-10-node.yml b/system-test/performance-testcases/gce-gpu-perf-10-node.yml
@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"
diff --git a/system-test/performance-testcases/gce-gpu-perf-25-node.yml b/system-test/performance-testcases/gce-gpu-perf-25-node.yml
@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"
diff --git a/system-test/performance-testcases/gce-gpu-perf-5-node.yml b/system-test/performance-testcases/gce-gpu-perf-5-node.yml
@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"
diff --git a/system-test/performance-testcases/gce-gpu-perf-50-node.yml b/system-test/performance-testcases/gce-gpu-perf-50-node.yml
@@ -15,5 +15,6 @@ steps:
       ALLOW_BOOT_FAILURES: "true"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"
diff --git a/system-test/sanity-testcases/colo-cpu-only-quick-sanity-test.yml b/system-test/sanity-testcases/colo-cpu-only-quick-sanity-test.yml
@@ -12,5 +12,6 @@ steps:
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 40000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
       BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/sanity-testcases/colo-partition-sanity-test.yml b/system-test/sanity-testcases/colo-partition-sanity-test.yml
@@ -16,5 +16,6 @@ steps:
       PARTITION_INACTIVE_DURATION: 30
       PARTITION_ITERATION_COUNT: 2
       BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
+      TEST_TYPE: "partition"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/stability-testcases/colo-long-duration-cpu-only-perf.yml b/system-test/stability-testcases/colo-long-duration-cpu-only-perf.yml
@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 30000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/stability-testcases/colo-long-duration-gpu-perf.yml b/system-test/stability-testcases/colo-long-duration-gpu-perf.yml
@@ -11,5 +11,6 @@ steps:
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 30000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/stability-testcases/gce-perf-stability-5-node-single-region.yml b/system-test/stability-testcases/gce-perf-stability-5-node-single-region.yml
@@ -14,5 +14,6 @@ steps:
       TESTNET_ZONES: "us-west1-a"
       USE_PUBLIC_IP_ADDRESSES: "false"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=gce-deploy"
diff --git a/system-test/stability-testcases/gce-stability-5-node.yml b/system-test/stability-testcases/gce-stability-5-node.yml
@@ -13,5 +13,6 @@ steps:
       TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
       USE_PUBLIC_IP_ADDRESSES: "true"
       ADDITIONAL_FLAGS: "--dedicated"
+      TEST_TYPE: "fixed_duration"
     agents:
       - "queue=stability-deploy"
diff --git a/system-test/stake-operations-testcases/offline_stake_gce.yml b/system-test/stake-operations-testcases/offline_stake_gce.yml
@@ -12,6 +12,7 @@ steps:
       NUMBER_OF_CLIENT_NODES: 0
       ADDITIONAL_FLAGS: ""
       BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
+      SKIP_PERF_RESULTS: "true"
       TEST_TYPE: "script"
       CUSTOM_SCRIPT: "system-test/stake-operations-testcases/stake_test_automation.sh"
     agents:

diff --git a/system-test/testnet-automation.sh b/system-test/testnet-automation.sh
@@ -185,7 +185,7 @@ function launch_testnet() {
   SLOTS_PER_SECOND="$(bc <<< "scale=3; ($END_SLOT - $START_SLOT)/($SLOT_COUNT_END_SECONDS - $SLOT_COUNT_START_SECONDS)")"
   execution_step "Average slot rate: $SLOTS_PER_SECOND slots/second over $((SLOT_COUNT_END_SECONDS - SLOT_COUNT_START_SECONDS)) seconds"
 
-  [[ -n $SKIP_PERF_RESULTS ]] || collect_performance_statistics
+  [[ "$SKIP_PERF_RESULTS" = "false" ]] || collect_performance_statistics
 
   echo "slots_per_second: $SLOTS_PER_SECOND" >>"$RESULT_FILE"
 
@@ -201,6 +201,7 @@ execution_step "Initialize Environment"
 [[ -n $TESTNET_TAG ]] || TESTNET_TAG=testnet-automation
 [[ -n $INFLUX_HOST ]] || INFLUX_HOST=https://metrics.solana.com:8086
 [[ -n $BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD ]] || BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD=66
+[[ -n $SKIP_PERF_RESULTS ]] || SKIP_PERF_RESULTS=false
 
 if [[ -z $NUMBER_OF_VALIDATOR_NODES ]]; then
   echo NUMBER_OF_VALIDATOR_NODES not defined