From 90c9462dd45f1b95de7e474d66e0e485fcdfe970 Mon Sep 17 00:00:00 2001 From: Dan Albert Date: Wed, 18 Mar 2020 14:57:19 -0700 Subject: [PATCH] Automated test framework can run scripts on launched clusters. Add offline stake operations test case and script. (#8510) automerge --- system-test/automation_utils.sh | 68 ++++- .../colo-cpu-only-perf.yml | 1 + .../colo-gpu-perf-high-txcount.yml | 1 + .../deprecated-testcases/colo-gpu-perf.yml | 1 + .../gce-gpu-perf-100-node.yml | 1 + .../partition-testcases/colo-3-partition.yml | 1 + .../colo-partition-long-sanity-test.yml | 1 + .../colo-partition-once-then-stabilize.yml | 1 + .../gce-5-node-3-partition.yml | 1 + .../gce-5-node-single-region-2-partitions.yml | 1 + .../gce-partition-once-then-stabilize.yml | 1 + .../aws-cpu-only-perf-10-node.yml | 1 + .../aws-cpu-only-perf-5-node.yml | 1 + .../azure-cpu-only-perf-5-node.yml | 1 + .../colo-cpu-only-perf-4-val-1-client.yml | 1 + .../colo-gpu-perf-4-val-1-client.yml | 1 + ...o-gpu-perf-high-txcount-4-val-1-client.yml | 1 + .../gce-cpu-only-perf-10-node.yml | 1 + .../gce-cpu-only-perf-5-node.yml | 1 + .../gce-gpu-perf-10-node.yml | 1 + .../gce-gpu-perf-25-node.yml | 1 + .../gce-gpu-perf-5-node.yml | 1 + .../gce-gpu-perf-50-node.yml | 1 + .../colo-cpu-only-quick-sanity-test.yml | 5 +- .../colo-partition-sanity-test.yml | 5 +- .../colo-long-duration-cpu-only-perf.yml | 1 + .../colo-long-duration-gpu-perf.yml | 1 + ...ce-perf-stability-5-node-single-region.yml | 1 + .../gce-stability-5-node.yml | 1 + .../offline_stake_colo.yml | 17 ++ .../offline_stake_gce.yml | 18 ++ .../offline_stake_operations.sh | 258 ++++++++++++++++++ .../stake_test_automation.sh | 16 ++ system-test/testnet-automation.sh | 164 +++++------ 34 files changed, 480 insertions(+), 97 deletions(-) create mode 100755 system-test/stake-operations-testcases/offline_stake_colo.yml create mode 100755 system-test/stake-operations-testcases/offline_stake_gce.yml create mode 100755 system-test/stake-operations-testcases/offline_stake_operations.sh create mode 100755 system-test/stake-operations-testcases/stake_test_automation.sh diff --git a/system-test/automation_utils.sh b/system-test/automation_utils.sh index 557d18cb6acfb4..581d242631ab40 100755 --- a/system-test/automation_utils.sh +++ b/system-test/automation_utils.sh @@ -62,6 +62,10 @@ function analyze_packet_loss { function wait_for_bootstrap_validator_stake_drop { max_stake="$1" + if [[ $max_stake -eq 100 ]]; then + return + fi + source "${REPO_ROOT}"/net/common.sh loadConfigFile @@ -85,6 +89,68 @@ function get_slot { ssh "${sshOptions[@]}" "${validatorIpList[0]}" '$HOME/.cargo/bin/solana slot' } +function get_bootstrap_validator_ip_address { + source "${REPO_ROOT}"/net/common.sh + loadConfigFile + echo "${validatorIpList[0]}" +} + +function collect_performance_statistics { + execution_step "Collect performance statistics about run" + declare q_mean_tps=' + SELECT ROUND(MEAN("median_sum")) as "mean_tps" FROM ( + SELECT MEDIAN(sum_count) AS "median_sum" FROM ( + SELECT SUM("count") AS "sum_count" + FROM "'$TESTNET_TAG'"."autogen"."bank-process_transactions" + WHERE time > now() - '"$TEST_DURATION_SECONDS"'s AND count > 0 + GROUP BY time(1s), host_id) + GROUP BY time(1s) + )' + + declare q_max_tps=' + SELECT MAX("median_sum") as "max_tps" FROM ( + SELECT MEDIAN(sum_count) AS "median_sum" FROM ( + SELECT SUM("count") AS "sum_count" + FROM "'$TESTNET_TAG'"."autogen"."bank-process_transactions" + WHERE time > now() - '"$TEST_DURATION_SECONDS"'s AND count > 0 + GROUP BY time(1s), host_id) + GROUP BY time(1s) + )' + + declare q_mean_confirmation=' + SELECT round(mean("duration_ms")) as "mean_confirmation_ms" + FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation" + WHERE time > now() - '"$TEST_DURATION_SECONDS"'s' + + declare q_max_confirmation=' + SELECT round(max("duration_ms")) as "max_confirmation_ms" + FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation" + WHERE time > now() - '"$TEST_DURATION_SECONDS"'s' + + declare q_99th_confirmation=' + SELECT round(percentile("duration_ms", 99)) as "99th_percentile_confirmation_ms" + FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation" + WHERE time > now() - '"$TEST_DURATION_SECONDS"'s' + + declare q_max_tower_distance_observed=' + SELECT MAX("tower_distance") as "max_tower_distance" FROM ( + SELECT last("slot") - last("root") as "tower_distance" + FROM "'$TESTNET_TAG'"."autogen"."tower-observed" + WHERE time > now() - '"$TEST_DURATION_SECONDS"'s + GROUP BY time(1s), host_id)' + + declare q_last_tower_distance_observed=' + SELECT MEAN("tower_distance") as "last_tower_distance" FROM ( + SELECT last("slot") - last("root") as "tower_distance" + FROM "'$TESTNET_TAG'"."autogen"."tower-observed" + GROUP BY host_id)' + + curl -G "${INFLUX_HOST}/query?u=ro&p=topsecret" \ + --data-urlencode "db=${TESTNET_TAG}" \ + --data-urlencode "q=$q_mean_tps;$q_max_tps;$q_mean_confirmation;$q_max_confirmation;$q_99th_confirmation;$q_max_tower_distance_observed;$q_last_tower_distance_observed" | + python "${REPO_ROOT}"/system-test/testnet-automation-json-parser.py >>"$RESULT_FILE" +} + function upload_results_to_slack() { echo --- Uploading results to Slack Performance Results App @@ -166,7 +232,7 @@ function upload_results_to_slack() { { "type": "divider" }, - { + { "type": "section", "text": { "type": "mrkdwn", diff --git a/system-test/deprecated-testcases/colo-cpu-only-perf.yml b/system-test/deprecated-testcases/colo-cpu-only-perf.yml index 2ac1954e5a8373..13e17665229add 100755 --- a/system-test/deprecated-testcases/colo-cpu-only-perf.yml +++ b/system-test/deprecated-testcases/colo-cpu-only-perf.yml @@ -11,5 +11,6 @@ steps: NUMBER_OF_CLIENT_NODES: 2 CLIENT_OPTIONS: "bench-tps=2=--tx_count 20000 --thread-batch-sleep-ms 250" ADDITIONAL_FLAGS: "" + TEST_TYPE: "fixed_duration" agents: - "queue=colo-deploy" diff --git a/system-test/deprecated-testcases/colo-gpu-perf-high-txcount.yml b/system-test/deprecated-testcases/colo-gpu-perf-high-txcount.yml index 50368c52e0f373..8e41bdbc6b80a6 100755 --- a/system-test/deprecated-testcases/colo-gpu-perf-high-txcount.yml +++ b/system-test/deprecated-testcases/colo-gpu-perf-high-txcount.yml @@ -11,5 +11,6 @@ steps: NUMBER_OF_CLIENT_NODES: 2 CLIENT_OPTIONS: "bench-tps=2=--tx_count 30000 --thread-batch-sleep-ms 250" ADDITIONAL_FLAGS: "" + TEST_TYPE: "fixed_duration" agents: - "queue=colo-deploy" diff --git a/system-test/deprecated-testcases/colo-gpu-perf.yml b/system-test/deprecated-testcases/colo-gpu-perf.yml index bf2dfe8cf3cc72..aceca0a187b835 100755 --- a/system-test/deprecated-testcases/colo-gpu-perf.yml +++ b/system-test/deprecated-testcases/colo-gpu-perf.yml @@ -11,5 +11,6 @@ steps: NUMBER_OF_CLIENT_NODES: 2 CLIENT_OPTIONS: "bench-tps=2=--tx_count 20000 --thread-batch-sleep-ms 250" ADDITIONAL_FLAGS: "" + TEST_TYPE: "fixed_duration" agents: - "queue=colo-deploy" diff --git a/system-test/deprecated-testcases/gce-gpu-perf-100-node.yml b/system-test/deprecated-testcases/gce-gpu-perf-100-node.yml index 6c2854fba01bf5..12c72ce494aae8 100755 --- a/system-test/deprecated-testcases/gce-gpu-perf-100-node.yml +++ b/system-test/deprecated-testcases/gce-gpu-perf-100-node.yml @@ -15,5 +15,6 @@ steps: ALLOW_BOOT_FAILURES: "true" USE_PUBLIC_IP_ADDRESSES: "true" ADDITIONAL_FLAGS: "--dedicated" + TEST_TYPE: "fixed_duration" agents: - "queue=gce-deploy" diff --git a/system-test/partition-testcases/colo-3-partition.yml b/system-test/partition-testcases/colo-3-partition.yml index 18c447023ae3a0..9b61d3d33f8877 100755 --- a/system-test/partition-testcases/colo-3-partition.yml +++ b/system-test/partition-testcases/colo-3-partition.yml @@ -15,5 +15,6 @@ steps: PARTITION_ACTIVE_DURATION: 30 PARTITION_INACTIVE_DURATION: 30 PARTITION_ITERATION_COUNT: 5 + TEST_TYPE: "partition" agents: - "queue=colo-deploy" diff --git a/system-test/partition-testcases/colo-partition-long-sanity-test.yml b/system-test/partition-testcases/colo-partition-long-sanity-test.yml index 64c3b75502bccf..3493a409c5d5ca 100755 --- a/system-test/partition-testcases/colo-partition-long-sanity-test.yml +++ b/system-test/partition-testcases/colo-partition-long-sanity-test.yml @@ -15,5 +15,6 @@ steps: PARTITION_ACTIVE_DURATION: 60 PARTITION_INACTIVE_DURATION: 60 PARTITION_ITERATION_COUNT: 10 + TEST_TYPE: "partition" agents: - "queue=colo-deploy" diff --git a/system-test/partition-testcases/colo-partition-once-then-stabilize.yml b/system-test/partition-testcases/colo-partition-once-then-stabilize.yml index a2ae25e52816f0..406c14705a987f 100755 --- a/system-test/partition-testcases/colo-partition-once-then-stabilize.yml +++ b/system-test/partition-testcases/colo-partition-once-then-stabilize.yml @@ -15,5 +15,6 @@ steps: PARTITION_ACTIVE_DURATION: 60 PARTITION_INACTIVE_DURATION: 300 PARTITION_ITERATION_COUNT: 1 + TEST_TYPE: "partition" agents: - "queue=colo-deploy" diff --git a/system-test/partition-testcases/gce-5-node-3-partition.yml b/system-test/partition-testcases/gce-5-node-3-partition.yml index a054a0b5da26f6..8197537b8eb795 100755 --- a/system-test/partition-testcases/gce-5-node-3-partition.yml +++ b/system-test/partition-testcases/gce-5-node-3-partition.yml @@ -18,5 +18,6 @@ steps: PARTITION_ACTIVE_DURATION: 30 PARTITION_INACTIVE_DURATION: 30 PARTITION_ITERATION_COUNT: 5 + TEST_TYPE: "partition" agents: - "queue=gce-deploy" diff --git a/system-test/partition-testcases/gce-5-node-single-region-2-partitions.yml b/system-test/partition-testcases/gce-5-node-single-region-2-partitions.yml index f0dcc75d536936..408b1a035080ad 100755 --- a/system-test/partition-testcases/gce-5-node-single-region-2-partitions.yml +++ b/system-test/partition-testcases/gce-5-node-single-region-2-partitions.yml @@ -18,5 +18,6 @@ steps: PARTITION_ACTIVE_DURATION: 60 PARTITION_INACTIVE_DURATION: 300 PARTITION_ITERATION_COUNT: 5 + TEST_TYPE: "partition" agents: - "queue=gce-deploy" diff --git a/system-test/partition-testcases/gce-partition-once-then-stabilize.yml b/system-test/partition-testcases/gce-partition-once-then-stabilize.yml index 9b06b1c16ca017..78668a1b06fc33 100755 --- a/system-test/partition-testcases/gce-partition-once-then-stabilize.yml +++ b/system-test/partition-testcases/gce-partition-once-then-stabilize.yml @@ -18,5 +18,6 @@ steps: PARTITION_ACTIVE_DURATION: 60 PARTITION_INACTIVE_DURATION: 300 PARTITION_ITERATION_COUNT: 1 + TEST_TYPE: "partition" agents: - "queue=gce-deploy" diff --git a/system-test/performance-testcases/aws-cpu-only-perf-10-node.yml b/system-test/performance-testcases/aws-cpu-only-perf-10-node.yml index 8c0e61cce47283..9d7add17045639 100755 --- a/system-test/performance-testcases/aws-cpu-only-perf-10-node.yml +++ b/system-test/performance-testcases/aws-cpu-only-perf-10-node.yml @@ -15,5 +15,6 @@ steps: TESTNET_ZONES: "us-west-1a,us-west-1c,us-east-1a,eu-west-1a" USE_PUBLIC_IP_ADDRESSES: "true" ADDITIONAL_FLAGS: "" + TEST_TYPE: "fixed_duration" agents: - "queue=aws-deploy" diff --git a/system-test/performance-testcases/aws-cpu-only-perf-5-node.yml b/system-test/performance-testcases/aws-cpu-only-perf-5-node.yml index 12f8d865e45fdb..702a99529037b2 100755 --- a/system-test/performance-testcases/aws-cpu-only-perf-5-node.yml +++ b/system-test/performance-testcases/aws-cpu-only-perf-5-node.yml @@ -15,5 +15,6 @@ steps: TESTNET_ZONES: "us-west-1a,us-west-1c,us-east-1a,eu-west-1a" USE_PUBLIC_IP_ADDRESSES: "true" ADDITIONAL_FLAGS: "" + TEST_TYPE: "fixed_duration" agents: - "queue=aws-deploy" diff --git a/system-test/performance-testcases/azure-cpu-only-perf-5-node.yml b/system-test/performance-testcases/azure-cpu-only-perf-5-node.yml index b2545b679e6d8a..5b7e9ced1da7d0 100755 --- a/system-test/performance-testcases/azure-cpu-only-perf-5-node.yml +++ b/system-test/performance-testcases/azure-cpu-only-perf-5-node.yml @@ -14,5 +14,6 @@ steps: TESTNET_ZONES: "westus" USE_PUBLIC_IP_ADDRESSES: "true" ADDITIONAL_FLAGS: "" + TEST_TYPE: "fixed_duration" agents: - "queue=azure-deploy" diff --git a/system-test/performance-testcases/colo-cpu-only-perf-4-val-1-client.yml b/system-test/performance-testcases/colo-cpu-only-perf-4-val-1-client.yml index faeade297fb64a..b808ff7c28b682 100755 --- a/system-test/performance-testcases/colo-cpu-only-perf-4-val-1-client.yml +++ b/system-test/performance-testcases/colo-cpu-only-perf-4-val-1-client.yml @@ -11,5 +11,6 @@ steps: NUMBER_OF_CLIENT_NODES: 1 CLIENT_OPTIONS: "bench-tps=1=--tx_count 40000 --thread-batch-sleep-ms 250" ADDITIONAL_FLAGS: "" + TEST_TYPE: "fixed_duration" agents: - "queue=colo-deploy" diff --git a/system-test/performance-testcases/colo-gpu-perf-4-val-1-client.yml b/system-test/performance-testcases/colo-gpu-perf-4-val-1-client.yml index 76e365835aa0e5..4be8f195d3d2f8 100755 --- a/system-test/performance-testcases/colo-gpu-perf-4-val-1-client.yml +++ b/system-test/performance-testcases/colo-gpu-perf-4-val-1-client.yml @@ -11,5 +11,6 @@ steps: NUMBER_OF_CLIENT_NODES: 1 CLIENT_OPTIONS: "bench-tps=1=--tx_count 40000 --thread-batch-sleep-ms 250" ADDITIONAL_FLAGS: "" + TEST_TYPE: "fixed_duration" agents: - "queue=colo-deploy" diff --git a/system-test/performance-testcases/colo-gpu-perf-high-txcount-4-val-1-client.yml b/system-test/performance-testcases/colo-gpu-perf-high-txcount-4-val-1-client.yml index a9cc89c0895734..59f14514181c18 100755 --- a/system-test/performance-testcases/colo-gpu-perf-high-txcount-4-val-1-client.yml +++ b/system-test/performance-testcases/colo-gpu-perf-high-txcount-4-val-1-client.yml @@ -11,5 +11,6 @@ steps: NUMBER_OF_CLIENT_NODES: 1 CLIENT_OPTIONS: "bench-tps=1=--tx_count 60000 --thread-batch-sleep-ms 250" ADDITIONAL_FLAGS: "" + TEST_TYPE: "fixed_duration" agents: - "queue=colo-deploy" diff --git a/system-test/performance-testcases/gce-cpu-only-perf-10-node.yml b/system-test/performance-testcases/gce-cpu-only-perf-10-node.yml index 16f65edfdbbd1b..7a9984f1129e66 100755 --- a/system-test/performance-testcases/gce-cpu-only-perf-10-node.yml +++ b/system-test/performance-testcases/gce-cpu-only-perf-10-node.yml @@ -14,5 +14,6 @@ steps: TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a" USE_PUBLIC_IP_ADDRESSES: "true" ADDITIONAL_FLAGS: "--dedicated" + TEST_TYPE: "fixed_duration" agents: - "queue=gce-deploy" diff --git a/system-test/performance-testcases/gce-cpu-only-perf-5-node.yml b/system-test/performance-testcases/gce-cpu-only-perf-5-node.yml index e7ac586ebf5698..19e2edf8026d14 100755 --- a/system-test/performance-testcases/gce-cpu-only-perf-5-node.yml +++ b/system-test/performance-testcases/gce-cpu-only-perf-5-node.yml @@ -14,5 +14,6 @@ steps: TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a" USE_PUBLIC_IP_ADDRESSES: "true" ADDITIONAL_FLAGS: "--dedicated" + TEST_TYPE: "fixed_duration" agents: - "queue=gce-deploy" diff --git a/system-test/performance-testcases/gce-gpu-perf-10-node.yml b/system-test/performance-testcases/gce-gpu-perf-10-node.yml index 3e7c7cbb572544..fe18204a074a7b 100755 --- a/system-test/performance-testcases/gce-gpu-perf-10-node.yml +++ b/system-test/performance-testcases/gce-gpu-perf-10-node.yml @@ -14,5 +14,6 @@ steps: TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a" USE_PUBLIC_IP_ADDRESSES: "true" ADDITIONAL_FLAGS: "--dedicated" + TEST_TYPE: "fixed_duration" agents: - "queue=gce-deploy" diff --git a/system-test/performance-testcases/gce-gpu-perf-25-node.yml b/system-test/performance-testcases/gce-gpu-perf-25-node.yml index 665466da3e4c24..3f9415cd5e9c80 100755 --- a/system-test/performance-testcases/gce-gpu-perf-25-node.yml +++ b/system-test/performance-testcases/gce-gpu-perf-25-node.yml @@ -14,5 +14,6 @@ steps: TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a" USE_PUBLIC_IP_ADDRESSES: "true" ADDITIONAL_FLAGS: "--dedicated" + TEST_TYPE: "fixed_duration" agents: - "queue=gce-deploy" diff --git a/system-test/performance-testcases/gce-gpu-perf-5-node.yml b/system-test/performance-testcases/gce-gpu-perf-5-node.yml index 8270309d1a8d7b..5972b8d1c67f96 100755 --- a/system-test/performance-testcases/gce-gpu-perf-5-node.yml +++ b/system-test/performance-testcases/gce-gpu-perf-5-node.yml @@ -14,5 +14,6 @@ steps: TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a" USE_PUBLIC_IP_ADDRESSES: "true" ADDITIONAL_FLAGS: "--dedicated" + TEST_TYPE: "fixed_duration" agents: - "queue=gce-deploy" diff --git a/system-test/performance-testcases/gce-gpu-perf-50-node.yml b/system-test/performance-testcases/gce-gpu-perf-50-node.yml index f730b9e67e286d..7e6d84fa428c60 100755 --- a/system-test/performance-testcases/gce-gpu-perf-50-node.yml +++ b/system-test/performance-testcases/gce-gpu-perf-50-node.yml @@ -15,5 +15,6 @@ steps: ALLOW_BOOT_FAILURES: "true" USE_PUBLIC_IP_ADDRESSES: "true" ADDITIONAL_FLAGS: "--dedicated" + TEST_TYPE: "fixed_duration" agents: - "queue=gce-deploy" diff --git a/system-test/sanity-testcases/colo-cpu-only-quick-sanity-test.yml b/system-test/sanity-testcases/colo-cpu-only-quick-sanity-test.yml index 6020020bae0397..57c73e5b71edf8 100755 --- a/system-test/sanity-testcases/colo-cpu-only-quick-sanity-test.yml +++ b/system-test/sanity-testcases/colo-cpu-only-quick-sanity-test.yml @@ -6,11 +6,12 @@ steps: CLOUD_PROVIDER: "colo" TESTNET_TAG: "colo-perf-cpu-only" ENABLE_GPU: "false" - TEST_DURATION_SECONDS: 30 + TEST_DURATION_SECONDS: 60 NUMBER_OF_VALIDATOR_NODES: 1 NUMBER_OF_CLIENT_NODES: 1 CLIENT_OPTIONS: "bench-tps=1=--tx_count 40000 --thread-batch-sleep-ms 250" ADDITIONAL_FLAGS: "" - BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100 + BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 99 + TEST_TYPE: "fixed_duration" agents: - "queue=colo-deploy" diff --git a/system-test/sanity-testcases/colo-partition-sanity-test.yml b/system-test/sanity-testcases/colo-partition-sanity-test.yml index 2aeceb257ee283..006052ed1b1d41 100755 --- a/system-test/sanity-testcases/colo-partition-sanity-test.yml +++ b/system-test/sanity-testcases/colo-partition-sanity-test.yml @@ -5,7 +5,7 @@ steps: UPLOAD_RESULTS_TO_SLACK: "true" CLOUD_PROVIDER: "colo" TESTNET_TAG: "colo-perf-cpu-only" - NUMBER_OF_VALIDATOR_NODES: 4 + NUMBER_OF_VALIDATOR_NODES: 3 ENABLE_GPU: "false" NUMBER_OF_CLIENT_NODES: 1 CLIENT_OPTIONS: "bench-tps=1=--tx_count 15000 --thread-batch-sleep-ms 250" @@ -15,6 +15,7 @@ steps: PARTITION_ACTIVE_DURATION: 30 PARTITION_INACTIVE_DURATION: 30 PARTITION_ITERATION_COUNT: 2 - BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100 + BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 66 + TEST_TYPE: "partition" agents: - "queue=colo-deploy" diff --git a/system-test/stability-testcases/colo-long-duration-cpu-only-perf.yml b/system-test/stability-testcases/colo-long-duration-cpu-only-perf.yml index f2feda95eaadb4..de36fa8aff0394 100755 --- a/system-test/stability-testcases/colo-long-duration-cpu-only-perf.yml +++ b/system-test/stability-testcases/colo-long-duration-cpu-only-perf.yml @@ -11,5 +11,6 @@ steps: NUMBER_OF_CLIENT_NODES: 1 CLIENT_OPTIONS: "bench-tps=1=--tx_count 30000 --thread-batch-sleep-ms 250" ADDITIONAL_FLAGS: "" + TEST_TYPE: "fixed_duration" agents: - "queue=colo-deploy" diff --git a/system-test/stability-testcases/colo-long-duration-gpu-perf.yml b/system-test/stability-testcases/colo-long-duration-gpu-perf.yml index 611b9dbd798f0a..8f5c576cd2deb2 100755 --- a/system-test/stability-testcases/colo-long-duration-gpu-perf.yml +++ b/system-test/stability-testcases/colo-long-duration-gpu-perf.yml @@ -11,5 +11,6 @@ steps: NUMBER_OF_CLIENT_NODES: 1 CLIENT_OPTIONS: "bench-tps=1=--tx_count 30000 --thread-batch-sleep-ms 250" ADDITIONAL_FLAGS: "" + TEST_TYPE: "fixed_duration" agents: - "queue=colo-deploy" diff --git a/system-test/stability-testcases/gce-perf-stability-5-node-single-region.yml b/system-test/stability-testcases/gce-perf-stability-5-node-single-region.yml index d55632b061e2ab..a4b7f8fd8bdda5 100755 --- a/system-test/stability-testcases/gce-perf-stability-5-node-single-region.yml +++ b/system-test/stability-testcases/gce-perf-stability-5-node-single-region.yml @@ -14,5 +14,6 @@ steps: TESTNET_ZONES: "us-west1-a" USE_PUBLIC_IP_ADDRESSES: "false" ADDITIONAL_FLAGS: "--dedicated" + TEST_TYPE: "fixed_duration" agents: - "queue=gce-deploy" diff --git a/system-test/stability-testcases/gce-stability-5-node.yml b/system-test/stability-testcases/gce-stability-5-node.yml index 281fee27c9e28f..bfb95356dd10ef 100755 --- a/system-test/stability-testcases/gce-stability-5-node.yml +++ b/system-test/stability-testcases/gce-stability-5-node.yml @@ -13,5 +13,6 @@ steps: TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a" USE_PUBLIC_IP_ADDRESSES: "true" ADDITIONAL_FLAGS: "--dedicated" + TEST_TYPE: "fixed_duration" agents: - "queue=stability-deploy" diff --git a/system-test/stake-operations-testcases/offline_stake_colo.yml b/system-test/stake-operations-testcases/offline_stake_colo.yml new file mode 100755 index 00000000000000..0aedbd54e1751e --- /dev/null +++ b/system-test/stake-operations-testcases/offline_stake_colo.yml @@ -0,0 +1,17 @@ +steps: + - command: "system-test/testnet-automation.sh" + label: "Running Offline Stake Operations Tests on Colo" + env: + UPLOAD_RESULTS_TO_SLACK: "true" + CLOUD_PROVIDER: "colo" + ENABLE_GPU: "false" + TEST_DURATION_SECONDS: 30 + NUMBER_OF_VALIDATOR_NODES: 1 + NUMBER_OF_CLIENT_NODES: 0 + ADDITIONAL_FLAGS: "" + BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100 + SKIP_PERF_RESULTS: "true" + TEST_TYPE: "script" + CUSTOM_SCRIPT: "system-test/stake-operations-testcases/stake_test_automation.sh" + agents: + - "queue=colo-deploy" diff --git a/system-test/stake-operations-testcases/offline_stake_gce.yml b/system-test/stake-operations-testcases/offline_stake_gce.yml new file mode 100755 index 00000000000000..1b88d6ba443da9 --- /dev/null +++ b/system-test/stake-operations-testcases/offline_stake_gce.yml @@ -0,0 +1,18 @@ +steps: + - command: "system-test/testnet-automation.sh" + label: "Running Offline Stake Operations Tests on GCE" + env: + UPLOAD_RESULTS_TO_SLACK: "true" + CLOUD_PROVIDER: "gce" + ENABLE_GPU: "false" + TEST_DURATION_SECONDS: 30 + NUMBER_OF_VALIDATOR_NODES: 1 + VALIDATOR_NODE_MACHINE_TYPE: "--machine-type n1-standard-16" + NUMBER_OF_CLIENT_NODES: 0 + ADDITIONAL_FLAGS: "--dedicated" + BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100 + SKIP_PERF_RESULTS: "true" + TEST_TYPE: "script" + CUSTOM_SCRIPT: "system-test/stake-operations-testcases/stake_test_automation.sh" + agents: + - "queue=gce-deploy" diff --git a/system-test/stake-operations-testcases/offline_stake_operations.sh b/system-test/stake-operations-testcases/offline_stake_operations.sh new file mode 100755 index 00000000000000..79a0abd609b2ea --- /dev/null +++ b/system-test/stake-operations-testcases/offline_stake_operations.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2086 +# shellcheck disable=SC2068 +# shellcheck disable=SC2206 +# shellcheck disable=SC2162 +# shellcheck disable=SC2178 +# shellcheck disable=SC2145 + +# shellcheck disable=SC1090 +# shellcheck disable=SC1091 +source "$(dirname "$0")"/../automation_utils.sh + +set -e + +function get_signers_string() { + sign_only_output="$1" + + signers=() + while read LINE; do + signers+=( --signer $LINE ) + done <<<"$(sed -Ee $'s/^ ([a-km-zA-HJ-NP-Z1-9]{32,44}=[a-km-zA-HJ-NP-Z1-9]{64,88})$/\\1/\nt\nd' <<<"$sign_only_output")" + + echo "${signers[@]}" +} + +if [[ -n "$1" ]]; then + url="$1" +fi + +if [[ -z "$url" ]]; then + echo Provide complete URL, ex: "$0" http://devnet.solana.com:8899 + exit 1 +fi +solana config set --url $url + +# Create a dummy keypair file with no balance for operations that require a "client keypair file" to exist even if they don't touch it +dummy_keypair=dummy.json +solana-keygen new -o "$dummy_keypair" --no-passphrase --force --silent +solana config set --keypair $dummy_keypair + +### Offline stake account creation + +# The nonce account and the system account funding its creation are online +online_nonce_account_keypair=nonce_keypair.json +online_system_account_keypair=online_system_account_keypair.json + +solana-keygen new -o "$online_system_account_keypair" --no-passphrase --force --silent +solana-keygen new -o "$online_nonce_account_keypair" --no-passphrase --force --silent + +online_system_account_pubkey="$(solana-keygen pubkey $online_system_account_keypair)" +nonce_account_pubkey="$(solana-keygen pubkey $online_nonce_account_keypair)" + +# System account funding the stake account is offline, and the auth staker and withdrawer keypairs are offline +offline_system_account_keypair=offline_system_account_keypair.json +offline_staker_keypair=offline_staker_keypair.json +offline_withdrawer_keypair=offline_withdrawer_keypair.json +offline_custodian_keypair=offline_custodian_keypair.json + +solana-keygen new -o "$offline_system_account_keypair" --no-passphrase --force --silent +solana-keygen new -o "$offline_staker_keypair" --no-passphrase --force --silent +solana-keygen new -o "$offline_withdrawer_keypair" --no-passphrase --force --silent +solana-keygen new -o "$offline_custodian_keypair" --no-passphrase --force --silent + +offline_system_account_pubkey="$(solana-keygen pubkey $offline_system_account_keypair)" +offline_withdrawer_pubkey="$(solana-keygen pubkey $offline_withdrawer_keypair)" +offline_staker_pubkey="$(solana-keygen pubkey $offline_staker_keypair)" +offline_custodian_pubkey="$(solana-keygen pubkey $offline_custodian_keypair)" + +# Airdrop some funds to the offline account. +solana airdrop 100 $offline_system_account_pubkey +solana airdrop 2 $online_system_account_pubkey + +# Create a nonce account funded by the online account, with the authority given to the offline account +solana create-nonce-account $online_nonce_account_keypair 1 --nonce-authority $offline_system_account_pubkey --keypair $online_system_account_keypair +nonce="$(solana nonce $nonce_account_pubkey)" + +execution_step OFFLINE SYSTEM ACCOUNT BALANCE BEFORE CREATING STAKE ACCOUNTS +( + set -x + solana balance $offline_system_account_pubkey +) + +################################ +execution_step CREATE OFFLINE STAKE ACCOUNT +################################ + +# Create a stake account funded by the offline system account + +stake_account_keypair=stake_account_keypair.json +solana-keygen new -o "$stake_account_keypair" --no-passphrase --force --silent +stake_account_address="$(solana-keygen pubkey $stake_account_keypair)" + +sign_only="$(solana create-stake-account $stake_account_keypair 50 \ + --sign-only --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \ + --stake-authority $offline_staker_pubkey --withdraw-authority $offline_withdrawer_pubkey \ + --custodian $offline_custodian_pubkey \ + --lockup-epoch 999 \ + --keypair $offline_system_account_keypair --url http://0.0.0.0)" + +signers="$(get_signers_string "${sign_only[@]})")" + +solana create-stake-account $stake_account_keypair 50 \ + --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \ + --stake-authority $offline_staker_pubkey --withdraw-authority $offline_withdrawer_pubkey \ + --custodian $offline_custodian_pubkey \ + --lockup-epoch 999 \ + --from $offline_system_account_pubkey --fee-payer $offline_system_account_pubkey ${signers[@]} + +execution_step VIEW STAKE ACCOUNT AFTER CREATION +( + set -x + solana stake-account $stake_account_address +) + + +execution_step VIEW OFFLINE SYSTEM ACCOUNT BALANCE AFTER CREATING FIRST STAKE ACCOUNT +( + set -x + solana balance $offline_system_account_pubkey +) + +##################### +execution_step SPLIT STAKE OFFLINE +##################### + +# Split the original stake account before delegating + +split_stake_account_keypair=split_stake_account_keypair.json +solana-keygen new -o $split_stake_account_keypair --no-passphrase --force --silent +split_stake_account_address=$(solana-keygen pubkey $split_stake_account_keypair) + +nonce="$(solana nonce $nonce_account_pubkey)" + +sign_only="$(solana split-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \ + --stake-authority $offline_staker_keypair $stake_account_address $split_stake_account_keypair 10 \ + --keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)" + +signers="$(get_signers_string "${sign_only[@]})")" + +solana split-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \ + --stake-authority $offline_staker_pubkey $stake_account_address $split_stake_account_keypair 10 \ + --fee-payer $offline_system_account_pubkey ${signers[@]} + +execution_step VIEW ORIGINAL STAKE ACCOUNT AFTER SPLITTING +( + set -x + solana stake-account $stake_account_address +) + +execution_step VIEW NEW STAKE ACCOUNT CREATED FROM SPLITTING ORIGINAL +( + set -x + solana stake-account $split_stake_account_address +) + +##################### +execution_step CHANGE CUSTODIAN LOCKUP +##################### + +# Set the lockup epoch to 0 to allow stake to be withdrawn + +nonce="$(solana nonce $nonce_account_pubkey)" + +sign_only="$(solana stake-set-lockup --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \ + $split_stake_account_address --custodian $offline_custodian_keypair --lockup-epoch 0 \ + --keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)" + +signers="$(get_signers_string "${sign_only[@]})")" + +solana stake-set-lockup --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \ + $split_stake_account_address --custodian $offline_custodian_pubkey --lockup-epoch 0 \ + --fee-payer $offline_system_account_pubkey ${signers[@]} + +execution_step VIEW SPLIT STAKE ACCOUNT AFTER CHANGING LOCKUP +( + set -x + solana stake-account $split_stake_account_address +) + +########################## +execution_step OFFLINE STAKE WITHDRAWAL +########################## + +# Withdraw the lamports from the stake account that was split off and return them to the offline system account + +nonce="$(solana nonce $nonce_account_pubkey)" + +sign_only="$(solana withdraw-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \ + $split_stake_account_address $offline_system_account_pubkey 10 \ + --withdraw-authority $offline_withdrawer_keypair \ + --keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)" + +signers="$(get_signers_string "${sign_only[@]})")" + +solana withdraw-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \ + $split_stake_account_address $offline_system_account_pubkey 10 \ + --withdraw-authority $offline_withdrawer_pubkey \ + --fee-payer $offline_system_account_pubkey ${signers[@]} + +execution_step VIEW OFFLINE SYSTEM ACCOUNT BALANCE AFTER WITHDRAWING SPLIT STAKE +( + set -x + solana balance $offline_system_account_pubkey +) + +########################## +execution_step OFFLINE STAKE DELEGATION +########################## + +# Delegate stake from the original account to a vote account + +vote_account_pubkey="$(awk '{if(NR==4) print $2}'<<<"$(solana show-validators)")" +nonce="$(solana nonce $nonce_account_pubkey)" + +# Sign a stake delegation, assuming the authorized staker is held offline +sign_only="$(solana delegate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \ +--stake-authority $offline_staker_keypair $stake_account_address $vote_account_pubkey \ +--keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)" + +signers="$(get_signers_string "${sign_only[@]})")" + +# Send the signed transaction on the cluster +solana delegate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \ +--stake-authority $offline_staker_pubkey $stake_account_address $vote_account_pubkey \ +--fee-payer $offline_system_account_pubkey ${signers[@]} + +execution_step VIEW ORIGINAL STAKE ACCOUNT AFTER DELEGATION +( + set -x + solana stake-account $stake_account_address +) + +########################## + execution_step OFFLINE STAKE DEACTIVATION +########################## + +# Deactivate delegated stake + +nonce="$(solana nonce $nonce_account_pubkey)" + +# Sign a stake delegation, assuming the authorized staker is held offline +sign_only="$(solana deactivate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_keypair \ +--stake-authority $offline_staker_keypair $stake_account_address \ +--keypair $offline_system_account_keypair --sign-only --url http://0.0.0.0)" + +signers="$(get_signers_string "${sign_only[@]})")" + +# Send the signed transaction on the cluster +solana deactivate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-authority $offline_system_account_pubkey \ +--stake-authority $offline_staker_pubkey $stake_account_address \ +--fee-payer $offline_system_account_pubkey ${signers[@]} + +execution_step VIEW ORIGINAL STAKE ACCOUNT AFTER DEACTIVATION +( + set -x + solana stake-account $stake_account_address +) diff --git a/system-test/stake-operations-testcases/stake_test_automation.sh b/system-test/stake-operations-testcases/stake_test_automation.sh new file mode 100755 index 00000000000000..77dd534343e68c --- /dev/null +++ b/system-test/stake-operations-testcases/stake_test_automation.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -ex + +# shellcheck disable=SC1090 +# shellcheck disable=SC1091 +source "$(dirname "$0")"/../automation_utils.sh + +RESULT_FILE="$1" + +# Runs offline stake operations tests against a running cluster launched from the automation framework +bootstrapper_ip_address="$(get_bootstrap_validator_ip_address)" +entrypoint=http://"${bootstrapper_ip_address}":8899 +PATH="$REPO_ROOT"/solana-release/bin:$PATH "$REPO_ROOT"/system-test/stake-operations-testcases/offline_stake_operations.sh "$entrypoint" + +echo "Offline Stake Operations Test Succeeded" >>"$RESULT_FILE" diff --git a/system-test/testnet-automation.sh b/system-test/testnet-automation.sh index abd1b31bab6a14..613cb4860556e7 100755 --- a/system-test/testnet-automation.sh +++ b/system-test/testnet-automation.sh @@ -147,30 +147,36 @@ function launch_testnet() { SLOT_COUNT_START_SECONDS=$SECONDS execution_step "Marking beginning of slot rate test - Slot: $START_SLOT, Seconds: $SLOT_COUNT_START_SECONDS" - if [[ -n $TEST_DURATION_SECONDS ]]; then - execution_step "Wait ${TEST_DURATION_SECONDS} seconds to complete test" - sleep "$TEST_DURATION_SECONDS" - elif [[ "$APPLY_PARTITIONS" = "true" ]]; then - STATS_START_SECONDS=$SECONDS - execution_step "Wait $PARTITION_INACTIVE_DURATION before beginning to apply partitions" - sleep "$PARTITION_INACTIVE_DURATION" - for (( i=1; i<=PARTITION_ITERATION_COUNT; i++ )); do - execution_step "Partition Iteration $i of $PARTITION_ITERATION_COUNT" - execution_step "Applying netem config $NETEM_CONFIG_FILE for $PARTITION_ACTIVE_DURATION seconds" - "${REPO_ROOT}"/net/net.sh netem --config-file "$NETEM_CONFIG_FILE" - sleep "$PARTITION_ACTIVE_DURATION" - - execution_step "Resolving partitions for $PARTITION_INACTIVE_DURATION seconds" - "${REPO_ROOT}"/net/net.sh netem --config-file "$NETEM_CONFIG_FILE" --netem-cmd cleanup + case $TEST_TYPE in + fixed_duration) + execution_step "Wait ${TEST_DURATION_SECONDS} seconds to complete test" + sleep "$TEST_DURATION_SECONDS" + ;; + partition) + STATS_START_SECONDS=$SECONDS + execution_step "Wait $PARTITION_INACTIVE_DURATION before beginning to apply partitions" sleep "$PARTITION_INACTIVE_DURATION" - done - STATS_FINISH_SECONDS=$SECONDS - TEST_DURATION_SECONDS=$((STATS_FINISH_SECONDS - STATS_START_SECONDS)) - else - # We should never get here - echo Test duration and partition config not defined - exit 1 - fi + for (( i=1; i<=PARTITION_ITERATION_COUNT; i++ )); do + execution_step "Partition Iteration $i of $PARTITION_ITERATION_COUNT" + execution_step "Applying netem config $NETEM_CONFIG_FILE for $PARTITION_ACTIVE_DURATION seconds" + "${REPO_ROOT}"/net/net.sh netem --config-file "$NETEM_CONFIG_FILE" + sleep "$PARTITION_ACTIVE_DURATION" + + execution_step "Resolving partitions for $PARTITION_INACTIVE_DURATION seconds" + "${REPO_ROOT}"/net/net.sh netem --config-file "$NETEM_CONFIG_FILE" --netem-cmd cleanup + sleep "$PARTITION_INACTIVE_DURATION" + done + STATS_FINISH_SECONDS=$SECONDS + TEST_DURATION_SECONDS=$((STATS_FINISH_SECONDS - STATS_START_SECONDS)) + ;; + script) + execution_step "Running custom script: ${REPO_ROOT}/${CUSTOM_SCRIPT}" + "$REPO_ROOT"/"$CUSTOM_SCRIPT" "$RESULT_FILE" + ;; + *) + echo "Error: Unsupported test type: $TEST_TYPE" + ;; + esac END_SLOT=$(get_slot) SLOT_COUNT_END_SECONDS=$SECONDS @@ -179,63 +185,11 @@ function launch_testnet() { SLOTS_PER_SECOND="$(bc <<< "scale=3; ($END_SLOT - $START_SLOT)/($SLOT_COUNT_END_SECONDS - $SLOT_COUNT_START_SECONDS)")" execution_step "Average slot rate: $SLOTS_PER_SECOND slots/second over $((SLOT_COUNT_END_SECONDS - SLOT_COUNT_START_SECONDS)) seconds" - execution_step "Collect statistics about run" - declare q_mean_tps=' - SELECT ROUND(MEAN("median_sum")) as "mean_tps" FROM ( - SELECT MEDIAN(sum_count) AS "median_sum" FROM ( - SELECT SUM("count") AS "sum_count" - FROM "'$TESTNET_TAG'"."autogen"."bank-process_transactions" - WHERE time > now() - '"$TEST_DURATION_SECONDS"'s AND count > 0 - GROUP BY time(1s), host_id) - GROUP BY time(1s) - )' - - declare q_max_tps=' - SELECT MAX("median_sum") as "max_tps" FROM ( - SELECT MEDIAN(sum_count) AS "median_sum" FROM ( - SELECT SUM("count") AS "sum_count" - FROM "'$TESTNET_TAG'"."autogen"."bank-process_transactions" - WHERE time > now() - '"$TEST_DURATION_SECONDS"'s AND count > 0 - GROUP BY time(1s), host_id) - GROUP BY time(1s) - )' - - declare q_mean_confirmation=' - SELECT round(mean("duration_ms")) as "mean_confirmation_ms" - FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation" - WHERE time > now() - '"$TEST_DURATION_SECONDS"'s' - - declare q_max_confirmation=' - SELECT round(max("duration_ms")) as "max_confirmation_ms" - FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation" - WHERE time > now() - '"$TEST_DURATION_SECONDS"'s' - - declare q_99th_confirmation=' - SELECT round(percentile("duration_ms", 99)) as "99th_percentile_confirmation_ms" - FROM "'$TESTNET_TAG'"."autogen"."validator-confirmation" - WHERE time > now() - '"$TEST_DURATION_SECONDS"'s' - - declare q_max_tower_distance_observed=' - SELECT MAX("tower_distance") as "max_tower_distance" FROM ( - SELECT last("slot") - last("root") as "tower_distance" - FROM "'$TESTNET_TAG'"."autogen"."tower-observed" - WHERE time > now() - '"$TEST_DURATION_SECONDS"'s - GROUP BY time(1s), host_id)' - - declare q_last_tower_distance_observed=' - SELECT MEAN("tower_distance") as "last_tower_distance" FROM ( - SELECT last("slot") - last("root") as "tower_distance" - FROM "'$TESTNET_TAG'"."autogen"."tower-observed" - GROUP BY host_id)' - - curl -G "${INFLUX_HOST}/query?u=ro&p=topsecret" \ - --data-urlencode "db=${TESTNET_TAG}" \ - --data-urlencode "q=$q_mean_tps;$q_max_tps;$q_mean_confirmation;$q_max_confirmation;$q_99th_confirmation;$q_max_tower_distance_observed;$q_last_tower_distance_observed" | - python "${REPO_ROOT}"/system-test/testnet-automation-json-parser.py >>"$RESULT_FILE" - - echo "slots_per_second: $SLOTS_PER_SECOND" >>"$RESULT_FILE" - - execution_step "Writing test results to ${RESULT_FILE}" + if [[ "$SKIP_PERF_RESULTS" = "false" ]]; then + collect_performance_statistics + echo "slots_per_second: $SLOTS_PER_SECOND" >>"$RESULT_FILE" + fi + RESULT_DETAILS=$(<"$RESULT_FILE") upload-ci-artifact "$RESULT_FILE" } @@ -245,9 +199,10 @@ RESULT_DETAILS= STEP= execution_step "Initialize Environment" -[[ -n $TESTNET_TAG ]] || TESTNET_TAG=testnet-automation +[[ -n $TESTNET_TAG ]] || TESTNET_TAG=${CLOUD_PROVIDER}-testnet-automation [[ -n $INFLUX_HOST ]] || INFLUX_HOST=https://metrics.solana.com:8086 [[ -n $BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD ]] || BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD=66 +[[ -n $SKIP_PERF_RESULTS ]] || SKIP_PERF_RESULTS=false if [[ -z $NUMBER_OF_VALIDATOR_NODES ]]; then echo NUMBER_OF_VALIDATOR_NODES not defined @@ -292,18 +247,41 @@ if [[ "$USE_PUBLIC_IP_ADDRESSES" = "true" ]]; then maybePublicIpAddresses="-P" fi -: "${CLIENT_DELAY_START:=0}" - -if [[ -z $APPLY_PARTITIONS ]]; then - APPLY_PARTITIONS=false -fi -if [[ "$APPLY_PARTITIONS" = "true" ]]; then - if [[ -n $TEST_DURATION_SECONDS ]]; then - echo Cannot accept TEST_DURATION_SECONDS and a parition looping config - exit 1 +execution_step "Checking for required parameters" +testTypeRequiredParameters= +case $TEST_TYPE in + fixed_duration) + testTypeRequiredParameters=( + TEST_DURATION_SECONDS \ + ) + ;; + partition) + testTypeRequiredParameters=( + NETEM_CONFIG_FILE \ + PARTITION_ACTIVE_DURATION \ + PARTITION_INACTIVE_DURATION \ + PARTITION_ITERATION_COUNT \ + ) + ;; + script) + testTypeRequiredParameters=( + CUSTOM_SCRIPT \ + ) + ;; + *) + echo "Error: Unsupported test type: $TEST_TYPE" + ;; +esac + +missingParameters= +for i in "${testTypeRequiredParameters[@]}"; do + if [[ -z ${!i} ]]; then + missingParameters+="${i}, " fi -elif [[ -z $TEST_DURATION_SECONDS ]]; then - echo TEST_DURATION_SECONDS not defined +done + +if [[ -n $missingParameters ]]; then + echo "Error: For test type $TEST_TYPE, the following required parameters are missing: ${missingParameters[*]}" exit 1 fi @@ -337,6 +315,8 @@ TEST_PARAMS_TO_DISPLAY=(CLOUD_PROVIDER \ PARTITION_ACTIVE_DURATION \ PARTITION_INACTIVE_DURATION \ PARTITION_ITERATION_COUNT \ + TEST_TYPE \ + CUSTOM_SCRIPT \ ) TEST_CONFIGURATION=