Add colo testcase and Report success to result file

solana-labs · Mar 17, 2020 · 30855b8 · 30855b8
1 parent 3a3bddf
commit 30855b8
Show file tree

Hide file tree

Showing 7 changed files with 66 additions and 33 deletions.
diff --git a/system-test/sanity-testcases/colo-cpu-only-quick-sanity-test.yml b/system-test/sanity-testcases/colo-cpu-only-quick-sanity-test.yml
@@ -6,12 +6,12 @@ steps:
       CLOUD_PROVIDER: "colo"
       TESTNET_TAG: "colo-perf-cpu-only"
       ENABLE_GPU: "false"
-      TEST_DURATION_SECONDS: 30
+      TEST_DURATION_SECONDS: 60
       NUMBER_OF_VALIDATOR_NODES: 1
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 40000 --thread-batch-sleep-ms 250"
       ADDITIONAL_FLAGS: ""
-      BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
+      BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 99
       TEST_TYPE: "fixed_duration"
     agents:
       - "queue=colo-deploy"
diff --git a/system-test/sanity-testcases/colo-partition-sanity-test.yml b/system-test/sanity-testcases/colo-partition-sanity-test.yml
@@ -5,7 +5,7 @@ steps:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "colo"
       TESTNET_TAG: "colo-perf-cpu-only"
-      NUMBER_OF_VALIDATOR_NODES: 4
+      NUMBER_OF_VALIDATOR_NODES: 3
       ENABLE_GPU: "false"
       NUMBER_OF_CLIENT_NODES: 1
       CLIENT_OPTIONS: "bench-tps=1=--tx_count 15000 --thread-batch-sleep-ms 250"

diff --git a/system-test/stake-operations-testcases/offline_stake_colo.yml b/system-test/stake-operations-testcases/offline_stake_colo.yml
@@ -0,0 +1,17 @@
+steps:
+  - command: "system-test/testnet-automation.sh"
+    label: "Running Offline Stake Operations Tests on Colo"
+    env:
+      UPLOAD_RESULTS_TO_SLACK: "true"
+      CLOUD_PROVIDER: "colo"
+      ENABLE_GPU: "false"
+      TEST_DURATION_SECONDS: 30
+      NUMBER_OF_VALIDATOR_NODES: 1
+      NUMBER_OF_CLIENT_NODES: 0
+      ADDITIONAL_FLAGS: ""
+      BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
+      SKIP_PERF_RESULTS: "true"
+      TEST_TYPE: "script"
+      CUSTOM_SCRIPT: "system-test/stake-operations-testcases/stake_test_automation.sh"
+    agents:
+      - "queue=colo-deploy"
diff --git a/system-test/stake-operations-testcases/offline_stake_gce.yml b/system-test/stake-operations-testcases/offline_stake_gce.yml
@@ -1,16 +1,15 @@
 steps:
   - command: "system-test/testnet-automation.sh"
-    label: "Running Offline Stake Operations Tests"
+    label: "Running Offline Stake Operations Tests on GCE"
     env:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "gce"
-      TESTNET_TAG: "gce-perf-cpu-only"
       ENABLE_GPU: "false"
       TEST_DURATION_SECONDS: 30
       NUMBER_OF_VALIDATOR_NODES: 1
       VALIDATOR_NODE_MACHINE_TYPE: "--machine-type n1-standard-16"
       NUMBER_OF_CLIENT_NODES: 0
-      ADDITIONAL_FLAGS: ""
+      ADDITIONAL_FLAGS: "--dedicated"
       BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 100
       SKIP_PERF_RESULTS: "true"
       TEST_TYPE: "script"

diff --git a/system-test/stake-operations-testcases/offline_stake_operations.sh b/system-test/stake-operations-testcases/offline_stake_operations.sh
@@ -5,6 +5,10 @@
 # shellcheck disable=SC2206
 # shellcheck disable=SC2162
 
+# shellcheck disable=SC1090
+# shellcheck disable=SC1091
+source "$(dirname "$0")"/../automation_utils.sh
+
 set -e
 
 if [[ -n "$1" ]]; then
@@ -58,14 +62,14 @@ solana airdrop 2 $online_system_account_pubkey
 solana create-nonce-account $online_nonce_account_keypair 1 --nonce-authority $offline_system_account_pubkey --keypair $online_system_account_keypair
 nonce="$(solana nonce $nonce_account_pubkey)"
 
-echo --- OFFLINE SYSTEM ACCOUNT BALANCE BEFORE CREATING STAKE ACCOUNTS
+execution_step OFFLINE SYSTEM ACCOUNT BALANCE BEFORE CREATING STAKE ACCOUNTS
 (
   set -x
   solana balance $offline_system_account_pubkey
 )
 
 ################################
-echo --- OFFLINE STAKE ACCOUNT CREATION
+execution_step CREATE OFFLINE STAKE ACCOUNT
 ################################
 
 # Create a stake account funded by the offline system account
@@ -93,21 +97,21 @@ solana create-stake-account $stake_account_keypair 50 \
   --lockup-epoch 999 \
   --from $offline_system_account_pubkey --fee-payer $offline_system_account_pubkey ${signers[@]}
 
-echo --- STAKE ACCOUNT AFTER CREATION
+execution_step VIEW STAKE ACCOUNT AFTER CREATION
 (
   set -x
   solana stake-account $stake_account_address
 )
 
 
-echo --- OFFLINE SYSTEM ACCOUNT BALANCE AFTER CREATING FIRST STAKE ACCOUNT
+execution_step VIEW OFFLINE SYSTEM ACCOUNT BALANCE AFTER CREATING FIRST STAKE ACCOUNT
 (
   set -x
   solana balance $offline_system_account_pubkey
 )
 
 #####################
-echo --- OFFLINE STAKE SPLIT
+execution_step SPLIT STAKE OFFLINE
 #####################
 
 # Split the original stake account before delegating
@@ -131,20 +135,20 @@ solana split-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-auth
   --stake-authority $offline_staker_pubkey $stake_account_address $split_stake_account_keypair 10 \
   --fee-payer $offline_system_account_pubkey ${signers[@]}
 
-echo --- ORIGINAL STAKE ACCOUNT AFTER SPLITTING
+execution_step VIEW ORIGINAL STAKE ACCOUNT AFTER SPLITTING
 (
   set -x
   solana stake-account $stake_account_address
 )
 
-echo --- NEW STAKE ACCOUNT CREATED FROM SPLITTING ORIGINAL
+execution_step VIEW NEW STAKE ACCOUNT CREATED FROM SPLITTING ORIGINAL
 (
   set -x
   solana stake-account $split_stake_account_address
 )
 
 #####################
-echo --- CUSTODIAN CHANGE LOCKUP
+execution_step CHANGE CUSTODIAN LOCKUP
 #####################
 
 # Set the lockup epoch to 0 to allow stake to be withdrawn
@@ -164,14 +168,14 @@ solana stake-set-lockup --blockhash $nonce --nonce $nonce_account_pubkey --nonce
   $split_stake_account_address --custodian $offline_custodian_pubkey --lockup-epoch 0 \
   --fee-payer $offline_system_account_pubkey ${signers[@]}
 
-echo --- SPLIT STAKE ACCOUNT AFTER CHANGING LOCKUP
+execution_step VIEW SPLIT STAKE ACCOUNT AFTER CHANGING LOCKUP
 (
   set -x
   solana stake-account $split_stake_account_address
 )
 
 ##########################
-echo --- OFFLINE STAKE WITHDRAWAL
+execution_step OFFLINE STAKE WITHDRAWAL
 ##########################
 
 # Withdraw the lamports from the stake account that was split off and return them to the offline system account
@@ -193,14 +197,14 @@ solana withdraw-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-a
   --withdraw-authority $offline_withdrawer_pubkey \
   --fee-payer $offline_system_account_pubkey ${signers[@]}
 
-echo --- OFFLINE SYSTEM ACCOUNT BALANCE AFTER WITHDRAWING SPLIT STAKE
+execution_step VIEW OFFLINE SYSTEM ACCOUNT BALANCE AFTER WITHDRAWING SPLIT STAKE
 (
   set -x
   solana balance $offline_system_account_pubkey
 )
 
 ##########################
-echo --- OFFLINE STAKE DELEGATION
+execution_step OFFLINE STAKE DELEGATION
 ##########################
 
 # Delegate stake from the original account to a vote account
@@ -223,14 +227,14 @@ solana delegate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce-a
 --stake-authority $offline_staker_pubkey $stake_account_address $vote_account_pubkey \
 --fee-payer $offline_system_account_pubkey ${signers[@]}
 
-echo --- ORIGINAL STAKE ACCOUNT AFTER DELEGATION
+execution_step VIEW ORIGINAL STAKE ACCOUNT AFTER DELEGATION
 (
   set -x
   solana stake-account $stake_account_address
 )
 
 ##########################
- echo --- OFFLINE STAKE DEACTIVATION
+ execution_step OFFLINE STAKE DEACTIVATION
 ##########################
 
 # Deactivate delegated stake
@@ -252,7 +256,7 @@ solana deactivate-stake --blockhash $nonce --nonce $nonce_account_pubkey --nonce
 --stake-authority $offline_staker_pubkey $stake_account_address \
 --fee-payer $offline_system_account_pubkey ${signers[@]}
 
-echo --- ORIGINAL STAKE ACCOUNT AFTER DEACTIVATION
+execution_step VIEW ORIGINAL STAKE ACCOUNT AFTER DEACTIVATION
 (
    set -x
    solana stake-account $stake_account_address

diff --git a/system-test/stake-operations-testcases/stake_test_automation.sh b/system-test/stake-operations-testcases/stake_test_automation.sh
@@ -6,9 +6,13 @@ set -ex
 # shellcheck disable=SC1091
 source "$(dirname "$0")"/../automation_utils.sh
 
+RESULT_FILE="$1"
+
 # Runs offline stake operations tests against a running cluster launched from the automation framework
 
 bootstrapper_ip_address="$(get_bootstrap_validator_ip_address)"
 entrypoint=http://"${bootstrapper_ip_address}":8899
 
 PATH="$REPO_ROOT"/solana-release/bin:$PATH "$REPO_ROOT"/system-test/stake-operations-testcases/offline_stake_operations.sh "$entrypoint"
+
+echo "Offline Stake Operations Test Succeeded" >>$RESULT_FILE
diff --git a/system-test/testnet-automation.sh b/system-test/testnet-automation.sh
@@ -171,7 +171,7 @@ function launch_testnet() {
       ;;
     script)
       execution_step "Running custom script: ${REPO_ROOT}/${CUSTOM_SCRIPT}"
-      "$REPO_ROOT"/"$CUSTOM_SCRIPT"
+      "$REPO_ROOT"/"$CUSTOM_SCRIPT" "$RESULT_FILE"
       ;;
     *)
       echo "Error: Unsupported test type: $TEST_TYPE"
@@ -185,9 +185,10 @@ function launch_testnet() {
   SLOTS_PER_SECOND="$(bc <<< "scale=3; ($END_SLOT - $START_SLOT)/($SLOT_COUNT_END_SECONDS - $SLOT_COUNT_START_SECONDS)")"
   execution_step "Average slot rate: $SLOTS_PER_SECOND slots/second over $((SLOT_COUNT_END_SECONDS - SLOT_COUNT_START_SECONDS)) seconds"
 
-  [[ "$SKIP_PERF_RESULTS" = "false" ]] || collect_performance_statistics
-
-  echo "slots_per_second: $SLOTS_PER_SECOND" >>"$RESULT_FILE"
+  if [[ "$SKIP_PERF_RESULTS" = "false" ]]; then
+    collect_performance_statistics
+    echo "slots_per_second: $SLOTS_PER_SECOND" >>"$RESULT_FILE"
+  fi
 
   RESULT_DETAILS=$(<"$RESULT_FILE")
   upload-ci-artifact "$RESULT_FILE"
@@ -198,7 +199,7 @@ RESULT_DETAILS=
 STEP=
 execution_step "Initialize Environment"
 
-[[ -n $TESTNET_TAG ]] || TESTNET_TAG=testnet-automation
+[[ -n $TESTNET_TAG ]] || TESTNET_TAG=${CLOUD_PROVIDER}-testnet-automation
 [[ -n $INFLUX_HOST ]] || INFLUX_HOST=https://metrics.solana.com:8086
 [[ -n $BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD ]] || BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD=66
 [[ -n $SKIP_PERF_RESULTS ]] || SKIP_PERF_RESULTS=false
@@ -250,16 +251,22 @@ execution_step "Checking for required parameters"
 testTypeRequiredParameters=
 case $TEST_TYPE in
   fixed_duration)
-    testTypeRequiredParameters+=TEST_DURATION_SECONDS
+    testTypeRequiredParameters=(
+      TEST_DURATION_SECONDS \
+      )
     ;;
   partition)
-    testTypeRequiredParameters+=NETEM_CONFIG_FILE
-    testTypeRequiredParameters+=PARTITION_ACTIVE_DURATION
-    testTypeRequiredParameters+=PARTITION_INACTIVE_DURATION
-    testTypeRequiredParameters+=PARTITION_ITERATION_COUNT
+    testTypeRequiredParameters=(
+      NETEM_CONFIG_FILE \
+      PARTITION_ACTIVE_DURATION \
+      PARTITION_INACTIVE_DURATION \
+      PARTITION_ITERATION_COUNT \
+    )
     ;;
   script)
-    testTypeRequiredParameters+=CUSTOM_SCRIPT
+    testTypeRequiredParameters=(
+      CUSTOM_SCRIPT \
+    )
     ;;
   *)
     echo "Error: Unsupported test type: $TEST_TYPE"
@@ -269,7 +276,7 @@ esac
 missingParameters=
 for i in "${testTypeRequiredParameters[@]}"; do
   if [[ -z ${!i} ]]; then
-    missingParameters+="${i} "
+    missingParameters+="${i}, "
   fi
 done
 
@@ -308,6 +315,8 @@ TEST_PARAMS_TO_DISPLAY=(CLOUD_PROVIDER \
                         PARTITION_ACTIVE_DURATION \
                         PARTITION_INACTIVE_DURATION \
                         PARTITION_ITERATION_COUNT \
+                        TEST_TYPE \
+                        CUSTOM_SCRIPT \
                         )
 
 TEST_CONFIGURATION=