-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
135231: scripts/ldr: record tpcc setup steps r=dt a=dt Release note: none. Epic: none. 135554: drtprod: update drt-chaos yaml and scripts r=shailendra-patel a=vidit-bhat This PR updates the deployment yaml for `drt-chaos` and generalises the common scripts for workloads Epic: none Release note: None Co-authored-by: David Taylor <[email protected]> Co-authored-by: Vidit Bhat <[email protected]>
- Loading branch information
Showing
4 changed files
with
253 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Copyright 2024 The Cockroach Authors. | ||
| # | ||
| # Use of this software is governed by the CockroachDB Software License | ||
| # included in the /LICENSE file. | ||
|
|
||
| # This script sets up the kv workload script in the workload nodes | ||
| # NOTE - This uses CLUSTER and WORKLOAD_CLUSTER environment variable, if not set the script fails | ||
| if [ "$#" -lt 1 ]; then | ||
| echo "Usage: $0 <execute:true|false>" | ||
| exit 1 | ||
| fi | ||
| execute_script=$1 | ||
| shift | ||
|
|
||
| if [ -z "${CLUSTER}" ]; then | ||
| echo "environment CLUSTER is not set" | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ -z "${WORKLOAD_CLUSTER}" ]; then | ||
| echo "environment WORKLOAD_CLUSTER is not set" | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ -z "${WORKLOAD_NODES}" ]; then | ||
| echo "environment WORKLOAD_NODES is not set" | ||
| exit 1 | ||
| fi | ||
|
|
||
| PGURLS=$(roachprod pgurl $CLUSTER --external | sed s/\'//g) | ||
|
|
||
| # Loop through each node | ||
| for NODE in $(seq 1 $WORKLOAD_NODES) | ||
| do | ||
| # Create the workload script | ||
| cat <<EOF >/tmp/kv_run.sh | ||
| #!/usr/bin/env bash | ||
| read -r -a PGURLS_ARR <<< "$PGURLS" | ||
| j=0 | ||
| while true; do | ||
| echo ">> Starting kv workload" | ||
| ((j++)) | ||
| LOG=./kv_$j.txt | ||
| ./cockroach workload run kv \ | ||
| --init \ | ||
| --drop \ | ||
| --concurrency 128 \ | ||
| --histograms kv/stats.json \ | ||
| --db kv \ | ||
| --splits 1000 \ | ||
| --read-percent 50 \ | ||
| --span-percent 20 \ | ||
| --cycle-length 100000 \ | ||
| --min-block-bytes 100 \ | ||
| --max-block-bytes 1000 \ | ||
| --prometheus-port 2114 \ | ||
| --max-rate 200 \ | ||
| --secure \ | ||
| --ramp 10m \ | ||
| --display-every 5s \ | ||
| --duration 12h \ | ||
| --tolerate-errors \ | ||
| --enum \ | ||
| "\${PGURLS_ARR[@]}" | tee "\$LOG" | ||
| if [ \$? -eq 0 ]; then | ||
| rm "\$LOG" | ||
| fi | ||
| sleep 1 | ||
| done | ||
| EOF | ||
|
|
||
| # Upload the script to the workload cluster | ||
| roachprod put $WORKLOAD_CLUSTER:$NODE /tmp/kv_run.sh | ||
| roachprod ssh $WORKLOAD_CLUSTER:$NODE -- "chmod +x kv_run.sh" | ||
| if [ "$execute_script" = "true" ]; then | ||
| roachprod run "${WORKLOAD_CLUSTER}":1 -- "sudo systemd-run --unit kv_run --same-dir --uid \$(id -u) --gid \$(id -g) bash ${pwd}/kv_run.sh" | ||
| fi | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Copyright 2024 The Cockroach Authors. | ||
| # | ||
| # Use of this software is governed by the CockroachDB Software License | ||
| # included in the /LICENSE file. | ||
|
|
||
| # This script sets up the tpcc drop workload script in the workload nodes | ||
| # NOTE - This uses CLUSTER and WORKLOAD_CLUSTER environment variable, if not set the script fails | ||
| if [ "$#" -lt 1 ]; then | ||
| echo "Usage: $0 <execute:true|false>" | ||
| exit 1 | ||
| fi | ||
| execute_script=$1 | ||
| shift | ||
|
|
||
| if [ -z "${CLUSTER}" ]; then | ||
| echo "environment CLUSTER is not set" | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ -z "${WORKLOAD_CLUSTER}" ]; then | ||
| echo "environment WORKLOAD_CLUSTER is not set" | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ -z "${WORKLOAD_NODES}" ]; then | ||
| echo "environment WORKLOAD_NODES is not set" | ||
| exit 1 | ||
| fi | ||
|
|
||
| PG_URL_N1=$(roachprod pgurl $CLUSTER:1 --external | sed s/\'//g) | ||
| PGURLS=$(roachprod pgurl $CLUSTER --external | sed s/\'//g) | ||
|
|
||
| # Loop through each node | ||
| for NODE in $(seq 1 $WORKLOAD_NODES) | ||
| do | ||
| # Create the workload script | ||
| cat <<EOF >/tmp/tpcc_drop.sh | ||
| #!/usr/bin/env bash | ||
| read -r -a PGURLS_ARR <<< "$PGURLS" | ||
| echo ">> Dropping old databases if they exist" | ||
| ./cockroach sql --url "${PG_URL_N1}" -e "DROP DATABASE IF EXISTS cct_tpcc_drop_old CASCADE;" | ||
| ./cockroach sql --url "${PG_URL_N1}" -e "DROP DATABASE IF EXISTS cct_tpcc_drop CASCADE;" | ||
| j=0 | ||
| while true; do | ||
| echo ">> Starting tpcc-drop-db workload" | ||
| echo ">> Importing tpcc" | ||
| ((j++)) | ||
| INIT_LOG=./cct_tpcc_drop_init_\$j.txt | ||
| RUN_LOG=./cct_tpcc_drop_run_\$j.txt | ||
| # Temporarily, we cleanup and paused IMPORT jobs that may exist due to | ||
| # node failures. Long-term, we will address these failures by making | ||
| # IMPORT more resilient and/or creating a variant of IMPORT which cancels | ||
| # itself instead of pausing. | ||
| ./cockroach sql --url "${PG_URL_N1}" -e "CANCEL JOBS (WITH x AS (SHOW JOBS) SELECT job_id FROM x WHERE status = 'paused' AND job_type = 'IMPORT');" | ||
| sleep 15 | ||
| ./cockroach workload init tpcc \ | ||
| --warehouses=3000 \ | ||
| --secure \ | ||
| --concurrency 4 \ | ||
| --db cct_tpcc_drop \ | ||
| \$PG_URL_N1 | tee "\$INIT_LOG" | ||
| echo ">> Dropping cct_tpcc_drop_old if it exists" | ||
| ./cockroach sql --url "${PG_URL_N1}" -e "DROP DATABASE cct_tpcc_drop_old CASCADE;" | ||
| sleep 5 | ||
| echo ">> Starting tpcc workload for 1h" | ||
| ./cockroach workload run tpcc \ | ||
| --warehouses 3000 \ | ||
| --active-warehouses 1000 \ | ||
| --db cct_tpcc_drop \ | ||
| --secure \ | ||
| --prometheus-port 2113 \ | ||
| --ramp 5m \ | ||
| --display-every 5s \ | ||
| --duration 60m \ | ||
| --tolerate-errors \ | ||
| "\${PGURLS_ARR[@]}" | tee "\$RUN_LOG" | ||
| echo ">> Renaming to cct_tpcc_drop_old" | ||
| ./cockroach sql --url "${PG_URL_N1}" -e "ALTER DATABASE cct_tpcc_drop RENAME TO cct_tpcc_drop_old;" | ||
| sleep 1 | ||
| done | ||
| EOF | ||
|
|
||
| # Upload the script to the workload cluster | ||
| roachprod put $WORKLOAD_CLUSTER:$NODE /tmp/tpcc_drop.sh | ||
| roachprod ssh $WORKLOAD_CLUSTER:$NODE -- "chmod +x tpcc_drop.sh" | ||
| if [ "$execute_script" = "true" ]; then | ||
| roachprod run "${WORKLOAD_CLUSTER}":1 -- "sudo systemd-run --unit tpcc_drop --same-dir --uid \$(id -u) --gid \$(id -g) bash ${pwd}/tpcc_drop.sh" | ||
| fi | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters