From eb4bf55a38907e24cd3f5c4e0e2858443f83ef86 Mon Sep 17 00:00:00 2001 From: Vidit Bhat Date: Mon, 28 Oct 2024 15:23:56 +0530 Subject: [PATCH] drtprod: increase drt-scale warehouses and reduce RF to 3 This PR increases the primary and secondary tpcc warehouses to 400k and 2M respectively. We also reduced the RF to 3 and removed some unnecessary steps. Epic: none Release note: None --- pkg/cmd/drtprod/configs/drt_scale.yaml | 34 ++++++------------- .../drtprod/scripts/create_run_operation.sh | 3 ++ pkg/cmd/drtprod/scripts/generate_tpcc_run.sh | 24 +++++++++++-- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/pkg/cmd/drtprod/configs/drt_scale.yaml b/pkg/cmd/drtprod/configs/drt_scale.yaml index adc644ee31c4..4b52961aa0c7 100644 --- a/pkg/cmd/drtprod/configs/drt_scale.yaml +++ b/pkg/cmd/drtprod/configs/drt_scale.yaml @@ -58,18 +58,6 @@ targets: - $CLUSTER - -- - "sudo systemctl unmask cron.service ; sudo systemctl enable cron.service ; echo \"crontab -l ; echo '@reboot sleep 100 && ~/cockroach.sh' | crontab -\" > t.sh ; sh t.sh ; rm t.sh" - - command: sql - args: - - $CLUSTER:1 - - -- - - -e - - "ALTER RANGE timeseries CONFIGURE ZONE USING num_replicas=5,num_voters=5" - - command: sql - args: - - $CLUSTER:1 - - -- - - -e - - "ALTER RANGE default CONFIGURE ZONE USING num_replicas=5,num_voters=5" # workload cluster specs - target_name: $WORKLOAD_CLUSTER steps: @@ -136,29 +124,27 @@ targets: - $WORKLOAD_CLUSTER - artifacts/roachtest - roachtest-operations - - command: put - args: - - $WORKLOAD_CLUSTER - - pkg/cmd/drt/scripts/roachtest_operations_run.sh - - roachtest_operations_run.sh - script: "pkg/cmd/drtprod/scripts/tpcc_init.sh" args: - - cct_tpcc_320k # suffix added to script name tpcc_init_cct_tpcc_320k.sh - - true # determines whether to execute the script immediately on workload node + - cct_tpcc_400k # suffix added to script name tpcc_init_cct_tpcc_400k.sh + - false # determines whether to execute the script immediately on workload node flags: - warehouses: 320000 + warehouses: 400000 db: cct_tpcc - script: "pkg/cmd/drtprod/scripts/tpcc_init.sh" args: - - cct_tpcc_640k # suffix added to script name tpcc_init_cct_tpcc_640k.sh + - cct_tpcc_2000k # suffix added to script name tpcc_init_cct_tpcc_2000k.sh - false # determines whether to execute the script immediately on workload node flags: - warehouses: 640000 + warehouses: 2000000 db: cct_tpcc_big - script: "pkg/cmd/drtprod/scripts/generate_tpcc_run.sh" + args: + - cct_tpcc_400k # suffix added to script name tpcc_run_400k.sh + - false # determines whether to execute the script immediately on workload node flags: db: cct_tpcc - warehouses: 320000 + warehouses: 400000 max-rate: 10000 workers: 5000 conns: 5000 @@ -168,7 +154,7 @@ targets: - script: "pkg/cmd/drtprod/scripts/tpch_init.sh" args: - scale_factor_1000 # suffix added to script name tpch_init_scale_factor_1000.sh - - true # determines whether to execute the script immediately on workload node + - false # determines whether to execute the script immediately on workload node flags: scale-factor: 1000 - script: "pkg/cmd/drtprod/scripts/generate_tpch_run.sh" diff --git a/pkg/cmd/drtprod/scripts/create_run_operation.sh b/pkg/cmd/drtprod/scripts/create_run_operation.sh index 014100cf5342..e5117eb0a5e7 100755 --- a/pkg/cmd/drtprod/scripts/create_run_operation.sh +++ b/pkg/cmd/drtprod/scripts/create_run_operation.sh @@ -31,6 +31,9 @@ if [ -z "${dd_api_key}" ]; then exit 1 fi +# sync cluster for operations +drtprod ssh ${WORKLOAD_CLUSTER} -- "ROACHPROD_GCE_DEFAULT_PROJECT=${ROACHPROD_GCE_DEFAULT_PROJECT} ./roachprod sync" + # the ssh keys of all workload nodes should be setup on the crdb nodes for the operations roachprod ssh ${CLUSTER} -- "echo \"$(roachprod run ${WORKLOAD_CLUSTER} -- cat ./.ssh/id_rsa.pub|grep ssh-rsa)\" >> ./.ssh/authorized_keys" diff --git a/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh b/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh index 2e8038d430a2..29bc5bd54866 100755 --- a/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh +++ b/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh @@ -8,6 +8,21 @@ # This script sets up the tpcc run workload script in the workload nodes # The --warehouses flag is passed as argument to this script # NOTE - This uses CLUSTER and WORKLOAD_CLUSTER environment variable, if not set the script fails +if [ "$#" -lt 2 ]; then + echo "Usage: $0 " + exit 1 +fi +suffix=$1 +shift +# The second argument represents whether the init process should be started in the workload cluster +# The value is true or false +if [ "$1" != "true" ] && [ "$1" != "false" ]; then + # $1 is used again because of the shift + echo "Error: The second argument must be 'true' or 'false' which implies whether the script should be started in background or not." + exit 1 +fi +execute_script=$1 +shift if [ -z "${CLUSTER}" ]; then echo "environment CLUSTER is not set" @@ -31,7 +46,7 @@ PGURLS=$(roachprod pgurl $CLUSTER | sed s/\'//g) for NODE in $(seq 1 $WORKLOAD_NODES) do # Create the workload script - cat </tmp/tpcc_run.sh + cat </tmp/tpcc_run_${suffix}.sh #!/usr/bin/env bash read -r -a PGURLS_ARR <<< "$PGURLS" @@ -53,6 +68,9 @@ done EOF # Upload the script to the workload cluster - roachprod put $WORKLOAD_CLUSTER:$NODE /tmp/tpcc_run.sh - roachprod ssh $WORKLOAD_CLUSTER:$NODE -- chmod +x tpcc_run.sh + roachprod put $WORKLOAD_CLUSTER:$NODE /tmp/tpcc_run_${suffix}.sh + roachprod ssh $WORKLOAD_CLUSTER:$NODE -- "chmod +x tpcc_run_${suffix}.sh" + if [ "$execute_script" = "true" ]; then + roachprod run "${WORKLOAD_CLUSTER}":1 -- "sudo systemd-run --unit tpcc_run_${suffix} --same-dir --uid \$(id -u) --gid \$(id -g) bash ${pwd}/tpcc_run_${suffix}.sh" + fi done