Merge cockroachdb#133553

133553: drtprod: increase drt-scale warehouses and reduce RF to 3 r=nameisbhaskar a=vidit-bhat This PR increases the primary and secondary `tpcc` warehouses to 400k and 2M respectively. We also reduced the RF to 3 and removed some unnecessary steps. Epic: none Release note: None Co-authored-by: Vidit Bhat <[email protected]>
herkolategan · Oct 29, 2024 · 987dd38 · 987dd38
2 parents c0ba836 + 1d70722
commit 987dd38
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 27 deletions.
diff --git a/pkg/cmd/drtprod/configs/drt_scale.yaml b/pkg/cmd/drtprod/configs/drt_scale.yaml
@@ -58,18 +58,6 @@ targets:
           - $CLUSTER
           - --
           - "sudo systemctl unmask cron.service ; sudo systemctl enable cron.service ; echo \"crontab -l ; echo '@reboot sleep 100 && ~/cockroach.sh' | crontab -\" > t.sh ; sh t.sh ; rm t.sh"
-      - command: sql
-        args:
-          - $CLUSTER:1
-          - --
-          - -e
-          - "ALTER RANGE timeseries CONFIGURE ZONE USING num_replicas=5,num_voters=5"
-      - command: sql
-        args:
-          - $CLUSTER:1
-          - --
-          - -e
-          - "ALTER RANGE default CONFIGURE ZONE USING num_replicas=5,num_voters=5"
   # workload cluster specs
   - target_name: $WORKLOAD_CLUSTER
     steps:
@@ -136,29 +124,27 @@ targets:
           - $WORKLOAD_CLUSTER
           - artifacts/roachtest
           - roachtest-operations
-      - command: put
-        args:
-          - $WORKLOAD_CLUSTER
-          - pkg/cmd/drt/scripts/roachtest_operations_run.sh
-          - roachtest_operations_run.sh
       - script: "pkg/cmd/drtprod/scripts/tpcc_init.sh"
         args:
-          - cct_tpcc_320k # suffix added to script name tpcc_init_cct_tpcc_320k.sh
-          - true # determines whether to execute the script immediately on workload node
+          - cct_tpcc_400k # suffix added to script name tpcc_init_cct_tpcc_400k.sh
+          - false # determines whether to execute the script immediately on workload node
         flags:
-          warehouses: 320000
+          warehouses: 400000
           db: cct_tpcc
       - script: "pkg/cmd/drtprod/scripts/tpcc_init.sh"
         args:
-          - cct_tpcc_640k # suffix added to script name tpcc_init_cct_tpcc_640k.sh
+          - cct_tpcc_2000k # suffix added to script name tpcc_init_cct_tpcc_2000k.sh
           - false # determines whether to execute the script immediately on workload node
         flags:
-          warehouses: 640000
+          warehouses: 2000000
           db: cct_tpcc_big
       - script: "pkg/cmd/drtprod/scripts/generate_tpcc_run.sh"
+        args:
+          - cct_tpcc_400k # suffix added to script name tpcc_run_400k.sh
+          - false # determines whether to execute the script immediately on workload node
         flags:
           db: cct_tpcc
-          warehouses: 320000
+          warehouses: 400000
           max-rate: 10000
           workers: 5000
           conns: 5000
@@ -168,7 +154,7 @@ targets:
       - script: "pkg/cmd/drtprod/scripts/tpch_init.sh"
         args:
           - scale_factor_1000 # suffix added to script name tpch_init_scale_factor_1000.sh
-          - true # determines whether to execute the script immediately on workload node
+          - false # determines whether to execute the script immediately on workload node
         flags:
           scale-factor: 1000
       - script: "pkg/cmd/drtprod/scripts/generate_tpch_run.sh"

diff --git a/pkg/cmd/drtprod/scripts/create_run_operation.sh b/pkg/cmd/drtprod/scripts/create_run_operation.sh
@@ -31,6 +31,9 @@ if [ -z "${dd_api_key}" ]; then
   exit 1
 fi
 
+# sync cluster is needed for operations
+drtprod ssh ${WORKLOAD_CLUSTER} -- "ROACHPROD_GCE_DEFAULT_PROJECT=${ROACHPROD_GCE_DEFAULT_PROJECT} ./roachprod sync"
+
 # the ssh keys of all workload nodes should be setup on the crdb nodes for the operations
 roachprod ssh ${CLUSTER} -- "echo \"$(roachprod run ${WORKLOAD_CLUSTER} -- cat ./.ssh/id_rsa.pub|grep ssh-rsa)\" >> ./.ssh/authorized_keys"
 

diff --git a/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh b/pkg/cmd/drtprod/scripts/generate_tpcc_run.sh
@@ -8,6 +8,21 @@
 # This script sets up the tpcc run workload script in the workload nodes
 # The --warehouses flag is passed as argument to this script
 # NOTE - This uses CLUSTER and WORKLOAD_CLUSTER environment variable, if not set the script fails
+if [ "$#" -lt 2 ]; then
+  echo "Usage: $0 <script_suffix> <execute:true|false> <flags to init:--warehouses,--db>"
+  exit 1
+fi
+suffix=$1
+shift
+# The second argument represents whether the init process should be started in the workload cluster
+# The value is true or false
+if [ "$1" != "true" ] && [ "$1" != "false" ]; then
+  # $1 is used again because of the shift
+  echo "Error: The second argument must be 'true' or 'false' which implies whether the script should be started in background or not."
+  exit 1
+fi
+execute_script=$1
+shift
 
 if [ -z "${CLUSTER}" ]; then
   echo "environment CLUSTER is not set"
@@ -31,7 +46,7 @@ PGURLS=$(roachprod pgurl $CLUSTER | sed s/\'//g)
 for NODE in $(seq 1 $WORKLOAD_NODES)
 do
   # Create the workload script
-  cat <<EOF >/tmp/tpcc_run.sh
+  cat <<EOF >/tmp/tpcc_run_${suffix}.sh
 #!/usr/bin/env bash
 
 read -r -a PGURLS_ARR <<< "$PGURLS"
@@ -53,6 +68,9 @@ done
 EOF
 
   # Upload the script to the workload cluster
-  roachprod put $WORKLOAD_CLUSTER:$NODE /tmp/tpcc_run.sh
-  roachprod ssh $WORKLOAD_CLUSTER:$NODE -- chmod +x tpcc_run.sh
+  roachprod put $WORKLOAD_CLUSTER:$NODE /tmp/tpcc_run_${suffix}.sh
+  roachprod ssh $WORKLOAD_CLUSTER:$NODE -- "chmod +x tpcc_run_${suffix}.sh"
+  if [ "$execute_script" = "true" ]; then
+    roachprod run "${WORKLOAD_CLUSTER}":1 -- "sudo systemd-run --unit tpcc_run_${suffix} --same-dir --uid \$(id -u) --gid \$(id -g) bash ${pwd}/tpcc_run_${suffix}.sh"
+  fi
 done