Parallel testing (#1419)

Parallel sync tests Signed-off-by: mambisi <[email protected]> Co-authored-by: jouzo <[email protected]> Co-authored-by: Prasanna Loganathar <[email protected]>
DeFiCh · Aug 25, 2022 · 5ad2ca3 · 5ad2ca3
1 parent 4c17904
commit 5ad2ca3
Show file tree

Hide file tree

Showing 2 changed files with 222 additions and 0 deletions.
diff --git a/.github/workflows/automated-run.yml b/.github/workflows/automated-run.yml
@@ -0,0 +1,129 @@
+name: Full Sync Tests
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+jobs:
+  build-binaries:
+    runs-on: [self-hosted, linux, x64,server-2]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Build Node
+      run: ./make.sh build
+    - name: Upload Binaries
+      uses: actions/upload-artifact@v3
+      with:
+        name: defibins
+        path: |
+         src/defid
+         src/defi-cli
+    - name: Archive Binaries
+      uses: actions/upload-artifact@v3
+      with:
+        name: defibins
+        path: |
+          src/defid
+          src/defi-cli
+    - name: Archive Shell Commands
+      uses: actions/upload-artifact@v3
+      with:
+        name: sync
+        path: |
+          ci/parallel_sync/sync_then_diff.sh
+
+  sync:
+    strategy:
+      matrix:
+        datadir:
+         - start: 450000
+           stop: 500000
+         - start: 500000
+           stop: 550000
+         - start: 550000
+           stop: 600000
+         - start: 600000
+           stop: 650000
+         - start: 650000
+           stop: 700000
+         - start: 700000
+           stop: 750000
+         - start: 750000
+           stop: 800000
+         - start: 800000
+           stop: 850000
+         - start: 850000
+           stop: 900000
+         - start: 900000
+           stop: 950000
+         - start: 950000
+           stop: 1000000
+         - start: 1000000
+           stop: 1050000
+         - start: 1050000
+           stop: 1100000
+         - start: 1100000
+           stop: 1150000
+         - start: 1150000
+           stop: 1200000
+         - start: 1200000
+           stop: 1250000
+         - start: 1250000
+           stop: 1300000
+         - start: 1300000
+           stop: 1350000
+         - start: 1350000
+           stop: 1400000
+         - start: 1400000
+           stop: 1450000
+         - start: 1450000
+           stop: 1500000
+         - start: 1500000
+           stop: 1550000
+         - start: 1550000
+           stop: 1600000
+         - start: 1600000
+           stop: 1650000
+         - start: 1650000
+           stop: 1700000
+         - start: 1700000
+           stop: 1750000
+         - start: 1750000
+           stop: 1800000
+         - start: 1800000
+           stop: 1850000
+         - start: 1850000
+           stop: 1900000
+    runs-on: [self-hosted, linux, x64]
+    needs: build-binaries
+    continue-on-error: true
+    container:
+      image : gcr.io/br-blockchains-dev/datadir-${{matrix.datadir.start}}
+      options: --privileged
+      env:
+        STOP_BLOCK: ${{matrix.datadir.stop}}
+        DEFID_BIN: ./defid
+        DEFI_CLI_BIN: ./defi-cli
+    timeout-minutes: 4320
+    steps:
+    - name: Download Binaries
+      uses: actions/download-artifact@v3
+      with:
+        name: defibins
+    - name: Download Shell Commands
+      uses: actions/download-artifact@v3
+      with:
+        name: sync
+    - name: Set Permissions
+      run: |
+        chmod 777 defid
+        chmod 777 defi-cli
+        chmod 777 sync_then_diff.sh
+    - name: Sync and Diff
+      run: ./sync_then_diff.sh
+    - name: Show Debug Log
+      run: cat $DATADIR/debug.log
diff --git a/ci/parallel_sync/sync_then_diff.sh b/ci/parallel_sync/sync_then_diff.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+
+# TODO: Resolve shellcheck errors
+
+export LC_ALL=C
+set -Eeuo pipefail
+
+setup_vars() {
+  # Binaries
+  DEFID_BIN=${DEFID_BIN:-"./defid"}
+  DEFI_CLI_BIN=${DEFI_CLI_BIN:-"./defi-cli"}
+
+  # Files and directories
+  DATADIR=${DATADIR:-".defi"}
+  DEBUG_FILE="$DATADIR/debug.log"
+  TMP_LOG=debug-tmp-$STOP_BLOCK.log
+  BASE_PATH=https://storage.googleapis.com
+  BUCKET=team-drop
+  REF_LOG_DIR=master-logs-full
+  REF_LOG=debug-$STOP_BLOCK.log
+  REF_LOG_PATH=$BASE_PATH/$BUCKET/$REF_LOG_DIR/$REF_LOG
+
+  # Commands
+  DEFID_CMD="$DEFID_BIN -datadir=$DATADIR -daemon -debug=accountchange"
+  DEFI_CLI_CMD="$DEFI_CLI_BIN -datadir=$DATADIR"
+  ACCOUNT_BALANCES_CMD="$DEFI_CLI_CMD logaccountbalances"
+  LIST_ANCHORS_CMD="$DEFI_CLI_CMD spv_listanchors"
+  FETCH="wget -q"
+  GREP="grep"
+
+  BLOCK=0
+  ATTEMPTS=0
+  MAX_ATTEMPTS=10
+  MAX_NODE_RESTARTS=5
+  NODE_RESTARTS=0
+}
+
+# Start defid
+start_node () {
+  echo "Syncing to block height: $STOP_BLOCK"
+  $DEFID_CMD -interrupt-block=$((STOP_BLOCK + 1))
+  sleep 30
+}
+
+main() {
+  setup_vars
+  start_node
+
+  $DEFI_CLI_CMD clearbanned || true
+
+  # Sync to target block height
+  while [ "$BLOCK" -lt "$STOP_BLOCK" ]; do
+    if [ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]; then
+      if [ "$NODE_RESTARTS" -lt "$MAX_NODE_RESTARTS" ]; then
+        echo "Node Stuck After $ATTEMPTS attempts, restarting node"
+        $DEFI_CLI_CMD stop
+        sleep 20
+        start_node
+        NODE_RESTARTS=$((NODE_RESTARTS + 1))
+        ATTEMPTS=0
+      else
+        exit 1
+      fi
+    fi
+    CUR_BLOCK=$($DEFI_CLI_CMD getblockcount || echo $BLOCK)
+    if [ "$CUR_BLOCK" -eq "$BLOCK" ]; then
+      ATTEMPTS=$((ATTEMPTS + 1))
+    else
+      ATTEMPTS=0
+    fi
+    BLOCK=${CUR_BLOCK:-$BLOCK}
+    echo "Current block: $BLOCK"
+    sleep 20
+  done
+
+  # Create temporary log file
+  $GREP "AccountChange:" $DEBUG_FILE | cut -d" " -f2- > $TMP_LOG
+  $ACCOUNT_BALANCES_CMD >> $TMP_LOG
+  $LIST_ANCHORS_CMD >> $TMP_LOG
+
+  $DEFI_CLI_CMD stop
+  # Download reference log file
+  echo "Downloading reference log file : $REF_LOG_PATH"
+  $FETCH $REF_LOG_PATH
+
+  echo "diff $TMP_LOG $REF_LOG"
+  diff $TMP_LOG $REF_LOG
+}
+
+main "$@"
+
+
+