diff --git a/.github/workflows/fullsync-tests.yml b/.github/workflows/fullsync-tests.yml index 3639254801c..c40706d2d7a 100644 --- a/.github/workflows/fullsync-tests.yml +++ b/.github/workflows/fullsync-tests.yml @@ -98,6 +98,17 @@ jobs: stop: 1850000 - start: 1850000 stop: 1900000 + - start: 1900000 + stop: 1950000 + - start: 1950000 + stop: 2000000 + - start: 2000000 + stop: 2050000 + - start: 2050000 + stop: 2100000 + - start: 2100000 + stop: 2150000 + runs-on: [self-hosted, linux, x64] needs: build-binaries if: contains(github.event.pull_request.labels.*.name, 'ci/sync') @@ -106,6 +117,7 @@ jobs: image : gcr.io/br-blockchains-dev/datadir-${{matrix.datadir.start}} options: --privileged env: + START_BLOCK: ${{matrix.datadir.start}} STOP_BLOCK: ${{matrix.datadir.stop}} DEFID_BIN: ./defid DEFI_CLI_BIN: ./defi-cli diff --git a/.github/workflows/fullsync-tip-tests.yml b/.github/workflows/fullsync-tip-tests.yml new file mode 100644 index 00000000000..cefd748799c --- /dev/null +++ b/.github/workflows/fullsync-tip-tests.yml @@ -0,0 +1,126 @@ +name: Full Sync Tip Tests +on: + pull_request: + branches: + - master + types: [labeled, opened, reopened, synchronize] + +jobs: + build-binaries: + if: contains(github.event.pull_request.labels.*.name, 'ci/sync-tip') + runs-on: [self-hosted, linux, x64,server-2] + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Build Node + run: ./make.sh build + - name: Upload Binaries + uses: actions/upload-artifact@v3 + with: + name: defibins + path: | + src/defid + src/defi-cli + - name: Archive Binaries + uses: actions/upload-artifact@v3 + with: + name: defibins + path: | + src/defid + src/defi-cli + - name: Archive Shell Commands + uses: actions/upload-artifact@v3 + with: + name: sync + path: | + ci/parallel_sync/sync_then_diff.sh + - name: Archive Sync to Tip Shell Command + uses: actions/upload-artifact@v3 + with: + name: sync-to-tip + path: | + ci/parallel_sync/sync_to_tip.sh + + sync: + strategy: + matrix: + datadir: + - start: 1900000 + stop: 1950000 + - start: 1950000 + stop: 2000000 + - start: 2000000 + stop: 2050000 + - start: 2050000 + stop: 2100000 + - start: 2100000 + stop: 2150000 + - start: 2150000 + stop: 2232254 + + runs-on: [self-hosted, linux, x64] + needs: build-binaries + if: contains(github.event.pull_request.labels.*.name, 'ci/sync-tip') + continue-on-error: true + container: + image : gcr.io/br-blockchains-dev/datadir-${{matrix.datadir.start}} + options: --privileged + env: + START_BLOCK: ${{matrix.datadir.start}} + STOP_BLOCK: ${{matrix.datadir.stop}} + DEFID_BIN: ./defid + DEFI_CLI_BIN: ./defi-cli + timeout-minutes: 4320 + steps: + - name: Download Binaries + uses: actions/download-artifact@v3 + with: + name: defibins + - name: Download Shell Commands + uses: actions/download-artifact@v3 + with: + name: sync + - name: Set Permissions + run: | + chmod 777 defid + chmod 777 defi-cli + chmod 777 sync_then_diff.sh + - name: Sync and Diff + run: ./sync_then_diff.sh + - name: Show Debug Log + run: cat $DATADIR/debug.log + if: ${{ failure() || success() }} + sync-to-tip: + runs-on: [self-hosted, linux, x64] + needs: build-binaries + if: contains(github.event.pull_request.labels.*.name, 'ci/sync-tip') + continue-on-error: true + container: + image : gcr.io/br-blockchains-dev/datadir-${{matrix.datadir.start}} + options: --privileged + env: + START_BLOCK: 2150000 + STOP_BLOCK: 2232254 + DEFID_BIN: ./defid + DEFI_CLI_BIN: ./defi-cli + timeout-minutes: 4320 + steps: + - name: Download Binaries + uses: actions/download-artifact@v3 + with: + name: defibins + - name: Download Shell Commands + uses: actions/download-artifact@v3 + with: + name: sync-to-tip + - name: Set Permissions + run: | + chmod 777 defid + chmod 777 defi-cli + chmod 777 sync_to_tip.sh + - name: Sync to Tip + run: ./sync_to_tip.sh + - name: Show Debug Log + run: cat $DATADIR/debug.log + if: ${{ failure() || success() }} diff --git a/ci/parallel_sync/sync_then_diff.sh b/ci/parallel_sync/sync_then_diff.sh index e3c866f9b00..199212eb614 100755 --- a/ci/parallel_sync/sync_then_diff.sh +++ b/ci/parallel_sync/sync_then_diff.sh @@ -13,6 +13,7 @@ setup_vars() { # Files and directories DATADIR=${DATADIR:-".defi"} DEBUG_FILE="$DATADIR/debug.log" + CONF_FILE="$DATADIR/defi.conf" TMP_LOG=debug-tmp-$STOP_BLOCK.log BASE_PATH=https://storage.googleapis.com BUCKET=team-drop @@ -30,11 +31,26 @@ setup_vars() { BLOCK=0 ATTEMPTS=0 + START_BLOCK=${START_BLOCK:-0} + STOP_BLOCK=${STOP_BLOCK:-0} MAX_ATTEMPTS=10 MAX_NODE_RESTARTS=5 NODE_RESTARTS=0 } +echo "======== Sync Test Info ========== +- Block range: ${START_BLOCK} - ${STOP_BLOCK} +- Base snapshot: https://gcr.io/br-blockchains-dev/datadir-${START_BLOCK} +- Reference logs: + - debug.log: $REF_LOG_PATH +- Commands used: + - $ACCOUNT_BALANCES_CMD + - $LIST_ANCHORS_CMD +- defid cmd: ${DEFI_CLI_CMD} +- defi.conf: + $(cat "$CONF_FILE") +---------------------------------- +" # Start defid start_node () { echo "Syncing to block height: $STOP_BLOCK" @@ -74,17 +90,17 @@ main() { done # Create temporary log file - $GREP "AccountChange:" $DEBUG_FILE | cut -d" " -f2- > $TMP_LOG - $ACCOUNT_BALANCES_CMD >> $TMP_LOG - $LIST_ANCHORS_CMD >> $TMP_LOG + $GREP "AccountChange:" "$DEBUG_FILE" | cut -d" " -f2- > "$TMP_LOG" + $ACCOUNT_BALANCES_CMD >> "$TMP_LOG" + $LIST_ANCHORS_CMD >> "$TMP_LOG" $DEFI_CLI_CMD stop # Download reference log file echo "Downloading reference log file : $REF_LOG_PATH" - $FETCH $REF_LOG_PATH + $FETCH "$REF_LOG_PATH" echo "diff $TMP_LOG $REF_LOG" - diff $TMP_LOG $REF_LOG + diff "$TMP_LOG" "$REF_LOG" } main "$@" diff --git a/ci/parallel_sync/sync_to_tip.sh b/ci/parallel_sync/sync_to_tip.sh new file mode 100755 index 00000000000..76397eadcca --- /dev/null +++ b/ci/parallel_sync/sync_to_tip.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +export LC_ALL=C +set -Eeuo pipefail + +setup_vars() { + # Binaries + DEFID_BIN=${DEFID_BIN:-"./defid"} + DEFI_CLI_BIN=${DEFI_CLI_BIN:-"./defi-cli"} + + # Files and directories + DATADIR=${DATADIR:-".defi"} + DEBUG_FILE="$DATADIR/debug.log" + CONF_FILE="$DATADIR/defi.conf" + TMP_LOG=debug-tmp-$STOP_BLOCK.log + BASE_PATH=https://storage.googleapis.com + BUCKET=team-drop + REF_LOG_DIR=master-logs-full + REF_LOG=debug-$STOP_BLOCK.log + REF_LOG_PATH=$BASE_PATH/$BUCKET/$REF_LOG_DIR/$REF_LOG + + # Commands + DEFID_CMD="$DEFID_BIN -datadir=$DATADIR -daemon -debug=accountchange" + DEFI_CLI_CMD="$DEFI_CLI_BIN -datadir=$DATADIR" + ACCOUNT_BALANCES_CMD="$DEFI_CLI_CMD logaccountbalances" + LIST_ANCHORS_CMD="$DEFI_CLI_CMD spv_listanchors" + GREP="grep" + + BLOCK=0 + ATTEMPTS=0 + START_BLOCK=${START_BLOCK:-0} + STOP_BLOCK=${STOP_BLOCK:-0} + MAX_ATTEMPTS=10 + MAX_NODE_RESTARTS=5 + NODE_RESTARTS=0 +} + +print_info() { + echo "======== Sync Test Info ========== + - Block range: ${START_BLOCK} - ${STOP_BLOCK} + - Base snapshot: https://gcr.io/br-blockchains-dev/datadir-${START_BLOCK} + - Reference logs: + - debug.log: $REF_LOG_PATH + - Commands used: + - $ACCOUNT_BALANCES_CMD + - $LIST_ANCHORS_CMD + - defid cmd: ${DEFI_CLI_CMD} + - defi.conf: + $(cat "$CONF_FILE") + ---------------------------------- + " +} + +print_info + +# Start defid +start_node () { + echo "Syncing to block height: $STOP_BLOCK" + $DEFID_CMD + sleep 30 + update_tip_node +} + +update_tip () { + CUR_TIP=$($DEFI_CLI_CMD "$DEFI_CLI_CMD" getblockchaininfo | jq .headers || echo "$STOP_BLOCK") + if [ "$CUR_TIP" -gt "$STOP_BLOCK" ]; then + echo "New Tip: $STOP_BLOCK" + STOP_BLOCK=${CUR_TIP:-$STOP_BLOCK} + fi +} + +main() { + setup_vars + start_node + + $DEFI_CLI_CMD clearbanned || true + + # Sync to target block height + while [ "$BLOCK" -lt "$STOP_BLOCK" ]; do + if [ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]; then + if [ "$NODE_RESTARTS" -lt "$MAX_NODE_RESTARTS" ]; then + echo "Node Stuck After $ATTEMPTS attempts, restarting node" + $DEFI_CLI_CMD stop + sleep 20 + start_node + NODE_RESTARTS=$((NODE_RESTARTS + 1)) + ATTEMPTS=0 + else + exit 1 + fi + fi + CUR_BLOCK=$($DEFI_CLI_CMD getblockcount || echo $BLOCK) + if [ "$CUR_BLOCK" -eq "$BLOCK" ]; then + ATTEMPTS=$((ATTEMPTS + 1)) + else + ATTEMPTS=0 + fi + BLOCK=${CUR_BLOCK:-$BLOCK} + echo "Current block: $BLOCK / $STOP_BLOCK" + sleep 20 + update_tip + done + + # Create temporary log file + $GREP "AccountChange:" "$DEBUG_FILE" | cut -d" " -f2- > "$TMP_LOG" + $ACCOUNT_BALANCES_CMD >> "$TMP_LOG" + $LIST_ANCHORS_CMD >> "$TMP_LOG" + $DEFI_CLI_CMD stop + cat "$TMP_LOG" +} + +main "$@" + + +