[WIP] Add basic full sync info + Add Snapshot Full sync for 2000k + height #1459
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| name: Full Sync Tip Tests | ||
| on: | ||
| pull_request: | ||
| branches: | ||
| - master | ||
| types: [labeled, opened, reopened, synchronize] | ||
|
|
||
| jobs: | ||
| build-binaries: | ||
| if: contains(github.event.pull_request.labels.*.name, 'ci/sync-tip') | ||
| runs-on: [self-hosted, linux, x64,server-2] | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| fetch-depth: 0 | ||
| - name: Build Node | ||
| run: ./make.sh build | ||
| - name: Upload Binaries | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: defibins | ||
| path: | | ||
| src/defid | ||
| src/defi-cli | ||
| - name: Archive Binaries | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: defibins | ||
| path: | | ||
| src/defid | ||
| src/defi-cli | ||
| - name: Archive Shell Commands | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: sync | ||
| path: | | ||
| ci/parallel_sync/sync_then_diff.sh | ||
| - name: Archive Sync to Tip Shell Command | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: sync-to-tip | ||
| path: | | ||
| ci/parallel_sync/sync_to_tip.sh | ||
|
|
||
| sync: | ||
| strategy: | ||
| matrix: | ||
| datadir: | ||
| - start: 1900000 | ||
| stop: 1950000 | ||
| - start: 1950000 | ||
| stop: 2000000 | ||
| - start: 2000000 | ||
| stop: 2050000 | ||
| - start: 2050000 | ||
| stop: 2100000 | ||
| - start: 2100000 | ||
| stop: 2150000 | ||
| - start: 2150000 | ||
| stop: 2232254 | ||
|
|
||
| runs-on: [self-hosted, linux, x64] | ||
| needs: build-binaries | ||
| if: contains(github.event.pull_request.labels.*.name, 'ci/sync-tip') | ||
| continue-on-error: true | ||
| container: | ||
| image : gcr.io/br-blockchains-dev/datadir-${{matrix.datadir.start}} | ||
| options: --privileged | ||
| env: | ||
| START_BLOCK: ${{matrix.datadir.start}} | ||
| STOP_BLOCK: ${{matrix.datadir.stop}} | ||
| DEFID_BIN: ./defid | ||
| DEFI_CLI_BIN: ./defi-cli | ||
| timeout-minutes: 4320 | ||
| steps: | ||
| - name: Download Binaries | ||
| uses: actions/download-artifact@v3 | ||
| with: | ||
| name: defibins | ||
| - name: Download Shell Commands | ||
| uses: actions/download-artifact@v3 | ||
| with: | ||
| name: sync | ||
| - name: Set Permissions | ||
| run: | | ||
| chmod 777 defid | ||
| chmod 777 defi-cli | ||
| chmod 777 sync_then_diff.sh | ||
| - name: Sync and Diff | ||
| run: ./sync_then_diff.sh | ||
| - name: Show Debug Log | ||
| run: cat $DATADIR/debug.log | ||
| if: ${{ failure() || success() }} | ||
| sync-to-tip: | ||
| runs-on: [self-hosted, linux, x64] | ||
| needs: build-binaries | ||
| if: contains(github.event.pull_request.labels.*.name, 'ci/sync-tip') | ||
| continue-on-error: true | ||
| container: | ||
| image : gcr.io/br-blockchains-dev/datadir-${{matrix.datadir.start}} | ||
| options: --privileged | ||
| env: | ||
| START_BLOCK: 2150000 | ||
| STOP_BLOCK: 2232254 | ||
| DEFID_BIN: ./defid | ||
| DEFI_CLI_BIN: ./defi-cli | ||
| timeout-minutes: 4320 | ||
| steps: | ||
| - name: Download Binaries | ||
| uses: actions/download-artifact@v3 | ||
| with: | ||
| name: defibins | ||
| - name: Download Shell Commands | ||
| uses: actions/download-artifact@v3 | ||
| with: | ||
| name: sync-to-tip | ||
| - name: Set Permissions | ||
| run: | | ||
| chmod 777 defid | ||
| chmod 777 defi-cli | ||
| chmod 777 sync_to_tip.sh | ||
| - name: Sync to Tip | ||
| run: ./sync_to_tip.sh | ||
| - name: Show Debug Log | ||
| run: cat $DATADIR/debug.log | ||
| if: ${{ failure() || success() }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| #!/bin/bash | ||
|
|
||
| export LC_ALL=C | ||
| set -Eeuo pipefail | ||
|
|
||
| setup_vars() { | ||
| # Binaries | ||
| DEFID_BIN=${DEFID_BIN:-"./defid"} | ||
| DEFI_CLI_BIN=${DEFI_CLI_BIN:-"./defi-cli"} | ||
|
|
||
| # Files and directories | ||
| DATADIR=${DATADIR:-".defi"} | ||
| DEBUG_FILE="$DATADIR/debug.log" | ||
| CONF_FILE="$DATADIR/defi.conf" | ||
| TMP_LOG=debug-tmp-$STOP_BLOCK.log | ||
| BASE_PATH=https://storage.googleapis.com | ||
| BUCKET=team-drop | ||
| REF_LOG_DIR=master-logs-full | ||
| REF_LOG=debug-$STOP_BLOCK.log | ||
| REF_LOG_PATH=$BASE_PATH/$BUCKET/$REF_LOG_DIR/$REF_LOG | ||
|
|
||
| # Commands | ||
| DEFID_CMD="$DEFID_BIN -datadir=$DATADIR -daemon -debug=accountchange" | ||
| DEFI_CLI_CMD="$DEFI_CLI_BIN -datadir=$DATADIR" | ||
| ACCOUNT_BALANCES_CMD="$DEFI_CLI_CMD logaccountbalances" | ||
| LIST_ANCHORS_CMD="$DEFI_CLI_CMD spv_listanchors" | ||
| GREP="grep" | ||
|
|
||
| BLOCK=0 | ||
| ATTEMPTS=0 | ||
| START_BLOCK=${START_BLOCK:-0} | ||
| STOP_BLOCK=${STOP_BLOCK:-0} | ||
| MAX_ATTEMPTS=10 | ||
| MAX_NODE_RESTARTS=5 | ||
| NODE_RESTARTS=0 | ||
| } | ||
|
|
||
| echo "======== Sync Test Info ========== | ||
|
There was a problem hiding this comment. Move this into a function and call it from main, so its cleaner. |
||
| - Block range: ${START_BLOCK} - ${STOP_BLOCK} | ||
| - Base snapshot: https://gcr.io/br-blockchains-dev/datadir-${START_BLOCK} | ||
|
There was a problem hiding this comment. Can we get a direct link for this? I assume this leads to a dir. There was a problem hiding this comment. its the link to snapshot(tar) file |
||
| - Reference logs: | ||
| - debug.log: $REF_LOG_PATH | ||
| - Commands used: | ||
| - $ACCOUNT_BALANCES_CMD | ||
| - $LIST_ANCHORS_CMD | ||
| - defid cmd: ${DEFI_CLI_CMD} | ||
| - defi.conf: | ||
| $(cat "$CONF_FILE") | ||
| ---------------------------------- | ||
| " | ||
| # Start defid | ||
| start_node () { | ||
| echo "Syncing to block height: $STOP_BLOCK" | ||
| $DEFID_CMD | ||
| sleep 30 | ||
| update_tip_node | ||
| } | ||
|
|
||
| update_tip () { | ||
| CUR_TIP=$($DEFI_CLI_CMD "$DEFI_CLI_CMD" getblockchaininfo | jq .headers || echo "$STOP_BLOCK") | ||
| if [ "$CUR_TIP" -gt "$STOP_BLOCK" ]; then | ||
| echo "New Tip: $STOP_BLOCK" | ||
| STOP_BLOCK=${CUR_TIP:-$STOP_BLOCK} | ||
| fi | ||
| } | ||
|
|
||
| main() { | ||
| setup_vars | ||
| start_node | ||
|
|
||
| $DEFI_CLI_CMD clearbanned || true | ||
|
|
||
| # Sync to target block height | ||
| while [ "$BLOCK" -lt "$STOP_BLOCK" ]; do | ||
| if [ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]; then | ||
| if [ "$NODE_RESTARTS" -lt "$MAX_NODE_RESTARTS" ]; then | ||
| echo "Node Stuck After $ATTEMPTS attempts, restarting node" | ||
| $DEFI_CLI_CMD stop | ||
| sleep 20 | ||
| start_node | ||
| NODE_RESTARTS=$((NODE_RESTARTS + 1)) | ||
| ATTEMPTS=0 | ||
| else | ||
| exit 1 | ||
| fi | ||
| fi | ||
| CUR_BLOCK=$($DEFI_CLI_CMD getblockcount || echo $BLOCK) | ||
| if [ "$CUR_BLOCK" -eq "$BLOCK" ]; then | ||
| ATTEMPTS=$((ATTEMPTS + 1)) | ||
| else | ||
| ATTEMPTS=0 | ||
| fi | ||
| BLOCK=${CUR_BLOCK:-$BLOCK} | ||
| echo "Current block: $BLOCK / $STOP_BLOCK" | ||
| sleep 20 | ||
| update_tip | ||
| done | ||
|
|
||
| # Create temporary log file | ||
| $GREP "AccountChange:" "$DEBUG_FILE" | cut -d" " -f2- > "$TMP_LOG" | ||
| $ACCOUNT_BALANCES_CMD >> "$TMP_LOG" | ||
| $LIST_ANCHORS_CMD >> "$TMP_LOG" | ||
| $DEFI_CLI_CMD stop | ||
| cat "$TMP_LOG" | ||
| } | ||
|
|
||
| main "$@" | ||
|
|
||
|
|
||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Similar adaptations to the other code block