Add raw parser for db integrity checks #5011
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests - Sync | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| block_ranges: | |
| description: 'Block ranges: Format: "<start-block-1> <start-block-2>" | |
| Eg: "350000 50000 750000". | |
| Default: All available ranges when empty' | |
| required: false | |
| name: | |
| description: 'Custom name for the workflow run' | |
| required: false | |
| pull_request: | |
| branches: | |
| - master | |
| types: [labeled, opened, reopened, synchronize] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref || github.run_id }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} | |
| run-name: ${{ inputs.name || github.event.pull_request.title || github.ref_name }} | |
| env: | |
| TARGET: x86_64-pc-linux-gnu | |
| MAKE_DEBUG: 0 | |
| jobs: | |
| build: | |
| if: contains(github.event.pull_request.labels.*.name, 'ci/sync') || github.event_name == 'workflow_dispatch' | |
| runs-on: [self-hosted, linux, x64, builder] | |
| env: | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: git config --global --add safe.directory '*' | |
| - name: Populate environment | |
| run: ./make.sh ci-export-vars | |
| - name: Setup dependencies | |
| run: sudo ./make.sh ci-setup-deps | |
| - name: Setup user dependencies | |
| run: ./make.sh ci-setup-user-deps | |
| - name: Restore cpp build cache | |
| id: cpp-cache-restore | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: | | |
| ./build/depends | |
| ./build/src | |
| ~/.ccache | |
| key: cpp-${{ env.TARGET }}-${{ env.BUILD_TYPE }} | |
| - name: Rust build cache | |
| uses: Swatinem/rust-cache@v2 | |
| id: rust-cache-restore | |
| with: | |
| workspaces: lib -> ../build/lib/target | |
| save-if: ${{ github.ref == 'refs/heads/master' }} | |
| shared-key: ${{ env.TARGET }} | |
| - name: Build binaries | |
| run: ./make.sh build | |
| - name: Upload binaries | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: defi-bins | |
| path: | | |
| build/src/defid | |
| build/src/defi-cli | |
| - name: Upload shell commands | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sync | |
| path: ci/sync/main.sh | |
| generate-matrix: | |
| if: contains(github.event.pull_request.labels.*.name, 'ci/sync') || github.event_name == 'workflow_dispatch' | |
| runs-on: [self-hosted, linux, x64] | |
| # Add "id-token" with the intended permissions. | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: 'auth' | |
| name: 'Authenticate to Google Cloud' | |
| uses: 'google-github-actions/auth@v0' | |
| with: | |
| workload_identity_provider: 'projects/965426322273/locations/global/workloadIdentityPools/br-blockchains-pool/providers/br-blockchains-provider' | |
| service_account: 'blockchain-dev-service@br-blockchains-dev.iam.gserviceaccount.com' | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v0' | |
| - id: set-matrix | |
| name: 'Set matrix output' | |
| run: | | |
| if [ -n "${{ inputs.block_ranges }}" ]; then | |
| BLOCKS=$(echo ${{ inputs.block_ranges }} | tr ' ' '\n') | |
| else | |
| SNAPSHOTS=$(gsutil ls gs://team-drop/master-datadir) | |
| BLOCKS=$(echo "$SNAPSHOTS" | sed -e 's/.*\-\(.*\)\.tar.*/\1/' | grep -v gs | sort -n | head -n -1) | |
| fi | |
| MATRIX_JSON=$(jq -n -c -M --arg blocks "$BLOCKS" '{blocks: ($blocks | split("\n") | .[] |= tonumber | to_entries | map({start: .value , stop: (.value + 50000)}))}') | |
| echo "MATRIX=$MATRIX_JSON" >> $GITHUB_OUTPUT | |
| sync: | |
| runs-on: [self-hosted, linux, x64] | |
| needs: [build, generate-matrix] | |
| strategy: | |
| matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}} | |
| continue-on-error: true | |
| env: | |
| DATADIR : datadir-${{matrix.blocks.start}} | |
| STOP_BLOCK: ${{matrix.blocks.stop}} | |
| START_BLOCK: ${{matrix.blocks.start}} | |
| DEFID_BIN: ./defid | |
| DEFI_CLI_BIN: ./defi-cli | |
| REF_LOG_DIR: master-datadir/log | |
| BASE_REF: master | |
| timeout-minutes: 4320 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download Snapshot | |
| run: aria2c -x16 -s16 https://storage.googleapis.com/team-drop/master-datadir/datadir-${{matrix.blocks.start}}.tar.gz | |
| - name: Create datadir | |
| run: mkdir $DATADIR && tar -C $DATADIR -xvf datadir-${{matrix.blocks.start}}.tar.gz | |
| - name: Download binaries | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: defi-bins | |
| - name: Download Shell Commands | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: sync | |
| - name: Rename Artifact | |
| run: mv main.sh sync.sh | |
| - name: Set Permissions | |
| run: | | |
| chmod 777 defid | |
| chmod 777 defi-cli | |
| chmod 777 sync.sh | |
| - name: Sync | |
| run: ./sync.sh | |
| - name: Diff log | |
| run: diff debug-${{matrix.blocks.stop}}.log debug-tmp-${{matrix.blocks.stop}}.log | |
| - name: Diff rollback log | |
| run: diff debug-pre-rollback.log debug-post-rollback.log | |
| if: ${{ failure() || success() }} | |
| - name: Show debug.log | |
| run: cat $DATADIR/debug.log | |
| if: ${{ failure() || success() }} | |
| - name: Show log file | |
| run: cat debug-tmp-$STOP_BLOCK.log | |
| if: ${{ failure() || success() }} |