continous-integration-docker.yml

name: CI Docker

# Ensures that only one workflow task will run at a time. Previous builds, if
# already in process, will get cancelled. Only the latest commit will be allowed
# to run, cancelling any workflows in between
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

on:
  workflow_dispatch:
    inputs:
      network:
        default: 'Mainnet'
        description: 'Network to deploy: Mainnet or Testnet'
        required: true
      checkpoint_sync:
        default: 'true'
        description: 'Configures `zebrad` to use as many checkpoints as possible'
        required: true
      regenerate-disks:
        type: boolean
        default: false
        description: 'Just run a Zebra checkpoint sync and update checkpoint disks'
        required: true
      run-full-sync:
        type: boolean
        default: false
        description: 'Just run a Zebra full sync and update tip disks'
        required: true
      run-lwd-sync:
        type: boolean
        default: false
        description: 'Just run a lightwalletd full sync and update tip disks'
        required: true

  pull_request:
    paths:
      # code and tests
      - '**/*.rs'
      # hard-coded checkpoints and proptest regressions
      - '**/*.txt'
      # test data snapshots
      - '**/*.snap'
      # dependencies
      - '**/Cargo.toml'
      - '**/Cargo.lock'
      # configuration files
      - '.cargo/config.toml'
      - '**/clippy.toml'
      # workflow definitions
      - 'docker/**'
      - '.github/workflows/continous-integration-docker.yml'
      - '.github/workflows/deploy-gcp-tests.yml'
      - '.github/workflows/build-docker-image.yml'

  push:
    branches:
      - main
    paths:
      # code and tests
      - '**/*.rs'
      # hard-coded checkpoints and proptest regressions
      - '**/*.txt'
      # test data snapshots
      - '**/*.snap'
      # dependencies
      - '**/Cargo.toml'
      - '**/Cargo.lock'
      # configuration files
      - '.cargo/config.toml'
      - '**/clippy.toml'
      # workflow definitions
      - 'docker/**'
      - '.github/workflows/continous-integration-docker.yml'
      - '.github/workflows/deploy-gcp-tests.yml'
      - '.github/workflows/build-docker-image.yml'

env:
  # TODO: use the output from ./.github/workflows/build-docker-image.yml
  IMAGE_NAME: zebrad-test
  GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra
  # TODO: use environmental secrets for dynamic values
  NETWORK: Mainnet

jobs:
  get-available-disks:
    runs-on: ubuntu-latest
    name: Find available cached state disks
    outputs:
      lwd_tip_disk: ${{ steps.get-available-disks.outputs.lwd_tip_disk }}
      zebra_tip_disk: ${{ steps.get-available-disks.outputs.zebra_tip_disk }}
      zebra_checkpoint_disk: ${{ steps.get-available-disks.outputs.zebra_checkpoint_disk }}
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - uses: actions/checkout@v3.3.0
        with:
          persist-credentials: false
          fetch-depth: 0

      # Setup gcloud CLI
      - name: Authenticate to Google Cloud
        id: auth
        uses: google-github-actions/auth@v1.0.0
        with:
          retries: '3'
          workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc'
          service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com'

      - name: Set up Cloud SDK
        uses: google-github-actions/setup-gcloud@v1.1.0

      # Disk images in GCP are required to be in lowercase, but the blockchain network
      # uses sentence case, so we need to downcase ${{ env.NETWORK or github.event.inputs.network }}
      #
      # Passes a lowercase Network name to subsequent steps using $NETWORK env variable
      - name: Downcase network name for disks
        run: |
          NETWORK_CAPS=${{ env.NETWORK || github.event.inputs.network }}
          echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV

      # Find a cached state disk for subsequent jobs needing a cached state without
      # restricting the result from any branch.
      #
      # This search is executed considering the actual version from constants.rs
      #
      # Generate one of the following outputs with a boolean to pass to subsequent jobs:
      # - lwd_tip_disk
      # - zebra_tip_disk
      # - zebra_checkpoint_disk
      - name: Find cached state disks
        id: get-available-disks
        run: |
          LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" "$GITHUB_WORKSPACE/zebra-state/src/constants.rs" | grep -oE "[0-9]+" | tail -n1)
          echo "STATE_VERSION: $LOCAL_STATE_VERSION"

          LWD_TIP_DISK=$(gcloud compute images list --filter="status=READY AND name~lwd-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-tip" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
          if [[ -z "$LWD_TIP_DISK" ]]; then
              echo "No TIP disk found for LWD"
              echo "lwd_tip_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT"
          else
              echo "Disk: $LWD_TIP_DISK"
              echo "lwd_tip_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT"
          fi

          ZEBRA_TIP_DISK=$(gcloud compute images list --filter="status=READY AND name~zebrad-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-tip" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
          if [[ -z "$ZEBRA_TIP_DISK" ]]; then
              echo "No TIP disk found for ZEBRA"
              echo "zebra_tip_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT"
          else
              echo "Disk: $ZEBRA_TIP_DISK"
              echo "zebra_tip_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT"
          fi

          ZEBRA_CHECKPOINT_DISK=$(gcloud compute images list --filter="status=READY AND name~zebrad-cache-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-checkpoint" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1)
          if [[ -z "$ZEBRA_CHECKPOINT_DISK" ]]; then
              echo "No CHECKPOINT found for ZEBRA"
              echo "zebra_checkpoint_disk=${{ toJSON(false) }}" >> "$GITHUB_OUTPUT"
          else
              echo "Disk: $ZEBRA_CHECKPOINT_DISK"
              echo "zebra_checkpoint_disk=${{ toJSON(true) }}" >> "$GITHUB_OUTPUT"
          fi

  build:
    name: Build CI Docker
    uses: ./.github/workflows/build-docker-image.yml
    with:
      dockerfile_path: ./docker/Dockerfile
      dockerfile_target: tests
      image_name: zebrad-test
      # TODO: validate how to use variable/conditional values for Testnet
      network: Mainnet
      checkpoint_sync: true
      rust_backtrace: full
      rust_lib_backtrace: full
      colorbt_show_hidden: '1'
      zebra_skip_ipv6_tests: '1'
      rust_log: info

  # zebrad tests without cached state

  # Run all the zebra tests, including tests that are ignored by default.
  # Skips tests that need a cached state disk or a lightwalletd binary.
  #
  # - We run all the tests behind the `getblocktemplate-rpcs` feature as a separated step.
  # - We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.
  #
  # TODO: turn this test and the getblocktemplate test into a matrix, so the jobs use exactly the same diagnostics settings
  test-all:
    name: Test all
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    steps:
      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4
        with:
          short-length: 7

      # Run unit, basic acceptance tests, and ignored tests, only showing command output if the test fails.
      #
      # If some tests hang, add "-- --nocapture" for just that test, or for all the tests.
      - name: Run zebrad tests
        run: |
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
          docker run --name zebrad-tests --tty ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests" --workspace -- --include-ignored

  # zebrad tests without cached state with `getblocktemplate-rpcs` feature
  #
  # Same as above but we run all the tests behind the `getblocktemplate-rpcs` feature.
  test-all-getblocktemplate-rpcs:
    name: Test all with getblocktemplate-rpcs feature
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    steps:
      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4
        with:
          short-length: 7

      - name: Run zebrad tests
        run: |
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
          docker run --name zebrad-tests --tty ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features "lightwalletd-grpc-tests getblocktemplate-rpcs" --workspace -- --include-ignored

  # Run state tests with fake activation heights.
  #
  # This test changes zebra-chain's activation heights,
  # which can recompile all the Zebra crates,
  # so we want its build products to be cached separately.
  #
  # Also, we don't want to accidentally use the fake heights in other tests.
  #
  # (The gRPC feature is a zebrad feature, so it isn't needed here.)
  test-fake-activation-heights:
    name: Test with fake activation heights
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    steps:
      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4
        with:
          short-length: 7

      - name: Run tests with fake activation heights
        run: |
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
          docker run -e TEST_FAKE_ACTIVATION_HEIGHTS --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights
        env:
          TEST_FAKE_ACTIVATION_HEIGHTS: '1'

  # Test that Zebra syncs and checkpoints a few thousand blocks from an empty state.
  #
  # (We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.)
  test-empty-sync:
    name: Test checkpoint sync from empty state
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    steps:
      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4
        with:
          short-length: 7

      - name: Run zebrad large sync tests
        run: |
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
          docker run --name zebrad-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_

  # Test launching lightwalletd with an empty lightwalletd and Zebra state.
  #
  # (We activate the gRPC feature to avoid recompiling `zebrad`, but we don't actually run any gRPC tests.)
  test-lightwalletd-integration:
    name: Test integration with lightwalletd
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    steps:
      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4
        with:
          short-length: 7

      - name: Run tests with empty lightwalletd launch
        run: |
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
          docker run -e ZEBRA_TEST_LIGHTWALLETD --name lightwalletd-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_integration
        env:
          ZEBRA_TEST_LIGHTWALLETD: '1'

  # Test that Zebra works using the default config with the latest Zebra version
  test-configuration-file:
    name: Test Zebra default Docker config file
    timeout-minutes: 5
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    steps:
      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4
        with:
          short-length: 7

      - name: Run tests using the default config
        run: |
          set -ex
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
          docker run --detach --name default-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start
          EXIT_STATUS=$(docker logs --tail all --follow default-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'estimated progress to chain tip.*BeforeOverwinter'; echo $?; )
          docker stop default-conf-tests
          docker logs default-conf-tests
          exit "$EXIT_STATUS"

  # Test that Zebra works using the $ZEBRA_CONF_PATH config
  test-zebra-conf-path:
    name: Test Zebra custom Docker config file
    timeout-minutes: 5
    runs-on: ubuntu-latest
    needs: build
    if: ${{ github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    steps:
      - name: Inject slug/short variables
        uses: rlespinasse/github-slug-action@v4
        with:
          short-length: 7

      - name: Run tests using the $ZEBRA_CONF_PATH
        run: |
          set -ex
          docker pull ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}
          docker run --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} -c $ZEBRA_CONF_PATH start
          EXIT_STATUS=$(docker logs --tail all --follow variable-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml'; echo $?; )
          docker stop variable-conf-tests
          docker logs variable-conf-tests
          exit "$EXIT_STATUS"
        env:
          ZEBRA_CONF_PATH: 'zebrad/tests/common/configs/v1.0.0-rc.2.toml'

  # zebrad cached checkpoint state tests

  # Regenerate mandatory checkpoint Zebra cached state disks.
  #
  # Runs:
  # - on every PR update, but only if there's no available disk matching the actual state version from constants.rs
  # - on request, using workflow_dispatch with regenerate-disks
  #
  # Note: the output from get-available-disks should match with the caller workflow inputs
  regenerate-stateful-disks:
    name: Zebra checkpoint
    needs: [ build, get-available-disks ]
    uses: ./.github/workflows/deploy-gcp-tests.yml
    if: ${{ !fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || github.event.inputs.regenerate-disks == 'true' }}
    with:
      app_name: zebrad
      test_id: sync-to-checkpoint
      test_description: Test sync up to mandatory checkpoint
      test_variables: '-e TEST_DISK_REBUILD=1 -e ZEBRA_FORCE_USE_COLOR=1'
      needs_zebra_state: false
      saves_to_disk: true
      disk_suffix: checkpoint
      height_grep_text: 'flushing database to disk .*height.*=.*Height.*\('
    secrets: inherit
    # We want to prevent multiple checkpoint syncs running at the same time,
    # but we don't want to cancel running syncs on `main` if a new PR gets merged,
    # because we might never get a finished sync.
    #
    # See the concurrency comment on the zebrad test-full-sync job for details.
    concurrency:
      group: ${{ github.workflow }}−${{ github.ref }}-regenerate-stateful-disks
      cancel-in-progress: false

  # Test that Zebra syncs and fully validates a few thousand blocks from a cached mandatory checkpoint disk
  #
  # If the state version has changed, waits for the new cached state to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  test-stateful-sync:
    name: Zebra checkpoint update
    needs: regenerate-stateful-disks
    uses: ./.github/workflows/deploy-gcp-tests.yml
    if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: zebrad
      test_id: sync-past-checkpoint
      test_description: Test full validation sync from a cached state
      test_variables: '-e TEST_CHECKPOINT_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1'
      needs_zebra_state: true
      saves_to_disk: false
      disk_suffix: checkpoint
    secrets: inherit

  # zebrad cached tip state tests

  # Test that Zebra can run a full mainnet sync,
  # and regenerate chain tip Zebra cached state disks.
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update, but only if there's no available disk matching the actual state version from constants.rs
  # - on request, using workflow_dispatch with run-full-sync
  #
  # Note: the output from get-available-disks should match with the caller workflow inputs
  test-full-sync:
    name: Zebra tip
    needs: [ build, get-available-disks ]
    uses: ./.github/workflows/deploy-gcp-tests.yml
    # to also run on Mergify head branches,
    # add `|| (github.event_name == 'push' && startsWith(github.head_ref, 'mergify/merge-queue/'))`:
    # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1
    if: ${{ (github.event_name == 'push' && github.ref_name == 'main') || !fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || github.event.inputs.run-full-sync == 'true' }}
    with:
      app_name: zebrad
      test_id: full-sync-to-tip
      test_description: Test a full sync up to the tip
      # The value of FULL_SYNC_MAINNET_TIMEOUT_MINUTES is currently ignored.
      test_variables: '-e TEST_FULL_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e FULL_SYNC_MAINNET_TIMEOUT_MINUTES=0'
      # This test runs for longer than 6 hours, so it needs multiple jobs
      is_long_test: true
      needs_zebra_state: false
      saves_to_disk: true
      disk_suffix: tip
      height_grep_text: 'current_height.*=.*Height.*\('
    secrets: inherit
    # We want to prevent multiple full zebrad syncs running at the same time,
    # but we don't want to cancel running syncs on `main` if a new PR gets merged,
    # because we might never get a finished sync.
    #
    # Instead, we let the first sync complete, then queue the latest pending sync, cancelling any syncs in between.
    # (As the general workflow concurrency group just gets matched in Pull Requests,
    # it has no impact on this job.)
    #
    # TODO:
    # - allow multiple manual syncs on a branch, and isolate manual syncs from automatic syncs, by adding '-${{ github.run_id }}' when github.event.inputs.run-full-sync is true
    # - stop multiple automatic full syncs across different PRs by removing '−${{ github.ref }}' when needs.get-available-disks.outputs.zebra_tip_disk is true
    concurrency:
      group: ${{ github.workflow }}−${{ github.ref }}-test-full-sync
      cancel-in-progress: false

  # Test that Zebra can sync to the chain tip, using a cached Zebra tip state,
  # without launching `lightwalletd`.
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached state to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  test-update-sync:
    name: Zebra tip update
    needs: test-full-sync
    uses: ./.github/workflows/deploy-gcp-tests.yml
    if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: zebrad
      test_id: update-to-tip
      test_description: Test syncing to tip with a Zebra tip state
      test_variables: '-e TEST_UPDATE_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
      needs_zebra_state: true
      # update the disk on every PR, to increase CI speed
      saves_to_disk: true
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
      height_grep_text: 'current_height.*=.*Height.*\('
    secrets: inherit

  # lightwalletd cached tip state tests

  # Test full sync of lightwalletd with a Zebra tip state
  #
  # Runs:
  # - after every PR is merged to `main`
  #
  # If the state version has changed, waits for the new cached state to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  lightwalletd-full-sync:
    name: lightwalletd tip
    needs: [ test-full-sync, get-available-disks ]
    uses: ./.github/workflows/deploy-gcp-tests.yml
    # to also run on Mergify head branches,
    # add `|| (github.event_name == 'push' && startsWith(github.head_ref, 'mergify/merge-queue/'))`:
    # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1
    if: ${{ !cancelled() && !failure() && ((github.event_name == 'push' && github.ref_name == 'main') || !fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || github.event.inputs.run-lwd-sync == 'true' ) }}
    with:
      app_name: lightwalletd
      test_id: lwd-full-sync
      test_description: Test lightwalletd full sync
      test_variables: '-e TEST_LWD_FULL_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache'
      # This test runs for (just) longer than 6 hours, so it needs multiple jobs
      is_long_test: true
      needs_zebra_state: true
      needs_lwd_state: false
      saves_to_disk: true
      disk_prefix: lwd-cache
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
      lwd_state_dir: 'lwd-cache'
      height_grep_text: '(current_height.*=.*Height.*\()|(Adding block to cache )'
    secrets: inherit
    # We want to prevent multiple lightwalletd full syncs running at the same time,
    # but we don't want to cancel running syncs on `main` if a new PR gets merged,
    # because we might never get a finished sync.
    #
    # See the concurrency comment on the zebrad test-full-sync job for details.
    concurrency:
      group: ${{ github.workflow }}−${{ github.ref }}-lightwalletd-full-sync
      cancel-in-progress: false

  # Test update sync of lightwalletd with a lightwalletd and Zebra tip state
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached states to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  lightwalletd-update-sync:
    name: lightwalletd tip update
    needs: lightwalletd-full-sync
    uses: ./.github/workflows/deploy-gcp-tests.yml
    if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: lightwalletd
      test_id: lwd-update-sync
      test_description: Test lightwalletd update sync with both states
      test_variables: '-e TEST_LWD_UPDATE_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache'
      needs_zebra_state: true
      needs_lwd_state: true
      # since we do a full sync in every PR, the new cached state will only be a few minutes newer than the original one
      saves_to_disk: false
      disk_prefix: lwd-cache
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
      lwd_state_dir: 'lwd-cache'
      height_grep_text: '(current_height.*=.*Height.*\()|(Adding block to cache )'
    secrets: inherit

  # Test that Zebra can answer a synthetic RPC call, using a cached Zebra tip state
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached state to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  lightwalletd-rpc-test:
    name: Zebra tip JSON-RPC
    needs: test-full-sync
    uses: ./.github/workflows/deploy-gcp-tests.yml
    if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: lightwalletd
      test_id: fully-synced-rpc
      test_description: Test lightwalletd RPC with a Zebra tip state
      test_variables: '-e TEST_LWD_RPC_CALL=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
      needs_zebra_state: true
      saves_to_disk: false
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
    secrets: inherit

  # Test that Zebra can handle a lightwalletd send transaction RPC call, using a cached Zebra tip state
  #
  # Runs:
  # - after every PR is merged to `main`
  #
  # If the state version has changed, waits for the new cached states to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  # TODO: move this job under lightwalletd-full-sync to have a sequential logic
  lightwalletd-transactions-test:
    name: lightwalletd tip send
    needs: lightwalletd-full-sync
    uses: ./.github/workflows/deploy-gcp-tests.yml
    if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: lightwalletd
      test_id: lwd-send-transactions
      test_description: Test sending transactions via lightwalletd
      test_variables: '-e TEST_LWD_TRANSACTIONS=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache'
      needs_zebra_state: true
      needs_lwd_state: true
      saves_to_disk: false
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
      lwd_state_dir: 'lwd-cache'
    secrets: inherit
    # We want to prevent multiple lightwalletd send transaction tests running at the same time,
    # but we don't want to cancel running tests on `main` if a new PR gets merged,
    # because we might never get a finished test.
    #
    # See the concurrency comment on the zebrad test-full-sync job for details.
    concurrency:
      group: ${{ github.workflow }}−${{ github.ref }}-lightwalletd-transactions-test
      cancel-in-progress: false

  # Test that Zebra can handle gRPC wallet calls, using a cached Zebra tip state
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached states to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  lightwalletd-grpc-test:
    name: lightwalletd GRPC tests
    needs: lightwalletd-full-sync
    uses: ./.github/workflows/deploy-gcp-tests.yml
    if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: lightwalletd
      test_id: lwd-grpc-wallet
      test_description: Test gRPC calls via lightwalletd
      test_variables: '-e TEST_LWD_GRPC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache'
      needs_zebra_state: true
      needs_lwd_state: true
      saves_to_disk: false
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
      lwd_state_dir: 'lwd-cache'
    secrets: inherit

  # Test that Zebra can handle a getblocktemplate RPC call, using a cached Zebra tip state
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached states to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  get-block-template-test:
    name: get block template
    needs: test-full-sync
    uses: ./.github/workflows/deploy-gcp-tests.yml
    if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: zebrad
      test_id: get-block-template
      test_description: Test getblocktemplate RPC method via Zebra's rpc server
      test_variables: '-e TEST_GET_BLOCK_TEMPLATE=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
      needs_zebra_state: true
      needs_lwd_state: false
      saves_to_disk: false
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
    secrets: inherit

  # Test that Zebra can handle a submit block RPC call, using a cached Zebra tip state
  #
  # Runs:
  # - after every PR is merged to `main`
  # - on every PR update
  #
  # If the state version has changed, waits for the new cached states to be created.
  # Otherwise, if the state rebuild was skipped, runs immediately after the build job.
  submit-block-test:
    name: submit block
    needs: test-full-sync
    uses: ./.github/workflows/deploy-gcp-tests.yml
    if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }}
    with:
      app_name: zebrad
      test_id: submit-block
      test_description: Test submitting blocks via Zebra's rpc server
      test_variables: '-e TEST_SUBMIT_BLOCK=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache'
      needs_zebra_state: true
      needs_lwd_state: false
      saves_to_disk: false
      disk_suffix: tip
      root_state_path: '/var/cache'
      zebra_state_dir: 'zebrad-cache'
    secrets: inherit