From 1f47193077286c8c3fe0de04a1834ae7bcf7ae59 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 2 Oct 2023 23:33:53 +0100 Subject: [PATCH 01/22] ref(docker): combine test and prod entrypoints into one This makes the following changes - Uses `-x` instead of echoing the variables values - Sets default values where required - Create a function to list directories - Create a function to run cargo tests - Use a better approach to handle different options in the `case` manegement for tests and production - Replaces all instances of `runtime-entrypoint.sh` with `entrypoint.sh` --- .dockerignore | 1 - .github/workflows/ci-sync-tests-gcp.yml | 616 ++++++++++++++++++++++++ book/src/user/docker.md | 4 +- docker/Dockerfile | 8 +- docker/entrypoint.sh | 381 ++++++++++----- docker/runtime-entrypoint.sh | 120 ----- 6 files changed, 890 insertions(+), 240 deletions(-) create mode 100644 .github/workflows/ci-sync-tests-gcp.yml delete mode 100755 docker/runtime-entrypoint.sh diff --git a/.dockerignore b/.dockerignore index 12057f20ac8..9d62f3c5c13 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,4 +21,3 @@ !zebra-* !zebrad !docker/entrypoint.sh -!docker/runtime-entrypoint.sh diff --git a/.github/workflows/ci-sync-tests-gcp.yml b/.github/workflows/ci-sync-tests-gcp.yml new file mode 100644 index 00000000000..2b9582a3153 --- /dev/null +++ b/.github/workflows/ci-sync-tests-gcp.yml @@ -0,0 +1,616 @@ +name: CI Docker + +# Ensures that only one workflow task will run at a time. Previous builds, if +# already in process, will get cancelled. Only the latest commit will be allowed +# to run, cancelling any workflows in between +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + schedule: + # Run this job every Friday at mid-day UTC + # This is limited to the Zebra and lightwalletd Full Sync jobs + - cron: '0 12 * * 5' + + workflow_dispatch: + inputs: + network: + default: 'Mainnet' + description: 'Network to deploy: Mainnet or Testnet' + required: true + regenerate-disks: + type: boolean + default: false + description: 'Just run a Zebra checkpoint sync and update checkpoint disks' + required: true + run-full-sync: + type: boolean + default: false + description: 'Just run a Zebra full sync on `network`, and update tip disks' + required: true + run-lwd-sync: + type: boolean + default: false + description: 'Just run a lightwalletd full sync and update tip disks' + required: true + force_save_to_disk: + required: false + type: boolean + default: false + description: 'Force tests to always create a cached state disk, if they already create disks' + no_cache: + description: 'Disable the Docker cache for this build' + required: false + type: boolean + default: false + + pull_request: + paths: + # code and tests + - '**/*.rs' + # hard-coded checkpoints and proptest regressions + - '**/*.txt' + # test data snapshots + - '**/*.snap' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # configuration files + - '.cargo/config.toml' + - '**/clippy.toml' + # workflow definitions + - 'docker/**' + - '.github/workflows/continous-integration-docker.yml' + - '.github/workflows/deploy-gcp-tests.yml' + - '.github/workflows/build-docker-image.yml' + - '.github/workflows/find-cached-disks.yml' + + push: + branches: + - main + paths: + # code and tests + - '**/*.rs' + # hard-coded checkpoints and proptest regressions + - '**/*.txt' + # test data snapshots + - '**/*.snap' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # configuration files + - '.cargo/config.toml' + - '**/clippy.toml' + # workflow definitions + - 'docker/**' + - '.dockerignore' + - '.github/workflows/continous-integration-docker.yml' + - '.github/workflows/deploy-gcp-tests.yml' + - '.github/workflows/find-cached-disks.yml' + - '.github/workflows/build-docker-image.yml' + +jobs: + # to also run a job on Mergify head branches, + # add `|| (github.event_name == 'push' && startsWith(github.head_ref, 'mergify/merge-queue/'))`: + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1 + + # Check if the cached state disks used by the tests are available for the default network. + # + # The default network is mainnet unless a manually triggered workflow or repository variable + # is configured differently. + # + # The outputs for this job have the same names as the workflow outputs in find-cached-disks.yml + get-available-disks: + name: Check if cached state disks exist for ${{ inputs.network || vars.ZCASH_NETWORK }} + uses: ./.github/workflows/find-cached-disks.yml + with: + network: ${{ inputs.network || vars.ZCASH_NETWORK }} + + # Check if the cached state disks used by the tests are available for testnet. + # + # The outputs for this job have the same names as the workflow outputs in find-cached-disks.yml + # Some outputs are ignored, because we don't run those jobs on testnet. + get-available-disks-testnet: + name: Check if cached state disks exist for testnet + uses: ./.github/workflows/find-cached-disks.yml + with: + network: 'Testnet' + + # Build the docker image used by the tests. + # + # The default network in the Zebra config in the image is mainnet, unless a manually triggered + # workflow or repository variable is configured differently. Testnet jobs change that config to + # testnet when running the image. + build: + name: Build CI Docker + uses: ./.github/workflows/build-docker-image.yml + with: + dockerfile_path: ./docker/Dockerfile + dockerfile_target: tests + image_name: ${{ vars.CI_IMAGE_NAME }} + no_cache: ${{ inputs.no_cache || false }} + rust_backtrace: full + rust_lib_backtrace: full + rust_log: info + + # END TODO: make the non-cached-state tests use: + # network: ${{ inputs.network || vars.ZCASH_NETWORK }} + + # zebrad cached checkpoint state tests + + # Regenerate mandatory checkpoint Zebra cached state disks. + # + # Runs: + # - on every PR update, but only if there's no available disk matching the actual state version from constants.rs + # - on request, using workflow_dispatch with regenerate-disks + # + # Note: the output from get-available-disks should match with the caller workflow inputs + regenerate-stateful-disks: + name: Zebra checkpoint + needs: [ build, get-available-disks ] + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || github.event.inputs.regenerate-disks == 'true' }} + with: + app_name: zebrad + test_id: sync-to-checkpoint + test_description: Test sync up to mandatory checkpoint + test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_DISK_REBUILD=1 -e ZEBRA_FORCE_USE_COLOR=1' + needs_zebra_state: false + saves_to_disk: true + force_save_to_disk: ${{ inputs.force_save_to_disk || false }} + disk_suffix: checkpoint + height_grep_text: 'flushing database to disk .*height.*=.*Height.*\(' + secrets: inherit + # We want to prevent multiple checkpoint syncs running at the same time, + # but we don't want to cancel running syncs on `main` if a new PR gets merged, + # because we might never get a finished sync. + # + # See the concurrency comment on the zebrad test-full-sync job for details. + concurrency: + group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.regenerate-disks == 'true') }}-regenerate-stateful-disks + cancel-in-progress: false + + # Test that Zebra syncs and fully validates a few thousand blocks from a cached mandatory checkpoint disk + # + # If the state version has changed, waits for the new cached state to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + test-stateful-sync: + name: Zebra checkpoint update + needs: [ regenerate-stateful-disks, get-available-disks ] + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || needs.regenerate-stateful-disks.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + with: + app_name: zebrad + test_id: sync-past-checkpoint + test_description: Test full validation sync from a cached state + test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_CHECKPOINT_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1' + needs_zebra_state: true + saves_to_disk: false + disk_suffix: checkpoint + secrets: inherit + + # zebrad cached tip state tests + + # Test that Zebra can run a full sync on mainnet, + # and regenerate chain tip Zebra cached state disks. + # + # This test always runs on mainnet. + # + # Runs: + # - on schedule, as defined at the top of the workflow + # - on every PR update, but only if the state version in constants.rs has no cached disk + # - in manual workflow runs, when run-full-sync is 'true' and network is 'Mainnet' + # + # Note: the output from get-available-disks should match with the caller workflow inputs + test-full-sync: + name: Zebra tip + needs: [ build, get-available-disks ] + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ github.event_name == 'schedule' || !fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet') }} + with: + app_name: zebrad + test_id: full-sync-to-tip + test_description: Test a full sync up to the tip + # The value of FULL_SYNC_MAINNET_TIMEOUT_MINUTES is currently ignored. + # TODO: update the test to use {{ input.network }} instead? + test_variables: '-e NETWORK=Mainnet -e FULL_SYNC_MAINNET_TIMEOUT_MINUTES=0 -e ZEBRA_FORCE_USE_COLOR=1' + # This test runs for longer than 6 hours, so it needs multiple jobs + is_long_test: true + needs_zebra_state: false + saves_to_disk: true + force_save_to_disk: ${{ inputs.force_save_to_disk || false }} + disk_suffix: tip + height_grep_text: 'current_height.*=.*Height.*\(' + secrets: inherit + # We want to prevent multiple full zebrad syncs running at the same time, + # but we don't want to cancel running syncs on `main` if a new PR gets merged, + # because we might never get a finished sync. + # + # Instead, we let the first sync complete, then queue the latest pending sync, cancelling any syncs in between. + # (As the general workflow concurrency group just gets matched in Pull Requests, + # it has no impact on this job.) + # + # TODO: + # - allow multiple manual syncs on a branch by adding '-${{ github.run_id }}' when github.event.inputs.run-full-sync is true + concurrency: + group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.run-full-sync == 'true') }}-test-full-sync + cancel-in-progress: false + + # Test that Zebra can sync to the chain tip, using a cached Zebra tip state, + # without launching `lightwalletd`. + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached state to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + test-update-sync: + name: Zebra tip update + needs: [ test-full-sync, get-available-disks ] + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + with: + app_name: zebrad + test_id: update-to-tip + test_description: Test syncing to tip with a Zebra tip state + test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_UPDATE_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' + needs_zebra_state: true + # update the disk on every PR, to increase CI speed + saves_to_disk: true + force_save_to_disk: ${{ inputs.force_save_to_disk || false }} + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + height_grep_text: 'current_height.*=.*Height.*\(' + secrets: inherit + + # zebra mainnet checkpoint generation tests + + # Test that Zebra can generate mainnet checkpoints after syncing to the chain tip, + # using a cached Zebra tip state, + # + # This test always runs on mainnet. + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached state to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + generate-checkpoints-mainnet: + name: Generate checkpoints mainnet + needs: [ test-full-sync, get-available-disks ] + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + with: + app_name: zebrad + test_id: generate-checkpoints-mainnet + test_description: Generate Zebra checkpoints on mainnet + # TODO: update the test to use {{ input.network }} instead? + test_variables: '-e NETWORK=Mainnet -e GENERATE_CHECKPOINTS_MAINNET=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' + needs_zebra_state: true + # test-update-sync updates the disk on every PR, so we don't need to do it here + saves_to_disk: false + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + height_grep_text: 'current_height.*=.*Height.*\(' + secrets: inherit + + # zebra testnet checkpoint generation tests + # + # These tests will fail when testnet is unstable, they should not be required to merge. + # + # TODO: ignore failures on testnet, so any failures don't appear in the GitHub interface. + + # Test that Zebra can run a full testnet sync, and regenerate chain tip Zebra cached state disks. + # This job always runs on testnet, regardless of any inputs or variable settings. + # + # Runs: + # - on schedule, as defined at the top of the workflow + # - on every PR update, but only if the state version in constants.rs has no cached disk + # - in manual workflow runs, when run-full-sync is 'true' and network is 'Testnet' + # + # Note: the output from get-available-disks-testnet should match with the caller workflow inputs + test-full-sync-testnet: + name: Zebra tip on testnet + needs: [ build, get-available-disks-testnet ] + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ (github.event_name == 'schedule' && vars.SCHEDULE_TESTNET_FULL_SYNC == 'true') || !fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Testnet') }} + with: + app_name: zebrad + test_id: full-sync-to-tip-testnet + test_description: Test a full sync up to the tip on testnet + # The value of FULL_SYNC_TESTNET_TIMEOUT_MINUTES is currently ignored. + test_variables: '-e NETWORK=Testnet -e FULL_SYNC_TESTNET_TIMEOUT_MINUTES=0 -e ZEBRA_FORCE_USE_COLOR=1' + network: "Testnet" + # A full testnet sync could take 2-10 hours in April 2023. + # The time varies a lot due to the small number of nodes. + is_long_test: true + needs_zebra_state: false + saves_to_disk: true + force_save_to_disk: ${{ inputs.force_save_to_disk || false }} + disk_suffix: tip + height_grep_text: 'current_height.*=.*Height.*\(' + secrets: inherit + # We want to prevent multiple full zebrad syncs running at the same time, + # but we don't want to cancel running syncs on `main` if a new PR gets merged, + # because we might never get a finished sync. + # + # Instead, we let the first sync complete, then queue the latest pending sync, cancelling any syncs in between. + # (As the general workflow concurrency group just gets matched in Pull Requests, + # it has no impact on this job.) + # + # TODO: + # - allow multiple manual syncs on a branch by adding '-${{ github.run_id }}' when github.event.inputs.run-full-sync is true + concurrency: + group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.run-full-sync == 'true') }}-test-full-sync-testnet + cancel-in-progress: false + + # Test that Zebra can generate testnet checkpoints after syncing to the chain tip, + # using a cached Zebra tip state. + # + # This test always runs on testnet. + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached state to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + generate-checkpoints-testnet: + name: Generate checkpoints testnet + needs: [ test-full-sync-testnet, get-available-disks-testnet ] + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk) || needs.test-full-sync-testnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + with: + app_name: zebrad + test_id: generate-checkpoints-testnet + test_description: Generate Zebra checkpoints on testnet + test_variables: '-e NETWORK=Testnet -e GENERATE_CHECKPOINTS_TESTNET=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' + network: "Testnet" + needs_zebra_state: true + # update the disk on every PR, to increase CI speed + # we don't have a test-update-sync-testnet job, so we need to update the disk here + saves_to_disk: true + force_save_to_disk: ${{ inputs.force_save_to_disk || false }} + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + height_grep_text: 'zebra_tip_height.*=.*Height.*\(' + secrets: inherit + + # lightwalletd cached tip state tests + + # Test full sync of lightwalletd with a Zebra tip state + # + # Runs: + # - on schedule, as defined at the top of the workflow + # - on every PR update, but only if the state version in constants.rs has no cached disk + # - in manual workflow runs, when run-lwd-sync is 'true' and network is 'Mainnet' (the network is required by the test-full-sync job) + # + # If the state version has changed, waits for the new cached state to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + lightwalletd-full-sync: + name: lightwalletd tip + needs: [ test-full-sync, get-available-disks ] + uses: ./.github/workflows/deploy-gcp-tests.yml + # Currently the lightwalletd tests only work on Mainnet + if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && (github.event_name == 'schedule' || !fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || github.event.inputs.run-lwd-sync == 'true' ) }} + with: + app_name: lightwalletd + test_id: lwd-full-sync + test_description: Test lightwalletd full sync + test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_FULL_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache' + # This test runs for longer than 6 hours, so it needs multiple jobs + is_long_test: true + needs_zebra_state: true + needs_lwd_state: false + saves_to_disk: true + force_save_to_disk: ${{ inputs.force_save_to_disk || false }} + disk_prefix: lwd-cache + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + lwd_state_dir: 'lwd-cache' + height_grep_text: 'Waiting for block: ' + secrets: inherit + # We want to prevent multiple lightwalletd full syncs running at the same time, + # but we don't want to cancel running syncs on `main` if a new PR gets merged, + # because we might never get a finished sync. + # + # See the concurrency comment on the zebrad test-full-sync job for details. + concurrency: + group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.run-lwd-sync == 'true') }}-lightwalletd-full-sync + cancel-in-progress: false + + # Test update sync of lightwalletd with a lightwalletd and Zebra tip state + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached states to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + lightwalletd-update-sync: + name: lightwalletd tip update + needs: [ lightwalletd-full-sync, get-available-disks ] + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + with: + app_name: lightwalletd + test_id: lwd-update-sync + test_description: Test lightwalletd update sync with both states + test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_UPDATE_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache' + needs_zebra_state: true + needs_lwd_state: true + saves_to_disk: true + force_save_to_disk: ${{ inputs.force_save_to_disk || false }} + disk_prefix: lwd-cache + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + lwd_state_dir: 'lwd-cache' + height_grep_text: 'Waiting for block: ' + secrets: inherit + + # Test that Zebra can answer a synthetic RPC call, using a cached Zebra tip state + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached state to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + # + # TODO: move this job below the rest of the mainnet jobs that just use Zebra cached state + lightwalletd-rpc-test: + name: Zebra tip JSON-RPC + needs: [ test-full-sync, get-available-disks ] + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + with: + app_name: lightwalletd + test_id: fully-synced-rpc + test_description: Test lightwalletd RPC with a Zebra tip state + test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_RPC_CALL=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' + needs_zebra_state: true + saves_to_disk: false + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + secrets: inherit + + # Test that Zebra can handle a lightwalletd send transaction RPC call, using a cached Zebra tip state + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached states to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + lightwalletd-transactions-test: + name: lightwalletd tip send + needs: [ lightwalletd-full-sync, get-available-disks ] + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + with: + app_name: lightwalletd + test_id: lwd-send-transactions + test_description: Test sending transactions via lightwalletd + test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_TRANSACTIONS=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache' + needs_zebra_state: true + needs_lwd_state: true + saves_to_disk: false + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + lwd_state_dir: 'lwd-cache' + secrets: inherit + + # Test that Zebra can handle gRPC wallet calls, using a cached Zebra tip state + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached states to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + lightwalletd-grpc-test: + name: lightwalletd GRPC tests + needs: [ lightwalletd-full-sync, get-available-disks ] + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + with: + app_name: lightwalletd + test_id: lwd-grpc-wallet + test_description: Test gRPC calls via lightwalletd + test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_GRPC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache' + needs_zebra_state: true + needs_lwd_state: true + saves_to_disk: false + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + lwd_state_dir: 'lwd-cache' + secrets: inherit + + ## getblocktemplate-rpcs using cached Zebra state on mainnet + # + # TODO: move these below the rest of the mainnet jobs that just use Zebra cached state + + # Test that Zebra can handle a getblocktemplate RPC call, using a cached Zebra tip state + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached states to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + get-block-template-test: + name: get block template + needs: [ test-full-sync, get-available-disks ] + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + with: + app_name: zebrad + test_id: get-block-template + test_description: Test getblocktemplate RPC method via Zebra's rpc server + test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_GET_BLOCK_TEMPLATE=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' + needs_zebra_state: true + needs_lwd_state: false + saves_to_disk: false + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + secrets: inherit + + # Test that Zebra can handle a submit block RPC call, using a cached Zebra tip state + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached states to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + submit-block-test: + name: submit block + needs: [ test-full-sync, get-available-disks ] + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + with: + app_name: zebrad + test_id: submit-block + test_description: Test submitting blocks via Zebra's rpc server + test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SUBMIT_BLOCK=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' + needs_zebra_state: true + needs_lwd_state: false + saves_to_disk: false + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + secrets: inherit + + failure-issue: + name: Open or update issues for main branch failures + # When a new test is added to this workflow, add it to this list. + # + # This list is for reliable tests that are run on the `main` branch. + # Testnet jobs are not in this list, because we expect testnet to fail occasionally. + needs: [ regenerate-stateful-disks, test-full-sync, lightwalletd-full-sync, test-all, test-all-getblocktemplate-rpcs, test-fake-activation-heights, test-empty-sync, test-lightwalletd-integration, test-configuration-file, test-zebra-conf-path, test-stateful-sync, test-update-sync, generate-checkpoints-mainnet, lightwalletd-update-sync, lightwalletd-rpc-test, lightwalletd-transactions-test, lightwalletd-grpc-test, get-block-template-test, submit-block-test ] + # Only open tickets for failed scheduled jobs, manual workflow runs, or `main` branch merges. + # (PR statuses are already reported in the PR jobs list, and checked by Mergify.) + # TODO: if a job times out, we want to create a ticket. Does failure() do that? Or do we need cancelled()? + if: failure() && github.event.pull_request == null + runs-on: ubuntu-latest + steps: + - uses: jayqi/failed-build-issue-action@v1 + with: + title-template: "{{refname}} branch CI failed: {{eventName}} in {{workflow}}" + # New failures open an issue with this label. + # TODO: do we want a different label for each workflow, or each kind of workflow? + label-name: S-ci-fail-auto-issue + # If there is already an open issue with this label, any failures become comments on that issue. + always-create-new-issue: false + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/book/src/user/docker.md b/book/src/user/docker.md index dc977069db8..5119017de01 100644 --- a/book/src/user/docker.md +++ b/book/src/user/docker.md @@ -51,7 +51,7 @@ And after our image has been built, we can run it on `Mainnet` with the followin docker run --env LOG_COLOR="true" -p 9999:9999 local/zebra.mining ``` -Based on our actual `runtime-entrypoint.sh` script, the following configuration file will be generated (on the fly, at startup) and used by Zebra: +Based on our actual `entrypoint.sh` script, the following configuration file will be generated (on the fly, at startup) and used by Zebra: ```toml [network] @@ -80,7 +80,7 @@ endpoint_addr = "127.0.0.1:9999" - `TEST_FEATURES`: Specifies the features for tests. Example: `"lightwalletd-grpc-tests zebra-checkpoints"` - `ZEBRA_SKIP_IPV6_TESTS`: Skips IPv6 tests. Example: `1` -- `ENTRYPOINT_FEATURES`: Overrides the specific features used to run tests in `runtime-entrypoint.sh`. Example: `"default-release-binaries lightwalletd-grpc-tests"` +- `ENTRYPOINT_FEATURES`: Overrides the specific features used to run tests in `entrypoint.sh`. Example: `"default-release-binaries lightwalletd-grpc-tests"` #### CI/CD diff --git a/docker/Dockerfile b/docker/Dockerfile index e16cac66930..6987354662a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -169,8 +169,8 @@ RUN rm -r zebra-original # Build zebrad RUN cargo build --locked --release --features "${FEATURES}" --package zebrad --bin zebrad -COPY ./docker/runtime-entrypoint.sh / -RUN chmod u+x /runtime-entrypoint.sh +COPY ./docker/entrypoint.sh / +RUN chmod u+x /entrypoint.sh # This stage is only used when deploying nodes or when only the resulting zebrad binary is needed # @@ -178,7 +178,7 @@ RUN chmod u+x /runtime-entrypoint.sh # binary from the `release` stage, and the Zcash Sprout & Sapling parameters from ZCash FROM debian:bullseye-slim AS runtime COPY --from=release /opt/zebrad/target/release/zebrad /usr/local/bin -COPY --from=release /runtime-entrypoint.sh / +COPY --from=release /entrypoint.sh / COPY --from=us-docker.pkg.dev/zfnd-dev-zebra/zebra/zcash-params:edge /root/.zcash-params /root/.zcash-params RUN apt-get update && \ @@ -194,4 +194,4 @@ EXPOSE 8233 18233 # Update the config file based on the Docker run variables, # and launch zebrad with it -ENTRYPOINT [ "/runtime-entrypoint.sh" ] +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 123d4f34dd9..af43531b303 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,128 +1,283 @@ #!/usr/bin/env bash -# show the commands we are executing +# Show the commands we are executing set -x -# exit if a command fails +# Exit if a command fails set -e -# exit if any command in a pipeline fails +# Exit if any command in a pipeline fails set -o pipefail +#### +# General Variables +# These variables are used to run the Zebra node. +#### + +# Set this to change the default cached state directory +# Path and name of the config file +: "${ZEBRA_CONF_DIR:=/etc/zebrad}" +: "${ZEBRA_CONF_FILE:=zebrad.toml}" +# [network] : "${NETWORK:=Mainnet}" -echo "Test variables:" -echo "NETWORK=$NETWORK" -echo "ZEBRA_TEST_LIGHTWALLETD=$ZEBRA_TEST_LIGHTWALLETD" -echo "Hard-coded Zebra full sync directory: /zebrad-cache" -echo "ZEBRA_CACHED_STATE_DIR=$ZEBRA_CACHED_STATE_DIR" -echo "LIGHTWALLETD_DATA_DIR=$LIGHTWALLETD_DATA_DIR" -echo "ENTRYPOINT_FEATURES=$ENTRYPOINT_FEATURES" +: "${ZEBRA_LISTEN_ADDR:=0.0.0.0}" +# [consensus] +: "${ZEBRA_CHECKPOINT_SYNC:=true}" +# [state] +: "${ZEBRA_CACHED_STATE_DIR:=/var/cache/zebrad-cache}" +# [metrics] +: "${METRICS_ENDPOINT_ADDR:=0.0.0.0}" +: "${METRICS_ENDPOINT_PORT:=9999}" +# [tracing] +: "${LOG_COLOR:=false}" +: "${TRACING_ENDPOINT_ADDR:=0.0.0.0}" +: "${TRACING_ENDPOINT_PORT:=3000}" +# [rpc] +: "${RPC_LISTEN_ADDR:=0.0.0.0}" +# if ${RPC_PORT} is not set and ${FEATURES} contains getblocktemplate-rpcs, +# set ${RPC_PORT} to the default value for the current network +if [[ -z "${RPC_PORT}" ]]; then + if [[ " ${FEATURES} " =~ " getblocktemplate-rpcs " ]]; then + if [[ "${NETWORK}" = "Mainnet" ]]; then + : "${RPC_PORT:=8232}" + elif [[ "${NETWORK}" = "Testnet" ]]; then + : "${RPC_PORT:=18232}" + fi + fi +fi + +#### +# Test Variables +# These variables are used to run tests in the Dockerfile. +#### + +: "${RUN_ALL_TESTS:=}" +: "${ZEBRA_TEST_LIGHTWALLETD:=}" +: "${FULL_SYNC_MAINNET_TIMEOUT_MINUTES:=}" +: "${FULL_SYNC_TESTNET_TIMEOUT_MINUTES:=}" +: "${TEST_DISK_REBUILD:=}" +: "${TEST_UPDATE_SYNC:=}" +: "${TEST_CHECKPOINT_SYNC:=}" +: "${GENERATE_CHECKPOINTS_MAINNET:=}" +: "${GENERATE_CHECKPOINTS_TESTNET:=}" +: "${TEST_LWD_RPC_CALL:=}" +: "${TEST_LWD_FULL_SYNC:=}" +: "${TEST_LWD_UPDATE_SYNC:=}" +: "${TEST_LWD_GRPC:=}" +: "${TEST_LWD_TRANSACTIONS:=}" +: "${TEST_GET_BLOCK_TEMPLATE:=}" +: "${TEST_SUBMIT_BLOCK:=}" +: "${ENTRYPOINT_FEATURES:=}" + +# Configuration file path +if [[ -n "${ZEBRA_CONF_DIR}" ]] && [[ -n "${ZEBRA_CONF_FILE}" ]]; then + ZEBRA_CONF_PATH="${ZEBRA_CONF_DIR}/${ZEBRA_CONF_FILE}" +fi + +# Populate `zebrad.toml` before starting zebrad, using the environmental +# variables set by the Dockerfile or the user. If the user has already created a config, don't replace it. +# +# We disable most ports by default, so the default config is secure. +# Users have to opt-in to additional functionality by setting environmental variables. +if [[ -n "${ZEBRA_CONF_PATH}" ]] && [[ ! -f "${ZEBRA_CONF_PATH}" ]]; then + # Create the conf path and file + mkdir -p "${ZEBRA_CONF_DIR}" || { echo "Error creating directory ${ZEBRA_CONF_DIR}"; exit 1; } + touch "${ZEBRA_CONF_PATH}" || { echo "Error creating file ${ZEBRA_CONF_PATH}"; exit 1; } + # Populate the conf file + cat < "${ZEBRA_CONF_PATH}" +[network] +network = "${NETWORK}" +listen_addr = "${ZEBRA_LISTEN_ADDR}" +[state] +cache_dir = "${ZEBRA_CACHED_STATE_DIR}" +EOF + + if [[ " ${FEATURES} " =~ " prometheus " ]]; then # spaces are important here to avoid partial matches + cat <> "${ZEBRA_CONF_PATH}" +[metrics] +endpoint_addr = "${METRICS_ENDPOINT_ADDR}:${METRICS_ENDPOINT_PORT}" +EOF + fi + + if [[ -n "${RPC_PORT}" ]]; then + cat <> "${ZEBRA_CONF_PATH}" +[rpc] +listen_addr = "${RPC_LISTEN_ADDR}:${RPC_PORT}" +EOF + fi + + if [[ -n "${LOG_FILE}" ]] || [[ -n "${LOG_COLOR}" ]] || [[ -n "${TRACING_ENDPOINT_ADDR}" ]]; then + cat <> "${ZEBRA_CONF_PATH}" +[tracing] +EOF + if [[ " ${FEATURES} " =~ " filter-reload " ]]; then # spaces are important here to avoid partial matches + cat <> "${ZEBRA_CONF_PATH}" +endpoint_addr = "${TRACING_ENDPOINT_ADDR}:${TRACING_ENDPOINT_PORT}" +EOF + fi + # Set this to log to a file, if not set, logs to standard output + if [[ -n "${LOG_FILE}" ]]; then + mkdir -p "$(dirname "${LOG_FILE}")" + cat <> "${ZEBRA_CONF_PATH}" +log_file = "${LOG_FILE}" +EOF + fi + # Zebra automatically detects if it is attached to a terminal, and uses colored output. + # Set this to 'true' to force using color even if the output is not a terminal. + # Set this to 'false' to disable using color even if the output is a terminal. + if [[ "${LOG_COLOR}" = "true" ]]; then + cat <> "${ZEBRA_CONF_PATH}" +force_use_color = true +EOF + elif [[ "${LOG_COLOR}" = "false" ]]; then + cat <> "${ZEBRA_CONF_PATH}" +use_color = false +EOF + fi + fi + + if [[ -n "${MINER_ADDRESS}" ]]; then + cat <> "${ZEBRA_CONF_PATH}" +[mining] +miner_address = "${MINER_ADDRESS}" +EOF + fi +fi + +echo "Using zebrad.toml:" +cat "${ZEBRA_CONF_PATH}" + +# Function to list directory +list_directory() { + local dir="$1" + # Using find instead of ls to better handle non-alphanumeric filenames, which also requires `-exec +` + find "${dir}" -type f -print0 | xargs -0 -I {} ls -lh "{}" || { echo "No files in ${dir}"; find "${dir}" -type d -exec ls -lhR {} + | head -50 || echo "No ${dir} directory"; } +} + +# Function to run cargo test +run_cargo_test() { + cargo test --locked --release --features "$1" --package zebrad --test acceptance -- --nocapture --include-ignored "$2" || { echo "Cargo test failed"; exit 1; } +} case "$1" in --* | -*) - exec zebrad "$@" + if [[ -n "${ZEBRA_CONF_PATH}" ]]; then + exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" || { echo "Execution with custom configuration failed"; exit 1; } + else + exec zebrad "$@" || { echo "Execution failed"; exit 1; } + fi + ;; + "") + if [[ -n "${ZEBRA_CONF_PATH}" ]]; then + exec zebrad -c "${ZEBRA_CONF_PATH}" || { echo "Execution with custom configuration failed"; exit 1; } + else + exec zebrad || { echo "Execution with default configuration failed"; exit 1; } + fi ;; *) - # For these tests, we activate the test features to avoid recompiling `zebrad`, - # but we don't actually run any gRPC tests. - if [[ "$RUN_ALL_TESTS" -eq "1" ]]; then - # Run all the available tests for the current environment. - # If the lightwalletd environmental variables are set, we will also run those tests. - cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --workspace -- --nocapture --include-ignored - - # For these tests, we activate the gRPC feature to avoid recompiling `zebrad`, - # but we don't actually run any gRPC tests. - elif [[ -n "$FULL_SYNC_MAINNET_TIMEOUT_MINUTES" ]]; then - # Run a Zebra full sync test on mainnet. - cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored full_sync_mainnet - # List directory generated by test - # TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows - ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory") - elif [[ -n "$FULL_SYNC_TESTNET_TIMEOUT_MINUTES" ]]; then - # Run a Zebra full sync test on testnet. - cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored full_sync_testnet - # List directory generated by test - # TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows - ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory") - elif [[ "$TEST_DISK_REBUILD" -eq "1" ]]; then - # Run a Zebra sync up to the mandatory checkpoint. - # - # TODO: use environmental variables instead of Rust features (part of #2995) - cargo test --locked --release --features "test_sync_to_mandatory_checkpoint_${NETWORK,,},$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_to_mandatory_checkpoint_${NETWORK,,}" - # TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows - ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory") - elif [[ "$TEST_UPDATE_SYNC" -eq "1" ]]; then - # Run a Zebra sync starting at the cached tip, and syncing to the latest tip. - # - # List directory used by test - ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored zebrad_update_sync - elif [[ "$TEST_CHECKPOINT_SYNC" -eq "1" ]]; then - # Run a Zebra sync starting at the cached mandatory checkpoint, and syncing past it. - # - # List directory used by test - # TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows - ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory") - # TODO: use environmental variables instead of Rust features (part of #2995) - cargo test --locked --release --features "test_sync_past_mandatory_checkpoint_${NETWORK,,},$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_past_mandatory_checkpoint_${NETWORK,,}" - - elif [[ "$GENERATE_CHECKPOINTS_MAINNET" -eq "1" ]]; then - # Generate checkpoints after syncing Zebra from a cached state on mainnet. - # - # TODO: disable or filter out logs like: - # test generate_checkpoints_mainnet has been running for over 60 seconds - # - # List directory used by test - ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored generate_checkpoints_mainnet - elif [[ "$GENERATE_CHECKPOINTS_TESTNET" -eq "1" ]]; then - # Generate checkpoints after syncing Zebra on testnet. - # - # This test might fail if testnet is unstable. - # - # List directory used by test - ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored generate_checkpoints_testnet - - elif [[ "$TEST_LWD_RPC_CALL" -eq "1" ]]; then - # Starting at a cached Zebra tip, test a JSON-RPC call to Zebra. - ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - # Run both the fully synced RPC test and the subtree snapshot test, one test at a time. - # Since these tests use the same cached state, a state problem in the first test can fail the second test. - cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored --test-threads 1 fully_synced_rpc_ - elif [[ "$TEST_LWD_FULL_SYNC" -eq "1" ]]; then - # Starting at a cached Zebra tip, run a lightwalletd sync to tip. - ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_full_sync - ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory") - elif [[ "$TEST_LWD_UPDATE_SYNC" -eq "1" ]]; then - # Starting with a cached Zebra and lightwalletd tip, run a quick update sync. - ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory") - cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_update_sync - - # These tests actually use gRPC. - elif [[ "$TEST_LWD_GRPC" -eq "1" ]]; then - # Starting with a cached Zebra and lightwalletd tip, test all gRPC calls to lightwalletd, which calls Zebra. - ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory") - cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored lightwalletd_wallet_grpc_tests - elif [[ "$TEST_LWD_TRANSACTIONS" -eq "1" ]]; then - # Starting with a cached Zebra and lightwalletd tip, test sending transactions gRPC call to lightwalletd, which calls Zebra. - ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory") - cargo test --locked --release --features "$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored sending_transactions_using_lightwalletd - - # These tests use mining code, but don't use gRPC. - # We add the mining feature here because our other code needs to pass tests without it. - elif [[ "$TEST_GET_BLOCK_TEMPLATE" -eq "1" ]]; then - # Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server. - ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - cargo test --locked --release --features "getblocktemplate-rpcs,$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored get_block_template - elif [[ "$TEST_SUBMIT_BLOCK" -eq "1" ]]; then - # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port. - ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") - cargo test --locked --release --features "getblocktemplate-rpcs,$ENTRYPOINT_FEATURES" --package zebrad --test acceptance -- --nocapture --include-ignored submit_block + if [[ -n "${ENTRYPOINT_FEATURES}" ]]; then + # Validate the test variables + # For these tests, we activate the test features to avoid recompiling `zebrad`, + # but we don't actually run any gRPC tests. + if [[ "${RUN_ALL_TESTS}" -eq "1" ]]; then + # Run all the available tests for the current environment. + # If the lightwalletd environmental variables are set, we will also run those tests. + cargo test --locked --release --features "${ENTRYPOINT_FEATURES}" --workspace -- --nocapture --include-ignored - else - exec "$@" + # For these tests, we activate the gRPC feature to avoid recompiling `zebrad`, + # but we don't actually run any gRPC tests. + elif [[ -n "${FULL_SYNC_MAINNET_TIMEOUT_MINUTES}" ]]; then + # Run a Zebra full sync test on mainnet. + run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_mainnet" + # List directory generated by test + # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows + list_directory "/zebrad-cache" + elif [[ -n "${FULL_SYNC_TESTNET_TIMEOUT_MINUTES}" ]]; then + # Run a Zebra full sync test on testnet. + run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_testnet" + # List directory generated by test + # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows + list_directory "/zebrad-cache" + elif [[ "${TEST_DISK_REBUILD}" -eq "1" ]]; then + # Run a Zebra sync up to the mandatory checkpoint. + # + # TODO: use environmental variables instead of Rust features (part of #2995) + run_cargo_test "test_sync_to_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_to_mandatory_checkpoint_${NETWORK,,}" + # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows + list_directory "/zebrad-cache" + elif [[ "${TEST_UPDATE_SYNC}" -eq "1" ]]; then + # Run a Zebra sync starting at the cached tip, and syncing to the latest tip. + # + # List directory used by test + list_directory "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "zebrad_update_sync" + elif [[ "${TEST_CHECKPOINT_SYNC}" -eq "1" ]]; then + # Run a Zebra sync starting at the cached mandatory checkpoint, and syncing past it. + # + # List directory used by test + # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows + list_directory "/zebrad-cache" + # TODO: use environmental variables instead of Rust features (part of #2995) + run_cargo_test "test_sync_to_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_past_mandatory_checkpoint_${NETWORK,,}" + + elif [[ "${GENERATE_CHECKPOINTS_MAINNET}" -eq "1" ]]; then + # Generate checkpoints after syncing Zebra from a cached state on mainnet. + # + # TODO: disable or filter out logs like: + # test generate_checkpoints_mainnet has been running for over 60 seconds + # + # List directory used by test + list_directory "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_mainnet" + elif [[ "${GENERATE_CHECKPOINTS_TESTNET}" -eq "1" ]]; then + # Generate checkpoints after syncing Zebra on testnet. + # + # This test might fail if testnet is unstable. + # + # List directory used by test + list_directory "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_testnet" + + elif [[ "${TEST_LWD_RPC_CALL}" -eq "1" ]]; then + # Starting at a cached Zebra tip, test a JSON-RPC call to Zebra. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + # Run both the fully synced RPC test and the subtree snapshot test, one test at a time. + # Since these tests use the same cached state, a state problem in the first test can fail the second test. + run_cargo_test "${ENTRYPOINT_FEATURES}" "fully_synced_rpc_" + elif [[ "${TEST_LWD_FULL_SYNC}" -eq "1" ]]; then + # Starting at a cached Zebra tip, run a lightwalletd sync to tip. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_full_sync" + list_directory "${LIGHTWALLETD_DATA_DIR}/db" + elif [[ "${TEST_LWD_UPDATE_SYNC}" -eq "1" ]]; then + # Starting with a cached Zebra and lightwalletd tip, run a quick update sync. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + list_directory "${LIGHTWALLETD_DATA_DIR}/db" + run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_update_sync" + + # These tests actually use gRPC. + elif [[ "${TEST_LWD_GRPC}" -eq "1" ]]; then + # Starting with a cached Zebra and lightwalletd tip, test all gRPC calls to lightwalletd, which calls Zebra. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + list_directory "${LIGHTWALLETD_DATA_DIR}/db" + run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_wallet_grpc_tests" + elif [[ "${TEST_LWD_TRANSACTIONS}" -eq "1" ]]; then + # Starting with a cached Zebra and lightwalletd tip, test sending transactions gRPC call to lightwalletd, which calls Zebra. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + list_directory "${LIGHTWALLETD_DATA_DIR}/db" + run_cargo_test "${ENTRYPOINT_FEATURES}" "sending_transactions_using_lightwalletd" + + # These tests use mining code, but don't use gRPC. + # We add the mining feature here because our other code needs to pass tests without it. + elif [[ "${TEST_GET_BLOCK_TEMPLATE}" -eq "1" ]]; then + # Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "getblocktemplate-rpcs,${ENTRYPOINT_FEATURES}" "get_block_template" + elif [[ "${TEST_SUBMIT_BLOCK}" -eq "1" ]]; then + # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "getblocktemplate-rpcs,${ENTRYPOINT_FEATURES}" "submit_block" + + else + exec "$@" + fi fi esac diff --git a/docker/runtime-entrypoint.sh b/docker/runtime-entrypoint.sh deleted file mode 100755 index 613619795f5..00000000000 --- a/docker/runtime-entrypoint.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash - -# Show the commands we are executing -set -x -# Exit if a command fails -set -e -# Exit if any command in a pipeline fails -set -o pipefail - -# Set this to change the default cached state directory -# Path and name of the config file -: "${ZEBRA_CONF_DIR:=/etc/zebrad}" -: "${ZEBRA_CONF_FILE:=zebrad.toml}" -if [[ -n "$ZEBRA_CONF_DIR" ]] && [[ -n "$ZEBRA_CONF_FILE" ]]; then - ZEBRA_CONF_PATH="$ZEBRA_CONF_DIR/$ZEBRA_CONF_FILE" -fi - -# [network] -: "${NETWORK:=Mainnet}" -: "${ZEBRA_LISTEN_ADDR:=0.0.0.0}" -# [consensus] -: "${ZEBRA_CHECKPOINT_SYNC:=true}" -# [state] -: "${ZEBRA_CACHED_STATE_DIR:=/var/cache/zebrad-cache}" -# [metrics] -: "${METRICS_ENDPOINT_ADDR:=0.0.0.0}" -: "${METRICS_ENDPOINT_PORT:=9999}" -# [tracing] -: "${LOG_COLOR:=false}" -: "${TRACING_ENDPOINT_ADDR:=0.0.0.0}" -: "${TRACING_ENDPOINT_PORT:=3000}" -# [rpc] -: "${RPC_LISTEN_ADDR:=0.0.0.0}" -if [[ -z "${RPC_PORT}" ]]; then -if [[ " ${FEATURES} " =~ " getblocktemplate-rpcs " ]]; then -if [[ "${NETWORK}" = "Mainnet" ]]; then -: "${RPC_PORT:=8232}" -elif [[ "${NETWORK}" = "Testnet" ]]; then -: "${RPC_PORT:=18232}" -fi -fi -fi - -# Populate `zebrad.toml` before starting zebrad, using the environmental -# variables set by the Dockerfile or the user. If the user has already created a config, don't replace it. -# -# We disable most ports by default, so the default config is secure. -# Users have to opt-in to additional functionality by setting environmental variables. -if [[ -n "$ZEBRA_CONF_PATH" ]] && [[ ! -f "$ZEBRA_CONF_PATH" ]]; then - -# Create the conf path and file -mkdir -p "$ZEBRA_CONF_DIR" -touch "$ZEBRA_CONF_PATH" - -# Populate the conf file -cat < "$ZEBRA_CONF_PATH" -[network] -network = "$NETWORK" -listen_addr = "$ZEBRA_LISTEN_ADDR" -[state] -cache_dir = "$ZEBRA_CACHED_STATE_DIR" -EOF - -if [[ " $FEATURES " =~ " prometheus " ]]; then # spaces are important here to avoid partial matches -cat <> "$ZEBRA_CONF_PATH" -[metrics] -endpoint_addr = "${METRICS_ENDPOINT_ADDR}:${METRICS_ENDPOINT_PORT}" -EOF -fi - -if [[ -n "${RPC_PORT}" ]]; then -cat <> "${ZEBRA_CONF_PATH}" -[rpc] -listen_addr = "${RPC_LISTEN_ADDR}:${RPC_PORT}" -EOF -fi - -if [[ -n "$LOG_FILE" ]] || [[ -n "$LOG_COLOR" ]] || [[ -n "$TRACING_ENDPOINT_ADDR" ]]; then -cat <> "$ZEBRA_CONF_PATH" -[tracing] -EOF -if [[ " $FEATURES " =~ " filter-reload " ]]; then # spaces are important here to avoid partial matches -cat <> "$ZEBRA_CONF_PATH" -endpoint_addr = "${TRACING_ENDPOINT_ADDR}:${TRACING_ENDPOINT_PORT}" -EOF -fi -# Set this to log to a file, if not set, logs to standard output -if [[ -n "$LOG_FILE" ]]; then -mkdir -p "$(dirname "$LOG_FILE")" -cat <> "$ZEBRA_CONF_PATH" -log_file = "${LOG_FILE}" -EOF -fi - -# Zebra automatically detects if it is attached to a terminal, and uses colored output. -# Set this to 'true' to force using color even if the output is not a terminal. -# Set this to 'false' to disable using color even if the output is a terminal. -if [[ "$LOG_COLOR" = "true" ]]; then -cat <> "$ZEBRA_CONF_PATH" -force_use_color = true -EOF -elif [[ "$LOG_COLOR" = "false" ]]; then -cat <> "$ZEBRA_CONF_PATH" -use_color = false -EOF -fi -fi - -if [[ -n "$MINER_ADDRESS" ]]; then -cat <> "$ZEBRA_CONF_PATH" -[mining] -miner_address = "${MINER_ADDRESS}" -EOF -fi -fi - -echo "Using zebrad.toml:" -cat "$ZEBRA_CONF_PATH" - -exec zebrad -c "$ZEBRA_CONF_PATH" "$@" From f3c85b4e32b71dba10452b99517f7e3fbb17875f Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 2 Oct 2023 23:42:06 +0100 Subject: [PATCH 02/22] chore: use 2 spaces insted of 4 for indentation --- docker/entrypoint.sh | 327 ++++++++++++++++++++++--------------------- 1 file changed, 168 insertions(+), 159 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index af43531b303..54529dd9487 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -35,13 +35,13 @@ set -o pipefail # if ${RPC_PORT} is not set and ${FEATURES} contains getblocktemplate-rpcs, # set ${RPC_PORT} to the default value for the current network if [[ -z "${RPC_PORT}" ]]; then - if [[ " ${FEATURES} " =~ " getblocktemplate-rpcs " ]]; then - if [[ "${NETWORK}" = "Mainnet" ]]; then - : "${RPC_PORT:=8232}" - elif [[ "${NETWORK}" = "Testnet" ]]; then - : "${RPC_PORT:=18232}" - fi + if [[ " ${FEATURES} " =~ " getblocktemplate-rpcs " ]]; then + if [[ "${NETWORK}" = "Mainnet" ]]; then + : "${RPC_PORT:=8232}" + elif [[ "${NETWORK}" = "Testnet" ]]; then + : "${RPC_PORT:=18232}" fi + fi fi #### @@ -69,7 +69,7 @@ fi # Configuration file path if [[ -n "${ZEBRA_CONF_DIR}" ]] && [[ -n "${ZEBRA_CONF_FILE}" ]]; then - ZEBRA_CONF_PATH="${ZEBRA_CONF_DIR}/${ZEBRA_CONF_FILE}" + ZEBRA_CONF_PATH="${ZEBRA_CONF_DIR}/${ZEBRA_CONF_FILE}" fi # Populate `zebrad.toml` before starting zebrad, using the environmental @@ -78,11 +78,11 @@ fi # We disable most ports by default, so the default config is secure. # Users have to opt-in to additional functionality by setting environmental variables. if [[ -n "${ZEBRA_CONF_PATH}" ]] && [[ ! -f "${ZEBRA_CONF_PATH}" ]]; then - # Create the conf path and file - mkdir -p "${ZEBRA_CONF_DIR}" || { echo "Error creating directory ${ZEBRA_CONF_DIR}"; exit 1; } - touch "${ZEBRA_CONF_PATH}" || { echo "Error creating file ${ZEBRA_CONF_PATH}"; exit 1; } - # Populate the conf file - cat < "${ZEBRA_CONF_PATH}" + # Create the conf path and file + mkdir -p "${ZEBRA_CONF_DIR}" || { echo "Error creating directory ${ZEBRA_CONF_DIR}"; exit 1; } + touch "${ZEBRA_CONF_PATH}" || { echo "Error creating file ${ZEBRA_CONF_PATH}"; exit 1; } + # Populate the conf file + cat < "${ZEBRA_CONF_PATH}" [network] network = "${NETWORK}" listen_addr = "${ZEBRA_LISTEN_ADDR}" @@ -90,56 +90,56 @@ listen_addr = "${ZEBRA_LISTEN_ADDR}" cache_dir = "${ZEBRA_CACHED_STATE_DIR}" EOF - if [[ " ${FEATURES} " =~ " prometheus " ]]; then # spaces are important here to avoid partial matches - cat <> "${ZEBRA_CONF_PATH}" + if [[ " ${FEATURES} " =~ " prometheus " ]]; then # spaces are important here to avoid partial matches + cat <> "${ZEBRA_CONF_PATH}" [metrics] endpoint_addr = "${METRICS_ENDPOINT_ADDR}:${METRICS_ENDPOINT_PORT}" EOF - fi + fi - if [[ -n "${RPC_PORT}" ]]; then - cat <> "${ZEBRA_CONF_PATH}" + if [[ -n "${RPC_PORT}" ]]; then + cat <> "${ZEBRA_CONF_PATH}" [rpc] listen_addr = "${RPC_LISTEN_ADDR}:${RPC_PORT}" EOF - fi + fi - if [[ -n "${LOG_FILE}" ]] || [[ -n "${LOG_COLOR}" ]] || [[ -n "${TRACING_ENDPOINT_ADDR}" ]]; then - cat <> "${ZEBRA_CONF_PATH}" + if [[ -n "${LOG_FILE}" ]] || [[ -n "${LOG_COLOR}" ]] || [[ -n "${TRACING_ENDPOINT_ADDR}" ]]; then + cat <> "${ZEBRA_CONF_PATH}" [tracing] EOF - if [[ " ${FEATURES} " =~ " filter-reload " ]]; then # spaces are important here to avoid partial matches - cat <> "${ZEBRA_CONF_PATH}" + if [[ " ${FEATURES} " =~ " filter-reload " ]]; then # spaces are important here to avoid partial matches + cat <> "${ZEBRA_CONF_PATH}" endpoint_addr = "${TRACING_ENDPOINT_ADDR}:${TRACING_ENDPOINT_PORT}" EOF - fi - # Set this to log to a file, if not set, logs to standard output - if [[ -n "${LOG_FILE}" ]]; then - mkdir -p "$(dirname "${LOG_FILE}")" - cat <> "${ZEBRA_CONF_PATH}" + fi + # Set this to log to a file, if not set, logs to standard output + if [[ -n "${LOG_FILE}" ]]; then + mkdir -p "$(dirname "${LOG_FILE}")" + cat <> "${ZEBRA_CONF_PATH}" log_file = "${LOG_FILE}" EOF - fi - # Zebra automatically detects if it is attached to a terminal, and uses colored output. - # Set this to 'true' to force using color even if the output is not a terminal. - # Set this to 'false' to disable using color even if the output is a terminal. - if [[ "${LOG_COLOR}" = "true" ]]; then - cat <> "${ZEBRA_CONF_PATH}" + fi + # Zebra automatically detects if it is attached to a terminal, and uses colored output. + # Set this to 'true' to force using color even if the output is not a terminal. + # Set this to 'false' to disable using color even if the output is a terminal. + if [[ "${LOG_COLOR}" = "true" ]]; then + cat <> "${ZEBRA_CONF_PATH}" force_use_color = true EOF - elif [[ "${LOG_COLOR}" = "false" ]]; then - cat <> "${ZEBRA_CONF_PATH}" + elif [[ "${LOG_COLOR}" = "false" ]]; then + cat <> "${ZEBRA_CONF_PATH}" use_color = false EOF - fi fi + fi - if [[ -n "${MINER_ADDRESS}" ]]; then - cat <> "${ZEBRA_CONF_PATH}" + if [[ -n "${MINER_ADDRESS}" ]]; then + cat <> "${ZEBRA_CONF_PATH}" [mining] miner_address = "${MINER_ADDRESS}" EOF - fi + fi fi echo "Using zebrad.toml:" @@ -147,137 +147,146 @@ cat "${ZEBRA_CONF_PATH}" # Function to list directory list_directory() { - local dir="$1" - # Using find instead of ls to better handle non-alphanumeric filenames, which also requires `-exec +` - find "${dir}" -type f -print0 | xargs -0 -I {} ls -lh "{}" || { echo "No files in ${dir}"; find "${dir}" -type d -exec ls -lhR {} + | head -50 || echo "No ${dir} directory"; } + local dir="$1" + # Using find instead of ls to better handle non-alphanumeric filenames, which also requires `-exec +` + find "${dir}" -type f -print0 | xargs -0 -I {} ls -lh "{}" || { echo "No files in ${dir}"; find "${dir}" -type d -exec ls -lhR {} + | head -50 || echo "No ${dir} directory"; } } # Function to run cargo test run_cargo_test() { - cargo test --locked --release --features "$1" --package zebrad --test acceptance -- --nocapture --include-ignored "$2" || { echo "Cargo test failed"; exit 1; } + cargo test --locked --release --features "$1" --package zebrad --test acceptance -- --nocapture --include-ignored "$2" || { echo "Cargo test failed"; exit 1; } } case "$1" in - --* | -*) - if [[ -n "${ZEBRA_CONF_PATH}" ]]; then - exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" || { echo "Execution with custom configuration failed"; exit 1; } - else - exec zebrad "$@" || { echo "Execution failed"; exit 1; } - fi - ;; - "") - if [[ -n "${ZEBRA_CONF_PATH}" ]]; then - exec zebrad -c "${ZEBRA_CONF_PATH}" || { echo "Execution with custom configuration failed"; exit 1; } - else - exec zebrad || { echo "Execution with default configuration failed"; exit 1; } - fi - ;; - *) - if [[ -n "${ENTRYPOINT_FEATURES}" ]]; then - # Validate the test variables - # For these tests, we activate the test features to avoid recompiling `zebrad`, - # but we don't actually run any gRPC tests. - if [[ "${RUN_ALL_TESTS}" -eq "1" ]]; then - # Run all the available tests for the current environment. - # If the lightwalletd environmental variables are set, we will also run those tests. - cargo test --locked --release --features "${ENTRYPOINT_FEATURES}" --workspace -- --nocapture --include-ignored + --* | -*) + if [[ -n "${ZEBRA_CONF_PATH}" ]]; then + exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" || { echo "Execution with custom configuration failed"; exit 1; } + else + exec zebrad "$@" || { echo "Execution failed"; exit 1; } + fi + ;; + "") + if [[ -n "${ZEBRA_CONF_PATH}" ]]; then + exec zebrad -c "${ZEBRA_CONF_PATH}" || { echo "Execution with custom configuration failed"; exit 1; } + else + exec zebrad || { echo "Execution with default configuration failed"; exit 1; } + fi + ;; + *) + if [[ -n "${ENTRYPOINT_FEATURES}" ]]; then + # Validate the test variables + # For these tests, we activate the test features to avoid recompiling `zebrad`, + # but we don't actually run any gRPC tests. + if [[ "${RUN_ALL_TESTS}" -eq "1" ]]; then + # Run all the available tests for the current environment. + # If the lightwalletd environmental variables are set, we will also run those tests. + cargo test --locked --release --features "${ENTRYPOINT_FEATURES}" --workspace -- --nocapture --include-ignored + + # For these tests, we activate the gRPC feature to avoid recompiling `zebrad`, + # but we don't actually run any gRPC tests. + elif [[ -n "${FULL_SYNC_MAINNET_TIMEOUT_MINUTES}" ]]; then + # Run a Zebra full sync test on mainnet. + run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_mainnet" + # List directory generated by test + # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows + list_directory "/zebrad-cache" + + elif [[ -n "${FULL_SYNC_TESTNET_TIMEOUT_MINUTES}" ]]; then + # Run a Zebra full sync test on testnet. + run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_testnet" + # List directory generated by test + # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows + list_directory "/zebrad-cache" - # For these tests, we activate the gRPC feature to avoid recompiling `zebrad`, - # but we don't actually run any gRPC tests. - elif [[ -n "${FULL_SYNC_MAINNET_TIMEOUT_MINUTES}" ]]; then - # Run a Zebra full sync test on mainnet. - run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_mainnet" - # List directory generated by test - # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows - list_directory "/zebrad-cache" - elif [[ -n "${FULL_SYNC_TESTNET_TIMEOUT_MINUTES}" ]]; then - # Run a Zebra full sync test on testnet. - run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_testnet" - # List directory generated by test - # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows - list_directory "/zebrad-cache" - elif [[ "${TEST_DISK_REBUILD}" -eq "1" ]]; then - # Run a Zebra sync up to the mandatory checkpoint. - # - # TODO: use environmental variables instead of Rust features (part of #2995) - run_cargo_test "test_sync_to_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_to_mandatory_checkpoint_${NETWORK,,}" - # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows - list_directory "/zebrad-cache" - elif [[ "${TEST_UPDATE_SYNC}" -eq "1" ]]; then - # Run a Zebra sync starting at the cached tip, and syncing to the latest tip. - # - # List directory used by test - list_directory "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "zebrad_update_sync" - elif [[ "${TEST_CHECKPOINT_SYNC}" -eq "1" ]]; then - # Run a Zebra sync starting at the cached mandatory checkpoint, and syncing past it. - # - # List directory used by test - # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows - list_directory "/zebrad-cache" - # TODO: use environmental variables instead of Rust features (part of #2995) - run_cargo_test "test_sync_to_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_past_mandatory_checkpoint_${NETWORK,,}" + elif [[ "${TEST_DISK_REBUILD}" -eq "1" ]]; then + # Run a Zebra sync up to the mandatory checkpoint. + # + # TODO: use environmental variables instead of Rust features (part of #2995) + run_cargo_test "test_sync_to_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_to_mandatory_checkpoint_${NETWORK,,}" + # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows + list_directory "/zebrad-cache" - elif [[ "${GENERATE_CHECKPOINTS_MAINNET}" -eq "1" ]]; then - # Generate checkpoints after syncing Zebra from a cached state on mainnet. - # - # TODO: disable or filter out logs like: - # test generate_checkpoints_mainnet has been running for over 60 seconds - # - # List directory used by test - list_directory "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_mainnet" - elif [[ "${GENERATE_CHECKPOINTS_TESTNET}" -eq "1" ]]; then - # Generate checkpoints after syncing Zebra on testnet. - # - # This test might fail if testnet is unstable. - # - # List directory used by test - list_directory "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_testnet" + elif [[ "${TEST_UPDATE_SYNC}" -eq "1" ]]; then + # Run a Zebra sync starting at the cached tip, and syncing to the latest tip. + # + # List directory used by test + list_directory "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "zebrad_update_sync" - elif [[ "${TEST_LWD_RPC_CALL}" -eq "1" ]]; then - # Starting at a cached Zebra tip, test a JSON-RPC call to Zebra. - list_directory "${ZEBRA_CACHED_STATE_DIR}" - # Run both the fully synced RPC test and the subtree snapshot test, one test at a time. - # Since these tests use the same cached state, a state problem in the first test can fail the second test. - run_cargo_test "${ENTRYPOINT_FEATURES}" "fully_synced_rpc_" - elif [[ "${TEST_LWD_FULL_SYNC}" -eq "1" ]]; then - # Starting at a cached Zebra tip, run a lightwalletd sync to tip. - list_directory "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_full_sync" - list_directory "${LIGHTWALLETD_DATA_DIR}/db" - elif [[ "${TEST_LWD_UPDATE_SYNC}" -eq "1" ]]; then - # Starting with a cached Zebra and lightwalletd tip, run a quick update sync. - list_directory "${ZEBRA_CACHED_STATE_DIR}" - list_directory "${LIGHTWALLETD_DATA_DIR}/db" - run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_update_sync" + elif [[ "${TEST_CHECKPOINT_SYNC}" -eq "1" ]]; then + # Run a Zebra sync starting at the cached mandatory checkpoint, and syncing past it. + # + # List directory used by test + # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows + list_directory "/zebrad-cache" + # TODO: use environmental variables instead of Rust features (part of #2995) + run_cargo_test "test_sync_to_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_past_mandatory_checkpoint_${NETWORK,,}" - # These tests actually use gRPC. - elif [[ "${TEST_LWD_GRPC}" -eq "1" ]]; then - # Starting with a cached Zebra and lightwalletd tip, test all gRPC calls to lightwalletd, which calls Zebra. - list_directory "${ZEBRA_CACHED_STATE_DIR}" - list_directory "${LIGHTWALLETD_DATA_DIR}/db" - run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_wallet_grpc_tests" - elif [[ "${TEST_LWD_TRANSACTIONS}" -eq "1" ]]; then - # Starting with a cached Zebra and lightwalletd tip, test sending transactions gRPC call to lightwalletd, which calls Zebra. - list_directory "${ZEBRA_CACHED_STATE_DIR}" - list_directory "${LIGHTWALLETD_DATA_DIR}/db" - run_cargo_test "${ENTRYPOINT_FEATURES}" "sending_transactions_using_lightwalletd" + elif [[ "${GENERATE_CHECKPOINTS_MAINNET}" -eq "1" ]]; then + # Generate checkpoints after syncing Zebra from a cached state on mainnet. + # + # TODO: disable or filter out logs like: + # test generate_checkpoints_mainnet has been running for over 60 seconds + # + # List directory used by test + list_directory "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_mainnet" - # These tests use mining code, but don't use gRPC. - # We add the mining feature here because our other code needs to pass tests without it. - elif [[ "${TEST_GET_BLOCK_TEMPLATE}" -eq "1" ]]; then - # Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server. - list_directory "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "getblocktemplate-rpcs,${ENTRYPOINT_FEATURES}" "get_block_template" - elif [[ "${TEST_SUBMIT_BLOCK}" -eq "1" ]]; then - # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port. - list_directory "${ZEBRA_CACHED_STATE_DIR}" - run_cargo_test "getblocktemplate-rpcs,${ENTRYPOINT_FEATURES}" "submit_block" + elif [[ "${GENERATE_CHECKPOINTS_TESTNET}" -eq "1" ]]; then + # Generate checkpoints after syncing Zebra on testnet. + # + # This test might fail if testnet is unstable. + # + # List directory used by test + list_directory "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_testnet" - else - exec "$@" - fi - fi + elif [[ "${TEST_LWD_RPC_CALL}" -eq "1" ]]; then + # Starting at a cached Zebra tip, test a JSON-RPC call to Zebra. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + # Run both the fully synced RPC test and the subtree snapshot test, one test at a time. + # Since these tests use the same cached state, a state problem in the first test can fail the second test. + run_cargo_test "${ENTRYPOINT_FEATURES}" "fully_synced_rpc_" + + elif [[ "${TEST_LWD_FULL_SYNC}" -eq "1" ]]; then + # Starting at a cached Zebra tip, run a lightwalletd sync to tip. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_full_sync" + list_directory "${LIGHTWALLETD_DATA_DIR}/db" + + elif [[ "${TEST_LWD_UPDATE_SYNC}" -eq "1" ]]; then + # Starting with a cached Zebra and lightwalletd tip, run a quick update sync. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + list_directory "${LIGHTWALLETD_DATA_DIR}/db" + run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_update_sync" + + # These tests actually use gRPC. + elif [[ "${TEST_LWD_GRPC}" -eq "1" ]]; then + # Starting with a cached Zebra and lightwalletd tip, test all gRPC calls to lightwalletd, which calls Zebra. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + list_directory "${LIGHTWALLETD_DATA_DIR}/db" + run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_wallet_grpc_tests" + + elif [[ "${TEST_LWD_TRANSACTIONS}" -eq "1" ]]; then + # Starting with a cached Zebra and lightwalletd tip, test sending transactions gRPC call to lightwalletd, which calls Zebra. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + list_directory "${LIGHTWALLETD_DATA_DIR}/db" + run_cargo_test "${ENTRYPOINT_FEATURES}" "sending_transactions_using_lightwalletd" + + # These tests use mining code, but don't use gRPC. + # We add the mining feature here because our other code needs to pass tests without it. + elif [[ "${TEST_GET_BLOCK_TEMPLATE}" -eq "1" ]]; then + # Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "getblocktemplate-rpcs,${ENTRYPOINT_FEATURES}" "get_block_template" + + elif [[ "${TEST_SUBMIT_BLOCK}" -eq "1" ]]; then + # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port. + list_directory "${ZEBRA_CACHED_STATE_DIR}" + run_cargo_test "getblocktemplate-rpcs,${ENTRYPOINT_FEATURES}" "submit_block" + + else + exec "$@" + fi + fi esac From 1561aafe67147b7f4822b1f84acc6b4af3c3a21d Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Tue, 3 Oct 2023 21:30:37 +0100 Subject: [PATCH 03/22] fix: add missing `test-threads` to `fully_synced_rpc_` --- docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 54529dd9487..6697d8fc0ae 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -246,7 +246,7 @@ case "$1" in list_directory "${ZEBRA_CACHED_STATE_DIR}" # Run both the fully synced RPC test and the subtree snapshot test, one test at a time. # Since these tests use the same cached state, a state problem in the first test can fail the second test. - run_cargo_test "${ENTRYPOINT_FEATURES}" "fully_synced_rpc_" + run_cargo_test "${ENTRYPOINT_FEATURES}" "--test-threads 1 fully_synced_rpc_" elif [[ "${TEST_LWD_FULL_SYNC}" -eq "1" ]]; then # Starting at a cached Zebra tip, run a lightwalletd sync to tip. From 3c9679756e5c540eec169aa092f354e299142686 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Tue, 3 Oct 2023 21:58:50 +0100 Subject: [PATCH 04/22] fix: allow 3 arguments instead of 2 to handle `fully_synced_rpc_` correctly --- docker/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 6697d8fc0ae..4fb59ac8476 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -154,7 +154,7 @@ list_directory() { # Function to run cargo test run_cargo_test() { - cargo test --locked --release --features "$1" --package zebrad --test acceptance -- --nocapture --include-ignored "$2" || { echo "Cargo test failed"; exit 1; } + cargo test --locked --release --features "$1" --package zebrad --test acceptance -- --nocapture --include-ignored "$2" "$3" || { echo "Cargo test failed"; exit 1; } } case "$1" in @@ -246,7 +246,7 @@ case "$1" in list_directory "${ZEBRA_CACHED_STATE_DIR}" # Run both the fully synced RPC test and the subtree snapshot test, one test at a time. # Since these tests use the same cached state, a state problem in the first test can fail the second test. - run_cargo_test "${ENTRYPOINT_FEATURES}" "--test-threads 1 fully_synced_rpc_" + run_cargo_test "${ENTRYPOINT_FEATURES}" "--test-threads 1" "fully_synced_rpc_" elif [[ "${TEST_LWD_FULL_SYNC}" -eq "1" ]]; then # Starting at a cached Zebra tip, run a lightwalletd sync to tip. From b2fd2282eaed9262e8915512bd9965d2128d97be Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 9 Oct 2023 21:47:32 +0100 Subject: [PATCH 05/22] chore: remove extra file --- .github/workflows/ci-sync-tests-gcp.yml | 616 ------------------------ 1 file changed, 616 deletions(-) delete mode 100644 .github/workflows/ci-sync-tests-gcp.yml diff --git a/.github/workflows/ci-sync-tests-gcp.yml b/.github/workflows/ci-sync-tests-gcp.yml deleted file mode 100644 index 2b9582a3153..00000000000 --- a/.github/workflows/ci-sync-tests-gcp.yml +++ /dev/null @@ -1,616 +0,0 @@ -name: CI Docker - -# Ensures that only one workflow task will run at a time. Previous builds, if -# already in process, will get cancelled. Only the latest commit will be allowed -# to run, cancelling any workflows in between -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -on: - schedule: - # Run this job every Friday at mid-day UTC - # This is limited to the Zebra and lightwalletd Full Sync jobs - - cron: '0 12 * * 5' - - workflow_dispatch: - inputs: - network: - default: 'Mainnet' - description: 'Network to deploy: Mainnet or Testnet' - required: true - regenerate-disks: - type: boolean - default: false - description: 'Just run a Zebra checkpoint sync and update checkpoint disks' - required: true - run-full-sync: - type: boolean - default: false - description: 'Just run a Zebra full sync on `network`, and update tip disks' - required: true - run-lwd-sync: - type: boolean - default: false - description: 'Just run a lightwalletd full sync and update tip disks' - required: true - force_save_to_disk: - required: false - type: boolean - default: false - description: 'Force tests to always create a cached state disk, if they already create disks' - no_cache: - description: 'Disable the Docker cache for this build' - required: false - type: boolean - default: false - - pull_request: - paths: - # code and tests - - '**/*.rs' - # hard-coded checkpoints and proptest regressions - - '**/*.txt' - # test data snapshots - - '**/*.snap' - # dependencies - - '**/Cargo.toml' - - '**/Cargo.lock' - # configuration files - - '.cargo/config.toml' - - '**/clippy.toml' - # workflow definitions - - 'docker/**' - - '.github/workflows/continous-integration-docker.yml' - - '.github/workflows/deploy-gcp-tests.yml' - - '.github/workflows/build-docker-image.yml' - - '.github/workflows/find-cached-disks.yml' - - push: - branches: - - main - paths: - # code and tests - - '**/*.rs' - # hard-coded checkpoints and proptest regressions - - '**/*.txt' - # test data snapshots - - '**/*.snap' - # dependencies - - '**/Cargo.toml' - - '**/Cargo.lock' - # configuration files - - '.cargo/config.toml' - - '**/clippy.toml' - # workflow definitions - - 'docker/**' - - '.dockerignore' - - '.github/workflows/continous-integration-docker.yml' - - '.github/workflows/deploy-gcp-tests.yml' - - '.github/workflows/find-cached-disks.yml' - - '.github/workflows/build-docker-image.yml' - -jobs: - # to also run a job on Mergify head branches, - # add `|| (github.event_name == 'push' && startsWith(github.head_ref, 'mergify/merge-queue/'))`: - # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-your-workflow-based-on-the-head-or-base-branch-of-a-pull-request-1 - - # Check if the cached state disks used by the tests are available for the default network. - # - # The default network is mainnet unless a manually triggered workflow or repository variable - # is configured differently. - # - # The outputs for this job have the same names as the workflow outputs in find-cached-disks.yml - get-available-disks: - name: Check if cached state disks exist for ${{ inputs.network || vars.ZCASH_NETWORK }} - uses: ./.github/workflows/find-cached-disks.yml - with: - network: ${{ inputs.network || vars.ZCASH_NETWORK }} - - # Check if the cached state disks used by the tests are available for testnet. - # - # The outputs for this job have the same names as the workflow outputs in find-cached-disks.yml - # Some outputs are ignored, because we don't run those jobs on testnet. - get-available-disks-testnet: - name: Check if cached state disks exist for testnet - uses: ./.github/workflows/find-cached-disks.yml - with: - network: 'Testnet' - - # Build the docker image used by the tests. - # - # The default network in the Zebra config in the image is mainnet, unless a manually triggered - # workflow or repository variable is configured differently. Testnet jobs change that config to - # testnet when running the image. - build: - name: Build CI Docker - uses: ./.github/workflows/build-docker-image.yml - with: - dockerfile_path: ./docker/Dockerfile - dockerfile_target: tests - image_name: ${{ vars.CI_IMAGE_NAME }} - no_cache: ${{ inputs.no_cache || false }} - rust_backtrace: full - rust_lib_backtrace: full - rust_log: info - - # END TODO: make the non-cached-state tests use: - # network: ${{ inputs.network || vars.ZCASH_NETWORK }} - - # zebrad cached checkpoint state tests - - # Regenerate mandatory checkpoint Zebra cached state disks. - # - # Runs: - # - on every PR update, but only if there's no available disk matching the actual state version from constants.rs - # - on request, using workflow_dispatch with regenerate-disks - # - # Note: the output from get-available-disks should match with the caller workflow inputs - regenerate-stateful-disks: - name: Zebra checkpoint - needs: [ build, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ !fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || github.event.inputs.regenerate-disks == 'true' }} - with: - app_name: zebrad - test_id: sync-to-checkpoint - test_description: Test sync up to mandatory checkpoint - test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_DISK_REBUILD=1 -e ZEBRA_FORCE_USE_COLOR=1' - needs_zebra_state: false - saves_to_disk: true - force_save_to_disk: ${{ inputs.force_save_to_disk || false }} - disk_suffix: checkpoint - height_grep_text: 'flushing database to disk .*height.*=.*Height.*\(' - secrets: inherit - # We want to prevent multiple checkpoint syncs running at the same time, - # but we don't want to cancel running syncs on `main` if a new PR gets merged, - # because we might never get a finished sync. - # - # See the concurrency comment on the zebrad test-full-sync job for details. - concurrency: - group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.regenerate-disks == 'true') }}-regenerate-stateful-disks - cancel-in-progress: false - - # Test that Zebra syncs and fully validates a few thousand blocks from a cached mandatory checkpoint disk - # - # If the state version has changed, waits for the new cached state to be created. - # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - test-stateful-sync: - name: Zebra checkpoint update - needs: [ regenerate-stateful-disks, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || needs.regenerate-stateful-disks.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - with: - app_name: zebrad - test_id: sync-past-checkpoint - test_description: Test full validation sync from a cached state - test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_CHECKPOINT_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1' - needs_zebra_state: true - saves_to_disk: false - disk_suffix: checkpoint - secrets: inherit - - # zebrad cached tip state tests - - # Test that Zebra can run a full sync on mainnet, - # and regenerate chain tip Zebra cached state disks. - # - # This test always runs on mainnet. - # - # Runs: - # - on schedule, as defined at the top of the workflow - # - on every PR update, but only if the state version in constants.rs has no cached disk - # - in manual workflow runs, when run-full-sync is 'true' and network is 'Mainnet' - # - # Note: the output from get-available-disks should match with the caller workflow inputs - test-full-sync: - name: Zebra tip - needs: [ build, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ github.event_name == 'schedule' || !fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet') }} - with: - app_name: zebrad - test_id: full-sync-to-tip - test_description: Test a full sync up to the tip - # The value of FULL_SYNC_MAINNET_TIMEOUT_MINUTES is currently ignored. - # TODO: update the test to use {{ input.network }} instead? - test_variables: '-e NETWORK=Mainnet -e FULL_SYNC_MAINNET_TIMEOUT_MINUTES=0 -e ZEBRA_FORCE_USE_COLOR=1' - # This test runs for longer than 6 hours, so it needs multiple jobs - is_long_test: true - needs_zebra_state: false - saves_to_disk: true - force_save_to_disk: ${{ inputs.force_save_to_disk || false }} - disk_suffix: tip - height_grep_text: 'current_height.*=.*Height.*\(' - secrets: inherit - # We want to prevent multiple full zebrad syncs running at the same time, - # but we don't want to cancel running syncs on `main` if a new PR gets merged, - # because we might never get a finished sync. - # - # Instead, we let the first sync complete, then queue the latest pending sync, cancelling any syncs in between. - # (As the general workflow concurrency group just gets matched in Pull Requests, - # it has no impact on this job.) - # - # TODO: - # - allow multiple manual syncs on a branch by adding '-${{ github.run_id }}' when github.event.inputs.run-full-sync is true - concurrency: - group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.run-full-sync == 'true') }}-test-full-sync - cancel-in-progress: false - - # Test that Zebra can sync to the chain tip, using a cached Zebra tip state, - # without launching `lightwalletd`. - # - # Runs: - # - after every PR is merged to `main` - # - on every PR update - # - # If the state version has changed, waits for the new cached state to be created. - # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - test-update-sync: - name: Zebra tip update - needs: [ test-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - with: - app_name: zebrad - test_id: update-to-tip - test_description: Test syncing to tip with a Zebra tip state - test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_UPDATE_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' - needs_zebra_state: true - # update the disk on every PR, to increase CI speed - saves_to_disk: true - force_save_to_disk: ${{ inputs.force_save_to_disk || false }} - disk_suffix: tip - root_state_path: '/var/cache' - zebra_state_dir: 'zebrad-cache' - height_grep_text: 'current_height.*=.*Height.*\(' - secrets: inherit - - # zebra mainnet checkpoint generation tests - - # Test that Zebra can generate mainnet checkpoints after syncing to the chain tip, - # using a cached Zebra tip state, - # - # This test always runs on mainnet. - # - # Runs: - # - after every PR is merged to `main` - # - on every PR update - # - # If the state version has changed, waits for the new cached state to be created. - # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - generate-checkpoints-mainnet: - name: Generate checkpoints mainnet - needs: [ test-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - with: - app_name: zebrad - test_id: generate-checkpoints-mainnet - test_description: Generate Zebra checkpoints on mainnet - # TODO: update the test to use {{ input.network }} instead? - test_variables: '-e NETWORK=Mainnet -e GENERATE_CHECKPOINTS_MAINNET=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' - needs_zebra_state: true - # test-update-sync updates the disk on every PR, so we don't need to do it here - saves_to_disk: false - disk_suffix: tip - root_state_path: '/var/cache' - zebra_state_dir: 'zebrad-cache' - height_grep_text: 'current_height.*=.*Height.*\(' - secrets: inherit - - # zebra testnet checkpoint generation tests - # - # These tests will fail when testnet is unstable, they should not be required to merge. - # - # TODO: ignore failures on testnet, so any failures don't appear in the GitHub interface. - - # Test that Zebra can run a full testnet sync, and regenerate chain tip Zebra cached state disks. - # This job always runs on testnet, regardless of any inputs or variable settings. - # - # Runs: - # - on schedule, as defined at the top of the workflow - # - on every PR update, but only if the state version in constants.rs has no cached disk - # - in manual workflow runs, when run-full-sync is 'true' and network is 'Testnet' - # - # Note: the output from get-available-disks-testnet should match with the caller workflow inputs - test-full-sync-testnet: - name: Zebra tip on testnet - needs: [ build, get-available-disks-testnet ] - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ (github.event_name == 'schedule' && vars.SCHEDULE_TESTNET_FULL_SYNC == 'true') || !fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Testnet') }} - with: - app_name: zebrad - test_id: full-sync-to-tip-testnet - test_description: Test a full sync up to the tip on testnet - # The value of FULL_SYNC_TESTNET_TIMEOUT_MINUTES is currently ignored. - test_variables: '-e NETWORK=Testnet -e FULL_SYNC_TESTNET_TIMEOUT_MINUTES=0 -e ZEBRA_FORCE_USE_COLOR=1' - network: "Testnet" - # A full testnet sync could take 2-10 hours in April 2023. - # The time varies a lot due to the small number of nodes. - is_long_test: true - needs_zebra_state: false - saves_to_disk: true - force_save_to_disk: ${{ inputs.force_save_to_disk || false }} - disk_suffix: tip - height_grep_text: 'current_height.*=.*Height.*\(' - secrets: inherit - # We want to prevent multiple full zebrad syncs running at the same time, - # but we don't want to cancel running syncs on `main` if a new PR gets merged, - # because we might never get a finished sync. - # - # Instead, we let the first sync complete, then queue the latest pending sync, cancelling any syncs in between. - # (As the general workflow concurrency group just gets matched in Pull Requests, - # it has no impact on this job.) - # - # TODO: - # - allow multiple manual syncs on a branch by adding '-${{ github.run_id }}' when github.event.inputs.run-full-sync is true - concurrency: - group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.run-full-sync == 'true') }}-test-full-sync-testnet - cancel-in-progress: false - - # Test that Zebra can generate testnet checkpoints after syncing to the chain tip, - # using a cached Zebra tip state. - # - # This test always runs on testnet. - # - # Runs: - # - after every PR is merged to `main` - # - on every PR update - # - # If the state version has changed, waits for the new cached state to be created. - # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - generate-checkpoints-testnet: - name: Generate checkpoints testnet - needs: [ test-full-sync-testnet, get-available-disks-testnet ] - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk) || needs.test-full-sync-testnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - with: - app_name: zebrad - test_id: generate-checkpoints-testnet - test_description: Generate Zebra checkpoints on testnet - test_variables: '-e NETWORK=Testnet -e GENERATE_CHECKPOINTS_TESTNET=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' - network: "Testnet" - needs_zebra_state: true - # update the disk on every PR, to increase CI speed - # we don't have a test-update-sync-testnet job, so we need to update the disk here - saves_to_disk: true - force_save_to_disk: ${{ inputs.force_save_to_disk || false }} - disk_suffix: tip - root_state_path: '/var/cache' - zebra_state_dir: 'zebrad-cache' - height_grep_text: 'zebra_tip_height.*=.*Height.*\(' - secrets: inherit - - # lightwalletd cached tip state tests - - # Test full sync of lightwalletd with a Zebra tip state - # - # Runs: - # - on schedule, as defined at the top of the workflow - # - on every PR update, but only if the state version in constants.rs has no cached disk - # - in manual workflow runs, when run-lwd-sync is 'true' and network is 'Mainnet' (the network is required by the test-full-sync job) - # - # If the state version has changed, waits for the new cached state to be created. - # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - lightwalletd-full-sync: - name: lightwalletd tip - needs: [ test-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml - # Currently the lightwalletd tests only work on Mainnet - if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && (github.event_name == 'schedule' || !fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || github.event.inputs.run-lwd-sync == 'true' ) }} - with: - app_name: lightwalletd - test_id: lwd-full-sync - test_description: Test lightwalletd full sync - test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_FULL_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache' - # This test runs for longer than 6 hours, so it needs multiple jobs - is_long_test: true - needs_zebra_state: true - needs_lwd_state: false - saves_to_disk: true - force_save_to_disk: ${{ inputs.force_save_to_disk || false }} - disk_prefix: lwd-cache - disk_suffix: tip - root_state_path: '/var/cache' - zebra_state_dir: 'zebrad-cache' - lwd_state_dir: 'lwd-cache' - height_grep_text: 'Waiting for block: ' - secrets: inherit - # We want to prevent multiple lightwalletd full syncs running at the same time, - # but we don't want to cancel running syncs on `main` if a new PR gets merged, - # because we might never get a finished sync. - # - # See the concurrency comment on the zebrad test-full-sync job for details. - concurrency: - group: ${{ github.workflow }}−manual-${{ format('{0}', github.event.inputs.run-lwd-sync == 'true') }}-lightwalletd-full-sync - cancel-in-progress: false - - # Test update sync of lightwalletd with a lightwalletd and Zebra tip state - # Runs: - # - after every PR is merged to `main` - # - on every PR update - # - # If the state version has changed, waits for the new cached states to be created. - # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - lightwalletd-update-sync: - name: lightwalletd tip update - needs: [ lightwalletd-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - with: - app_name: lightwalletd - test_id: lwd-update-sync - test_description: Test lightwalletd update sync with both states - test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_UPDATE_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache' - needs_zebra_state: true - needs_lwd_state: true - saves_to_disk: true - force_save_to_disk: ${{ inputs.force_save_to_disk || false }} - disk_prefix: lwd-cache - disk_suffix: tip - root_state_path: '/var/cache' - zebra_state_dir: 'zebrad-cache' - lwd_state_dir: 'lwd-cache' - height_grep_text: 'Waiting for block: ' - secrets: inherit - - # Test that Zebra can answer a synthetic RPC call, using a cached Zebra tip state - # - # Runs: - # - after every PR is merged to `main` - # - on every PR update - # - # If the state version has changed, waits for the new cached state to be created. - # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - # - # TODO: move this job below the rest of the mainnet jobs that just use Zebra cached state - lightwalletd-rpc-test: - name: Zebra tip JSON-RPC - needs: [ test-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - with: - app_name: lightwalletd - test_id: fully-synced-rpc - test_description: Test lightwalletd RPC with a Zebra tip state - test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_RPC_CALL=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' - needs_zebra_state: true - saves_to_disk: false - disk_suffix: tip - root_state_path: '/var/cache' - zebra_state_dir: 'zebrad-cache' - secrets: inherit - - # Test that Zebra can handle a lightwalletd send transaction RPC call, using a cached Zebra tip state - # - # Runs: - # - after every PR is merged to `main` - # - on every PR update - # - # If the state version has changed, waits for the new cached states to be created. - # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - lightwalletd-transactions-test: - name: lightwalletd tip send - needs: [ lightwalletd-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - with: - app_name: lightwalletd - test_id: lwd-send-transactions - test_description: Test sending transactions via lightwalletd - test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_TRANSACTIONS=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache' - needs_zebra_state: true - needs_lwd_state: true - saves_to_disk: false - disk_suffix: tip - root_state_path: '/var/cache' - zebra_state_dir: 'zebrad-cache' - lwd_state_dir: 'lwd-cache' - secrets: inherit - - # Test that Zebra can handle gRPC wallet calls, using a cached Zebra tip state - # - # Runs: - # - after every PR is merged to `main` - # - on every PR update - # - # If the state version has changed, waits for the new cached states to be created. - # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - lightwalletd-grpc-test: - name: lightwalletd GRPC tests - needs: [ lightwalletd-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lightwalletd-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - with: - app_name: lightwalletd - test_id: lwd-grpc-wallet - test_description: Test gRPC calls via lightwalletd - test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_LWD_GRPC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache' - needs_zebra_state: true - needs_lwd_state: true - saves_to_disk: false - disk_suffix: tip - root_state_path: '/var/cache' - zebra_state_dir: 'zebrad-cache' - lwd_state_dir: 'lwd-cache' - secrets: inherit - - ## getblocktemplate-rpcs using cached Zebra state on mainnet - # - # TODO: move these below the rest of the mainnet jobs that just use Zebra cached state - - # Test that Zebra can handle a getblocktemplate RPC call, using a cached Zebra tip state - # - # Runs: - # - after every PR is merged to `main` - # - on every PR update - # - # If the state version has changed, waits for the new cached states to be created. - # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - get-block-template-test: - name: get block template - needs: [ test-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - with: - app_name: zebrad - test_id: get-block-template - test_description: Test getblocktemplate RPC method via Zebra's rpc server - test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_GET_BLOCK_TEMPLATE=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' - needs_zebra_state: true - needs_lwd_state: false - saves_to_disk: false - disk_suffix: tip - root_state_path: '/var/cache' - zebra_state_dir: 'zebrad-cache' - secrets: inherit - - # Test that Zebra can handle a submit block RPC call, using a cached Zebra tip state - # - # Runs: - # - after every PR is merged to `main` - # - on every PR update - # - # If the state version has changed, waits for the new cached states to be created. - # Otherwise, if the state rebuild was skipped, runs immediately after the build job. - submit-block-test: - name: submit block - needs: [ test-full-sync, get-available-disks ] - uses: ./.github/workflows/deploy-gcp-tests.yml - if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.test-full-sync.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} - with: - app_name: zebrad - test_id: submit-block - test_description: Test submitting blocks via Zebra's rpc server - test_variables: '-e NETWORK=${{ inputs.network || vars.ZCASH_NETWORK }} -e TEST_SUBMIT_BLOCK=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' - needs_zebra_state: true - needs_lwd_state: false - saves_to_disk: false - disk_suffix: tip - root_state_path: '/var/cache' - zebra_state_dir: 'zebrad-cache' - secrets: inherit - - failure-issue: - name: Open or update issues for main branch failures - # When a new test is added to this workflow, add it to this list. - # - # This list is for reliable tests that are run on the `main` branch. - # Testnet jobs are not in this list, because we expect testnet to fail occasionally. - needs: [ regenerate-stateful-disks, test-full-sync, lightwalletd-full-sync, test-all, test-all-getblocktemplate-rpcs, test-fake-activation-heights, test-empty-sync, test-lightwalletd-integration, test-configuration-file, test-zebra-conf-path, test-stateful-sync, test-update-sync, generate-checkpoints-mainnet, lightwalletd-update-sync, lightwalletd-rpc-test, lightwalletd-transactions-test, lightwalletd-grpc-test, get-block-template-test, submit-block-test ] - # Only open tickets for failed scheduled jobs, manual workflow runs, or `main` branch merges. - # (PR statuses are already reported in the PR jobs list, and checked by Mergify.) - # TODO: if a job times out, we want to create a ticket. Does failure() do that? Or do we need cancelled()? - if: failure() && github.event.pull_request == null - runs-on: ubuntu-latest - steps: - - uses: jayqi/failed-build-issue-action@v1 - with: - title-template: "{{refname}} branch CI failed: {{eventName}} in {{workflow}}" - # New failures open an issue with this label. - # TODO: do we want a different label for each workflow, or each kind of workflow? - label-name: S-ci-fail-auto-issue - # If there is already an open issue with this label, any failures become comments on that issue. - always-create-new-issue: false - github-token: ${{ secrets.GITHUB_TOKEN }} From 11d68817c81d64cd0d8c9bf94ca56bae62b5bf49 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 9 Oct 2023 22:00:01 +0100 Subject: [PATCH 06/22] chore: add comments to main parts of the file --- docker/entrypoint.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 4fb59ac8476..78882b95785 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,5 +1,15 @@ #!/usr/bin/env bash +# This script serves as the entrypoint for the Zebra Docker container. +# +# Description: +# This script serves as the primary entrypoint for the Docker container. Its main responsibilities include: +# 1. Environment Setup: Prepares the environment by setting various flags and parameters. +# 2. Configuration Management: Dynamically generates the `zebrad.toml` configuration file based on environment variables, ensuring the node starts with the desired settings. +# 3. Test Execution: Can run a series of tests to validate functionality based on specified environment variables. +# 4. Node Startup: Starts the node, allowing it to begin its operations. +# + # Show the commands we are executing set -x # Exit if a command fails @@ -157,6 +167,12 @@ run_cargo_test() { cargo test --locked --release --features "$1" --package zebrad --test acceptance -- --nocapture --include-ignored "$2" "$3" || { echo "Cargo test failed"; exit 1; } } +# Main Execution Logic: +# This section determines the primary operation of the script based on the first argument passed. +# - If the argument starts with '--' or '-', the script attempts to execute `zebrad` with the provided arguments. +# - If no argument is provided, the script defaults to running `zebrad` with either a custom or default configuration. +# - For any other argument, the script checks for specific environment variables to determine which tests or operations to run. +# This design allows for flexible execution, catering to various use cases like node startup, testing, or other operations. case "$1" in --* | -*) if [[ -n "${ZEBRA_CONF_PATH}" ]]; then From 172c576615af9b934ac7048e4227fa206587d50d Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Tue, 10 Oct 2023 11:23:46 +0100 Subject: [PATCH 07/22] fix(ci): just create the `$ZEBRA_CONF_PATH` if `$ENTRYPOINT_FEATURES` is not set --- docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 78882b95785..e7901baef4b 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -87,7 +87,7 @@ fi # # We disable most ports by default, so the default config is secure. # Users have to opt-in to additional functionality by setting environmental variables. -if [[ -n "${ZEBRA_CONF_PATH}" ]] && [[ ! -f "${ZEBRA_CONF_PATH}" ]]; then +if [[ -n "${ZEBRA_CONF_PATH}" ]] && [[ ! -f "${ZEBRA_CONF_PATH}" ]] && [[ -z "${ENTRYPOINT_FEATURES}" ]]; then # Create the conf path and file mkdir -p "${ZEBRA_CONF_DIR}" || { echo "Error creating directory ${ZEBRA_CONF_DIR}"; exit 1; } touch "${ZEBRA_CONF_PATH}" || { echo "Error creating file ${ZEBRA_CONF_PATH}"; exit 1; } From de9f8eacc2cf6e89f03743ca7897db54963a23ca Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Tue, 10 Oct 2023 12:00:20 +0100 Subject: [PATCH 08/22] fix(entrypoint): just print ${ZEBRA_CONF_PATH} if exists --- docker/entrypoint.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index e7901baef4b..f09966527a9 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -152,8 +152,11 @@ EOF fi fi -echo "Using zebrad.toml:" -cat "${ZEBRA_CONF_PATH}" +if [[ -n "${ZEBRA_CONF_PATH}" ]]; then + # Print the config file + echo "Using zebrad.toml:" + cat "${ZEBRA_CONF_PATH}" +fi # Function to list directory list_directory() { From b8cafde11575f09f2ce596bd836656ce6d02f4b7 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Tue, 10 Oct 2023 15:20:03 +0100 Subject: [PATCH 09/22] fix: missing condition --- docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index f09966527a9..af292c052fe 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -152,7 +152,7 @@ EOF fi fi -if [[ -n "${ZEBRA_CONF_PATH}" ]]; then +if [[ -n "${ZEBRA_CONF_PATH}" ]] && [[ -z "${ENTRYPOINT_FEATURES}" ]]; then # Print the config file echo "Using zebrad.toml:" cat "${ZEBRA_CONF_PATH}" From 60446c06be89117312fda07d6c58c3da770d93c1 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Tue, 10 Oct 2023 23:13:53 +0100 Subject: [PATCH 10/22] ref: handle tests better if `$ENTRYPOINT_FEATURES` is set --- docker/entrypoint.sh | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index af292c052fe..6c640a49f42 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -172,23 +172,20 @@ run_cargo_test() { # Main Execution Logic: # This section determines the primary operation of the script based on the first argument passed. -# - If the argument starts with '--' or '-', the script attempts to execute `zebrad` with the provided arguments. -# - If no argument is provided, the script defaults to running `zebrad` with either a custom or default configuration. -# - For any other argument, the script checks for specific environment variables to determine which tests or operations to run. +# - If the ENTRYPOINT_FEATURES environment variable is not set: +# - If the argument starts with '--' or '-', the script attempts to execute `zebrad` with the provided arguments. +# - If no argument is provided, the script defaults to running `zebrad` with either a custom or default configuration. +# - If the ENTRYPOINT_FEATURES environment variable is set: +# - The script checks for specific environment variables to determine which tests or operations to run. # This design allows for flexible execution, catering to various use cases like node startup, testing, or other operations. case "$1" in - --* | -*) - if [[ -n "${ZEBRA_CONF_PATH}" ]]; then - exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" || { echo "Execution with custom configuration failed"; exit 1; } - else - exec zebrad "$@" || { echo "Execution failed"; exit 1; } - fi - ;; - "") - if [[ -n "${ZEBRA_CONF_PATH}" ]]; then - exec zebrad -c "${ZEBRA_CONF_PATH}" || { echo "Execution with custom configuration failed"; exit 1; } - else - exec zebrad || { echo "Execution with default configuration failed"; exit 1; } + --* | -* | "") + if [[ -z "${ENTRYPOINT_FEATURES}" ]]; then + if [[ -n "${ZEBRA_CONF_PATH}" ]]; then + exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" || { echo "Execution with custom configuration failed"; exit 1; } + else + exec zebrad "$@" || { echo "Execution failed"; exit 1; } + fi fi ;; *) From 76e1f90458f41c70098cbd06281cf6aa6f5c7398 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 11 Oct 2023 11:30:44 +0100 Subject: [PATCH 11/22] fix(ci): We just want `ZEBRA_CONF_PATH` to be set in the `release` image --- docker/Dockerfile | 4 ++++ docker/entrypoint.sh | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6987354662a..9fb70ccc7ca 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -189,6 +189,10 @@ RUN apt-get update && \ ARG FEATURES ENV FEATURES=${FEATURES} +# Path and name of the config file +ENV ZEBRA_CONF_DIR=/etc/zebrad +ENV ZEBRA_CONF_FILE=zebrad.toml + # Expose configured ports EXPOSE 8233 18233 diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 6c640a49f42..be28de77700 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -22,16 +22,16 @@ set -o pipefail # These variables are used to run the Zebra node. #### -# Set this to change the default cached state directory -# Path and name of the config file -: "${ZEBRA_CONF_DIR:=/etc/zebrad}" -: "${ZEBRA_CONF_FILE:=zebrad.toml}" +# Path and name of the config file. These two have defaults set in the Dockerfile. +: "${ZEBRA_CONF_DIR:=}" +: "${ZEBRA_CONF_FILE:=}" # [network] : "${NETWORK:=Mainnet}" : "${ZEBRA_LISTEN_ADDR:=0.0.0.0}" # [consensus] : "${ZEBRA_CHECKPOINT_SYNC:=true}" # [state] +# Set this to change the default cached state directory : "${ZEBRA_CACHED_STATE_DIR:=/var/cache/zebrad-cache}" # [metrics] : "${METRICS_ENDPOINT_ADDR:=0.0.0.0}" From a13831c95b35d3364941c510915d2c4ecd948a05 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 11 Oct 2023 14:08:34 +0100 Subject: [PATCH 12/22] fix(entrypoint): fix the overall `case` logic --- docker/Dockerfile | 1 + docker/entrypoint.sh | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9fb70ccc7ca..2b4c770e328 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -199,3 +199,4 @@ EXPOSE 8233 18233 # Update the config file based on the Docker run variables, # and launch zebrad with it ENTRYPOINT [ "/entrypoint.sh" ] +CMD ["zebrad"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index be28de77700..0341a24d18f 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -171,15 +171,17 @@ run_cargo_test() { } # Main Execution Logic: -# This section determines the primary operation of the script based on the first argument passed. -# - If the ENTRYPOINT_FEATURES environment variable is not set: -# - If the argument starts with '--' or '-', the script attempts to execute `zebrad` with the provided arguments. -# - If no argument is provided, the script defaults to running `zebrad` with either a custom or default configuration. -# - If the ENTRYPOINT_FEATURES environment variable is set: -# - The script checks for specific environment variables to determine which tests or operations to run. -# This design allows for flexible execution, catering to various use cases like node startup, testing, or other operations. +# This script orchestrates the execution flow based on the provided arguments and environment variables. +# - If "$1" is '--', '-', or 'zebrad', the script processes the subsequent arguments for the 'zebrad' command. +# - If ENTRYPOINT_FEATURES is unset, it checks for ZEBRA_CONF_PATH. If set, 'zebrad' runs with this custom configuration; otherwise, it runs with the provided arguments. +# - If "$1" is an empty string and ENTRYPOINT_FEATURES is set, the script enters the testing phase, checking various environment variables to determine the specific tests to run. +# - Different tests or operations are triggered based on the respective conditions being met. +# - If "$1" doesn't match any of the above, it's assumed to be a command, which is executed directly. +# This structure ensures a flexible execution strategy, accommodating various scenarios such as custom configurations, different testing phases, or direct command execution. + case "$1" in - --* | -* | "") + --* | -* | zebrad) + shift if [[ -z "${ENTRYPOINT_FEATURES}" ]]; then if [[ -n "${ZEBRA_CONF_PATH}" ]]; then exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" || { echo "Execution with custom configuration failed"; exit 1; } @@ -188,7 +190,7 @@ case "$1" in fi fi ;; - *) + "") if [[ -n "${ENTRYPOINT_FEATURES}" ]]; then # Validate the test variables # For these tests, we activate the test features to avoid recompiling `zebrad`, @@ -305,4 +307,8 @@ case "$1" in exec "$@" fi fi + ;; + *) + exec "$@" + ;; esac From 4f938871633f25cac3e14f3d5d7b8ac287dac17e Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 11 Oct 2023 14:50:28 +0100 Subject: [PATCH 13/22] fix(ci): allos to run with custom config in CI image --- docker/entrypoint.sh | 66 ++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 0341a24d18f..f7289eaf0f6 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -159,10 +159,24 @@ if [[ -n "${ZEBRA_CONF_PATH}" ]] && [[ -z "${ENTRYPOINT_FEATURES}" ]]; then fi # Function to list directory -list_directory() { +check_directory_files() { local dir="$1" - # Using find instead of ls to better handle non-alphanumeric filenames, which also requires `-exec +` - find "${dir}" -type f -print0 | xargs -0 -I {} ls -lh "{}" || { echo "No files in ${dir}"; find "${dir}" -type d -exec ls -lhR {} + | head -50 || echo "No ${dir} directory"; } + # Check if the directory exists + if [[ -d "${dir}" ]]; then + # Check if the directory contains any files + if find "${dir}" -type f | read -r; then + # Directory exists and has files, so we continue + : + else + # Directory exists but is empty, print message and exit with status 1 + echo "Directory ${dir} exists but contains no files." + exit 1 + fi + else + # Directory doesn't exist, print message and exit with status 1 + echo "Directory ${dir} does not exist." + exit 1 + fi } # Function to run cargo test @@ -182,12 +196,10 @@ run_cargo_test() { case "$1" in --* | -* | zebrad) shift - if [[ -z "${ENTRYPOINT_FEATURES}" ]]; then - if [[ -n "${ZEBRA_CONF_PATH}" ]]; then - exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" || { echo "Execution with custom configuration failed"; exit 1; } - else - exec zebrad "$@" || { echo "Execution failed"; exit 1; } - fi + if [[ -n "${ZEBRA_CONF_PATH}" ]]; then + exec zebrad -c "${ZEBRA_CONF_PATH}" "$@" || { echo "Execution with custom configuration failed"; exit 1; } + else + exec zebrad "$@" || { echo "Execution failed"; exit 1; } fi ;; "") @@ -207,14 +219,14 @@ case "$1" in run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_mainnet" # List directory generated by test # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows - list_directory "/zebrad-cache" + check_directory_files "/zebrad-cache" elif [[ -n "${FULL_SYNC_TESTNET_TIMEOUT_MINUTES}" ]]; then # Run a Zebra full sync test on testnet. run_cargo_test "${ENTRYPOINT_FEATURES}" "full_sync_testnet" # List directory generated by test # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows - list_directory "/zebrad-cache" + check_directory_files "/zebrad-cache" elif [[ "${TEST_DISK_REBUILD}" -eq "1" ]]; then # Run a Zebra sync up to the mandatory checkpoint. @@ -222,13 +234,13 @@ case "$1" in # TODO: use environmental variables instead of Rust features (part of #2995) run_cargo_test "test_sync_to_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_to_mandatory_checkpoint_${NETWORK,,}" # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows - list_directory "/zebrad-cache" + check_directory_files "/zebrad-cache" elif [[ "${TEST_UPDATE_SYNC}" -eq "1" ]]; then # Run a Zebra sync starting at the cached tip, and syncing to the latest tip. # # List directory used by test - list_directory "${ZEBRA_CACHED_STATE_DIR}" + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" run_cargo_test "${ENTRYPOINT_FEATURES}" "zebrad_update_sync" elif [[ "${TEST_CHECKPOINT_SYNC}" -eq "1" ]]; then @@ -236,7 +248,7 @@ case "$1" in # # List directory used by test # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows - list_directory "/zebrad-cache" + check_directory_files "/zebrad-cache" # TODO: use environmental variables instead of Rust features (part of #2995) run_cargo_test "test_sync_to_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_past_mandatory_checkpoint_${NETWORK,,}" @@ -247,7 +259,7 @@ case "$1" in # test generate_checkpoints_mainnet has been running for over 60 seconds # # List directory used by test - list_directory "${ZEBRA_CACHED_STATE_DIR}" + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_mainnet" elif [[ "${GENERATE_CHECKPOINTS_TESTNET}" -eq "1" ]]; then @@ -256,51 +268,51 @@ case "$1" in # This test might fail if testnet is unstable. # # List directory used by test - list_directory "${ZEBRA_CACHED_STATE_DIR}" + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" run_cargo_test "${ENTRYPOINT_FEATURES}" "generate_checkpoints_testnet" elif [[ "${TEST_LWD_RPC_CALL}" -eq "1" ]]; then # Starting at a cached Zebra tip, test a JSON-RPC call to Zebra. - list_directory "${ZEBRA_CACHED_STATE_DIR}" + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" # Run both the fully synced RPC test and the subtree snapshot test, one test at a time. # Since these tests use the same cached state, a state problem in the first test can fail the second test. run_cargo_test "${ENTRYPOINT_FEATURES}" "--test-threads 1" "fully_synced_rpc_" elif [[ "${TEST_LWD_FULL_SYNC}" -eq "1" ]]; then # Starting at a cached Zebra tip, run a lightwalletd sync to tip. - list_directory "${ZEBRA_CACHED_STATE_DIR}" + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_full_sync" - list_directory "${LIGHTWALLETD_DATA_DIR}/db" + check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" elif [[ "${TEST_LWD_UPDATE_SYNC}" -eq "1" ]]; then # Starting with a cached Zebra and lightwalletd tip, run a quick update sync. - list_directory "${ZEBRA_CACHED_STATE_DIR}" - list_directory "${LIGHTWALLETD_DATA_DIR}/db" + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_update_sync" # These tests actually use gRPC. elif [[ "${TEST_LWD_GRPC}" -eq "1" ]]; then # Starting with a cached Zebra and lightwalletd tip, test all gRPC calls to lightwalletd, which calls Zebra. - list_directory "${ZEBRA_CACHED_STATE_DIR}" - list_directory "${LIGHTWALLETD_DATA_DIR}/db" + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" run_cargo_test "${ENTRYPOINT_FEATURES}" "lightwalletd_wallet_grpc_tests" elif [[ "${TEST_LWD_TRANSACTIONS}" -eq "1" ]]; then # Starting with a cached Zebra and lightwalletd tip, test sending transactions gRPC call to lightwalletd, which calls Zebra. - list_directory "${ZEBRA_CACHED_STATE_DIR}" - list_directory "${LIGHTWALLETD_DATA_DIR}/db" + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" + check_directory_files "${LIGHTWALLETD_DATA_DIR}/db" run_cargo_test "${ENTRYPOINT_FEATURES}" "sending_transactions_using_lightwalletd" # These tests use mining code, but don't use gRPC. # We add the mining feature here because our other code needs to pass tests without it. elif [[ "${TEST_GET_BLOCK_TEMPLATE}" -eq "1" ]]; then # Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server. - list_directory "${ZEBRA_CACHED_STATE_DIR}" + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" run_cargo_test "getblocktemplate-rpcs,${ENTRYPOINT_FEATURES}" "get_block_template" elif [[ "${TEST_SUBMIT_BLOCK}" -eq "1" ]]; then # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port. - list_directory "${ZEBRA_CACHED_STATE_DIR}" + check_directory_files "${ZEBRA_CACHED_STATE_DIR}" run_cargo_test "getblocktemplate-rpcs,${ENTRYPOINT_FEATURES}" "submit_block" else From 86302b53d8e50a31aab1fb43cd82be4b693a8c5d Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 11 Oct 2023 15:34:09 +0100 Subject: [PATCH 14/22] fix(ci): more edgecases --- .github/workflows/continous-integration-docker.yml | 2 +- docker/entrypoint.sh | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index aeafb9a0bea..0abb7d64753 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -316,7 +316,7 @@ jobs: run: | set -ex docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} - docker run -e NETWORK --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} -c $ZEBRA_CONF_PATH start + docker run -e NETWORK --detach -e ZEBRA_CONF_PATH --name variable-conf-tests -t ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} zebrad start EXIT_STATUS=$(docker logs --tail all --follow variable-conf-tests 2>&1 | grep -q --extended-regexp --max-count=1 -e 'v1.0.0-rc.2.toml'; echo $?; ) docker stop variable-conf-tests docker logs variable-conf-tests diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index f7289eaf0f6..fa563dc43be 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -163,13 +163,13 @@ check_directory_files() { local dir="$1" # Check if the directory exists if [[ -d "${dir}" ]]; then - # Check if the directory contains any files - if find "${dir}" -type f | read -r; then - # Directory exists and has files, so we continue + # Check if any subdirectory contains any files + if find "${dir}" -mindepth 2 -type f | read -r; then + # Subdirectories exist and have files, so we continue : else - # Directory exists but is empty, print message and exit with status 1 - echo "Directory ${dir} exists but contains no files." + # Subdirectories exists but are empty, print message and exit with status 1 + echo "Subdirectories in ${dir} exists but contain no files." exit 1 fi else From 0a8f335f4591394401f8c55bcf834594ec4f4762 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 11 Oct 2023 16:18:38 +0100 Subject: [PATCH 15/22] fix: we don't need to find files, but subdirectories --- docker/entrypoint.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index fa563dc43be..665d776be04 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -163,13 +163,13 @@ check_directory_files() { local dir="$1" # Check if the directory exists if [[ -d "${dir}" ]]; then - # Check if any subdirectory contains any files - if find "${dir}" -mindepth 2 -type f | read -r; then - # Subdirectories exist and have files, so we continue + # Check if there are any subdirectories + if find "${dir}" -mindepth 1 -type d | read -r; then + # Subdirectories exist, so we continue : else - # Subdirectories exists but are empty, print message and exit with status 1 - echo "Subdirectories in ${dir} exists but contain no files." + # No subdirectories, print message and exit with status 1 + echo "No subdirectories found in ${dir}." exit 1 fi else From fba1af4d7d2395100f8b9587bf7a972e18bddf33 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 11 Oct 2023 16:40:15 +0100 Subject: [PATCH 16/22] fix(ci): handle Signal Forwarding and exit codes for `cargo` --- docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 665d776be04..cf966b3ebb8 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -210,7 +210,7 @@ case "$1" in if [[ "${RUN_ALL_TESTS}" -eq "1" ]]; then # Run all the available tests for the current environment. # If the lightwalletd environmental variables are set, we will also run those tests. - cargo test --locked --release --features "${ENTRYPOINT_FEATURES}" --workspace -- --nocapture --include-ignored + exec cargo test --locked --release --features "${ENTRYPOINT_FEATURES}" --workspace -- --nocapture --include-ignored # For these tests, we activate the gRPC feature to avoid recompiling `zebrad`, # but we don't actually run any gRPC tests. From 998687f1089df351b33595a9fdac550f385e0113 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 11 Oct 2023 16:59:01 +0100 Subject: [PATCH 17/22] fix(ci): parse `fully_synced_rpc_` test correctly --- docker/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index cf966b3ebb8..7e1c5f43b08 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -181,7 +181,7 @@ check_directory_files() { # Function to run cargo test run_cargo_test() { - cargo test --locked --release --features "$1" --package zebrad --test acceptance -- --nocapture --include-ignored "$2" "$3" || { echo "Cargo test failed"; exit 1; } + cargo test --locked --release --features "$1" --package zebrad --test acceptance -- --nocapture --include-ignored "$2" "$3" "$4" || { echo "Cargo test failed"; exit 1; } } # Main Execution Logic: @@ -276,7 +276,7 @@ case "$1" in check_directory_files "${ZEBRA_CACHED_STATE_DIR}" # Run both the fully synced RPC test and the subtree snapshot test, one test at a time. # Since these tests use the same cached state, a state problem in the first test can fail the second test. - run_cargo_test "${ENTRYPOINT_FEATURES}" "--test-threads 1" "fully_synced_rpc_" + run_cargo_test "${ENTRYPOINT_FEATURES}" "--test-threads" "1" "fully_synced_rpc_" elif [[ "${TEST_LWD_FULL_SYNC}" -eq "1" ]]; then # Starting at a cached Zebra tip, run a lightwalletd sync to tip. From efce396719e2dcdda030cd12d4429dc40903ae14 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 11 Oct 2023 17:34:13 +0100 Subject: [PATCH 18/22] chore: add missing cache dir variable for LWD --- docker/entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 7e1c5f43b08..01e8dbd2c1c 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -33,6 +33,7 @@ set -o pipefail # [state] # Set this to change the default cached state directory : "${ZEBRA_CACHED_STATE_DIR:=/var/cache/zebrad-cache}" +: "${LIGHTWALLETD_DATA_DIR:=/var/cache/lwd-cache}" # [metrics] : "${METRICS_ENDPOINT_ADDR:=0.0.0.0}" : "${METRICS_ENDPOINT_PORT:=9999}" From 8fb43ca93499008f46e061915f7a5b38359d94bc Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 11 Oct 2023 17:45:30 +0100 Subject: [PATCH 19/22] fix(entrypoint): handle an arbitrary number of arguments --- docker/entrypoint.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 01e8dbd2c1c..2b716edd66a 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -180,9 +180,24 @@ check_directory_files() { fi } -# Function to run cargo test +# Function to run cargo test with an arbitrary number of arguments run_cargo_test() { - cargo test --locked --release --features "$1" --package zebrad --test acceptance -- --nocapture --include-ignored "$2" "$3" "$4" || { echo "Cargo test failed"; exit 1; } + # Start constructing the command + local cmd="exec cargo test --locked --release --features $1 --package zebrad --test acceptance -- --nocapture --include-ignored" + + # Shift the first argument, as it's already included in the cmd + shift + + # Loop through the remaining arguments + for arg in "$@"; do + if [[ -n ${arg} ]]; then + # If the argument is non-empty, add it to the command + cmd+=" ${arg}" + fi + done + + # Run the command + eval "${cmd}" || { echo "Cargo test failed"; exit 1; } } # Main Execution Logic: From da834f5f5d45deea60818e4734f1312ab46dc44b Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 11 Oct 2023 18:10:13 +0100 Subject: [PATCH 20/22] fix(entrypoint): handle features list --- docker/entrypoint.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 2b716edd66a..31604ea3a85 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -182,8 +182,8 @@ check_directory_files() { # Function to run cargo test with an arbitrary number of arguments run_cargo_test() { - # Start constructing the command - local cmd="exec cargo test --locked --release --features $1 --package zebrad --test acceptance -- --nocapture --include-ignored" + # Start constructing the command, ensuring that $1 is enclosed in single quotes as it's a feature list + local cmd="exec cargo test --locked --release --features '$1' --package zebrad --test acceptance -- --nocapture --include-ignored" # Shift the first argument, as it's already included in the cmd shift @@ -196,7 +196,7 @@ run_cargo_test() { fi done - # Run the command + # Run the command using eval, this will replace the current process with the cargo command eval "${cmd}" || { echo "Cargo test failed"; exit 1; } } From 3fe1031f08ed55bde2858e2c1606104b1c2d90f8 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 11 Oct 2023 18:46:05 +0100 Subject: [PATCH 21/22] fix(entrypoint): typo --- docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 31604ea3a85..20e297bf2d8 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -266,7 +266,7 @@ case "$1" in # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows check_directory_files "/zebrad-cache" # TODO: use environmental variables instead of Rust features (part of #2995) - run_cargo_test "test_sync_to_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_past_mandatory_checkpoint_${NETWORK,,}" + run_cargo_test "sync_past_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_past_mandatory_checkpoint_${NETWORK,,}" elif [[ "${GENERATE_CHECKPOINTS_MAINNET}" -eq "1" ]]; then # Generate checkpoints after syncing Zebra from a cached state on mainnet. From a784f34fbb28edc08ebf64a0daf4a323d81f2d82 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 11 Oct 2023 19:15:54 +0100 Subject: [PATCH 22/22] chore: typo --- docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 20e297bf2d8..66b8f586586 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -266,7 +266,7 @@ case "$1" in # TODO: replace with ${ZEBRA_CACHED_STATE_DIR} in Rust and workflows check_directory_files "/zebrad-cache" # TODO: use environmental variables instead of Rust features (part of #2995) - run_cargo_test "sync_past_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_past_mandatory_checkpoint_${NETWORK,,}" + run_cargo_test "test_sync_past_mandatory_checkpoint_${NETWORK,,},${ENTRYPOINT_FEATURES}" "sync_past_mandatory_checkpoint_${NETWORK,,}" elif [[ "${GENERATE_CHECKPOINTS_MAINNET}" -eq "1" ]]; then # Generate checkpoints after syncing Zebra from a cached state on mainnet.