diff --git a/.cargo/config.toml b/.cargo/config.toml index 4e644a328ad..0d373727220 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,4 +1,6 @@ +# Zebra cargo configuration +# Flags that apply to all Zebra crates and configurations [target.'cfg(all())'] rustflags = [ # Zebra standard lints for Rust 1.58+ @@ -47,6 +49,13 @@ rustflags = [ # Code styles we want to accept "-Aclippy::try_err", + # Links in public docs can point to private items. + "-Arustdoc::private_intra_doc_links", + + # Panics + "-Wclippy::fallible_impl_from", + "-Wclippy::unwrap_in_result", + # TODOs: # `cargo fix` might help do these fixes, # or add a config.toml to sub-directories which should allow these lints, diff --git a/.github/PULL_REQUEST_TEMPLATE/release-checklist.md b/.github/PULL_REQUEST_TEMPLATE/release-checklist.md index 3d137f43e8d..092ead62e41 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release-checklist.md +++ b/.github/PULL_REQUEST_TEMPLATE/release-checklist.md @@ -11,8 +11,19 @@ assignees: '' ### Which Crates to Increment -To check if any of the top-level crates need version increments, go to the zebra GitHub code page: https://github.com/ZcashFoundation/zebra and use the last modified dates of each crate. Alternatively you can use the github compare tool and check the `main` branch against the last tag ([Example](https://github.com/ZcashFoundation/zebra/compare/v1.0.0-alpha.15...main)). `git diff --stat origin/main` is also useful to see what's changed. - +To check if any of the top-level crates need version increments: +1. Go to the zebra GitHub code page: https://github.com/ZcashFoundation/zebra +2. Check if the last commit was a Zebra version bump, or something else + +
+ +Alternatively you can: +- Use the github compare tool and check the `main` branch against the last tag ([Example](https://github.com/ZcashFoundation/zebra/compare/v1.0.0-alpha.15...main)) +- Use `git diff --stat origin/main` + +
+ +Once you know which crates have changed: - [ ] Increment the crates that have new commits since the last version update - [ ] Increment any crates that depend on crates that have changed - [ ] Keep a list of the crates that haven't been incremented, to include in the PR @@ -21,7 +32,9 @@ To check if any of the top-level crates need version increments, go to the zebra Zebra follows [semantic versioning](https://semver.org). -Semantic versions look like: `MAJOR`.`MINOR`.`PATCH[`-`TAG`.`PRE-RELEASE]` +Semantic versions look like: MAJOR`.`MINOR`.`PATCH[`-`TAG`.`PRE-RELEASE] + +
#### Pre-Release Crates @@ -46,6 +59,17 @@ Increment the first version component in this list, and reset the other componen 3. PATCH versions for bug fixes * includes dependency updates that don't impact the public API +### Reviewing Version Bumps + +Check for missed changes by going to: +`https://github.com/ZcashFoundation/zebra/tree//` +Where `` is the hash of the last commit in the version bump PR. + +If any Zebra or Tower crates have commit messages that are **not** a version bump, we have missed an update. +Also check for crates that depend on crates that have changed. They should get a version bump as well. + +
+ ### Version Locations Once you know which versions you want to increment, you can find them in the: @@ -62,35 +86,29 @@ You can use `fastmod` to interactively find and replace versions. For example, you can do something like: ``` -fastmod --extensions rs,toml,md --fixed-strings '1.0.0-alpha.12' '1.0.0-alpha.13' -fastmod --extensions rs,toml,md --fixed-strings '0.2.9' '0.2.10' tower-batch -fastmod --extensions rs,toml,md --fixed-strings '0.2.8' '0.2.9' tower-fallback +fastmod --extensions rs,toml,md --fixed-strings '1.0.0-beta.11' '1.0.0-beta.12' +fastmod --extensions rs,toml,md --fixed-strings '0.2.26' '0.2.27' tower-batch tower-fallback ``` -### Reviewing Version Bumps - -Check for missed changes by going to: -`https://github.com/ZcashFoundation/zebra/tree//` -Where `` is the hash of the last commit in the version bump PR. - -If any Zebra or Tower crates have commit messages that are **not** a version bump, we have missed an update. - -Also check for crates that depend on crates that have changed. They should get a version bump as well. +If you use `fastmod`, don't update versions in `CHANGELOG.md`. ## README -As we resolve various outstanding known issues and implement new functionality with each release, we should double check the README for any necessary updates. +We should update the README to: +- [ ] Remove any "Known Issues" that have been fixed +- [ ] Update the "Build and Run Instructions" with any new dependencies. + Check for changes in the `Dockerfile` since the last tag: `git diff docker/Dockerfile`. -We should check and update if necessary: +## Checkpoints -- [ ] The "Known Issues" section to ensure that any items that are resolved in the latest release are no longer listed in the README. -- [ ] The "Build and Run Instructions" section. Check if any new dependencies were introduced and - list them if needed; one possible approach is to check for changes in the `Dockerfile` - since the last tag: `git diff docker/Dockerfile`. +With every release and for performance reasons, we want to update the zebra checkpoints. More information on how to do this can be found in [the zebra-checkpoints README](https://github.com/ZcashFoundation/zebra/blob/main/zebra-consensus/src/checkpoint/README.md). + +To do this you will need a synchronized `zcashd` node. You can request help from other zebra team members to submit this PR if you can't make it yourself at the moment of the release. ## Change Log -**Important**: Any merge into `main` deletes any edits to the draft changelog. Once you are ready to tag a release, copy the draft changelog into `CHANGELOG.md`. +**Important**: Any merge into `main` deletes any edits to the draft changelog. +Once you are ready to tag a release, copy the draft changelog into `CHANGELOG.md`. We follow the [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. @@ -102,8 +120,10 @@ To create the final change log: - [ ] Combine duplicate changes - [ ] Edit change descriptions so they are consistent, and make sense to non-developers - [ ] Check the category for each change - - prefer the "Fix" category if you're not sure + - Prefer the "Fix" category if you're not sure +
+ #### Change Categories From "Keep a Changelog": @@ -114,32 +134,35 @@ From "Keep a Changelog": * `Fixed` for any bug fixes. * `Security` in case of vulnerabilities. +
+ ## Create the Release ### Create the Release PR -After you have the version increments and the updated changelog: +After you have the version increments, the updated checkpoints and the updated changelog: +- [ ] Make sure the PR with the new checkpoint hashes is already merged. - [ ] Push the version increments and the updated changelog into a branch (name suggestion, example: `v1.0.0-alpha.0-release`) - [ ] Create a release PR by adding `&template=release-checklist.md` to the comparing url ([Example](https://github.com/ZcashFoundation/zebra/compare/v1.0.0-alpha.0-release?expand=1&template=release-checklist.md)). - - [ ] Add the list of deleted changelog entries as a comment to make reviewing easier. - - [ ] Also add the list of not-bumped crates as a comment (can use the same comment as the previous one). + - [ ] Add the list of deleted changelog entries as a comment to make reviewing easier. + - [ ] Also add the list of not-bumped crates as a comment (can use the same comment as the previous one). - [ ] Turn on [Merge Freeze](https://www.mergefreeze.com/installations/3676/branches). -- [ ] Once the PR is ready to be merged, unfreeze it - [here](https://www.mergefreeze.com/installations/3676/branches). Do not - unfreeze the whole repository. +- [ ] Once the PR is ready to be merged, unfreeze it [here](https://www.mergefreeze.com/installations/3676/branches). + Do not unfreeze the whole repository. ### Create the Release -- [ ] Once the PR has been merged, create a new release using the draft release - as a base, by clicking the Edit icon in the [draft - release](https://github.com/ZcashFoundation/zebra/releases). -- [ ] Set the tag name to the version tag, for example: `v1.0.0-alpha.0` +- [ ] Once the PR has been merged, + create a new release using the draft release as a base, + by clicking the Edit icon in the [draft release](https://github.com/ZcashFoundation/zebra/releases) +- [ ] Set the tag name to the version tag, + for example: `v1.0.0-alpha.0` - [ ] Set the release to target the `main` branch -- [ ] Set the release title to `Zebra ` followed by the version tag, for example: - `Zebra 1.0.0-alpha.0` +- [ ] Set the release title to `Zebra ` followed by the version tag, + for example: `Zebra 1.0.0-alpha.0` - [ ] Copy the final changelog of this release to the release description (starting just _after_ the title `## [Zebra ...`) - [ ] Mark the release as 'pre-release' (until we are no longer alpha/beta) @@ -147,15 +170,13 @@ After you have the version increments and the updated changelog: ## Final Testing -- [ ] After tagging the release, test that the exact `cargo install` command in - `README.md` works (`--git` behaves a bit differently to `--path`) -- [ ] Turn off [Merge - Freeze](https://www.mergefreeze.com/installations/3676/branches) for the - whole repository. +- [ ] After tagging the release, test that the exact `cargo install` command in `README.md` works + (`--git` behaves a bit differently to `--path`) +- [ ] Turn off [Merge Freeze](https://www.mergefreeze.com/installations/3676/branches) for the whole repository If the build fails after tagging: -1. fix the build -2. increment versions again, following these instructions from the start -3. update `README.md` with a **new** git tag -4. update `CHANGELOG.md` with details about the fix -5. tag a **new** release +1. Fix the build +2. Increment versions again, following these instructions from the start +3. Update `README.md` with a **new** git tag +4. Update `CHANGELOG.md` with details about the fix +5. Tag a **new** release diff --git a/.github/workflows/build-crates-individually.patch.yml b/.github/workflows/build-crates-individually.patch.yml new file mode 100644 index 00000000000..f78e69a2731 --- /dev/null +++ b/.github/workflows/build-crates-individually.patch.yml @@ -0,0 +1,69 @@ +name: Build crates individually + +# We need to keep the `matrix` job in this workflow as-is, as we need the results +# to actually match the same `build` job names from the original file. +on: + pull_request: + paths-ignore: + # production code and test code + - '**/*.rs' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # workflow definitions + - '.github/workflows/build-crates-individually.yml' + +jobs: + matrix: + name: Crates matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v3.0.2 + + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + + - uses: actions-rs/cargo@v1.0.3 + # This step is meant to dynamically create a JSON containing the values of each crate + # available in this repo in the root directory. We use `cargo tree` to accomplish this task. + # + # The result from `cargo tree` is then transform to JSON values between double quotes, + # and separated by commas, then added to a `crates.txt` and assigned to a $JSON_CRATES variable. + # + # A JSON object is created and assigned to a $MATRIX variable, which is use to create an output + # named `matrix`, which is then used as the input in following steps, + # using ` ${{ fromJson(needs.matrix.outputs.matrix) }}` + - id: set-matrix + name: Dynamically build crates JSON + run: | + TEMP_DIR=$(mktemp -d) + echo "$(cargo tree --depth 0 --edges no-normal,no-dev,no-build,no-proc-macro --prefix none | cut -d ' ' -f1 | sed '/^$/d' | awk '{ printf "\"%s\",\n", $0 }' | sed '$ s/.$//')" > $TEMP_DIR/crates.txt + MATRIX=$( ( + echo '{ "crate" : [' + echo "$(cat $TEMP_DIR/crates.txt)" + echo " ]}" + ) | jq -c .) + echo $MATRIX + echo $MATRIX | jq . + echo "::set-output name=matrix::$MATRIX" + + check-matrix: + runs-on: ubuntu-latest + needs: [ matrix ] + steps: + - run: 'echo "No job required"' + + build: + name: Build ${{ matrix.crate }} crate + needs: [ matrix, check-matrix ] + runs-on: ubuntu-latest + strategy: + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} + + steps: + - run: 'echo "No job required"' \ No newline at end of file diff --git a/.github/workflows/build-crates-individually.yml b/.github/workflows/build-crates-individually.yml new file mode 100644 index 00000000000..4e65f6b8ad8 --- /dev/null +++ b/.github/workflows/build-crates-individually.yml @@ -0,0 +1,125 @@ +name: Build crates individually + +on: + workflow_dispatch: + push: + branches: + - main + paths: + # production code and test code + - '**/*.rs' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # workflow definitions + - '.github/workflows/build-crates-individually.yml' + pull_request: + paths: + # production code and test code + - '**/*.rs' + # dependencies + - '**/Cargo.toml' + - '**/Cargo.lock' + # workflow definitions + - '.github/workflows/build-crates-individually.yml' + +env: + CARGO_INCREMENTAL: 0 + RUST_BACKTRACE: full + RUST_LIB_BACKTRACE: full + COLORBT_SHOW_HIDDEN: '1' + +jobs: + matrix: + name: Crates matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v3.0.2 + + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + + - uses: actions-rs/cargo@v1.0.3 + # This step is meant to dynamically create a JSON containing the values of each crate + # available in this repo in the root directory. We use `cargo tree` to accomplish this task. + # + # The result from `cargo tree` is then transform to JSON values between double quotes, + # and separated by commas, then added to a `crates.txt` and assigned to a $JSON_CRATES variable. + # + # A JSON object is created and assigned to a $MATRIX variable, which is use to create an output + # named `matrix`, which is then used as the input in following steps, + # using ` ${{ fromJson(needs.matrix.outputs.matrix) }}` + - id: set-matrix + name: Dynamically build crates JSON + run: | + TEMP_DIR=$(mktemp -d) + echo "$(cargo tree --depth 0 --edges no-normal,no-dev,no-build,no-proc-macro --prefix none | cut -d ' ' -f1 | sed '/^$/d' | awk '{ printf "\"%s\",\n", $0 }' | sed '$ s/.$//')" > $TEMP_DIR/crates.txt + MATRIX=$( ( + echo '{ "crate" : [' + echo "$(cat $TEMP_DIR/crates.txt)" + echo " ]}" + ) | jq -c .) + echo $MATRIX + echo $MATRIX | jq . + echo "::set-output name=matrix::$MATRIX" + + check-matrix: + runs-on: ubuntu-latest + needs: [ matrix ] + steps: + - name: Install json2yaml + run: | + sudo npm install -g json2yaml + + - name: Check matrix definition + run: | + matrix='${{ needs.matrix.outputs.matrix }}' + echo $matrix + echo $matrix | jq . + echo $matrix | json2yaml + + build: + name: Build ${{ matrix.crate }} crate + needs: [ matrix, check-matrix ] + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} + + steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + + # We could use `features: ['', '--all-features', '--no-default-features']` as a matrix argument, + # but it's faster to run these commands sequentially, so they can re-use the local cargo cache. + # + # Some Zebra crates do not have any features, and most don't have any default features. + - name: Build ${{ matrix.crate }} crate with no default features + uses: actions-rs/cargo@v1.0.3 + with: + command: build + args: --package ${{ matrix.crate }} --no-default-features + + - name: Build ${{ matrix.crate }} crate normally + uses: actions-rs/cargo@v1.0.3 + with: + command: build + args: --package ${{ matrix.crate }} + + - name: Build ${{ matrix.crate }} crate with all features + uses: actions-rs/cargo@v1.0.3 + with: + command: build + args: --package ${{ matrix.crate }} --all-features diff --git a/.github/workflows/continous-integration-docker.patch-always.yml b/.github/workflows/continous-integration-docker.patch-always.yml index 82b7acecb0f..5a9e115442f 100644 --- a/.github/workflows/continous-integration-docker.patch-always.yml +++ b/.github/workflows/continous-integration-docker.patch-always.yml @@ -7,8 +7,6 @@ name: CI Docker on: pull_request: - branches: - - main jobs: regenerate-stateful-disks: diff --git a/.github/workflows/continous-integration-docker.patch.yml b/.github/workflows/continous-integration-docker.patch.yml index 87c10930d3a..b96f03e6b01 100644 --- a/.github/workflows/continous-integration-docker.patch.yml +++ b/.github/workflows/continous-integration-docker.patch.yml @@ -4,8 +4,6 @@ name: CI Docker # so they can be skipped when the modified files make the actual workflow run. on: pull_request: - branches: - - main paths-ignore: # code and tests - '**/*.rs' @@ -65,6 +63,12 @@ jobs: steps: - run: 'echo "No build required"' + test-update-sync: + name: Zebra tip update / Run update-to-tip test + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + lightwalletd-rpc-test: name: Zebra tip JSON-RPC / Run fully-synced-rpc test runs-on: ubuntu-latest diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index f6fea3d8f96..14fdd108d4a 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -23,8 +23,6 @@ on: required: true pull_request: - branches: - - main paths: # code and tests - '**/*.rs' @@ -311,6 +309,33 @@ jobs: disk_suffix: tip height_grep_text: 'current_height.*=.*Height' + # Test that Zebra can sync to the chain tip, using a cached Zebra tip state, + # without launching `lightwalletd`. + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached state to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + test-update-sync: + name: Zebra tip update + needs: test-full-sync + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' }} + with: + app_name: zebrad + test_id: update-to-tip + test_description: Test syncing to tip with a Zebra tip state + test_variables: '-e TEST_UPDATE_SYNC=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' + needs_zebra_state: true + # TODO: do we want to update the disk on every PR, to increase CI speed? + saves_to_disk: false + disk_suffix: tip + root_state_path: '/var/cache' + # TODO: do we also want to test the `zebrad` part of the `lwd-cache`? (But not update it.) + zebra_state_dir: 'zebrad-cache' + # Test that Zebra can answer a synthetic RPC call, using a cached Zebra tip state # # Runs: @@ -412,6 +437,7 @@ jobs: test_variables: '-e TEST_LWD_UPDATE_SYNC=1 -e ZEBRA_TEST_LIGHTWALLETD=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache -e LIGHTWALLETD_DATA_DIR=/var/cache/lwd-cache' needs_zebra_state: true needs_lwd_state: true + # TODO: do we want to update the disk on every PR, to increase CI speed? saves_to_disk: false disk_prefix: lwd-cache disk_suffix: tip diff --git a/.github/workflows/continous-integration-os.patch.yml b/.github/workflows/continous-integration-os.patch.yml index 88d05fa12a3..ef965a6433a 100644 --- a/.github/workflows/continous-integration-os.patch.yml +++ b/.github/workflows/continous-integration-os.patch.yml @@ -14,46 +14,21 @@ on: jobs: test: name: Test ${{ matrix.rust }} on ${{ matrix.os }} - # The large timeout is to accommodate: - # - Windows builds (75 minutes, typically 30-50 minutes) - # - parameter downloads (40 minutes, but only when the cache expires) - timeout-minutes: 115 runs-on: ${{ matrix.os }} strategy: - fail-fast: false matrix: # TODO: Windows was removed for now, see https://github.com/ZcashFoundation/zebra/issues/3801 os: [ubuntu-latest, macos-latest] - rust: [stable] - - steps: - - run: 'echo "No build required"' - - test-fake-activation-heights: - name: Test ${{ matrix.rust }} zebra-state with fake activation heights on ubuntu-latest - timeout-minutes: 60 - runs-on: ubuntu-latest - strategy: - matrix: - rust: [stable] - - steps: - - run: 'echo "No build required"' - - build-chain-no-features: - name: Build ${{ matrix.rust }} zebra-chain w/o features on ubuntu-latest - timeout-minutes: 60 - runs-on: ubuntu-latest - strategy: - matrix: rust: [stable, beta] + exclude: + - os: macos-latest + rust: beta steps: - run: 'echo "No build required"' install-from-lockfile-no-cache: name: Install zebrad from lockfile without cache on ubuntu-latest - timeout-minutes: 60 runs-on: ubuntu-latest steps: @@ -61,23 +36,20 @@ jobs: check-cargo-lock: name: Check Cargo.lock is up to date - timeout-minutes: 60 runs-on: ubuntu-latest steps: - run: 'echo "No build required"' cargo-deny: - name: Check deny.toml ${{ matrix.checks }} + name: Check deny.toml ${{ matrix.checks }} ${{ matrix.features }} runs-on: ubuntu-latest strategy: matrix: checks: - bans - sources - - # Prevent sudden announcement of a new advisory from failing ci: - continue-on-error: ${{ matrix.checks == 'advisories' }} + features: ['', '--all-features', '--no-default-features'] steps: - run: 'echo "No build required"' diff --git a/.github/workflows/continous-integration-os.yml b/.github/workflows/continous-integration-os.yml index ba29bdba073..18614b51c64 100644 --- a/.github/workflows/continous-integration-os.yml +++ b/.github/workflows/continous-integration-os.yml @@ -53,7 +53,14 @@ jobs: matrix: # TODO: Windows was removed for now, see https://github.com/ZcashFoundation/zebra/issues/3801 os: [ubuntu-latest, macos-latest] - rust: [stable] + rust: [stable, beta] + # We're excluding macOS for the following reasons: + # - the concurrent macOS runner limit is much lower than the Linux limit + # - macOS is slower than Linux, and shouldn't have a build or test difference with Linux + # - macOS is a second-tier Zebra support platform + exclude: + - os: macos-latest + rust: beta steps: - uses: actions/checkout@v3.0.2 @@ -146,77 +153,6 @@ jobs: # Note: this only runs the zebrad acceptance tests, because re-running all the test binaries is slow on Windows args: --verbose --package zebrad --test acceptance -- --nocapture --include-ignored sync_large_checkpoints_ - test-fake-activation-heights: - name: Test ${{ matrix.rust }} zebra-state with fake activation heights on ubuntu-latest - timeout-minutes: 60 - runs-on: ubuntu-latest - strategy: - matrix: - rust: [stable] - - steps: - - uses: actions/checkout@v3.0.2 - with: - persist-credentials: false - - - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ matrix.rust }} - profile: minimal - override: true - - - uses: Swatinem/rust-cache@v1 - - - name: cargo fetch - uses: actions-rs/cargo@v1.0.3 - with: - command: fetch - - # This test changes zebra-chain's activation heights, - # which can recompile all the Zebra crates, - # so we want its build products to be cached separately. - # - # Also, we don't want to accidentally use the fake heights in other tests. - - name: Run tests with fake activation heights - uses: actions-rs/cargo@v1.0.3 - env: - TEST_FAKE_ACTIVATION_HEIGHTS: '' - with: - command: test - # Note: this only runs the zebra-state crate tests, - # because re-running all the test binaries can be slow - args: --verbose --package zebra-state --lib -- --nocapture --include-ignored with_fake_activation_heights - - build-chain-no-features: - name: Build ${{ matrix.rust }} zebra-chain w/o features on ubuntu-latest - timeout-minutes: 60 - runs-on: ubuntu-latest - strategy: - matrix: - rust: [stable, beta] - - steps: - - uses: actions/checkout@v3.0.2 - with: - persist-credentials: false - - - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ matrix.rust }} - profile: minimal - override: true - - - uses: Swatinem/rust-cache@v1 - - - name: cargo fetch - uses: actions-rs/cargo@v1.0.3 - with: - command: fetch - - - name: Run build without features enabled - working-directory: ./zebra-chain - run: cargo build --verbose --no-default-features - # Install Zebra with lockfile dependencies, with no caching and default features install-from-lockfile-no-cache: name: Install zebrad from lockfile without cache on ubuntu-latest @@ -267,13 +203,14 @@ jobs: args: --locked --all-features --all-targets cargo-deny: - name: Check deny.toml ${{ matrix.checks }} + name: Check deny.toml ${{ matrix.checks }} ${{ matrix.features }} runs-on: ubuntu-latest strategy: matrix: checks: - bans - sources + features: ['', '--all-features', '--no-default-features'] # Prevent sudden announcement of a new advisory from failing ci: continue-on-error: ${{ matrix.checks == 'advisories' }} @@ -283,14 +220,10 @@ jobs: with: persist-credentials: false - - uses: EmbarkStudios/cargo-deny-action@v1 - with: - command: check ${{ matrix.checks }} - args: --all-features --workspace - - # this check runs with optional features off + # this check also runs with optional features off # so we expect some warnings about "skip tree root was not found" - - uses: EmbarkStudios/cargo-deny-action@v1 + - name: Check ${{ matrix.checks }} with features ${{ matrix.features }} + uses: EmbarkStudios/cargo-deny-action@v1 with: command: check ${{ matrix.checks }} - args: --workspace + args: --workspace with features ${{ matrix.features }} diff --git a/.github/workflows/delete-gcp-resources.yml b/.github/workflows/delete-gcp-resources.yml new file mode 100644 index 00000000000..6cb58f7a857 --- /dev/null +++ b/.github/workflows/delete-gcp-resources.yml @@ -0,0 +1,33 @@ +name: Delete GCP resources + +on: + schedule: + - cron: "0 0 1 * *" + workflow_dispatch: + +jobs: + delete-resources: + name: Delete old GCP resources + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.8.0 + with: + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + # Deletes all the instances template older than 30 days + - name: Delete old instance templates + run: | + TEMPLATES=$(gcloud compute instance-templates list --sort-by=creationTimestamp --filter="creationTimestamp < $(date --date='30 days ago' '+%Y%m%d')" --format='value(NAME)') + + for TEMPLATE in $TEMPLATES + do + gcloud compute instance-templates delete ${TEMPLATE} --quiet || continue + done diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/deploy-gcp-tests.yml index 2ad0f0dfb5e..6051ac13119 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/deploy-gcp-tests.yml @@ -3,72 +3,97 @@ name: Deploy GCP tests on: workflow_call: inputs: - network: - required: false - type: string - default: Mainnet - app_name: - required: false - type: string - default: 'zebra' + # Status and logging test_id: required: true type: string + description: 'Unique identifier for the test' test_description: required: true type: string + description: 'Explains what the test does' + # Test selection and parameters test_variables: required: true type: string + description: 'Environmental variables used to select and configure the test' + network: + required: false + type: string + default: Mainnet + description: 'Zcash network to test against' + # Cached state + # # TODO: find a better name root_state_path: required: false type: string default: '/zebrad-cache' + description: 'Cached state base directory path' # TODO: find a better name zebra_state_dir: required: false type: string default: '' - description: 'Name of the Zebra cached state directory and input image prefix to search in GCP' + description: 'Zebra cached state directory and input image prefix to search in GCP' # TODO: find a better name lwd_state_dir: required: false type: string default: '' - description: 'Name of the Lightwalletd cached state directory and input image prefix to search in GCP' + description: 'Lightwalletd cached state directory and input image prefix to search in GCP' disk_prefix: required: false type: string default: 'zebrad-cache' - description: 'Used to name the image, and for tests that do not use a `zebra_state_dir` to work, but builds a cached state' + description: 'Image name prefix, and `zebra_state_dir` name for newly created cached states' disk_suffix: required: false type: string + description: 'Image name suffix' needs_zebra_state: required: true type: boolean - description: 'Indicates if a test needs a disk with a Zebra cached state to run' + description: 'Does the test use Zebra cached state?' needs_lwd_state: required: false type: boolean - description: 'Indicates if a test needs a disk with Lightwalletd cached state to run (which also includes a Zebra cached state)' + description: 'Does the test use Lightwalletd and Zebra cached state?' saves_to_disk: required: true type: boolean + description: 'Does the test create a new cached state disk?' + # Metadata height_grep_text: required: false type: string + description: 'Regular expression to find the tip height in test logs, and add it to newly created cached state image metadata' + app_name: + required: false + type: string + default: 'zebra' + description: 'Application name for Google Cloud instance metadata' env: + # where we get the Docker image from IMAGE_NAME: zebrad-test GAR_BASE: us-docker.pkg.dev/zealous-zebra/zebra + # what kind of Google Cloud instance we want to launch ZONE: us-central1-a MACHINE_TYPE: c2d-standard-16 + # How many previous log lines we show at the start of each new log job. + # Increase this number if some log lines are skipped between jobs + # + # We want to show all the logs since the last job finished, + # but we don't know how long it will be between jobs. + # 200 lines is about 6-15 minutes of sync logs, or one panic log. + EXTRA_LOG_LINES: 200 jobs: - test-without-cached-state: - name: Run ${{ inputs.test_id }} test + # set up the test, if it doesn't use any cached state + # each test runs one of the *-with/without-cached-state job series, and skips the other + setup-without-cached-state: + name: Setup ${{ inputs.test_id }} test if: ${{ !inputs.needs_zebra_state }} runs-on: ubuntu-latest permissions: @@ -78,6 +103,7 @@ jobs: - uses: actions/checkout@v3.0.2 with: persist-credentials: false + fetch-depth: '2' - name: Inject slug/short variables uses: rlespinasse/github-slug-action@v4 @@ -98,7 +124,8 @@ jobs: service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' token_format: 'access_token' - - name: Create GCP compute instance + # Create a Compute Engine virtual machine + - name: Create ${{ inputs.test_id }} GCP compute instance id: create-instance run: | gcloud compute instances create-with-container "${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \ @@ -114,7 +141,7 @@ jobs: --zone ${{ env.ZONE }} sleep 60 - - name: Run ${{ inputs.test_id }} test + - name: Create ${{ inputs.test_id }} Docker volume run: | gcloud compute ssh \ ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ @@ -127,13 +154,67 @@ jobs: && \ docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \ ${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ - && \ - docker run ${{ inputs.test_variables }} -t --name ${{ inputs.test_id }} \ + " + + # launch the test, if it doesn't use any cached state + launch-without-cached-state: + name: Launch ${{ inputs.test_id }} test + needs: [ setup-without-cached-state ] + # If creating the Google Cloud instance fails, we don't want to launch another docker instance. + if: ${{ !cancelled() && !failure() && !inputs.needs_zebra_state }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + fetch-depth: '2' + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Downcase network name for disks + run: | + NETWORK_CAPS=${{ inputs.network }} + echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.8.0 + with: + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + # Launch the test without any cached state + - name: Launch ${{ inputs.test_id }} test + run: | + gcloud compute ssh \ + ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command \ + "\ + docker run \ + --name ${{ inputs.test_id }} \ + --tty \ + --detach \ + ${{ inputs.test_variables }} \ --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \ - ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}" + ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \ + " - test-with-cached-state: - name: Run ${{ inputs.test_id }} test + + # set up the test, if it uses cached state + # each test runs one of the *-with/without-cached-state job series, and skips the other + setup-with-cached-state: + name: Setup ${{ inputs.test_id }} test if: ${{ inputs.needs_zebra_state }} runs-on: ubuntu-latest permissions: @@ -175,12 +256,12 @@ jobs: # - To ${{ inputs.zebra_state_dir || inputs.disk_prefix }} if not # # If there are multiple disks: - # - prefer images generated from the `main` branch, then any other branch + # - prefer images generated from this branch, then the `main` branch, then any other branch # - prefer newer images to older images # # Passes the disk name to subsequent steps using $CACHED_DISK_NAME env variable # Passes the state version to subsequent steps using $STATE_VERSION env variable - - name: Find cached state disk + - name: Find ${{ inputs.test_id }} cached state disk id: get-disk-name run: | LOCAL_STATE_VERSION=$(grep -oE "DATABASE_FORMAT_VERSION: .* [0-9]+" "$GITHUB_WORKSPACE/zebra-state/src/constants.rs" | grep -oE "[0-9]+" | tail -n1) @@ -192,20 +273,28 @@ jobs: DISK_PREFIX=${{ inputs.zebra_state_dir || inputs.disk_prefix }} fi - # Try to find an image generated from the main branch + # Try to find an image generated from this branch and commit # Fields are listed in the "Create image from state disk" step - CACHED_DISK_NAME=$(gcloud compute images list --filter="name~${DISK_PREFIX}-main-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-${{ inputs.disk_suffix }}" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1) - echo "main Disk: $CACHED_DISK_NAME" + BRANCH_DISK_NAME="${DISK_PREFIX}-${GITHUB_REF_SLUG_URL}-${GITHUB_SHA_SHORT}-v${LOCAL_STATE_VERSION}-${NETWORK}-${{ inputs.disk_suffix }}" + CACHED_DISK_NAME=$(gcloud compute images list --filter="name~${BRANCH_DISK_NAME}" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1) + echo "${GITHUB_REF_SLUG_URL}-${GITHUB_SHA_SHORT} Disk: $CACHED_DISK_NAME" + + if [[ -z "$CACHED_DISK_NAME" ]]; then + # Try to find an image generated from the main branch + CACHED_DISK_NAME=$(gcloud compute images list --filter="name~${DISK_PREFIX}-main-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-${{ inputs.disk_suffix }}" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1) + echo "main Disk: $CACHED_DISK_NAME" + fi if [[ -z "$CACHED_DISK_NAME" ]]; then # Try to find an image generated from any other branch CACHED_DISK_NAME=$(gcloud compute images list --filter="name~${DISK_PREFIX}-.+-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-${{ inputs.disk_suffix }}" --format="value(NAME)" --sort-by=~creationTimestamp --limit=1) - echo "Disk: $CACHED_DISK_NAME" + echo "any branch Disk: $CACHED_DISK_NAME" fi if [[ -z "$CACHED_DISK_NAME" ]]; then echo "No cached state disk available" - echo "Expected ${DISK_PREFIX}-(branch)-[0-9a-f]+-v${LOCAL_STATE_VERSION}-${NETWORK}-${{ inputs.disk_suffix }}" + echo "Expected ${BRANCH_DISK_NAME}" + echo "Also searched for any commit on main, and any commit on any branch" echo "Cached state test jobs must depend on the cached state rebuild job" exit 1 fi @@ -215,15 +304,15 @@ jobs: echo "STATE_VERSION=$LOCAL_STATE_VERSION" >> $GITHUB_ENV echo "CACHED_DISK_NAME=$CACHED_DISK_NAME" >> $GITHUB_ENV - # Creates Compute Engine virtual machine and attach a cached state disk using the + # Create a Compute Engine virtual machine and attach a cached state disk using the # $CACHED_DISK_NAME variable as the source image to populate the disk cached state - - name: Create GCP compute instance + - name: Create ${{ inputs.test_id }} GCP compute instance id: create-instance run: | gcloud compute instances create-with-container "${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \ --boot-disk-size 100GB \ --boot-disk-type pd-ssd \ - --create-disk image=${{ env.CACHED_DISK_NAME }},name="${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=100GB,type=pd-ssd \ + --create-disk image=${{ env.CACHED_DISK_NAME }},name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=100GB,type=pd-ssd \ --container-image debian:buster \ --container-restart-policy=never \ --machine-type ${{ env.MACHINE_TYPE }} \ @@ -233,9 +322,62 @@ jobs: --zone ${{ env.ZONE }} sleep 60 - # SSH into the just created VM, and create a Docker container to run the incoming test - # from ${{ inputs.test_id }}, then create a docker volume with the recently attached disk. - # The disk will be mounted in ${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }}. + # Create a docker volume with the selected cached state. + # + # SSH into the just created VM, and create a docker volume with the recently attached disk. + - name: Create ${{ inputs.test_id }} Docker volume + run: | + gcloud compute ssh \ + ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command \ + "\ + docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \ + ${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ + " + + # launch the test, if it uses cached state + launch-with-cached-state: + name: Launch ${{ inputs.test_id }} test + needs: [ setup-with-cached-state ] + # If creating the Google Cloud instance fails, we don't want to launch another docker instance. + if: ${{ !cancelled() && !failure() && inputs.needs_zebra_state }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + fetch-depth: '2' + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Downcase network name for disks + run: | + NETWORK_CAPS=${{ inputs.network }} + echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.8.0 + with: + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + # Launch the test with the previously created Zebra-only cached state. + # Each test runs one of the "Launch test" steps, and skips the other. + # + # SSH into the just created VM, and create a Docker container to run the incoming test + # from ${{ inputs.test_id }}, then mount the docker volume created in the previous job. # # The disk mounted in the VM is located at /dev/sdb, we mount the root `/` of this disk to the docker # container in one path: @@ -244,14 +386,13 @@ jobs: # This path must match the variable used by the tests in Rust, which are also set in # `continous-integration-docker.yml` to be able to run this tests. # - # Although we're mounting the disk root, Zebra will only respect the values from + # Although we're mounting the disk root, Zebra will only respect the values from # $ZEBRA_CACHED_STATE_DIR. The inputs like ${{ inputs.zebra_state_dir }} are only used # to match that variable paths. - - name: Run ${{ inputs.test_id }} test - # This step mounts the volume only when a single cached state is needed, in this case - # the cached state from Zebra. - # lightwalletd-full-sync test is an exception to this rule, as it does not need a lwd cached state, - # but it does saves a lwd cached state + - name: Launch ${{ inputs.test_id }} test + # This step only runs for tests that just read or write a Zebra state. + # + # lightwalletd-full-sync reads Zebra and writes lwd, so it is handled specially. # TODO: we should find a better logic for this use cases if: ${{ (inputs.needs_zebra_state && !inputs.needs_lwd_state) && inputs.test_id != 'lwd-full-sync' }} run: | @@ -262,17 +403,20 @@ jobs: --ssh-flag="-o ServerAliveInterval=5" \ --command \ "\ - docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \ - ${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ - && \ - docker run ${{ inputs.test_variables }} -t --name ${{ inputs.test_id }} \ - --mount type=volume,src=${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \ - ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}" - - # SSH into the just created VM, and create a Docker container to run the incoming test - # from ${{ inputs.test_id }}, then create a docker volume with the recently attached disk. - # The disk will be mounted in ${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }}, - # and ${{ inputs.root_state_path }}/${{ inputs.lwd_state_dir }} + docker run \ + --name ${{ inputs.test_id }} \ + --tty \ + --detach \ + ${{ inputs.test_variables }} \ + --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \ + ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \ + " + + # Launch the test with the previously created Lightwalletd and Zebra cached state. + # Each test runs one of the "Launch test" steps, and skips the other. + # + # SSH into the just created VM, and create a Docker container to run the incoming test + # from ${{ inputs.test_id }}, then mount the docker volume created in the previous job. # # In this step we're using the same disk for simplicity, as mounting multiple disks to the # VM and to the container might require more steps in this workflow, and additional @@ -293,11 +437,10 @@ jobs: # Although we're mounting the disk root to both directories, Zebra and Lightwalletd # will only respect the values from $ZEBRA_CACHED_STATE_DIR and $LIGHTWALLETD_DATA_DIR, # the inputs like ${{ inputs.lwd_state_dir }} are only used to match those variables paths. - - name: Run ${{ inputs.test_id }} test - # This step mounts the volume only when both cached states are needed, in this case - # the cached state from Zebra and Lightwalletd - # lightwalletd-full-sync test is an exception to this rule, as it does not need a lwd cached state, - # but it does saves a lwd cached state + - name: Launch ${{ inputs.test_id }} test + # This step only runs for tests that read or write Lightwalletd and Zebra states. + # + # lightwalletd-full-sync reads Zebra and writes lwd, so it is handled specially. # TODO: we should find a better logic for this use cases if: ${{ (inputs.needs_zebra_state && inputs.needs_lwd_state) || inputs.test_id == 'lwd-full-sync' }} run: | @@ -308,23 +451,433 @@ jobs: --ssh-flag="-o ServerAliveInterval=5" \ --command \ "\ - docker volume create --driver local --opt type=ext4 --opt device=/dev/sdb \ - ${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ - && \ - docker run ${{ inputs.test_variables }} -t --name ${{ inputs.test_id }} \ - --mount type=volume,src=${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \ - --mount type=volume,src=${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.lwd_state_dir }} \ - ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }}" + docker run \ + --name ${{ inputs.test_id }} \ + --tty \ + --detach \ + ${{ inputs.test_variables }} \ + --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.zebra_state_dir }} \ + --mount type=volume,src=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }},dst=${{ inputs.root_state_path }}/${{ inputs.lwd_state_dir }} \ + ${{ env.GAR_BASE }}/${{ env.IMAGE_NAME }}:sha-${{ env.GITHUB_SHA_SHORT }} \ + " + + + # follow the logs of the test we just launched, up to Sapling activation (or the test finishing) + logs-sprout: + name: Log ${{ inputs.test_id }} test (sprout) + # We run exactly one of without-cached-state or with-cached-state, and we always skip the other one. + needs: [ launch-with-cached-state, launch-without-cached-state ] + # If the previous job fails, we still want to show the logs. + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + fetch-depth: '2' + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Downcase network name for disks + run: | + NETWORK_CAPS=${{ inputs.network }} + echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.8.0 + with: + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + # Show all the logs since the container launched, + # following until Sapling activation (or the test finishes). + # + # The log pipeline ignores the exit status of `docker logs`. + # Errors in the tests are caught by the final test status job. + - name: Show logs for ${{ inputs.test_id }} test (sprout) + run: | + gcloud compute ssh \ + ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command \ + "\ + docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + '(estimated progress.*network_upgrade.*=.*Sapling)|(estimated progress.*network_upgrade.*=.*Blossom)|(estimated progress.*network_upgrade.*=.*Heartwood)|(estimated progress.*network_upgrade.*=.*Canopy)|(estimated progress.*network_upgrade.*=.*Nu5)|(test result:.*finished in)' \ + " + + # follow the logs of the test we just launched, up to Canopy activation (or the test finishing) + logs-heartwood: + name: Log ${{ inputs.test_id }} test (heartwood) + needs: [ logs-sprout ] + # If the previous job fails, we still want to show the logs. + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + fetch-depth: '2' + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Downcase network name for disks + run: | + NETWORK_CAPS=${{ inputs.network }} + echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.8.0 + with: + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + # Show recent logs, following until Canopy activation (or the test finishes) + - name: Show logs for ${{ inputs.test_id }} test (heartwood) + run: | + gcloud compute ssh \ + ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command \ + "\ + docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + '(estimated progress.*network_upgrade.*=.*Canopy)|(estimated progress.*network_upgrade.*=.*Nu5)|(test result:.*finished in)' \ + " + + # follow the logs of the test we just launched, up to NU5 activation (or the test finishing) + logs-canopy: + name: Log ${{ inputs.test_id }} test (canopy) + needs: [ logs-heartwood ] + # If the previous job fails, we still want to show the logs. + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + fetch-depth: '2' + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Downcase network name for disks + run: | + NETWORK_CAPS=${{ inputs.network }} + echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.8.0 + with: + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + # Show recent logs, following until NU5 activation (or the test finishes) + - name: Show logs for ${{ inputs.test_id }} test (canopy) + run: | + gcloud compute ssh \ + ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command \ + "\ + docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + '(estimated progress.*network_upgrade.*=.*Nu5)|(test result:.*finished in)' \ + " + + # follow the logs of the test we just launched, up to block 1,740,000 (or the test finishing) + # + # We chose this height because it was about 5 hours into the NU5 sync, at the end of July 2022. + # This is a temporary workaround until we improve sync speeds. + logs-1740k: + name: Log ${{ inputs.test_id }} test (1740k) + needs: [ logs-canopy ] + # If the previous job fails, we still want to show the logs. + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + fetch-depth: '2' + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Downcase network name for disks + run: | + NETWORK_CAPS=${{ inputs.network }} + echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.8.0 + with: + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + # Show recent logs, following until block 1,740,000 (or the test finishes) + - name: Show logs for ${{ inputs.test_id }} test (1740k) + run: | + gcloud compute ssh \ + ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command \ + "\ + docker logs \ + --tail all \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + '(estimated progress.*current_height.*=.*174[0-9][0-9][0-9][0-9].*remaining_sync_blocks)|(test result:.*finished in)' \ + " + + # follow the logs of the test we just launched, up to the last checkpoint (or the test finishing) + logs-checkpoint: + name: Log ${{ inputs.test_id }} test (checkpoint) + needs: [ logs-1740k ] + # If the previous job fails, we still want to show the logs. + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + fetch-depth: '2' + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Downcase network name for disks + run: | + NETWORK_CAPS=${{ inputs.network }} + echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.8.0 + with: + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + # Show recent logs, following until the last checkpoint (or the test finishes) + # + # TODO: when doing obtain/extend tips, log the verifier in use, and check for full verification here + - name: Show logs for ${{ inputs.test_id }} test (checkpoint) + run: | + gcloud compute ssh \ + ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command \ + "\ + docker logs \ + --tail ${{ env.EXTRA_LOG_LINES }} \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + '(verified final checkpoint)|(test result:.*finished in)' \ + " + + # follow the logs of the test we just launched, until it finishes + logs-end: + name: Log ${{ inputs.test_id }} test (end) + needs: [ logs-checkpoint ] + # If the previous job fails, we still want to show the logs. + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + fetch-depth: '2' + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Downcase network name for disks + run: | + NETWORK_CAPS=${{ inputs.network }} + echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.8.0 + with: + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + # Show recent logs, following until the test finishes + - name: Show logs for ${{ inputs.test_id }} test (end) + run: | + gcloud compute ssh \ + ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command \ + "\ + docker logs \ + --tail ${{ env.EXTRA_LOG_LINES }} \ + --follow \ + ${{ inputs.test_id }} | \ + tee --output-error=exit /dev/stderr | \ + grep --max-count=1 --extended-regexp --color=always \ + 'test result:.*finished in' \ + " + + + # wait for the result of the test + test-result: + # TODO: update the job name here, and in the branch protection rules + name: Run ${{ inputs.test_id }} test + needs: [ logs-end ] + # If the previous job fails, we also want to run and fail this job, + # so that the branch protection rule fails in Mergify and GitHub. + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + fetch-depth: '2' + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + with: + short-length: 7 + + - name: Downcase network name for disks + run: | + NETWORK_CAPS=${{ inputs.network }} + echo "NETWORK=${NETWORK_CAPS,,}" >> $GITHUB_ENV + + # Setup gcloud CLI + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v0.8.0 + with: + workload_identity_provider: 'projects/143793276228/locations/global/workloadIdentityPools/github-actions/providers/github-oidc' + service_account: 'github-service-account@zealous-zebra.iam.gserviceaccount.com' + token_format: 'access_token' + + # Wait for the container to finish, then exit with the test's exit status. + # + # If the container has already finished, `docker wait` should return its status. + # But sometimes this doesn't work, so we use `docker inspect` as a fallback. + # + # `docker wait` prints the container exit status as a string, but we need to exit the `ssh` command + # with that status. + # (`docker wait` can also wait for multiple containers, but we only ever wait for a single container.) + - name: Result of ${{ inputs.test_id }} test + run: | + gcloud compute ssh \ + ${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ + --zone ${{ env.ZONE }} \ + --quiet \ + --ssh-flag="-o ServerAliveInterval=5" \ + --command=' \ + EXIT_STATUS=$( \ + docker wait ${{ inputs.test_id }} || \ + docker inspect --format "{{.State.ExitCode}}" ${{ inputs.test_id }} || \ + echo "missing container, or missing exit status for container" \ + ); \ + echo "docker exit status: $EXIT_STATUS"; \ + exit "$EXIT_STATUS" \ + ' + + + # create a state image from the instance's state disk, if requested by the caller create-state-image: name: Create ${{ inputs.test_id }} cached state image runs-on: ubuntu-latest - needs: [ test-without-cached-state, test-with-cached-state ] - if: ${{ inputs.saves_to_disk }} + needs: [ test-result ] + # We run exactly one of without-cached-state or with-cached-state, and we always skip the other one. + # Normally, if a job is skipped, all the jobs that depend on it are also skipped. + # So we need to override the default success() check to make this job run. + if: ${{ !cancelled() && !failure() && inputs.saves_to_disk }} permissions: contents: 'read' id-token: 'write' steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + fetch-depth: '2' + - name: Inject slug/short variables uses: rlespinasse/github-slug-action@v4 with: @@ -394,11 +947,12 @@ jobs: run: | gcloud compute images create ${{ inputs.disk_prefix }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}-v${{ env.STATE_VERSION }}-${{ env.NETWORK }}-${{ inputs.disk_suffix }} \ --force \ - --source-disk=${{ inputs.disk_prefix }}-${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ + --source-disk=${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }} \ --source-disk-zone=${{ env.ZONE }} \ --storage-location=us \ --description="Created from commit ${{ env.GITHUB_SHA_SHORT }} with height ${{ env.SYNC_HEIGHT }}" + # delete the Google Cloud instance for this test delete-instance: name: Delete ${{ inputs.test_id }} instance runs-on: ubuntu-latest @@ -411,6 +965,11 @@ jobs: contents: 'read' id-token: 'write' steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + fetch-depth: '2' + - name: Inject slug/short variables uses: rlespinasse/github-slug-action@v4 with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ba57228e7b5..f989f57b488 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -37,9 +37,14 @@ jobs: - uses: Swatinem/rust-cache@v1 + - name: Setup mdBook + uses: peaceiris/actions-mdbook@v1.1.14 + with: + mdbook-version: '0.4.18' + + # TODO: actions-mdbook does not yet have an option to install mdbook-mermaid https://github.com/peaceiris/actions-mdbook/issues/426 - name: Install mdbook run: | - cargo install mdbook cargo install mdbook-mermaid - name: Build Zebra book diff --git a/.github/workflows/lint.patch.yml b/.github/workflows/lint.patch.yml index cbb85228d72..d6462a1a43d 100644 --- a/.github/workflows/lint.patch.yml +++ b/.github/workflows/lint.patch.yml @@ -17,3 +17,9 @@ jobs: runs-on: ubuntu-latest steps: - run: 'echo "No build required"' + + docs: + name: Rust doc + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3b74716df11..cac408bb78e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,7 +25,7 @@ jobs: - name: Rust files id: changed-files-rust - uses: tj-actions/changed-files@v23 + uses: tj-actions/changed-files@v24 with: files: | **/*.rs @@ -37,7 +37,7 @@ jobs: - name: Workflow files id: changed-files-workflows - uses: tj-actions/changed-files@v23 + uses: tj-actions/changed-files@v24 with: files: | .github/workflows/*.yml @@ -95,7 +95,7 @@ jobs: with: persist-credentials: false - - uses: actions-rs/toolchain@v1.0.1 + - uses: actions-rs/toolchain@v1.0.6 with: toolchain: stable components: rustfmt @@ -108,6 +108,29 @@ jobs: command: fmt args: --all -- --check + docs: + name: Rust doc + timeout-minutes: 30 + runs-on: ubuntu-latest + needs: changed-files + if: ${{ needs.changed-files.outputs.rust == 'true' }} + + steps: + - uses: actions/checkout@v3.0.2 + with: + persist-credentials: false + + - uses: actions-rs/toolchain@v1.0.6 + with: + toolchain: stable + profile: minimal + override: true + + - uses: actions-rs/cargo@v1.0.3 + with: + command: doc + args: --no-deps --document-private-items --all-features + actionlint: runs-on: ubuntu-latest continue-on-error: true @@ -115,7 +138,7 @@ jobs: if: ${{ needs.changed-files.outputs.workflows == 'true' }} steps: - uses: actions/checkout@v3.0.2 - - uses: reviewdog/action-actionlint@v1.25.1 + - uses: reviewdog/action-actionlint@v1.27.0 with: level: warning fail_on_error: false diff --git a/CHANGELOG.md b/CHANGELOG.md index be4a6529c0f..efb2641103b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,194 @@ All notable changes to Zebra are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org). + +## [Zebra 1.0.0-beta.13](https://github.com/ZcashFoundation/zebra/releases/tag/v1.0.0-beta.13) - 2022-07-29 + +This release fixes multiple bugs in proof and signature verification, which were causing big performance issues near the blockchain tip. +It also improves Zebra's sync performance and reliability under heavy load. + +### Disk and Network Usage Changes + +Zebra now uses around 50 - 100 GB of disk space, because many large transactions were recently added to the block chain. (In the longer term, several hundred GB are likely to be needed.) + +When there are a lot of large user-generated transactions on the network, Zebra can upload or download 1 GB or more per day. + +### Configuration Changes + +- Split the checkpoint and full verification [`sync` concurrency options](https://doc.zebra.zfnd.org/zebrad/config/struct.SyncSection.html) (#4726, #4758): + - Add a new `full_verify_concurrency_limit` + - Rename `max_concurrent_block_requests` to `download_concurrency_limit` + - Rename `lookahead_limit` to `checkpoint_verify_concurrency_limit` + For backwards compatibility, the old names are still accepted as aliases. +- Add a new `parallel_cpu_threads` [`sync` concurrency option](https://doc.zebra.zfnd.org/zebrad/config/struct.SyncSection.html) (#4776). + This option sets the number of threads to use for CPU-bound tasks, such as proof and signature verification. + By default, Zebra uses all available CPU cores. + +### Rust Compiler Bug Fixes + +- The Rust team has recently [fixed compilation bugs](https://blog.rust-lang.org/2022/07/19/Rust-1.62.1.html) in function coercions of `impl Trait` return types, and `async fn` lifetimes. + We recommend that you update your Rust compiler to release 1.62.1, and re-compile Zebra. + +### Added + +- Add a `rayon` thread pool for CPU-bound tasks (#4776) +- Run multiple cryptographic batches concurrently within each verifier (#4776) +- Run deserialization of transaction in a rayon thread (#4801) +- Run CPU-intensive state updates in parallel rayon threads (#4802) +- Run CPU-intensive state reads in parallel rayon threads (#4805) +- Support Tiers and supported platforms per Tier doc (#4773) + +### Changed + +- Update column family names to match Zebra's database design (#4639) +- Update Zebra's mainnet and testnet checkpoints (#4777, #4833) +- Process more blocks and batch items concurrently, so there's a batch ready for each available CPU (#4776) +- Wrap note commitment trees in an `Arc`, to reduce CPU and memory usage (#4757) +- Increment `tokio` dependency from 1.19.2 to 1.20.0, to improve diagnostics (#4780) + +### Fixed + +- Only verify halo2 proofs once per transaction (#4752) +- Use a separate channel for each cryptographic batch, to avoid dropped batch results (#4750) +- Improve batch fairness and latency under heavy load (#4750, #4776) +- Run all verifier cryptography on blocking CPU-bound threads, using `tokio` and `rayon` (#4750, #4776) +- Use `tokio`'s `PollSemaphore`, instead of an outdated `Semaphore` impl (#4750) +- Check batch worker tasks for panics and task termination (#4750, #4777) +- Limit the length of the `reject` network message's `message` and `reason` fields (#4687) +- Change the `bitvec` dependency from 1.0.0 to 1.0.1, to fix a performance regression (#4769) +- Fix an occasional panic when a `zebra-network` connection closes (#4782) +- Stop panicking when the connection error slot is not set (#4770) +- When writing blocks to disk, don't block other async tasks (#4199) +- When sending headers to peers, only deserialize the header data from disk (#4792) +- Return errors from `send_periodic_heartbeats_with_shutdown_handle` (#4756) +- Make FindHeaders and FindHashes run concurrently with state updates (#4826) +- Stop reading redundant blocks for every FindHashes and FindHeaders request (#4825) +- Generate sapling point outside the method (#4799) + +#### CI + +- Workaround lightwalletd hangs by waiting until we're near the tip (#4763) +- Split out Canopy logs into a separate job (#4730) +- Make full sync go all the way to the tip (#4709) +- Split Docker logs into sprout, other checkpoints, and full validation (#4704) +- Add a Zebra cached state update test, fix lightwalletd tests (#4813) + + +## [Zebra 1.0.0-beta.12](https://github.com/ZcashFoundation/zebra/releases/tag/v1.0.0-beta.12) - 2022-06-29 + +This release improves Zebra's Orchard proof verification performance and sync performance. +Zebra prefers to connect to peers on the canonical Zcash ports. + +This release also contains some breaking changes which: +- improve usability, and +- make Zebra compile faster. + +### Breaking Changes + +#### Cache Deletion + +- Zebra deletes unused cached state directories in `/zebra` (#4586) + These caches only contain public chain data, so it is safe to delete them. + +#### Compile-Time Features + +- Most of Zebra's [tracing](https://github.com/ZcashFoundation/zebra/blob/main/book/src/user/tracing.md) + and [metrics](https://github.com/ZcashFoundation/zebra/blob/main/book/src/user/metrics.md) features + are off by default at compile time (#4539, #4680) +- The `enable-sentry` feature has been renamed to `sentry` (#4623) + +#### Config + +- Times in `zebrad.config` change from seconds/nanoseconds to a + [human-readable format](https://docs.rs/humantime/latest/humantime/). + Remove times in the old format, or use `zebrad generate` to create a new config. (#4587) + +### Added + +#### Diagnostics + +- Show the current network upgrade in progress logs (#4694) +- Add some missing tracing spans (#4660) +- Add tokio-console support to zebrad (#4519, #4641) +- Add `fallible_impl_from` clippy lint (#4609) +- Add `unwrap_in_result` clippy lint (#4667) + +#### Testing + +- Check that old `zebrad.toml` configs can be parsed by the latest version (#4676) +- Test `cargo doc` warnings and errors (#4635, #4654) +- Document how to run full sync and lightwalletd tests (#4523) + +#### Continuous Integration + +- Add `beta` rust to CI (#4637, #4668) +- Build each Zebra crate individually (#4640) + +### Changed + +#### Chain Sync + +- Update mainnet and testnet checkpoint hashes (#4708) + +#### Diagnostics + +- Update transaction verification dashboard to show all shielded pool sigs, proofs, nullifiers (#4585) + +#### Testing + +- Add an identifiable suffix to zcash-rpc-diff temp directories (#4577) + +#### Dependencies + +- Manage`cargo-mdbook` as a GitHub action (#4636) + +#### Continuous Integration + +- Automatically delete old GCP resources (#4598) + +#### Documentation + +- Improve the release checklist (#4568, #4595) + +### Removed + +#### Continuous Integration + +- Remove redundant build-chain-no-features job (#4656) + +### Fixed + +#### Performance + +- Upgrade `halo2` and related dependencies to improve proof verification speed (#4699) +- Change default sync config to improve reliability (#4662, #4670, #4679) +- Fix a lookahead config panic (#4662) + +#### Continuous Integration + +- Actually create a cached state image after running a sync (#4669) +- Split `docker run` into launch, `logs`, and `wait`, to avoid GitHub job timeouts (#4675, #4690) +- Ignore lightwalletd test hangs for now (#4663) +- Disable `zcash_rpc_conflict` test on macOS (#4614) +- Use `latest` lightwalletd image for Zebra's Dockerfile (#4599) +- Increase lightwalletd timeout, remove testnet tests (#4584) + +#### Documentation + +- Fix various `cargo doc` warnings (#4561, #4611, #4627) +- Clarify how Zebra and `zcashd` interact in `README.md` (#4570) +- Improve `lightwalletd` tutorial (#4566) +- Simplify README and link to detailed documentation (#4680) + +#### Diagnostics + +- Resolve some lifetime and reference lints (#4578) + +### Security + +- When connecting to peers, ignore invalid ports, and prefer canonical ports (#4564) + + ## [Zebra 1.0.0-beta.11](https://github.com/ZcashFoundation/zebra/releases/tag/v1.0.0-beta.11) - 2022-06-03 This release cleans up a lot of tech dept accumulated in the previous diff --git a/Cargo.lock b/Cargo.lock index b5b72c7e23d..8dab1379d56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -231,7 +231,7 @@ dependencies = [ "futures-core", "futures-task", "futures-util", - "pin-project 1.0.10", + "pin-project 1.0.11", "rustc_version", "tokio", ] @@ -459,9 +459,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitvec" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1489fcb93a5bb47da0462ca93ad252ad6af2145cce58d10d46a83931ba9f016b" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" dependencies = [ "funty", "radium", @@ -854,6 +854,42 @@ dependencies = [ "winapi", ] +[[package]] +name = "console-api" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c5fd425783d81668ed68ec98408a80498fb4ae2fd607797539e1a9dfa3618f" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31432bc31ff8883bf6a693a79371862f73087822470c82d6a1ec778781ee3978" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures", + "hdrhistogram", + "humantime", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber 0.3.11", +] + [[package]] name = "const-oid" version = "0.6.2" @@ -1451,8 +1487,18 @@ dependencies = [ [[package]] name = "equihash" -version = "0.1.0" -source = "git+https://github.com/ZcashFoundation/librustzcash.git?rev=6d75718076e592a41b6bd6ec916dc15420e4cc3c#6d75718076e592a41b6bd6ec916dc15420e4cc3c" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab579d7cf78477773b03e80bc2f89702ef02d7112c711d54ca93dcdce68533d5" +dependencies = [ + "blake2b_simd 1.0.0", + "byteorder", +] + +[[package]] +name = "equihash" +version = "0.2.0" +source = "git+https://github.com/ZcashFoundation/librustzcash.git?rev=4567a37ceccbd506a58aaaded39ba14c952c1510#4567a37ceccbd506a58aaaded39ba14c952c1510" dependencies = [ "blake2b_simd 1.0.0", "byteorder", @@ -1855,7 +1901,25 @@ dependencies = [ "bitvec", "ff", "group", - "halo2_proofs", + "halo2_proofs 0.1.0", + "lazy_static", + "pasta_curves", + "rand 0.8.5", + "subtle", + "uint", +] + +[[package]] +name = "halo2_gadgets" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e10bf9924da1754e443641c9e7f9f00483749f8fb837fde696ef6ed6e2f079" +dependencies = [ + "arrayvec 0.7.2", + "bitvec", + "ff", + "group", + "halo2_proofs 0.2.0", "lazy_static", "pasta_curves", "rand 0.8.5", @@ -1877,6 +1941,21 @@ dependencies = [ "rayon", ] +[[package]] +name = "halo2_proofs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff771b9a2445cd2545c9ef26d863c290fbb44ae440c825a20eb7156f67a949a" +dependencies = [ + "blake2b_simd 1.0.0", + "ff", + "group", + "pasta_curves", + "rand_core 0.6.3", + "rayon", + "tracing", +] + [[package]] name = "hashbrown" version = "0.11.2" @@ -1886,13 +1965,19 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" + [[package]] name = "hashlink" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" dependencies = [ - "hashbrown", + "hashbrown 0.11.2", ] [[package]] @@ -1903,7 +1988,6 @@ checksum = "31672b7011be2c4f7456c4ddbcb40e7e9a4a9fad8efe49a6ebaf5f307d0109c0" dependencies = [ "base64", "byteorder", - "crossbeam-channel", "flate2", "nom", "num-traits", @@ -2157,27 +2241,27 @@ checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" [[package]] name = "indexmap" -version = "1.8.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ "autocfg 1.1.0", - "hashbrown", + "hashbrown 0.12.1", "serde", ] [[package]] name = "inferno" -version = "0.11.4" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3cb215599901c8f491666421d44ffaed08d6872b4c7ced6f425683b951271e" +checksum = "9a262875c8e10820b9366e991ed6710cd80dc93578375e5d499fcbd408985937" dependencies = [ "ahash", "atty", "itoa 1.0.1", - "lazy_static", "log", "num-format", + "once_cell", "quick-xml", "rgb", "str_stack", @@ -2185,9 +2269,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.14.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc3e639bcba360d9237acabd22014c16f3df772db463b7446cd81b070714767" +checksum = "4126dd76ebfe2561486a1bd6738a33d2029ffb068a99ac446b7f8c77b2e58dbc" dependencies = [ "console", "once_cell", @@ -2604,7 +2688,7 @@ dependencies = [ "atomic-shim", "crossbeam-epoch", "crossbeam-utils", - "hashbrown", + "hashbrown 0.11.2", "metrics", "num_cpus", "parking_lot 0.11.2", @@ -2910,8 +2994,8 @@ dependencies = [ "ff", "fpe", "group", - "halo2_gadgets", - "halo2_proofs", + "halo2_gadgets 0.1.0", + "halo2_proofs 0.1.0", "hex", "incrementalmerkletree", "lazy_static", @@ -2925,6 +3009,34 @@ dependencies = [ "zcash_note_encryption 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "orchard" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7619db7f917afd9b1139044c595fab1b6166de2db62317794b5f5e34a2104ae1" +dependencies = [ + "aes", + "bitvec", + "blake2b_simd 1.0.0", + "ff", + "fpe", + "group", + "halo2_gadgets 0.2.0", + "halo2_proofs 0.2.0", + "hex", + "incrementalmerkletree", + "lazy_static", + "memuse", + "nonempty", + "pasta_curves", + "rand 0.8.5", + "reddsa", + "serde", + "subtle", + "tracing", + "zcash_note_encryption 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ordered-map" version = "0.4.2" @@ -3198,11 +3310,11 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260" dependencies = [ - "pin-project-internal 1.0.10", + "pin-project-internal 1.0.11", ] [[package]] @@ -3218,9 +3330,9 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74" dependencies = [ "proc-macro2 1.0.36", "quote 1.0.15", @@ -3317,7 +3429,7 @@ dependencies = [ "atomic", "crossbeam-queue", "futures", - "pin-project 1.0.10", + "pin-project 1.0.11", "static_assertions", "thiserror", ] @@ -3680,9 +3792,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.1" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" dependencies = [ "autocfg 1.1.0", "crossbeam-deque", @@ -3692,14 +3804,13 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.1" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" dependencies = [ "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "lazy_static", "num_cpus", ] @@ -3792,9 +3903,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.10" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" +checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" dependencies = [ "base64", "bytes", @@ -3823,6 +3934,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-rustls", + "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", @@ -3966,7 +4078,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.10", + "semver 1.0.12", ] [[package]] @@ -3983,9 +4095,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "0.3.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360" +checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" dependencies = [ "base64", ] @@ -4123,9 +4235,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41d061efea015927ac527063765e73601444cdc344ba855bc7bd44578b25e1c" +checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1" [[package]] name = "semver-parser" @@ -4135,9 +4247,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "sentry" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904eca4fb30c6112a1dae60c0a9e29cfb42f42129da4260f1ee20e94151b62e3" +checksum = "73642819e7fa63eb264abc818a2f65ac8764afbe4870b5ee25bcecc491be0d4c" dependencies = [ "httpdate", "reqwest", @@ -4149,21 +4261,21 @@ dependencies = [ [[package]] name = "sentry-backtrace" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1671189d1b759879fa4bdde46c50a499abb14332ed81f84fc6f60658f41b2fdb" +checksum = "49bafa55eefc6dbc04c7dac91e8c8ab9e89e9414f3193c105cabd991bbc75134" dependencies = [ "backtrace", - "lazy_static", + "once_cell", "regex", "sentry-core", ] [[package]] name = "sentry-contexts" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db80ceff16bb1a4b2689b8758e5e61e405fc4d8ff9f2d1b5b845b76ce37fa34e" +checksum = "c63317c4051889e73f0b00ce4024cae3e6a225f2e18a27d2c1522eb9ce2743da" dependencies = [ "hostname", "libc", @@ -4174,11 +4286,11 @@ dependencies = [ [[package]] name = "sentry-core" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c9f509d3959ed4dbbd80ca42572caad682aaa1cdd92c719e0815d0e87f82c96" +checksum = "5a4591a2d128af73b1b819ab95f143bc6a2fbe48cd23a4c45e1ee32177e66ae6" dependencies = [ - "lazy_static", + "once_cell", "rand 0.8.5", "sentry-types", "serde", @@ -4187,9 +4299,9 @@ dependencies = [ [[package]] name = "sentry-tracing" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e2796e40502893ed0d04646e507f91e74f1cbf09e370d42bb1cdbcaeeca9bb" +checksum = "ea50bcf843510179a7ba41c9fe4d83a8958e9f8adf707ec731ff999297536344" dependencies = [ "sentry-core", "tracing-core", @@ -4198,9 +4310,9 @@ dependencies = [ [[package]] name = "sentry-types" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "254b600e93e9ef00a48382c9f1e86d27884bd9a5489efa4eb9210c20c72e88a6" +checksum = "823923ae5f54a729159d720aa12181673044ee5c79cbda3be09e56f885e5468f" dependencies = [ "debugid", "getrandom 0.2.5", @@ -4747,10 +4859,11 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.19.2" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439" +checksum = "57aec3cfa4c296db7255446efb4928a6be304b431a806216105542a67b6ca82e" dependencies = [ + "autocfg 1.1.0", "bytes", "libc", "memchr", @@ -4890,7 +5003,7 @@ dependencies = [ "hyper", "hyper-timeout", "percent-encoding", - "pin-project 1.0.10", + "pin-project 1.0.11", "prost", "prost-derive", "tokio", @@ -5003,7 +5116,7 @@ dependencies = [ "futures", "humantime-serde", "itertools", - "pin-project 1.0.10", + "pin-project 1.0.11", "rand 0.8.5", "retry-error", "serde", @@ -5117,7 +5230,7 @@ dependencies = [ "futures", "humantime-serde", "itertools", - "pin-project 1.0.10", + "pin-project 1.0.11", "rand 0.8.5", "retain_mut", "serde", @@ -5287,7 +5400,7 @@ dependencies = [ "async_executors", "futures", "native-tls", - "pin-project 1.0.10", + "pin-project 1.0.11", "tokio", "tokio-native-tls", "tokio-util 0.6.9", @@ -5305,15 +5418,15 @@ dependencies = [ [[package]] name = "tower" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", "hdrhistogram", "indexmap", - "pin-project 1.0.10", + "pin-project 1.0.11", "pin-project-lite", "rand 0.8.5", "slab", @@ -5326,27 +5439,30 @@ dependencies = [ [[package]] name = "tower-batch" -version = "0.2.26" +version = "0.2.28" dependencies = [ "color-eyre", "ed25519-zebra", "futures", "futures-core", - "pin-project 1.0.10", + "pin-project 1.0.11", "rand 0.8.5", + "rayon", "tokio", "tokio-test", + "tokio-util 0.7.3", "tower", "tower-fallback", "tower-test", "tracing", "tracing-futures", + "zebra-consensus", "zebra-test", ] [[package]] name = "tower-fallback" -version = "0.2.22" +version = "0.2.28" dependencies = [ "futures-core", "pin-project 0.4.29", @@ -5394,7 +5510,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4546773ffeab9e4ea02b8872faa49bb616a80a7da66afc2f32688943f97efa7" dependencies = [ "futures-util", - "pin-project 1.0.10", + "pin-project 1.0.11", "tokio", "tokio-test", "tower-layer", @@ -5427,9 +5543,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.22" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03cfcb51380632a72d3111cb8d3447a8d908e577d31beeac006f836383d29a23" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" dependencies = [ "lazy_static", "valuable", @@ -5462,7 +5578,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "pin-project 1.0.10", + "pin-project 1.0.11", "tracing", ] @@ -5984,7 +6100,7 @@ dependencies = [ [[package]] name = "zcash_encoding" version = "0.1.0" -source = "git+https://github.com/ZcashFoundation/librustzcash.git?rev=6d75718076e592a41b6bd6ec916dc15420e4cc3c#6d75718076e592a41b6bd6ec916dc15420e4cc3c" +source = "git+https://github.com/ZcashFoundation/librustzcash.git?rev=4567a37ceccbd506a58aaaded39ba14c952c1510#4567a37ceccbd506a58aaaded39ba14c952c1510" dependencies = [ "byteorder", "nonempty", @@ -6016,7 +6132,7 @@ dependencies = [ [[package]] name = "zcash_note_encryption" version = "0.1.0" -source = "git+https://github.com/ZcashFoundation/librustzcash.git?rev=6d75718076e592a41b6bd6ec916dc15420e4cc3c#6d75718076e592a41b6bd6ec916dc15420e4cc3c" +source = "git+https://github.com/ZcashFoundation/librustzcash.git?rev=4567a37ceccbd506a58aaaded39ba14c952c1510#4567a37ceccbd506a58aaaded39ba14c952c1510" dependencies = [ "chacha20", "chacha20poly1305", @@ -6039,7 +6155,7 @@ dependencies = [ "bs58", "byteorder", "chacha20poly1305", - "equihash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "equihash 0.1.0", "ff", "fpe", "group", @@ -6050,7 +6166,7 @@ dependencies = [ "lazy_static", "memuse", "nonempty", - "orchard", + "orchard 0.1.0", "rand 0.8.5", "rand_core 0.6.3", "ripemd", @@ -6063,8 +6179,9 @@ dependencies = [ [[package]] name = "zcash_primitives" -version = "0.6.0" -source = "git+https://github.com/ZcashFoundation/librustzcash.git?rev=6d75718076e592a41b6bd6ec916dc15420e4cc3c#6d75718076e592a41b6bd6ec916dc15420e4cc3c" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fbb401f5dbc482b831954aaa7cba0a8fe148241db6d19fe7cebda78252ca680" dependencies = [ "aes", "bip0039", @@ -6072,9 +6189,45 @@ dependencies = [ "blake2b_simd 1.0.0", "blake2s_simd", "bls12_381", + "bs58", "byteorder", "chacha20poly1305", - "equihash 0.1.0 (git+https://github.com/ZcashFoundation/librustzcash.git?rev=6d75718076e592a41b6bd6ec916dc15420e4cc3c)", + "equihash 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ff", + "fpe", + "group", + "hdwallet", + "hex", + "incrementalmerkletree", + "jubjub", + "lazy_static", + "memuse", + "nonempty", + "orchard 0.2.0", + "rand 0.8.5", + "rand_core 0.6.3", + "ripemd", + "secp256k1", + "sha2", + "subtle", + "zcash_encoding 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "zcash_note_encryption 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "zcash_primitives" +version = "0.7.0" +source = "git+https://github.com/ZcashFoundation/librustzcash.git?rev=4567a37ceccbd506a58aaaded39ba14c952c1510#4567a37ceccbd506a58aaaded39ba14c952c1510" +dependencies = [ + "aes", + "bip0039", + "bitvec", + "blake2b_simd 1.0.0", + "blake2s_simd", + "bls12_381", + "byteorder", + "chacha20poly1305", + "equihash 0.2.0 (git+https://github.com/ZcashFoundation/librustzcash.git?rev=4567a37ceccbd506a58aaaded39ba14c952c1510)", "ff", "fpe", "group", @@ -6084,19 +6237,19 @@ dependencies = [ "lazy_static", "memuse", "nonempty", - "orchard", + "orchard 0.2.0", "rand 0.8.5", "rand_core 0.6.3", "sha2", "subtle", - "zcash_encoding 0.1.0 (git+https://github.com/ZcashFoundation/librustzcash.git?rev=6d75718076e592a41b6bd6ec916dc15420e4cc3c)", - "zcash_note_encryption 0.1.0 (git+https://github.com/ZcashFoundation/librustzcash.git?rev=6d75718076e592a41b6bd6ec916dc15420e4cc3c)", + "zcash_encoding 0.1.0 (git+https://github.com/ZcashFoundation/librustzcash.git?rev=4567a37ceccbd506a58aaaded39ba14c952c1510)", + "zcash_note_encryption 0.1.0 (git+https://github.com/ZcashFoundation/librustzcash.git?rev=4567a37ceccbd506a58aaaded39ba14c952c1510)", ] [[package]] name = "zcash_proofs" -version = "0.6.0" -source = "git+https://github.com/ZcashFoundation/librustzcash.git?rev=6d75718076e592a41b6bd6ec916dc15420e4cc3c#6d75718076e592a41b6bd6ec916dc15420e4cc3c" +version = "0.7.0" +source = "git+https://github.com/ZcashFoundation/librustzcash.git?rev=4567a37ceccbd506a58aaaded39ba14c952c1510#4567a37ceccbd506a58aaaded39ba14c952c1510" dependencies = [ "bellman", "blake2b_simd 1.0.0", @@ -6109,7 +6262,7 @@ dependencies = [ "lazy_static", "minreq", "rand_core 0.6.3", - "zcash_primitives 0.6.0 (git+https://github.com/ZcashFoundation/librustzcash.git?rev=6d75718076e592a41b6bd6ec916dc15420e4cc3c)", + "zcash_primitives 0.7.0 (git+https://github.com/ZcashFoundation/librustzcash.git?rev=4567a37ceccbd506a58aaaded39ba14c952c1510)", ] [[package]] @@ -6123,17 +6276,17 @@ dependencies = [ "cc", "libc", "memuse", - "orchard", + "orchard 0.1.0", "rand_core 0.6.3", "tracing", "zcash_encoding 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "zcash_note_encryption 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "zcash_primitives 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "zcash_primitives 0.6.0", ] [[package]] name = "zebra-chain" -version = "1.0.0-beta.11" +version = "1.0.0-beta.13" dependencies = [ "aes", "bech32", @@ -6149,23 +6302,25 @@ dependencies = [ "criterion", "displaydoc", "ed25519-zebra", - "equihash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "equihash 0.1.0", "fpe", "futures", "group", - "halo2_proofs", + "halo2_proofs 0.2.0", "hex", + "humantime", "incrementalmerkletree", "itertools", "jubjub", "lazy_static", - "orchard", + "orchard 0.2.0", "primitive-types", "proptest", "proptest-derive", "rand 0.8.5", "rand_chacha 0.3.1", "rand_core 0.6.3", + "rayon", "redjubjub", "ripemd", "secp256k1", @@ -6184,7 +6339,7 @@ dependencies = [ "zcash_encoding 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "zcash_history", "zcash_note_encryption 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "zcash_primitives 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "zcash_primitives 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "zebra-test", ] @@ -6194,7 +6349,7 @@ version = "1.0.0-beta.0" [[package]] name = "zebra-consensus" -version = "1.0.0-beta.11" +version = "1.0.0-beta.13" dependencies = [ "bellman", "blake2b_simd 1.0.0", @@ -6205,17 +6360,18 @@ dependencies = [ "displaydoc", "futures", "futures-util", - "halo2_proofs", + "halo2_proofs 0.2.0", "hex", "jubjub", "lazy_static", "metrics", "once_cell", - "orchard", + "orchard 0.2.0", "proptest", "proptest-derive", "rand 0.7.3", "rand 0.8.5", + "rayon", "serde", "spandoc", "thiserror", @@ -6236,7 +6392,7 @@ dependencies = [ [[package]] name = "zebra-network" -version = "1.0.0-beta.11" +version = "1.0.0-beta.13" dependencies = [ "arti-client", "bitflags", @@ -6246,13 +6402,15 @@ dependencies = [ "futures", "hex", "humantime-serde", + "indexmap", "lazy_static", "metrics", "ordered-map", - "pin-project 1.0.10", + "pin-project 1.0.11", "proptest", "proptest-derive", "rand 0.8.5", + "rayon", "regex", "serde", "static_assertions", @@ -6272,14 +6430,14 @@ dependencies = [ [[package]] name = "zebra-node-services" -version = "1.0.0-beta.11" +version = "1.0.0-beta.13" dependencies = [ "zebra-chain", ] [[package]] name = "zebra-rpc" -version = "1.0.0-beta.11" +version = "1.0.0-beta.13" dependencies = [ "chrono", "futures", @@ -6308,7 +6466,7 @@ dependencies = [ [[package]] name = "zebra-script" -version = "1.0.0-beta.11" +version = "1.0.0-beta.13" dependencies = [ "displaydoc", "hex", @@ -6321,7 +6479,7 @@ dependencies = [ [[package]] name = "zebra-state" -version = "1.0.0-beta.11" +version = "1.0.0-beta.13" dependencies = [ "bincode", "chrono", @@ -6329,7 +6487,7 @@ dependencies = [ "dirs", "displaydoc", "futures", - "halo2_proofs", + "halo2_proofs 0.2.0", "hex", "insta", "itertools", @@ -6340,6 +6498,7 @@ dependencies = [ "once_cell", "proptest", "proptest-derive", + "rayon", "regex", "rlimit", "rocksdb", @@ -6356,7 +6515,7 @@ dependencies = [ [[package]] name = "zebra-test" -version = "1.0.0-beta.11" +version = "1.0.0-beta.13" dependencies = [ "color-eyre", "futures", @@ -6381,7 +6540,7 @@ dependencies = [ [[package]] name = "zebra-utils" -version = "1.0.0-beta.11" +version = "1.0.0-beta.13" dependencies = [ "color-eyre", "hex", @@ -6396,16 +6555,18 @@ dependencies = [ [[package]] name = "zebrad" -version = "1.0.0-beta.11" +version = "1.0.0-beta.13" dependencies = [ "abscissa_core", "atty", "chrono", "color-eyre", + "console-subscriber", "dirs", "futures", "gumdrop", "hex", + "humantime", "humantime-serde", "hyper", "indexmap", @@ -6416,14 +6577,15 @@ dependencies = [ "metrics-exporter-prometheus", "num-integer", "once_cell", - "pin-project 1.0.10", + "pin-project 1.0.11", "proptest", "proptest-derive", "prost", "rand 0.8.5", + "rayon", "regex", "reqwest", - "semver 1.0.10", + "semver 1.0.12", "sentry", "sentry-tracing", "serde", diff --git a/Cargo.toml b/Cargo.toml index 9f89983a57e..e2958c15afe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,4 +66,4 @@ lto = "thin" [patch.crates-io] # Currently pointing to `download-sprout-params` branch. -zcash_proofs = { git = "https://github.com/ZcashFoundation/librustzcash.git", rev = "6d75718076e592a41b6bd6ec916dc15420e4cc3c" } +zcash_proofs = { git = "https://github.com/ZcashFoundation/librustzcash.git", rev = "4567a37ceccbd506a58aaaded39ba14c952c1510" } diff --git a/README.md b/README.md index 25b4d08017a..b16beef0ca2 100644 --- a/README.md +++ b/README.md @@ -25,51 +25,25 @@ consensus-compatible implementation of a Zcash node, currently under development. It can be used to join the Zcash peer-to-peer network, which helps keeping Zcash working by validating and broadcasting transactions, and maintaining the Zcash blockchain state in a distributed manner. + +[Zcash](https://doc.zebra.zfnd.org/zebrad/index.html#about-zcash) +is a cryptocurrency designed to preserve the user's privacy. +If you just want to send and receive Zcash then you don't need to use Zebra +directly. You can download a Zcash wallet application which will handle that +for you. + Please [join us on Discord](https://discord.gg/na6QZNd) if you'd like to find out more or get involved! -Zcash is a cryptocurrency designed to preserve the user's privacy. Like most -cryptocurrencies, it works by a collection of software nodes run by members of -the Zcash community or any other interested parties. The nodes talk to each -other in peer-to-peer fashion in order to maintain the state of the Zcash -blockchain. They also communicate with miners who create new blocks. When a -Zcash user sends Zcash, their wallet broadcasts transactions to these nodes -which will eventually reach miners, and the mined transaction will then go -through Zcash nodes until they reach the recipient's wallet which will report -the received Zcash to the recipient. - -The original Zcash node is named `zcashd` and is developed by the Electric Coin -Company as a fork of the original Bitcoin node. Zebra, on the other hand, is -an independent Zcash node implementation developed from scratch. Since they -implement the same protocol, `zcashd` and Zebra nodes can communicate with each -other and maintain the Zcash network together. +### Using Zebra -If you just want to send and receive Zcash then you don't need to use Zebra -directly. You can download a Zcash wallet application which will handle that -for you. (Eventually, Zebra can be used by wallets to implement their -functionality.) You would want to run Zebra if you want to contribute to the +You would want to run Zebra if you want to contribute to the Zcash network: the more nodes are run, the more reliable the network will be in terms of speed and resistance to denial of service attacks, for example. -These are some of the advantages or benefits of Zebra: - -- Better performance: since it was implemented from scratch in an async, parallelized way, Zebra - is currently faster than `zcashd`. -- Better security: since it is developed in a memory-safe language (Rust), Zebra - is less likely to be affected by memory-safety and correctness security bugs that - could compromise the environment where it is run. -- Better governance: with a new node deployment, there will be more developers - who can implement different features for the Zcash network. -- Dev accessibility: supports more developers, which gives new developers - options for contributing to Zcash protocol development. -- Runtime safety: with an independent implementation, the detection of consensus bugs - can happen quicker, reducing the risk of consensus splits. -- Spec safety: with several node implementations, it is much easier to notice - bugs and ambiguity in protocol specification. -- User options: different nodes present different features and tradeoffs for - users to decide on their preferred options. -- Additional contexts: wider target deployments for people to use a consensus - node in more contexts e.g. mobile, wasm, etc. +Zebra aims to be +[faster, more secure, and more easily extensible](https://doc.zebra.zfnd.org/zebrad/index.html#zebra-advantages) +than other Zcash implementations. ## Beta Releases @@ -78,11 +52,7 @@ Every few weeks, we release a new Zebra beta [release](https://github.com/ZcashF Zebra's network stack is interoperable with `zcashd`, and Zebra implements all the features required to reach Zcash network consensus. -The goals of the beta release series are for Zebra to act as a fully validating Zcash node, -for all active consensus rules as of NU5 activation. - Currently, Zebra validates all of the Zcash consensus rules for the NU5 network upgrade. -(As of the second NU5 activation on testnet.) But it may not validate any: - Undocumented rules derived from Bitcoin @@ -106,20 +76,26 @@ for your platform: 2. Install Zebra's build dependencies: - **libclang:** the `libclang`, `libclang-dev`, `llvm`, or `llvm-dev` packages, depending on your package manager - **clang** or another C++ compiler: `g++`, `Xcode`, or `MSVC` - 3. Run `cargo install --locked --git https://github.com/ZcashFoundation/zebra --tag v1.0.0-beta.11 zebrad` +3. Run `cargo install --locked --git https://github.com/ZcashFoundation/zebra --tag v1.0.0-beta.13 zebrad` 4. Run `zebrad start` (see [Running Zebra](https://zebra.zfnd.org/user/run.html) for more information) -If you're interested in testing out `zebrad` please feel free, but keep in mind -that there is a lot of key functionality still missing. - For more detailed instructions, refer to the [documentation](https://zebra.zfnd.org/user/install.html). +### Optional Features + +For performance reasons, some debugging and monitoring features are disabled in release builds. + +You can [enable these features](https://doc.zebra.zfnd.org/zebrad/index.html#zebra-feature-flags) using: +```sh +cargo install --features= ... +``` + ### System Requirements The recommended requirements for compiling and running `zebrad` are: - 4+ CPU cores - 16+ GB RAM -- 50GB+ available disk space for building binaries and storing finalized state +- 100 GB+ available disk space for building binaries and storing cached chain state - 100+ Mbps network connections We continuously test that our builds and tests pass on: @@ -171,8 +147,8 @@ If this is a problem for you, please [open a ticket.](https://github.com/ZcashFoundation/zebra/issues/new/choose) `zebrad`'s typical mainnet network usage is: -- Initial sync: 31 GB download -- Ongoing updates: 10-100 MB upload and download per day, depending on peer requests +- Initial sync: 40 GB download (in the longer term, several hundred GB are likely to be downloaded). +- Ongoing updates: 10 MB - 1 GB upload and download per day, depending on user-created transaction size, and peer requests Zebra also performs an initial sync every time its internal database version changes. @@ -206,18 +182,31 @@ So Zebra's state should always be valid, unless your OS or disk hardware is corr ## Known Issues There are a few bugs in Zebra that we're still working on fixing: -- [Old state versions are not deleted #1213](https://github.com/ZcashFoundation/zebra/issues/1213) - - When Zebra changes its state format, it does not delete the old state directory. You can delete old state directories if you need the space. -- [No Windows support #3801](https://github.com/ZcashFoundation/zebra/issues/3801) +- No Windows support [#3801](https://github.com/ZcashFoundation/zebra/issues/3801) - We used to test with Windows Server 2019, but not anymore; see issue for details +### Performance + +We are working on improving Zebra performance, the following are known issues: +- Send note commitment and history trees from the non-finalized state to the finalized state [#4824](https://github.com/ZcashFoundation/zebra/issues/4824) +- Speed up opening the database [#4822](https://github.com/ZcashFoundation/zebra/issues/4822) +- Revert note commitment and history trees when forking non-finalized chains [#4794](https://github.com/ZcashFoundation/zebra/issues/4794) +- Store only the first tree state in each identical series of tree states [#4784](https://github.com/ZcashFoundation/zebra/issues/4784) + +RPCs might also be slower than they used to be, we need to check: +- Revert deserializing state transactions in rayon threads [#4831](https://github.com/ZcashFoundation/zebra/issues/4831) + +Ongoing investigations: +- Find out which parts of CommitBlock/CommitFinalizedBlock are slow [#4823](https://github.com/ZcashFoundation/zebra/issues/4823) +- Mini-Epic: Stop tokio tasks running for a long time and blocking other tasks [#4747](https://github.com/ZcashFoundation/zebra/issues/4747) +- Investigate busiest tasks per tokio-console [#4583](https://github.com/ZcashFoundation/zebra/issues/4583) + ## Future Work Features: - Wallet functionality Performance and Reliability: -- Reliable syncing on Testnet - Reliable syncing under poor network conditions - Additional batch verification - Performance tuning diff --git a/book/src/dev/rfcs/0004-asynchronous-script-verification.md b/book/src/dev/rfcs/0004-asynchronous-script-verification.md index c442de9d936..23800002119 100644 --- a/book/src/dev/rfcs/0004-asynchronous-script-verification.md +++ b/book/src/dev/rfcs/0004-asynchronous-script-verification.md @@ -422,16 +422,3 @@ cleaner and the cost is probably not too large. - We need to pick a timeout for UTXO lookup. This should be long enough to account for the fact that we may start verifying blocks before all of their ancestors are downloaded. - -These optimisations can be delayed until after the initial implementation is -complete, and covered by tests: - -- Should we stop storing heights for non-coinbase UTXOs? (#2455) - -- Should we avoid storing any extra data for UTXOs, and just lookup the coinbase - flag and height using `outpoint.hash` and `tx_by_hash`? (#2455) - -- The maturity check can be skipped for UTXOs from the finalized state, -because Zebra only finalizes mature UTXOs. We could implement this -optimisation by adding a `Utxo::MatureCoinbase { output: transparent::Output }` -variant, which only performs the spend checks. (#2455) diff --git a/book/src/dev/rfcs/0005-state-updates.md b/book/src/dev/rfcs/0005-state-updates.md index 4f4e065d354..e47245ad175 100644 --- a/book/src/dev/rfcs/0005-state-updates.md +++ b/book/src/dev/rfcs/0005-state-updates.md @@ -268,20 +268,90 @@ is completely empty. The `Chain` type is defined by the following struct and API: ```rust -#[derive(Debug, Default, Clone)] -struct Chain { - blocks: BTreeMap>, - height_by_hash: HashMap, - tx_by_hash: HashMap, - - created_utxos: HashSet, - spent_utxos: HashSet, - sprout_anchors: HashSet, - sapling_anchors: HashSet, - sprout_nullifiers: HashSet, - sapling_nullifiers: HashSet, - orchard_nullifiers: HashSet, - partial_cumulative_work: PartialCumulativeWork, +#[derive(Debug, Clone)] +pub struct Chain { + // The function `eq_internal_state` must be updated every time a field is added to [`Chain`]. + /// The configured network for this chain. + network: Network, + + /// The contextually valid blocks which form this non-finalized partial chain, in height order. + pub(crate) blocks: BTreeMap, + + /// An index of block heights for each block hash in `blocks`. + pub height_by_hash: HashMap, + + /// An index of [`TransactionLocation`]s for each transaction hash in `blocks`. + pub tx_loc_by_hash: HashMap, + + /// The [`transparent::Utxo`]s created by `blocks`. + /// + /// Note that these UTXOs may not be unspent. + /// Outputs can be spent by later transactions or blocks in the chain. + // + // TODO: replace OutPoint with OutputLocation? + pub(crate) created_utxos: HashMap, + /// The [`transparent::OutPoint`]s spent by `blocks`, + /// including those created by earlier transactions or blocks in the chain. + pub(crate) spent_utxos: HashSet, + + /// The Sprout note commitment tree of the tip of this [`Chain`], + /// including all finalized notes, and the non-finalized notes in this chain. + pub(super) sprout_note_commitment_tree: sprout::tree::NoteCommitmentTree, + /// The Sprout note commitment tree for each anchor. + /// This is required for interstitial states. + pub(crate) sprout_trees_by_anchor: + HashMap, + /// The Sapling note commitment tree of the tip of this [`Chain`], + /// including all finalized notes, and the non-finalized notes in this chain. + pub(super) sapling_note_commitment_tree: sapling::tree::NoteCommitmentTree, + /// The Sapling note commitment tree for each height. + pub(crate) sapling_trees_by_height: BTreeMap, + /// The Orchard note commitment tree of the tip of this [`Chain`], + /// including all finalized notes, and the non-finalized notes in this chain. + pub(super) orchard_note_commitment_tree: orchard::tree::NoteCommitmentTree, + /// The Orchard note commitment tree for each height. + pub(crate) orchard_trees_by_height: BTreeMap, + /// The ZIP-221 history tree of the tip of this [`Chain`], + /// including all finalized blocks, and the non-finalized `blocks` in this chain. + pub(crate) history_tree: HistoryTree, + + /// The Sprout anchors created by `blocks`. + pub(crate) sprout_anchors: MultiSet, + /// The Sprout anchors created by each block in `blocks`. + pub(crate) sprout_anchors_by_height: BTreeMap, + /// The Sapling anchors created by `blocks`. + pub(crate) sapling_anchors: MultiSet, + /// The Sapling anchors created by each block in `blocks`. + pub(crate) sapling_anchors_by_height: BTreeMap, + /// The Orchard anchors created by `blocks`. + pub(crate) orchard_anchors: MultiSet, + /// The Orchard anchors created by each block in `blocks`. + pub(crate) orchard_anchors_by_height: BTreeMap, + + /// The Sprout nullifiers revealed by `blocks`. + pub(super) sprout_nullifiers: HashSet, + /// The Sapling nullifiers revealed by `blocks`. + pub(super) sapling_nullifiers: HashSet, + /// The Orchard nullifiers revealed by `blocks`. + pub(super) orchard_nullifiers: HashSet, + + /// Partial transparent address index data from `blocks`. + pub(super) partial_transparent_transfers: HashMap, + + /// The cumulative work represented by `blocks`. + /// + /// Since the best chain is determined by the largest cumulative work, + /// the work represented by finalized blocks can be ignored, + /// because they are common to all non-finalized chains. + pub(super) partial_cumulative_work: PartialCumulativeWork, + + /// The chain value pool balances of the tip of this [`Chain`], + /// including the block value pool changes from all finalized blocks, + /// and the non-finalized blocks in this chain. + /// + /// When a new chain is created from the finalized tip, + /// it is initialized with the finalized tip chain value pool balances. + pub(crate) chain_value_pools: ValueBalance, } ``` @@ -293,7 +363,7 @@ Push a block into a chain as the new tip - Add the block's hash to `height_by_hash` - Add work to `self.partial_cumulative_work` - For each `transaction` in `block` - - Add key: `transaction.hash` and value: `(height, tx_index)` to `tx_by_hash` + - Add key: `transaction.hash` and value: `(height, tx_index)` to `tx_loc_by_hash` - Add created utxos to `self.created_utxos` - Add spent utxos to `self.spent_utxos` - Add nullifiers to the appropriate `self._nullifiers` @@ -310,7 +380,7 @@ Remove the lowest height block of the non-finalized portion of a chain. - Remove the block's hash from `self.height_by_hash` - Subtract work from `self.partial_cumulative_work` - For each `transaction` in `block` - - Remove `transaction.hash` from `tx_by_hash` + - Remove `transaction.hash` from `tx_loc_by_hash` - Remove created utxos from `self.created_utxos` - Remove spent utxos from `self.spent_utxos` - Remove the nullifiers from the appropriate `self._nullifiers` @@ -340,7 +410,7 @@ Remove the highest height block of the non-finalized portion of a chain. - Remove the corresponding hash from `self.height_by_hash` - Subtract work from `self.partial_cumulative_work` - for each `transaction` in `block` - - remove `transaction.hash` from `tx_by_hash` + - remove `transaction.hash` from `tx_loc_by_hash` - Remove created utxos from `self.created_utxos` - Remove spent utxos from `self.spent_utxos` - Remove the nullifiers from the appropriate `self._nullifiers` @@ -365,7 +435,7 @@ parent block is the tip of the finalized state. This implementation should be handled by `#[derive(Default)]`. 1. initialise cumulative data members - - Construct an empty `self.blocks`, `height_by_hash`, `tx_by_hash`, + - Construct an empty `self.blocks`, `height_by_hash`, `tx_loc_by_hash`, `self.created_utxos`, `self.spent_utxos`, `self._anchors`, `self._nullifiers` - Zero `self.partial_cumulative_work` @@ -1102,13 +1172,14 @@ Returns Implemented by querying: -- (non-finalized) the `tx_by_hash` map (to get the block that contains the +- (non-finalized) the `tx_loc_by_hash` map (to get the block that contains the transaction) of each chain starting with the best chain, and then find block that chain's `blocks` (to get the block containing the transaction data) -- (finalized) the `tx_by_hash` tree (to get the block that contains the - transaction) and then `block_by_height` tree (to get the block containing - the transaction data), if the transaction is not in any non-finalized chain +- (finalized) the `tx_loc_by_hash` tree (to get the block that contains the + transaction) and then `block_header_by_height` tree (to get the block + containing the transaction data), if the transaction is not in any + non-finalized chain ### `Request::Block(block::Hash)` [request-block]: #request-block @@ -1125,8 +1196,9 @@ Implemented by querying: - (non-finalized) the `height_by_hash` of each chain starting with the best chain, then find block that chain's `blocks` (to get the block data) -- (finalized) the `height_by_hash` tree (to get the block height) and then - the `block_by_height` tree (to get the block data), if the block is not in any non-finalized chain +- (finalized) the `height_by_hash` tree (to get the block height) and then the + `block_header_by_height` tree (to get the block data), if the block is not in + any non-finalized chain ### `Request::AwaitSpendableUtxo { outpoint: OutPoint, spend_height: Height, spend_restriction: SpendRestriction }` diff --git a/book/src/dev/tokio-console.md b/book/src/dev/tokio-console.md new file mode 100644 index 00000000000..4a10d27e5cf --- /dev/null +++ b/book/src/dev/tokio-console.md @@ -0,0 +1,38 @@ +# `tokio-console` support + +`tokio-console` is a diagnostics and debugging tool for asynchronous Rust programs. This tool can be +useful to lint runtime behavior, collect diagnostic data from processes, and debugging performance +issues. ["it's like top(1) for tasks!"][top] + +### Setup + +Support for `tokio-console` is not enabled by default for zebrad. To activate this feature, run: + ```sh + $ RUSTFLAGS="--cfg tokio_unstable" cargo build --no-default-features --features="tokio-console" --bin zebrad + ``` + +Install [`tokio-console`][install]. + +Then start `zebrad` however you wish. + +When `zebrad` is running, run: +``` +$ tokio-console +``` + +The default options are used, so `tokio-console` should connect to the running `zebrad` without other configuration. + +### Example + +image + + +### More + +For more details, see the [`tokio` docs][enabling_tokio_instrumentation]. + + +[top]: https://github.com/tokio-rs/console#extremely-cool-and-amazing-screenshots +[install]: https://github.com/tokio-rs/console#running-the-console] +[enabling_tokio_instrumentation]: https://github.com/tokio-rs/console/blob/main/console-subscriber/README.md#enabling-tokio-instrumentation + diff --git a/book/src/user/install.md b/book/src/user/install.md index 4a48d9261ff..7ce8fafbb8c 100644 --- a/book/src/user/install.md +++ b/book/src/user/install.md @@ -9,7 +9,7 @@ for your platform: 2. Install Zebra's build dependencies: - **libclang:** the `libclang`, `libclang-dev`, `llvm`, or `llvm-dev` packages, depending on your package manager - **clang** or another C++ compiler: `g++`, `Xcode`, or `MSVC` -3. Run `cargo install --locked --git https://github.com/ZcashFoundation/zebra --tag v1.0.0-beta.11 zebrad` +3. Run `cargo install --locked --git https://github.com/ZcashFoundation/zebra --tag v1.0.0-beta.13 zebrad` 4. Run `zebrad start` (see [Running Zebra](run.md) for more information) If you're interested in testing out `zebrad` please feel free, but keep in mind diff --git a/book/src/user/lightwalletd.md b/book/src/user/lightwalletd.md index 8c0fdd9586f..5930a89c874 100644 --- a/book/src/user/lightwalletd.md +++ b/book/src/user/lightwalletd.md @@ -1,6 +1,6 @@ # Running lightwalletd with zebra -Starting on [v1.0.0-beta.11](https://github.com/ZcashFoundation/zebra/releases/tag/v1.0.0-beta.11), the Zebra RPC methods are fully featured to run a lightwalletd service backed by zebrad. +Zebra's RPC methods can support a lightwalletd service backed by zebrad. Contents: @@ -9,7 +9,7 @@ Contents: - [RPC section](#rpc-section) - [Sync Zebra](#sync-zebra) - [Download and build lightwalletd](#download-and-build-lightwalletd) -- [Sync lightwalled](#sync-lightwalled) +- [Sync lightwalletd](#sync-lightwalletd) - [Run tests](#run-tests) - [Connect wallet to lightwalletd](#connect-wallet-to-lightwalletd) - [Download and build the cli-wallet](#download-and-build-the-cli-wallet) @@ -96,7 +96,7 @@ make install If everything went good you should have a `lightwalletd` binary in `~/go/bin/`. -## Sync lightwalled +## Sync lightwalletd [#sync-lightwalletd]: (#sync-lightwalletd) Please make sure you have zebrad running (with RPC endpoint and up to date blockchain) to synchronize lightwalletd. diff --git a/book/src/user/metrics.md b/book/src/user/metrics.md index 5fc69e9304d..c2fb19ed742 100644 --- a/book/src/user/metrics.md +++ b/book/src/user/metrics.md @@ -1,6 +1,7 @@ # Zebra Metrics -Zebra has support for Prometheus, configured using the [`MetricsSection`][metrics_section]. +Zebra has support for Prometheus, configured using the `prometheus` compile-time feature, +and the [`MetricsSection`][metrics_section] runtime configuration. This requires supporting infrastructure to collect and visualize metrics, for example: diff --git a/book/src/user/requirements.md b/book/src/user/requirements.md index f8421eb399c..c5701045693 100644 --- a/book/src/user/requirements.md +++ b/book/src/user/requirements.md @@ -1,6 +1,7 @@ # System Requirements We usually build `zebrad` on systems with: + - 2+ CPU cores - 7+ GB RAM - 14+ GB of disk space @@ -9,11 +10,13 @@ On many-core machines (like, 32-core) the build is very fast; on 2-core machines it's less fast. We continuously test that our builds and tests pass on: + - macOS Big Sur 11.0 - Ubuntu 18.04 / the latest LTS - Debian Buster We usually run `zebrad` on systems with: + - 4+ CPU cores - 16+ GB RAM - 50GB+ available disk space for finalized state @@ -26,11 +29,12 @@ tested its exact limits yet. ## Sentry Production Monitoring -Compile Zebra with `--features enable-sentry` to monitor it using Sentry in production. +Compile Zebra with `--features sentry` to monitor it using Sentry in production. ## Lightwalletd Test Requirements To test Zebra's `lightwalletd` RPC methods: + - compile Zebra with the `--features lightwalletd-grpc-tests` - install a `lightwalletd` binary - Zebra's tests currently target [adityapk00/lightwalletd](https://github.com/adityapk00/lightwalletd) diff --git a/book/src/user/supported-platforms.md b/book/src/user/supported-platforms.md new file mode 100644 index 00000000000..c51e621cbc8 --- /dev/null +++ b/book/src/user/supported-platforms.md @@ -0,0 +1,49 @@ +# Platform Support + +Support for different platforms are organized into three tiers, each with a +different set of guarantees. For more information on the policies for platforms +at each tier, see the [Platform Tier Policy](platform-tier-policy.md). + +Platforms are identified by their Rust "target triple" which is a string composed by +`--`. + +## Tier 1 + +Tier 1 platforms can be thought of as "guaranteed to work". The Zebra project +builds official binary releases for each tier 1 platform, and automated testing +ensures that each tier 1 platform builds and passes tests after each change. + +For the full requirements, see [Tier 1 platform policy](platform-tier-policy.md#tier-1-platform-policy) in the Platform Tier Policy. + +| platform | os | notes | rust | artifacts +| -------|-------|-------|-------|------- +| `x86_64-unknown-linux-gnu` | [Debian 11](https://www.debian.org/releases/bullseye/) | 64-bit | [latest stable release](https://github.com/rust-lang/rust/releases) | Docker + +## Tier 2 + +Tier 2 platforms can be thought of as "guaranteed to build". The Zebra project +builds in CI for each tier 2 platform, and automated builds ensure that each +tier 2 platform builds after each change. Not all automated tests are run so it's +not guaranteed to produce a working build, and official builds are not available, +but tier 2 platforms often work to quite a good degree and patches are always +welcome! + +For the full requirements, see [Tier 2 platform policy](platform-tier-policy.md#tier-2-platform-policy) in the Platform Tier Policy. + +| platform | os | notes | rust | artifacts +| -------|-------|-------|-------|------- +| `x86_64-apple-darwin` | [GitHub macos-latest](https://github.com/actions/virtual-environments#available-environments) | 64-bit | [latest stable release](https://github.com/rust-lang/rust/releases) | N/A +| `x86_64-unknown-linux-gnu` | [GitHub ubuntu-latest](https://github.com/actions/virtual-environments#available-environments) | 64-bit | [latest stable release](https://github.com/rust-lang/rust/releases) | N/A +| `x86_64-unknown-linux-gnu` | [GitHub ubuntu-latest](https://github.com/actions/virtual-environments#available-environments) | 64-bit | [latest beta release](https://github.com/rust-lang/rust/blob/beta/src/version) | N/A + +## Tier 3 + +Tier 3 platforms are those which the Zebra codebase has support for, but which +the Zebra project does not build or test automatically, so they may or may not +work. Official builds are not available. + +For the full requirements, see [Tier 3 platform policy](platform-tier-policy.md#tier-3-platform-policy) in the Platform Tier Policy. + +| platform | os | notes | rust | artifacts +| -------|-------|-------|-------|------- +| `aarch64-unknown-linux-gnu` | [Debian 11](https://www.debian.org/releases/bullseye/) | 64-bit | [latest stable release](https://github.com/rust-lang/rust/releases) | N/A diff --git a/book/src/user/target-tier-policies.md b/book/src/user/target-tier-policies.md new file mode 100644 index 00000000000..d213c13f5b4 --- /dev/null +++ b/book/src/user/target-tier-policies.md @@ -0,0 +1,241 @@ +# Platform Tier Policy + +## Table of Contents + +- [Platform Tier Policy](#platform-tier-policy) + - [Table of Contents](#table-of-contents) + - [General](#general) + - [Tier 3 platform policy](#tier-3-platform-policy) + - [Tier 2 platform policy](#tier-2-platform-policy) + - [Tier 1 platform policy](#tier-1-platform-policy) + +## General + +The Zcash Foundation provides three tiers of platform support, modeled after the +[Rust Target Tier Policy](https://doc.rust-lang.org/stable/rustc/target-tier-policy.html): + +- The Zcash Foundation provides no guarantees about tier 3 platforms; they may + or may not build with the actual codebase. +- Zebra's continuous integration checks that tier 2 platforms will always build, + but they may or may not pass tests. +- Zebra's continuous integration checks that tier 1 platforms will always build + and pass tests. + +Adding a new tier 3 platform imposes minimal requirements; but we focus +primarily on avoiding disruption to ongoing Zebra development. + +Tier 2 and tier 1 platforms place work on Zcash Foundation developers as a whole, +to avoid breaking the platform. These tiers require commensurate and ongoing +efforts from the maintainers of the platform, to demonstrate value and to +minimize any disruptions to ongoing Zebra development. + +This policy defines the requirements for accepting a proposed platform at a +given level of support. + +Each tier is based on all the requirements from the previous tier, unless +overridden by a stronger requirement. + +While these criteria attempt to document the policy, that policy still involves +human judgment. Targets must fulfill the spirit of the requirements as well, as +determined by the judgment of the Zebra team. + +For a list of all supported platforms and their corresponding tiers ("tier 3", +"tier 2", or "tier 1"), see +[platform support](platform-support.md). + +Note that a platform must have already received approval for the next lower +tier, and spent a reasonable amount of time at that tier, before making a +proposal for promotion to the next higher tier; this is true even if a platform +meets the requirements for several tiers at once. This policy leaves the +precise interpretation of "reasonable amount of time" up to the Zebra team. + +The availability or tier of a platform in stable Zebra is not a hard stability +guarantee about the future availability or tier of that platform. Higher-level +platform tiers are an increasing commitment to the support of a platform, and +we will take that commitment and potential disruptions into account when +evaluating the potential demotion or removal of a platform that has been part +of a stable release. The promotion or demotion of a platform will not generally +affect existing stable releases, only current development and future releases. + +In this policy, the words "must" and "must not" specify absolute requirements +that a platform must meet to qualify for a tier. The words "should" and "should +not" specify requirements that apply in almost all cases, but for which the +Zebra team may grant an exception for good reason. The word "may" indicates +something entirely optional, and does not indicate guidance or recommendations. +This language is based on [IETF RFC 2119](https://tools.ietf.org/html/rfc2119). + +## Tier 3 platform policy + +At this tier, the Zebra project provides no official support for a platform, so +we place minimal requirements on the introduction of platforms. + +- A tier 3 platform must have a designated developer or developers (the "platform + maintainers") on record to be CCed when issues arise regarding the platform. + (The mechanism to track and CC such developers may evolve over time.) +- Target names should not introduce undue confusion or ambiguity unless + absolutely necessary to maintain ecosystem compatibility. For example, if + the name of the platform makes people extremely likely to form incorrect + beliefs about what it targets, the name should be changed or augmented to + disambiguate it. +- Tier 3 platforms must not impose burden on the authors of pull requests, or + other developers in the community, to maintain the platform. In particular, + do not post comments (automated or manual) on a PR that derail or suggest a + block on the PR based on a tier 3 platform. Do not send automated messages or + notifications (via any medium, including via `@`) to a PR author or others + involved with a PR regarding a tier 3 platform, unless they have opted into + such messages. +- Patches adding or updating tier 3 platforms must not break any existing tier 2 + or tier 1 platform, and must not knowingly break another tier 3 platform + without approval of either the Zebra team of the other tier 3 platform. + +If a tier 3 platform stops meeting these requirements, or the platform maintainers +no longer have interest or time, or the platform shows no signs of activity and +has not built for some time, or removing the platform would improve the quality +of the Zebra codebase, we may post a PR to remove support for that platform. Any such PR will be CCed +to the platform maintainers (and potentially other people who have previously +worked on the platform), to check potential interest in improving the situation. + +## Tier 2 platform policy + +At this tier, the Zebra project guarantees that a platform builds, and will reject +patches that fail to build on a platform. Thus, we place requirements that ensure +the platform will not block forward progress of the Zebra project. + +A proposed new tier 2 platform must be reviewed and approved by Zebra team based +on these requirements. + +In addition, the devops team must approve the integration of the platform +into Continuous Integration (CI), and the tier 2 CI-related requirements. This +review and approval may take place in a PR adding the platform to CI, or simply +by a devops team member reporting the outcome of a team discussion. + +- Tier 2 platforms must implement all the Zcash consensus rules. + Other Zebra features and binaries may be disabled, on a case-by-case basis. +- A tier 2 platform must have a designated team of developers (the "platform + maintainers") available to consult on platform-specific build-breaking issues. + This team must have at least 1 developer. +- The platform must not place undue burden on Zebra developers not specifically + concerned with that platform. Zebra developers are expected to not gratuitously + break a tier 2 platform, but are not expected to become experts in every tier 2 + platform, and are not expected to provide platform-specific implementations for + every tier 2 platform. +- The platform must provide documentation for the Zcash community explaining how + to build for their platform, and explaining how to run tests for the platform. + If at all possible, this documentation should show how to run Zebra programs + and tests for the platform using emulation, to allow anyone to do so. If the + platform cannot be feasibly emulated, the documentation should document the + required physical hardware or cloud systems. +- The platform must document its baseline expectations for the features or + versions of CPUs, operating systems, and any other dependencies. +- The platform must build reliably in CI, for all components that Zebra's CI + considers mandatory. + - Since a working Rust compiler is required to build Zebra, + the platform must be a [Rust tier 1 platform](https://rust-lang.github.io/rustup-components-history/). +- The Zebra team may additionally require that a subset of tests pass in + CI. In particular, this requirement may apply if the tests in question provide + substantial value via early detection of critical problems. +- Building the platform in CI must not take substantially longer than the current + slowest platform in CI, and should not substantially raise the maintenance + burden of the CI infrastructure. This requirement is subjective, to be + evaluated by the devops team, and will take the community importance + of the platform into account. +- Test failures on tier 2 platforms will be handled on a case-by-case basis. + Depending on the severity of the failure, the Zebra team may decide to: + - disable the test on that platform, + - require a fix before the next release, or + - remove the platform from tier 2. +- The platform maintainers should regularly run the testsuite for the platform, + and should fix any test failures in a reasonably timely fashion. +- All requirements for tier 3 apply. + +A tier 2 platform may be demoted or removed if it no longer meets these +requirements. Any proposal for demotion or removal will be CCed to the platform +maintainers, and will be communicated widely to the Zcash community before being +dropped from a stable release. (The amount of time between such communication +and the next stable release may depend on the nature and severity of the failed +requirement, the timing of its discovery, whether the platform has been part of a +stable release yet, and whether the demotion or removal can be a planned and +scheduled action.) + +## Tier 1 platform policy + +At this tier, the Zebra project guarantees that a platform builds and passes all +tests, and will reject patches that fail to build or pass the testsuite on a +platform. We hold tier 1 platforms to our highest standard of requirements. + +A proposed new tier 1 platform must be reviewed and approved by the Zebra team +based on these requirements. In addition, the release team must approve the +viability and value of supporting the platform. + +In addition, the devops team must approve the integration of the platform +into Continuous Integration (CI), and the tier 1 CI-related requirements. This +review and approval may take place in a PR adding the platform to CI, by a +devops team member reporting the outcome of a team discussion. + +- Tier 1 platforms must implement Zebra's standard production feature set, + including the network protocol, mempool, cached state, and RPCs. + Exceptions may be made on a case-by-case basis. + - Zebra must have reasonable security, performance, and robustness on that platform. + These requirements are subjective, and determined by consensus of the Zebra team. + - Internal developer tools and manual testing tools may be disabled for that platform. +- The platform must serve the ongoing needs of multiple production + users of Zebra across multiple organizations or projects. These requirements + are subjective, and determined by consensus of the Zebra team. A tier 1 + platform may be demoted or removed if it becomes obsolete or no longer meets + this requirement. +- The platform must build and pass tests reliably in CI, for all components that + Zebra's CI considers mandatory. + - Test failures on tier 1 platforms will be handled on a case-by-case basis. + Depending on the severity of the failure, the Zebra team may decide to: + - disable the test on that platform, + - require a fix before the next release, + - require a fix before any other PRs merge, or + - remove the platform from tier 1. + - The platform must not disable an excessive number of tests or pieces of tests + in the testsuite in order to do so. This is a subjective requirement. +- Building the platform and running the testsuite for the platform must not take + substantially longer than other platforms, and should not substantially raise + the maintenance burden of the CI infrastructure. + - In particular, if building the platform takes a reasonable amount of time, + but the platform cannot run the testsuite in a timely fashion due to low + performance, that alone may prevent the platform from qualifying as tier 1. +- If running the testsuite requires additional infrastructure (such as physical + systems running the platform), the platform maintainers must arrange to provide + such resources to the Zebra project, to the satisfaction and approval of the + Zebra devops team. + - Such resources may be provided via cloud systems, via emulation, or via + physical hardware. + - If the platform requires the use of emulation to meet any of the tier + requirements, the Zebra team must have high + confidence in the accuracy of the emulation, such that discrepancies + between emulation and native operation that affect test results will + constitute a high-priority bug in either the emulation, the Rust + implementation of the platform, or the Zebra implementation for the platform. + - If it is not possible to run the platform via emulation, these resources + must additionally be sufficient for the Zebra devops team to make them + available for access by Zebra team members, for the purposes of development + and testing. (Note that the responsibility for doing platform-specific + development to keep the platform well maintained remains with the platform + maintainers. This requirement ensures that it is possible for other + Zebra developers to test the platform, but does not obligate other Zebra + developers to make platform-specific fixes.) + - Resources provided for CI and similar infrastructure must be available for + continuous exclusive use by the Zebra project. Resources provided + for access by Zebra team members for development and testing must be + available on an exclusive basis when in use, but need not be available on a + continuous basis when not in use. +- All requirements for tier 2 apply. + +A tier 1 platform may be demoted if it no longer meets these requirements but +still meets the requirements for a lower tier. Any such proposal will be +communicated widely to the Zcash community, both when initially proposed and +before being dropped from a stable release. A tier 1 platform is highly unlikely +to be directly removed without first being demoted to tier 2 or tier 3. +(The amount of time between such communication and the next stable release may +depend on the nature and severity of the failed requirement, the timing of its +discovery, whether the platform has been part of a stable release yet, and +whether the demotion or removal can be a planned and scheduled action.) + +Raising the baseline expectations of a tier 1 platform (such as the minimum CPU +features or OS version required) requires the approval of the Zebra team, and +should be widely communicated as well. diff --git a/book/src/user/tracing.md b/book/src/user/tracing.md index 620ef406583..3bb65d1fd45 100644 --- a/book/src/user/tracing.md +++ b/book/src/user/tracing.md @@ -1,7 +1,12 @@ # Tracing Zebra -Zebra supports dynamic tracing, configured using the config's -[`TracingSection`][tracing_section] and (optionally) an HTTP RPC endpoint. +## Dynamic Tracing + +Zebra supports dynamic tracing, configured using the config's +[`TracingSection`][tracing_section] and an HTTP RPC endpoint. + +Activate this feature using the `filter-reload` compile-time feature, +and the [`filter`][filter] and `endpoint_addr` runtime config options. If the `endpoint_addr` is specified, `zebrad` will open an HTTP endpoint allowing dynamic runtime configuration of the tracing filter. For instance, @@ -12,13 +17,20 @@ if the config had `endpoint_addr = '127.0.0.1:3000'`, then See the [`filter`][filter] documentation for more details. -Zebra also has support for: +## `journald` Logging + +Zebra can send tracing spans and events to [systemd-journald][systemd_journald], +on Linux distributions that use `systemd`. + +Activate `journald` logging using the `journald` compile-time feature, +and the [`use_journald`][use_journald] runtime config option. + +## Flamegraphs + +Zebra can generate [flamegraphs] of tracing spans. -* Generating [flamegraphs] of tracing spans, configured using the -[`flamegraph`][flamegraph] option. -* Sending tracing spans and events to [systemd-journald][systemd_journald], -on Linux distributions that use `systemd`. Configured using the -[`use_journald`][use_journald] option. +Activate flamegraphs using the `flamegraph` compile-time feature, +and the [`flamegraph`][flamegraph] runtime config option. [tracing_section]: https://doc.zebra.zfnd.org/zebrad/config/struct.TracingSection.html [filter]: https://doc.zebra.zfnd.org/zebrad/config/struct.TracingSection.html#structfield.filter diff --git a/deny.toml b/deny.toml index cdacef6c535..59f86626e8e 100644 --- a/deny.toml +++ b/deny.toml @@ -26,10 +26,16 @@ allow = [ skip = [ # wait for zcash_proofs fork be merged back into upstream # https://github.com/ZcashFoundation/zebra/issues/3831 - { name = "equihash", version = "=0.1.0" }, + { name = "equihash", version = "=0.2.0" }, { name = "zcash_encoding", version = "=0.1.0" }, { name = "zcash_note_encryption", version = "=0.1.0" }, - { name = "zcash_primitives", version = "=0.6.0" }, + { name = "zcash_primitives", version = "=0.7.0" }, + + # wait until zcash updates its halo2, orchard, etc. dependencies + # (which is likely to happen in the release after 5.0.0) + { name = "halo2_gadgets", version = "=0.1.0" }, + { name = "halo2_proofs", version = "=0.1.0" }, + { name = "orchard", version = "=0.1.0" }, ] # Similarly to `skip` allows you to skip certain crates during duplicate # detection. Unlike skip, it also includes the entire tree of transitive diff --git a/docker/Dockerfile b/docker/Dockerfile index a564599e5e9..5bb39b96361 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -100,7 +100,7 @@ COPY --from=us-docker.pkg.dev/zealous-zebra/zebra/lightwalletd /lightwalletd /us # This is the caching Docker layer for Rust! # # TODO: is it faster to use --tests here? -RUN cargo chef cook --release --features enable-sentry,lightwalletd-grpc-tests --workspace --recipe-path recipe.json +RUN cargo chef cook --release --features sentry,lightwalletd-grpc-tests --workspace --recipe-path recipe.json COPY . . RUN cargo test --locked --release --features lightwalletd-grpc-tests --workspace --no-run @@ -118,11 +118,11 @@ CMD [ "cargo"] # `test` stage. This step is a dependency for the `runtime` stage, which uses the resulting # zebrad binary from this step. FROM deps AS release -RUN cargo chef cook --release --features enable-sentry --recipe-path recipe.json +RUN cargo chef cook --release --features sentry --recipe-path recipe.json COPY . . # Build zebra -RUN cargo build --locked --release --features enable-sentry --package zebrad --bin zebrad +RUN cargo build --locked --release --features sentry --package zebrad --bin zebrad # This stage is only used when deploying nodes or when only the resulting zebrad binary is needed # diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 002a3e3017b..0e4ecf65b07 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -38,6 +38,12 @@ case "$1" in cargo test --locked --release --features "test_sync_to_mandatory_checkpoint_${NETWORK,,},lightwalletd-grpc-tests" --package zebrad --test acceptance -- --nocapture --include-ignored "sync_to_mandatory_checkpoint_${NETWORK,,}" # TODO: replace with $ZEBRA_CACHED_STATE_DIR in Rust and workflows ls -lh "/zebrad-cache"/*/* || (echo "No /zebrad-cache/*/*"; ls -lhR "/zebrad-cache" | head -50 || echo "No /zebrad-cache directory") + elif [[ "$TEST_UPDATE_SYNC" -eq "1" ]]; then + # Run a Zebra sync starting at the cached tip, and syncing to the latest tip. + # + # List directory used by test + ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") + cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored zebrad_update_sync elif [[ "$TEST_CHECKPOINT_SYNC" -eq "1" ]]; then # Run a Zebra sync starting at the cached mandatory checkpoint, and syncing past it. # diff --git a/docker/zcash-params/Dockerfile b/docker/zcash-params/Dockerfile index 65da3839c32..c441e171951 100644 --- a/docker/zcash-params/Dockerfile +++ b/docker/zcash-params/Dockerfile @@ -23,7 +23,7 @@ RUN apt-get -qq update && \ ENV CARGO_HOME /app/.cargo/ # Build dependencies - this is the caching Docker layer! -RUN cargo chef cook --release --features enable-sentry --package zebrad --recipe-path recipe.json +RUN cargo chef cook --release --features sentry --package zebrad --recipe-path recipe.json ARG RUST_BACKTRACE=0 ENV RUST_BACKTRACE ${RUST_BACKTRACE} @@ -36,4 +36,4 @@ ENV COLORBT_SHOW_HIDDEN ${COLORBT_SHOW_HIDDEN} COPY . . # Pre-download Zcash Sprout and Sapling parameters -RUN cargo run --locked --release --features enable-sentry --package zebrad --bin zebrad download +RUN cargo run --locked --release --features sentry --package zebrad --bin zebrad download diff --git a/tower-batch/Cargo.toml b/tower-batch/Cargo.toml index 88202f82cb2..915cd66b602 100644 --- a/tower-batch/Cargo.toml +++ b/tower-batch/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tower-batch" -version = "0.2.26" +version = "0.2.28" authors = ["Zcash Foundation "] license = "MIT" edition = "2021" @@ -9,8 +9,10 @@ edition = "2021" futures = "0.3.21" futures-core = "0.3.21" pin-project = "1.0.10" -tokio = { version = "1.19.2", features = ["time", "sync", "tracing", "macros"] } -tower = { version = "0.4.12", features = ["util", "buffer"] } +rayon = "1.5.3" +tokio = { version = "1.20.0", features = ["time", "sync", "tracing", "macros"] } +tokio-util = "0.7.3" +tower = { version = "0.4.13", features = ["util", "buffer"] } tracing = "0.1.31" tracing-futures = "0.2.5" @@ -18,9 +20,11 @@ tracing-futures = "0.2.5" color-eyre = "0.6.1" ed25519-zebra = "3.0.0" rand = { version = "0.8.5", package = "rand" } -tokio = { version = "1.19.2", features = ["full"] } + +tokio = { version = "1.20.0", features = ["full", "tracing", "test-util"] } tokio-test = "0.4.2" tower-fallback = { path = "../tower-fallback/" } tower-test = "0.4.0" -tracing = "0.1.31" + +zebra-consensus = { path = "../zebra-consensus/" } zebra-test = { path = "../zebra-test/" } diff --git a/tower-batch/src/layer.rs b/tower-batch/src/layer.rs index 8fda7ba307f..c3757eef663 100644 --- a/tower-batch/src/layer.rs +++ b/tower-batch/src/layer.rs @@ -1,8 +1,12 @@ -use super::{service::Batch, BatchControl}; +//! Tower service layer for batch processing. + use std::{fmt, marker::PhantomData}; + use tower::layer::Layer; use tower::Service; +use super::{service::Batch, BatchControl}; + /// Adds a layer performing batch processing of requests. /// /// The default Tokio executor is used to run the given service, @@ -10,24 +14,31 @@ use tower::Service; /// /// See the module documentation for more details. pub struct BatchLayer { - max_items: usize, + max_items_in_batch: usize, + max_batches: Option, max_latency: std::time::Duration, - _p: PhantomData, + + // TODO: is the variance correct here? + // https://doc.rust-lang.org/1.33.0/nomicon/subtyping.html#variance + // https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns + _handles_requests: PhantomData, } impl BatchLayer { /// Creates a new `BatchLayer`. /// /// The wrapper is responsible for telling the inner service when to flush a - /// batch of requests. Two parameters control this policy: - /// - /// * `max_items` gives the maximum number of items per batch. - /// * `max_latency` gives the maximum latency for a batch item. - pub fn new(max_items: usize, max_latency: std::time::Duration) -> Self { + /// batch of requests. See [`Batch::new()`] for details. + pub fn new( + max_items_in_batch: usize, + max_batches: impl Into>, + max_latency: std::time::Duration, + ) -> Self { BatchLayer { - max_items, + max_items_in_batch, + max_batches: max_batches.into(), max_latency, - _p: PhantomData, + _handles_requests: PhantomData, } } } @@ -36,20 +47,27 @@ impl Layer for BatchLayer where S: Service> + Send + 'static, S::Future: Send, + S::Response: Send, S::Error: Into + Send + Sync, Request: Send + 'static, { type Service = Batch; fn layer(&self, service: S) -> Self::Service { - Batch::new(service, self.max_items, self.max_latency) + Batch::new( + service, + self.max_items_in_batch, + self.max_batches, + self.max_latency, + ) } } impl fmt::Debug for BatchLayer { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("BufferLayer") - .field("max_items", &self.max_items) + .field("max_items_in_batch", &self.max_items_in_batch) + .field("max_batches", &self.max_batches) .field("max_latency", &self.max_latency) .finish() } diff --git a/tower-batch/src/lib.rs b/tower-batch/src/lib.rs index 2cf9f770337..855b1a962b0 100644 --- a/tower-batch/src/lib.rs +++ b/tower-batch/src/lib.rs @@ -89,22 +89,23 @@ pub mod error; pub mod future; mod layer; mod message; -mod semaphore; mod service; mod worker; type BoxError = Box; /// Signaling mechanism for batchable services that allows explicit flushing. -pub enum BatchControl { +/// +/// This request type is a generic wrapper for the inner `Req` type. +pub enum BatchControl { /// A new batch item. - Item(R), + Item(Req), /// The current batch should be flushed. Flush, } -impl From for BatchControl { - fn from(req: R) -> BatchControl { +impl From for BatchControl { + fn from(req: Req) -> BatchControl { BatchControl::Item(req) } } diff --git a/tower-batch/src/message.rs b/tower-batch/src/message.rs index 287076f26ce..05415e2b4ca 100644 --- a/tower-batch/src/message.rs +++ b/tower-batch/src/message.rs @@ -1,5 +1,8 @@ +//! Batch message types. + +use tokio::sync::{oneshot, OwnedSemaphorePermit}; + use super::error::ServiceError; -use tokio::sync::oneshot; /// Message sent to the batch worker #[derive(Debug)] @@ -7,7 +10,7 @@ pub(crate) struct Message { pub(crate) request: Request, pub(crate) tx: Tx, pub(crate) span: tracing::Span, - pub(super) _permit: crate::semaphore::Permit, + pub(super) _permit: OwnedSemaphorePermit, } /// Response sender diff --git a/tower-batch/src/semaphore.rs b/tower-batch/src/semaphore.rs deleted file mode 100644 index f09e31bcf61..00000000000 --- a/tower-batch/src/semaphore.rs +++ /dev/null @@ -1,128 +0,0 @@ -// Copied from tower/src/semaphore.rs, commit: -// d4d1c67 hedge: use auto-resizing histograms (#484) -// -// When we upgrade to tower 0.4, we can use tokio's PollSemaphore, like tower's: -// ccfaffc buffer, limit: use `tokio-util`'s `PollSemaphore` (#556) - -// Ignore lints on this copied code -#![allow(dead_code)] - -pub(crate) use self::sync::OwnedSemaphorePermit as Permit; -use futures::FutureExt; -use futures_core::ready; -use std::{ - fmt, - future::Future, - mem, - pin::Pin, - sync::{Arc, Weak}, - task::{Context, Poll}, -}; -use tokio::sync; - -#[derive(Debug)] -pub(crate) struct Semaphore { - semaphore: Arc, - state: State, -} - -#[derive(Debug)] -pub(crate) struct Close { - semaphore: Weak, - permits: usize, -} - -enum State { - Waiting(Pin + Send + Sync + 'static>>), - Ready(Permit), - Empty, -} - -impl Semaphore { - pub(crate) fn new_with_close(permits: usize) -> (Self, Close) { - let semaphore = Arc::new(sync::Semaphore::new(permits)); - let close = Close { - semaphore: Arc::downgrade(&semaphore), - permits, - }; - let semaphore = Self { - semaphore, - state: State::Empty, - }; - (semaphore, close) - } - - pub(crate) fn new(permits: usize) -> Self { - Self { - semaphore: Arc::new(sync::Semaphore::new(permits)), - state: State::Empty, - } - } - - pub(crate) fn poll_acquire(&mut self, cx: &mut Context<'_>) -> Poll<()> { - loop { - self.state = match self.state { - State::Ready(_) => return Poll::Ready(()), - State::Waiting(ref mut fut) => { - let permit = ready!(Pin::new(fut).poll(cx)); - State::Ready(permit) - } - State::Empty => State::Waiting(Box::pin( - self.semaphore - .clone() - .acquire_owned() - .map(|result| result.expect("internal semaphore is never closed")), - )), - }; - } - } - - pub(crate) fn take_permit(&mut self) -> Option { - if let State::Ready(permit) = mem::replace(&mut self.state, State::Empty) { - return Some(permit); - } - None - } -} - -impl Clone for Semaphore { - fn clone(&self) -> Self { - Self { - semaphore: self.semaphore.clone(), - state: State::Empty, - } - } -} - -impl fmt::Debug for State { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - State::Waiting(_) => f - .debug_tuple("State::Waiting") - .field(&format_args!("...")) - .finish(), - State::Ready(ref r) => f.debug_tuple("State::Ready").field(&r).finish(), - State::Empty => f.debug_tuple("State::Empty").finish(), - } - } -} - -impl Close { - /// Close the semaphore, waking any remaining tasks currently awaiting a permit. - pub(crate) fn close(self) { - // The maximum number of permits that a `tokio::sync::Semaphore` - // can hold is usize::MAX >> 3. If we attempt to add more than that - // number of permits, the semaphore will panic. - // XXX(eliza): another shift is kinda janky but if we add (usize::MAX - // > 3 - initial permits) the semaphore impl panics (I think due to a - // bug in tokio?). - // TODO(eliza): Tokio should _really_ just expose `Semaphore::close` - // publicly so we don't have to do this nonsense... - const MAX: usize = std::usize::MAX >> 4; - if let Some(semaphore) = self.semaphore.upgrade() { - // If we added `MAX - available_permits`, any tasks that are - // currently holding permits could drop them, overflowing the max. - semaphore.add_permits(MAX - self.permits); - } - } -} diff --git a/tower-batch/src/service.rs b/tower-batch/src/service.rs index 2cc7b3740b6..29df281f642 100644 --- a/tower-batch/src/service.rs +++ b/tower-batch/src/service.rs @@ -1,40 +1,75 @@ -use super::{ - future::ResponseFuture, - message::Message, - worker::{Handle, Worker}, - BatchControl, -}; +//! Wrapper service for batching items to an underlying service. -use crate::semaphore::Semaphore; -use futures_core::ready; use std::{ + cmp::max, fmt, + future::Future, + pin::Pin, + sync::{Arc, Mutex}, task::{Context, Poll}, }; -use tokio::sync::{mpsc, oneshot}; + +use futures_core::ready; +use tokio::{ + pin, + sync::{mpsc, oneshot, OwnedSemaphorePermit, Semaphore}, + task::JoinHandle, +}; +use tokio_util::sync::PollSemaphore; use tower::Service; +use tracing::{info_span, Instrument}; + +use super::{ + future::ResponseFuture, + message::Message, + worker::{ErrorHandle, Worker}, + BatchControl, +}; + +/// The maximum number of batches in the queue. +/// +/// This avoids having very large queues on machines with hundreds or thousands of cores. +pub const QUEUE_BATCH_LIMIT: usize = 64; /// Allows batch processing of requests. /// -/// See the module documentation for more details. +/// See the crate documentation for more details. pub struct Batch where T: Service>, { - // Note: this actually _is_ bounded, but rather than using Tokio's unbounded - // channel, we use tokio's semaphore separately to implement the bound. + // Batch management + // + /// A custom-bounded channel for sending requests to the batch worker. + /// + /// Note: this actually _is_ bounded, but rather than using Tokio's unbounded + /// channel, we use tokio's semaphore separately to implement the bound. tx: mpsc::UnboundedSender>, - // When the buffer's channel is full, we want to exert backpressure in - // `poll_ready`, so that callers such as load balancers could choose to call - // another service rather than waiting for buffer capacity. + + /// A semaphore used to bound the channel. + /// + /// When the buffer's channel is full, we want to exert backpressure in + /// `poll_ready`, so that callers such as load balancers could choose to call + /// another service rather than waiting for buffer capacity. + /// + /// Unfortunately, this can't be done easily using Tokio's bounded MPSC + /// channel, because it doesn't wake pending tasks on close. Therefore, we implement our + /// own bounded MPSC on top of the unbounded channel, using a semaphore to + /// limit how many items are in the channel. + semaphore: PollSemaphore, + + /// A semaphore permit that allows this service to send one message on `tx`. + permit: Option, + + // Errors // - // Unfortunately, this can't be done easily using Tokio's bounded MPSC - // channel, because it doesn't expose a polling-based interface, only an - // `async fn ready`, which borrows the sender. Therefore, we implement our - // own bounded MPSC on top of the unbounded channel, using a semaphore to - // limit how many items are in the channel. - semaphore: Semaphore, - handle: Handle, + /// An error handle shared between all service clones for the same worker. + error_handle: ErrorHandle, + + /// A worker task handle shared between all service clones for the same worker. + /// + /// Only used when the worker is spawned on the tokio runtime. + worker_handle: Arc>>>, } impl fmt::Debug for Batch @@ -46,7 +81,9 @@ where f.debug_struct(name) .field("tx", &self.tx) .field("semaphore", &self.semaphore) - .field("handle", &self.handle) + .field("permit", &self.permit) + .field("error_handle", &self.error_handle) + .field("worker_handle", &self.worker_handle) .finish() } } @@ -54,27 +91,58 @@ where impl Batch where T: Service>, + T::Future: Send + 'static, T::Error: Into, { /// Creates a new `Batch` wrapping `service`. /// /// The wrapper is responsible for telling the inner service when to flush a - /// batch of requests. Two parameters control this policy: + /// batch of requests. These parameters control this policy: /// - /// * `max_items` gives the maximum number of items per batch. - /// * `max_latency` gives the maximum latency for a batch item. + /// * `max_items_in_batch` gives the maximum number of items per batch. + /// * `max_batches` is an upper bound on the number of batches in the queue, + /// and the number of concurrently executing batches. + /// If this is `None`, we use the current number of [`rayon`] threads. + /// The number of batches in the queue is also limited by [`QUEUE_BATCH_LIMIT`]. + /// * `max_latency` gives the maximum latency for a batch item to start verifying. /// /// The default Tokio executor is used to run the given service, which means /// that this method must be called while on the Tokio runtime. - pub fn new(service: T, max_items: usize, max_latency: std::time::Duration) -> Self + pub fn new( + service: T, + max_items_in_batch: usize, + max_batches: impl Into>, + max_latency: std::time::Duration, + ) -> Self where T: Send + 'static, T::Future: Send, + T::Response: Send, T::Error: Send + Sync, Request: Send + 'static, { - let (batch, worker) = Self::pair(service, max_items, max_latency); - tokio::spawn(worker.run()); + let (mut batch, worker) = Self::pair(service, max_items_in_batch, max_batches, max_latency); + + let span = info_span!("batch worker", kind = std::any::type_name::()); + + #[cfg(tokio_unstable)] + let worker_handle = { + let batch_kind = std::any::type_name::(); + + // TODO: identify the unique part of the type name generically, + // or make it an argument to this method + let batch_kind = batch_kind.trim_start_matches("zebra_consensus::primitives::"); + let batch_kind = batch_kind.trim_end_matches("::Verifier"); + + tokio::task::Builder::new() + .name(&format!("{} batch", batch_kind)) + .spawn(worker.run().instrument(span)) + }; + #[cfg(not(tokio_unstable))] + let worker_handle = tokio::spawn(worker.run().instrument(span)); + + batch.register_worker(worker_handle); + batch } @@ -85,7 +153,8 @@ where /// `Batch` and the background `Worker` that you can then spawn. pub fn pair( service: T, - max_items: usize, + max_items_in_batch: usize, + max_batches: impl Into>, max_latency: std::time::Duration, ) -> (Self, Worker) where @@ -95,32 +164,64 @@ where { let (tx, rx) = mpsc::unbounded_channel(); + // Clamp config to sensible values. + let max_items_in_batch = max(max_items_in_batch, 1); + let max_batches = max_batches + .into() + .unwrap_or_else(rayon::current_num_threads); + let max_batches_in_queue = max_batches.clamp(1, QUEUE_BATCH_LIMIT); + // The semaphore bound limits the maximum number of concurrent requests // (specifically, requests which got a `Ready` from `poll_ready`, but haven't // used their semaphore reservation in a `call` yet). - // We choose a bound that allows callers to check readiness for every item in - // a batch, then actually submit those items. - let bound = max_items; - let (semaphore, close) = Semaphore::new_with_close(bound); + // + // We choose a bound that allows callers to check readiness for one batch per rayon CPU thread. + // This helps keep all CPUs filled with work: there is one batch executing, and another ready to go. + // Often there is only one verifier running, when that happens we want it to take all the cores. + let semaphore = Semaphore::new(max_items_in_batch * max_batches_in_queue); + let semaphore = PollSemaphore::new(Arc::new(semaphore)); + + let (error_handle, worker) = Worker::new( + service, + rx, + max_items_in_batch, + max_batches, + max_latency, + semaphore.clone(), + ); - let (handle, worker) = Worker::new(service, rx, max_items, max_latency, close); let batch = Batch { tx, semaphore, - handle, + permit: None, + error_handle, + worker_handle: Arc::new(Mutex::new(None)), }; (batch, worker) } + /// Ask the `Batch` to monitor the spawned worker task's [`JoinHandle`](tokio::task::JoinHandle). + /// + /// Only used when the task is spawned on the tokio runtime. + pub fn register_worker(&mut self, worker_handle: JoinHandle<()>) { + *self + .worker_handle + .lock() + .expect("previous task panicked while holding the worker handle mutex") = + Some(worker_handle); + } + + /// Returns the error from the batch worker's `error_handle`. fn get_worker_error(&self) -> crate::BoxError { - self.handle.get_error_on_closed() + self.error_handle.get_error_on_closed() } } impl Service for Batch where T: Service>, + T::Future: Send + 'static, T::Error: Into, { type Response = T::Response; @@ -128,26 +229,59 @@ where type Future = ResponseFuture; fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - // First, check if the worker is still alive. - if self.tx.is_closed() { - // If the inner service has errored, then we error here. + // Check to see if the worker has returned or panicked. + // + // Correctness: Registers this task for wakeup when the worker finishes. + if let Some(worker_handle) = self + .worker_handle + .lock() + .expect("previous task panicked while holding the worker handle mutex") + .as_mut() + { + match Pin::new(worker_handle).poll(cx) { + Poll::Ready(Ok(())) => return Poll::Ready(Err(self.get_worker_error())), + Poll::Ready(task_panic) => { + task_panic.expect("unexpected panic in batch worker task") + } + Poll::Pending => {} + } + } + + // Check if the worker has set an error and closed its channels. + // + // Correctness: Registers this task for wakeup when the channel is closed. + let tx = self.tx.clone(); + let closed = tx.closed(); + pin!(closed); + if closed.poll(cx).is_ready() { return Poll::Ready(Err(self.get_worker_error())); } + // Poll to acquire a semaphore permit. + // // CORRECTNESS // - // Poll to acquire a semaphore permit. If we acquire a permit, then - // there's enough buffer capacity to send a new request. Otherwise, we - // need to wait for capacity. + // If we acquire a permit, then there's enough buffer capacity to send a new request. + // Otherwise, we need to wait for capacity. When that happens, `poll_acquire()` registers + // this task for wakeup when the next permit is available, or when the semaphore is closed. // - // In tokio 0.3.7, `acquire_owned` panics if its semaphore returns an - // error, so we don't need to handle errors until we upgrade to - // tokio 1.0. + // When `poll_ready()` is called multiple times, and channel capacity is 1, + // avoid deadlocks by dropping any previous permit before acquiring another one. + // This also stops tasks holding a permit after an error. // - // The current task must be scheduled for wakeup every time we return - // `Poll::Pending`. If it returns Pending, the semaphore also schedules - // the task for wakeup when the next permit is available. - ready!(self.semaphore.poll_acquire(cx)); + // Calling `poll_ready()` multiple times can make tasks lose their previous permit + // to another concurrent task. + self.permit = None; + + let permit = ready!(self.semaphore.poll_acquire(cx)); + if let Some(permit) = permit { + // Calling poll_ready() more than once will drop any previous permit, + // releasing its capacity back to the semaphore. + self.permit = Some(permit); + } else { + // The semaphore has been closed. + return Poll::Ready(Err(self.get_worker_error())); + } Poll::Ready(Ok(())) } @@ -155,9 +289,9 @@ where fn call(&mut self, request: Request) -> Self::Future { tracing::trace!("sending request to buffer worker"); let _permit = self - .semaphore - .take_permit() - .expect("buffer full; poll_ready must be called first"); + .permit + .take() + .expect("poll_ready must be called before a batch request"); // get the current Span so that we can explicitly propagate it to the worker // if we didn't do this, events on the worker related to this span wouldn't be counted @@ -187,8 +321,10 @@ where fn clone(&self) -> Self { Self { tx: self.tx.clone(), - handle: self.handle.clone(), semaphore: self.semaphore.clone(), + permit: None, + error_handle: self.error_handle.clone(), + worker_handle: self.worker_handle.clone(), } } } diff --git a/tower-batch/src/worker.rs b/tower-batch/src/worker.rs index 75c39215dbd..f2266e67100 100644 --- a/tower-batch/src/worker.rs +++ b/tower-batch/src/worker.rs @@ -1,19 +1,24 @@ +//! Batch worker item handling and run loop implementation. + use std::{ pin::Pin, sync::{Arc, Mutex}, }; -use futures::future::TryFutureExt; +use futures::{ + future::{BoxFuture, OptionFuture}, + stream::FuturesUnordered, + FutureExt, StreamExt, +}; use pin_project::pin_project; use tokio::{ sync::mpsc, time::{sleep, Sleep}, }; +use tokio_util::sync::PollSemaphore; use tower::{Service, ServiceExt}; use tracing_futures::Instrument; -use crate::semaphore; - use super::{ error::{Closed, ServiceError}, message::{self, Message}, @@ -32,162 +37,264 @@ use super::{ pub struct Worker where T: Service>, + T::Future: Send + 'static, T::Error: Into, { + // Batch management + // + /// A semaphore-bounded channel for receiving requests from the batch wrapper service. rx: mpsc::UnboundedReceiver>, + + /// The wrapped service that processes batches. service: T, + + /// The number of pending items sent to `service`, since the last batch flush. + pending_items: usize, + + /// The timer for the pending batch, if it has any items. + /// + /// The timer is started when the first entry of a new batch is + /// submitted, so that the batch latency of all entries is at most + /// self.max_latency. However, we don't keep the timer running unless + /// there is a pending request to prevent wakeups on idle services. + pending_batch_timer: Option>>, + + /// The batches that the worker is concurrently executing. + concurrent_batches: FuturesUnordered>>, + + // Errors and termination + // + /// An error that's populated on permanent service failure. failed: Option, - handle: Handle, - max_items: usize, + + /// A shared error handle that's populated on permanent service failure. + error_handle: ErrorHandle, + + /// A cloned copy of the wrapper service's semaphore, used to close the semaphore. + close: PollSemaphore, + + // Config + // + /// The maximum number of items allowed in a batch. + max_items_in_batch: usize, + + /// The maximum number of batches that are allowed to run concurrently. + max_concurrent_batches: usize, + + /// The maximum delay before processing a batch with fewer than `max_items_in_batch`. max_latency: std::time::Duration, - close: Option, } /// Get the error out #[derive(Debug)] -pub(crate) struct Handle { +pub(crate) struct ErrorHandle { inner: Arc>>, } impl Worker where T: Service>, + T::Future: Send + 'static, T::Error: Into, { + /// Creates a new batch worker. + /// + /// See [`Batch::new()`](crate::Batch::new) for details. pub(crate) fn new( service: T, rx: mpsc::UnboundedReceiver>, - max_items: usize, + max_items_in_batch: usize, + max_concurrent_batches: usize, max_latency: std::time::Duration, - close: semaphore::Close, - ) -> (Handle, Worker) { - let handle = Handle { + close: PollSemaphore, + ) -> (ErrorHandle, Worker) { + let error_handle = ErrorHandle { inner: Arc::new(Mutex::new(None)), }; let worker = Worker { rx, service, - handle: handle.clone(), + pending_items: 0, + pending_batch_timer: None, + concurrent_batches: FuturesUnordered::new(), failed: None, - max_items, + error_handle: error_handle.clone(), + close, + max_items_in_batch, + max_concurrent_batches, max_latency, - close: Some(close), }; - (handle, worker) + (error_handle, worker) } + /// Process a single worker request. async fn process_req(&mut self, req: Request, tx: message::Tx) { - if let Some(ref failed) = self.failed { - tracing::trace!("notifying caller about worker failure"); - let _ = tx.send(Err(failed.clone())); - } else { - match self.service.ready().await { - Ok(svc) => { - let rsp = svc.call(req.into()); - let _ = tx.send(Ok(rsp)); - } - Err(e) => { - self.failed(e.into()); - let _ = tx.send(Err(self - .failed - .as_ref() - .expect("Worker::failed did not set self.failed?") - .clone())); - - // Wake any tasks waiting on channel capacity. - if let Some(close) = self.close.take() { - tracing::debug!("waking pending tasks"); - close.close(); - } - } + if let Some(ref error) = self.failed { + tracing::trace!( + ?error, + "notifying batch request caller about worker failure", + ); + let _ = tx.send(Err(error.clone())); + return; + } + + match self.service.ready().await { + Ok(svc) => { + let rsp = svc.call(req.into()); + let _ = tx.send(Ok(rsp)); + + self.pending_items += 1; + } + Err(e) => { + self.failed(e.into()); + let _ = tx.send(Err(self + .failed + .as_ref() + .expect("Worker::failed did not set self.failed?") + .clone())); } } } + /// Tell the inner service to flush the current batch. + /// + /// Waits until the inner service is ready, + /// then stores a future which resolves when the batch finishes. async fn flush_service(&mut self) { - if let Err(e) = self - .service - .ready() - .and_then(|svc| svc.call(BatchControl::Flush)) - .await - { - self.failed(e.into()); + if self.failed.is_some() { + tracing::trace!("worker failure: skipping flush"); + return; + } + + match self.service.ready().await { + Ok(ready_service) => { + let flush_future = ready_service.call(BatchControl::Flush); + self.concurrent_batches.push(flush_future.boxed()); + + // Now we have an empty batch. + self.pending_items = 0; + self.pending_batch_timer = None; + } + Err(error) => { + self.failed(error.into()); + } } } + /// Is the current number of concurrent batches above the configured limit? + fn can_spawn_new_batches(&self) -> bool { + self.concurrent_batches.len() < self.max_concurrent_batches + } + + /// Run loop for batch requests, which implements the batch policies. + /// + /// See [`Batch::new()`](crate::Batch::new) for details. pub async fn run(mut self) { - // The timer is started when the first entry of a new batch is - // submitted, so that the batch latency of all entries is at most - // self.max_latency. However, we don't keep the timer running unless - // there is a pending request to prevent wakeups on idle services. - let mut timer: Option>> = None; - let mut pending_items = 0usize; loop { - match timer.as_mut() { - None => match self.rx.recv().await { - // The first message in a new batch. + // Wait on either a new message or the batch timer. + // + // If both are ready, end the batch now, because the timer has elapsed. + // If the timer elapses, any pending messages are preserved: + // https://docs.rs/tokio/latest/tokio/sync/mpsc/struct.UnboundedReceiver.html#cancel-safety + tokio::select! { + biased; + + batch_result = self.concurrent_batches.next(), if !self.concurrent_batches.is_empty() => match batch_result.expect("only returns None when empty") { + Ok(_response) => { + tracing::trace!( + pending_items = self.pending_items, + batch_deadline = ?self.pending_batch_timer.as_ref().map(|sleep| sleep.deadline()), + running_batches = self.concurrent_batches.len(), + "batch finished executing", + ); + } + Err(error) => { + let error = error.into(); + tracing::trace!(?error, "batch execution failed"); + self.failed(error); + } + }, + + Some(()) = OptionFuture::from(self.pending_batch_timer.as_mut()), if self.pending_batch_timer.as_ref().is_some() => { + tracing::trace!( + pending_items = self.pending_items, + batch_deadline = ?self.pending_batch_timer.as_ref().map(|sleep| sleep.deadline()), + running_batches = self.concurrent_batches.len(), + "batch timer expired", + ); + + // TODO: use a batch-specific span to instrument this future. + self.flush_service().await; + }, + + maybe_msg = self.rx.recv(), if self.can_spawn_new_batches() => match maybe_msg { Some(msg) => { + tracing::trace!( + pending_items = self.pending_items, + batch_deadline = ?self.pending_batch_timer.as_ref().map(|sleep| sleep.deadline()), + running_batches = self.concurrent_batches.len(), + "batch message received", + ); + let span = msg.span; + self.process_req(msg.request, msg.tx) - // Apply the provided span to request processing + // Apply the provided span to request processing. .instrument(span) .await; - timer = Some(Box::pin(sleep(self.max_latency))); - pending_items = 1; - } - // No more messages, ever. - None => return, - }, - Some(sleep) => { - // Wait on either a new message or the batch timer. - // If both are ready, select! chooses one of them at random. - tokio::select! { - maybe_msg = self.rx.recv() => match maybe_msg { - Some(msg) => { - let span = msg.span; - self.process_req(msg.request, msg.tx) - // Apply the provided span to request processing. - .instrument(span) - .await; - pending_items += 1; - // Check whether we have too many pending items. - if pending_items >= self.max_items { - // XXX(hdevalence): what span should instrument this? - self.flush_service().await; - // Now we have an empty batch. - timer = None; - pending_items = 0; - } else { - // The timer is still running. - } - } - None => { - // No more messages, ever. - return; - } - }, - () = sleep => { - // The batch timer elapsed. - // XXX(hdevalence): what span should instrument this? + + // Check whether we have too many pending items. + if self.pending_items >= self.max_items_in_batch { + tracing::trace!( + pending_items = self.pending_items, + batch_deadline = ?self.pending_batch_timer.as_ref().map(|sleep| sleep.deadline()), + running_batches = self.concurrent_batches.len(), + "batch is full", + ); + + // TODO: use a batch-specific span to instrument this future. self.flush_service().await; - timer = None; - pending_items = 0; + } else if self.pending_items == 1 { + tracing::trace!( + pending_items = self.pending_items, + batch_deadline = ?self.pending_batch_timer.as_ref().map(|sleep| sleep.deadline()), + running_batches = self.concurrent_batches.len(), + "batch is new, starting timer", + ); + + // The first message in a new batch. + self.pending_batch_timer = Some(Box::pin(sleep(self.max_latency))); + } else { + tracing::trace!( + pending_items = self.pending_items, + batch_deadline = ?self.pending_batch_timer.as_ref().map(|sleep| sleep.deadline()), + running_batches = self.concurrent_batches.len(), + "waiting for full batch or batch timer", + ); } } - } + None => { + tracing::trace!("batch channel closed and emptied, exiting worker task"); + + return; + } + }, } } } + /// Register an inner service failure. + /// + /// The underlying service failed when we called `poll_ready` on it with the given `error`. We + /// need to communicate this to all the `Buffer` handles. To do so, we wrap up the error in + /// an `Arc`, send that `Arc` to all pending requests, and store it so that subsequent + /// requests will also fail with the same error. fn failed(&mut self, error: crate::BoxError) { - // The underlying service failed when we called `poll_ready` on it with the given `error`. We - // need to communicate this to all the `Buffer` handles. To do so, we wrap up the error in - // an `Arc`, send that `Arc` to all pending requests, and store it so that subsequent - // requests will also fail with the same error. + tracing::debug!(?error, "batch worker error"); - // Note that we need to handle the case where some handle is concurrently trying to send us + // Note that we need to handle the case where some error_handle is concurrently trying to send us // a request. We need to make sure that *either* the send of the request fails *or* it // receives an error on the `oneshot` it constructed. Specifically, we want to avoid the // case where we send errors to all outstanding requests, and *then* the caller sends its @@ -196,17 +303,25 @@ where // sending the error to all outstanding requests. let error = ServiceError::new(error); - let mut inner = self.handle.inner.lock().unwrap(); + let mut inner = self.error_handle.inner.lock().unwrap(); + // Ignore duplicate failures if inner.is_some() { - // Future::poll was called after we've already errored out! return; } *inner = Some(error.clone()); drop(inner); + tracing::trace!( + ?error, + "worker failure: waking pending requests so they can be failed", + ); self.rx.close(); + self.close.close(); + + // We don't schedule any batches on an errored service + self.pending_batch_timer = None; // By closing the mpsc::Receiver, we know that that the run() loop will // drain all pending requests. We just need to make sure that any @@ -216,20 +331,20 @@ where } } -impl Handle { +impl ErrorHandle { pub(crate) fn get_error_on_closed(&self) -> crate::BoxError { self.inner .lock() - .unwrap() + .expect("previous task panicked while holding the error handle mutex") .as_ref() .map(|svc_err| svc_err.clone().into()) .unwrap_or_else(|| Closed::new().into()) } } -impl Clone for Handle { - fn clone(&self) -> Handle { - Handle { +impl Clone for ErrorHandle { + fn clone(&self) -> ErrorHandle { + ErrorHandle { inner: self.inner.clone(), } } @@ -239,11 +354,39 @@ impl Clone for Handle { impl PinnedDrop for Worker where T: Service>, + T::Future: Send + 'static, T::Error: Into, { fn drop(mut self: Pin<&mut Self>) { - if let Some(close) = self.as_mut().close.take() { - close.close(); + tracing::trace!( + pending_items = self.pending_items, + batch_deadline = ?self.pending_batch_timer.as_ref().map(|sleep| sleep.deadline()), + running_batches = self.concurrent_batches.len(), + error = ?self.failed, + "dropping batch worker", + ); + + // Fail pending tasks + self.failed(Closed::new().into()); + + // Fail queued requests + while let Ok(msg) = self.rx.try_recv() { + let _ = msg + .tx + .send(Err(self.failed.as_ref().expect("just set failed").clone())); } + + // Clear any finished batches, ignoring any errors. + // Ignore any batches that are still executing, because we can't cancel them. + // + // now_or_never() can stop futures waking up, but that's ok here, + // because we're manually polling, then dropping the stream. + while let Some(Some(_)) = self + .as_mut() + .project() + .concurrent_batches + .next() + .now_or_never() + {} } } diff --git a/tower-batch/tests/ed25519.rs b/tower-batch/tests/ed25519.rs index 7bef3971f72..76fa153852a 100644 --- a/tower-batch/tests/ed25519.rs +++ b/tower-batch/tests/ed25519.rs @@ -1,88 +1,18 @@ -use std::{ - future::Future, - mem, - pin::Pin, - task::{Context, Poll}, - time::Duration, -}; +//! Test batching using ed25519 verification. + +use std::time::Duration; use color_eyre::{eyre::eyre, Report}; use ed25519_zebra::*; use futures::stream::{FuturesUnordered, StreamExt}; use rand::thread_rng; -use tokio::sync::broadcast::{channel, error::RecvError, Sender}; use tower::{Service, ServiceExt}; -use tower_batch::{Batch, BatchControl}; +use tower_batch::Batch; use tower_fallback::Fallback; // ============ service impl ============ -pub struct Ed25519Verifier { - batch: batch::Verifier, - // This uses a "broadcast" channel, which is an mpmc channel. Tokio also - // provides a spmc channel, "watch", but it only keeps the latest value, so - // using it would require thinking through whether it was possible for - // results from one batch to be mixed with another. - tx: Sender>, -} - -#[allow(clippy::new_without_default)] -impl Ed25519Verifier { - pub fn new() -> Self { - let batch = batch::Verifier::default(); - // XXX(hdevalence) what's a reasonable choice here? - let (tx, _) = channel(10); - Self { batch, tx } - } -} - -pub type Ed25519Item = batch::Item; - -impl Service> for Ed25519Verifier { - type Response = (); - type Error = Error; - type Future = Pin> + Send + 'static>>; - - fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn call(&mut self, req: BatchControl) -> Self::Future { - match req { - BatchControl::Item(item) => { - tracing::trace!("got item"); - self.batch.queue(item); - let mut rx = self.tx.subscribe(); - Box::pin(async move { - match rx.recv().await { - Ok(result) => result, - Err(RecvError::Lagged(_)) => { - tracing::warn!( - "missed channel updates for the correct signature batch!" - ); - Err(Error::InvalidSignature) - } - Err(RecvError::Closed) => panic!("verifier was dropped without flushing"), - } - }) - } - BatchControl::Flush => { - tracing::trace!("got flush command"); - let batch = mem::take(&mut self.batch); - let _ = self.tx.send(batch.verify(thread_rng())); - Box::pin(async { Ok(()) }) - } - } - } -} - -impl Drop for Ed25519Verifier { - fn drop(&mut self) { - // We need to flush the current batch in case there are still any pending futures. - let batch = mem::take(&mut self.batch); - let _ = self.tx.send(batch.verify(thread_rng())); - } -} +use zebra_consensus::ed25519::{Item as Ed25519Item, Verifier as Ed25519Verifier}; // =============== testing code ======== @@ -122,14 +52,16 @@ where Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn batch_flushes_on_max_items() -> Result<(), Report> { use tokio::time::timeout; zebra_test::init(); // Use a very long max_latency and a short timeout to check that // flushing is happening based on hitting max_items. - let verifier = Batch::new(Ed25519Verifier::new(), 10, Duration::from_secs(1000)); + // + // Create our own verifier, so we don't shut down a shared verifier used by other tests. + let verifier = Batch::new(Ed25519Verifier::default(), 10, 5, Duration::from_secs(1000)); timeout(Duration::from_secs(1), sign_and_verify(verifier, 100, None)) .await .map_err(|e| eyre!(e))? @@ -138,14 +70,21 @@ async fn batch_flushes_on_max_items() -> Result<(), Report> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn batch_flushes_on_max_latency() -> Result<(), Report> { use tokio::time::timeout; zebra_test::init(); // Use a very high max_items and a short timeout to check that // flushing is happening based on hitting max_latency. - let verifier = Batch::new(Ed25519Verifier::new(), 100, Duration::from_millis(500)); + // + // Create our own verifier, so we don't shut down a shared verifier used by other tests. + let verifier = Batch::new( + Ed25519Verifier::default(), + 100, + 10, + Duration::from_millis(500), + ); timeout(Duration::from_secs(1), sign_and_verify(verifier, 10, None)) .await .map_err(|e| eyre!(e))? @@ -154,12 +93,18 @@ async fn batch_flushes_on_max_latency() -> Result<(), Report> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn fallback_verification() -> Result<(), Report> { zebra_test::init(); + // Create our own verifier, so we don't shut down a shared verifier used by other tests. let verifier = Fallback::new( - Batch::new(Ed25519Verifier::new(), 10, Duration::from_millis(100)), + Batch::new( + Ed25519Verifier::default(), + 10, + 1, + Duration::from_millis(100), + ), tower::service_fn(|item: Ed25519Item| async move { item.verify_single() }), ); diff --git a/tower-batch/tests/worker.rs b/tower-batch/tests/worker.rs index 7bec206b316..7b59148058b 100644 --- a/tower-batch/tests/worker.rs +++ b/tower-batch/tests/worker.rs @@ -1,4 +1,7 @@ +//! Fixed test cases for batch worker tasks. + use std::time::Duration; + use tokio_test::{assert_pending, assert_ready, assert_ready_err, task}; use tower::{Service, ServiceExt}; use tower_batch::{error, Batch}; @@ -10,7 +13,7 @@ async fn wakes_pending_waiters_on_close() { let (service, mut handle) = mock::pair::<_, ()>(); - let (mut service, worker) = Batch::pair(service, 1, Duration::from_secs(1)); + let (mut service, worker) = Batch::pair(service, 1, 1, Duration::from_secs(1)); let mut worker = task::spawn(worker.run()); // // keep the request in the worker @@ -37,29 +40,29 @@ async fn wakes_pending_waiters_on_close() { assert!( err.is::(), "response should fail with a Closed, got: {:?}", - err + err, ); assert!( ready1.is_woken(), - "dropping worker should wake ready task 1" + "dropping worker should wake ready task 1", ); let err = assert_ready_err!(ready1.poll()); assert!( - err.is::(), - "ready 1 should fail with a Closed, got: {:?}", - err + err.is::(), + "ready 1 should fail with a ServiceError {{ Closed }}, got: {:?}", + err, ); assert!( ready2.is_woken(), - "dropping worker should wake ready task 2" + "dropping worker should wake ready task 2", ); let err = assert_ready_err!(ready1.poll()); assert!( - err.is::(), - "ready 2 should fail with a Closed, got: {:?}", - err + err.is::(), + "ready 2 should fail with a ServiceError {{ Closed }}, got: {:?}", + err, ); } @@ -69,7 +72,7 @@ async fn wakes_pending_waiters_on_failure() { let (service, mut handle) = mock::pair::<_, ()>(); - let (mut service, worker) = Batch::pair(service, 1, Duration::from_secs(1)); + let (mut service, worker) = Batch::pair(service, 1, 1, Duration::from_secs(1)); let mut worker = task::spawn(worker.run()); // keep the request in the worker diff --git a/tower-fallback/Cargo.toml b/tower-fallback/Cargo.toml index 34ba264628e..c2b7e8946ed 100644 --- a/tower-fallback/Cargo.toml +++ b/tower-fallback/Cargo.toml @@ -1,16 +1,17 @@ [package] name = "tower-fallback" -version = "0.2.22" +version = "0.2.28" authors = ["Zcash Foundation "] license = "MIT" edition = "2021" [dependencies] pin-project = "0.4.29" -tower = "0.4.12" +tower = "0.4.13" futures-core = "0.3.21" tracing = "0.1.31" [dev-dependencies] +tokio = { version = "1.20.0", features = ["full", "tracing", "test-util"] } + zebra-test = { path = "../zebra-test/" } -tokio = { version = "1.19.2", features = ["full"] } diff --git a/tower-fallback/src/future.rs b/tower-fallback/src/future.rs index 92dd9ea9a9e..dbbea911ce9 100644 --- a/tower-fallback/src/future.rs +++ b/tower-fallback/src/future.rs @@ -1,5 +1,8 @@ //! Future types for the `Fallback` middleware. +// TODO: remove this lint exception after upgrading to pin-project 1.0.11 or later (#2355) +#![allow(dead_code)] + use std::{ fmt::Debug, future::Future, diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml index c44ed0bbb96..9c310937894 100644 --- a/zebra-chain/Cargo.toml +++ b/zebra-chain/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zebra-chain" -version = "1.0.0-beta.11" +version = "1.0.0-beta.13" authors = ["Zcash Foundation "] license = "MIT OR Apache-2.0" edition = "2021" @@ -13,76 +13,95 @@ proptest-impl = ["proptest", "proptest-derive", "zebra-test", "rand", "rand_chac bench = ["zebra-test"] [dependencies] + +# Cryptography aes = "0.7.5" bech32 = "0.9.0" -bitvec = "1.0.0" +bitvec = "1.0.1" bitflags = "1.3.2" blake2b_simd = "1.0.0" blake2s_simd = "1.0.0" bls12_381 = "0.7.0" bs58 = { version = "0.4.0", features = ["check"] } byteorder = "1.4.3" -chrono = { version = "0.4.19", features = ["serde"] } -displaydoc = "0.2.3" +equihash = "0.1.0" fpe = "0.5.1" -futures = "0.3.21" group = "0.12.0" -halo2 = { package = "halo2_proofs", version = "0.1.0" } -hex = { version = "0.4.3", features = ["serde"] } incrementalmerkletree = "0.3.0" -itertools = "0.10.3" jubjub = "0.9.0" lazy_static = "1.4.0" primitive-types = "0.11.1" rand_core = "0.6.3" ripemd = "0.1.1" - -serde = { version = "1.0.137", features = ["serde_derive", "rc"] } -serde_with = "1.14.0" -serde-big-array = "0.4.1" # Matches version used by hdwallet secp256k1 = { version = "0.21.3", features = ["serde"] } sha2 = { version = "0.9.9", features=["compress"] } -static_assertions = "1.1.0" subtle = "2.4.1" -thiserror = "1.0.31" uint = "0.9.1" x25519-dalek = { version = "1.2.0", features = ["serde"] } -orchard = "0.1.0" - -equihash = "0.1.0" -zcash_note_encryption = "0.1" -zcash_primitives = { version = "0.6.0", features = ["transparent-inputs"] } +# ECC deps +halo2 = { package = "halo2_proofs", version = "0.2.0" } +orchard = "0.2.0" zcash_encoding = "0.1.0" zcash_history = "0.3.0" +zcash_note_encryption = "0.1" +zcash_primitives = { version = "0.7.0", features = ["transparent-inputs"] } +# Time +chrono = { version = "0.4.19", features = ["serde"] } +humantime = "2.1.0" + +# Error Handling & Formatting +displaydoc = "0.2.3" +static_assertions = "1.1.0" +thiserror = "1.0.31" +tracing = "0.1.31" + +# Serialization +hex = { version = "0.4.3", features = ["serde"] } +serde = { version = "1.0.137", features = ["serde_derive", "rc"] } +serde_with = "1.14.0" +serde-big-array = "0.4.1" + +# Processing +futures = "0.3.21" +itertools = "0.10.3" +rayon = "1.5.3" + +# ZF deps +ed25519-zebra = "3.0.0" +redjubjub = "0.5.0" + +# Optional testing dependencies proptest = { version = "0.10.1", optional = true } proptest-derive = { version = "0.3.0", optional = true } rand = { version = "0.8.5", optional = true, package = "rand" } rand_chacha = { version = "0.3.1", optional = true } -tokio = { version = "1.19.2", optional = true } -# ZF deps -ed25519-zebra = "3.0.0" -redjubjub = "0.5.0" +tokio = { version = "1.20.0", features = ["tracing"], optional = true } zebra-test = { path = "../zebra-test/", optional = true } [dev-dependencies] -color-eyre = "0.6.1" + +# Benchmarks criterion = { version = "0.3.5", features = ["html_reports"] } -itertools = "0.10.3" + +# Error Handling & Formatting +color-eyre = "0.6.1" spandoc = "0.2.2" tracing = "0.1.31" +# Make the optional testing dependencies required proptest = "0.10.1" proptest-derive = "0.3.0" + rand = { version = "0.8.5", package = "rand" } rand_chacha = "0.3.1" -tokio = "1.19.2" +tokio = { version = "1.20.0", features = ["full", "tracing", "test-util"] } zebra-test = { path = "../zebra-test/" } diff --git a/zebra-chain/src/amount.rs b/zebra-chain/src/amount.rs index 7ca036accc2..8a130e4ad7e 100644 --- a/zebra-chain/src/amount.rs +++ b/zebra-chain/src/amount.rs @@ -536,6 +536,7 @@ impl ZcashDeserialize for Amount { } impl ZcashSerialize for Amount { + #[allow(clippy::unwrap_in_result)] fn zcash_serialize(&self, mut writer: W) -> Result<(), std::io::Error> { let amount = self .0 diff --git a/zebra-chain/src/amount/tests/vectors.rs b/zebra-chain/src/amount/tests/vectors.rs index d1f764b4b94..b4fe100d5a7 100644 --- a/zebra-chain/src/amount/tests/vectors.rs +++ b/zebra-chain/src/amount/tests/vectors.rs @@ -167,6 +167,8 @@ fn add_with_diff_constraints() -> Result<()> { } #[test] +// The borrows are actually needed to call the correct trait impl +#[allow(clippy::needless_borrow)] fn deserialize_checks_bounds() -> Result<()> { zebra_test::init(); diff --git a/zebra-chain/src/block.rs b/zebra-chain/src/block.rs index ada83e3e2ce..390f5fac7b1 100644 --- a/zebra-chain/src/block.rs +++ b/zebra-chain/src/block.rs @@ -1,5 +1,21 @@ //! Blocks and block-related structures (heights, headers, etc.) +use std::{collections::HashMap, fmt, ops::Neg, sync::Arc}; + +use crate::{ + amount::NegativeAllowed, + block::merkle::AuthDataRoot, + fmt::DisplayToDebug, + orchard, + parameters::{Network, NetworkUpgrade}, + sapling, + serialization::{TrustedPreallocate, MAX_PROTOCOL_MESSAGE_LEN}, + sprout, + transaction::Transaction, + transparent, + value_balance::{ValueBalance, ValueBalanceError}, +}; + mod commitment; mod error; mod hash; @@ -14,8 +30,6 @@ pub mod arbitrary; #[cfg(any(test, feature = "bench", feature = "proptest-impl"))] pub mod tests; -use std::{collections::HashMap, fmt, ops::Neg}; - pub use commitment::{ ChainHistoryBlockTxAuthCommitmentHash, ChainHistoryMmrRootHash, Commitment, CommitmentError, }; @@ -27,28 +41,14 @@ pub use serialize::{SerializedBlock, MAX_BLOCK_BYTES}; #[cfg(any(test, feature = "proptest-impl"))] pub use arbitrary::LedgerState; -use crate::{ - amount::NegativeAllowed, - block::merkle::AuthDataRoot, - fmt::DisplayToDebug, - orchard, - parameters::{Network, NetworkUpgrade}, - sapling, - serialization::{TrustedPreallocate, MAX_PROTOCOL_MESSAGE_LEN}, - sprout, - transaction::Transaction, - transparent, - value_balance::{ValueBalance, ValueBalanceError}, -}; - /// A Zcash block, containing a header and a list of transactions. #[derive(Clone, Debug, Eq, PartialEq)] #[cfg_attr(any(test, feature = "proptest-impl"), derive(Serialize))] pub struct Block { /// The block header, containing block metadata. - pub header: Header, + pub header: Arc
, /// The block transactions. - pub transactions: Vec>, + pub transactions: Vec>, } impl fmt::Display for Block { @@ -113,6 +113,7 @@ impl Block { /// /// /// [ZIP-244]: https://zips.z.cash/zip-0244 + #[allow(clippy::unwrap_in_result)] pub fn check_transaction_network_upgrade_consistency( &self, network: Network, @@ -218,7 +219,7 @@ impl Block { impl<'a> From<&'a Block> for Hash { fn from(block: &'a Block) -> Hash { - (&block.header).into() + block.header.as_ref().into() } } diff --git a/zebra-chain/src/block/arbitrary.rs b/zebra-chain/src/block/arbitrary.rs index c7979523820..be078394936 100644 --- a/zebra-chain/src/block/arbitrary.rs +++ b/zebra-chain/src/block/arbitrary.rs @@ -359,7 +359,7 @@ impl Arbitrary for Block { (Header::arbitrary_with(ledger_state), transactions_strategy) .prop_map(move |(header, transactions)| Self { - header, + header: header.into(), transactions, }) .boxed() @@ -431,7 +431,7 @@ impl Block { for (height, block) in vec.iter_mut() { // fixup the previous block hash if let Some(previous_block_hash) = previous_block_hash { - block.header.previous_block_hash = previous_block_hash; + Arc::make_mut(&mut block.header).previous_block_hash = previous_block_hash; } let mut new_transactions = Vec::new(); @@ -449,6 +449,9 @@ impl Block { // would be awkward since the genesis block is handled separatedly there. // This forces us to skip the genesis block here too in order to able to use // this to test the finalized state. + // + // TODO: run note commitment tree updates in parallel rayon threads, + // using `NoteCommitmentTrees::update_trees_parallel()` if generate_valid_commitments && *height != Height(0) { for sapling_note_commitment in transaction.sapling_note_commitments() { sapling_tree.append(*sapling_note_commitment).unwrap(); @@ -471,18 +474,21 @@ impl Block { .activation_height(current.network) .unwrap(); let nu5_height = NetworkUpgrade::Nu5.activation_height(current.network); + match current_height.cmp(&heartwood_height) { std::cmp::Ordering::Less => { // In pre-Heartwood blocks this is the Sapling note commitment tree root. // We don't validate it since we checkpoint on Canopy, but it // needs to be well-formed, i.e. smaller than 𝑞_J, so we // arbitrarily set it to 1. - block.header.commitment_bytes = [0u8; 32]; - block.header.commitment_bytes[0] = 1; + let block_header = Arc::make_mut(&mut block.header); + block_header.commitment_bytes = [0u8; 32]; + block_header.commitment_bytes[0] = 1; } std::cmp::Ordering::Equal => { // The Heartwood activation block has a hardcoded all-zeroes commitment. - block.header.commitment_bytes = [0u8; 32]; + let block_header = Arc::make_mut(&mut block.header); + block_header.commitment_bytes = [0u8; 32]; } std::cmp::Ordering::Greater => { // Set the correct commitment bytes according to the network upgrade. @@ -498,9 +504,11 @@ impl Block { &history_tree_root, &auth_data_root, ); - block.header.commitment_bytes = hash_block_commitments.into(); + let block_header = Arc::make_mut(&mut block.header); + block_header.commitment_bytes = hash_block_commitments.into(); } else { - block.header.commitment_bytes = history_tree_root.into(); + let block_header = Arc::make_mut(&mut block.header); + block_header.commitment_bytes = history_tree_root.into(); } } } diff --git a/zebra-chain/src/block/hash.rs b/zebra-chain/src/block/hash.rs index ce679a8d8b1..ccf3217ea6f 100644 --- a/zebra-chain/src/block/hash.rs +++ b/zebra-chain/src/block/hash.rs @@ -1,4 +1,4 @@ -use std::{fmt, io}; +use std::{fmt, io, sync::Arc}; use hex::{FromHex, ToHex}; @@ -97,6 +97,30 @@ impl<'a> From<&'a Header> for Hash { } } +impl From
for Hash { + // The borrow is actually needed to use From<&Header> + #[allow(clippy::needless_borrow)] + fn from(block_header: Header) -> Self { + (&block_header).into() + } +} + +impl From<&Arc
> for Hash { + // The borrow is actually needed to use From<&Header> + #[allow(clippy::needless_borrow)] + fn from(block_header: &Arc
) -> Self { + block_header.as_ref().into() + } +} + +impl From> for Hash { + // The borrow is actually needed to use From<&Header> + #[allow(clippy::needless_borrow)] + fn from(block_header: Arc
) -> Self { + block_header.as_ref().into() + } +} + impl ZcashSerialize for Hash { fn zcash_serialize(&self, mut writer: W) -> Result<(), io::Error> { writer.write_all(&self.0)?; diff --git a/zebra-chain/src/block/header.rs b/zebra-chain/src/block/header.rs index c7a297b0285..e30f3eb12e9 100644 --- a/zebra-chain/src/block/header.rs +++ b/zebra-chain/src/block/header.rs @@ -1,12 +1,12 @@ //! The block header. -use std::usize; +use std::sync::Arc; use chrono::{DateTime, Duration, Utc}; use thiserror::Error; use crate::{ - serialization::{CompactSizeMessage, TrustedPreallocate, MAX_PROTOCOL_MESSAGE_LEN}, + serialization::{TrustedPreallocate, MAX_PROTOCOL_MESSAGE_LEN}, work::{difficulty::CompactDifficulty, equihash::Solution}, }; @@ -101,6 +101,7 @@ pub enum BlockTimeError { impl Header { /// TODO: Inline this function into zebra_consensus::block::check::time_is_valid_at. /// See for more details. + #[allow(clippy::unwrap_in_result)] pub fn time_is_valid_at( &self, now: DateTime, @@ -124,18 +125,14 @@ impl Header { } /// A header with a count of the number of transactions in its block. -/// /// This structure is used in the Bitcoin network protocol. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +/// +/// The transaction count field is always zero, so we don't store it in the struct. +#[derive(Clone, Debug, Eq, PartialEq)] #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] pub struct CountedHeader { /// The header for a block - pub header: Header, - - /// The number of transactions that come after the header - /// - /// TODO: should this always be zero? (#1924) - pub transaction_count: CompactSizeMessage, + pub header: Arc
, } /// The serialized size of a Zcash block header. diff --git a/zebra-chain/src/block/serialize.rs b/zebra-chain/src/block/serialize.rs index 3cf19ae60cf..8c0907aee99 100644 --- a/zebra-chain/src/block/serialize.rs +++ b/zebra-chain/src/block/serialize.rs @@ -1,11 +1,14 @@ -use std::{borrow::Borrow, convert::TryInto, io}; +//! Serialization and deserialization for Zcash blocks. + +use std::{borrow::Borrow, io}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use chrono::{TimeZone, Utc}; use crate::{ serialization::{ - ReadZcashExt, SerializationError, ZcashDeserialize, ZcashDeserializeInto, ZcashSerialize, + CompactSizeMessage, ReadZcashExt, SerializationError, ZcashDeserialize, + ZcashDeserializeInto, ZcashSerialize, }, work::{difficulty::CompactDifficulty, equihash}, }; @@ -21,6 +24,7 @@ use super::{merkle, Block, CountedHeader, Hash, Header}; pub const MAX_BLOCK_BYTES: u64 = 2_000_000; impl ZcashSerialize for Header { + #[allow(clippy::unwrap_in_result)] fn zcash_serialize(&self, mut writer: W) -> Result<(), io::Error> { writer.write_u32::(self.version)?; self.previous_block_hash.zcash_serialize(&mut writer)?; @@ -92,19 +96,30 @@ impl ZcashDeserialize for Header { } impl ZcashSerialize for CountedHeader { + #[allow(clippy::unwrap_in_result)] fn zcash_serialize(&self, mut writer: W) -> Result<(), io::Error> { self.header.zcash_serialize(&mut writer)?; - self.transaction_count.zcash_serialize(&mut writer)?; + + // A header-only message has zero transactions in it. + let transaction_count = + CompactSizeMessage::try_from(0).expect("0 is below the message size limit"); + transaction_count.zcash_serialize(&mut writer)?; + Ok(()) } } impl ZcashDeserialize for CountedHeader { fn zcash_deserialize(mut reader: R) -> Result { - Ok(CountedHeader { + let header = CountedHeader { header: (&mut reader).zcash_deserialize_into()?, - transaction_count: (&mut reader).zcash_deserialize_into()?, - }) + }; + + // We ignore the number of transactions in a header-only message, + // it should always be zero. + let _transaction_count: CompactSizeMessage = (&mut reader).zcash_deserialize_into()?; + + Ok(header) } } diff --git a/zebra-chain/src/block/tests/generate.rs b/zebra-chain/src/block/tests/generate.rs index 1f93fabc88d..4e10299b6b2 100644 --- a/zebra-chain/src/block/tests/generate.rs +++ b/zebra-chain/src/block/tests/generate.rs @@ -148,7 +148,7 @@ fn multi_transaction_block(oversized: bool) -> Block { // Add the transactions into a block let block = Block { - header: block_header, + header: block_header.into(), transactions, }; @@ -228,7 +228,7 @@ fn single_transaction_block_many_inputs(oversized: bool) -> Block { let transactions = vec![Arc::new(big_transaction)]; let block = Block { - header: block_header, + header: block_header.into(), transactions, }; @@ -306,7 +306,7 @@ fn single_transaction_block_many_outputs(oversized: bool) -> Block { let transactions = vec![Arc::new(big_transaction)]; let block = Block { - header: block_header, + header: block_header.into(), transactions, }; diff --git a/zebra-chain/src/block/tests/preallocate.rs b/zebra-chain/src/block/tests/preallocate.rs index 902d251b4c2..97ee92dae63 100644 --- a/zebra-chain/src/block/tests/preallocate.rs +++ b/zebra-chain/src/block/tests/preallocate.rs @@ -1,6 +1,6 @@ //! Tests for trusted preallocation during deserialization. -use std::convert::TryInto; +use std::sync::Arc; use proptest::prelude::*; @@ -9,10 +9,7 @@ use crate::{ header::MIN_COUNTED_HEADER_LEN, CountedHeader, Hash, Header, BLOCK_HASH_SIZE, MAX_PROTOCOL_MESSAGE_LEN, }, - serialization::{ - arbitrary::max_allocation_is_big_enough, CompactSizeMessage, TrustedPreallocate, - ZcashSerialize, - }, + serialization::{arbitrary::max_allocation_is_big_enough, TrustedPreallocate, ZcashSerialize}, }; proptest! { @@ -49,10 +46,9 @@ proptest! { /// Confirm that each counted header takes at least COUNTED_HEADER_LEN bytes when serialized. /// This verifies that our calculated [`TrustedPreallocate::max_allocation`] is indeed an upper bound. #[test] - fn counted_header_min_length(header in any::
(), transaction_count in any::()) { + fn counted_header_min_length(header in any::>()) { let header = CountedHeader { header, - transaction_count, }; let serialized_header = header.zcash_serialize_to_vec().expect("Serialization to vec must succeed"); prop_assert!(serialized_header.len() >= MIN_COUNTED_HEADER_LEN) @@ -66,10 +62,9 @@ proptest! { /// 1. The smallest disallowed vector of `CountedHeaders`s is too large to send via the Zcash Wire Protocol /// 2. The largest allowed vector is small enough to fit in a legal Zcash Wire Protocol message #[test] - fn counted_header_max_allocation(header in any::
()) { + fn counted_header_max_allocation(header in any::>()) { let header = CountedHeader { header, - transaction_count: 0.try_into().expect("zero is less than MAX_PROTOCOL_MESSAGE_LEN"), }; let ( diff --git a/zebra-chain/src/diagnostic.rs b/zebra-chain/src/diagnostic.rs new file mode 100644 index 00000000000..1d662ec2c55 --- /dev/null +++ b/zebra-chain/src/diagnostic.rs @@ -0,0 +1,128 @@ +//! Tracing the execution time of functions. +//! +//! TODO: also trace polling time for futures, using a `Future` wrapper + +use std::time::{Duration, Instant}; + +use crate::fmt::duration_short; + +/// The default minimum info-level message time. +pub const DEFAULT_MIN_INFO_TIME: Duration = Duration::from_secs(5); + +/// The default minimum warning message time. +pub const DEFAULT_MIN_WARN_TIME: Duration = Duration::from_secs(20); + +/// A guard that logs code execution time when dropped. +#[derive(Debug)] +pub struct CodeTimer { + /// The time that the code started executing. + start: Instant, + + /// The minimum duration for info-level messages. + min_info_time: Duration, + + /// The minimum duration for warning messages. + min_warn_time: Duration, + + /// `true` if this timer has finished. + has_finished: bool, +} + +impl CodeTimer { + /// Start timing the execution of a function, method, or other code region. + /// + /// Returns a guard that finishes timing the code when dropped, + /// or when [`CodeTimer::finish()`] is called. + #[track_caller] + pub fn start() -> Self { + let start = Instant::now(); + trace!( + target: "run time", + ?start, + "started code timer", + ); + + Self { + start, + min_info_time: DEFAULT_MIN_INFO_TIME, + min_warn_time: DEFAULT_MIN_WARN_TIME, + has_finished: false, + } + } + + /// Finish timing the execution of a function, method, or other code region. + pub fn finish( + mut self, + module_path: &'static str, + line: u32, + description: impl Into>, + ) where + S: ToString, + { + self.finish_inner(Some(module_path), Some(line), description); + } + + /// Finish timing the execution of a function, method, or other code region. + /// + /// This private method can be called from [`CodeTimer::finish()`] or `drop()`. + fn finish_inner( + &mut self, + module_path: impl Into>, + line: impl Into>, + description: impl Into>, + ) where + S: ToString, + { + if self.has_finished { + return; + } + + self.has_finished = true; + + let execution = self.start.elapsed(); + let time = duration_short(execution); + let time = time.as_str(); + + let module = module_path.into().unwrap_or_default(); + + let line = line.into().map(|line| line.to_string()).unwrap_or_default(); + let line = line.as_str(); + + let description = description + .into() + .map(|desc| desc.to_string()) + .unwrap_or_default(); + + if execution >= self.min_warn_time { + warn!( + target: "run time", + %time, + %module, + %line, + "very long {description}", + ); + } else if execution >= self.min_info_time { + info!( + target: "run time", + %time, + %module, + %line, + "long {description}", + ); + } else { + trace!( + target: "run time", + %time, + %module, + %line, + "finished {description} code timer", + ); + } + } +} + +impl Drop for CodeTimer { + fn drop(&mut self) { + self.finish_inner(None, None, "(dropped, cancelled, or aborted)") + } +} diff --git a/zebra-chain/src/fmt.rs b/zebra-chain/src/fmt.rs index ff3bbd09d22..aaacf9737b7 100644 --- a/zebra-chain/src/fmt.rs +++ b/zebra-chain/src/fmt.rs @@ -7,6 +7,10 @@ use proptest::prelude::*; #[cfg(any(test, feature = "proptest-impl"))] use proptest_derive::Arbitrary; +pub mod time; + +pub use time::{duration_short, humantime_milliseconds, humantime_seconds}; + /// Wrapper to override `Debug`, redirecting it to only output the type's name. #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] diff --git a/zebra-chain/src/fmt/time.rs b/zebra-chain/src/fmt/time.rs new file mode 100644 index 00000000000..4e8a3c340a5 --- /dev/null +++ b/zebra-chain/src/fmt/time.rs @@ -0,0 +1,44 @@ +//! Human-readable formats for times and durations. + +use std::time::Duration; + +/// The minimum amount of time displayed with only seconds (no milliseconds). +pub const MIN_SECONDS_ONLY_TIME: Duration = Duration::from_secs(5); + +/// Returns a human-friendly formatted string for the whole number of seconds in `duration`. +pub fn duration_short(duration: impl Into) -> String { + let duration = duration.into(); + + if duration >= MIN_SECONDS_ONLY_TIME { + humantime_seconds(duration) + } else { + humantime_milliseconds(duration) + } +} + +// TODO: rename these functions to duration_* + +/// Returns a human-friendly formatted string for the whole number of seconds in `duration`. +pub fn humantime_seconds(duration: impl Into) -> String { + let duration = duration.into(); + + // Truncate fractional seconds. + let duration = Duration::from_secs(duration.as_secs()); + + let duration = humantime::format_duration(duration); + + format!("{}", duration) +} + +/// Returns a human-friendly formatted string for the whole number of milliseconds in `duration`. +pub fn humantime_milliseconds(duration: impl Into) -> String { + let duration = duration.into(); + + // Truncate fractional seconds. + let duration_secs = Duration::from_secs(duration.as_secs()); + let duration_millis = Duration::from_millis(duration.subsec_millis().into()); + + let duration = humantime::format_duration(duration_secs + duration_millis); + + format!("{}", duration) +} diff --git a/zebra-chain/src/history_tree.rs b/zebra-chain/src/history_tree.rs index de0e305f423..fcd19fb478a 100644 --- a/zebra-chain/src/history_tree.rs +++ b/zebra-chain/src/history_tree.rs @@ -128,6 +128,7 @@ impl NonEmptyHistoryTree { /// `sapling_root` is the root of the Sapling note commitment tree of the block. /// `orchard_root` is the root of the Orchard note commitment tree of the block; /// (ignored for pre-Orchard blocks). + #[allow(clippy::unwrap_in_result)] pub fn from_block( network: Network, block: Arc, @@ -186,6 +187,7 @@ impl NonEmptyHistoryTree { /// # Panics /// /// If the block height is not one more than the previously pushed block. + #[allow(clippy::unwrap_in_result)] pub fn push( &mut self, block: Arc, @@ -419,6 +421,7 @@ impl HistoryTree { /// Create a HistoryTree from a block. /// /// If the block is pre-Heartwood, it returns an empty history tree. + #[allow(clippy::unwrap_in_result)] pub fn from_block( network: Network, block: Arc, @@ -444,6 +447,7 @@ impl HistoryTree { /// /// The tree is updated in-place. It is created when pushing the Heartwood /// activation block. + #[allow(clippy::unwrap_in_result)] pub fn push( &mut self, network: Network, diff --git a/zebra-chain/src/lib.rs b/zebra-chain/src/lib.rs index 755cd48c9cf..a9d20dc2ec6 100644 --- a/zebra-chain/src/lib.rs +++ b/zebra-chain/src/lib.rs @@ -9,18 +9,23 @@ // Required by bitvec! macro #![recursion_limit = "256"] +#[macro_use] +extern crate bitflags; + #[macro_use] extern crate serde; #[macro_use] -extern crate bitflags; +extern crate tracing; pub mod amount; pub mod block; pub mod chain_tip; +pub mod diagnostic; pub mod fmt; pub mod history_tree; pub mod orchard; +pub mod parallel; pub mod parameters; pub mod primitives; pub mod sapling; diff --git a/zebra-chain/src/orchard/commitment.rs b/zebra-chain/src/orchard/commitment.rs index 64c523061d1..b09721172e7 100644 --- a/zebra-chain/src/orchard/commitment.rs +++ b/zebra-chain/src/orchard/commitment.rs @@ -113,6 +113,7 @@ impl NoteCommitment { /// /// #[allow(non_snake_case)] + #[allow(clippy::unwrap_in_result)] pub fn new(note: Note) -> Option { // s as in the argument name for WindowedPedersenCommit_r(s) let mut s: BitVec = BitVec::new(); @@ -300,17 +301,16 @@ impl ValueCommitment { /// #[allow(non_snake_case)] pub fn new(rcv: pallas::Scalar, value: Amount) -> Self { - lazy_static! { - static ref V: pallas::Point = pallas_group_hash(b"z.cash:Orchard-cv", b"v"); - static ref R: pallas::Point = pallas_group_hash(b"z.cash:Orchard-cv", b"r"); - } - let v = pallas::Scalar::from(value); - Self::from(*V * v + *R * rcv) } } +lazy_static! { + static ref V: pallas::Point = pallas_group_hash(b"z.cash:Orchard-cv", b"v"); + static ref R: pallas::Point = pallas_group_hash(b"z.cash:Orchard-cv", b"r"); +} + #[cfg(test)] mod tests { diff --git a/zebra-chain/src/orchard/keys.rs b/zebra-chain/src/orchard/keys.rs index e0601b6f01f..759b1156075 100644 --- a/zebra-chain/src/orchard/keys.rs +++ b/zebra-chain/src/orchard/keys.rs @@ -1,7 +1,7 @@ //! Orchard key types. //! //! - +#![allow(clippy::fallible_impl_from)] #![allow(dead_code)] #[cfg(test)] @@ -161,13 +161,15 @@ impl ConstantTimeEq for SpendingKey { } impl fmt::Display for SpendingKey { + #[allow(clippy::unwrap_in_result)] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let hrp = match self.network { Network::Mainnet => sk_hrp::MAINNET, Network::Testnet => sk_hrp::TESTNET, }; - bech32::encode_to_fmt(f, hrp, &self.bytes.to_base32(), Variant::Bech32).unwrap() + bech32::encode_to_fmt(f, hrp, &self.bytes.to_base32(), Variant::Bech32) + .expect("hrp is valid") } } @@ -308,7 +310,9 @@ impl From for [u8; 32] { impl From for SpendValidatingKey { fn from(ask: SpendAuthorizingKey) -> Self { - let sk = redpallas::SigningKey::::try_from(<[u8; 32]>::from(ask)).unwrap(); + let sk = redpallas::SigningKey::::try_from(<[u8; 32]>::from(ask)).expect( + "a scalar converted to byte array and then converted back to a scalar should not fail", + ); Self(redpallas::VerificationKey::from(&sk)) } diff --git a/zebra-chain/src/orchard/tree.rs b/zebra-chain/src/orchard/tree.rs index eb7c1dfd989..1ca12267ea4 100644 --- a/zebra-chain/src/orchard/tree.rs +++ b/zebra-chain/src/orchard/tree.rs @@ -11,7 +11,6 @@ //! A root of a note commitment tree is associated with each treestate. #![allow(clippy::derive_hash_xor_eq)] -#![allow(dead_code)] use std::{ fmt, @@ -34,6 +33,11 @@ use crate::serialization::{ serde_helpers, ReadZcashExt, SerializationError, ZcashDeserialize, ZcashSerialize, }; +/// The type that is used to update the note commitment tree. +/// +/// Unfortunately, this is not the same as `orchard::NoteCommitment`. +pub type NoteCommitmentUpdate = pallas::Base; + pub(super) const MERKLE_DEPTH: usize = 32; /// MerkleCRH^Orchard Hash Function @@ -248,8 +252,8 @@ impl<'de> serde::Deserialize<'de> for Node { } } -#[allow(dead_code, missing_docs)] -#[derive(Error, Debug, Clone, PartialEq, Eq)] +#[derive(Error, Copy, Clone, Debug, Eq, PartialEq, Hash)] +#[allow(missing_docs)] pub enum NoteCommitmentTreeError { #[error("The note commitment tree is full")] FullTree, @@ -301,7 +305,8 @@ impl NoteCommitmentTree { /// chain and input into the proof. /// /// Returns an error if the tree is full. - pub fn append(&mut self, cm_x: pallas::Base) -> Result<(), NoteCommitmentTreeError> { + #[allow(clippy::unwrap_in_result)] + pub fn append(&mut self, cm_x: NoteCommitmentUpdate) -> Result<(), NoteCommitmentTreeError> { if self.inner.append(&cm_x.into()) { // Invalidate cached root let cached_root = self diff --git a/zebra-chain/src/parallel.rs b/zebra-chain/src/parallel.rs new file mode 100644 index 00000000000..0a2dcffd720 --- /dev/null +++ b/zebra-chain/src/parallel.rs @@ -0,0 +1,3 @@ +//! Parallel chain update methods. + +pub mod tree; diff --git a/zebra-chain/src/parallel/tree.rs b/zebra-chain/src/parallel/tree.rs new file mode 100644 index 00000000000..265c3db9b11 --- /dev/null +++ b/zebra-chain/src/parallel/tree.rs @@ -0,0 +1,195 @@ +//! Parallel note commitment tree update methods. + +use std::{collections::BTreeMap, sync::Arc}; + +use thiserror::Error; + +use crate::{ + block::{Block, Height}, + orchard, sapling, sprout, +}; + +/// An argument wrapper struct for note commitment trees. +#[derive(Clone, Debug)] +pub struct NoteCommitmentTrees { + /// The sprout note commitment tree. + pub sprout: Arc, + + /// The sapling note commitment tree. + pub sapling: Arc, + + /// The orchard note commitment tree. + pub orchard: Arc, +} + +/// Note commitment tree errors. +#[derive(Error, Copy, Clone, Debug, Eq, PartialEq, Hash)] +pub enum NoteCommitmentTreeError { + /// A sprout tree error + #[error("sprout error: {0}")] + Sprout(#[from] sprout::tree::NoteCommitmentTreeError), + + /// A sapling tree error + #[error("sapling error: {0}")] + Sapling(#[from] sapling::tree::NoteCommitmentTreeError), + + /// A orchard tree error + #[error("orchard error: {0}")] + Orchard(#[from] orchard::tree::NoteCommitmentTreeError), +} + +impl NoteCommitmentTrees { + /// Updates the note commitment trees using the transactions in `block`, + /// then re-calculates the cached tree roots, using parallel `rayon` threads. + /// + /// If any of the tree updates cause an error, + /// it will be returned at the end of the parallel batches. + #[allow(clippy::unwrap_in_result)] + pub fn update_trees_parallel( + &mut self, + block: &Arc, + ) -> Result<(), NoteCommitmentTreeError> { + self.update_trees_parallel_list( + [( + block + .coinbase_height() + .expect("height was already validated"), + block.clone(), + )] + .into_iter() + .collect(), + ) + } + + /// Updates the note commitment trees using the transactions in `block`, + /// then re-calculates the cached tree roots, using parallel `rayon` threads. + /// + /// If any of the tree updates cause an error, + /// it will be returned at the end of the parallel batches. + pub fn update_trees_parallel_list( + &mut self, + block_list: BTreeMap>, + ) -> Result<(), NoteCommitmentTreeError> { + // Prepare arguments for parallel threads + let NoteCommitmentTrees { + sprout, + sapling, + orchard, + } = self.clone(); + + let sprout_note_commitments: Vec<_> = block_list + .values() + .flat_map(|block| block.transactions.iter()) + .flat_map(|tx| tx.sprout_note_commitments()) + .cloned() + .collect(); + let sapling_note_commitments: Vec<_> = block_list + .values() + .flat_map(|block| block.transactions.iter()) + .flat_map(|tx| tx.sapling_note_commitments()) + .cloned() + .collect(); + let orchard_note_commitments: Vec<_> = block_list + .values() + .flat_map(|block| block.transactions.iter()) + .flat_map(|tx| tx.orchard_note_commitments()) + .cloned() + .collect(); + + let mut sprout_result = None; + let mut sapling_result = None; + let mut orchard_result = None; + + rayon::in_place_scope_fifo(|scope| { + if !sprout_note_commitments.is_empty() { + scope.spawn_fifo(|_scope| { + sprout_result = Some(Self::update_sprout_note_commitment_tree( + sprout, + sprout_note_commitments, + )); + }); + } + + if !sapling_note_commitments.is_empty() { + scope.spawn_fifo(|_scope| { + sapling_result = Some(Self::update_sapling_note_commitment_tree( + sapling, + sapling_note_commitments, + )); + }); + } + + if !orchard_note_commitments.is_empty() { + scope.spawn_fifo(|_scope| { + orchard_result = Some(Self::update_orchard_note_commitment_tree( + orchard, + orchard_note_commitments, + )); + }); + } + }); + + if let Some(sprout_result) = sprout_result { + self.sprout = sprout_result?; + } + if let Some(sapling_result) = sapling_result { + self.sapling = sapling_result?; + } + if let Some(orchard_result) = orchard_result { + self.orchard = orchard_result?; + } + + Ok(()) + } + + /// Update the sprout note commitment tree. + fn update_sprout_note_commitment_tree( + mut sprout: Arc, + sprout_note_commitments: Vec, + ) -> Result, NoteCommitmentTreeError> { + let sprout_nct = Arc::make_mut(&mut sprout); + + for sprout_note_commitment in sprout_note_commitments { + sprout_nct.append(sprout_note_commitment)?; + } + + // Re-calculate and cache the tree root. + let _ = sprout_nct.root(); + + Ok(sprout) + } + + /// Update the sapling note commitment tree. + fn update_sapling_note_commitment_tree( + mut sapling: Arc, + sapling_note_commitments: Vec, + ) -> Result, NoteCommitmentTreeError> { + let sapling_nct = Arc::make_mut(&mut sapling); + + for sapling_note_commitment in sapling_note_commitments { + sapling_nct.append(sapling_note_commitment)?; + } + + // Re-calculate and cache the tree root. + let _ = sapling_nct.root(); + + Ok(sapling) + } + + /// Update the orchard note commitment tree. + fn update_orchard_note_commitment_tree( + mut orchard: Arc, + orchard_note_commitments: Vec, + ) -> Result, NoteCommitmentTreeError> { + let orchard_nct = Arc::make_mut(&mut orchard); + + for orchard_note_commitment in orchard_note_commitments { + orchard_nct.append(orchard_note_commitment)?; + } + + // Re-calculate and cache the tree root. + let _ = orchard_nct.root(); + + Ok(orchard) + } +} diff --git a/zebra-chain/src/parameters/network.rs b/zebra-chain/src/parameters/network.rs index b6abd95e3dd..aa2178b0476 100644 --- a/zebra-chain/src/parameters/network.rs +++ b/zebra-chain/src/parameters/network.rs @@ -56,8 +56,8 @@ pub enum Network { Testnet, } -impl From<&Network> for &'static str { - fn from(network: &Network) -> &'static str { +impl From for &'static str { + fn from(network: Network) -> &'static str { match network { Network::Mainnet => "Mainnet", Network::Testnet => "Testnet", @@ -65,9 +65,9 @@ impl From<&Network> for &'static str { } } -impl From for &'static str { - fn from(network: Network) -> &'static str { - (&network).into() +impl From<&Network> for &'static str { + fn from(network: &Network) -> &'static str { + (*network).into() } } diff --git a/zebra-chain/src/primitives/zcash_history.rs b/zebra-chain/src/primitives/zcash_history.rs index 2f0967c8798..3d20a042678 100644 --- a/zebra-chain/src/primitives/zcash_history.rs +++ b/zebra-chain/src/primitives/zcash_history.rs @@ -105,6 +105,7 @@ impl Tree { /// # Panics /// /// Will panic if `peaks` is empty. + #[allow(clippy::unwrap_in_result)] pub fn new_from_cache( network: Network, network_upgrade: NetworkUpgrade, @@ -138,6 +139,7 @@ impl Tree { /// `sapling_root` is the root of the Sapling note commitment tree of the block. /// `orchard_root` is the root of the Orchard note commitment tree of the block; /// (ignored for V1 trees). + #[allow(clippy::unwrap_in_result)] pub fn new_from_block( network: Network, block: Arc, @@ -171,6 +173,7 @@ impl Tree { /// /// Panics if the network upgrade of the given block is different from /// the network upgrade of the other blocks in the tree. + #[allow(clippy::unwrap_in_result)] pub fn append_leaf( &mut self, block: Arc, diff --git a/zebra-chain/src/primitives/zcash_primitives.rs b/zebra-chain/src/primitives/zcash_primitives.rs index 8ddf35cbb66..517d66dda3c 100644 --- a/zebra-chain/src/primitives/zcash_primitives.rs +++ b/zebra-chain/src/primitives/zcash_primitives.rs @@ -198,6 +198,15 @@ impl From<&Script> for zcash_primitives::legacy::Script { } } +/// Convert a Zebra Script into a librustzcash one. +impl From