From 7b73aa0c849497e8b925b24b81ee0a119757048e Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 16 Nov 2022 19:08:28 -0400 Subject: [PATCH 1/8] ci: use Container-Optimized OS public image on the VM (#5617) * ci(compute): use debian public image on the VM, not the container Previous behavior: We were pulling the debian image the wrong way, as this was being used as a container but it was meant to be the VM image The image being pulled to create the internal container has been causing crashes as this images do not exists on Google's container repositories Expected behavior: Use a public image as debian-11 to get multiple benefits from it, as being able to use machine-images (#5615) and automatic disk resizing (which is now possible as we're using COS images, but those are more restrictive) Solution Add `--image-project=debian-cloud` and `--image-family=debian-11` as stated in the official documentation: https://cloud.google.com/sdk/gcloud/reference/compute/instances/create-with-container#--image-project More info: https://cloud.google.com/compute/docs/images/os-details#import * fix: use a public image with docker on the host * fix(logs): missing sudo before docker command Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/continous-delivery.yml | 4 ++++ .github/workflows/deploy-gcp-tests.yml | 10 ++++++---- .github/workflows/zcashd-manual-deploy.yml | 2 ++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continous-delivery.yml b/.github/workflows/continous-delivery.yml index 744f934f4a9..3f1bb77514f 100644 --- a/.github/workflows/continous-delivery.yml +++ b/.github/workflows/continous-delivery.yml @@ -120,6 +120,8 @@ jobs: run: | gcloud compute instance-templates create-with-container zebrad-${{ needs.versioning.outputs.major_version || env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ --boot-disk-type=pd-ssd \ + --image-project=cos-cloud \ + --image-family=cos-stable \ --container-image ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \ --create-disk name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }},auto-delete=yes,size=300GB,type=pd-ssd \ --container-mount-disk mount-path="/zebrad-cache",name=zebrad-cache-${{ env.GITHUB_SHA_SHORT }} \ @@ -196,6 +198,8 @@ jobs: gcloud compute instances create-with-container "zebrad-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \ --boot-disk-size 300GB \ --boot-disk-type=pd-ssd \ + --image-project=cos-cloud \ + --image-family=cos-stable \ --container-stdin \ --container-tty \ --container-image ${{ env.GAR_BASE }}/zebrad@${{ needs.build.outputs.image_digest }} \ diff --git a/.github/workflows/deploy-gcp-tests.yml b/.github/workflows/deploy-gcp-tests.yml index ecd9f10fce7..f0274ac9b60 100644 --- a/.github/workflows/deploy-gcp-tests.yml +++ b/.github/workflows/deploy-gcp-tests.yml @@ -144,9 +144,10 @@ jobs: gcloud compute instances create-with-container "${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \ --boot-disk-size 300GB \ --boot-disk-type pd-ssd \ + --image-project=cos-cloud \ + --image-family=cos-stable \ --create-disk name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=300GB,type=pd-ssd \ - --container-image gcr.io/google-containers/busybox \ - --container-restart-policy=never \ + --container-image=gcr.io/google-containers/busybox \ --machine-type ${{ env.MACHINE_TYPE }} \ --scopes cloud-platform \ --metadata=google-monitoring-enabled=TRUE,google-logging-enabled=TRUE,enable-oslogin=TRUE \ @@ -363,9 +364,10 @@ jobs: gcloud compute instances create-with-container "${{ inputs.test_id }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \ --boot-disk-size 300GB \ --boot-disk-type pd-ssd \ + --image-project=cos-cloud \ + --image-family=cos-stable \ --create-disk image=${{ env.CACHED_DISK_NAME }},name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",device-name="${{ inputs.test_id }}-${{ env.GITHUB_SHA_SHORT }}",size=300GB,type=pd-ssd \ - --container-image gcr.io/google-containers/busybox \ - --container-restart-policy=never \ + --container-image=gcr.io/google-containers/busybox \ --machine-type ${{ env.MACHINE_TYPE }} \ --scopes cloud-platform \ --metadata=google-monitoring-enabled=TRUE,google-logging-enabled=TRUE,enable-oslogin=TRUE \ diff --git a/.github/workflows/zcashd-manual-deploy.yml b/.github/workflows/zcashd-manual-deploy.yml index c7482935e5e..8ab7eb499ff 100644 --- a/.github/workflows/zcashd-manual-deploy.yml +++ b/.github/workflows/zcashd-manual-deploy.yml @@ -55,6 +55,8 @@ jobs: gcloud compute instance-templates create-with-container zcashd-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \ --boot-disk-size 10GB \ --boot-disk-type=pd-ssd \ + --image-project=cos-cloud \ + --image-family=cos-stable \ --container-stdin \ --container-tty \ --container-image electriccoinco/zcashd \ From 44f46dd1644c9fd2f62ff1adda3244b733317d54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Nov 2022 15:08:35 +0000 Subject: [PATCH 2/8] build(deps): bump insta from 1.21.0 to 1.21.1 (#5633) Bumps [insta](https://github.com/mitsuhiko/insta) from 1.21.0 to 1.21.1. - [Release notes](https://github.com/mitsuhiko/insta/releases) - [Changelog](https://github.com/mitsuhiko/insta/blob/master/CHANGELOG.md) - [Commits](https://github.com/mitsuhiko/insta/commits/1.21.1) --- updated-dependencies: - dependency-name: insta dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- zebra-rpc/Cargo.toml | 2 +- zebra-state/Cargo.toml | 2 +- zebra-test/Cargo.toml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e68d68b8472..0f118723c13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1977,9 +1977,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.21.0" +version = "1.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581d4e3314cae4536e5d22ffd23189d4a374696c5ef733eadafae0ed273fd303" +checksum = "ba1e75aa1530e7385af7b2685478dece08dafb9db3b4225c753286decea83bef" dependencies = [ "console", "lazy_static", diff --git a/zebra-rpc/Cargo.toml b/zebra-rpc/Cargo.toml index 57ade6bacc7..27d0e8ad835 100644 --- a/zebra-rpc/Cargo.toml +++ b/zebra-rpc/Cargo.toml @@ -65,7 +65,7 @@ zebra-script = { path = "../zebra-script" } zebra-state = { path = "../zebra-state" } [dev-dependencies] -insta = { version = "1.21.0", features = ["redactions", "json"] } +insta = { version = "1.21.1", features = ["redactions", "json"] } proptest = "0.10.1" proptest-derive = "0.3.0" thiserror = "1.0.37" diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml index 1a1bd6bc602..5e3aad78d48 100644 --- a/zebra-state/Cargo.toml +++ b/zebra-state/Cargo.toml @@ -62,7 +62,7 @@ once_cell = "1.16.0" spandoc = "0.2.2" hex = { version = "0.4.3", features = ["serde"] } -insta = { version = "1.21.0", features = ["ron"] } +insta = { version = "1.21.1", features = ["ron"] } proptest = "0.10.1" proptest-derive = "0.3.0" diff --git a/zebra-test/Cargo.toml b/zebra-test/Cargo.toml index b72dfea5202..2e66f14627a 100644 --- a/zebra-test/Cargo.toml +++ b/zebra-test/Cargo.toml @@ -11,7 +11,7 @@ edition = "2021" hex = "0.4.3" indexmap = "1.9.1" lazy_static = "1.4.0" -insta = "1.21.0" +insta = "1.21.1" proptest = "0.10.1" once_cell = "1.16.0" rand = { version = "0.8.5", package = "rand" } From 2f6170ee4a3811c2ef786df073da856bb5e9e1df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Nov 2022 20:36:27 +0000 Subject: [PATCH 3/8] build(deps): bump reviewdog/action-actionlint from 1.34.0 to 1.34.1 (#5646) Bumps [reviewdog/action-actionlint](https://github.com/reviewdog/action-actionlint) from 1.34.0 to 1.34.1. - [Release notes](https://github.com/reviewdog/action-actionlint/releases) - [Commits](https://github.com/reviewdog/action-actionlint/compare/v1.34.0...v1.34.1) --- updated-dependencies: - dependency-name: reviewdog/action-actionlint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 54363311b44..136c85a0bba 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -173,7 +173,7 @@ jobs: if: ${{ needs.changed-files.outputs.workflows == 'true' }} steps: - uses: actions/checkout@v3.1.0 - - uses: reviewdog/action-actionlint@v1.34.0 + - uses: reviewdog/action-actionlint@v1.34.1 with: level: warning fail_on_error: false From 57fde15e5ef5ef8313e3d225e541450b2a2fb283 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 17 Nov 2022 19:12:10 -0500 Subject: [PATCH 4/8] change(rpc): return an error from getblocktemplate method if Zebra is not synced to network tip (#5623) * Returns error from getblocktemplate if not synced * sets max estimated distance to 1 * removes unnecessary calls to best_tip_height adds info log when estimated_distance_to_chain_tip is too high * trigger GitHub actions Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- zebra-chain/src/chain_tip.rs | 22 +++++++++++++ zebra-chain/src/chain_tip/mock.rs | 31 ++++++++++++++++++- .../src/methods/get_block_template_rpcs.rs | 30 +++++++++++++++++- zebra-rpc/src/methods/tests/snapshot.rs | 1 - .../tests/snapshot/get_block_template_rpcs.rs | 3 +- zebra-rpc/src/methods/tests/vectors.rs | 12 +++++++ 6 files changed, 94 insertions(+), 5 deletions(-) diff --git a/zebra-chain/src/chain_tip.rs b/zebra-chain/src/chain_tip.rs index 923302acdf9..27fa08e13eb 100644 --- a/zebra-chain/src/chain_tip.rs +++ b/zebra-chain/src/chain_tip.rs @@ -58,6 +58,28 @@ pub trait ChainTip { Some(estimator.estimate_height_at(now)) } + + /// Return an estimate of how many blocks there are ahead of Zebra's best chain tip + /// until the network chain tip, and Zebra's best chain tip height. + /// + /// The estimate is calculated based on the current local time, the block time of the best tip + /// and the height of the best tip. + /// + /// This estimate may be negative if the current local time is behind the chain tip block's timestamp. + fn estimate_distance_to_network_chain_tip( + &self, + network: Network, + ) -> Option<(i32, block::Height)> { + let (current_height, current_block_time) = self.best_tip_height_and_block_time()?; + + let estimator = + NetworkChainTipHeightEstimator::new(current_block_time, current_height, network); + + Some(( + estimator.estimate_height_at(Utc::now()) - current_height, + current_height, + )) + } } /// A chain tip that is always empty. diff --git a/zebra-chain/src/chain_tip/mock.rs b/zebra-chain/src/chain_tip/mock.rs index b13231d60f5..fec056a3749 100644 --- a/zebra-chain/src/chain_tip/mock.rs +++ b/zebra-chain/src/chain_tip/mock.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; use tokio::sync::watch; -use crate::{block, chain_tip::ChainTip, transaction}; +use crate::{block, chain_tip::ChainTip, parameters::Network, transaction}; /// A sender to sets the values read by a [`MockChainTip`]. pub struct MockChainTipSender { @@ -17,6 +17,9 @@ pub struct MockChainTipSender { /// A sender that sets the `best_tip_block_time` of a [`MockChainTip`]. best_tip_block_time: watch::Sender>>, + + /// A sender that sets the `estimate_distance_to_network_chain_tip` of a [`MockChainTip`]. + estimated_distance_to_network_chain_tip: watch::Sender>, } /// A mock [`ChainTip`] implementation that allows setting the `best_tip_height` externally. @@ -30,6 +33,9 @@ pub struct MockChainTip { /// A mocked `best_tip_height` value set by the [`MockChainTipSender`]. best_tip_block_time: watch::Receiver>>, + + /// A mocked `estimate_distance_to_network_chain_tip` value set by the [`MockChainTipSender`]. + estimated_distance_to_network_chain_tip: watch::Receiver>, } impl MockChainTip { @@ -43,17 +49,21 @@ impl MockChainTip { let (height_sender, height_receiver) = watch::channel(None); let (hash_sender, hash_receiver) = watch::channel(None); let (time_sender, time_receiver) = watch::channel(None); + let (estimated_distance_to_tip_sender, estimated_distance_to_tip_receiver) = + watch::channel(None); let mock_chain_tip = MockChainTip { best_tip_height: height_receiver, best_tip_hash: hash_receiver, best_tip_block_time: time_receiver, + estimated_distance_to_network_chain_tip: estimated_distance_to_tip_receiver, }; let mock_chain_tip_sender = MockChainTipSender { best_tip_height: height_sender, best_tip_hash: hash_sender, best_tip_block_time: time_sender, + estimated_distance_to_network_chain_tip: estimated_distance_to_tip_sender, }; (mock_chain_tip, mock_chain_tip_sender) @@ -90,6 +100,18 @@ impl ChainTip for MockChainTip { fn best_tip_mined_transaction_ids(&self) -> Arc<[transaction::Hash]> { unreachable!("Method not used in tests"); } + + fn estimate_distance_to_network_chain_tip( + &self, + _network: Network, + ) -> Option<(i32, block::Height)> { + self.estimated_distance_to_network_chain_tip + .borrow() + .and_then(|estimated_distance| { + self.best_tip_height() + .map(|tip_height| (estimated_distance, tip_height)) + }) + } } impl MockChainTipSender { @@ -113,4 +135,11 @@ impl MockChainTipSender { .send(block_time.into()) .expect("attempt to send a best tip block time to a dropped `MockChainTip`"); } + + /// Send a new estimated distance to network chain tip to the [`MockChainTip`]. + pub fn send_estimated_distance_to_network_chain_tip(&self, distance: impl Into>) { + self.estimated_distance_to_network_chain_tip + .send(distance.into()) + .expect("attempt to send a best tip height to a dropped `MockChainTip`"); + } } diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index eda2cbc3026..9b8a1bbb08e 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -40,6 +40,11 @@ pub mod config; pub mod constants; pub(crate) mod types; +/// The max estimated distance to the chain tip for the getblocktemplate method +// Set to 30 in case the local time is a little ahead. +// TODO: Replace this with SyncStatus +const MAX_ESTIMATED_DISTANCE_TO_NETWORK_CHAIN_TIP: i32 = 30; + /// getblocktemplate RPC method signatures. #[rpc(server)] pub trait GetBlockTemplateRpc { @@ -281,7 +286,30 @@ where data: None, })?; - let tip_height = best_chain_tip_height(&latest_chain_tip)?; + let (estimated_distance_to_chain_tip, tip_height) = latest_chain_tip + .estimate_distance_to_network_chain_tip(network) + .ok_or_else(|| Error { + code: ErrorCode::ServerError(0), + message: "No Chain tip available yet".to_string(), + data: None, + })?; + + if estimated_distance_to_chain_tip > MAX_ESTIMATED_DISTANCE_TO_NETWORK_CHAIN_TIP { + tracing::info!( + estimated_distance_to_chain_tip, + ?tip_height, + "Zebra has not synced to the chain tip" + ); + + return Err(Error { + // Return error code -10 (https://github.com/s-nomp/node-stratum-pool/blob/d86ae73f8ff968d9355bb61aac05e0ebef36ccb5/lib/pool.js#L140) + // TODO: Confirm that this is the expected error code for !synced + code: ErrorCode::ServerError(-10), + message: format!("Zebra has not synced to the chain tip, estimated distance: {estimated_distance_to_chain_tip}"), + data: None, + }); + } + let mempool_txs = select_mempool_transactions(mempool).await?; let miner_fee = miner_fee(&mempool_txs); diff --git a/zebra-rpc/src/methods/tests/snapshot.rs b/zebra-rpc/src/methods/tests/snapshot.rs index c7cb6fcd6e7..ca333a909b7 100644 --- a/zebra-rpc/src/methods/tests/snapshot.rs +++ b/zebra-rpc/src/methods/tests/snapshot.rs @@ -61,7 +61,6 @@ async fn test_rpc_response_data_for_network(network: Network) { mempool.clone(), state, read_state.clone(), - latest_chain_tip.clone(), settings.clone(), ) .await; diff --git a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs index a97557d60be..1b54844cb35 100644 --- a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs @@ -16,7 +16,6 @@ use zebra_chain::{ transparent, }; use zebra_node_services::mempool; -use zebra_state::LatestChainTip; use zebra_test::mock_service::{MockService, PanicAssertion}; @@ -38,7 +37,6 @@ pub async fn test_responses( >, state: State, read_state: ReadState, - _latest_chain_tip: LatestChainTip, settings: Settings, ) where State: Service< @@ -79,6 +77,7 @@ pub async fn test_responses( let (mock_chain_tip, mock_chain_tip_sender) = MockChainTip::new(); mock_chain_tip_sender.send_best_tip_height(NetworkUpgrade::Nu5.activation_height(network)); + mock_chain_tip_sender.send_estimated_distance_to_network_chain_tip(Some(0)); let get_block_template_rpc = GetBlockTemplateRpcImpl::new( network, diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index c8967141d8c..b99fa58819b 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -827,6 +827,7 @@ async fn rpc_getblocktemplate() { let (mock_chain_tip, mock_chain_tip_sender) = MockChainTip::new(); mock_chain_tip_sender.send_best_tip_height(NetworkUpgrade::Nu5.activation_height(Mainnet)); + mock_chain_tip_sender.send_estimated_distance_to_network_chain_tip(Some(0)); // Init RPC let get_block_template_rpc = get_block_template_rpcs::GetBlockTemplateRpcImpl::new( @@ -887,6 +888,17 @@ async fn rpc_getblocktemplate() { ); mempool.expect_no_requests().await; + + mock_chain_tip_sender.send_estimated_distance_to_network_chain_tip(Some(100)); + let get_block_template_sync_error = get_block_template_rpc + .get_block_template() + .await + .expect_err("needs an error when estimated distance to network chain tip is far"); + + assert_eq!( + get_block_template_sync_error.code, + ErrorCode::ServerError(-10) + ); } #[cfg(feature = "getblocktemplate-rpcs")] From fa6cee426c6c6ad7398ae73fda526242e68a65f4 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 17 Nov 2022 23:30:46 -0500 Subject: [PATCH 5/8] build(deps): bump sentry and sentry-tracing from 0.28.0 to 0.29.0 (#5654) --- Cargo.lock | 25 ++++++++++++------------- zebrad/Cargo.toml | 4 ++-- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0f118723c13..63402a8754c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3699,9 +3699,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "sentry" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a120fb5e8b7975736bf1fc57de380531e617a6a8f5a55d037bcea25a7f5e8371" +checksum = "c6425e2a14006415449fb0a3e9a119df5032f59e7a2d9350cf8738eca290dfc5" dependencies = [ "httpdate", "reqwest", @@ -3716,9 +3716,9 @@ dependencies = [ [[package]] name = "sentry-backtrace" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac56ff9aae25b024a5aad4f0242808dfde29161c82d183adce778338c6822ef" +checksum = "04d79c194e5c20fe602e81faf39f3cff0f275ec61283f437a892cfd6544da592" dependencies = [ "backtrace", "once_cell", @@ -3728,9 +3728,9 @@ dependencies = [ [[package]] name = "sentry-contexts" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "188506b08b5e64004c71b7a5edb34959083e6e1288fada3b8d18d0bc7449ce1e" +checksum = "e1c2a57601eeb870521cc241caee27e57a012f297ece3c1b7eee87f2a531edb5" dependencies = [ "hostname", "libc", @@ -3742,9 +3742,9 @@ dependencies = [ [[package]] name = "sentry-core" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff58433a7ad557b586a09c42c4298d5f3ddb0c777e1a79d950e510d7b93fce0e" +checksum = "8be90ea119c6d0664c8ab534013bc9e90355e7004d782d5d1492ca513393b929" dependencies = [ "once_cell", "rand 0.8.5", @@ -3755,9 +3755,9 @@ dependencies = [ [[package]] name = "sentry-tracing" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc8d5bae1e1c06d96a966efc425bf1479a90464de99757d40601ce449f91fbed" +checksum = "073a872ab639da1bee23354025ddc9c1b9e5d70eabfe8cd6f10a81914cf6563d" dependencies = [ "sentry-core", "tracing-core", @@ -3766,11 +3766,10 @@ dependencies = [ [[package]] name = "sentry-types" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb30d75498a041005a774ec1b6b7d9589c5906d17ebaca338cb685dc92170f9b" +checksum = "67ad85f0addf16310a1fbcf3facc7acb17ef5dbf6ae059d2f3c38442a471404d" dependencies = [ - "chrono", "debugid", "getrandom 0.2.5", "hex", diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index dfc2014e047..29148b32326 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -129,8 +129,8 @@ num-integer = "0.1.45" rand = { version = "0.8.5", package = "rand" } # prod feature sentry -sentry-tracing = { version = "0.28.0", optional = true } -sentry = { version = "0.28.0", default-features = false, features = ["backtrace", "contexts", "reqwest", "rustls"], optional = true } +sentry-tracing = { version = "0.29.0", optional = true } +sentry = { version = "0.29.0", default-features = false, features = ["backtrace", "contexts", "reqwest", "rustls"], optional = true } # prod feature flamegraph tracing-flame = { version = "0.2.0", optional = true } From f27651251c2da9afb086b6f7e99af71c0d4c686f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Nov 2022 08:28:22 +0000 Subject: [PATCH 6/8] build(deps): bump serde_with from 2.0.1 to 2.1.0 (#5649) Bumps [serde_with](https://github.com/jonasbb/serde_with) from 2.0.1 to 2.1.0. - [Release notes](https://github.com/jonasbb/serde_with/releases) - [Commits](https://github.com/jonasbb/serde_with/compare/v2.0.1...v2.1.0) --- updated-dependencies: - dependency-name: serde_with dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- zebra-chain/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 63402a8754c..723f83df913 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3836,9 +3836,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f2d60d049ea019a84dcd6687b0d1e0030fe663ae105039bdf967ed5e6a9a7" +checksum = "25bf4a5a814902cd1014dbccfa4d4560fb8432c779471e96e035602519f82eef" dependencies = [ "base64", "chrono", @@ -3852,9 +3852,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ccadfacf6cf10faad22bbadf55986bdd0856edfb5d9210aa1dcf1f516e84e93" +checksum = "e3452b4c0f6c1e357f73fdb87cd1efabaa12acf328c7a528e252893baeb3f4aa" dependencies = [ "darling 0.14.1", "proc-macro2 1.0.42", diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml index 0102e7a0808..5bdbc3a5b5f 100644 --- a/zebra-chain/Cargo.toml +++ b/zebra-chain/Cargo.toml @@ -76,7 +76,7 @@ tracing = "0.1.37" # Serialization hex = { version = "0.4.3", features = ["serde"] } serde = { version = "1.0.147", features = ["serde_derive", "rc"] } -serde_with = "2.0.1" +serde_with = "2.1.0" serde-big-array = "0.4.1" # Processing From bd54a2f40ea62ea19efd2b43c9309d23aa1f4ea5 Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 18 Nov 2022 06:49:54 -0500 Subject: [PATCH 7/8] change(tests): add acceptance test for getblocktemplate method in CI (#5653) * adds test for getblocktemplate rpc method * adds the new test to CI * adds a couple logs * Adds example for running the test in acceptance.rs Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../continous-integration-docker.patch.yml | 6 ++ .../continous-integration-docker.yml | 25 ++++++ docker/entrypoint.sh | 4 + zebrad/tests/acceptance.rs | 18 ++++- .../tests/common/get_block_template_rpcs.rs | 1 + .../get_block_template.rs | 76 +++++++++++++++++++ .../get_block_template_rpcs/submit_block.rs | 1 - zebrad/tests/common/test_type.rs | 5 ++ 8 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 zebrad/tests/common/get_block_template_rpcs/get_block_template.rs diff --git a/.github/workflows/continous-integration-docker.patch.yml b/.github/workflows/continous-integration-docker.patch.yml index 06225b59b13..c0a34842233 100644 --- a/.github/workflows/continous-integration-docker.patch.yml +++ b/.github/workflows/continous-integration-docker.patch.yml @@ -90,6 +90,12 @@ jobs: steps: - run: 'echo "No build required"' + get-block-template-test: + name: get block template / Run get-block-template test + runs-on: ubuntu-latest + steps: + - run: 'echo "No build required"' + submit-block-test: name: submit block / Run submit-block test runs-on: ubuntu-latest diff --git a/.github/workflows/continous-integration-docker.yml b/.github/workflows/continous-integration-docker.yml index f5ee2d9e87b..08b77f834fa 100644 --- a/.github/workflows/continous-integration-docker.yml +++ b/.github/workflows/continous-integration-docker.yml @@ -581,6 +581,31 @@ jobs: zebra_state_dir: 'zebrad-cache' lwd_state_dir: 'lwd-cache' + # Test that Zebra can handle a getblocktemplate RPC call, using a cached Zebra tip state + # + # Runs: + # - after every PR is merged to `main` + # - on every PR update + # + # If the state version has changed, waits for the new cached states to be created. + # Otherwise, if the state rebuild was skipped, runs immediately after the build job. + get-block-template-test: + name: get block template + needs: test-full-sync + uses: ./.github/workflows/deploy-gcp-tests.yml + if: ${{ !cancelled() && !failure() && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' && github.event.inputs.run-lwd-send-tx != 'true' }} + with: + app_name: zebrad + test_id: get-block-template + test_description: Test getblocktemplate RPC method via Zebra's rpc server + test_variables: '-e TEST_GET_BLOCK_TEMPLATE=1 -e ZEBRA_FORCE_USE_COLOR=1 -e ZEBRA_CACHED_STATE_DIR=/var/cache/zebrad-cache' + needs_zebra_state: true + needs_lwd_state: false + saves_to_disk: false + disk_suffix: tip + root_state_path: '/var/cache' + zebra_state_dir: 'zebrad-cache' + # Test that Zebra can handle a submit block RPC call, using a cached Zebra tip state # # Runs: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 6129f8bf32e..b7fa28f15c3 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -78,6 +78,10 @@ case "$1" in ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") ls -lhR "$LIGHTWALLETD_DATA_DIR/db" || (echo "No $LIGHTWALLETD_DATA_DIR/db"; ls -lhR "$LIGHTWALLETD_DATA_DIR" | head -50 || echo "No $LIGHTWALLETD_DATA_DIR directory") cargo test --locked --release --features lightwalletd-grpc-tests --package zebrad --test acceptance -- --nocapture --include-ignored sending_transactions_using_lightwalletd + elif [[ "$TEST_GET_BLOCK_TEMPLATE" -eq "1" ]]; then + # Starting with a cached Zebra tip, test getting a block template from Zebra's RPC server. + ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") + cargo test --locked --release --features getblocktemplate-rpcs --package zebrad --test acceptance -- --nocapture --include-ignored get_block_template elif [[ "$TEST_SUBMIT_BLOCK" -eq "1" ]]; then # Starting with a cached Zebra tip, test sending a block to Zebra's RPC port. ls -lh "$ZEBRA_CACHED_STATE_DIR"/*/* || (echo "No $ZEBRA_CACHED_STATE_DIR/*/*"; ls -lhR "$ZEBRA_CACHED_STATE_DIR" | head -50 || echo "No $ZEBRA_CACHED_STATE_DIR directory") diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 03b5e55d1ab..7fe48c9913e 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -103,6 +103,12 @@ //! //! ## Getblocktemplate tests //! +//! Example of how to run the get_block_template test: +//! +//! ```console +//! ZEBRA_CACHED_STATE_DIR=/path/to/zebra/chain cargo test get_block_template --features getblocktemplate-rpcs --release -- --ignored --nocapture +//! ``` +//! //! Example of how to run the submit_block test: //! //! ```console @@ -2154,9 +2160,19 @@ async fn lightwalletd_wallet_grpc_tests() -> Result<()> { common::lightwalletd::wallet_grpc_test::run().await } +/// Test successful getblocktemplate rpc call +/// +/// See [`common::get_block_template_rpcs::get_block_template`] for more information. +#[tokio::test] +#[ignore] +#[cfg(feature = "getblocktemplate-rpcs")] +async fn get_block_template() -> Result<()> { + common::get_block_template_rpcs::get_block_template::run().await +} + /// Test successful submitblock rpc call /// -/// See [`common::getblocktemplate`] for more information. +/// See [`common::get_block_template_rpcs::submit_block`] for more information. #[tokio::test] #[ignore] #[cfg(feature = "getblocktemplate-rpcs")] diff --git a/zebrad/tests/common/get_block_template_rpcs.rs b/zebrad/tests/common/get_block_template_rpcs.rs index 03c046814ee..6fe12254a3e 100644 --- a/zebrad/tests/common/get_block_template_rpcs.rs +++ b/zebrad/tests/common/get_block_template_rpcs.rs @@ -1,3 +1,4 @@ //! Acceptance tests for getblocktemplate RPC methods in Zebra. +pub(crate) mod get_block_template; pub(crate) mod submit_block; diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs new file mode 100644 index 00000000000..16ebadbbe87 --- /dev/null +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -0,0 +1,76 @@ +//! Test getblocktemplate RPC method. +//! +//! This test requires a cached chain state that is partially synchronized close to the +//! network chain tip height. It will finish the sync and update the cached chain state. +//! +//! After finishing the sync, it will call getblocktemplate. + +use color_eyre::eyre::{eyre, Context, Result}; + +use zebra_chain::parameters::Network; + +use crate::common::{ + launch::{can_spawn_zebrad_for_rpc, spawn_zebrad_for_rpc}, + rpc_client::RPCRequestClient, + sync::{check_sync_logs_until, MempoolBehavior, SYNC_FINISHED_REGEX}, + test_type::TestType, +}; + +pub(crate) async fn run() -> Result<()> { + let _init_guard = zebra_test::init(); + + // We want a zebra state dir in place, + let test_type = TestType::UpdateZebraCachedStateWithRpc; + let test_name = "get_block_template_test"; + let network = Network::Mainnet; + + // Skip the test unless the user specifically asked for it and there is a zebrad_state_path + if !can_spawn_zebrad_for_rpc(test_name, test_type) { + return Ok(()); + } + + tracing::info!( + ?network, + ?test_type, + "running getblocktemplate test using zebrad", + ); + + let should_sync = true; + let (zebrad, zebra_rpc_address) = + spawn_zebrad_for_rpc(network, test_name, test_type, should_sync)? + .ok_or_else(|| eyre!("getblocktemplate test requires a cached state"))?; + + let rpc_address = zebra_rpc_address.expect("test type must have RPC port"); + + let mut zebrad = check_sync_logs_until( + zebrad, + network, + SYNC_FINISHED_REGEX, + MempoolBehavior::ShouldAutomaticallyActivate, + true, + )?; + + tracing::info!("calling getblocktemplate RPC method at {rpc_address}...",); + let getblocktemplate_response = RPCRequestClient::new(rpc_address) + .call("getblocktemplate", "[]".to_string()) + .await?; + + let is_response_success = getblocktemplate_response.status().is_success(); + let response_text = getblocktemplate_response.text().await?; + + tracing::info!(response_text, "got getblocktemplate response",); + + assert!(is_response_success); + + zebrad.kill(false)?; + + let output = zebrad.wait_with_output()?; + let output = output.assert_failure()?; + + // [Note on port conflict](#Note on port conflict) + output + .assert_was_killed() + .wrap_err("Possible port conflict. Are there other acceptance tests running?")?; + + Ok(()) +} diff --git a/zebrad/tests/common/get_block_template_rpcs/submit_block.rs b/zebrad/tests/common/get_block_template_rpcs/submit_block.rs index 9d979bb93fa..1f16e7ab235 100644 --- a/zebrad/tests/common/get_block_template_rpcs/submit_block.rs +++ b/zebrad/tests/common/get_block_template_rpcs/submit_block.rs @@ -22,7 +22,6 @@ use crate::common::{ /// Number of blocks past the finalized to retrieve and submit. const MAX_NUM_FUTURE_BLOCKS: u32 = 3; -#[allow(clippy::print_stderr)] pub(crate) async fn run() -> Result<()> { let _init_guard = zebra_test::init(); diff --git a/zebrad/tests/common/test_type.rs b/zebrad/tests/common/test_type.rs index 4f03f86e90a..385a38ddf39 100644 --- a/zebrad/tests/common/test_type.rs +++ b/zebrad/tests/common/test_type.rs @@ -176,6 +176,11 @@ impl TestType { return Some(Ok(config)); } + #[cfg(feature = "getblocktemplate-rpcs")] + let _ = config.mining.miner_address.insert( + zebra_chain::transparent::Address::from_script_hash(config.network.network, [0x7e; 20]), + ); + let zebra_state_path = self.zebrad_state_path(test_name)?; config.sync.checkpoint_verify_concurrency_limit = From a2f2a1402e88454f7b9764c6a97529ef4745ab9e Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 18 Nov 2022 09:24:10 -0500 Subject: [PATCH 8/8] fix(zebrad) print usage info for --help flag (#5634) * print usage info for --help flag * update known issues and help command's help text * Update README.md Co-authored-by: Alfredo Garcia * updates README Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Alfredo Garcia --- README.md | 2 ++ zebrad/src/application.rs | 2 +- zebrad/src/application/entry_point.rs | 7 +++++++ zebrad/src/commands.rs | 9 ++++++--- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a8f28569c38..a7201605ad6 100644 --- a/README.md +++ b/README.md @@ -242,6 +242,8 @@ There are a few bugs in Zebra that we're still working on fixing: - Experimental Tor support is disabled until [`arti-client` upgrades to `x25519-dalek` 2.0.0 or later](https://github.com/ZcashFoundation/zebra/issues/5492) - This happens due to a Rust dependency conflict, which can only be resolved by changing the dependencies of `x25519-dalek` +- Output of `help`, `--help` flag, and usage of invalid commands or options are inconsistent. Reports of these issues can be found [here](https://github.com/ZcashFoundation/zebra/issues/5502) and are planned to be fixed in the context of [upgrading Abscissa](https://github.com/ZcashFoundation/zebra/issues/5502). + ## Future Work Performance and Reliability: diff --git a/zebrad/src/application.rs b/zebrad/src/application.rs index 69b5734c052..b07c41c19ae 100644 --- a/zebrad/src/application.rs +++ b/zebrad/src/application.rs @@ -376,7 +376,7 @@ impl Application for ZebradApp { let default_filter = command .command .as_ref() - .map(|zcmd| zcmd.default_tracing_filter(command.verbose)) + .map(|zcmd| zcmd.default_tracing_filter(command.verbose, command.help)) .unwrap_or("warn"); let is_server = command .command diff --git a/zebrad/src/application/entry_point.rs b/zebrad/src/application/entry_point.rs index bbcff6cb8e7..16f262d1394 100644 --- a/zebrad/src/application/entry_point.rs +++ b/zebrad/src/application/entry_point.rs @@ -42,6 +42,13 @@ pub struct EntryPoint { impl EntryPoint { /// Borrow the underlying command type fn command(&self) -> &ZebradCmd { + if self.help { + let _ = Usage::for_command::().print_info(); + let _ = Usage::for_command::().print_usage(); + let _ = Usage::for_command::().print_usage(); + std::process::exit(0); + } + self.command .as_ref() .expect("Some(ZebradCmd::Start(StartCmd::default()) as default value") diff --git a/zebrad/src/commands.rs b/zebrad/src/commands.rs index c53d177d2b9..a306a5ab840 100644 --- a/zebrad/src/commands.rs +++ b/zebrad/src/commands.rs @@ -42,7 +42,9 @@ pub enum ZebradCmd { Generate(GenerateCmd), /// The `help` subcommand - #[options(help = "get usage information")] + #[options(help = "get usage information, \ + use help for subcommand usage information, \ + or --help flag to see top-level options")] Help(Help), /// The `start` subcommand @@ -78,7 +80,7 @@ impl ZebradCmd { /// Returns the default log level for this command, based on the `verbose` command line flag. /// /// Some commands need to be quiet by default. - pub(crate) fn default_tracing_filter(&self, verbose: bool) -> &'static str { + pub(crate) fn default_tracing_filter(&self, verbose: bool, help: bool) -> &'static str { let only_show_warnings = match self { // Commands that generate quiet output by default. // This output: @@ -90,7 +92,8 @@ impl ZebradCmd { CopyState(_) | Download(_) | Start(_) => false, }; - if only_show_warnings && !verbose { + // set to warn so that usage info is printed without info-level logs from component registration + if help || (only_show_warnings && !verbose) { "warn" } else if only_show_warnings || !verbose { "info"