From 37d5afd76fca1e95f539ee2e7686bc5741b667bf Mon Sep 17 00:00:00 2001 From: arya2 Date: Wed, 14 Dec 2022 21:21:42 -0500 Subject: [PATCH 01/44] adds ValidateBlock request to state --- zebra-state/src/error.rs | 5 ++ zebra-state/src/request.rs | 25 ++++++ zebra-state/src/response.rs | 11 +++ zebra-state/src/service.rs | 131 +++++++++++++++++++++++++++++++ zebra-state/src/service/check.rs | 15 ++-- zebra-state/src/service/write.rs | 7 +- 6 files changed, 186 insertions(+), 8 deletions(-) diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index edb651a26b3..3ef94445809 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -55,6 +55,11 @@ pub enum ValidateContextError { #[non_exhaustive] NotReadyToBeCommitted, + #[cfg(feature = "getblocktemplate-rpcs")] + #[error("block parent not found in non-finalized best chain")] + #[non_exhaustive] + NotReadyToBeValidated, + #[error("block height {candidate_height:?} is lower than the current finalized height {finalized_tip_height:?}")] #[non_exhaustive] OrphanedBlock { diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index c7f04eb79af..9cc6ed1c7f9 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -558,6 +558,15 @@ pub enum Request { /// /// Returns [`Response::ValidBestChainTipNullifiersAndAnchors`] CheckBestChainTipNullifiersAndAnchors(UnminedTx), + + #[cfg(feature = "getblocktemplate-rpcs")] + /// Performs contextual validation of the given block. + /// + /// It is the caller's responsibility to perform semantic validation. + /// + /// Returns [`Response::Validated`] when successful, or an error if + /// the block fails contextual validation. + CheckBlockValidity(PreparedBlock), } impl Request { @@ -577,6 +586,8 @@ impl Request { Request::CheckBestChainTipNullifiersAndAnchors(_) => { "best_chain_tip_nullifiers_anchors" } + #[cfg(feature = "getblocktemplate-rpcs")] + Request::CheckBlockValidity(_) => "check_block_validity", } } @@ -790,6 +801,15 @@ pub enum ReadRequest { /// Optionally estimate the network speed at the time when a certain block was found height: Option, }, + + #[cfg(feature = "getblocktemplate-rpcs")] + /// Performs contextual validation of the given block. + /// + /// It is the caller's responsibility to perform semantic validation. + /// + /// Returns [`Response::Validated`] when successful, or an error if + /// the block fails contextual validation. + CheckBlockValidity(PreparedBlock), } impl ReadRequest { @@ -819,6 +839,8 @@ impl ReadRequest { ReadRequest::ChainInfo => "chain_info", #[cfg(feature = "getblocktemplate-rpcs")] ReadRequest::SolutionRate { .. } => "solution_rate", + #[cfg(feature = "getblocktemplate-rpcs")] + ReadRequest::CheckBlockValidity(_) => "check_block_validity", } } @@ -866,6 +888,9 @@ impl TryFrom for ReadRequest { Err("ReadService does not write blocks") } + #[cfg(feature = "getblocktemplate-rpcs")] + Request::CheckBlockValidity(prepared) => Ok(ReadRequest::CheckBlockValidity(prepared)), + Request::AwaitUtxo(_) => Err("ReadService does not track pending UTXOs. \ Manually convert the request to ReadRequest::AnyChainUtxo, \ and handle pending UTXOs"), diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs index 74386de9d90..cd96b3de284 100644 --- a/zebra-state/src/response.rs +++ b/zebra-state/src/response.rs @@ -59,6 +59,10 @@ pub enum Response { /// /// Does not check transparent UTXO inputs ValidBestChainTipNullifiersAndAnchors, + + #[cfg(feature = "getblocktemplate-rpcs")] + /// Response to [`Request::CheckBlockValidity`](crate::Request::CheckBlockValidity) + Validated, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -137,6 +141,10 @@ pub enum ReadResponse { #[cfg(feature = "getblocktemplate-rpcs")] /// Response to [`ReadRequest::SolutionRate`](crate::ReadRequest::SolutionRate) SolutionRate(Option), + + #[cfg(feature = "getblocktemplate-rpcs")] + /// Response to [`ReadRequest::CheckBlockValidity`](crate::ReadRequest::CheckBlockValidity) + Validated, } /// A structure with the information needed from the state to build a `getblocktemplate` RPC response. @@ -215,6 +223,9 @@ impl TryFrom for Response { Err("there is no corresponding Response for this ReadResponse") } + #[cfg(feature = "getblocktemplate-rpcs")] + ReadResponse::Validated => Ok(Response::Validated), + #[cfg(feature = "getblocktemplate-rpcs")] ReadResponse::BlockHash(_) => { Err("there is no corresponding Response for this ReadResponse") diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 452624ff2d7..ecaf98edccf 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -803,6 +803,91 @@ impl ReadStateService { fn latest_non_finalized_state(&self) -> NonFinalizedState { self.non_finalized_state_receiver.cloned_watch_data() } + + /// Check the contextual validity of a block in the best chain + #[cfg(feature = "getblocktemplate-rpcs")] + #[allow(clippy::unwrap_in_result)] + #[tracing::instrument(level = "debug", skip_all)] + fn check_best_chain_contextual_validity( + &self, + prepared: PreparedBlock, + ) -> Result<(), crate::ValidateContextError> { + let latest_non_finalized_state = self.latest_non_finalized_state(); + + let Some(best_chain) = latest_non_finalized_state.best_chain() else { + Err(crate::ValidateContextError::NotReadyToBeValidated)? + }; + + if best_chain.non_finalized_tip_hash() != prepared.block.header.previous_block_hash { + Err(crate::ValidateContextError::NotReadyToBeValidated)? + } + + check::initial_contextual_validity( + self.network, + &self.db, + &latest_non_finalized_state, + &prepared, + )?; + + // Reads from disk + let sprout_final_treestates = + check::anchors::block_fetch_sprout_final_treestates(&self.db, best_chain, &prepared); + + let spent_utxos = check::utxo::transparent_spend( + &prepared, + &best_chain.unspent_utxos(), + &best_chain.spent_utxos, + &self.db, + )?; + + check::anchors::block_sapling_orchard_anchors_refer_to_final_treestates( + &self.db, best_chain, &prepared, + )?; + + let contextual = crate::request::ContextuallyValidBlock::with_block_and_spent_utxos( + prepared.clone(), + spent_utxos.clone(), + ) + .map_err(|value_balance_error| { + crate::ValidateContextError::CalculateBlockChainValueChange { + value_balance_error, + height: prepared.height, + block_hash: prepared.hash, + transaction_count: prepared.block.transactions.len(), + spent_utxo_count: spent_utxos.len(), + } + })?; + let mut block_commitment_result = None; + let mut sprout_anchor_result = None; + + rayon::in_place_scope_fifo(|scope| { + // Clone function arguments for different threads + let block = Arc::clone(&contextual.block); + + scope.spawn_fifo(|_scope| { + block_commitment_result = + Some(check::anchors::block_sprout_anchors_refer_to_treestates( + sprout_final_treestates, + block, + contextual.transaction_hashes, + contextual.height, + )); + }); + + scope.spawn_fifo(|_scope| { + sprout_anchor_result = Some(check::block_commitment_is_valid_for_chain_history( + contextual.block, + self.network, + &best_chain.history_tree, + )); + }); + }); + + block_commitment_result.expect("scope has finished")?; + sprout_anchor_result.expect("scope has finished")?; + + Ok(()) + } } impl Service for StateService { @@ -1047,6 +1132,24 @@ impl Service for StateService { } .boxed() } + + #[cfg(feature = "getblocktemplate-rpcs")] + Request::CheckBlockValidity(_) => { + // Redirect the request to the concurrent ReadStateService + let read_service = self.read_service.clone(); + + async move { + let req = req + .try_into() + .expect("ReadRequest conversion should not fail"); + + let rsp = read_service.oneshot(req).await?; + let rsp = rsp.try_into().expect("Response conversion should not fail"); + + Ok(rsp) + } + .boxed() + } } } } @@ -1684,6 +1787,34 @@ impl Service for ReadStateService { .map(|join_result| join_result.expect("panic in ReadRequest::ChainInfo")) .boxed() } + + // Used by getmininginfo, getnetworksolps, and getnetworkhashps RPCs. + #[cfg(feature = "getblocktemplate-rpcs")] + ReadRequest::CheckBlockValidity(prepared) => { + let timer = CodeTimer::start(); + + let state = self.clone(); + + // # Performance + // + // Allow other async tasks to make progress while concurrently reading blocks from disk. + let span = Span::current(); + tokio::task::spawn_blocking(move || { + span.in_scope(move || { + state.check_best_chain_contextual_validity(prepared)?; + // The work is done in the future. + timer.finish( + module_path!(), + line!(), + "ReadRequest::CheckContextualValidity", + ); + + Ok(ReadResponse::Validated) + }) + }) + .map(|join_result| join_result.expect("panic in ReadRequest::ChainInfo")) + .boxed() + } } } } diff --git a/zebra-state/src/service/check.rs b/zebra-state/src/service/check.rs index 15e172f67ed..483a8eb534b 100644 --- a/zebra-state/src/service/check.rs +++ b/zebra-state/src/service/check.rs @@ -14,13 +14,13 @@ use zebra_chain::{ use crate::{ service::{ block_iter::any_ancestor_blocks, check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN, - finalized_state::FinalizedState, non_finalized_state::NonFinalizedState, + non_finalized_state::NonFinalizedState, }, BoxError, PreparedBlock, ValidateContextError, }; // use self as check -use super::check; +use super::{check, finalized_state::ZebraDb}; // These types are used in doc links #[allow(unused_imports)] @@ -365,25 +365,26 @@ where /// /// Additional contextual validity checks are performed by the non-finalized [`Chain`]. pub(crate) fn initial_contextual_validity( - finalized_state: &FinalizedState, + network: Network, + finalized_state: &ZebraDb, non_finalized_state: &NonFinalizedState, prepared: &PreparedBlock, ) -> Result<(), ValidateContextError> { let relevant_chain = any_ancestor_blocks( non_finalized_state, - &finalized_state.db, + finalized_state, prepared.block.header.previous_block_hash, ); // Security: check proof of work before any other checks check::block_is_valid_for_recent_chain( prepared, - finalized_state.network(), - finalized_state.db.finalized_tip_height(), + network, + finalized_state.finalized_tip_height(), relevant_chain, )?; - check::nullifier::no_duplicates_in_finalized_chain(prepared, &finalized_state.db)?; + check::nullifier::no_duplicates_in_finalized_chain(prepared, finalized_state)?; Ok(()) } diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 5ba92218431..a3226081c71 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -40,7 +40,12 @@ pub(crate) fn validate_and_commit_non_finalized( non_finalized_state: &mut NonFinalizedState, prepared: PreparedBlock, ) -> Result<(), CommitBlockError> { - check::initial_contextual_validity(finalized_state, non_finalized_state, &prepared)?; + check::initial_contextual_validity( + finalized_state.network(), + &finalized_state.db, + non_finalized_state, + &prepared, + )?; let parent_hash = prepared.block.header.previous_block_hash; if finalized_state.db.finalized_tip_hash() == parent_hash { From 6eb60482a449fe223b32f2402ec6ef838add9ade Mon Sep 17 00:00:00 2001 From: arya2 Date: Wed, 14 Dec 2022 21:51:39 -0500 Subject: [PATCH 02/44] adds `Request` enum in block verifier skips solution check for BlockProposal requests calls CheckBlockValidity instead of Commit block for BlockProposal requests --- zebra-consensus/src/block.rs | 70 +++++++++++++++++++--------- zebra-consensus/src/block/request.rs | 34 ++++++++++++++ zebra-consensus/src/chain.rs | 16 +++---- zebra-state/src/request.rs | 4 +- zebra-state/src/response.rs | 6 +-- 5 files changed, 94 insertions(+), 36 deletions(-) create mode 100644 zebra-consensus/src/block/request.rs diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs index a822323fe0d..b4edf7584e8 100644 --- a/zebra-consensus/src/block.rs +++ b/zebra-consensus/src/block.rs @@ -21,19 +21,15 @@ use thiserror::Error; use tower::{Service, ServiceExt}; use tracing::Instrument; -use zebra_chain::{ - amount::Amount, - block::{self, Block}, - parameters::Network, - transparent, - work::equihash, -}; +use zebra_chain::{amount::Amount, block, parameters::Network, transparent, work::equihash}; use zebra_state as zs; use crate::{error::*, transaction as tx, BoxError}; pub mod check; +pub mod request; pub mod subsidy; +pub use request::Request; #[cfg(test)] mod tests; @@ -74,6 +70,10 @@ pub enum VerifyBlockError { // TODO: make this into a concrete type, and add it to is_duplicate_request() (#2908) Commit(#[source] BoxError), + #[error("unable to validate block after semantic verification")] + // TODO: make this into a concrete type + Validate(#[source] BoxError), + #[error("invalid transaction")] Transaction(#[from] TransactionError), } @@ -115,7 +115,7 @@ where } } -impl Service> for BlockVerifier +impl Service for BlockVerifier where S: Service + Send + Clone + 'static, S::Future: Send + 'static, @@ -134,11 +134,13 @@ where Poll::Ready(Ok(())) } - fn call(&mut self, block: Arc) -> Self::Future { + fn call(&mut self, request: Request) -> Self::Future { let mut state_service = self.state_service.clone(); let mut transaction_verifier = self.transaction_verifier.clone(); let network = self.network; + let block = request.block(); + // We don't include the block hash, because it's likely already in a parent span let span = tracing::debug_span!("block", height = ?block.coinbase_height()); @@ -175,7 +177,10 @@ where // Do the difficulty checks first, to raise the threshold for // attacks that use any other fields. check::difficulty_is_valid(&block.header, network, &height, &hash)?; - check::equihash_solution_is_valid(&block.header)?; + + if !request.is_proposal() { + check::equihash_solution_is_valid(&block.header)?; + } // Next, check the Merkle root validity, to ensure that // the header binds to the transactions in the blocks. @@ -279,19 +284,38 @@ where new_outputs, transaction_hashes, }; - match state_service - .ready() - .await - .map_err(VerifyBlockError::Commit)? - .call(zs::Request::CommitBlock(prepared_block)) - .await - .map_err(VerifyBlockError::Commit)? - { - zs::Response::Committed(committed_hash) => { - assert_eq!(committed_hash, hash, "state must commit correct hash"); - Ok(hash) - } - _ => unreachable!("wrong response for CommitBlock"), + + match request { + Request::Block(_) => match state_service + .ready() + .await + .map_err(VerifyBlockError::Commit)? + .call(zs::Request::CommitBlock(prepared_block)) + .await + .map_err(VerifyBlockError::Commit)? + { + zs::Response::Committed(committed_hash) => { + assert_eq!(committed_hash, hash, "state must commit correct hash"); + Ok(hash) + } + _ => unreachable!("wrong response for CommitBlock"), + }, + + #[cfg(feature = "getblocktemplate-rpcs")] + Request::BlockProposal(_) => match state_service + .ready() + .await + .map_err(VerifyBlockError::Validate)? + .call(zs::Request::CheckBlockValidity(prepared_block)) + .await + .map_err(VerifyBlockError::Validate)? + { + zs::Response::Committed(committed_hash) => { + assert_eq!(committed_hash, hash, "state must validate correct hash"); + Ok(hash) + } + _ => unreachable!("wrong response for CheckBlockValidity"), + }, } } .instrument(span) diff --git a/zebra-consensus/src/block/request.rs b/zebra-consensus/src/block/request.rs new file mode 100644 index 00000000000..083c742beae --- /dev/null +++ b/zebra-consensus/src/block/request.rs @@ -0,0 +1,34 @@ +//! Block verifier request type. + +use std::sync::Arc; + +use zebra_chain::block::Block; + +pub enum Request { + Block(Arc), + + #[cfg(feature = "getblocktemplate-rpcs")] + BlockProposal(Arc), +} + +impl Request { + /// Returns inner block + pub fn block(&self) -> Arc { + Arc::clone(match self { + Request::Block(block) => block, + + #[cfg(feature = "getblocktemplate-rpcs")] + Request::BlockProposal(block) => block, + }) + } + + /// Checks if request is a proposal + pub fn is_proposal(&self) -> bool { + match self { + Request::Block(_) => false, + + #[cfg(feature = "getblocktemplate-rpcs")] + Request::BlockProposal(_) => true, + } + } +} diff --git a/zebra-consensus/src/chain.rs b/zebra-consensus/src/chain.rs index 59803180612..373a4b8d467 100644 --- a/zebra-consensus/src/chain.rs +++ b/zebra-consensus/src/chain.rs @@ -15,7 +15,6 @@ use std::{ future::Future, pin::Pin, - sync::Arc, task::{Context, Poll}, }; @@ -27,15 +26,14 @@ use tower::{buffer::Buffer, util::BoxService, Service, ServiceExt}; use tracing::{instrument, Span}; use zebra_chain::{ - block::{self, Block, Height}, + block::{self, Height}, parameters::Network, }; use zebra_state as zs; use crate::{ - block::BlockVerifier, - block::VerifyBlockError, + block::{BlockVerifier, Request, VerifyBlockError}, checkpoint::{CheckpointList, CheckpointVerifier, VerifyCheckpointError}, error::TransactionError, transaction, BoxError, Config, @@ -129,7 +127,7 @@ impl VerifyChainError { } } -impl Service> for ChainVerifier +impl Service for ChainVerifier where S: Service + Send + Clone + 'static, S::Future: Send + 'static, @@ -167,14 +165,16 @@ where Poll::Ready(Ok(())) } - fn call(&mut self, block: Arc) -> Self::Future { + fn call(&mut self, request: Request) -> Self::Future { + let block = request.block(); + match block.coinbase_height() { Some(height) if height <= self.max_checkpoint_height => { self.checkpoint.call(block).map_err(Into::into).boxed() } // This also covers blocks with no height, which the block verifier // will reject immediately. - _ => self.block.call(block).map_err(Into::into).boxed(), + _ => self.block.call(request).map_err(Into::into).boxed(), } } } @@ -219,7 +219,7 @@ pub async fn init( mut state_service: S, debug_skip_parameter_preload: bool, ) -> ( - Buffer, block::Hash, VerifyChainError>, Arc>, + Buffer, Request>, Buffer< BoxService, transaction::Request, diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index 9cc6ed1c7f9..446af0d92ba 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -564,7 +564,7 @@ pub enum Request { /// /// It is the caller's responsibility to perform semantic validation. /// - /// Returns [`Response::Validated`] when successful, or an error if + /// Returns [`Response::ValidBlock`] with the hash of the block when successful, or an error if /// the block fails contextual validation. CheckBlockValidity(PreparedBlock), } @@ -807,7 +807,7 @@ pub enum ReadRequest { /// /// It is the caller's responsibility to perform semantic validation. /// - /// Returns [`Response::Validated`] when successful, or an error if + /// Returns [`Response::ValidBlock`] with the hash of the block when successful, or an error if /// the block fails contextual validation. CheckBlockValidity(PreparedBlock), } diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs index cd96b3de284..116a252410f 100644 --- a/zebra-state/src/response.rs +++ b/zebra-state/src/response.rs @@ -62,7 +62,7 @@ pub enum Response { #[cfg(feature = "getblocktemplate-rpcs")] /// Response to [`Request::CheckBlockValidity`](crate::Request::CheckBlockValidity) - Validated, + ValidBlock(block::Hash), } #[derive(Clone, Debug, PartialEq, Eq)] @@ -144,7 +144,7 @@ pub enum ReadResponse { #[cfg(feature = "getblocktemplate-rpcs")] /// Response to [`ReadRequest::CheckBlockValidity`](crate::ReadRequest::CheckBlockValidity) - Validated, + ValidBlock(block::Hash), } /// A structure with the information needed from the state to build a `getblocktemplate` RPC response. @@ -224,7 +224,7 @@ impl TryFrom for Response { } #[cfg(feature = "getblocktemplate-rpcs")] - ReadResponse::Validated => Ok(Response::Validated), + ReadResponse::ValidBlock(block_hash) => Ok(Response::ValidBlock(block_hash)), #[cfg(feature = "getblocktemplate-rpcs")] ReadResponse::BlockHash(_) => { From 162dedb73ef4fe17ba472a401b77b33a222ad565 Mon Sep 17 00:00:00 2001 From: arya2 Date: Wed, 14 Dec 2022 22:32:17 -0500 Subject: [PATCH 03/44] uses new Request in references to chain verifier --- zebra-consensus/src/block.rs | 4 +- zebra-consensus/src/block/request.rs | 17 ++++--- zebra-consensus/src/block/tests.rs | 50 ++++++++++++------- zebra-consensus/src/chain/tests.rs | 24 +++++---- zebra-consensus/src/lib.rs | 2 +- .../src/methods/get_block_template_rpcs.rs | 10 ++-- zebra-rpc/src/server.rs | 16 +++--- zebra-state/src/service.rs | 14 +++--- zebrad/src/components/inbound.rs | 10 ++-- zebrad/src/components/inbound/downloads.rs | 23 +++++---- .../components/inbound/tests/real_peer_set.rs | 6 +-- zebrad/src/components/sync.rs | 8 +-- zebrad/src/components/sync/downloads.rs | 10 ++-- zebrad/src/components/sync/tests/vectors.rs | 30 +++++------ 14 files changed, 125 insertions(+), 99 deletions(-) diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs index b4edf7584e8..dc6d05ed341 100644 --- a/zebra-consensus/src/block.rs +++ b/zebra-consensus/src/block.rs @@ -286,7 +286,7 @@ where }; match request { - Request::Block(_) => match state_service + Request::Commit(_) => match state_service .ready() .await .map_err(VerifyBlockError::Commit)? @@ -302,7 +302,7 @@ where }, #[cfg(feature = "getblocktemplate-rpcs")] - Request::BlockProposal(_) => match state_service + Request::CheckProposal(_) => match state_service .ready() .await .map_err(VerifyBlockError::Validate)? diff --git a/zebra-consensus/src/block/request.rs b/zebra-consensus/src/block/request.rs index 083c742beae..b89df5f0c0d 100644 --- a/zebra-consensus/src/block/request.rs +++ b/zebra-consensus/src/block/request.rs @@ -4,31 +4,36 @@ use std::sync::Arc; use zebra_chain::block::Block; +#[derive(Debug, Clone, PartialEq, Eq)] +/// A request to the chain or block verifier pub enum Request { - Block(Arc), + /// Performs semantic validation then calls state with CommitBlock request + Commit(Arc), #[cfg(feature = "getblocktemplate-rpcs")] - BlockProposal(Arc), + /// Performs semantic validation but skips checking the solution, + /// then calls the state with CheckBlockValid request + CheckProposal(Arc), } impl Request { /// Returns inner block pub fn block(&self) -> Arc { Arc::clone(match self { - Request::Block(block) => block, + Request::Commit(block) => block, #[cfg(feature = "getblocktemplate-rpcs")] - Request::BlockProposal(block) => block, + Request::CheckProposal(block) => block, }) } /// Checks if request is a proposal pub fn is_proposal(&self) -> bool { match self { - Request::Block(_) => false, + Request::Commit(_) => false, #[cfg(feature = "getblocktemplate-rpcs")] - Request::BlockProposal(_) => true, + Request::CheckProposal(_) => true, } } } diff --git a/zebra-consensus/src/block/tests.rs b/zebra-consensus/src/block/tests.rs index 1901984541b..09512633b1c 100644 --- a/zebra-consensus/src/block/tests.rs +++ b/zebra-consensus/src/block/tests.rs @@ -28,19 +28,18 @@ use crate::{parameters::SLOW_START_SHIFT, transaction}; use super::*; -static VALID_BLOCK_TRANSCRIPT: Lazy< - Vec<(Arc, Result)>, -> = Lazy::new(|| { - let block: Arc<_> = - Block::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..]) - .unwrap() - .into(); - let hash = Ok(block.as_ref().into()); - vec![(block, hash)] -}); +static VALID_BLOCK_TRANSCRIPT: Lazy)>> = + Lazy::new(|| { + let block: Arc<_> = + Block::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..]) + .unwrap() + .into(); + let hash = Ok(block.as_ref().into()); + vec![(Request::Commit(block), hash)] + }); static INVALID_TIME_BLOCK_TRANSCRIPT: Lazy< - Vec<(Arc, Result)>, + Vec<(Request, Result)>, > = Lazy::new(|| { let mut block: Block = Block::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..]).unwrap(); @@ -55,11 +54,14 @@ static INVALID_TIME_BLOCK_TRANSCRIPT: Lazy< .unwrap(); Arc::make_mut(&mut block.header).time = three_hours_in_the_future; - vec![(Arc::new(block), Err(ExpectedTranscriptError::Any))] + vec![( + Request::Commit(Arc::new(block)), + Err(ExpectedTranscriptError::Any), + )] }); static INVALID_HEADER_SOLUTION_TRANSCRIPT: Lazy< - Vec<(Arc, Result)>, + Vec<(Request, Result)>, > = Lazy::new(|| { let mut block: Block = Block::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..]).unwrap(); @@ -67,11 +69,14 @@ static INVALID_HEADER_SOLUTION_TRANSCRIPT: Lazy< // Change nonce to something invalid Arc::make_mut(&mut block.header).nonce = [0; 32]; - vec![(Arc::new(block), Err(ExpectedTranscriptError::Any))] + vec![( + Request::Commit(Arc::new(block)), + Err(ExpectedTranscriptError::Any), + )] }); static INVALID_COINBASE_TRANSCRIPT: Lazy< - Vec<(Arc, Result)>, + Vec<(Request, Result)>, > = Lazy::new(|| { let header = block::Header::zcash_deserialize(&zebra_test::vectors::DUMMY_HEADER[..]).unwrap(); @@ -105,9 +110,18 @@ static INVALID_COINBASE_TRANSCRIPT: Lazy< assert_eq!(block3.transactions.len(), 2); vec![ - (Arc::new(block1), Err(ExpectedTranscriptError::Any)), - (Arc::new(block2), Err(ExpectedTranscriptError::Any)), - (Arc::new(block3), Err(ExpectedTranscriptError::Any)), + ( + Request::Commit(Arc::new(block1)), + Err(ExpectedTranscriptError::Any), + ), + ( + Request::Commit(Arc::new(block2)), + Err(ExpectedTranscriptError::Any), + ), + ( + Request::Commit(Arc::new(block3)), + Err(ExpectedTranscriptError::Any), + ), ] }); diff --git a/zebra-consensus/src/chain/tests.rs b/zebra-consensus/src/chain/tests.rs index 015e1d75f1a..308be754bdb 100644 --- a/zebra-consensus/src/chain/tests.rs +++ b/zebra-consensus/src/chain/tests.rs @@ -49,7 +49,7 @@ async fn verifiers_from_network( network: Network, ) -> ( impl Service< - Arc, + Request, Response = block::Hash, Error = BoxError, Future = impl Future>, @@ -77,7 +77,7 @@ async fn verifiers_from_network( } static BLOCK_VERIFY_TRANSCRIPT_GENESIS: Lazy< - Vec<(Arc, Result)>, + Vec<(Request, Result)>, > = Lazy::new(|| { let block: Arc<_> = Block::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..]) @@ -85,27 +85,29 @@ static BLOCK_VERIFY_TRANSCRIPT_GENESIS: Lazy< .into(); let hash = Ok(block.hash()); - vec![(block, hash)] + vec![(Request::Commit(block), hash)] }); static BLOCK_VERIFY_TRANSCRIPT_GENESIS_FAIL: Lazy< - Vec<(Arc, Result)>, + Vec<(Request, Result)>, > = Lazy::new(|| { let block: Arc<_> = Block::zcash_deserialize(&zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES[..]) .unwrap() .into(); - vec![(block, Err(ExpectedTranscriptError::Any))] + vec![(Request::Commit(block), Err(ExpectedTranscriptError::Any))] }); -static NO_COINBASE_TRANSCRIPT: Lazy< - Vec<(Arc, Result)>, -> = Lazy::new(|| { - let block = block_no_transactions(); +static NO_COINBASE_TRANSCRIPT: Lazy)>> = + Lazy::new(|| { + let block = block_no_transactions(); - vec![(Arc::new(block), Err(ExpectedTranscriptError::Any))] -}); + vec![( + Request::Commit(Arc::new(block)), + Err(ExpectedTranscriptError::Any), + )] + }); static NO_COINBASE_STATE_TRANSCRIPT: Lazy< Vec<(zs::Request, Result)>, diff --git a/zebra-consensus/src/lib.rs b/zebra-consensus/src/lib.rs index a10850b701a..de80af5f94c 100644 --- a/zebra-consensus/src/lib.rs +++ b/zebra-consensus/src/lib.rs @@ -49,7 +49,7 @@ pub mod error; pub use block::{ subsidy::funding_streams::funding_stream_address, subsidy::funding_streams::funding_stream_values, subsidy::funding_streams::new_coinbase_script, - subsidy::general::miner_subsidy, VerifyBlockError, MAX_BLOCK_SIGOPS, + subsidy::general::miner_subsidy, Request, VerifyBlockError, MAX_BLOCK_SIGOPS, }; pub use chain::VerifyChainError; pub use checkpoint::{ diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 895bb7543d2..c0b97c7dc83 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -165,7 +165,7 @@ where Response = zebra_state::ReadResponse, Error = zebra_state::BoxError, >, - ChainVerifier: Service, Response = block::Hash, Error = zebra_consensus::BoxError> + ChainVerifier: Service + Clone + Send + Sync @@ -217,7 +217,7 @@ where + Sync + 'static, Tip: ChainTip + Clone + Send + Sync + 'static, - ChainVerifier: Service, Response = block::Hash, Error = zebra_consensus::BoxError> + ChainVerifier: Service + Clone + Send + Sync @@ -265,12 +265,12 @@ where + 'static, >::Future: Send, Tip: ChainTip + Clone + Send + Sync + 'static, - ChainVerifier: Service, Response = block::Hash, Error = zebra_consensus::BoxError> + ChainVerifier: Service + Clone + Send + Sync + 'static, - >>::Future: Send, + >::Future: Send, SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, { fn get_block_count(&self) -> Result { @@ -601,7 +601,7 @@ where message: error.to_string(), data: None, })? - .call(Arc::new(block)) + .call(zebra_consensus::Request::Commit(Arc::new(block))) .await; let chain_error = match chain_verifier_response { diff --git a/zebra-rpc/src/server.rs b/zebra-rpc/src/server.rs index bfa373e6cc0..d3548a0572e 100644 --- a/zebra-rpc/src/server.rs +++ b/zebra-rpc/src/server.rs @@ -7,7 +7,7 @@ //! See the full list of //! [Differences between JSON-RPC 1.0 and 2.0.](https://www.simple-is-better.org/rpc/#differences-between-1-0-and-2-0) -use std::{fmt, panic, sync::Arc}; +use std::{fmt, panic}; use jsonrpc_core::{Compatibility, MetaIoHandler}; use jsonrpc_http_server::{CloseHandle, ServerBuilder}; @@ -17,10 +17,7 @@ use tower::{buffer::Buffer, Service}; use tracing::{Instrument, *}; use zebra_chain::{ - block::{self, Block}, - chain_sync_status::ChainSyncStatus, - chain_tip::ChainTip, - parameters::Network, + block, chain_sync_status::ChainSyncStatus, chain_tip::ChainTip, parameters::Network, }; use zebra_node_services::mempool; @@ -107,12 +104,15 @@ impl RpcServer { + 'static, State::Future: Send, Tip: ChainTip + Clone + Send + Sync + 'static, - ChainVerifier: Service, Response = block::Hash, Error = zebra_consensus::BoxError> - + Clone + ChainVerifier: Service< + zebra_consensus::Request, + Response = block::Hash, + Error = zebra_consensus::BoxError, + > + Clone + Send + Sync + 'static, - >>::Future: Send, + >::Future: Send, SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, { if let Some(listen_addr) = config.listen_addr { diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index ecaf98edccf..c5e22522e5f 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -810,7 +810,7 @@ impl ReadStateService { #[tracing::instrument(level = "debug", skip_all)] fn check_best_chain_contextual_validity( &self, - prepared: PreparedBlock, + prepared: &PreparedBlock, ) -> Result<(), crate::ValidateContextError> { let latest_non_finalized_state = self.latest_non_finalized_state(); @@ -826,22 +826,22 @@ impl ReadStateService { self.network, &self.db, &latest_non_finalized_state, - &prepared, + prepared, )?; // Reads from disk let sprout_final_treestates = - check::anchors::block_fetch_sprout_final_treestates(&self.db, best_chain, &prepared); + check::anchors::block_fetch_sprout_final_treestates(&self.db, best_chain, prepared); let spent_utxos = check::utxo::transparent_spend( - &prepared, + prepared, &best_chain.unspent_utxos(), &best_chain.spent_utxos, &self.db, )?; check::anchors::block_sapling_orchard_anchors_refer_to_final_treestates( - &self.db, best_chain, &prepared, + &self.db, best_chain, prepared, )?; let contextual = crate::request::ContextuallyValidBlock::with_block_and_spent_utxos( @@ -1801,7 +1801,7 @@ impl Service for ReadStateService { let span = Span::current(); tokio::task::spawn_blocking(move || { span.in_scope(move || { - state.check_best_chain_contextual_validity(prepared)?; + state.check_best_chain_contextual_validity(&prepared)?; // The work is done in the future. timer.finish( module_path!(), @@ -1809,7 +1809,7 @@ impl Service for ReadStateService { "ReadRequest::CheckContextualValidity", ); - Ok(ReadResponse::Validated) + Ok(ReadResponse::ValidBlock(prepared.hash)) }) }) .map(|join_result| join_result.expect("panic in ReadRequest::ChainInfo")) diff --git a/zebrad/src/components/inbound.rs b/zebrad/src/components/inbound.rs index 363b6411ac1..880f58cb49f 100644 --- a/zebrad/src/components/inbound.rs +++ b/zebrad/src/components/inbound.rs @@ -24,10 +24,7 @@ use tower::{buffer::Buffer, timeout::Timeout, util::BoxService, Service, Service use zebra_network as zn; use zebra_state as zs; -use zebra_chain::{ - block::{self, Block}, - transaction::UnminedTxId, -}; +use zebra_chain::{block, transaction::UnminedTxId}; use zebra_consensus::chain::VerifyChainError; use zebra_network::{ constants::{ADDR_RESPONSE_LIMIT_DENOMINATOR, MAX_ADDRS_IN_MESSAGE}, @@ -53,7 +50,10 @@ type BlockDownloadPeerSet = Buffer, zn::Request>; type State = Buffer, zs::Request>; type Mempool = Buffer, mempool::Request>; -type BlockVerifier = Buffer, block::Hash, VerifyChainError>, Arc>; +type BlockVerifier = Buffer< + BoxService, + zebra_consensus::Request, +>; type GossipedBlockDownloads = BlockDownloads, Timeout, State>; diff --git a/zebrad/src/components/inbound/downloads.rs b/zebrad/src/components/inbound/downloads.rs index 250b904831f..130d324ba45 100644 --- a/zebrad/src/components/inbound/downloads.rs +++ b/zebrad/src/components/inbound/downloads.rs @@ -4,7 +4,6 @@ use std::{ collections::HashMap, convert::TryFrom, pin::Pin, - sync::Arc, task::{Context, Poll}, }; @@ -18,10 +17,7 @@ use tokio::{sync::oneshot, task::JoinHandle}; use tower::{Service, ServiceExt}; use tracing_futures::Instrument; -use zebra_chain::{ - block::{self, Block}, - chain_tip::ChainTip, -}; +use zebra_chain::{block, chain_tip::ChainTip}; use zebra_network as zn; use zebra_state as zs; @@ -77,7 +73,10 @@ pub struct Downloads where ZN: Service + Send + Clone + 'static, ZN::Future: Send, - ZV: Service, Response = block::Hash, Error = BoxError> + Send + Clone + 'static, + ZV: Service + + Send + + Clone + + 'static, ZV::Future: Send, ZS: Service + Send + Clone + 'static, ZS::Future: Send, @@ -117,7 +116,10 @@ impl Stream for Downloads where ZN: Service + Send + Clone + 'static, ZN::Future: Send, - ZV: Service, Response = block::Hash, Error = BoxError> + Send + Clone + 'static, + ZV: Service + + Send + + Clone + + 'static, ZV::Future: Send, ZS: Service + Send + Clone + 'static, ZS::Future: Send, @@ -160,7 +162,10 @@ impl Downloads where ZN: Service + Send + Clone + 'static, ZN::Future: Send, - ZV: Service, Response = block::Hash, Error = BoxError> + Send + Clone + 'static, + ZV: Service + + Send + + Clone + + 'static, ZV::Future: Send, ZS: Service + Send + Clone + 'static, ZS::Future: Send, @@ -338,7 +343,7 @@ where } verifier - .oneshot(block) + .oneshot(zebra_consensus::Request::Commit(block)) .await .map(|hash| (hash, block_height)) } diff --git a/zebrad/src/components/inbound/tests/real_peer_set.rs b/zebrad/src/components/inbound/tests/real_peer_set.rs index e1611fa90ff..c610f1a1f71 100644 --- a/zebrad/src/components/inbound/tests/real_peer_set.rs +++ b/zebrad/src/components/inbound/tests/real_peer_set.rs @@ -1,6 +1,6 @@ //! Inbound service tests with a real peer set. -use std::{iter, net::SocketAddr, sync::Arc}; +use std::{iter, net::SocketAddr}; use futures::FutureExt; use indexmap::IndexSet; @@ -13,7 +13,7 @@ use tower::{ }; use zebra_chain::{ - block::{self, Block, Height}, + block::{self, Height}, parameters::Network, serialization::ZcashDeserializeInto, transaction::{AuthDigest, Hash as TxHash, Transaction, UnminedTx, UnminedTxId, WtxId}, @@ -603,7 +603,7 @@ async fn setup( Buffer, mempool::Request>, Buffer, zebra_state::Request>, // mocked services - MockService, block::Hash, PanicAssertion, VerifyChainError>, + MockService, MockService, // real tasks JoinHandle>, diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs index 1a502945bc1..33f42622559 100644 --- a/zebrad/src/components/sync.rs +++ b/zebrad/src/components/sync.rs @@ -2,7 +2,7 @@ //! //! It is used when Zebra is a long way behind the current chain tip. -use std::{cmp::max, collections::HashSet, pin::Pin, sync::Arc, task::Poll, time::Duration}; +use std::{cmp::max, collections::HashSet, pin::Pin, task::Poll, time::Duration}; use color_eyre::eyre::{eyre, Report}; use futures::stream::{FuturesUnordered, StreamExt}; @@ -15,7 +15,7 @@ use tower::{ }; use zebra_chain::{ - block::{self, Block, Height}, + block::{self, Height}, chain_tip::ChainTip, parameters::genesis_hash, }; @@ -300,7 +300,7 @@ where + Clone + 'static, ZS::Future: Send, - ZV: Service, Response = block::Hash, Error = BoxError> + ZV: Service + Send + Sync + Clone @@ -381,7 +381,7 @@ where + Clone + 'static, ZS::Future: Send, - ZV: Service, Response = block::Hash, Error = BoxError> + ZV: Service + Send + Sync + Clone diff --git a/zebrad/src/components/sync/downloads.rs b/zebrad/src/components/sync/downloads.rs index 10c4bda8f31..4b6b5f00256 100644 --- a/zebrad/src/components/sync/downloads.rs +++ b/zebrad/src/components/sync/downloads.rs @@ -24,7 +24,7 @@ use tower::{hedge, Service, ServiceExt}; use tracing_futures::Instrument; use zebra_chain::{ - block::{self, Block, Height}, + block::{self, Height}, chain_tip::ChainTip, }; use zebra_network as zn; @@ -163,7 +163,7 @@ pub struct Downloads where ZN: Service + Send + Sync + 'static, ZN::Future: Send, - ZV: Service, Response = block::Hash, Error = BoxError> + ZV: Service + Send + Sync + Clone @@ -217,7 +217,7 @@ impl Stream for Downloads where ZN: Service + Send + Sync + 'static, ZN::Future: Send, - ZV: Service, Response = block::Hash, Error = BoxError> + ZV: Service + Send + Sync + Clone @@ -264,7 +264,7 @@ impl Downloads where ZN: Service + Send + Sync + 'static, ZN::Future: Send, - ZV: Service, Response = block::Hash, Error = BoxError> + ZV: Service + Send + Sync + Clone @@ -516,7 +516,7 @@ where // Verify the block. let mut rsp = verifier .map_err(|error| BlockDownloadVerifyError::VerifierServiceError { error })? - .call(block).boxed(); + .call(zebra_consensus::Request::Commit(block)).boxed(); // Add a shorter timeout to workaround a known bug (#5125) let short_timeout_max = (max_checkpoint_height + FINAL_CHECKPOINT_BLOCK_VERIFY_TIMEOUT_LIMIT).expect("checkpoint block height is in valid range"); diff --git a/zebrad/src/components/sync/tests/vectors.rs b/zebrad/src/components/sync/tests/vectors.rs index 4acc67760c7..468d74958ab 100644 --- a/zebrad/src/components/sync/tests/vectors.rs +++ b/zebrad/src/components/sync/tests/vectors.rs @@ -89,7 +89,7 @@ async fn sync_blocks_ok() -> Result<(), crate::BoxError> { .respond(zn::Response::Blocks(vec![Available(block0.clone())])); chain_verifier - .expect_request(block0) + .expect_request(zebra_consensus::Request::Commit(block0)) .await .respond(block0_hash); @@ -175,9 +175,9 @@ async fn sync_blocks_ok() -> Result<(), crate::BoxError> { for _ in 1..=2 { chain_verifier - .expect_request_that(|req| remaining_blocks.remove(&req.hash()).is_some()) + .expect_request_that(|req| remaining_blocks.remove(&req.block().hash()).is_some()) .await - .respond_with(|req| req.hash()); + .respond_with(|req| req.block().hash()); } assert_eq!( remaining_blocks, @@ -239,9 +239,9 @@ async fn sync_blocks_ok() -> Result<(), crate::BoxError> { for _ in 3..=4 { chain_verifier - .expect_request_that(|req| remaining_blocks.remove(&req.hash()).is_some()) + .expect_request_that(|req| remaining_blocks.remove(&req.block().hash()).is_some()) .await - .respond_with(|req| req.hash()); + .respond_with(|req| req.block().hash()); } assert_eq!( remaining_blocks, @@ -316,7 +316,7 @@ async fn sync_blocks_duplicate_hashes_ok() -> Result<(), crate::BoxError> { .respond(zn::Response::Blocks(vec![Available(block0.clone())])); chain_verifier - .expect_request(block0) + .expect_request(zebra_consensus::Request::Commit(block0)) .await .respond(block0_hash); @@ -404,9 +404,9 @@ async fn sync_blocks_duplicate_hashes_ok() -> Result<(), crate::BoxError> { for _ in 1..=2 { chain_verifier - .expect_request_that(|req| remaining_blocks.remove(&req.hash()).is_some()) + .expect_request_that(|req| remaining_blocks.remove(&req.block().hash()).is_some()) .await - .respond_with(|req| req.hash()); + .respond_with(|req| req.block().hash()); } assert_eq!( remaining_blocks, @@ -470,9 +470,9 @@ async fn sync_blocks_duplicate_hashes_ok() -> Result<(), crate::BoxError> { for _ in 3..=4 { chain_verifier - .expect_request_that(|req| remaining_blocks.remove(&req.hash()).is_some()) + .expect_request_that(|req| remaining_blocks.remove(&req.block().hash()).is_some()) .await - .respond_with(|req| req.hash()); + .respond_with(|req| req.block().hash()); } assert_eq!( remaining_blocks, @@ -598,7 +598,7 @@ async fn sync_block_too_high_obtain_tips() -> Result<(), crate::BoxError> { .respond(zn::Response::Blocks(vec![Available(block0.clone())])); chain_verifier - .expect_request(block0) + .expect_request(zebra_consensus::Request::Commit(block0)) .await .respond(block0_hash); @@ -759,7 +759,7 @@ async fn sync_block_too_high_extend_tips() -> Result<(), crate::BoxError> { .respond(zn::Response::Blocks(vec![Available(block0.clone())])); chain_verifier - .expect_request(block0) + .expect_request(zebra_consensus::Request::Commit(block0)) .await .respond(block0_hash); @@ -845,9 +845,9 @@ async fn sync_block_too_high_extend_tips() -> Result<(), crate::BoxError> { for _ in 1..=2 { chain_verifier - .expect_request_that(|req| remaining_blocks.remove(&req.hash()).is_some()) + .expect_request_that(|req| remaining_blocks.remove(&req.block().hash()).is_some()) .await - .respond_with(|req| req.hash()); + .respond_with(|req| req.block().hash()); } assert_eq!( remaining_blocks, @@ -927,7 +927,7 @@ fn setup() -> ( impl Future> + Send, SyncStatus, // ChainVerifier - MockService, block::Hash, PanicAssertion>, + MockService, // PeerSet MockService, // StateService From f7c594185c701ba967b070c25b941b2cd32dcb83 Mon Sep 17 00:00:00 2001 From: arya2 Date: Thu, 15 Dec 2022 01:22:20 -0500 Subject: [PATCH 04/44] adds getblocktemplate proposal mode response type --- .../src/methods/get_block_template_rpcs.rs | 501 +++++++++--------- .../get_block_template.rs | 71 ++- .../types/get_block_template.rs | 59 ++- .../types/get_block_template/parameters.rs | 7 +- .../tests/snapshot/get_block_template_rpcs.rs | 14 +- zebra-rpc/src/methods/tests/vectors.rs | 6 +- zebra-state/src/service.rs | 2 +- 7 files changed, 399 insertions(+), 261 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index c0b97c7dc83..e7b1610aa73 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -26,9 +26,8 @@ use crate::methods::{ DEFAULT_SOLUTION_RATE_WINDOW_SIZE, GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, }, get_block_template::{ - check_block_template_parameters, check_miner_address, check_synced_to_tip, - fetch_mempool_transactions, fetch_state_tip_and_local_time, - generate_coinbase_and_roots, + check_miner_address, check_synced_to_tip, fetch_mempool_transactions, + fetch_state_tip_and_local_time, generate_coinbase_and_roots, }, types::{ get_block_template::GetBlockTemplate, get_mining_info, hex_data::HexData, @@ -101,7 +100,7 @@ pub trait GetBlockTemplateRpc { fn get_block_template( &self, parameters: Option, - ) -> BoxFuture>; + ) -> BoxFuture>; /// Submits block to the node to be validated and committed. /// Returns the [`submit_block::Response`] for the operation, as a JSON string. @@ -316,7 +315,7 @@ where fn get_block_template( &self, parameters: Option, - ) -> BoxFuture> { + ) -> BoxFuture> { // Clone Config let network = self.network; let miner_address = self.miner_address; @@ -326,256 +325,284 @@ where let mut latest_chain_tip = self.latest_chain_tip.clone(); let sync_status = self.sync_status.clone(); let state = self.state.clone(); + let chain_verifier = get_block_template::JsonParameters::is_proposal_mode(¶meters) + .then(|| self.chain_verifier.clone()); // To implement long polling correctly, we split this RPC into multiple phases. async move { - // - One-off checks - - // Check config and parameters. - // These checks always have the same result during long polling. - let miner_address = check_miner_address(miner_address)?; - - let mut client_long_poll_id = None; - if let Some(parameters) = parameters { - check_block_template_parameters(¶meters)?; - - client_long_poll_id = parameters.long_poll_id; - } - - // - Checks and fetches that can change during long polling - // - // Set up the loop. - let mut max_time_reached = false; - - // The loop returns the server long poll ID, - // which should be different to the client long poll ID. - let (server_long_poll_id, chain_tip_and_local_time, mempool_txs, submit_old) = loop { - // Check if we are synced to the tip. - // The result of this check can change during long polling. - // - // Optional TODO: - // - add `async changed()` method to ChainSyncStatus (like `ChainTip`) - check_synced_to_tip(network, latest_chain_tip.clone(), sync_status.clone())?; - - // We're just about to fetch state data, then maybe wait for any changes. - // Mark all the changes before the fetch as seen. - // Changes are also ignored in any clones made after the mark. - latest_chain_tip.mark_best_tip_seen(); - - // Fetch the state data and local time for the block template: - // - if the tip block hash changes, we must return from long polling, - // - if the local clock changes on testnet, we might return from long polling + let client_long_poll_id = parameters.as_ref().and_then(|params| params.long_poll_id.clone()); + + if let Some(HexData(block_proposal_bytes)) = + get_block_template::JsonParameters::block_proposal(parameters)? + { + let mut chain_verifier = + chain_verifier.expect("chain_verifier is Some in proposal mode"); + + let block: Block = match block_proposal_bytes.zcash_deserialize_into() { + Ok(block_bytes) => block_bytes, + Err(_) => return Ok(get_block_template::ProposalRejectReason::Rejected.into()), + }; + + let chain_verifier_response = chain_verifier + .ready() + .await + .map_err(|error| Error { + code: ErrorCode::ServerError(0), + message: error.to_string(), + data: None, + })? + .call(zebra_consensus::Request::CheckProposal(Arc::new(block))) + .await; + + let response = chain_verifier_response + .map(|_| get_block_template::ProposalResponse::Valid) + .unwrap_or_else(|_| get_block_template::ProposalRejectReason::Rejected.into()); + + Ok(get_block_template::Response::ProposalMode(response)) + } else { + // - One-off checks + + // Check config and parameters. + // These checks always have the same result during long polling. + let miner_address = check_miner_address(miner_address)?; + + // - Checks and fetches that can change during long polling // - // We always return after 90 minutes on mainnet, even if we have the same response, - // because the max time has been reached. - let chain_tip_and_local_time = - fetch_state_tip_and_local_time(state.clone()).await?; + // Set up the loop. + let mut max_time_reached = false; + + // The loop returns the server long poll ID, + // which should be different to the client long poll ID. + let (server_long_poll_id, chain_tip_and_local_time, mempool_txs, submit_old) = loop { + // Check if we are synced to the tip. + // The result of this check can change during long polling. + // + // Optional TODO: + // - add `async changed()` method to ChainSyncStatus (like `ChainTip`) + check_synced_to_tip(network, latest_chain_tip.clone(), sync_status.clone())?; + + // We're just about to fetch state data, then maybe wait for any changes. + // Mark all the changes before the fetch as seen. + // Changes are also ignored in any clones made after the mark. + latest_chain_tip.mark_best_tip_seen(); + + // Fetch the state data and local time for the block template: + // - if the tip block hash changes, we must return from long polling, + // - if the local clock changes on testnet, we might return from long polling + // + // We always return after 90 minutes on mainnet, even if we have the same response, + // because the max time has been reached. + let chain_tip_and_local_time = + fetch_state_tip_and_local_time(state.clone()).await?; + + // Fetch the mempool data for the block template: + // - if the mempool transactions change, we might return from long polling. + // + // If the chain fork has just changed, miners want to get the new block as fast + // as possible, rather than wait for transactions to re-verify. This increases + // miner profits (and any delays can cause chain forks). So we don't wait between + // the chain tip changing and getting mempool transactions. + // + // Optional TODO: + // - add a `MempoolChange` type with an `async changed()` method (like `ChainTip`) + let mempool_txs = fetch_mempool_transactions(mempool.clone()).await?; + + // - Long poll ID calculation + let server_long_poll_id = LongPollInput::new( + chain_tip_and_local_time.tip_height, + chain_tip_and_local_time.tip_hash, + chain_tip_and_local_time.max_time, + mempool_txs.iter().map(|tx| tx.transaction.id), + ) + .generate_id(); + + // The loop finishes if: + // - the client didn't pass a long poll ID, + // - the server long poll ID is different to the client long poll ID, or + // - the previous loop iteration waited until the max time. + if Some(&server_long_poll_id) != client_long_poll_id.as_ref() || max_time_reached { + let mut submit_old = client_long_poll_id + .as_ref() + .map(|old_long_poll_id| server_long_poll_id.submit_old(old_long_poll_id)); + + // On testnet, the max time changes the block difficulty, so old shares are + // invalid. On mainnet, this means there has been 90 minutes without a new + // block or mempool transaction, which is very unlikely. So the miner should + // probably reset anyway. + if max_time_reached { + submit_old = Some(false); + } - // Fetch the mempool data for the block template: - // - if the mempool transactions change, we might return from long polling. - // - // If the chain fork has just changed, miners want to get the new block as fast - // as possible, rather than wait for transactions to re-verify. This increases - // miner profits (and any delays can cause chain forks). So we don't wait between - // the chain tip changing and getting mempool transactions. - // - // Optional TODO: - // - add a `MempoolChange` type with an `async changed()` method (like `ChainTip`) - let mempool_txs = fetch_mempool_transactions(mempool.clone()).await?; - - // - Long poll ID calculation - let server_long_poll_id = LongPollInput::new( - chain_tip_and_local_time.tip_height, - chain_tip_and_local_time.tip_hash, - chain_tip_and_local_time.max_time, - mempool_txs.iter().map(|tx| tx.transaction.id), - ) - .generate_id(); - - // The loop finishes if: - // - the client didn't pass a long poll ID, - // - the server long poll ID is different to the client long poll ID, or - // - the previous loop iteration waited until the max time. - if Some(&server_long_poll_id) != client_long_poll_id.as_ref() || max_time_reached { - let mut submit_old = client_long_poll_id - .as_ref() - .map(|old_long_poll_id| server_long_poll_id.submit_old(old_long_poll_id)); - - // On testnet, the max time changes the block difficulty, so old shares are - // invalid. On mainnet, this means there has been 90 minutes without a new - // block or mempool transaction, which is very unlikely. So the miner should - // probably reset anyway. - if max_time_reached { - submit_old = Some(false); + break ( + server_long_poll_id, + chain_tip_and_local_time, + mempool_txs, + submit_old, + ); } - break ( - server_long_poll_id, - chain_tip_and_local_time, - mempool_txs, - submit_old, - ); - } - - // - Polling wait conditions - // - // TODO: when we're happy with this code, split it into a function. - // - // Periodically check the mempool for changes. - // - // Optional TODO: - // Remove this polling wait if we switch to using futures to detect sync status - // and mempool changes. - let wait_for_mempool_request = tokio::time::sleep(Duration::from_secs( - GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, - )); - - // Return immediately if the chain tip has changed. - let wait_for_best_tip_change = latest_chain_tip.best_tip_changed(); - - // Wait for the maximum block time to elapse. This can change the block header - // on testnet. (On mainnet it can happen due to a network disconnection, or a - // rapid drop in hash rate.) - // - // This duration might be slightly lower than the actual maximum, - // if cur_time was clamped to min_time. In that case the wait is very long, - // and it's ok to return early. - // - // It can also be zero if cur_time was clamped to max_time. In that case, - // we want to wait for another change, and ignore this timeout. So we use an - // `OptionFuture::None`. - let duration_until_max_time = chain_tip_and_local_time - .max_time - .saturating_duration_since(chain_tip_and_local_time.cur_time); - let wait_for_max_time: OptionFuture<_> = if duration_until_max_time.seconds() > 0 { - Some(tokio::time::sleep(duration_until_max_time.to_std())) - } else { - None - } - .into(); - - // Optional TODO: - // `zcashd` generates the next coinbase transaction while waiting for changes. - // When Zebra supports shielded coinbase, we might want to do this in parallel. - // But the coinbase value depends on the selected transactions, so this needs - // further analysis to check if it actually saves us any time. - - // TODO: change logging to debug after testing - tokio::select! { - // Poll the futures in the listed order, for efficiency. - // We put the most frequent conditions first. - biased; - - // This timer elapses every few seconds - _elapsed = wait_for_mempool_request => { - tracing::info!( - max_time = ?chain_tip_and_local_time.max_time, - cur_time = ?chain_tip_and_local_time.cur_time, - ?server_long_poll_id, - ?client_long_poll_id, - GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, - "checking for a new mempool change after waiting a few seconds" - ); + // - Polling wait conditions + // + // TODO: when we're happy with this code, split it into a function. + // + // Periodically check the mempool for changes. + // + // Optional TODO: + // Remove this polling wait if we switch to using futures to detect sync status + // and mempool changes. + let wait_for_mempool_request = tokio::time::sleep(Duration::from_secs( + GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, + )); + + // Return immediately if the chain tip has changed. + let wait_for_best_tip_change = latest_chain_tip.best_tip_changed(); + + // Wait for the maximum block time to elapse. This can change the block header + // on testnet. (On mainnet it can happen due to a network disconnection, or a + // rapid drop in hash rate.) + // + // This duration might be slightly lower than the actual maximum, + // if cur_time was clamped to min_time. In that case the wait is very long, + // and it's ok to return early. + // + // It can also be zero if cur_time was clamped to max_time. In that case, + // we want to wait for another change, and ignore this timeout. So we use an + // `OptionFuture::None`. + let duration_until_max_time = chain_tip_and_local_time + .max_time + .saturating_duration_since(chain_tip_and_local_time.cur_time); + let wait_for_max_time: OptionFuture<_> = if duration_until_max_time.seconds() > 0 { + Some(tokio::time::sleep(duration_until_max_time.to_std())) + } else { + None } + .into(); + + // Optional TODO: + // `zcashd` generates the next coinbase transaction while waiting for changes. + // When Zebra supports shielded coinbase, we might want to do this in parallel. + // But the coinbase value depends on the selected transactions, so this needs + // further analysis to check if it actually saves us any time. + + // TODO: change logging to debug after testing + tokio::select! { + // Poll the futures in the listed order, for efficiency. + // We put the most frequent conditions first. + biased; + + // This timer elapses every few seconds + _elapsed = wait_for_mempool_request => { + tracing::info!( + max_time = ?chain_tip_and_local_time.max_time, + cur_time = ?chain_tip_and_local_time.cur_time, + ?server_long_poll_id, + ?client_long_poll_id, + GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, + "checking for a new mempool change after waiting a few seconds" + ); + } - // The state changes after around a target block interval (75s) - tip_changed_result = wait_for_best_tip_change => { - match tip_changed_result { - Ok(()) => { - tracing::info!( - max_time = ?chain_tip_and_local_time.max_time, - cur_time = ?chain_tip_and_local_time.cur_time, - ?server_long_poll_id, - ?client_long_poll_id, - "returning from long poll because state has changed" - ); + // The state changes after around a target block interval (75s) + tip_changed_result = wait_for_best_tip_change => { + match tip_changed_result { + Ok(()) => { + tracing::info!( + max_time = ?chain_tip_and_local_time.max_time, + cur_time = ?chain_tip_and_local_time.cur_time, + ?server_long_poll_id, + ?client_long_poll_id, + "returning from long poll because state has changed" + ); + } + + Err(recv_error) => { + // This log should stay at info when the others go to debug, + // it will help with debugging. + tracing::info!( + ?recv_error, + max_time = ?chain_tip_and_local_time.max_time, + cur_time = ?chain_tip_and_local_time.cur_time, + ?server_long_poll_id, + ?client_long_poll_id, + "returning from long poll due to a state error.\ + Is Zebra shutting down?" + ); + + return Err(Error { + code: ErrorCode::ServerError(0), + message: recv_error.to_string(), + data: None, + }); + } } + } - Err(recv_error) => { - // This log should stay at info when the others go to debug, - // it will help with debugging. - tracing::info!( - ?recv_error, - max_time = ?chain_tip_and_local_time.max_time, - cur_time = ?chain_tip_and_local_time.cur_time, - ?server_long_poll_id, - ?client_long_poll_id, - "returning from long poll due to a state error.\ - Is Zebra shutting down?" - ); - - return Err(Error { - code: ErrorCode::ServerError(0), - message: recv_error.to_string(), - data: None, - }); - } + // The max time does not elapse during normal operation on mainnet, + // and it rarely elapses on testnet. + Some(_elapsed) = wait_for_max_time => { + // This log should stay at info when the others go to debug, + // it's very rare. + tracing::info!( + max_time = ?chain_tip_and_local_time.max_time, + cur_time = ?chain_tip_and_local_time.cur_time, + ?server_long_poll_id, + ?client_long_poll_id, + "returning from long poll because max time was reached" + ); + + max_time_reached = true; } } + }; - // The max time does not elapse during normal operation on mainnet, - // and it rarely elapses on testnet. - Some(_elapsed) = wait_for_max_time => { - // This log should stay at info when the others go to debug, - // it's very rare. - tracing::info!( - max_time = ?chain_tip_and_local_time.max_time, - cur_time = ?chain_tip_and_local_time.cur_time, - ?server_long_poll_id, - ?client_long_poll_id, - "returning from long poll because max time was reached" - ); + // - Processing fetched data to create a transaction template + // + // Apart from random weighted transaction selection, + // the template only depends on the previously fetched data. + // This processing never fails. - max_time_reached = true; - } - } - }; + // Calculate the next block height. + let next_block_height = + (chain_tip_and_local_time.tip_height + 1).expect("tip is far below Height::MAX"); - // - Processing fetched data to create a transaction template - // - // Apart from random weighted transaction selection, - // the template only depends on the previously fetched data. - // This processing never fails. + // Randomly select some mempool transactions. + // + // TODO: sort these transactions to match zcashd's order, to make testing easier. + let mempool_txs = zip317::select_mempool_transactions( + network, + next_block_height, + miner_address, + mempool_txs, + ) + .await; - // Calculate the next block height. - let next_block_height = - (chain_tip_and_local_time.tip_height + 1).expect("tip is far below Height::MAX"); + // - After this point, the template only depends on the previously fetched data. - // Randomly select some mempool transactions. - // - // TODO: sort these transactions to match zcashd's order, to make testing easier. - let mempool_txs = zip317::select_mempool_transactions( - network, - next_block_height, - miner_address, - mempool_txs, - ) - .await; - - // - After this point, the template only depends on the previously fetched data. - - // Generate the coinbase transaction and default roots - // - // TODO: move expensive root, hash, and tree cryptography to a rayon thread? - let (coinbase_txn, default_roots) = generate_coinbase_and_roots( - network, - next_block_height, - miner_address, - &mempool_txs, - chain_tip_and_local_time.history_tree.clone(), - ); - - let response = GetBlockTemplate::new( - next_block_height, - &chain_tip_and_local_time, - server_long_poll_id, - coinbase_txn, - &mempool_txs, - default_roots, - submit_old, - ); - - Ok(response) + // Generate the coinbase transaction and default roots + // + // TODO: move expensive root, hash, and tree cryptography to a rayon thread? + let (coinbase_txn, default_roots) = generate_coinbase_and_roots( + network, + next_block_height, + miner_address, + &mempool_txs, + chain_tip_and_local_time.history_tree.clone(), + ); + + let response = GetBlockTemplate::new( + next_block_height, + &chain_tip_and_local_time, + server_long_poll_id, + coinbase_txn, + &mempool_txs, + default_roots, + submit_old, + ); + + Ok(get_block_template::Response::TemplateMode(Box::new( + response, + ))) + } } .boxed() } diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs index 8af438884a7..9b6225a0a6c 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs @@ -28,21 +28,70 @@ use crate::methods::get_block_template_rpcs::{ pub use crate::methods::get_block_template_rpcs::types::get_block_template::*; +use super::types::hex_data::HexData; + // - Parameter checks -/// Returns an error if the get block template RPC `parameters` are invalid. -pub fn check_block_template_parameters( - parameters: &get_block_template::JsonParameters, -) -> Result<()> { - if parameters.data.is_some() || parameters.mode == GetBlockTemplateRequestMode::Proposal { - return Err(Error { - code: ErrorCode::InvalidParams, - message: "\"proposal\" mode is currently unsupported by Zebra".to_string(), - data: None, - }); +impl JsonParameters { + /// Checks that `data` is omitted in `Template` mode or provided in `Proposal` mode, + /// + /// Returns an error if there's a mismatch between the mode and whether `data` is provided. + /// Returns Ok(Some(data)) with the block proposal hexdata if in `Proposal` mode. + /// Returns Ok(None) if in `Template` mode. + pub fn block_proposal(parameters: Option) -> Result> { + let Some(parameters) = parameters else { + return Ok(None) + }; + + match parameters { + Self { + mode: GetBlockTemplateRequestMode::Template, + data: None, + .. + } => Ok(None), + + Self { + mode: GetBlockTemplateRequestMode::Proposal, + data: data @ Some(_), + .. + } => Ok(data), + + Self { + mode: GetBlockTemplateRequestMode::Proposal, + data: None, + .. + } => Err(Error { + code: ErrorCode::InvalidParams, + message: "\"data\" parameter must be \ + provided in \"proposal\" mode" + .to_string(), + data: None, + }), + + Self { + mode: GetBlockTemplateRequestMode::Template, + data: Some(_), + .. + } => Err(Error { + code: ErrorCode::InvalidParams, + message: "\"data\" parameter must be \ + omitted in \"template\" mode" + .to_string(), + data: None, + }), + } } - Ok(()) + /// Checks if `mode` parameters is `Proposal`. + pub fn is_proposal_mode(parameters: &Option) -> bool { + matches!( + parameters, + Some(get_block_template::JsonParameters { + mode: GetBlockTemplateRequestMode::Proposal, + .. + }) + ) + } } /// Returns the miner address, or an error if it is invalid. diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs index 77134e16c56..08cfe80d03d 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs @@ -27,7 +27,7 @@ pub mod parameters; pub use parameters::*; -/// A serialized `getblocktemplate` RPC response. +/// A serialized `getblocktemplate` RPC response in template mode. #[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] pub struct GetBlockTemplate { /// The getblocktemplate RPC capabilities supported by Zebra. @@ -234,3 +234,60 @@ impl GetBlockTemplate { } } } + +/// Error response to a `getblocktemplate` RPC request in proposal mode. +#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ProposalRejectReason { + /// Block rejected as invalid + Rejected, +} + +/// Response to a `getblocktemplate` RPC request in proposal mode. +#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(untagged, rename_all = "kebab-case")] +pub enum ProposalResponse { + /// Block was not successfully submitted, return error + ErrorResponse { + /// Reason the proposal was invalid as-is + reject_reason: ProposalRejectReason, + + /// The getblocktemplate RPC capabilities supported by Zebra. + capabilities: Vec, + }, + + /// Block successfully proposed, returns null + Valid, +} + +#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(untagged)] +/// A `getblocktemplate` RPC response. +pub enum Response { + /// `getblocktemplate` RPC request in template mode. + TemplateMode(Box), + + /// `getblocktemplate` RPC request in proposal mode. + ProposalMode(ProposalResponse), +} + +impl From for ProposalResponse { + fn from(reject_reason: ProposalRejectReason) -> Self { + // Convert default values + let capabilities: Vec = GET_BLOCK_TEMPLATE_CAPABILITIES_FIELD + .iter() + .map(ToString::to_string) + .collect(); + + Self::ErrorResponse { + reject_reason, + capabilities, + } + } +} + +impl From for Response { + fn from(error_response: ProposalRejectReason) -> Self { + Self::ProposalMode(ProposalResponse::from(error_response)) + } +} diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs index 01152891816..d693b2a18ff 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs @@ -11,7 +11,6 @@ pub enum GetBlockTemplateRequestMode { Template, /// Indicates a request to validate block data. - /// Currently unsupported and will return an error. Proposal, } @@ -68,14 +67,14 @@ pub struct JsonParameters { /// Defines whether the RPC method should generate a block template or attempt to /// validate block data, checking against all of the server's usual acceptance rules /// (excluding the check for a valid proof-of-work). - // TODO: Support `proposal` mode. #[serde(default)] pub mode: GetBlockTemplateRequestMode, - /// Must be omitted as "proposal" mode is currently unsupported. + /// Must be omitted when `getblocktemplate` RPC is called in "template" mode (or when `mode` is omitted). + /// Must be provided when `getblocktemplate` RPC is called in "proposal" mode. /// /// Hex-encoded block data to be validated and checked against the server's usual acceptance rules - /// (excluding the check for a valid proof-of-work) when `mode` is set to `proposal`. + /// (excluding the check for a valid proof-of-work). pub data: Option, /// A list of client-side supported capability features diff --git a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs index 2da8102dfb7..f1e6b4b49ee 100644 --- a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs @@ -195,10 +195,12 @@ pub async fn test_responses( .await .respond(mempool::Response::FullTransactions(vec![])); - let get_block_template = get_block_template + let get_block_template::Response::TemplateMode(get_block_template) = get_block_template .await .expect("unexpected panic in getblocktemplate RPC task") - .expect("unexpected error in getblocktemplate RPC call"); + .expect("unexpected error in getblocktemplate RPC call") else { + panic!("this getblocktemplate call without parameters should return the `TemplateMode` variant of the response") + }; let coinbase_tx: Transaction = get_block_template .coinbase_txn @@ -250,10 +252,12 @@ pub async fn test_responses( .await .respond(mempool::Response::FullTransactions(vec![])); - let get_block_template = get_block_template + let get_block_template::Response::TemplateMode(get_block_template) = get_block_template .await .expect("unexpected panic in getblocktemplate RPC task") - .expect("unexpected error in getblocktemplate RPC call"); + .expect("unexpected error in getblocktemplate RPC call") else { + panic!("this getblocktemplate call without parameters should return the `TemplateMode` variant of the response") + }; let coinbase_tx: Transaction = get_block_template .coinbase_txn @@ -287,7 +291,7 @@ fn snapshot_rpc_getblockhash(block_hash: GetBlockHash, settings: &insta::Setting /// Snapshot `getblocktemplate` response, using `cargo insta` and JSON serialization. fn snapshot_rpc_getblocktemplate( variant: &'static str, - block_template: GetBlockTemplate, + block_template: Box, coinbase_tx: Transaction, settings: &insta::Settings, ) { diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index 465917319da..385464822f0 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -996,7 +996,7 @@ async fn rpc_getblocktemplate_mining_address(use_p2pkh: bool) { .await .respond(mempool::Response::FullTransactions(vec![])); - let get_block_template = get_block_template + let get_block_template::Response::TemplateMode(get_block_template) = get_block_template .await .unwrap_or_else(|error| match error.try_into_panic() { Ok(panic_object) => panic::resume_unwind(panic_object), @@ -1004,7 +1004,9 @@ async fn rpc_getblocktemplate_mining_address(use_p2pkh: bool) { panic!("getblocktemplate task was unexpectedly cancelled: {cancelled_error:?}") } }) - .expect("unexpected error in getblocktemplate RPC call"); + .expect("unexpected error in getblocktemplate RPC call") else { + panic!("this getblocktemplate call without parameters should return the `TemplateMode` variant of the response") + }; assert_eq!( get_block_template.capabilities, diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index c5e22522e5f..351cd5cab36 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -815,7 +815,7 @@ impl ReadStateService { let latest_non_finalized_state = self.latest_non_finalized_state(); let Some(best_chain) = latest_non_finalized_state.best_chain() else { - Err(crate::ValidateContextError::NotReadyToBeValidated)? + Err(crate::ValidateContextError::NotReadyToBeValidated)? }; if best_chain.non_finalized_tip_hash() != prepared.block.header.previous_block_hash { From 9e0b8d1a645027f8dd402b0f6ec7f4034011e784 Mon Sep 17 00:00:00 2001 From: arya2 Date: Thu, 15 Dec 2022 01:26:29 -0500 Subject: [PATCH 05/44] makes getblocktemplate-rpcs feature in zebra-consensus select getblocktemplate-rpcs in zebra-state --- zebra-consensus/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-consensus/Cargo.toml b/zebra-consensus/Cargo.toml index 71d66d6d442..b7404bed822 100644 --- a/zebra-consensus/Cargo.toml +++ b/zebra-consensus/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [features] default = [] -getblocktemplate-rpcs = [] +getblocktemplate-rpcs = ["zebra-state/getblocktemplate-rpcs"] proptest-impl = ["proptest", "proptest-derive", "zebra-chain/proptest-impl", "zebra-state/proptest-impl"] [dependencies] From 8420b75013c1a05ea11a9799d1f87bd338c3c27c Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 16 Dec 2022 14:00:57 -0500 Subject: [PATCH 06/44] Adds PR review revisions --- zebra-consensus/Cargo.toml | 5 +- zebra-consensus/src/block.rs | 54 +- zebra-consensus/src/block/check.rs | 26 +- zebra-consensus/src/block/request.rs | 9 +- .../src/methods/get_block_template_rpcs.rs | 480 +++++++++--------- .../get_block_template.rs | 98 ++-- .../types/get_block_template.rs | 14 + .../types/get_block_template/parameters.rs | 28 +- zebra-state/src/error.rs | 5 - zebra-state/src/request.rs | 24 +- zebra-state/src/response.rs | 10 +- zebra-state/src/service.rs | 108 +--- zebra-state/src/service/write.rs | 26 +- 13 files changed, 424 insertions(+), 463 deletions(-) diff --git a/zebra-consensus/Cargo.toml b/zebra-consensus/Cargo.toml index b7404bed822..f346e3752b8 100644 --- a/zebra-consensus/Cargo.toml +++ b/zebra-consensus/Cargo.toml @@ -7,7 +7,10 @@ edition = "2021" [features] default = [] -getblocktemplate-rpcs = ["zebra-state/getblocktemplate-rpcs"] +getblocktemplate-rpcs = [ + "zebra-state/getblocktemplate-rpcs", + "zebra-chain/getblocktemplate-rpcs", +] proptest-impl = ["proptest", "proptest-derive", "zebra-chain/proptest-impl", "zebra-state/proptest-impl"] [dependencies] diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs index dc6d05ed341..38c3534debd 100644 --- a/zebra-consensus/src/block.rs +++ b/zebra-consensus/src/block.rs @@ -70,9 +70,9 @@ pub enum VerifyBlockError { // TODO: make this into a concrete type, and add it to is_duplicate_request() (#2908) Commit(#[source] BoxError), - #[error("unable to validate block after semantic verification")] - // TODO: make this into a concrete type - Validate(#[source] BoxError), + #[error("unable to validate block proposal: failed semantic verification (proof of work is not checked for proposals)")] + // TODO: make this into a concrete type (see #5732) + ValidateProposal(#[source] BoxError), #[error("invalid transaction")] Transaction(#[from] TransactionError), @@ -174,11 +174,15 @@ where Err(BlockError::MaxHeight(height, hash, block::Height::MAX))?; } - // Do the difficulty checks first, to raise the threshold for - // attacks that use any other fields. - check::difficulty_is_valid(&block.header, network, &height, &hash)?; - - if !request.is_proposal() { + // > The block data MUST be validated and checked against the server's usual + // > acceptance rules (excluding the check for a valid proof-of-work). + // + if request.is_proposal() { + check::difficulty_threshold_is_valid(&block.header, network, &height, &hash)?; + } else { + // Do the difficulty checks first, to raise the threshold for + // attacks that use any other fields. + check::difficulty_is_valid(&block.header, network, &height, &hash)?; check::equihash_solution_is_valid(&block.header)?; } @@ -285,8 +289,20 @@ where transaction_hashes, }; - match request { - Request::Commit(_) => match state_service + if request.is_proposal() { + match state_service + .ready() + .await + .map_err(VerifyBlockError::ValidateProposal)? + .call(zs::Request::CheckBlockProposalValidity(prepared_block)) + .await + .map_err(VerifyBlockError::ValidateProposal)? + { + zs::Response::ValidBlockProposal => Ok(hash), + _ => unreachable!("wrong response for CheckBlockProposalValidity"), + } + } else { + match state_service .ready() .await .map_err(VerifyBlockError::Commit)? @@ -299,23 +315,7 @@ where Ok(hash) } _ => unreachable!("wrong response for CommitBlock"), - }, - - #[cfg(feature = "getblocktemplate-rpcs")] - Request::CheckProposal(_) => match state_service - .ready() - .await - .map_err(VerifyBlockError::Validate)? - .call(zs::Request::CheckBlockValidity(prepared_block)) - .await - .map_err(VerifyBlockError::Validate)? - { - zs::Response::Committed(committed_hash) => { - assert_eq!(committed_hash, hash, "state must validate correct hash"); - Ok(hash) - } - _ => unreachable!("wrong response for CheckBlockValidity"), - }, + } } } .instrument(span) diff --git a/zebra-consensus/src/block/check.rs b/zebra-consensus/src/block/check.rs index 094adf47b2c..761ea93352d 100644 --- a/zebra-consensus/src/block/check.rs +++ b/zebra-consensus/src/block/check.rs @@ -58,18 +58,17 @@ pub fn coinbase_is_first(block: &Block) -> Result, Ok(first.clone()) } -/// Returns `Ok(())` if `hash` passes: +/// Returns `Ok(ExpandedDifficulty)` if `difficulty_threshold` of `header` passes: /// - the target difficulty limit for `network` (PoWLimit), and /// - the difficulty filter, -/// based on the fields in `header`. /// -/// If the block is invalid, returns an error containing `height` and `hash`. -pub fn difficulty_is_valid( +/// If the header is invalid, returns an error containing `height` and `hash`. +pub fn difficulty_threshold_is_valid( header: &Header, network: Network, height: &Height, hash: &Hash, -) -> Result<(), BlockError> { +) -> Result { let difficulty_threshold = header .difficulty_threshold .to_expanded() @@ -90,6 +89,23 @@ pub fn difficulty_is_valid( ))?; } + Ok(difficulty_threshold) +} + +/// Returns `Ok(())` if `hash` passes: +/// - the target difficulty limit for `network` (PoWLimit), and +/// - the difficulty filter, +/// based on the fields in `header`. +/// +/// If the block is invalid, returns an error containing `height` and `hash`. +pub fn difficulty_is_valid( + header: &Header, + network: Network, + height: &Height, + hash: &Hash, +) -> Result<(), BlockError> { + let difficulty_threshold = difficulty_threshold_is_valid(header, network, height, hash)?; + // # Consensus // // > The block MUST pass the difficulty filter. diff --git a/zebra-consensus/src/block/request.rs b/zebra-consensus/src/block/request.rs index b89df5f0c0d..9a0dbcc9cf9 100644 --- a/zebra-consensus/src/block/request.rs +++ b/zebra-consensus/src/block/request.rs @@ -7,12 +7,13 @@ use zebra_chain::block::Block; #[derive(Debug, Clone, PartialEq, Eq)] /// A request to the chain or block verifier pub enum Request { - /// Performs semantic validation then calls state with CommitBlock request + /// Performs semantic validation, then asks the state to perform contextual validation and commit the block Commit(Arc), #[cfg(feature = "getblocktemplate-rpcs")] - /// Performs semantic validation but skips checking the solution, - /// then calls the state with CheckBlockValid request + /// Performs semantic validation but skips checking proof of work, + /// then asks the state to perform contextual validation. + /// Does not commit the block to the state. CheckProposal(Arc), } @@ -27,7 +28,7 @@ impl Request { }) } - /// Checks if request is a proposal + /// Returns `true` if the request is a proposal pub fn is_proposal(&self) -> bool { match self { Request::Commit(_) => false, diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index e7b1610aa73..07adcbbc960 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -26,8 +26,9 @@ use crate::methods::{ DEFAULT_SOLUTION_RATE_WINDOW_SIZE, GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, }, get_block_template::{ - check_miner_address, check_synced_to_tip, fetch_mempool_transactions, - fetch_state_tip_and_local_time, generate_coinbase_and_roots, + check_get_block_template_parameters, check_miner_address, check_synced_to_tip, + fetch_mempool_transactions, fetch_state_tip_and_local_time, + generate_coinbase_and_roots, }, types::{ get_block_template::GetBlockTemplate, get_mining_info, hex_data::HexData, @@ -311,10 +312,10 @@ where .boxed() } - // TODO: use HexData to handle block proposal data, and a generic error constructor (#5548) + // TODO: use a generic error constructor (#5548) fn get_block_template( &self, - parameters: Option, + mut parameters: Option, ) -> BoxFuture> { // Clone Config let network = self.network; @@ -325,19 +326,20 @@ where let mut latest_chain_tip = self.latest_chain_tip.clone(); let sync_status = self.sync_status.clone(); let state = self.state.clone(); - let chain_verifier = get_block_template::JsonParameters::is_proposal_mode(¶meters) - .then(|| self.chain_verifier.clone()); + let mut chain_verifier = self.chain_verifier.clone(); // To implement long polling correctly, we split this RPC into multiple phases. async move { - let client_long_poll_id = parameters.as_ref().and_then(|params| params.long_poll_id.clone()); + check_get_block_template_parameters(¶meters)?; - if let Some(HexData(block_proposal_bytes)) = - get_block_template::JsonParameters::block_proposal(parameters)? - { - let mut chain_verifier = - chain_verifier.expect("chain_verifier is Some in proposal mode"); + let client_long_poll_id = parameters + .as_mut() + .and_then(|params| params.long_poll_id.take()); + if let Some(HexData(block_proposal_bytes)) = parameters + .as_mut() + .and_then(get_block_template::JsonParameters::block_proposal_data) + { let block: Block = match block_proposal_bytes.zcash_deserialize_into() { Ok(block_bytes) => block_bytes, Err(_) => return Ok(get_block_template::ProposalRejectReason::Rejected.into()), @@ -358,251 +360,249 @@ where .map(|_| get_block_template::ProposalResponse::Valid) .unwrap_or_else(|_| get_block_template::ProposalRejectReason::Rejected.into()); - Ok(get_block_template::Response::ProposalMode(response)) - } else { - // - One-off checks + return Ok(response.into()); + } - // Check config and parameters. - // These checks always have the same result during long polling. - let miner_address = check_miner_address(miner_address)?; + // - One-off checks - // - Checks and fetches that can change during long polling + // Check config and parameters. + // These checks always have the same result during long polling. + let miner_address = check_miner_address(miner_address)?; + + // - Checks and fetches that can change during long polling + // + // Set up the loop. + let mut max_time_reached = false; + + // The loop returns the server long poll ID, + // which should be different to the client long poll ID. + let (server_long_poll_id, chain_tip_and_local_time, mempool_txs, submit_old) = loop { + // Check if we are synced to the tip. + // The result of this check can change during long polling. // - // Set up the loop. - let mut max_time_reached = false; - - // The loop returns the server long poll ID, - // which should be different to the client long poll ID. - let (server_long_poll_id, chain_tip_and_local_time, mempool_txs, submit_old) = loop { - // Check if we are synced to the tip. - // The result of this check can change during long polling. - // - // Optional TODO: - // - add `async changed()` method to ChainSyncStatus (like `ChainTip`) - check_synced_to_tip(network, latest_chain_tip.clone(), sync_status.clone())?; - - // We're just about to fetch state data, then maybe wait for any changes. - // Mark all the changes before the fetch as seen. - // Changes are also ignored in any clones made after the mark. - latest_chain_tip.mark_best_tip_seen(); - - // Fetch the state data and local time for the block template: - // - if the tip block hash changes, we must return from long polling, - // - if the local clock changes on testnet, we might return from long polling - // - // We always return after 90 minutes on mainnet, even if we have the same response, - // because the max time has been reached. - let chain_tip_and_local_time = - fetch_state_tip_and_local_time(state.clone()).await?; - - // Fetch the mempool data for the block template: - // - if the mempool transactions change, we might return from long polling. - // - // If the chain fork has just changed, miners want to get the new block as fast - // as possible, rather than wait for transactions to re-verify. This increases - // miner profits (and any delays can cause chain forks). So we don't wait between - // the chain tip changing and getting mempool transactions. - // - // Optional TODO: - // - add a `MempoolChange` type with an `async changed()` method (like `ChainTip`) - let mempool_txs = fetch_mempool_transactions(mempool.clone()).await?; - - // - Long poll ID calculation - let server_long_poll_id = LongPollInput::new( - chain_tip_and_local_time.tip_height, - chain_tip_and_local_time.tip_hash, - chain_tip_and_local_time.max_time, - mempool_txs.iter().map(|tx| tx.transaction.id), - ) - .generate_id(); - - // The loop finishes if: - // - the client didn't pass a long poll ID, - // - the server long poll ID is different to the client long poll ID, or - // - the previous loop iteration waited until the max time. - if Some(&server_long_poll_id) != client_long_poll_id.as_ref() || max_time_reached { - let mut submit_old = client_long_poll_id - .as_ref() - .map(|old_long_poll_id| server_long_poll_id.submit_old(old_long_poll_id)); - - // On testnet, the max time changes the block difficulty, so old shares are - // invalid. On mainnet, this means there has been 90 minutes without a new - // block or mempool transaction, which is very unlikely. So the miner should - // probably reset anyway. - if max_time_reached { - submit_old = Some(false); - } + // Optional TODO: + // - add `async changed()` method to ChainSyncStatus (like `ChainTip`) + check_synced_to_tip(network, latest_chain_tip.clone(), sync_status.clone())?; + + // We're just about to fetch state data, then maybe wait for any changes. + // Mark all the changes before the fetch as seen. + // Changes are also ignored in any clones made after the mark. + latest_chain_tip.mark_best_tip_seen(); + + // Fetch the state data and local time for the block template: + // - if the tip block hash changes, we must return from long polling, + // - if the local clock changes on testnet, we might return from long polling + // + // We always return after 90 minutes on mainnet, even if we have the same response, + // because the max time has been reached. + let chain_tip_and_local_time = + fetch_state_tip_and_local_time(state.clone()).await?; - break ( - server_long_poll_id, - chain_tip_and_local_time, - mempool_txs, - submit_old, - ); + // Fetch the mempool data for the block template: + // - if the mempool transactions change, we might return from long polling. + // + // If the chain fork has just changed, miners want to get the new block as fast + // as possible, rather than wait for transactions to re-verify. This increases + // miner profits (and any delays can cause chain forks). So we don't wait between + // the chain tip changing and getting mempool transactions. + // + // Optional TODO: + // - add a `MempoolChange` type with an `async changed()` method (like `ChainTip`) + let mempool_txs = fetch_mempool_transactions(mempool.clone()).await?; + + // - Long poll ID calculation + let server_long_poll_id = LongPollInput::new( + chain_tip_and_local_time.tip_height, + chain_tip_and_local_time.tip_hash, + chain_tip_and_local_time.max_time, + mempool_txs.iter().map(|tx| tx.transaction.id), + ) + .generate_id(); + + // The loop finishes if: + // - the client didn't pass a long poll ID, + // - the server long poll ID is different to the client long poll ID, or + // - the previous loop iteration waited until the max time. + if Some(&server_long_poll_id) != client_long_poll_id.as_ref() || max_time_reached { + let mut submit_old = client_long_poll_id + .as_ref() + .map(|old_long_poll_id| server_long_poll_id.submit_old(old_long_poll_id)); + + // On testnet, the max time changes the block difficulty, so old shares are + // invalid. On mainnet, this means there has been 90 minutes without a new + // block or mempool transaction, which is very unlikely. So the miner should + // probably reset anyway. + if max_time_reached { + submit_old = Some(false); } - // - Polling wait conditions - // - // TODO: when we're happy with this code, split it into a function. - // - // Periodically check the mempool for changes. - // - // Optional TODO: - // Remove this polling wait if we switch to using futures to detect sync status - // and mempool changes. - let wait_for_mempool_request = tokio::time::sleep(Duration::from_secs( - GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, - )); - - // Return immediately if the chain tip has changed. - let wait_for_best_tip_change = latest_chain_tip.best_tip_changed(); - - // Wait for the maximum block time to elapse. This can change the block header - // on testnet. (On mainnet it can happen due to a network disconnection, or a - // rapid drop in hash rate.) - // - // This duration might be slightly lower than the actual maximum, - // if cur_time was clamped to min_time. In that case the wait is very long, - // and it's ok to return early. - // - // It can also be zero if cur_time was clamped to max_time. In that case, - // we want to wait for another change, and ignore this timeout. So we use an - // `OptionFuture::None`. - let duration_until_max_time = chain_tip_and_local_time - .max_time - .saturating_duration_since(chain_tip_and_local_time.cur_time); - let wait_for_max_time: OptionFuture<_> = if duration_until_max_time.seconds() > 0 { - Some(tokio::time::sleep(duration_until_max_time.to_std())) - } else { - None + break ( + server_long_poll_id, + chain_tip_and_local_time, + mempool_txs, + submit_old, + ); + } + + // - Polling wait conditions + // + // TODO: when we're happy with this code, split it into a function. + // + // Periodically check the mempool for changes. + // + // Optional TODO: + // Remove this polling wait if we switch to using futures to detect sync status + // and mempool changes. + let wait_for_mempool_request = tokio::time::sleep(Duration::from_secs( + GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, + )); + + // Return immediately if the chain tip has changed. + let wait_for_best_tip_change = latest_chain_tip.best_tip_changed(); + + // Wait for the maximum block time to elapse. This can change the block header + // on testnet. (On mainnet it can happen due to a network disconnection, or a + // rapid drop in hash rate.) + // + // This duration might be slightly lower than the actual maximum, + // if cur_time was clamped to min_time. In that case the wait is very long, + // and it's ok to return early. + // + // It can also be zero if cur_time was clamped to max_time. In that case, + // we want to wait for another change, and ignore this timeout. So we use an + // `OptionFuture::None`. + let duration_until_max_time = chain_tip_and_local_time + .max_time + .saturating_duration_since(chain_tip_and_local_time.cur_time); + let wait_for_max_time: OptionFuture<_> = if duration_until_max_time.seconds() > 0 { + Some(tokio::time::sleep(duration_until_max_time.to_std())) + } else { + None + } + .into(); + + // Optional TODO: + // `zcashd` generates the next coinbase transaction while waiting for changes. + // When Zebra supports shielded coinbase, we might want to do this in parallel. + // But the coinbase value depends on the selected transactions, so this needs + // further analysis to check if it actually saves us any time. + + // TODO: change logging to debug after testing + tokio::select! { + // Poll the futures in the listed order, for efficiency. + // We put the most frequent conditions first. + biased; + + // This timer elapses every few seconds + _elapsed = wait_for_mempool_request => { + tracing::info!( + max_time = ?chain_tip_and_local_time.max_time, + cur_time = ?chain_tip_and_local_time.cur_time, + ?server_long_poll_id, + ?client_long_poll_id, + GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, + "checking for a new mempool change after waiting a few seconds" + ); } - .into(); - - // Optional TODO: - // `zcashd` generates the next coinbase transaction while waiting for changes. - // When Zebra supports shielded coinbase, we might want to do this in parallel. - // But the coinbase value depends on the selected transactions, so this needs - // further analysis to check if it actually saves us any time. - - // TODO: change logging to debug after testing - tokio::select! { - // Poll the futures in the listed order, for efficiency. - // We put the most frequent conditions first. - biased; - - // This timer elapses every few seconds - _elapsed = wait_for_mempool_request => { - tracing::info!( - max_time = ?chain_tip_and_local_time.max_time, - cur_time = ?chain_tip_and_local_time.cur_time, - ?server_long_poll_id, - ?client_long_poll_id, - GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, - "checking for a new mempool change after waiting a few seconds" - ); - } - // The state changes after around a target block interval (75s) - tip_changed_result = wait_for_best_tip_change => { - match tip_changed_result { - Ok(()) => { - tracing::info!( - max_time = ?chain_tip_and_local_time.max_time, - cur_time = ?chain_tip_and_local_time.cur_time, - ?server_long_poll_id, - ?client_long_poll_id, - "returning from long poll because state has changed" - ); - } - - Err(recv_error) => { - // This log should stay at info when the others go to debug, - // it will help with debugging. - tracing::info!( - ?recv_error, - max_time = ?chain_tip_and_local_time.max_time, - cur_time = ?chain_tip_and_local_time.cur_time, - ?server_long_poll_id, - ?client_long_poll_id, - "returning from long poll due to a state error.\ - Is Zebra shutting down?" - ); - - return Err(Error { - code: ErrorCode::ServerError(0), - message: recv_error.to_string(), - data: None, - }); - } + // The state changes after around a target block interval (75s) + tip_changed_result = wait_for_best_tip_change => { + match tip_changed_result { + Ok(()) => { + tracing::info!( + max_time = ?chain_tip_and_local_time.max_time, + cur_time = ?chain_tip_and_local_time.cur_time, + ?server_long_poll_id, + ?client_long_poll_id, + "returning from long poll because state has changed" + ); } - } - // The max time does not elapse during normal operation on mainnet, - // and it rarely elapses on testnet. - Some(_elapsed) = wait_for_max_time => { - // This log should stay at info when the others go to debug, - // it's very rare. - tracing::info!( - max_time = ?chain_tip_and_local_time.max_time, - cur_time = ?chain_tip_and_local_time.cur_time, - ?server_long_poll_id, - ?client_long_poll_id, - "returning from long poll because max time was reached" - ); - - max_time_reached = true; + Err(recv_error) => { + // This log should stay at info when the others go to debug, + // it will help with debugging. + tracing::info!( + ?recv_error, + max_time = ?chain_tip_and_local_time.max_time, + cur_time = ?chain_tip_and_local_time.cur_time, + ?server_long_poll_id, + ?client_long_poll_id, + "returning from long poll due to a state error.\ + Is Zebra shutting down?" + ); + + return Err(Error { + code: ErrorCode::ServerError(0), + message: recv_error.to_string(), + data: None, + }); + } } } - }; - // - Processing fetched data to create a transaction template - // - // Apart from random weighted transaction selection, - // the template only depends on the previously fetched data. - // This processing never fails. + // The max time does not elapse during normal operation on mainnet, + // and it rarely elapses on testnet. + Some(_elapsed) = wait_for_max_time => { + // This log should stay at info when the others go to debug, + // it's very rare. + tracing::info!( + max_time = ?chain_tip_and_local_time.max_time, + cur_time = ?chain_tip_and_local_time.cur_time, + ?server_long_poll_id, + ?client_long_poll_id, + "returning from long poll because max time was reached" + ); + + max_time_reached = true; + } + } + }; - // Calculate the next block height. - let next_block_height = - (chain_tip_and_local_time.tip_height + 1).expect("tip is far below Height::MAX"); + // - Processing fetched data to create a transaction template + // + // Apart from random weighted transaction selection, + // the template only depends on the previously fetched data. + // This processing never fails. - // Randomly select some mempool transactions. - // - // TODO: sort these transactions to match zcashd's order, to make testing easier. - let mempool_txs = zip317::select_mempool_transactions( - network, - next_block_height, - miner_address, - mempool_txs, - ) - .await; + // Calculate the next block height. + let next_block_height = + (chain_tip_and_local_time.tip_height + 1).expect("tip is far below Height::MAX"); - // - After this point, the template only depends on the previously fetched data. + // Randomly select some mempool transactions. + // + // TODO: sort these transactions to match zcashd's order, to make testing easier. + let mempool_txs = zip317::select_mempool_transactions( + network, + next_block_height, + miner_address, + mempool_txs, + ) + .await; + + // - After this point, the template only depends on the previously fetched data. + + // Generate the coinbase transaction and default roots + // + // TODO: move expensive root, hash, and tree cryptography to a rayon thread? + let (coinbase_txn, default_roots) = generate_coinbase_and_roots( + network, + next_block_height, + miner_address, + &mempool_txs, + chain_tip_and_local_time.history_tree.clone(), + ); + + let response = GetBlockTemplate::new( + next_block_height, + &chain_tip_and_local_time, + server_long_poll_id, + coinbase_txn, + &mempool_txs, + default_roots, + submit_old, + ); - // Generate the coinbase transaction and default roots - // - // TODO: move expensive root, hash, and tree cryptography to a rayon thread? - let (coinbase_txn, default_roots) = generate_coinbase_and_roots( - network, - next_block_height, - miner_address, - &mempool_txs, - chain_tip_and_local_time.history_tree.clone(), - ); - - let response = GetBlockTemplate::new( - next_block_height, - &chain_tip_and_local_time, - server_long_poll_id, - coinbase_txn, - &mempool_txs, - default_roots, - submit_old, - ); - - Ok(get_block_template::Response::TemplateMode(Box::new( - response, - ))) - } + Ok(response.into()) } .boxed() } diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs index 9b6225a0a6c..d1c8437fd61 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs @@ -23,74 +23,56 @@ use zebra_state::GetBlockTemplateChainInfo; use crate::methods::get_block_template_rpcs::{ constants::{MAX_ESTIMATED_DISTANCE_TO_NETWORK_CHAIN_TIP, NOT_SYNCED_ERROR_CODE}, - types::{default_roots::DefaultRoots, get_block_template, transaction::TransactionTemplate}, + types::{default_roots::DefaultRoots, transaction::TransactionTemplate}, }; pub use crate::methods::get_block_template_rpcs::types::get_block_template::*; -use super::types::hex_data::HexData; - // - Parameter checks -impl JsonParameters { - /// Checks that `data` is omitted in `Template` mode or provided in `Proposal` mode, - /// - /// Returns an error if there's a mismatch between the mode and whether `data` is provided. - /// Returns Ok(Some(data)) with the block proposal hexdata if in `Proposal` mode. - /// Returns Ok(None) if in `Template` mode. - pub fn block_proposal(parameters: Option) -> Result> { - let Some(parameters) = parameters else { - return Ok(None) +/// Checks that `data` is omitted in `Template` mode or provided in `Proposal` mode, +/// +/// Returns an error if there's a mismatch between the mode and whether `data` is provided. +pub fn check_get_block_template_parameters(parameters: &Option) -> Result<()> { + let Some(parameters) = parameters else { + return Ok(()) }; - match parameters { - Self { - mode: GetBlockTemplateRequestMode::Template, - data: None, - .. - } => Ok(None), - - Self { - mode: GetBlockTemplateRequestMode::Proposal, - data: data @ Some(_), - .. - } => Ok(data), - - Self { - mode: GetBlockTemplateRequestMode::Proposal, - data: None, - .. - } => Err(Error { - code: ErrorCode::InvalidParams, - message: "\"data\" parameter must be \ + match parameters { + JsonParameters { + mode: GetBlockTemplateRequestMode::Template, + data: None, + .. + } + | JsonParameters { + mode: GetBlockTemplateRequestMode::Proposal, + data: Some(_), + .. + } => Ok(()), + + JsonParameters { + mode: GetBlockTemplateRequestMode::Proposal, + data: None, + .. + } => Err(Error { + code: ErrorCode::InvalidParams, + message: "\"data\" parameter must be \ provided in \"proposal\" mode" - .to_string(), - data: None, - }), - - Self { - mode: GetBlockTemplateRequestMode::Template, - data: Some(_), - .. - } => Err(Error { - code: ErrorCode::InvalidParams, - message: "\"data\" parameter must be \ + .to_string(), + data: None, + }), + + JsonParameters { + mode: GetBlockTemplateRequestMode::Template, + data: Some(_), + .. + } => Err(Error { + code: ErrorCode::InvalidParams, + message: "\"data\" parameter must be \ omitted in \"template\" mode" - .to_string(), - data: None, - }), - } - } - - /// Checks if `mode` parameters is `Proposal`. - pub fn is_proposal_mode(parameters: &Option) -> bool { - matches!( - parameters, - Some(get_block_template::JsonParameters { - mode: GetBlockTemplateRequestMode::Proposal, - .. - }) - ) + .to_string(), + data: None, + }), } } diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs index 08cfe80d03d..0923d49ecb7 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs @@ -244,6 +244,8 @@ pub enum ProposalRejectReason { } /// Response to a `getblocktemplate` RPC request in proposal mode. +/// +/// See #[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(untagged, rename_all = "kebab-case")] pub enum ProposalResponse { @@ -291,3 +293,15 @@ impl From for Response { Self::ProposalMode(ProposalResponse::from(error_response)) } } + +impl From for Response { + fn from(proposal_response: ProposalResponse) -> Self { + Self::ProposalMode(proposal_response) + } +} + +impl From for Response { + fn from(template: GetBlockTemplate) -> Self { + Self::TemplateMode(Box::new(template)) + } +} diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs index d693b2a18ff..eea36a78d35 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs @@ -3,7 +3,6 @@ use crate::methods::get_block_template_rpcs::types::{hex_data::HexData, long_poll::LongPollId}; /// Defines whether the RPC method should generate a block template or attempt to validate a block proposal. -/// `Proposal` mode is currently unsupported and will return an error. #[derive(Clone, Debug, serde::Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum GetBlockTemplateRequestMode { @@ -62,8 +61,6 @@ pub enum GetBlockTemplateCapability { /// All other fields are optional. #[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, Default)] pub struct JsonParameters { - /// Must be set to "template" or omitted, as "proposal" mode is currently unsupported. - /// /// Defines whether the RPC method should generate a block template or attempt to /// validate block data, checking against all of the server's usual acceptance rules /// (excluding the check for a valid proof-of-work). @@ -86,4 +83,29 @@ pub struct JsonParameters { /// In Zebra, the ID represents the chain tip, max time, and mempool contents. #[serde(rename = "longpollid")] pub long_poll_id: Option, + + /// The workid for the block template. + /// + /// currently unused. + #[serde(rename = "workid")] + pub _work_id: Option, +} + +impl JsonParameters { + /// Returns Some(data) with the block proposal hexdata if in `Proposal` mode and `data` is provided. + pub fn block_proposal_data(&mut self) -> Option { + match self { + Self { data: None, .. } + | Self { + mode: GetBlockTemplateRequestMode::Template, + .. + } => None, + + Self { + mode: GetBlockTemplateRequestMode::Proposal, + data: block_proposal_data @ Some(_), + .. + } => block_proposal_data.take(), + } + } } diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index 3ef94445809..edb651a26b3 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -55,11 +55,6 @@ pub enum ValidateContextError { #[non_exhaustive] NotReadyToBeCommitted, - #[cfg(feature = "getblocktemplate-rpcs")] - #[error("block parent not found in non-finalized best chain")] - #[non_exhaustive] - NotReadyToBeValidated, - #[error("block height {candidate_height:?} is lower than the current finalized height {finalized_tip_height:?}")] #[non_exhaustive] OrphanedBlock { diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index 446af0d92ba..0080379c8fa 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -560,13 +560,10 @@ pub enum Request { CheckBestChainTipNullifiersAndAnchors(UnminedTx), #[cfg(feature = "getblocktemplate-rpcs")] - /// Performs contextual validation of the given block. + /// Performs contextual validation of the given block, but does not commit it to the state. /// - /// It is the caller's responsibility to perform semantic validation. - /// - /// Returns [`Response::ValidBlock`] with the hash of the block when successful, or an error if - /// the block fails contextual validation. - CheckBlockValidity(PreparedBlock), + /// See `[ReadRequest::CheckBlockProposalValidity]` for details. + CheckBlockProposalValidity(PreparedBlock), } impl Request { @@ -587,7 +584,7 @@ impl Request { "best_chain_tip_nullifiers_anchors" } #[cfg(feature = "getblocktemplate-rpcs")] - Request::CheckBlockValidity(_) => "check_block_validity", + Request::CheckBlockProposalValidity(_) => "check_block_proposal_validity", } } @@ -803,13 +800,14 @@ pub enum ReadRequest { }, #[cfg(feature = "getblocktemplate-rpcs")] - /// Performs contextual validation of the given block. + /// Performs contextual validation of the given block, but does not commit it to the state. /// /// It is the caller's responsibility to perform semantic validation. + /// (The caller does not need to check proof of work for block proposals.) /// - /// Returns [`Response::ValidBlock`] with the hash of the block when successful, or an error if + /// Returns [`Response::ValidBlockProposal`] with the hash of the block when successful, or an error if /// the block fails contextual validation. - CheckBlockValidity(PreparedBlock), + CheckBlockProposalValidity(PreparedBlock), } impl ReadRequest { @@ -840,7 +838,7 @@ impl ReadRequest { #[cfg(feature = "getblocktemplate-rpcs")] ReadRequest::SolutionRate { .. } => "solution_rate", #[cfg(feature = "getblocktemplate-rpcs")] - ReadRequest::CheckBlockValidity(_) => "check_block_validity", + ReadRequest::CheckBlockProposalValidity(_) => "check_block_proposal_validity", } } @@ -889,7 +887,9 @@ impl TryFrom for ReadRequest { } #[cfg(feature = "getblocktemplate-rpcs")] - Request::CheckBlockValidity(prepared) => Ok(ReadRequest::CheckBlockValidity(prepared)), + Request::CheckBlockProposalValidity(prepared) => { + Ok(ReadRequest::CheckBlockProposalValidity(prepared)) + } Request::AwaitUtxo(_) => Err("ReadService does not track pending UTXOs. \ Manually convert the request to ReadRequest::AnyChainUtxo, \ diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs index 116a252410f..1d529c2b991 100644 --- a/zebra-state/src/response.rs +++ b/zebra-state/src/response.rs @@ -61,8 +61,8 @@ pub enum Response { ValidBestChainTipNullifiersAndAnchors, #[cfg(feature = "getblocktemplate-rpcs")] - /// Response to [`Request::CheckBlockValidity`](crate::Request::CheckBlockValidity) - ValidBlock(block::Hash), + /// Response to [`Request::CheckBlockProposalValidity`](crate::Request::CheckBlockProposalValidity) + ValidBlockProposal, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -143,8 +143,8 @@ pub enum ReadResponse { SolutionRate(Option), #[cfg(feature = "getblocktemplate-rpcs")] - /// Response to [`ReadRequest::CheckBlockValidity`](crate::ReadRequest::CheckBlockValidity) - ValidBlock(block::Hash), + /// Response to [`ReadRequest::CheckBlockProposalValidity`](crate::ReadRequest::CheckBlockProposalValidity) + ValidBlockProposal, } /// A structure with the information needed from the state to build a `getblocktemplate` RPC response. @@ -224,7 +224,7 @@ impl TryFrom for Response { } #[cfg(feature = "getblocktemplate-rpcs")] - ReadResponse::ValidBlock(block_hash) => Ok(Response::ValidBlock(block_hash)), + ReadResponse::ValidBlockProposal => Ok(Response::ValidBlockProposal), #[cfg(feature = "getblocktemplate-rpcs")] ReadResponse::BlockHash(_) => { diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 351cd5cab36..9d1932ac922 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -803,91 +803,6 @@ impl ReadStateService { fn latest_non_finalized_state(&self) -> NonFinalizedState { self.non_finalized_state_receiver.cloned_watch_data() } - - /// Check the contextual validity of a block in the best chain - #[cfg(feature = "getblocktemplate-rpcs")] - #[allow(clippy::unwrap_in_result)] - #[tracing::instrument(level = "debug", skip_all)] - fn check_best_chain_contextual_validity( - &self, - prepared: &PreparedBlock, - ) -> Result<(), crate::ValidateContextError> { - let latest_non_finalized_state = self.latest_non_finalized_state(); - - let Some(best_chain) = latest_non_finalized_state.best_chain() else { - Err(crate::ValidateContextError::NotReadyToBeValidated)? - }; - - if best_chain.non_finalized_tip_hash() != prepared.block.header.previous_block_hash { - Err(crate::ValidateContextError::NotReadyToBeValidated)? - } - - check::initial_contextual_validity( - self.network, - &self.db, - &latest_non_finalized_state, - prepared, - )?; - - // Reads from disk - let sprout_final_treestates = - check::anchors::block_fetch_sprout_final_treestates(&self.db, best_chain, prepared); - - let spent_utxos = check::utxo::transparent_spend( - prepared, - &best_chain.unspent_utxos(), - &best_chain.spent_utxos, - &self.db, - )?; - - check::anchors::block_sapling_orchard_anchors_refer_to_final_treestates( - &self.db, best_chain, prepared, - )?; - - let contextual = crate::request::ContextuallyValidBlock::with_block_and_spent_utxos( - prepared.clone(), - spent_utxos.clone(), - ) - .map_err(|value_balance_error| { - crate::ValidateContextError::CalculateBlockChainValueChange { - value_balance_error, - height: prepared.height, - block_hash: prepared.hash, - transaction_count: prepared.block.transactions.len(), - spent_utxo_count: spent_utxos.len(), - } - })?; - let mut block_commitment_result = None; - let mut sprout_anchor_result = None; - - rayon::in_place_scope_fifo(|scope| { - // Clone function arguments for different threads - let block = Arc::clone(&contextual.block); - - scope.spawn_fifo(|_scope| { - block_commitment_result = - Some(check::anchors::block_sprout_anchors_refer_to_treestates( - sprout_final_treestates, - block, - contextual.transaction_hashes, - contextual.height, - )); - }); - - scope.spawn_fifo(|_scope| { - sprout_anchor_result = Some(check::block_commitment_is_valid_for_chain_history( - contextual.block, - self.network, - &best_chain.history_tree, - )); - }); - }); - - block_commitment_result.expect("scope has finished")?; - sprout_anchor_result.expect("scope has finished")?; - - Ok(()) - } } impl Service for StateService { @@ -1134,7 +1049,7 @@ impl Service for StateService { } #[cfg(feature = "getblocktemplate-rpcs")] - Request::CheckBlockValidity(_) => { + Request::CheckBlockProposalValidity(_) => { // Redirect the request to the concurrent ReadStateService let read_service = self.read_service.clone(); @@ -1790,7 +1705,7 @@ impl Service for ReadStateService { // Used by getmininginfo, getnetworksolps, and getnetworkhashps RPCs. #[cfg(feature = "getblocktemplate-rpcs")] - ReadRequest::CheckBlockValidity(prepared) => { + ReadRequest::CheckBlockProposalValidity(prepared) => { let timer = CodeTimer::start(); let state = self.clone(); @@ -1801,18 +1716,29 @@ impl Service for ReadStateService { let span = Span::current(); tokio::task::spawn_blocking(move || { span.in_scope(move || { - state.check_best_chain_contextual_validity(&prepared)?; + // This clone of the non-finalized state is dropped when this closure returns. + // The non-finalized state that's used in the rest of the state (including finalizing + // blocks into the db) is not mutated here. + write::validate_and_commit_non_finalized( + state.network, + &state.db, + &mut state.latest_non_finalized_state(), + prepared, + )?; + // The work is done in the future. timer.finish( module_path!(), line!(), - "ReadRequest::CheckContextualValidity", + "ReadRequest::CheckBlockProposalValidity", ); - Ok(ReadResponse::ValidBlock(prepared.hash)) + Ok(ReadResponse::ValidBlockProposal) }) }) - .map(|join_result| join_result.expect("panic in ReadRequest::ChainInfo")) + .map(|join_result| { + join_result.expect("panic in ReadRequest::CheckBlockProposalValidity") + }) .boxed() } } diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index a3226081c71..e39ddc05aef 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -6,7 +6,10 @@ use tokio::sync::{ watch, }; -use zebra_chain::block::{self, Height}; +use zebra_chain::{ + block::{self, Height}, + parameters::Network, +}; use crate::{ constants::MAX_BLOCK_REORG_HEIGHT, @@ -27,6 +30,8 @@ use crate::service::{ non_finalized_state::Chain, }; +use super::finalized_state::ZebraDb; + /// The maximum size of the parent error map. /// /// We allow enough space for multiple concurrent chain forks with errors. @@ -36,22 +41,18 @@ const PARENT_ERROR_MAP_LIMIT: usize = MAX_BLOCK_REORG_HEIGHT as usize * 2; /// non-finalized state if it is contextually valid. #[tracing::instrument(level = "debug", skip(prepared), fields(height = ?prepared.height, hash = %prepared.hash))] pub(crate) fn validate_and_commit_non_finalized( - finalized_state: &FinalizedState, + network: Network, + finalized_state: &ZebraDb, non_finalized_state: &mut NonFinalizedState, prepared: PreparedBlock, ) -> Result<(), CommitBlockError> { - check::initial_contextual_validity( - finalized_state.network(), - &finalized_state.db, - non_finalized_state, - &prepared, - )?; + check::initial_contextual_validity(network, finalized_state, non_finalized_state, &prepared)?; let parent_hash = prepared.block.header.previous_block_hash; - if finalized_state.db.finalized_tip_hash() == parent_hash { - non_finalized_state.commit_new_chain(prepared, &finalized_state.db)?; + if finalized_state.finalized_tip_hash() == parent_hash { + non_finalized_state.commit_new_chain(prepared, finalized_state)?; } else { - non_finalized_state.commit_block(prepared, &finalized_state.db)?; + non_finalized_state.commit_block(prepared, finalized_state)?; } Ok(()) @@ -210,7 +211,8 @@ pub fn write_blocks_from_channels( } else { tracing::trace!(?child_hash, "validating queued child"); result = validate_and_commit_non_finalized( - &finalized_state, + finalized_state.network(), + &finalized_state.db, &mut non_finalized_state, queued_child, ) From e3b6d287b0d2b9a51489e16896e0f0232cc04b5e Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 16 Dec 2022 14:46:49 -0500 Subject: [PATCH 07/44] adds info log in CheckBlockProposalValidity --- zebra-state/src/service.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 9d1932ac922..fe69676ad4c 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1716,6 +1716,9 @@ impl Service for ReadStateService { let span = Span::current(); tokio::task::spawn_blocking(move || { span.in_scope(move || { + + tracing::info!("attempting to validate and commit block proposal onto a cloned non-finalized state"); + // This clone of the non-finalized state is dropped when this closure returns. // The non-finalized state that's used in the rest of the state (including finalizing // blocks into the db) is not mutated here. From 95cea02d3ac2b9c17110b69c74ac8af7ae20a948 Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 16 Dec 2022 15:03:25 -0500 Subject: [PATCH 08/44] Reverts replacement of match statement --- zebra-consensus/src/block.rs | 53 ++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs index 38c3534debd..3a505bfd559 100644 --- a/zebra-consensus/src/block.rs +++ b/zebra-consensus/src/block.rs @@ -289,32 +289,37 @@ where transaction_hashes, }; - if request.is_proposal() { - match state_service - .ready() - .await - .map_err(VerifyBlockError::ValidateProposal)? - .call(zs::Request::CheckBlockProposalValidity(prepared_block)) - .await - .map_err(VerifyBlockError::ValidateProposal)? - { - zs::Response::ValidBlockProposal => Ok(hash), - _ => unreachable!("wrong response for CheckBlockProposalValidity"), + match request { + Request::Commit(_) => { + match state_service + .ready() + .await + .map_err(VerifyBlockError::Commit)? + .call(zs::Request::CommitBlock(prepared_block)) + .await + .map_err(VerifyBlockError::Commit)? + { + zs::Response::Committed(committed_hash) => { + assert_eq!(committed_hash, hash, "state must commit correct hash"); + Ok(hash) + } + _ => unreachable!("wrong response for CommitBlock"), + } } - } else { - match state_service - .ready() - .await - .map_err(VerifyBlockError::Commit)? - .call(zs::Request::CommitBlock(prepared_block)) - .await - .map_err(VerifyBlockError::Commit)? - { - zs::Response::Committed(committed_hash) => { - assert_eq!(committed_hash, hash, "state must commit correct hash"); - Ok(hash) + + #[cfg(feature = "getblocktemplate-rpcs")] + Request::CheckProposal(_) => { + match state_service + .ready() + .await + .map_err(VerifyBlockError::ValidateProposal)? + .call(zs::Request::CheckBlockProposalValidity(prepared_block)) + .await + .map_err(VerifyBlockError::ValidateProposal)? + { + zs::Response::ValidBlockProposal => Ok(hash), + _ => unreachable!("wrong response for CheckBlockProposalValidity"), } - _ => unreachable!("wrong response for CommitBlock"), } } } From 31472a86b1d0da1ff6f6a9ea577fb52b04017b81 Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 16 Dec 2022 15:13:41 -0500 Subject: [PATCH 09/44] adds `GetBlockTemplate::capabilities` fn --- .../types/get_block_template.rs | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs index 0923d49ecb7..fac1b1b4046 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs @@ -162,6 +162,14 @@ pub struct GetBlockTemplate { } impl GetBlockTemplate { + /// Returns a `Vec` of capabilities supported by the `getblocktemplate` RPC + pub fn capabilities() -> Vec { + GET_BLOCK_TEMPLATE_CAPABILITIES_FIELD + .iter() + .map(ToString::to_string) + .collect() + } + /// Returns a new [`GetBlockTemplate`] struct, based on the supplied arguments and defaults. /// /// The result of this method only depends on the supplied arguments and constants. @@ -184,10 +192,7 @@ impl GetBlockTemplate { .expect("state always returns a valid difficulty value"); // Convert default values - let capabilities: Vec = GET_BLOCK_TEMPLATE_CAPABILITIES_FIELD - .iter() - .map(ToString::to_string) - .collect(); + let capabilities: Vec = Self::capabilities(); let mutable: Vec = GET_BLOCK_TEMPLATE_MUTABLE_FIELD .iter() .map(ToString::to_string) @@ -275,15 +280,9 @@ pub enum Response { impl From for ProposalResponse { fn from(reject_reason: ProposalRejectReason) -> Self { - // Convert default values - let capabilities: Vec = GET_BLOCK_TEMPLATE_CAPABILITIES_FIELD - .iter() - .map(ToString::to_string) - .collect(); - Self::ErrorResponse { reject_reason, - capabilities, + capabilities: GetBlockTemplate::capabilities(), } } } From ba302f71d06659c2aa9f38b132ca7d8da67cd6db Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 16 Dec 2022 15:32:30 -0500 Subject: [PATCH 10/44] conditions calling checkpoint verifier on !request.is_proposal --- zebra-consensus/src/block.rs | 7 +++++-- zebra-consensus/src/chain.rs | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs index 3a505bfd559..03b62da8ccd 100644 --- a/zebra-consensus/src/block.rs +++ b/zebra-consensus/src/block.rs @@ -24,7 +24,7 @@ use tracing::Instrument; use zebra_chain::{amount::Amount, block, parameters::Network, transparent, work::equihash}; use zebra_state as zs; -use crate::{error::*, transaction as tx, BoxError}; +use crate::{error::*, parameters::SLOW_START_INTERVAL, transaction as tx, BoxError}; pub mod check; pub mod request; @@ -203,7 +203,10 @@ where check::time_is_valid_at(&block.header, now, &height, &hash) .map_err(VerifyBlockError::Time)?; let coinbase_tx = check::coinbase_is_first(&block)?; - check::subsidy_is_valid(&block, network)?; + + if !request.is_proposal() || height > SLOW_START_INTERVAL { + check::subsidy_is_valid(&block, network)?; + } // Now do the slower checks diff --git a/zebra-consensus/src/chain.rs b/zebra-consensus/src/chain.rs index 373a4b8d467..bcd36ee1401 100644 --- a/zebra-consensus/src/chain.rs +++ b/zebra-consensus/src/chain.rs @@ -169,7 +169,7 @@ where let block = request.block(); match block.coinbase_height() { - Some(height) if height <= self.max_checkpoint_height => { + Some(height) if height <= self.max_checkpoint_height && !request.is_proposal() => { self.checkpoint.call(block).map_err(Into::into).boxed() } // This also covers blocks with no height, which the block verifier From 02b6758d9e724627fcf91613ac568ec2a2b5d46a Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 16 Dec 2022 15:52:33 -0500 Subject: [PATCH 11/44] updates references to validate_and_commit_non_finalized --- .../src/service/check/tests/anchors.rs | 36 +++++--- .../src/service/check/tests/nullifier.rs | 88 +++++++++++++------ zebra-state/src/service/check/tests/utxo.rs | 66 ++++++++++---- 3 files changed, 135 insertions(+), 55 deletions(-) diff --git a/zebra-state/src/service/check/tests/anchors.rs b/zebra-state/src/service/check/tests/anchors.rs index f79c1ff896e..7ecf388352d 100644 --- a/zebra-state/src/service/check/tests/anchors.rs +++ b/zebra-state/src/service/check/tests/anchors.rs @@ -81,10 +81,13 @@ fn check_sprout_anchors() { // Validate and commit [`block_1`]. This will add an anchor referencing the // empty note commitment tree to the state. - assert!( - validate_and_commit_non_finalized(&finalized_state, &mut non_finalized_state, block_1) - .is_ok() - ); + assert!(validate_and_commit_non_finalized( + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block_1 + ) + .is_ok()); let check_unmined_tx_anchors_result = unmined_txs.iter().try_for_each(|unmined_tx| { tx_anchors_refer_to_final_treestates( @@ -98,7 +101,12 @@ fn check_sprout_anchors() { // Validate and commit [`block_2`]. This will also check the anchors. assert_eq!( - validate_and_commit_non_finalized(&finalized_state, &mut non_finalized_state, block_2), + validate_and_commit_non_finalized( + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block_2 + ), Ok(()) ); } @@ -288,10 +296,13 @@ fn check_sapling_anchors() { Err(ValidateContextError::UnknownSaplingAnchor { .. }) )); - assert!( - validate_and_commit_non_finalized(&finalized_state, &mut non_finalized_state, block1) - .is_ok() - ); + assert!(validate_and_commit_non_finalized( + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block1 + ) + .is_ok()); let check_unmined_tx_anchors_result = unmined_txs.iter().try_for_each(|unmined_tx| { tx_anchors_refer_to_final_treestates( @@ -304,7 +315,12 @@ fn check_sapling_anchors() { assert!(check_unmined_tx_anchors_result.is_ok()); assert_eq!( - validate_and_commit_non_finalized(&finalized_state, &mut non_finalized_state, block2), + validate_and_commit_non_finalized( + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block2 + ), Ok(()) ); } diff --git a/zebra-state/src/service/check/tests/nullifier.rs b/zebra-state/src/service/check/tests/nullifier.rs index 3b1e43d343d..964ac1a8e29 100644 --- a/zebra-state/src/service/check/tests/nullifier.rs +++ b/zebra-state/src/service/check/tests/nullifier.rs @@ -102,7 +102,8 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, + finalized_state.network(), + &finalized_state.db, &mut non_finalized_state, block1.clone() ); @@ -156,7 +157,8 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, + finalized_state.network(), + &finalized_state.db, &mut non_finalized_state, block1 ); @@ -217,7 +219,8 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, + finalized_state.network(), + &finalized_state.db, &mut non_finalized_state, block1 ); @@ -278,8 +281,11 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block1); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block1 + ); prop_assert_eq!( commit_result, @@ -364,7 +370,8 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, + finalized_state.network(), + &finalized_state.db, &mut non_finalized_state, block1.clone() ); @@ -383,7 +390,8 @@ proptest! { let block2 = Arc::new(block2).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, + finalized_state.network(), + &finalized_state.db, &mut non_finalized_state, block2 ); @@ -459,8 +467,11 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block1.clone()); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block1.clone() + ); prop_assert_eq!(commit_result, Ok(())); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -506,8 +517,11 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block1); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block1 + ); prop_assert_eq!( commit_result, @@ -560,7 +574,8 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, + finalized_state.network(), + &finalized_state.db, &mut non_finalized_state, block1 ); @@ -639,8 +654,11 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block1.clone()); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block1.clone() + ); prop_assert_eq!(commit_result, Ok(())); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -655,8 +673,11 @@ proptest! { let block2 = Arc::new(block2).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block2); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block2 + ); prop_assert_eq!( commit_result, @@ -731,8 +752,11 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block1.clone()); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block1.clone() + ); prop_assert_eq!(commit_result, Ok(())); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -779,8 +803,11 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block1); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block1 + ); prop_assert_eq!( commit_result, @@ -837,8 +864,11 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block1); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block1 + ); prop_assert_eq!( commit_result, @@ -918,8 +948,11 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block1.clone()); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block1.clone() + ); prop_assert_eq!(commit_result, Ok(())); prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -933,8 +966,11 @@ proptest! { let block2 = Arc::new(block2).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block2); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block2 + ); prop_assert_eq!( commit_result, diff --git a/zebra-state/src/service/check/tests/utxo.rs b/zebra-state/src/service/check/tests/utxo.rs index efb70c66504..5dc5a25c8cb 100644 --- a/zebra-state/src/service/check/tests/utxo.rs +++ b/zebra-state/src/service/check/tests/utxo.rs @@ -194,8 +194,11 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block1.clone()); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block1.clone() + ); // the block was committed prop_assert_eq!(commit_result, Ok(())); @@ -279,8 +282,11 @@ proptest! { } else { let block2 = Arc::new(block2).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block2.clone()); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block2.clone() + ); // the block was committed prop_assert_eq!(commit_result, Ok(())); @@ -357,8 +363,11 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block1); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block1 + ); // the block was rejected prop_assert_eq!( @@ -419,8 +428,11 @@ proptest! { let block2 = Arc::new(block2).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block2); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block2 + ); // the block was rejected prop_assert_eq!( @@ -503,8 +515,11 @@ proptest! { let block2 = Arc::new(block2).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block2); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block2 + ); // the block was rejected prop_assert_eq!( @@ -615,8 +630,11 @@ proptest! { } else { let block2 = block2.clone().prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block2.clone()); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block2.clone() + ); // the block was committed prop_assert_eq!(commit_result, Ok(())); @@ -651,8 +669,11 @@ proptest! { let block3 = Arc::new(block3).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block3); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block3 + ); // the block was rejected if use_finalized_state_spend { @@ -725,8 +746,11 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block1); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block1 + ); // the block was rejected prop_assert_eq!( @@ -790,8 +814,11 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, - &mut non_finalized_state, block1); + finalized_state.network(), + &finalized_state.db, + &mut non_finalized_state, + block1 + ); // the block was rejected prop_assert_eq!( @@ -885,7 +912,8 @@ fn new_state_with_mainnet_transparent_data( } else { let block1 = block1.clone().prepare(); let commit_result = validate_and_commit_non_finalized( - &finalized_state, + finalized_state.network(), + &finalized_state.db, &mut non_finalized_state, block1.clone(), ); From 2faa82317e4d44858b30bccc1c2abed58380df7f Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 16 Dec 2022 17:03:02 -0500 Subject: [PATCH 12/44] adds snapshot test, updates test vectors --- .../tests/snapshot/get_block_template_rpcs.rs | 96 ++++++++++++++++--- ..._template_invalid-proposal@mainnet_10.snap | 8 ++ ..._template_invalid-proposal@testnet_10.snap | 8 ++ ...et_block_template_proposal@mainnet_10.snap | 5 + ...et_block_template_proposal@testnet_10.snap | 5 + zebra-rpc/src/methods/tests/vectors.rs | 4 +- 6 files changed, 109 insertions(+), 17 deletions(-) create mode 100644 zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@mainnet_10.snap create mode 100644 zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@testnet_10.snap create mode 100644 zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_proposal@mainnet_10.snap create mode 100644 zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_proposal@testnet_10.snap diff --git a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs index f1e6b4b49ee..5a6108bac18 100644 --- a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs @@ -23,13 +23,16 @@ use zebra_node_services::mempool; use zebra_state::{GetBlockTemplateChainInfo, ReadRequest, ReadResponse}; -use zebra_test::mock_service::{MockService, PanicAssertion}; +use zebra_test::{ + mock_service::{MockService, PanicAssertion}, + vectors::BLOCK_MAINNET_1_BYTES, +}; use crate::methods::{ get_block_template_rpcs::{ self, types::{ - get_block_template::{self, GetBlockTemplate}, + get_block_template::{self, GetBlockTemplateRequestMode}, get_mining_info, hex_data::HexData, long_poll::{LongPollId, LONG_POLL_ID_LENGTH}, @@ -159,12 +162,12 @@ pub async fn test_responses( // create a new rpc instance with new state and mock let get_block_template_rpc = GetBlockTemplateRpcImpl::new( network, - mining_config, + mining_config.clone(), Buffer::new(mempool.clone(), 1), new_read_state.clone(), - mock_chain_tip, + mock_chain_tip.clone(), chain_verifier, - mock_sync_status, + mock_sync_status.clone(), ); // Basic variant (default mode and no extra features) @@ -209,7 +212,12 @@ pub async fn test_responses( .zcash_deserialize_into() .expect("coinbase bytes are valid"); - snapshot_rpc_getblocktemplate("basic", get_block_template, coinbase_tx, &settings); + snapshot_rpc_getblocktemplate( + "basic", + (*get_block_template).into(), + Some(coinbase_tx), + &settings, + ); // long polling feature with submit old field @@ -266,7 +274,62 @@ pub async fn test_responses( .zcash_deserialize_into() .expect("coinbase bytes are valid"); - snapshot_rpc_getblocktemplate("long_poll", get_block_template, coinbase_tx, &settings); + snapshot_rpc_getblocktemplate( + "long_poll", + (*get_block_template).into(), + Some(coinbase_tx), + &settings, + ); + + // `getblocktemplate` proposal mode variant + + let get_block_template = tokio::spawn(get_block_template_rpc.get_block_template(Some( + get_block_template::JsonParameters { + mode: GetBlockTemplateRequestMode::Proposal, + data: Some(HexData("".into())), + ..Default::default() + }, + ))); + + let get_block_template = get_block_template + .await + .expect("unexpected panic in getblocktemplate RPC task") + .expect("unexpected error in getblocktemplate RPC call"); + + snapshot_rpc_getblocktemplate("invalid-proposal", get_block_template, None, &settings); + + let mut mock_chain_verifier = MockService::build().for_unit_tests(); + let get_block_template_rpc = GetBlockTemplateRpcImpl::new( + network, + mining_config, + Buffer::new(mempool.clone(), 1), + new_read_state.clone(), + mock_chain_tip, + mock_chain_verifier.clone(), + mock_sync_status, + ); + + let get_block_template = tokio::spawn(get_block_template_rpc.get_block_template(Some( + get_block_template::JsonParameters { + mode: GetBlockTemplateRequestMode::Proposal, + data: Some(HexData(BLOCK_MAINNET_1_BYTES.to_vec())), + ..Default::default() + }, + ))); + + tokio::spawn(async move { + mock_chain_verifier + .expect_request_that(|req| matches!(req, zebra_consensus::Request::CheckProposal(_))) + .await + .respond(Hash::from([0; 32])); + }); + + let get_block_template = get_block_template + .await + .expect("unexpected panic in getblocktemplate RPC task") + .expect("unexpected error in getblocktemplate RPC call"); + + snapshot_rpc_getblocktemplate("proposal", get_block_template, None, &settings); // `submitblock` @@ -291,19 +354,22 @@ fn snapshot_rpc_getblockhash(block_hash: GetBlockHash, settings: &insta::Setting /// Snapshot `getblocktemplate` response, using `cargo insta` and JSON serialization. fn snapshot_rpc_getblocktemplate( variant: &'static str, - block_template: Box, - coinbase_tx: Transaction, + block_template: get_block_template::Response, + coinbase_tx: Option, settings: &insta::Settings, ) { settings.bind(|| { insta::assert_json_snapshot!(format!("get_block_template_{variant}"), block_template) }); - settings.bind(|| { - insta::assert_ron_snapshot!( - format!("get_block_template_{variant}.coinbase_tx"), - coinbase_tx - ) - }); + + if let Some(coinbase_tx) = coinbase_tx { + settings.bind(|| { + insta::assert_ron_snapshot!( + format!("get_block_template_{variant}.coinbase_tx"), + coinbase_tx + ) + }); + }; } /// Snapshot `submitblock` response, using `cargo insta` and JSON serialization. diff --git a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@mainnet_10.snap b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@mainnet_10.snap new file mode 100644 index 00000000000..4dfe2d32f11 --- /dev/null +++ b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@mainnet_10.snap @@ -0,0 +1,8 @@ +--- +source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs +expression: block_template +--- +{ + "reject_reason": "rejected", + "capabilities": [] +} diff --git a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@testnet_10.snap b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@testnet_10.snap new file mode 100644 index 00000000000..4dfe2d32f11 --- /dev/null +++ b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@testnet_10.snap @@ -0,0 +1,8 @@ +--- +source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs +expression: block_template +--- +{ + "reject_reason": "rejected", + "capabilities": [] +} diff --git a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_proposal@mainnet_10.snap b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_proposal@mainnet_10.snap new file mode 100644 index 00000000000..600686f4fe5 --- /dev/null +++ b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_proposal@mainnet_10.snap @@ -0,0 +1,5 @@ +--- +source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs +expression: block_template +--- +null diff --git a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_proposal@testnet_10.snap b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_proposal@testnet_10.snap new file mode 100644 index 00000000000..600686f4fe5 --- /dev/null +++ b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_proposal@testnet_10.snap @@ -0,0 +1,5 @@ +--- +source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs +expression: block_template +--- +null diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index 385464822f0..e45be5a2d1b 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -1100,7 +1100,7 @@ async fn rpc_getblocktemplate_mining_address(use_p2pkh: bool) { ..Default::default() })) .await - .expect_err("needs an error when using unsupported mode"); + .expect_err("needs an error when called in proposal mode without data"); assert_eq!(get_block_template_sync_error.code, ErrorCode::InvalidParams); @@ -1110,7 +1110,7 @@ async fn rpc_getblocktemplate_mining_address(use_p2pkh: bool) { ..Default::default() })) .await - .expect_err("needs an error when passing in block data"); + .expect_err("needs an error when passing in block data in template mode"); assert_eq!(get_block_template_sync_error.code, ErrorCode::InvalidParams); From 32e6ac98a920c89a982cf57e78a5962441a47cc2 Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 16 Dec 2022 17:17:09 -0500 Subject: [PATCH 13/44] adds `should_count_metrics` to NonFinalizedState --- zebra-state/src/service.rs | 5 +++-- zebra-state/src/service/non_finalized_state.rs | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index fe69676ad4c..cab97b8a31a 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1718,14 +1718,15 @@ impl Service for ReadStateService { span.in_scope(move || { tracing::info!("attempting to validate and commit block proposal onto a cloned non-finalized state"); - + let mut latest_non_finalized_state = state.latest_non_finalized_state(); + latest_non_finalized_state.should_count_metrics = false; // This clone of the non-finalized state is dropped when this closure returns. // The non-finalized state that's used in the rest of the state (including finalizing // blocks into the db) is not mutated here. write::validate_and_commit_non_finalized( state.network, &state.db, - &mut state.latest_non_finalized_state(), + &mut latest_non_finalized_state, prepared, )?; diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs index 74fb3d7ff25..2d0aa903709 100644 --- a/zebra-state/src/service/non_finalized_state.rs +++ b/zebra-state/src/service/non_finalized_state.rs @@ -47,6 +47,12 @@ pub struct NonFinalizedState { // // Note: this field is currently unused, but it's useful for debugging. pub network: Network, + + #[cfg(feature = "getblocktemplate-rpcs")] + /// Configures the non-finalized state to count metrics. + /// + /// Used for testing block proposals on a cloned non-finalized state. + pub should_count_metrics: bool, } impl NonFinalizedState { @@ -55,6 +61,8 @@ impl NonFinalizedState { NonFinalizedState { chain_set: Default::default(), network, + #[cfg(feature = "getblocktemplate-rpcs")] + should_count_metrics: true, } } @@ -494,6 +502,11 @@ impl NonFinalizedState { /// Update the metrics after `block` is committed fn update_metrics_for_committed_block(&self, height: block::Height, hash: block::Hash) { + #[cfg(feature = "getblocktemplate-rpcs")] + if !self.should_count_metrics { + return; + } + metrics::counter!("state.memory.committed.block.count", 1); metrics::gauge!("state.memory.committed.block.height", height.0 as f64); @@ -517,6 +530,11 @@ impl NonFinalizedState { /// Update the metrics after `self.chain_set` is modified fn update_metrics_for_chains(&self) { + #[cfg(feature = "getblocktemplate-rpcs")] + if !self.should_count_metrics { + return; + } + metrics::gauge!("state.memory.chain.count", self.chain_set.len() as f64); metrics::gauge!( "state.memory.best.chain.length", From a2ae3fbd8f27ba6a55224c9b742b20ef35973bc4 Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 16 Dec 2022 17:46:54 -0500 Subject: [PATCH 14/44] Returns an error from chain verifier for block proposal requests below checkpoint height adds feature flags --- zebra-consensus/src/block.rs | 8 +++----- zebra-consensus/src/chain.rs | 12 +++++++++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs index 03b62da8ccd..29dbca4640e 100644 --- a/zebra-consensus/src/block.rs +++ b/zebra-consensus/src/block.rs @@ -24,7 +24,7 @@ use tracing::Instrument; use zebra_chain::{amount::Amount, block, parameters::Network, transparent, work::equihash}; use zebra_state as zs; -use crate::{error::*, parameters::SLOW_START_INTERVAL, transaction as tx, BoxError}; +use crate::{error::*, transaction as tx, BoxError}; pub mod check; pub mod request; @@ -70,6 +70,7 @@ pub enum VerifyBlockError { // TODO: make this into a concrete type, and add it to is_duplicate_request() (#2908) Commit(#[source] BoxError), + #[cfg(feature = "getblocktemplate-rpcs")] #[error("unable to validate block proposal: failed semantic verification (proof of work is not checked for proposals)")] // TODO: make this into a concrete type (see #5732) ValidateProposal(#[source] BoxError), @@ -203,10 +204,7 @@ where check::time_is_valid_at(&block.header, now, &height, &hash) .map_err(VerifyBlockError::Time)?; let coinbase_tx = check::coinbase_is_first(&block)?; - - if !request.is_proposal() || height > SLOW_START_INTERVAL { - check::subsidy_is_valid(&block, network)?; - } + check::subsidy_is_valid(&block, network)?; // Now do the slower checks diff --git a/zebra-consensus/src/chain.rs b/zebra-consensus/src/chain.rs index bcd36ee1401..c39f991493e 100644 --- a/zebra-consensus/src/chain.rs +++ b/zebra-consensus/src/chain.rs @@ -169,7 +169,17 @@ where let block = request.block(); match block.coinbase_height() { - Some(height) if height <= self.max_checkpoint_height && !request.is_proposal() => { + #[cfg(feature = "getblocktemplate-rpcs")] + Some(height) if height <= self.max_checkpoint_height && request.is_proposal() => { + async { + Err(VerifyBlockError::ValidateProposal( + "block proposals must be above checkpoint height".into(), + ))? + } + .boxed() + } + + Some(height) if height <= self.max_checkpoint_height => { self.checkpoint.call(block).map_err(Into::into).boxed() } // This also covers blocks with no height, which the block verifier From 40d002ab6bfe1cc9b5c651654f47caa75068115b Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 16 Dec 2022 19:19:33 -0500 Subject: [PATCH 15/44] adds "proposal" to GET_BLOCK_TEMPLATE_CAPABILITIES_FIELD --- zebra-rpc/src/methods/get_block_template_rpcs/constants.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/constants.rs b/zebra-rpc/src/methods/get_block_template_rpcs/constants.rs index 1fcc28e5743..305ab744156 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/constants.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/constants.rs @@ -27,7 +27,7 @@ pub const GET_BLOCK_TEMPLATE_MUTABLE_FIELD: &[&str] = &[ /// A hardcoded list of Zebra's getblocktemplate RPC capabilities. /// Currently empty. -pub const GET_BLOCK_TEMPLATE_CAPABILITIES_FIELD: &[&str] = &[]; +pub const GET_BLOCK_TEMPLATE_CAPABILITIES_FIELD: &[&str] = &["proposal"]; /// The max estimated distance to the chain tip for the getblocktemplate method. /// From da1b3085b869afd8e2c1e7cd0a53382640849b7a Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 19 Dec 2022 14:47:00 -0500 Subject: [PATCH 16/44] adds back block::Request to zebra-consensus lib --- zebra-consensus/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-consensus/src/lib.rs b/zebra-consensus/src/lib.rs index 669e14dd83b..f07511d2a26 100644 --- a/zebra-consensus/src/lib.rs +++ b/zebra-consensus/src/lib.rs @@ -51,7 +51,7 @@ pub use block::{ funding_streams::{funding_stream_address, funding_stream_values, new_coinbase_script}, general::miner_subsidy, }, - VerifyBlockError, MAX_BLOCK_SIGOPS, + Request, VerifyBlockError, MAX_BLOCK_SIGOPS, }; pub use chain::VerifyChainError; pub use checkpoint::{ From cb26b8ae27f8a929bf0101561be859edfa3f2699 Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 19 Dec 2022 17:33:55 -0500 Subject: [PATCH 17/44] updates snapshots --- .../snapshots/get_block_template_basic@mainnet_10.snap | 4 +++- .../snapshots/get_block_template_basic@testnet_10.snap | 4 +++- .../get_block_template_invalid-proposal@mainnet_10.snap | 4 +++- .../get_block_template_invalid-proposal@testnet_10.snap | 4 +++- .../snapshots/get_block_template_long_poll@mainnet_10.snap | 4 +++- .../snapshots/get_block_template_long_poll@testnet_10.snap | 4 +++- 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_basic@mainnet_10.snap b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_basic@mainnet_10.snap index d0853e4a2ea..9506416935e 100644 --- a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_basic@mainnet_10.snap +++ b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_basic@mainnet_10.snap @@ -3,7 +3,9 @@ source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs expression: block_template --- { - "capabilities": [], + "capabilities": [ + "proposal" + ], "version": 4, "previousblockhash": "0000000000d723156d9b65ffcf4984da7a19675ed7e2f06d9e5d5188af087bf8", "blockcommitmentshash": "02990723c6b62a724651322d141b4a72a4ffd66518167d809badbd5117d5518a", diff --git a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_basic@testnet_10.snap b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_basic@testnet_10.snap index 0b6e9e0bd54..8a5dd4bc8f5 100644 --- a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_basic@testnet_10.snap +++ b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_basic@testnet_10.snap @@ -3,7 +3,9 @@ source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs expression: block_template --- { - "capabilities": [], + "capabilities": [ + "proposal" + ], "version": 4, "previousblockhash": "0000000000d723156d9b65ffcf4984da7a19675ed7e2f06d9e5d5188af087bf8", "blockcommitmentshash": "3b25791957f9383b6ce851d728a78309664d5d7a82ca87b6a9125a2f2c529792", diff --git a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@mainnet_10.snap b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@mainnet_10.snap index 4dfe2d32f11..3a63ad3879a 100644 --- a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@mainnet_10.snap +++ b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@mainnet_10.snap @@ -4,5 +4,7 @@ expression: block_template --- { "reject_reason": "rejected", - "capabilities": [] + "capabilities": [ + "proposal" + ] } diff --git a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@testnet_10.snap b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@testnet_10.snap index 4dfe2d32f11..3a63ad3879a 100644 --- a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@testnet_10.snap +++ b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_invalid-proposal@testnet_10.snap @@ -4,5 +4,7 @@ expression: block_template --- { "reject_reason": "rejected", - "capabilities": [] + "capabilities": [ + "proposal" + ] } diff --git a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_long_poll@mainnet_10.snap b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_long_poll@mainnet_10.snap index 686028d69c8..043c937cd8b 100644 --- a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_long_poll@mainnet_10.snap +++ b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_long_poll@mainnet_10.snap @@ -3,7 +3,9 @@ source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs expression: block_template --- { - "capabilities": [], + "capabilities": [ + "proposal" + ], "version": 4, "previousblockhash": "0000000000d723156d9b65ffcf4984da7a19675ed7e2f06d9e5d5188af087bf8", "blockcommitmentshash": "02990723c6b62a724651322d141b4a72a4ffd66518167d809badbd5117d5518a", diff --git a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_long_poll@testnet_10.snap b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_long_poll@testnet_10.snap index ba44908e62e..e258d55e638 100644 --- a/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_long_poll@testnet_10.snap +++ b/zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template_long_poll@testnet_10.snap @@ -3,7 +3,9 @@ source: zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs expression: block_template --- { - "capabilities": [], + "capabilities": [ + "proposal" + ], "version": 4, "previousblockhash": "0000000000d723156d9b65ffcf4984da7a19675ed7e2f06d9e5d5188af087bf8", "blockcommitmentshash": "3b25791957f9383b6ce851d728a78309664d5d7a82ca87b6a9125a2f2c529792", From 78e95d9d517db8728592c1719d721816e6abefaa Mon Sep 17 00:00:00 2001 From: arya2 Date: Thu, 22 Dec 2022 13:46:48 -0500 Subject: [PATCH 18/44] Removes unnecessary network arg --- zebra-state/src/service.rs | 1 - zebra-state/src/service/check.rs | 3 +-- zebra-state/src/service/check/tests/anchors.rs | 16 ++-------------- zebra-state/src/service/check/tests/nullifier.rs | 16 ---------------- zebra-state/src/service/check/tests/utxo.rs | 10 ---------- zebra-state/src/service/write.rs | 9 ++------- 6 files changed, 5 insertions(+), 50 deletions(-) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index cab97b8a31a..c751561a225 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1724,7 +1724,6 @@ impl Service for ReadStateService { // The non-finalized state that's used in the rest of the state (including finalizing // blocks into the db) is not mutated here. write::validate_and_commit_non_finalized( - state.network, &state.db, &mut latest_non_finalized_state, prepared, diff --git a/zebra-state/src/service/check.rs b/zebra-state/src/service/check.rs index 483a8eb534b..603eb4f3e8d 100644 --- a/zebra-state/src/service/check.rs +++ b/zebra-state/src/service/check.rs @@ -365,7 +365,6 @@ where /// /// Additional contextual validity checks are performed by the non-finalized [`Chain`]. pub(crate) fn initial_contextual_validity( - network: Network, finalized_state: &ZebraDb, non_finalized_state: &NonFinalizedState, prepared: &PreparedBlock, @@ -379,7 +378,7 @@ pub(crate) fn initial_contextual_validity( // Security: check proof of work before any other checks check::block_is_valid_for_recent_chain( prepared, - network, + non_finalized_state.network, finalized_state.finalized_tip_height(), relevant_chain, )?; diff --git a/zebra-state/src/service/check/tests/anchors.rs b/zebra-state/src/service/check/tests/anchors.rs index 7ecf388352d..11564201e12 100644 --- a/zebra-state/src/service/check/tests/anchors.rs +++ b/zebra-state/src/service/check/tests/anchors.rs @@ -82,7 +82,6 @@ fn check_sprout_anchors() { // Validate and commit [`block_1`]. This will add an anchor referencing the // empty note commitment tree to the state. assert!(validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block_1 @@ -101,12 +100,7 @@ fn check_sprout_anchors() { // Validate and commit [`block_2`]. This will also check the anchors. assert_eq!( - validate_and_commit_non_finalized( - finalized_state.network(), - &finalized_state.db, - &mut non_finalized_state, - block_2 - ), + validate_and_commit_non_finalized(&finalized_state.db, &mut non_finalized_state, block_2), Ok(()) ); } @@ -297,7 +291,6 @@ fn check_sapling_anchors() { )); assert!(validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1 @@ -315,12 +308,7 @@ fn check_sapling_anchors() { assert!(check_unmined_tx_anchors_result.is_ok()); assert_eq!( - validate_and_commit_non_finalized( - finalized_state.network(), - &finalized_state.db, - &mut non_finalized_state, - block2 - ), + validate_and_commit_non_finalized(&finalized_state.db, &mut non_finalized_state, block2), Ok(()) ); } diff --git a/zebra-state/src/service/check/tests/nullifier.rs b/zebra-state/src/service/check/tests/nullifier.rs index 964ac1a8e29..904426d4256 100644 --- a/zebra-state/src/service/check/tests/nullifier.rs +++ b/zebra-state/src/service/check/tests/nullifier.rs @@ -102,7 +102,6 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1.clone() @@ -157,7 +156,6 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1 @@ -219,7 +217,6 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1 @@ -281,7 +278,6 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1 @@ -370,7 +366,6 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1.clone() @@ -390,7 +385,6 @@ proptest! { let block2 = Arc::new(block2).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block2 @@ -467,7 +461,6 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1.clone() @@ -517,7 +510,6 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1 @@ -574,7 +566,6 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1 @@ -654,7 +645,6 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1.clone() @@ -673,7 +663,6 @@ proptest! { let block2 = Arc::new(block2).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block2 @@ -752,7 +741,6 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1.clone() @@ -803,7 +791,6 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1 @@ -864,7 +851,6 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1 @@ -948,7 +934,6 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1.clone() @@ -966,7 +951,6 @@ proptest! { let block2 = Arc::new(block2).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block2 diff --git a/zebra-state/src/service/check/tests/utxo.rs b/zebra-state/src/service/check/tests/utxo.rs index 5dc5a25c8cb..6d14b8a13dd 100644 --- a/zebra-state/src/service/check/tests/utxo.rs +++ b/zebra-state/src/service/check/tests/utxo.rs @@ -194,7 +194,6 @@ proptest! { } else { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1.clone() @@ -282,7 +281,6 @@ proptest! { } else { let block2 = Arc::new(block2).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block2.clone() @@ -363,7 +361,6 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1 @@ -428,7 +425,6 @@ proptest! { let block2 = Arc::new(block2).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block2 @@ -515,7 +511,6 @@ proptest! { let block2 = Arc::new(block2).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block2 @@ -630,7 +625,6 @@ proptest! { } else { let block2 = block2.clone().prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block2.clone() @@ -669,7 +663,6 @@ proptest! { let block3 = Arc::new(block3).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block3 @@ -746,7 +739,6 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1 @@ -814,7 +806,6 @@ proptest! { let block1 = Arc::new(block1).prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1 @@ -912,7 +903,6 @@ fn new_state_with_mainnet_transparent_data( } else { let block1 = block1.clone().prepare(); let commit_result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, block1.clone(), diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index e39ddc05aef..7b91c09227c 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -6,10 +6,7 @@ use tokio::sync::{ watch, }; -use zebra_chain::{ - block::{self, Height}, - parameters::Network, -}; +use zebra_chain::block::{self, Height}; use crate::{ constants::MAX_BLOCK_REORG_HEIGHT, @@ -41,12 +38,11 @@ const PARENT_ERROR_MAP_LIMIT: usize = MAX_BLOCK_REORG_HEIGHT as usize * 2; /// non-finalized state if it is contextually valid. #[tracing::instrument(level = "debug", skip(prepared), fields(height = ?prepared.height, hash = %prepared.hash))] pub(crate) fn validate_and_commit_non_finalized( - network: Network, finalized_state: &ZebraDb, non_finalized_state: &mut NonFinalizedState, prepared: PreparedBlock, ) -> Result<(), CommitBlockError> { - check::initial_contextual_validity(network, finalized_state, non_finalized_state, &prepared)?; + check::initial_contextual_validity(finalized_state, non_finalized_state, &prepared)?; let parent_hash = prepared.block.header.previous_block_hash; if finalized_state.finalized_tip_hash() == parent_hash { @@ -211,7 +207,6 @@ pub fn write_blocks_from_channels( } else { tracing::trace!(?child_hash, "validating queued child"); result = validate_and_commit_non_finalized( - finalized_state.network(), &finalized_state.db, &mut non_finalized_state, queued_child, From 0c55a6702d6b2c92b9afa066e31a54be5fdd5518 Mon Sep 17 00:00:00 2001 From: arya2 Date: Thu, 22 Dec 2022 13:51:59 -0500 Subject: [PATCH 19/44] skips req in tracing intstrument for read state --- zebra-state/src/service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index c751561a225..63dcb4fdeef 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1100,7 +1100,7 @@ impl Service for ReadStateService { Poll::Ready(Ok(())) } - #[instrument(name = "read_state", skip(self))] + #[instrument(name = "read_state", skip(self, req))] fn call(&mut self, req: ReadRequest) -> Self::Future { req.count_metric(); From 616d2809736e5573c23e919c405b4875d02aee3a Mon Sep 17 00:00:00 2001 From: arya2 Date: Thu, 22 Dec 2022 16:38:49 -0500 Subject: [PATCH 20/44] Moves out block proposal validation to its own fn --- .../src/methods/get_block_template_rpcs.rs | 39 +++++------------- .../get_block_template.rs | 40 ++++++++++++++++++- 2 files changed, 48 insertions(+), 31 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 8d55057f4d5..fc290dd49f6 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -26,9 +26,8 @@ use crate::methods::{ DEFAULT_SOLUTION_RATE_WINDOW_SIZE, GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL, }, get_block_template::{ - check_get_block_template_parameters, check_miner_address, check_synced_to_tip, - fetch_mempool_transactions, fetch_state_tip_and_local_time, - generate_coinbase_and_roots, + check_miner_address, check_synced_to_tip, fetch_mempool_transactions, + fetch_state_tip_and_local_time, generate_coinbase_and_roots, validate_block_proposal, }, types::{ get_block_template::GetBlockTemplate, get_mining_info, hex_data::HexData, @@ -331,43 +330,23 @@ where let mut latest_chain_tip = self.latest_chain_tip.clone(); let sync_status = self.sync_status.clone(); let state = self.state.clone(); - let mut chain_verifier = self.chain_verifier.clone(); + let chain_verifier = self.chain_verifier.clone(); // To implement long polling correctly, we split this RPC into multiple phases. async move { - check_get_block_template_parameters(¶meters)?; - - let client_long_poll_id = parameters - .as_mut() - .and_then(|params| params.long_poll_id.take()); + get_block_template::check_parameters(¶meters)?; if let Some(HexData(block_proposal_bytes)) = parameters .as_mut() .and_then(get_block_template::JsonParameters::block_proposal_data) { - let block: Block = match block_proposal_bytes.zcash_deserialize_into() { - Ok(block_bytes) => block_bytes, - Err(_) => return Ok(get_block_template::ProposalRejectReason::Rejected.into()), - }; - - let chain_verifier_response = chain_verifier - .ready() - .await - .map_err(|error| Error { - code: ErrorCode::ServerError(0), - message: error.to_string(), - data: None, - })? - .call(zebra_consensus::Request::CheckProposal(Arc::new(block))) - .await; - - let response = chain_verifier_response - .map(|_| get_block_template::ProposalResponse::Valid) - .unwrap_or_else(|_| get_block_template::ProposalRejectReason::Rejected.into()); - - return Ok(response.into()); + return validate_block_proposal(chain_verifier, block_proposal_bytes).await; } + let client_long_poll_id = parameters + .as_mut() + .and_then(|params| params.long_poll_id.take()); + // - One-off checks // Check config and parameters. diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs index 8764c380497..6182df60b6f 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs @@ -8,12 +8,14 @@ use tower::{Service, ServiceExt}; use zebra_chain::{ amount::{self, Amount, NegativeOrZero, NonNegative}, block::{ + self, merkle::{self, AuthDataRoot}, ChainHistoryBlockTxAuthCommitmentHash, Height, }, chain_sync_status::ChainSyncStatus, chain_tip::ChainTip, parameters::Network, + serialization::ZcashDeserializeInto, transaction::{Transaction, UnminedTx, VerifiedUnminedTx}, transparent, }; @@ -35,7 +37,7 @@ pub use crate::methods::get_block_template_rpcs::types::get_block_template::*; /// Checks that `data` is omitted in `Template` mode or provided in `Proposal` mode, /// /// Returns an error if there's a mismatch between the mode and whether `data` is provided. -pub fn check_get_block_template_parameters(parameters: &Option) -> Result<()> { +pub fn check_parameters(parameters: &Option) -> Result<()> { let Some(parameters) = parameters else { return Ok(()) }; @@ -91,6 +93,42 @@ pub fn check_miner_address( }) } +/// Attempts to validate block proposal against all of the server's +/// usual acceptance rules (except proof-of-work). +/// +/// Returns a `getblocktemplate` [`Response`]. +pub async fn validate_block_proposal( + mut chain_verifier: ChainVerifier, + block_proposal_bytes: Vec, +) -> Result +where + ChainVerifier: Service + + Clone + + Send + + Sync + + 'static, +{ + let Ok(block) = block_proposal_bytes.zcash_deserialize_into() else { + return Ok(ProposalRejectReason::Rejected.into()) + }; + + let chain_verifier_response = chain_verifier + .ready() + .await + .map_err(|error| Error { + code: ErrorCode::ServerError(0), + message: error.to_string(), + data: None, + })? + .call(zebra_consensus::Request::CheckProposal(Arc::new(block))) + .await; + + Ok(chain_verifier_response + .map(|_| ProposalResponse::Valid) + .unwrap_or_else(|_| ProposalRejectReason::Rejected.into()) + .into()) +} + // - State and syncer checks /// Returns an error if Zebra is not synced to the consensus chain tip. From 89785418947780226bb1e16d1d808008d13e5f7c Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 23 Dec 2022 15:24:03 -0500 Subject: [PATCH 21/44] corrects `difficulty_threshold_is_valid` docs adds/fixes some comments, adds TODOs general cleanup from a self-review. --- zebra-consensus/src/block.rs | 57 +++++++++---------- zebra-consensus/src/block/check.rs | 10 ++-- zebra-consensus/src/chain.rs | 3 + .../src/methods/get_block_template_rpcs.rs | 16 +++--- .../get_block_template_rpcs/constants.rs | 1 - .../get_block_template.rs | 2 +- .../types/get_block_template.rs | 8 +-- .../types/get_block_template/parameters.rs | 4 +- zebra-state/src/request.rs | 3 +- zebra-state/src/service.rs | 3 + .../src/service/non_finalized_state.rs | 5 +- 11 files changed, 58 insertions(+), 54 deletions(-) diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs index 29dbca4640e..8f37d411236 100644 --- a/zebra-consensus/src/block.rs +++ b/zebra-consensus/src/block.rs @@ -290,38 +290,35 @@ where transaction_hashes, }; - match request { - Request::Commit(_) => { - match state_service - .ready() - .await - .map_err(VerifyBlockError::Commit)? - .call(zs::Request::CommitBlock(prepared_block)) - .await - .map_err(VerifyBlockError::Commit)? - { - zs::Response::Committed(committed_hash) => { - assert_eq!(committed_hash, hash, "state must commit correct hash"); - Ok(hash) - } - _ => unreachable!("wrong response for CommitBlock"), - } - } + // Return early for proposal requests when getblocktemplate-rpcs feature is enabled + #[cfg(feature = "getblocktemplate-rpcs")] + if request.is_proposal() { + return match state_service + .ready() + .await + .map_err(VerifyBlockError::ValidateProposal)? + .call(zs::Request::CheckBlockProposalValidity(prepared_block)) + .await + .map_err(VerifyBlockError::ValidateProposal)? + { + zs::Response::ValidBlockProposal => Ok(hash), + _ => unreachable!("wrong response for CheckBlockProposalValidity"), + }; + } - #[cfg(feature = "getblocktemplate-rpcs")] - Request::CheckProposal(_) => { - match state_service - .ready() - .await - .map_err(VerifyBlockError::ValidateProposal)? - .call(zs::Request::CheckBlockProposalValidity(prepared_block)) - .await - .map_err(VerifyBlockError::ValidateProposal)? - { - zs::Response::ValidBlockProposal => Ok(hash), - _ => unreachable!("wrong response for CheckBlockProposalValidity"), - } + match state_service + .ready() + .await + .map_err(VerifyBlockError::Commit)? + .call(zs::Request::CommitBlock(prepared_block)) + .await + .map_err(VerifyBlockError::Commit)? + { + zs::Response::Committed(committed_hash) => { + assert_eq!(committed_hash, hash, "state must commit correct hash"); + Ok(hash) } + _ => unreachable!("wrong response for CommitBlock"), } } .instrument(span) diff --git a/zebra-consensus/src/block/check.rs b/zebra-consensus/src/block/check.rs index 761ea93352d..717e81ea402 100644 --- a/zebra-consensus/src/block/check.rs +++ b/zebra-consensus/src/block/check.rs @@ -58,9 +58,8 @@ pub fn coinbase_is_first(block: &Block) -> Result, Ok(first.clone()) } -/// Returns `Ok(ExpandedDifficulty)` if `difficulty_threshold` of `header` passes: -/// - the target difficulty limit for `network` (PoWLimit), and -/// - the difficulty filter, +/// Returns `Ok(ExpandedDifficulty)` if `difficulty_threshold` of `header` passes +/// the target difficulty limit for `network` (PoWLimit) /// /// If the header is invalid, returns an error containing `height` and `hash`. pub fn difficulty_threshold_is_valid( @@ -74,7 +73,7 @@ pub fn difficulty_threshold_is_valid( .to_expanded() .ok_or(BlockError::InvalidDifficulty(*height, *hash))?; - // Note: the comparisons in this function are u256 integer comparisons, like + // Note: the comparison in this function is a u256 integer comparison, like // zcashd and bitcoin. Greater values represent *less* work. // The PowLimit check is part of `Threshold()` in the spec, but it doesn't @@ -106,6 +105,9 @@ pub fn difficulty_is_valid( ) -> Result<(), BlockError> { let difficulty_threshold = difficulty_threshold_is_valid(header, network, height, hash)?; + // Note: the comparison in this function is a u256 integer comparison, like + // zcashd and bitcoin. Greater values represent *less* work. + // # Consensus // // > The block MUST pass the difficulty filter. diff --git a/zebra-consensus/src/chain.rs b/zebra-consensus/src/chain.rs index c39f991493e..4f54f417768 100644 --- a/zebra-consensus/src/chain.rs +++ b/zebra-consensus/src/chain.rs @@ -170,8 +170,11 @@ where match block.coinbase_height() { #[cfg(feature = "getblocktemplate-rpcs")] + // There's currently no known use case for block proposals below the checkpoint height, + // so it's okay to immediately return an error here. Some(height) if height <= self.max_checkpoint_height && request.is_proposal() => { async { + // TODO: Add a `ValidateProposalError` enum with a `BelowCheckpoint` variant? Err(VerifyBlockError::ValidateProposal( "block proposals must be above checkpoint height".into(), ))? diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index fc290dd49f6..3b247878045 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -330,19 +330,19 @@ where let mut latest_chain_tip = self.latest_chain_tip.clone(); let sync_status = self.sync_status.clone(); let state = self.state.clone(); - let chain_verifier = self.chain_verifier.clone(); + + if let Some(HexData(block_proposal_bytes)) = parameters + .as_mut() + .and_then(get_block_template::JsonParameters::block_proposal_data) + { + return validate_block_proposal(self.chain_verifier.clone(), block_proposal_bytes) + .boxed(); + } // To implement long polling correctly, we split this RPC into multiple phases. async move { get_block_template::check_parameters(¶meters)?; - if let Some(HexData(block_proposal_bytes)) = parameters - .as_mut() - .and_then(get_block_template::JsonParameters::block_proposal_data) - { - return validate_block_proposal(chain_verifier, block_proposal_bytes).await; - } - let client_long_poll_id = parameters .as_mut() .and_then(|params| params.long_poll_id.take()); diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/constants.rs b/zebra-rpc/src/methods/get_block_template_rpcs/constants.rs index 305ab744156..2c0e7814e44 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/constants.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/constants.rs @@ -26,7 +26,6 @@ pub const GET_BLOCK_TEMPLATE_MUTABLE_FIELD: &[&str] = &[ ]; /// A hardcoded list of Zebra's getblocktemplate RPC capabilities. -/// Currently empty. pub const GET_BLOCK_TEMPLATE_CAPABILITIES_FIELD: &[&str] = &["proposal"]; /// The max estimated distance to the chain tip for the getblocktemplate method. diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs index 6182df60b6f..7270e810f3c 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs @@ -124,7 +124,7 @@ where .await; Ok(chain_verifier_response - .map(|_| ProposalResponse::Valid) + .map(|_hash| ProposalResponse::Valid) .unwrap_or_else(|_| ProposalRejectReason::Rejected.into()) .into()) } diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs index 9d1ef88f669..dcf62d32fa4 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs @@ -263,7 +263,7 @@ impl GetBlockTemplate { #[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "kebab-case")] pub enum ProposalRejectReason { - /// Block rejected as invalid + /// Block proposal rejected as invalid. Rejected, } @@ -273,16 +273,16 @@ pub enum ProposalRejectReason { #[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(untagged, rename_all = "kebab-case")] pub enum ProposalResponse { - /// Block was not successfully submitted, return error + /// Block proposal was rejected as invalid, returns `reject-reason` and server `capabilities`. ErrorResponse { - /// Reason the proposal was invalid as-is + /// Reason the proposal was invalid as-is. reject_reason: ProposalRejectReason, /// The getblocktemplate RPC capabilities supported by Zebra. capabilities: Vec, }, - /// Block successfully proposed, returns null + /// Block proposal was successfully validated, returns null. Valid, } diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs index eea36a78d35..0d64e0e9cf2 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs @@ -103,9 +103,9 @@ impl JsonParameters { Self { mode: GetBlockTemplateRequestMode::Proposal, - data: block_proposal_data @ Some(_), + data, .. - } => block_proposal_data.take(), + } => data.take(), } } } diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index 0080379c8fa..1ab59694f09 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -562,6 +562,7 @@ pub enum Request { #[cfg(feature = "getblocktemplate-rpcs")] /// Performs contextual validation of the given block, but does not commit it to the state. /// + /// Returns [`Response::ValidBlockProposal`] when successful. /// See `[ReadRequest::CheckBlockProposalValidity]` for details. CheckBlockProposalValidity(PreparedBlock), } @@ -805,7 +806,7 @@ pub enum ReadRequest { /// It is the caller's responsibility to perform semantic validation. /// (The caller does not need to check proof of work for block proposals.) /// - /// Returns [`Response::ValidBlockProposal`] with the hash of the block when successful, or an error if + /// Returns [`ReadResponse::ValidBlockProposal`] when successful, or an error if /// the block fails contextual validation. CheckBlockProposalValidity(PreparedBlock), } diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 63dcb4fdeef..5b036f432c9 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1720,9 +1720,12 @@ impl Service for ReadStateService { tracing::info!("attempting to validate and commit block proposal onto a cloned non-finalized state"); let mut latest_non_finalized_state = state.latest_non_finalized_state(); latest_non_finalized_state.should_count_metrics = false; + // This clone of the non-finalized state is dropped when this closure returns. // The non-finalized state that's used in the rest of the state (including finalizing // blocks into the db) is not mutated here. + // + // TODO: Convert `CommitBlockError` to a new `ValidateProposalError`? write::validate_and_commit_non_finalized( &state.db, &mut latest_non_finalized_state, diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs index 2d0aa903709..3e343da7405 100644 --- a/zebra-state/src/service/non_finalized_state.rs +++ b/zebra-state/src/service/non_finalized_state.rs @@ -44,14 +44,13 @@ pub struct NonFinalizedState { pub chain_set: BTreeSet>, /// The configured Zcash network. - // - // Note: this field is currently unused, but it's useful for debugging. pub network: Network, #[cfg(feature = "getblocktemplate-rpcs")] /// Configures the non-finalized state to count metrics. /// - /// Used for testing block proposals on a cloned non-finalized state. + /// Used for skipping metrics counting when testing block proposals + /// with a commit to a cloned non-finalized state. pub should_count_metrics: bool, } From 4551cf040c916015042d6bd56c6d70756ebbce04 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 5 Jan 2023 02:33:23 -0500 Subject: [PATCH 22/44] Update zebra-state/src/service.rs --- zebra-state/src/service.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 5b036f432c9..81e2d2dd1eb 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1703,7 +1703,6 @@ impl Service for ReadStateService { .boxed() } - // Used by getmininginfo, getnetworksolps, and getnetworkhashps RPCs. #[cfg(feature = "getblocktemplate-rpcs")] ReadRequest::CheckBlockProposalValidity(prepared) => { let timer = CodeTimer::start(); From 5cc737874d403d58110d386736017fb8dcb9a87f Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 9 Jan 2023 19:42:52 -0500 Subject: [PATCH 23/44] Apply suggestions from code review Co-authored-by: teor --- zebra-consensus/src/block.rs | 1 + zebra-consensus/src/block/check.rs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/zebra-consensus/src/block.rs b/zebra-consensus/src/block.rs index 8f37d411236..231b4a94ba4 100644 --- a/zebra-consensus/src/block.rs +++ b/zebra-consensus/src/block.rs @@ -29,6 +29,7 @@ use crate::{error::*, transaction as tx, BoxError}; pub mod check; pub mod request; pub mod subsidy; + pub use request::Request; #[cfg(test)] diff --git a/zebra-consensus/src/block/check.rs b/zebra-consensus/src/block/check.rs index 717e81ea402..ddd3dbefa63 100644 --- a/zebra-consensus/src/block/check.rs +++ b/zebra-consensus/src/block/check.rs @@ -58,10 +58,10 @@ pub fn coinbase_is_first(block: &Block) -> Result, Ok(first.clone()) } -/// Returns `Ok(ExpandedDifficulty)` if `difficulty_threshold` of `header` passes +/// Returns `Ok(ExpandedDifficulty)` if the`difficulty_threshold` of `header` is at least as difficult as /// the target difficulty limit for `network` (PoWLimit) /// -/// If the header is invalid, returns an error containing `height` and `hash`. +/// If the header difficulty threshold is invalid, returns an error containing `height` and `hash`. pub fn difficulty_threshold_is_valid( header: &Header, network: Network, From 67bf095a157d41aaee62d3f2af1f7b3bed65eed4 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 9 Jan 2023 20:00:01 -0500 Subject: [PATCH 24/44] Update zebra-rpc/src/methods/get_block_template_rpcs.rs Co-authored-by: teor --- zebra-rpc/src/methods/get_block_template_rpcs.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 3b247878045..4c1f32d2d20 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -332,7 +332,7 @@ where let state = self.state.clone(); if let Some(HexData(block_proposal_bytes)) = parameters - .as_mut() + .as_ref() .and_then(get_block_template::JsonParameters::block_proposal_data) { return validate_block_proposal(self.chain_verifier.clone(), block_proposal_bytes) @@ -344,8 +344,8 @@ where get_block_template::check_parameters(¶meters)?; let client_long_poll_id = parameters - .as_mut() - .and_then(|params| params.long_poll_id.take()); + .as_ref() + .and_then(|params| params.long_poll_id.clone()); // - One-off checks From f77482281518cccf0a51474c2886e4d00795a1ab Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 9 Jan 2023 03:21:25 -0500 Subject: [PATCH 25/44] check best chain tip --- zebra-rpc/src/methods/get_block_template_rpcs.rs | 2 +- .../types/get_block_template/parameters.rs | 4 ++-- zebra-state/src/service.rs | 9 +++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 4c1f32d2d20..10aeb96ffde 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -314,7 +314,7 @@ where // TODO: use a generic error constructor (#5548) fn get_block_template( &self, - mut parameters: Option, + parameters: Option, ) -> BoxFuture> { // Should we generate coinbase transactions that are exactly like zcashd's? // diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs index 0d64e0e9cf2..312ebf51f92 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs @@ -93,7 +93,7 @@ pub struct JsonParameters { impl JsonParameters { /// Returns Some(data) with the block proposal hexdata if in `Proposal` mode and `data` is provided. - pub fn block_proposal_data(&mut self) -> Option { + pub fn block_proposal_data(&self) -> Option { match self { Self { data: None, .. } | Self { @@ -105,7 +105,7 @@ impl JsonParameters { mode: GetBlockTemplateRequestMode::Proposal, data, .. - } => data.take(), + } => data.clone(), } } } diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 81e2d2dd1eb..0bf7207d8ab 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1715,16 +1715,21 @@ impl Service for ReadStateService { let span = Span::current(); tokio::task::spawn_blocking(move || { span.in_scope(move || { - tracing::info!("attempting to validate and commit block proposal onto a cloned non-finalized state"); let mut latest_non_finalized_state = state.latest_non_finalized_state(); - latest_non_finalized_state.should_count_metrics = false; + + // Check that the previous block hash is the best chain tip if there are blocks in the + // non-finalized state. The previous block of a valid proposal must be on the best chain tip. + if let Some(true) = latest_non_finalized_state.best_tip().map(|(_, tip_hash)| tip_hash != prepared.hash) { + return Err("previous block hash is not the best chain tip".into()) + } // This clone of the non-finalized state is dropped when this closure returns. // The non-finalized state that's used in the rest of the state (including finalizing // blocks into the db) is not mutated here. // // TODO: Convert `CommitBlockError` to a new `ValidateProposalError`? + latest_non_finalized_state.should_count_metrics = false; write::validate_and_commit_non_finalized( &state.db, &mut latest_non_finalized_state, From d75e17a6cb61f9b5af30bd042ef36874e975f647 Mon Sep 17 00:00:00 2001 From: Arya Date: Tue, 10 Jan 2023 12:33:37 -0500 Subject: [PATCH 26/44] Update zebra-state/src/service.rs Co-authored-by: teor --- zebra-state/src/service.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 0bf7207d8ab..5ed7f1e9d34 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1718,11 +1718,14 @@ impl Service for ReadStateService { tracing::info!("attempting to validate and commit block proposal onto a cloned non-finalized state"); let mut latest_non_finalized_state = state.latest_non_finalized_state(); - // Check that the previous block hash is the best chain tip if there are blocks in the - // non-finalized state. The previous block of a valid proposal must be on the best chain tip. - if let Some(true) = latest_non_finalized_state.best_tip().map(|(_, tip_hash)| tip_hash != prepared.hash) { - return Err("previous block hash is not the best chain tip".into()) - } + // The previous block of a valid proposal must be on the best chain tip. + let Some((_best_tip_height, best_tip_hash) = read::best_tip(&latest_non_finalized_state, &state.db) { + if prepared.block.header.previous_block_hash != best_tip_hash { + return Err("proposal is not based on the current best chain tip: previous block hash must be the best chain tip".into()); + } + } else { + return Err("state is empty: wait for Zebra to sync before submitting a proposal".into()); + } // This clone of the non-finalized state is dropped when this closure returns. // The non-finalized state that's used in the rest of the state (including finalizing From 9ce7d8afa45a660173f3a1610506cf89244a499a Mon Sep 17 00:00:00 2001 From: arya2 Date: Tue, 10 Jan 2023 13:37:19 -0500 Subject: [PATCH 27/44] Applies cleanup suggestions from code review --- .../methods/get_block_template_rpcs/constants.rs | 6 ++++++ zebra-state/src/service.rs | 16 ++++++++-------- zebra-state/src/service/check.rs | 4 ++-- zebra-state/src/service/write.rs | 4 +--- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/constants.rs b/zebra-rpc/src/methods/get_block_template_rpcs/constants.rs index 2c0e7814e44..ed624e4dc83 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/constants.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/constants.rs @@ -18,6 +18,8 @@ pub const GET_BLOCK_TEMPLATE_MEMPOOL_LONG_POLL_INTERVAL: u64 = 5; pub const GET_BLOCK_TEMPLATE_NONCE_RANGE_FIELD: &str = "00000000ffffffff"; /// A hardcoded list of fields that the miner can change from the block template. +/// +/// pub const GET_BLOCK_TEMPLATE_MUTABLE_FIELD: &[&str] = &[ // Standard mutations, copied from zcashd "time", @@ -26,6 +28,8 @@ pub const GET_BLOCK_TEMPLATE_MUTABLE_FIELD: &[&str] = &[ ]; /// A hardcoded list of Zebra's getblocktemplate RPC capabilities. +/// +/// pub const GET_BLOCK_TEMPLATE_CAPABILITIES_FIELD: &[&str] = &["proposal"]; /// The max estimated distance to the chain tip for the getblocktemplate method. @@ -34,6 +38,8 @@ pub const GET_BLOCK_TEMPLATE_CAPABILITIES_FIELD: &[&str] = &["proposal"]; /// > A full validator MUST NOT accept blocks with nTime more than two hours in the future /// > according to its clock. This is not strictly a consensus rule because it is nondeterministic, /// > and clock time varies between nodes. +/// > +/// > pub const MAX_ESTIMATED_DISTANCE_TO_NETWORK_CHAIN_TIP: i32 = 100; /// The RPC error code used by `zcashd` for when it's still downloading initial blocks. diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 5ed7f1e9d34..b8638520dec 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1718,14 +1718,14 @@ impl Service for ReadStateService { tracing::info!("attempting to validate and commit block proposal onto a cloned non-finalized state"); let mut latest_non_finalized_state = state.latest_non_finalized_state(); - // The previous block of a valid proposal must be on the best chain tip. - let Some((_best_tip_height, best_tip_hash) = read::best_tip(&latest_non_finalized_state, &state.db) { - if prepared.block.header.previous_block_hash != best_tip_hash { - return Err("proposal is not based on the current best chain tip: previous block hash must be the best chain tip".into()); - } - } else { - return Err("state is empty: wait for Zebra to sync before submitting a proposal".into()); - } + // The previous block of a valid proposal must be on the best chain tip. + let Some((_best_tip_height, best_tip_hash)) = read::best_tip(&latest_non_finalized_state, &state.db) else { + return Err("state is empty: wait for Zebra to sync before submitting a proposal".into()); + }; + + if prepared.block.header.previous_block_hash != best_tip_hash { + return Err("proposal is not based on the current best chain tip: previous block hash must be the best chain tip".into()); + } // This clone of the non-finalized state is dropped when this closure returns. // The non-finalized state that's used in the rest of the state (including finalizing diff --git a/zebra-state/src/service/check.rs b/zebra-state/src/service/check.rs index 603eb4f3e8d..0b3600ad89b 100644 --- a/zebra-state/src/service/check.rs +++ b/zebra-state/src/service/check.rs @@ -14,13 +14,13 @@ use zebra_chain::{ use crate::{ service::{ block_iter::any_ancestor_blocks, check::difficulty::POW_ADJUSTMENT_BLOCK_SPAN, - non_finalized_state::NonFinalizedState, + finalized_state::ZebraDb, non_finalized_state::NonFinalizedState, }, BoxError, PreparedBlock, ValidateContextError, }; // use self as check -use super::{check, finalized_state::ZebraDb}; +use super::check; // These types are used in doc links #[allow(unused_imports)] diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 7b91c09227c..8f33af7b6d1 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -12,7 +12,7 @@ use crate::{ constants::MAX_BLOCK_REORG_HEIGHT, service::{ check, - finalized_state::FinalizedState, + finalized_state::{FinalizedState, ZebraDb}, non_finalized_state::NonFinalizedState, queued_blocks::{QueuedFinalized, QueuedNonFinalized}, BoxError, ChainTipBlock, ChainTipSender, CloneError, @@ -27,8 +27,6 @@ use crate::service::{ non_finalized_state::Chain, }; -use super::finalized_state::ZebraDb; - /// The maximum size of the parent error map. /// /// We allow enough space for multiple concurrent chain forks with errors. From 8b89cbadb1bd70ff3dfa012a342052bda9b8c64c Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 19 Dec 2022 14:57:21 -0500 Subject: [PATCH 28/44] updates gbt acceptance test to make a block proposal --- zebra-chain/src/work/equihash.rs | 7 ++ zebra-rpc/src/methods.rs | 2 +- .../types/transaction.rs | 2 +- .../get_block_template.rs | 87 +++++++++++++++++-- 4 files changed, 91 insertions(+), 7 deletions(-) diff --git a/zebra-chain/src/work/equihash.rs b/zebra-chain/src/work/equihash.rs index 230ba6d5a94..30ebebc5e5a 100644 --- a/zebra-chain/src/work/equihash.rs +++ b/zebra-chain/src/work/equihash.rs @@ -61,6 +61,13 @@ impl Solution { } } +#[cfg(feature = "getblocktemplate-rpcs")] +impl Default for Solution { + fn default() -> Self { + Self([0; SOLUTION_SIZE]) + } +} + impl PartialEq for Solution { fn eq(&self, other: &Solution) -> bool { self.0.as_ref() == other.0.as_ref() diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index 4b4fd72888f..43b53a8e240 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -1193,7 +1193,7 @@ pub enum GetBlock { /// /// Also see the notes for the [`Rpc::get_best_block_hash`] and `get_block_hash` methods. #[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] -pub struct GetBlockHash(#[serde(with = "hex")] block::Hash); +pub struct GetBlockHash(#[serde(with = "hex")] pub block::Hash); /// Response to a `z_gettreestate` RPC request. /// diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/transaction.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/transaction.rs index e72d75e8175..c373722a362 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/transaction.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/transaction.rs @@ -16,7 +16,7 @@ where { /// The hex-encoded serialized data for this transaction. #[serde(with = "hex")] - pub(crate) data: SerializedTransaction, + pub data: SerializedTransaction, /// The transaction ID of this transaction. #[serde(with = "hex")] diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs index 953bfeff3cf..91bc978e6d1 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -5,11 +5,23 @@ //! //! After finishing the sync, it will call getblocktemplate. -use std::time::Duration; +use std::{sync::Arc, time::Duration}; +use chrono::Utc; use color_eyre::eyre::{eyre, Context, Result}; -use zebra_chain::parameters::Network; +use zebra_chain::{ + block::{self, Block, Height}, + parameters::Network, + serialization::{ZcashDeserializeInto, ZcashSerialize}, + work::equihash::Solution, +}; +use zebra_rpc::methods::{ + get_block_template_rpcs::{ + get_block_template::GetBlockTemplate, types::default_roots::DefaultRoots, + }, + GetBlockHash, +}; use crate::common::{ launch::{can_spawn_zebrad_for_rpc, spawn_zebrad_for_rpc}, @@ -90,9 +102,9 @@ pub(crate) async fn run() -> Result<()> { "calling getblocktemplate RPC method at {rpc_address}, \ with a mempool that likely has transactions...", ); - let getblocktemplate_response = RPCRequestClient::new(rpc_address) - .call("getblocktemplate", "[]".to_string()) - .await?; + + let client = RPCRequestClient::new(rpc_address); + let getblocktemplate_response = client.call("getblocktemplate", "[]".to_string()).await?; let is_response_success = getblocktemplate_response.status().is_success(); let response_text = getblocktemplate_response.text().await?; @@ -104,6 +116,28 @@ pub(crate) async fn run() -> Result<()> { assert!(is_response_success); + // Propose a new block with an empty solution and nonce field + + let raw_block_proposal = hex::encode( + proposal_block_from_template(serde_json::from_str(&response_text)?)? + .zcash_serialize_to_vec()?, + ); + + let getblocktemplate_proposal_response = client + .call( + "getblocktemplate", + format!(r#"["{{"mode":"proposal","data":"{raw_block_proposal}"}}"])"#), + ) + .await?; + + let is_response_success = getblocktemplate_proposal_response.status().is_success(); + let response_text = getblocktemplate_proposal_response.text().await?; + + tracing::info!(response_text, "got getblocktemplate proposal response"); + + assert!(is_response_success); + assert!(response_text.contains(r#""result":null"#)); + zebrad.kill(false)?; let output = zebrad.wait_with_output()?; @@ -116,3 +150,46 @@ pub(crate) async fn run() -> Result<()> { Ok(()) } + +/// Make a block proposal from [`GetBlockTemplate`] +fn proposal_block_from_template( + GetBlockTemplate { + version, + height, + previous_block_hash: GetBlockHash(previous_block_hash), + default_roots: + DefaultRoots { + merkle_root, + block_commitments_hash, + .. + }, + target, + coinbase_txn, + transactions: tx_templates, + .. + }: GetBlockTemplate, +) -> Result { + if Height(height) > Height::MAX { + Err(eyre!("height field must be lower than Height::MAX"))?; + }; + + let mut transactions = vec![coinbase_txn.data.as_ref().zcash_deserialize_into()?]; + + for tx_template in tx_templates { + transactions.push(tx_template.data.as_ref().zcash_deserialize_into()?); + } + + Ok(Block { + header: Arc::new(block::Header { + version, + previous_block_hash, + merkle_root, + commitment_bytes: block_commitments_hash.into(), + time: Utc::now(), + difficulty_threshold: target.to_compact(), + nonce: [0; 32], + solution: Solution::default(), + }), + transactions, + }) +} From 0bf81c7c600be366aec28f81177e12e3939ada69 Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 19 Dec 2022 18:34:14 -0500 Subject: [PATCH 29/44] fixes json parsing mistake --- Cargo.lock | 1 + zebrad/Cargo.toml | 1 + .../get_block_template_rpcs/get_block_template.rs | 10 ++++++---- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1855779ba44..321b2bd54a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5664,6 +5664,7 @@ dependencies = [ "hyper", "indexmap", "inferno", + "jsonrpc-core", "lazy_static", "log", "metrics", diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index dc2c124f718..3329aac07f8 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -100,6 +100,7 @@ indexmap = "1.9.2" lazy_static = "1.4.0" serde = { version = "1.0.152", features = ["serde_derive"] } toml = "0.5.10" +jsonrpc-core = "18.0.0" futures = "0.3.25" rayon = "1.6.1" diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs index 91bc978e6d1..83012bab422 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -118,15 +118,17 @@ pub(crate) async fn run() -> Result<()> { // Propose a new block with an empty solution and nonce field + let response_json: jsonrpc_core::Success = serde_json::from_str(&response_text)?; + let raw_block_proposal = hex::encode( - proposal_block_from_template(serde_json::from_str(&response_text)?)? + proposal_block_from_template(serde_json::from_value(response_json.result)?)? .zcash_serialize_to_vec()?, ); let getblocktemplate_proposal_response = client .call( "getblocktemplate", - format!(r#"["{{"mode":"proposal","data":"{raw_block_proposal}"}}"])"#), + format!(r#"[{{"mode":"proposal","data":"{raw_block_proposal}"}}]"#), ) .await?; @@ -163,7 +165,7 @@ fn proposal_block_from_template( block_commitments_hash, .. }, - target, + bits: difficulty_threshold, coinbase_txn, transactions: tx_templates, .. @@ -186,7 +188,7 @@ fn proposal_block_from_template( merkle_root, commitment_bytes: block_commitments_hash.into(), time: Utc::now(), - difficulty_threshold: target.to_compact(), + difficulty_threshold, nonce: [0; 32], solution: Solution::default(), }), From 1d043b17170e50f027d24346c84406e931d83974 Mon Sep 17 00:00:00 2001 From: arya2 Date: Wed, 21 Dec 2022 21:31:14 -0500 Subject: [PATCH 30/44] adds retries --- .../get_block_template.rs | 96 +++++++++++++------ 1 file changed, 65 insertions(+), 31 deletions(-) diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs index 83012bab422..d1b934ee784 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -35,6 +35,11 @@ use crate::common::{ /// We've seen it take anywhere from 1-45 seconds for the mempool to have some transactions in it. pub const EXPECTED_MEMPOOL_TRANSACTION_TIME: Duration = Duration::from_secs(45); +/// Number of times to retry a block proposal request with the template returned by the getblocktemplate RPC. +/// +/// We've seen spurious rejections of block proposals. +const NUM_BLOCK_PROPOSAL_RETRIES: usize = 5; + /// Launch Zebra, wait for it to sync, and check the getblocktemplate RPC returns without errors. pub(crate) async fn run() -> Result<()> { let _init_guard = zebra_test::init(); @@ -70,11 +75,13 @@ pub(crate) async fn run() -> Result<()> { true, )?; + let client = RPCRequestClient::new(rpc_address); + tracing::info!( "calling getblocktemplate RPC method at {rpc_address}, \ with a mempool that is likely empty...", ); - let getblocktemplate_response = RPCRequestClient::new(rpc_address) + let getblocktemplate_response = client .call( "getblocktemplate", // test that unknown capabilities are parsed as valid input @@ -96,14 +103,36 @@ pub(crate) async fn run() -> Result<()> { "waiting {EXPECTED_MEMPOOL_TRANSACTION_TIME:?} for the mempool \ to download and verify some transactions...", ); - tokio::time::sleep(EXPECTED_MEMPOOL_TRANSACTION_TIME).await; - tracing::info!( - "calling getblocktemplate RPC method at {rpc_address}, \ - with a mempool that likely has transactions...", - ); + let mut num_remaining_retries = NUM_BLOCK_PROPOSAL_RETRIES; + loop { + tokio::time::sleep(EXPECTED_MEMPOOL_TRANSACTION_TIME).await; + + tracing::info!( + "calling getblocktemplate RPC method at {rpc_address}, \ + with a mempool that likely has transactions...", + ); + + if try_validate_block_template(&client).await.is_ok() { + break; + } else { + assert!(num_remaining_retries > 0); + num_remaining_retries -= 1; + } + } - let client = RPCRequestClient::new(rpc_address); + zebrad.kill(false)?; + + let output = zebrad.wait_with_output()?; + let output = output.assert_failure()?; + + // [Note on port conflict](#Note on port conflict) + output + .assert_was_killed() + .wrap_err("Possible port conflict. Are there other acceptance tests running?") +} + +async fn try_validate_block_template(client: &RPCRequestClient) -> Result<()> { let getblocktemplate_response = client.call("getblocktemplate", "[]".to_string()).await?; let is_response_success = getblocktemplate_response.status().is_success(); @@ -117,40 +146,45 @@ pub(crate) async fn run() -> Result<()> { assert!(is_response_success); // Propose a new block with an empty solution and nonce field + tracing::info!("calling getblocktemplate with a block proposal...",); + + let (is_response_success, response_text) = + propose_block_from_response_text(client, response_text).await?; - let response_json: jsonrpc_core::Success = serde_json::from_str(&response_text)?; + tracing::info!(response_text, "got getblocktemplate proposal response"); + + assert!(is_response_success); + + response_text + .contains(r#""result":null"#) + .then_some(()) + .ok_or_else(|| { + eyre!("response result must be null to indicate successful block proposal validation") + }) +} - let raw_block_proposal = hex::encode( - proposal_block_from_template(serde_json::from_value(response_json.result)?)? +async fn propose_block_from_response_text( + client: &RPCRequestClient, + template_response_text: String, +) -> Result<(bool, String)> { + let template_response: jsonrpc_core::Success = serde_json::from_str(&template_response_text)?; + + let raw_proposal_block = hex::encode( + proposal_block_from_template(serde_json::from_value(template_response.result)?)? .zcash_serialize_to_vec()?, ); - let getblocktemplate_proposal_response = client + let proposal_response = client .call( "getblocktemplate", - format!(r#"[{{"mode":"proposal","data":"{raw_block_proposal}"}}]"#), + format!(r#"[{{"mode":"proposal","data":"{raw_proposal_block}"}}]"#), ) .await?; - let is_response_success = getblocktemplate_proposal_response.status().is_success(); - let response_text = getblocktemplate_proposal_response.text().await?; - - tracing::info!(response_text, "got getblocktemplate proposal response"); - - assert!(is_response_success); - assert!(response_text.contains(r#""result":null"#)); - - zebrad.kill(false)?; - - let output = zebrad.wait_with_output()?; - let output = output.assert_failure()?; - - // [Note on port conflict](#Note on port conflict) - output - .assert_was_killed() - .wrap_err("Possible port conflict. Are there other acceptance tests running?")?; - - Ok(()) + Ok(( + proposal_response.status().is_success(), + proposal_response.text().await?, + )) } /// Make a block proposal from [`GetBlockTemplate`] From da54e85137c775e02ea26493ec526bde14ea4a8c Mon Sep 17 00:00:00 2001 From: arya2 Date: Thu, 22 Dec 2022 14:35:30 -0500 Subject: [PATCH 31/44] returns reject reason if there are no retries left --- .../get_block_template.rs | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs index d1b934ee784..1bfcc4f2e68 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -18,7 +18,8 @@ use zebra_chain::{ }; use zebra_rpc::methods::{ get_block_template_rpcs::{ - get_block_template::GetBlockTemplate, types::default_roots::DefaultRoots, + get_block_template::{GetBlockTemplate, ProposalResponse}, + types::default_roots::DefaultRoots, }, GetBlockHash, }; @@ -113,12 +114,11 @@ pub(crate) async fn run() -> Result<()> { with a mempool that likely has transactions...", ); - if try_validate_block_template(&client).await.is_ok() { - break; - } else { - assert!(num_remaining_retries > 0); - num_remaining_retries -= 1; - } + match try_validate_block_template(&client).await { + Ok(()) => break, + Err(_) if num_remaining_retries > 0 => num_remaining_retries -= 1, + err => return err, + }; } zebrad.kill(false)?; @@ -155,12 +155,13 @@ async fn try_validate_block_template(client: &RPCRequestClient) -> Result<()> { assert!(is_response_success); - response_text - .contains(r#""result":null"#) - .then_some(()) - .ok_or_else(|| { - eyre!("response result must be null to indicate successful block proposal validation") - }) + let proposal_response: jsonrpc_core::Success = serde_json::from_str(&response_text)?; + match serde_json::from_value(proposal_response.result)? { + ProposalResponse::Valid => Ok(()), + ProposalResponse::ErrorResponse { reject_reason, .. } => Err(eyre!( + "unsuccessful block proposal validation, reason: {reject_reason:?}" + )), + } } async fn propose_block_from_response_text( From ee0fc0b835f522135605b9c7fd9ec7805613f8b6 Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 2 Jan 2023 15:54:08 -0500 Subject: [PATCH 32/44] moves result deserialization to RPCRequestClient method, adds docs, moves jsonrpc_core to dev-dependencies --- zebrad/Cargo.toml | 2 +- .../get_block_template.rs | 66 ++++++++----------- zebrad/tests/common/rpc_client.rs | 27 ++++++++ 3 files changed, 57 insertions(+), 38 deletions(-) diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index 3329aac07f8..195e950ae01 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -100,7 +100,6 @@ indexmap = "1.9.2" lazy_static = "1.4.0" serde = { version = "1.0.152", features = ["serde_derive"] } toml = "0.5.10" -jsonrpc-core = "18.0.0" futures = "0.3.25" rayon = "1.6.1" @@ -168,6 +167,7 @@ tonic-build = { version = "0.8.0", optional = true } [dev-dependencies] abscissa_core = { version = "0.5", features = ["testing"] } hex = "0.4.3" +jsonrpc-core = "18.0.0" once_cell = "1.17.0" regex = "1.7.0" semver = "1.0.16" diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs index 1bfcc4f2e68..9124db3fd44 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -132,31 +132,47 @@ pub(crate) async fn run() -> Result<()> { .wrap_err("Possible port conflict. Are there other acceptance tests running?") } +/// Accepts an [`RPCRequestClient`], calls getblocktemplate in template mode, +/// deserializes and transforms the block template in the response into block proposal data, +/// then calls getblocktemplate RPC in proposal mode with the serialized and hex-encoded data. +/// +/// Returns an error if it fails to transform template to block proposal or serialize the block proposal +/// Returns `Ok(())` if the block proposal is valid or an error with the reject-reason if the result is +/// an `ErrorResponse`. +/// +/// ## Panics +/// +/// If an RPC call returns a failure +/// If the response result cannot be deserialized to `GetBlockTemplate` in 'template' mode +/// or `ProposalResponse` in 'proposal' mode. async fn try_validate_block_template(client: &RPCRequestClient) -> Result<()> { - let getblocktemplate_response = client.call("getblocktemplate", "[]".to_string()).await?; - - let is_response_success = getblocktemplate_response.status().is_success(); - let response_text = getblocktemplate_response.text().await?; + let response_json_result = client + .json_result_from_call("getblocktemplate", "[]".to_string()) + .await + .expect("response should be success output with with a serialized `GetBlockTemplate`"); tracing::info!( - response_text, + ?response_json_result, "got getblocktemplate response, hopefully with transactions" ); - assert!(is_response_success); - // Propose a new block with an empty solution and nonce field tracing::info!("calling getblocktemplate with a block proposal...",); - let (is_response_success, response_text) = - propose_block_from_response_text(client, response_text).await?; + let raw_proposal_block = + hex::encode(proposal_block_from_template(response_json_result)?.zcash_serialize_to_vec()?); - tracing::info!(response_text, "got getblocktemplate proposal response"); + let json_result = client + .json_result_from_call( + "getblocktemplate", + format!(r#"[{{"mode":"proposal","data":"{raw_proposal_block}"}}]"#), + ) + .await + .expect("response should be success output with with a serialized `ProposalResponse`"); - assert!(is_response_success); + tracing::info!(?json_result, "got getblocktemplate proposal response"); - let proposal_response: jsonrpc_core::Success = serde_json::from_str(&response_text)?; - match serde_json::from_value(proposal_response.result)? { + match json_result { ProposalResponse::Valid => Ok(()), ProposalResponse::ErrorResponse { reject_reason, .. } => Err(eyre!( "unsuccessful block proposal validation, reason: {reject_reason:?}" @@ -164,30 +180,6 @@ async fn try_validate_block_template(client: &RPCRequestClient) -> Result<()> { } } -async fn propose_block_from_response_text( - client: &RPCRequestClient, - template_response_text: String, -) -> Result<(bool, String)> { - let template_response: jsonrpc_core::Success = serde_json::from_str(&template_response_text)?; - - let raw_proposal_block = hex::encode( - proposal_block_from_template(serde_json::from_value(template_response.result)?)? - .zcash_serialize_to_vec()?, - ); - - let proposal_response = client - .call( - "getblocktemplate", - format!(r#"[{{"mode":"proposal","data":"{raw_proposal_block}"}}]"#), - ) - .await?; - - Ok(( - proposal_response.status().is_success(), - proposal_response.text().await?, - )) -} - /// Make a block proposal from [`GetBlockTemplate`] fn proposal_block_from_template( GetBlockTemplate { diff --git a/zebrad/tests/common/rpc_client.rs b/zebrad/tests/common/rpc_client.rs index f825c93fcd4..bc6e350aa5c 100644 --- a/zebrad/tests/common/rpc_client.rs +++ b/zebrad/tests/common/rpc_client.rs @@ -2,6 +2,8 @@ use std::net::SocketAddr; +use color_eyre::{eyre::eyre, Result}; +use jsonrpc_core::Output; use reqwest::Client; /// An http client for making Json-RPC requests @@ -44,4 +46,29 @@ impl RPCRequestClient { ) -> reqwest::Result { self.call(method, params).await?.text().await } + + /// Builds an RPC request, awaits its response, and attempts to deserialize + /// it to the expected result type. + /// + /// Returns Ok with json result from response if successful. + /// Returns an error if the call or result deserialization fail. + pub async fn json_result_from_call( + &self, + method: &'static str, + params: impl Into, + ) -> Result { + Self::json_result_from_response_text(&self.text_from_call(method, params).await?) + } + + /// Accepts response text from an RPC call + /// Returns `Ok` with a deserialized `result` value in the expected type, or an error report. + fn json_result_from_response_text( + response_text: &str, + ) -> Result { + let output: Output = serde_json::from_str(response_text)?; + match output { + Output::Success(success) => Ok(serde_json::from_value(success.result)?), + Output::Failure(failure) => Err(eyre!("RPC call failed with: {failure:?}")), + } + } } From a6ac270ec9d69b11a3a29ebdd5a050e70f95360a Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 2 Jan 2023 16:26:24 -0500 Subject: [PATCH 33/44] moves sleep(EXPECTED_TX_TIME) out of loop --- .../tests/common/get_block_template_rpcs/get_block_template.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs index 9124db3fd44..57c3fdf2e3f 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -104,11 +104,10 @@ pub(crate) async fn run() -> Result<()> { "waiting {EXPECTED_MEMPOOL_TRANSACTION_TIME:?} for the mempool \ to download and verify some transactions...", ); + tokio::time::sleep(EXPECTED_MEMPOOL_TRANSACTION_TIME).await; let mut num_remaining_retries = NUM_BLOCK_PROPOSAL_RETRIES; loop { - tokio::time::sleep(EXPECTED_MEMPOOL_TRANSACTION_TIME).await; - tracing::info!( "calling getblocktemplate RPC method at {rpc_address}, \ with a mempool that likely has transactions...", From d58d474ced42fca4db014c6815a971e4a6a91d0b Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 2 Jan 2023 16:40:13 -0500 Subject: [PATCH 34/44] updates/adds info logs in retry loop --- .../get_block_template_rpcs/get_block_template.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs index 57c3fdf2e3f..2d5852025df 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -110,12 +110,21 @@ pub(crate) async fn run() -> Result<()> { loop { tracing::info!( "calling getblocktemplate RPC method at {rpc_address}, \ - with a mempool that likely has transactions...", + with a mempool that likely has transactions and attempting \ + to validate response result as a block proposal", ); match try_validate_block_template(&client).await { Ok(()) => break, - Err(_) if num_remaining_retries > 0 => num_remaining_retries -= 1, + Err(error) if num_remaining_retries > 0 => { + num_remaining_retries -= 1; + + tracing::info!( + ?error, + num_remaining_retries, + "block proposal validation failed" + ); + } err => return err, }; } From f105b580fc9dc54eda8c9387e8eb91df40801c10 Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 2 Jan 2023 17:47:45 -0500 Subject: [PATCH 35/44] Revert "moves sleep(EXPECTED_TX_TIME) out of loop" This reverts commit f7f0926f4050519687a79afc16656c3f345c004b. --- .../tests/common/get_block_template_rpcs/get_block_template.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs index 2d5852025df..ef40f5cb462 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -104,10 +104,11 @@ pub(crate) async fn run() -> Result<()> { "waiting {EXPECTED_MEMPOOL_TRANSACTION_TIME:?} for the mempool \ to download and verify some transactions...", ); - tokio::time::sleep(EXPECTED_MEMPOOL_TRANSACTION_TIME).await; let mut num_remaining_retries = NUM_BLOCK_PROPOSAL_RETRIES; loop { + tokio::time::sleep(EXPECTED_MEMPOOL_TRANSACTION_TIME).await; + tracing::info!( "calling getblocktemplate RPC method at {rpc_address}, \ with a mempool that likely has transactions and attempting \ From fc4bd13b6e5a9ba1595baff897059a8309ca238d Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 2 Jan 2023 19:36:43 -0500 Subject: [PATCH 36/44] adds `allow(dead_code)` --- zebrad/tests/common/rpc_client.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zebrad/tests/common/rpc_client.rs b/zebrad/tests/common/rpc_client.rs index bc6e350aa5c..b81b1bb94f4 100644 --- a/zebrad/tests/common/rpc_client.rs +++ b/zebrad/tests/common/rpc_client.rs @@ -52,6 +52,7 @@ impl RPCRequestClient { /// /// Returns Ok with json result from response if successful. /// Returns an error if the call or result deserialization fail. + #[allow(dead_code)] // currently only used with getblocktemplate-rpcs feature pub async fn json_result_from_call( &self, method: &'static str, @@ -62,6 +63,7 @@ impl RPCRequestClient { /// Accepts response text from an RPC call /// Returns `Ok` with a deserialized `result` value in the expected type, or an error report. + #[allow(dead_code)] // currently only used with getblocktemplate-rpcs feature fn json_result_from_response_text( response_text: &str, ) -> Result { From 516f17c257e3f39f63a9fa91ec0f0d253ca02b99 Mon Sep 17 00:00:00 2001 From: arya2 Date: Thu, 12 Jan 2023 17:59:05 -0500 Subject: [PATCH 37/44] tests with curtime, mintime, & maxtime --- zebra-chain/src/work/equihash.rs | 7 ++- .../get_block_template.rs | 54 +++++++++++-------- zebrad/tests/common/rpc_client.rs | 4 +- 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/zebra-chain/src/work/equihash.rs b/zebra-chain/src/work/equihash.rs index 30ebebc5e5a..72b7b2cbb30 100644 --- a/zebra-chain/src/work/equihash.rs +++ b/zebra-chain/src/work/equihash.rs @@ -59,11 +59,10 @@ impl Solution { Ok(()) } -} -#[cfg(feature = "getblocktemplate-rpcs")] -impl Default for Solution { - fn default() -> Self { + #[cfg(feature = "getblocktemplate-rpcs")] + /// Returns a [`Solution`] of `[0; SOLUTION_SIZE]` to be used in block proposals. + pub fn for_proposal() -> Self { Self([0; SOLUTION_SIZE]) } } diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs index ef40f5cb462..cd7a0b7f3ef 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -7,7 +7,6 @@ use std::{sync::Arc, time::Duration}; -use chrono::Utc; use color_eyre::eyre::{eyre, Context, Result}; use zebra_chain::{ @@ -168,25 +167,33 @@ async fn try_validate_block_template(client: &RPCRequestClient) -> Result<()> { // Propose a new block with an empty solution and nonce field tracing::info!("calling getblocktemplate with a block proposal...",); - let raw_proposal_block = - hex::encode(proposal_block_from_template(response_json_result)?.zcash_serialize_to_vec()?); + for proposal_block in proposal_block_from_template(response_json_result)? { + let raw_proposal_block = hex::encode(proposal_block.zcash_serialize_to_vec()?); - let json_result = client - .json_result_from_call( - "getblocktemplate", - format!(r#"[{{"mode":"proposal","data":"{raw_proposal_block}"}}]"#), - ) - .await - .expect("response should be success output with with a serialized `ProposalResponse`"); + let json_result = client + .json_result_from_call( + "getblocktemplate", + format!(r#"[{{"mode":"proposal","data":"{raw_proposal_block}"}}]"#), + ) + .await + .expect("response should be success output with with a serialized `ProposalResponse`"); - tracing::info!(?json_result, "got getblocktemplate proposal response"); + tracing::info!( + ?json_result, + ?proposal_block.header.time, + "got getblocktemplate proposal response" + ); - match json_result { - ProposalResponse::Valid => Ok(()), - ProposalResponse::ErrorResponse { reject_reason, .. } => Err(eyre!( - "unsuccessful block proposal validation, reason: {reject_reason:?}" - )), + if let ProposalResponse::ErrorResponse { reject_reason, .. } = json_result { + Err(eyre!( + "unsuccessful block proposal validation, reason: {reject_reason:?}" + ))?; + } else { + assert_eq!(ProposalResponse::Valid, json_result); + } } + + Ok(()) } /// Make a block proposal from [`GetBlockTemplate`] @@ -204,9 +211,12 @@ fn proposal_block_from_template( bits: difficulty_threshold, coinbase_txn, transactions: tx_templates, + cur_time, + min_time, + max_time, .. }: GetBlockTemplate, -) -> Result { +) -> Result<[Block; 3]> { if Height(height) > Height::MAX { Err(eyre!("height field must be lower than Height::MAX"))?; }; @@ -217,17 +227,17 @@ fn proposal_block_from_template( transactions.push(tx_template.data.as_ref().zcash_deserialize_into()?); } - Ok(Block { + Ok([cur_time, min_time, max_time].map(|time| Block { header: Arc::new(block::Header { version, previous_block_hash, merkle_root, commitment_bytes: block_commitments_hash.into(), - time: Utc::now(), + time: time.into(), difficulty_threshold, nonce: [0; 32], - solution: Solution::default(), + solution: Solution::for_proposal(), }), - transactions, - }) + transactions: transactions.clone(), + })) } diff --git a/zebrad/tests/common/rpc_client.rs b/zebrad/tests/common/rpc_client.rs index b81b1bb94f4..edda8230ec9 100644 --- a/zebrad/tests/common/rpc_client.rs +++ b/zebrad/tests/common/rpc_client.rs @@ -52,7 +52,7 @@ impl RPCRequestClient { /// /// Returns Ok with json result from response if successful. /// Returns an error if the call or result deserialization fail. - #[allow(dead_code)] // currently only used with getblocktemplate-rpcs feature + #[cfg(feature = "getblocktemplate-rpcs")] pub async fn json_result_from_call( &self, method: &'static str, @@ -63,7 +63,7 @@ impl RPCRequestClient { /// Accepts response text from an RPC call /// Returns `Ok` with a deserialized `result` value in the expected type, or an error report. - #[allow(dead_code)] // currently only used with getblocktemplate-rpcs feature + #[cfg(feature = "getblocktemplate-rpcs")] fn json_result_from_response_text( response_text: &str, ) -> Result { From 60c7a13e800d23062d89ee4cba7bd9956d3c2734 Mon Sep 17 00:00:00 2001 From: arya2 Date: Thu, 12 Jan 2023 18:03:21 -0500 Subject: [PATCH 38/44] Fixes doc comment --- .../common/get_block_template_rpcs/get_block_template.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs index cd7a0b7f3ef..d5379c58ec5 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -196,7 +196,10 @@ async fn try_validate_block_template(client: &RPCRequestClient) -> Result<()> { Ok(()) } -/// Make a block proposal from [`GetBlockTemplate`] +/// Make block proposals from [`GetBlockTemplate`] +/// +/// Returns an array of 3 block proposals using `curtime`, `mintime`, and `maxtime` +/// for their `block.header.time` fields. fn proposal_block_from_template( GetBlockTemplate { version, From d2cae9959bbd49215e187c1017ded2ef3c1316fe Mon Sep 17 00:00:00 2001 From: arya2 Date: Thu, 12 Jan 2023 19:17:09 -0500 Subject: [PATCH 39/44] Logs error responses from chain_verifier CheckProposal requests --- .../get_block_template.rs | 9 ++- .../types/get_block_template.rs | 59 ++----------------- .../types/get_block_template/proposal.rs | 59 +++++++++++++++++++ 3 files changed, 71 insertions(+), 56 deletions(-) create mode 100644 zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/proposal.rs diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs index 7270e810f3c..eb9fd6ef770 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs @@ -125,7 +125,14 @@ where Ok(chain_verifier_response .map(|_hash| ProposalResponse::Valid) - .unwrap_or_else(|_| ProposalRejectReason::Rejected.into()) + .unwrap_or_else(|verify_chain_error| { + tracing::info!( + verify_chain_error, + "Got error response from chain_verifier CheckProposal request" + ); + + ProposalRejectReason::Rejected.into() + }) .into()) } diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs index dcf62d32fa4..2191f7ef407 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs @@ -1,4 +1,5 @@ -//! The `GetBlockTempate` type is the output of the `getblocktemplate` RPC method. +//! The `GetBlockTempate` type is the output of the `getblocktemplate` RPC method in the +//! default 'template' mode. See [`ProposalResponse`] for the output in 'proposal' mode. use zebra_chain::{ amount, @@ -24,8 +25,10 @@ use crate::methods::{ }; pub mod parameters; +pub mod proposal; pub use parameters::*; +pub use proposal::*; /// A serialized `getblocktemplate` RPC response in template mode. #[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] @@ -259,33 +262,6 @@ impl GetBlockTemplate { } } -/// Error response to a `getblocktemplate` RPC request in proposal mode. -#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum ProposalRejectReason { - /// Block proposal rejected as invalid. - Rejected, -} - -/// Response to a `getblocktemplate` RPC request in proposal mode. -/// -/// See -#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(untagged, rename_all = "kebab-case")] -pub enum ProposalResponse { - /// Block proposal was rejected as invalid, returns `reject-reason` and server `capabilities`. - ErrorResponse { - /// Reason the proposal was invalid as-is. - reject_reason: ProposalRejectReason, - - /// The getblocktemplate RPC capabilities supported by Zebra. - capabilities: Vec, - }, - - /// Block proposal was successfully validated, returns null. - Valid, -} - #[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(untagged)] /// A `getblocktemplate` RPC response. @@ -296,30 +272,3 @@ pub enum Response { /// `getblocktemplate` RPC request in proposal mode. ProposalMode(ProposalResponse), } - -impl From for ProposalResponse { - fn from(reject_reason: ProposalRejectReason) -> Self { - Self::ErrorResponse { - reject_reason, - capabilities: GetBlockTemplate::capabilities(), - } - } -} - -impl From for Response { - fn from(error_response: ProposalRejectReason) -> Self { - Self::ProposalMode(ProposalResponse::from(error_response)) - } -} - -impl From for Response { - fn from(proposal_response: ProposalResponse) -> Self { - Self::ProposalMode(proposal_response) - } -} - -impl From for Response { - fn from(template: GetBlockTemplate) -> Self { - Self::TemplateMode(Box::new(template)) - } -} diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/proposal.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/proposal.rs new file mode 100644 index 00000000000..5434390c648 --- /dev/null +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/proposal.rs @@ -0,0 +1,59 @@ +//! `ProposalResponse` is the output of the `getblocktemplate` RPC method in 'proposal' mode. + +use super::{GetBlockTemplate, Response}; + +/// Error response to a `getblocktemplate` RPC request in proposal mode. +/// +/// See +#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ProposalRejectReason { + /// Block proposal rejected as invalid. + Rejected, +} + +/// Response to a `getblocktemplate` RPC request in proposal mode. +/// +/// See +#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(untagged, rename_all = "kebab-case")] +pub enum ProposalResponse { + /// Block proposal was rejected as invalid, returns `reject-reason` and server `capabilities`. + ErrorResponse { + /// Reason the proposal was invalid as-is. + reject_reason: ProposalRejectReason, + + /// The getblocktemplate RPC capabilities supported by Zebra. + capabilities: Vec, + }, + + /// Block proposal was successfully validated, returns null. + Valid, +} + +impl From for ProposalResponse { + fn from(reject_reason: ProposalRejectReason) -> Self { + Self::ErrorResponse { + reject_reason, + capabilities: GetBlockTemplate::capabilities(), + } + } +} + +impl From for Response { + fn from(error_response: ProposalRejectReason) -> Self { + Self::ProposalMode(ProposalResponse::from(error_response)) + } +} + +impl From for Response { + fn from(proposal_response: ProposalResponse) -> Self { + Self::ProposalMode(proposal_response) + } +} + +impl From for Response { + fn from(template: GetBlockTemplate) -> Self { + Self::TemplateMode(Box::new(template)) + } +} From bb32c52066f8f2bf30036961fc2e76f5862d3252 Mon Sep 17 00:00:00 2001 From: arya2 Date: Thu, 12 Jan 2023 20:48:47 -0500 Subject: [PATCH 40/44] Removes retry loop, adds num_txs log --- .../get_block_template.rs | 8 ++++- .../get_block_template.rs | 32 ++++--------------- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs index eb9fd6ef770..e7409c65caa 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs @@ -108,10 +108,16 @@ where + Sync + 'static, { - let Ok(block) = block_proposal_bytes.zcash_deserialize_into() else { + let Ok(block) = block_proposal_bytes.zcash_deserialize_into::() else { return Ok(ProposalRejectReason::Rejected.into()) }; + tracing::info!( + ?block.header, + num_transactions = block.transactions.len(), + "Deserialized block proposal data, calling chain_verifier with CheckProposal request" + ); + let chain_verifier_response = chain_verifier .ready() .await diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs index d5379c58ec5..ccca903518c 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -35,11 +35,6 @@ use crate::common::{ /// We've seen it take anywhere from 1-45 seconds for the mempool to have some transactions in it. pub const EXPECTED_MEMPOOL_TRANSACTION_TIME: Duration = Duration::from_secs(45); -/// Number of times to retry a block proposal request with the template returned by the getblocktemplate RPC. -/// -/// We've seen spurious rejections of block proposals. -const NUM_BLOCK_PROPOSAL_RETRIES: usize = 5; - /// Launch Zebra, wait for it to sync, and check the getblocktemplate RPC returns without errors. pub(crate) async fn run() -> Result<()> { let _init_guard = zebra_test::init(); @@ -104,30 +99,17 @@ pub(crate) async fn run() -> Result<()> { to download and verify some transactions...", ); - let mut num_remaining_retries = NUM_BLOCK_PROPOSAL_RETRIES; - loop { - tokio::time::sleep(EXPECTED_MEMPOOL_TRANSACTION_TIME).await; + tokio::time::sleep(EXPECTED_MEMPOOL_TRANSACTION_TIME).await; - tracing::info!( - "calling getblocktemplate RPC method at {rpc_address}, \ + tracing::info!( + "calling getblocktemplate RPC method at {rpc_address}, \ with a mempool that likely has transactions and attempting \ to validate response result as a block proposal", - ); + ); - match try_validate_block_template(&client).await { - Ok(()) => break, - Err(error) if num_remaining_retries > 0 => { - num_remaining_retries -= 1; - - tracing::info!( - ?error, - num_remaining_retries, - "block proposal validation failed" - ); - } - err => return err, - }; - } + try_validate_block_template(&client) + .await + .expect("block proposal validation failed"); zebrad.kill(false)?; From 71c335ff368fbda42bc38c084fcadf0502d7ae08 Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 13 Jan 2023 15:17:18 -0500 Subject: [PATCH 41/44] removes verbose info log --- .../methods/get_block_template_rpcs/get_block_template.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs index e7409c65caa..16cb2a9e212 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs @@ -112,12 +112,6 @@ where return Ok(ProposalRejectReason::Rejected.into()) }; - tracing::info!( - ?block.header, - num_transactions = block.transactions.len(), - "Deserialized block proposal data, calling chain_verifier with CheckProposal request" - ); - let chain_verifier_response = chain_verifier .ready() .await From f052448ba0cedb3f64ea721889a7a342a48b8540 Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 13 Jan 2023 15:44:23 -0500 Subject: [PATCH 42/44] sorts mempool_txs before generating merkle root --- .../src/methods/get_block_template_rpcs.rs | 18 ++------- .../types/get_block_template.rs | 40 +++++++++++++++---- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 10aeb96ffde..8182f822ee8 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -27,7 +27,7 @@ use crate::methods::{ }, get_block_template::{ check_miner_address, check_synced_to_tip, fetch_mempool_transactions, - fetch_state_tip_and_local_time, generate_coinbase_and_roots, validate_block_proposal, + fetch_state_tip_and_local_time, validate_block_proposal, }, types::{ get_block_template::GetBlockTemplate, get_mining_info, hex_data::HexData, @@ -566,24 +566,12 @@ where // - After this point, the template only depends on the previously fetched data. - // Generate the coinbase transaction and default roots - // - // TODO: move expensive root, hash, and tree cryptography to a rayon thread? - let (coinbase_txn, default_roots) = generate_coinbase_and_roots( + let response = GetBlockTemplate::new( network, - next_block_height, miner_address, - &mempool_txs, - chain_tip_and_local_time.history_tree.clone(), - COINBASE_LIKE_ZCASHD, - ); - - let response = GetBlockTemplate::new( &chain_tip_and_local_time, server_long_poll_id, - coinbase_txn, - &mempool_txs, - default_roots, + mempool_txs, submit_old, COINBASE_LIKE_ZCASHD, ); diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs index dcf62d32fa4..8cdf99411af 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs @@ -3,8 +3,10 @@ use zebra_chain::{ amount, block::{ChainHistoryBlockTxAuthCommitmentHash, MAX_BLOCK_BYTES, ZCASH_BLOCK_VERSION}, + parameters::Network, serialization::DateTime32, transaction::VerifiedUnminedTx, + transparent, work::difficulty::{CompactDifficulty, ExpandedDifficulty}, }; use zebra_consensus::MAX_BLOCK_SIGOPS; @@ -16,6 +18,7 @@ use crate::methods::{ GET_BLOCK_TEMPLATE_CAPABILITIES_FIELD, GET_BLOCK_TEMPLATE_MUTABLE_FIELD, GET_BLOCK_TEMPLATE_NONCE_RANGE_FIELD, }, + get_block_template::generate_coinbase_and_roots, types::{ default_roots::DefaultRoots, long_poll::LongPollId, transaction::TransactionTemplate, }, @@ -177,11 +180,11 @@ impl GetBlockTemplate { /// If `like_zcashd` is true, try to match the coinbase transactions generated by `zcashd` /// in the `getblocktemplate` RPC. pub fn new( + network: Network, + miner_address: transparent::Address, chain_tip_and_local_time: &GetBlockTemplateChainInfo, long_poll_id: LongPollId, - coinbase_txn: TransactionTemplate, - mempool_txs: &[VerifiedUnminedTx], - default_roots: DefaultRoots, + mempool_txs: Vec, submit_old: Option, like_zcashd: bool, ) -> Self { @@ -190,20 +193,41 @@ impl GetBlockTemplate { (chain_tip_and_local_time.tip_height + 1).expect("tip is far below Height::MAX"); // Convert transactions into TransactionTemplates - let mut mempool_txs: Vec> = - mempool_txs.iter().map(Into::into).collect(); + let mut mempool_txs_with_templates: Vec<( + TransactionTemplate, + VerifiedUnminedTx, + )> = mempool_txs + .into_iter() + .map(|tx| ((&tx).into(), tx)) + .collect(); // Transaction selection returns transactions in an arbitrary order, // but Zebra's snapshot tests expect the same order every time. if like_zcashd { // Sort in serialized data order, excluding the length byte. // `zcashd` sometimes seems to do this, but other times the order is arbitrary. - mempool_txs.sort_by_key(|tx| tx.data.clone()); + mempool_txs_with_templates.sort_by_key(|(tx_template, _tx)| tx_template.data.clone()); } else { // Sort by hash, this is faster. - mempool_txs.sort_by_key(|tx| tx.hash.bytes_in_display_order()); + mempool_txs_with_templates + .sort_by_key(|(tx_template, _tx)| tx_template.hash.bytes_in_display_order()); } + let (mempool_tx_templates, mempool_txs): (Vec<_>, Vec<_>) = + mempool_txs_with_templates.into_iter().unzip(); + + // Generate the coinbase transaction and default roots + // + // TODO: move expensive root, hash, and tree cryptography to a rayon thread? + let (coinbase_txn, default_roots) = generate_coinbase_and_roots( + network, + next_block_height, + miner_address, + &mempool_txs, + chain_tip_and_local_time.history_tree.clone(), + like_zcashd, + ); + // Convert difficulty let target = chain_tip_and_local_time .expected_difficulty @@ -228,7 +252,7 @@ impl GetBlockTemplate { final_sapling_root_hash: default_roots.block_commitments_hash, default_roots, - transactions: mempool_txs, + transactions: mempool_tx_templates, coinbase_txn, From 8c51e13c65da257e98b5c5e7d84c23f56369475d Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 16 Jan 2023 13:52:38 -0500 Subject: [PATCH 43/44] Make imports conditional on a feature --- zebrad/tests/common/rpc_client.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zebrad/tests/common/rpc_client.rs b/zebrad/tests/common/rpc_client.rs index edda8230ec9..5e459327719 100644 --- a/zebrad/tests/common/rpc_client.rs +++ b/zebrad/tests/common/rpc_client.rs @@ -2,10 +2,11 @@ use std::net::SocketAddr; -use color_eyre::{eyre::eyre, Result}; -use jsonrpc_core::Output; use reqwest::Client; +#[cfg(feature = "getblocktemplate-rpcs")] +use color_eyre::{eyre::eyre, Result}; + /// An http client for making Json-RPC requests pub struct RPCRequestClient { client: Client, @@ -67,6 +68,8 @@ impl RPCRequestClient { fn json_result_from_response_text( response_text: &str, ) -> Result { + use jsonrpc_core::Output; + let output: Output = serde_json::from_str(response_text)?; match output { Output::Success(success) => Ok(serde_json::from_value(success.result)?), From dd2438b31158f5cfc52334cbf313df33cc72cb87 Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 16 Jan 2023 13:52:52 -0500 Subject: [PATCH 44/44] Disable new CI tests until bugs are fixed --- .../common/get_block_template_rpcs/get_block_template.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs index ccca903518c..b3509428de6 100644 --- a/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs +++ b/zebrad/tests/common/get_block_template_rpcs/get_block_template.rs @@ -101,6 +101,8 @@ pub(crate) async fn run() -> Result<()> { tokio::time::sleep(EXPECTED_MEMPOOL_TRANSACTION_TIME).await; + /* TODO: activate this test after #5925 and #5953 have merged, + and we've checked for any other bugs using #5944. tracing::info!( "calling getblocktemplate RPC method at {rpc_address}, \ with a mempool that likely has transactions and attempting \ @@ -110,6 +112,7 @@ pub(crate) async fn run() -> Result<()> { try_validate_block_template(&client) .await .expect("block proposal validation failed"); + */ zebrad.kill(false)?; @@ -135,6 +138,7 @@ pub(crate) async fn run() -> Result<()> { /// If an RPC call returns a failure /// If the response result cannot be deserialized to `GetBlockTemplate` in 'template' mode /// or `ProposalResponse` in 'proposal' mode. +#[allow(dead_code)] async fn try_validate_block_template(client: &RPCRequestClient) -> Result<()> { let response_json_result = client .json_result_from_call("getblocktemplate", "[]".to_string()) @@ -182,6 +186,7 @@ async fn try_validate_block_template(client: &RPCRequestClient) -> Result<()> { /// /// Returns an array of 3 block proposals using `curtime`, `mintime`, and `maxtime` /// for their `block.header.time` fields. +#[allow(dead_code)] fn proposal_block_from_template( GetBlockTemplate { version,