From fde3370980a948d6854f22e02c28989b103357f5 Mon Sep 17 00:00:00 2001 From: Arya Date: Tue, 3 Sep 2024 22:25:42 -0400 Subject: [PATCH 01/69] Adds a parameter to `zebra_consensus::router::init()` for accepting a mempool setup argument, adds and uses an `init_test()` fn for passing a closed channel receiver in tests where no mempool service is needed in the transaction verifier. --- zebra-consensus/src/router.rs | 38 +++++++++++++++++- zebra-consensus/src/router/tests.rs | 4 +- .../tests/snapshot/get_block_template_rpcs.rs | 8 +++- zebra-rpc/src/methods/tests/vectors.rs | 40 ++++++++++++++----- zebrad/src/commands/start.rs | 2 + .../components/inbound/tests/fake_peer_set.rs | 10 +++-- zebrad/tests/acceptance.rs | 11 +++-- 7 files changed, 91 insertions(+), 22 deletions(-) diff --git a/zebra-consensus/src/router.rs b/zebra-consensus/src/router.rs index ba42896e56f..9f5311d7f43 100644 --- a/zebra-consensus/src/router.rs +++ b/zebra-consensus/src/router.rs @@ -21,7 +21,7 @@ use std::{ use futures::{FutureExt, TryFutureExt}; use thiserror::Error; -use tokio::task::JoinHandle; +use tokio::{sync::oneshot, task::JoinHandle}; use tower::{buffer::Buffer, util::BoxService, Service, ServiceExt}; use tracing::{instrument, Instrument, Span}; @@ -30,6 +30,7 @@ use zebra_chain::{ parameters::Network, }; +use zebra_node_services::mempool; use zebra_state as zs; use crate::{ @@ -230,11 +231,14 @@ where /// Block and transaction verification requests should be wrapped in a timeout, /// so that out-of-order and invalid requests do not hang indefinitely. /// See the [`router`](`crate::router`) module documentation for details. -#[instrument(skip(state_service))] +#[instrument(skip(state_service, mempool))] pub async fn init( config: Config, network: &Network, mut state_service: S, + mempool: oneshot::Receiver< + Buffer, mempool::Request>, + >, ) -> ( Buffer, Request>, Buffer< @@ -397,3 +401,33 @@ pub struct BackgroundTaskHandles { /// Finishes when all the checkpoints are verified, or when the state tip is reached. pub state_checkpoint_verify_handle: JoinHandle<()>, } + +/// Calls [`init`] with a closed mempool setup channel for conciseness in tests. +/// +/// See [`init`] for more details. +#[cfg(any(test, feature = "proptest-impl"))] +pub async fn init_test( + config: Config, + network: &Network, + state_service: S, +) -> ( + Buffer, Request>, + Buffer< + BoxService, + transaction::Request, + >, + BackgroundTaskHandles, + Height, +) +where + S: Service + Send + Clone + 'static, + S::Future: Send + 'static, +{ + init( + config.clone(), + network, + state_service.clone(), + oneshot::channel().1, + ) + .await +} diff --git a/zebra-consensus/src/router/tests.rs b/zebra-consensus/src/router/tests.rs index 8fe304e3364..063cc7394cf 100644 --- a/zebra-consensus/src/router/tests.rs +++ b/zebra-consensus/src/router/tests.rs @@ -68,7 +68,7 @@ async fn verifiers_from_network( _transaction_verifier, _groth16_download_handle, _max_checkpoint_height, - ) = crate::router::init(Config::default(), &network, state_service.clone()).await; + ) = crate::router::init_test(Config::default(), &network, state_service.clone()).await; // We can drop the download task handle here, because: // - if the download task fails, the tests will panic, and @@ -169,7 +169,7 @@ async fn verify_checkpoint(config: Config) -> Result<(), Report> { _transaction_verifier, _groth16_download_handle, _max_checkpoint_height, - ) = super::init(config.clone(), &network, zs::init_test(&network)).await; + ) = super::init_test(config.clone(), &network, zs::init_test(&network)).await; // Add a timeout layer let block_verifier_router = diff --git a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs index 8afb7dd312d..2fbd11c3978 100644 --- a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs @@ -86,8 +86,12 @@ pub async fn test_responses( _transaction_verifier, _parameter_download_task_handle, _max_checkpoint_height, - ) = zebra_consensus::router::init(zebra_consensus::Config::default(), network, state.clone()) - .await; + ) = zebra_consensus::router::init_test( + zebra_consensus::Config::default(), + network, + state.clone(), + ) + .await; let mut mock_sync_status = MockSyncStatus::default(); mock_sync_status.set_is_close_to_tip(true); diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index 5b5a21e23d0..f2a62e9e2bd 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -921,8 +921,12 @@ async fn rpc_getblockcount() { _transaction_verifier, _parameter_download_task_handle, _max_checkpoint_height, - ) = zebra_consensus::router::init(zebra_consensus::Config::default(), &Mainnet, state.clone()) - .await; + ) = zebra_consensus::router::init_test( + zebra_consensus::Config::default(), + &Mainnet, + state.clone(), + ) + .await; // Init RPC let get_block_template_rpc = GetBlockTemplateRpcImpl::new( @@ -966,8 +970,12 @@ async fn rpc_getblockcount_empty_state() { _transaction_verifier, _parameter_download_task_handle, _max_checkpoint_height, - ) = zebra_consensus::router::init(zebra_consensus::Config::default(), &Mainnet, state.clone()) - .await; + ) = zebra_consensus::router::init_test( + zebra_consensus::Config::default(), + &Mainnet, + state.clone(), + ) + .await; // Init RPC let get_block_template_rpc = get_block_template_rpcs::GetBlockTemplateRpcImpl::new( @@ -1013,8 +1021,12 @@ async fn rpc_getpeerinfo() { _transaction_verifier, _parameter_download_task_handle, _max_checkpoint_height, - ) = zebra_consensus::router::init(zebra_consensus::Config::default(), &network, state.clone()) - .await; + ) = zebra_consensus::router::init_test( + zebra_consensus::Config::default(), + &network, + state.clone(), + ) + .await; let mock_peer_address = zebra_network::types::MetaAddr::new_initial_peer( std::net::SocketAddr::new( @@ -1083,8 +1095,12 @@ async fn rpc_getblockhash() { _transaction_verifier, _parameter_download_task_handle, _max_checkpoint_height, - ) = zebra_consensus::router::init(zebra_consensus::Config::default(), &Mainnet, state.clone()) - .await; + ) = zebra_consensus::router::init_test( + zebra_consensus::Config::default(), + &Mainnet, + state.clone(), + ) + .await; // Init RPC let get_block_template_rpc = get_block_template_rpcs::GetBlockTemplateRpcImpl::new( @@ -1569,8 +1585,12 @@ async fn rpc_submitblock_errors() { _transaction_verifier, _parameter_download_task_handle, _max_checkpoint_height, - ) = zebra_consensus::router::init(zebra_consensus::Config::default(), &Mainnet, state.clone()) - .await; + ) = zebra_consensus::router::init_test( + zebra_consensus::Config::default(), + &Mainnet, + state.clone(), + ) + .await; // Init RPC let get_block_template_rpc = GetBlockTemplateRpcImpl::new( diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 887f1cc0242..13486ecd41f 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -184,6 +184,8 @@ impl StartCmd { config.consensus.clone(), &config.network.network, state.clone(), + // TODO: Pass actual setup channel receiver + oneshot::channel().1, ) .await; diff --git a/zebrad/src/components/inbound/tests/fake_peer_set.rs b/zebrad/src/components/inbound/tests/fake_peer_set.rs index 3ca30c5759a..b85dc3f2cb0 100644 --- a/zebrad/src/components/inbound/tests/fake_peer_set.rs +++ b/zebrad/src/components/inbound/tests/fake_peer_set.rs @@ -785,7 +785,7 @@ async fn caches_getaddr_response() { _transaction_verifier, _groth16_download_handle, _max_checkpoint_height, - ) = zebra_consensus::router::init( + ) = zebra_consensus::router::init_test( consensus_config.clone(), &network, state_service.clone(), @@ -894,8 +894,12 @@ async fn setup( // Download task panics and timeouts are propagated to the tests that use Groth16 verifiers. let (block_verifier, _transaction_verifier, _groth16_download_handle, _max_checkpoint_height) = - zebra_consensus::router::init(consensus_config.clone(), &network, state_service.clone()) - .await; + zebra_consensus::router::init_test( + consensus_config.clone(), + &network, + state_service.clone(), + ) + .await; let mut peer_set = MockService::build() .with_max_request_delay(MAX_PEER_SET_REQUEST_DELAY) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index c21b0a0e3e3..6c6594159f8 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -2910,7 +2910,8 @@ async fn validate_regtest_genesis_block() { _transaction_verifier, _parameter_download_task_handle, _max_checkpoint_height, - ) = zebra_consensus::router::init(zebra_consensus::Config::default(), &network, state).await; + ) = zebra_consensus::router::init_test(zebra_consensus::Config::default(), &network, state) + .await; let genesis_hash = block_verifier_router .oneshot(zebra_consensus::Request::Commit(regtest_genesis_block())) @@ -3310,8 +3311,12 @@ async fn nu6_funding_streams_and_coinbase_balance() -> Result<()> { _transaction_verifier, _parameter_download_task_handle, _max_checkpoint_height, - ) = zebra_consensus::router::init(zebra_consensus::Config::default(), &network, state.clone()) - .await; + ) = zebra_consensus::router::init_test( + zebra_consensus::Config::default(), + &network, + state.clone(), + ) + .await; tracing::info!("started state service and block verifier, committing Regtest genesis block"); From 85fdd88d8452949637e76d9a2bc4e0884edc1873 Mon Sep 17 00:00:00 2001 From: Arya Date: Tue, 3 Sep 2024 22:40:19 -0400 Subject: [PATCH 02/69] Adds a `mempool` argument to the transaction::Verifier constructor (and a `new_for_tests()` constructor for convenience) --- zebra-consensus/src/block/tests.rs | 2 +- zebra-consensus/src/router.rs | 7 +- zebra-consensus/src/transaction.rs | 16 +++- zebra-consensus/src/transaction/tests.rs | 78 +++++++++---------- zebra-consensus/src/transaction/tests/prop.rs | 2 +- 5 files changed, 57 insertions(+), 48 deletions(-) diff --git a/zebra-consensus/src/block/tests.rs b/zebra-consensus/src/block/tests.rs index d8f74bf7b2f..97191266ccd 100644 --- a/zebra-consensus/src/block/tests.rs +++ b/zebra-consensus/src/block/tests.rs @@ -137,7 +137,7 @@ async fn check_transcripts() -> Result<(), Report> { let network = Network::Mainnet; let state_service = zebra_state::init_test(&network); - let transaction = transaction::Verifier::new(&network, state_service.clone()); + let transaction = transaction::Verifier::new_for_tests(&network, state_service.clone()); let transaction = Buffer::new(BoxService::new(transaction), 1); let block_verifier = Buffer::new( SemanticBlockVerifier::new(&network, state_service.clone(), transaction), diff --git a/zebra-consensus/src/router.rs b/zebra-consensus/src/router.rs index 9f5311d7f43..169fb32affb 100644 --- a/zebra-consensus/src/router.rs +++ b/zebra-consensus/src/router.rs @@ -30,7 +30,6 @@ use zebra_chain::{ parameters::Network, }; -use zebra_node_services::mempool; use zebra_state as zs; use crate::{ @@ -236,9 +235,7 @@ pub async fn init( config: Config, network: &Network, mut state_service: S, - mempool: oneshot::Receiver< - Buffer, mempool::Request>, - >, + mempool: oneshot::Receiver, ) -> ( Buffer, Request>, Buffer< @@ -337,7 +334,7 @@ where // transaction verification - let transaction = transaction::Verifier::new(network, state_service.clone()); + let transaction = transaction::Verifier::new(network, state_service.clone(), mempool); let transaction = Buffer::new(BoxService::new(transaction), VERIFIER_BUFFER_BOUND); // block verification diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 1c303003615..ea328ad3c6d 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -13,7 +13,8 @@ use futures::{ stream::{FuturesUnordered, StreamExt}, FutureExt, }; -use tower::{timeout::Timeout, Service, ServiceExt}; +use tokio::sync::oneshot; +use tower::{buffer::Buffer, timeout::Timeout, util::BoxService, Service, ServiceExt}; use tracing::Instrument; use zebra_chain::{ @@ -29,6 +30,7 @@ use zebra_chain::{ transparent::{self, OrderedUtxo}, }; +use zebra_node_services::mempool; use zebra_script::CachedFfiTransaction; use zebra_state as zs; @@ -52,6 +54,10 @@ mod tests; /// chain in the correct order.) const UTXO_LOOKUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(6 * 60); +/// Mempool service type used by the transaction verifier +pub type MempoolService = + Buffer, mempool::Request>; + /// Asynchronous transaction verification. /// /// # Correctness @@ -72,13 +78,19 @@ where ZS::Future: Send + 'static, { /// Create a new transaction verifier. - pub fn new(network: &Network, state: ZS) -> Self { + pub fn new(network: &Network, state: ZS, mempool: oneshot::Receiver) -> Self { Self { network: network.clone(), state: Timeout::new(state, UTXO_LOOKUP_TIMEOUT), script_verifier: script::Verifier, } } + + /// Create a new transaction verifier with a closed channel receiver for mempool setup for tests. + #[cfg(test)] + pub fn new_for_tests(network: &Network, state: ZS) -> Self { + Self::new(network, state, oneshot::channel().1) + } } /// Specifies whether a transaction should be verified as part of a block or as diff --git a/zebra-consensus/src/transaction/tests.rs b/zebra-consensus/src/transaction/tests.rs index 0a4c21bb039..3aee0f40959 100644 --- a/zebra-consensus/src/transaction/tests.rs +++ b/zebra-consensus/src/transaction/tests.rs @@ -181,7 +181,7 @@ fn v5_transaction_with_no_inputs_fails_validation() { #[tokio::test] async fn mempool_request_with_missing_input_is_rejected() { let mut state: MockService<_, _, _, _> = MockService::build().for_prop_tests(); - let verifier = Verifier::new(&Network::Mainnet, state.clone()); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state.clone()); let (height, tx) = transactions_from_blocks(zebra_test::vectors::MAINNET_BLOCKS.iter()) .find(|(_, tx)| !(tx.is_coinbase() || tx.inputs().is_empty())) @@ -230,7 +230,7 @@ async fn mempool_request_with_missing_input_is_rejected() { #[tokio::test] async fn mempool_request_with_present_input_is_accepted() { let mut state: MockService<_, _, _, _> = MockService::build().for_prop_tests(); - let verifier = Verifier::new(&Network::Mainnet, state.clone()); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state.clone()); let height = NetworkUpgrade::Canopy .activation_height(&Network::Mainnet) @@ -297,7 +297,7 @@ async fn mempool_request_with_present_input_is_accepted() { #[tokio::test] async fn mempool_request_with_invalid_lock_time_is_rejected() { let mut state: MockService<_, _, _, _> = MockService::build().for_prop_tests(); - let verifier = Verifier::new(&Network::Mainnet, state.clone()); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state.clone()); let height = NetworkUpgrade::Canopy .activation_height(&Network::Mainnet) @@ -376,7 +376,7 @@ async fn mempool_request_with_invalid_lock_time_is_rejected() { #[tokio::test] async fn mempool_request_with_unlocked_lock_time_is_accepted() { let mut state: MockService<_, _, _, _> = MockService::build().for_prop_tests(); - let verifier = Verifier::new(&Network::Mainnet, state.clone()); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state.clone()); let height = NetworkUpgrade::Canopy .activation_height(&Network::Mainnet) @@ -443,7 +443,7 @@ async fn mempool_request_with_unlocked_lock_time_is_accepted() { #[tokio::test] async fn mempool_request_with_lock_time_max_sequence_number_is_accepted() { let mut state: MockService<_, _, _, _> = MockService::build().for_prop_tests(); - let verifier = Verifier::new(&Network::Mainnet, state.clone()); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state.clone()); let height = NetworkUpgrade::Canopy .activation_height(&Network::Mainnet) @@ -513,7 +513,7 @@ async fn mempool_request_with_lock_time_max_sequence_number_is_accepted() { #[tokio::test] async fn mempool_request_with_past_lock_time_is_accepted() { let mut state: MockService<_, _, _, _> = MockService::build().for_prop_tests(); - let verifier = Verifier::new(&Network::Mainnet, state.clone()); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state.clone()); let height = NetworkUpgrade::Canopy .activation_height(&Network::Mainnet) @@ -592,7 +592,7 @@ async fn mempool_request_with_immature_spend_is_rejected() { let _init_guard = zebra_test::init(); let mut state: MockService<_, _, _, _> = MockService::build().for_prop_tests(); - let verifier = Verifier::new(&Network::Mainnet, state.clone()); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state.clone()); let height = NetworkUpgrade::Canopy .activation_height(&Network::Mainnet) @@ -695,7 +695,7 @@ async fn state_error_converted_correctly() { use zebra_state::DuplicateNullifierError; let mut state: MockService<_, _, _, _> = MockService::build().for_prop_tests(); - let verifier = Verifier::new(&Network::Mainnet, state.clone()); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state.clone()); let height = NetworkUpgrade::Canopy .activation_height(&Network::Mainnet) @@ -856,7 +856,7 @@ async fn v5_transaction_is_rejected_before_nu5_activation() { for network in Network::iter() { let state_service = service_fn(|_| async { unreachable!("Service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let transaction = fake_v5_transactions_for_network(&network, network.block_iter()) .next_back() @@ -903,7 +903,7 @@ fn v5_transaction_is_accepted_after_nu5_activation_for_network(network: Network) let blocks = network.block_iter(); let state_service = service_fn(|_| async { unreachable!("Service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let mut transaction = fake_v5_transactions_for_network(&network, blocks) .next_back() @@ -975,7 +975,7 @@ async fn v4_transaction_with_transparent_transfer_is_accepted() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let result = verifier .oneshot(Request::Block { @@ -998,7 +998,7 @@ async fn v4_transaction_with_transparent_transfer_is_accepted() { async fn v4_transaction_with_last_valid_expiry_height() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&Network::Mainnet, state_service); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state_service); let block_height = NetworkUpgrade::Canopy .activation_height(&Network::Mainnet) @@ -1045,7 +1045,7 @@ async fn v4_transaction_with_last_valid_expiry_height() { async fn v4_coinbase_transaction_with_low_expiry_height() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&Network::Mainnet, state_service); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state_service); let block_height = NetworkUpgrade::Canopy .activation_height(&Network::Mainnet) @@ -1086,7 +1086,7 @@ async fn v4_coinbase_transaction_with_low_expiry_height() { async fn v4_transaction_with_too_low_expiry_height() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&Network::Mainnet, state_service); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state_service); let block_height = NetworkUpgrade::Canopy .activation_height(&Network::Mainnet) @@ -1138,7 +1138,7 @@ async fn v4_transaction_with_too_low_expiry_height() { async fn v4_transaction_with_exceeding_expiry_height() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&Network::Mainnet, state_service); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state_service); let block_height = block::Height::MAX; @@ -1189,7 +1189,7 @@ async fn v4_transaction_with_exceeding_expiry_height() { async fn v4_coinbase_transaction_with_exceeding_expiry_height() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&Network::Mainnet, state_service); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state_service); // Use an arbitrary pre-NU5 block height. // It can't be NU5-onward because the expiry height limit is not enforced @@ -1265,7 +1265,7 @@ async fn v4_coinbase_transaction_is_accepted() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let result = verifier .oneshot(Request::Block { @@ -1320,7 +1320,7 @@ async fn v4_transaction_with_transparent_transfer_is_rejected_by_the_script() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let result = verifier .oneshot(Request::Block { @@ -1375,7 +1375,7 @@ async fn v4_transaction_with_conflicting_transparent_spend_is_rejected() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let result = verifier .oneshot(Request::Block { @@ -1446,7 +1446,7 @@ fn v4_transaction_with_conflicting_sprout_nullifier_inside_joinsplit_is_rejected let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let result = verifier .oneshot(Request::Block { @@ -1522,7 +1522,7 @@ fn v4_transaction_with_conflicting_sprout_nullifier_across_joinsplits_is_rejecte let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let result = verifier .oneshot(Request::Block { @@ -1581,7 +1581,7 @@ async fn v5_transaction_with_transparent_transfer_is_accepted() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let result = verifier .oneshot(Request::Block { @@ -1605,7 +1605,7 @@ async fn v5_transaction_with_last_valid_expiry_height() { let network = Network::new_default_testnet(); let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let block_height = NetworkUpgrade::Nu5 .activation_height(&network) @@ -1651,7 +1651,7 @@ async fn v5_coinbase_transaction_expiry_height() { let network = Network::new_default_testnet(); let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let block_height = NetworkUpgrade::Nu5 .activation_height(&network) @@ -1768,7 +1768,7 @@ async fn v5_transaction_with_too_low_expiry_height() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let block_height = NetworkUpgrade::Nu5 .activation_height(&network) @@ -1820,7 +1820,7 @@ async fn v5_transaction_with_too_low_expiry_height() { async fn v5_transaction_with_exceeding_expiry_height() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&Network::Mainnet, state_service); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state_service); let block_height = block::Height::MAX; @@ -1898,7 +1898,7 @@ async fn v5_coinbase_transaction_is_accepted() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let result = verifier .oneshot(Request::Block { @@ -1955,7 +1955,7 @@ async fn v5_transaction_with_transparent_transfer_is_rejected_by_the_script() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let result = verifier .oneshot(Request::Block { @@ -2012,7 +2012,7 @@ async fn v5_transaction_with_conflicting_transparent_spend_is_rejected() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); let result = verifier .oneshot(Request::Block { @@ -2055,7 +2055,7 @@ fn v4_with_signed_sprout_transfer_is_accepted() { // Initialize the verifier let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); // Test the transaction verifier let result = verifier @@ -2135,7 +2135,7 @@ async fn v4_with_joinsplit_is_rejected_for_modification( // Initialize the verifier let state_service = service_fn(|_| async { unreachable!("State service should not be called.") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); // Test the transaction verifier. // @@ -2186,7 +2186,7 @@ fn v4_with_sapling_spends() { // Initialize the verifier let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); // Test the transaction verifier let result = verifier @@ -2229,7 +2229,7 @@ fn v4_with_duplicate_sapling_spends() { // Initialize the verifier let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); // Test the transaction verifier let result = verifier @@ -2274,7 +2274,7 @@ fn v4_with_sapling_outputs_and_no_spends() { // Initialize the verifier let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); // Test the transaction verifier let result = verifier @@ -2323,7 +2323,7 @@ fn v5_with_sapling_spends() { // Initialize the verifier let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); // Test the transaction verifier let result = verifier @@ -2367,7 +2367,7 @@ fn v5_with_duplicate_sapling_spends() { // Initialize the verifier let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); // Test the transaction verifier let result = verifier @@ -2430,7 +2430,7 @@ fn v5_with_duplicate_orchard_action() { // Initialize the verifier let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = Verifier::new(&network, state_service); + let verifier = Verifier::new_for_tests(&network, state_service); // Test the transaction verifier let result = verifier @@ -2933,7 +2933,7 @@ fn shielded_outputs_are_not_decryptable_for_fake_v5_blocks() { #[tokio::test] async fn mempool_zip317_error() { let mut state: MockService<_, _, _, _> = MockService::build().for_prop_tests(); - let verifier = Verifier::new(&Network::Mainnet, state.clone()); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state.clone()); let height = NetworkUpgrade::Nu5 .activation_height(&Network::Mainnet) @@ -3005,7 +3005,7 @@ async fn mempool_zip317_error() { #[tokio::test] async fn mempool_zip317_ok() { let mut state: MockService<_, _, _, _> = MockService::build().for_prop_tests(); - let verifier = Verifier::new(&Network::Mainnet, state.clone()); + let verifier = Verifier::new_for_tests(&Network::Mainnet, state.clone()); let height = NetworkUpgrade::Nu5 .activation_height(&Network::Mainnet) diff --git a/zebra-consensus/src/transaction/tests/prop.rs b/zebra-consensus/src/transaction/tests/prop.rs index f45b4731de0..24b4cd2c982 100644 --- a/zebra-consensus/src/transaction/tests/prop.rs +++ b/zebra-consensus/src/transaction/tests/prop.rs @@ -450,7 +450,7 @@ fn validate( // Initialize the verifier let state_service = tower::service_fn(|_| async { unreachable!("State service should not be called") }); - let verifier = transaction::Verifier::new(&network, state_service); + let verifier = transaction::Verifier::new_for_tests(&network, state_service); // Test the transaction verifier verifier From 174f4ea280c776e3d09760b1584496b8db1f94e7 Mon Sep 17 00:00:00 2001 From: Arya Date: Tue, 3 Sep 2024 22:54:25 -0400 Subject: [PATCH 03/69] Removes `Clone` impl on `transaction::Verifier` to add mempool oneshot receiver, updates tests. --- zebra-consensus/src/transaction.rs | 11 +++++-- zebra-consensus/src/transaction/tests.rs | 29 ++++++++++++------- zebra-consensus/src/transaction/tests/prop.rs | 7 ++++- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index ea328ad3c6d..8d6bcff63cb 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -65,11 +65,12 @@ pub type MempoolService = /// Transaction verification requests should be wrapped in a timeout, so that /// out-of-order and invalid requests do not hang indefinitely. See the [`router`](`crate::router`) /// module documentation for details. -#[derive(Debug, Clone)] pub struct Verifier { network: Network, state: Timeout, + mempool: Option, script_verifier: script::Verifier, + mempool_setup_rx: oneshot::Receiver, } impl Verifier @@ -78,11 +79,17 @@ where ZS::Future: Send + 'static, { /// Create a new transaction verifier. - pub fn new(network: &Network, state: ZS, mempool: oneshot::Receiver) -> Self { + pub fn new( + network: &Network, + state: ZS, + mempool_setup_rx: oneshot::Receiver, + ) -> Self { Self { network: network.clone(), state: Timeout::new(state, UTXO_LOOKUP_TIMEOUT), + mempool: None, script_verifier: script::Verifier, + mempool_setup_rx, } } diff --git a/zebra-consensus/src/transaction/tests.rs b/zebra-consensus/src/transaction/tests.rs index 3aee0f40959..a463565f252 100644 --- a/zebra-consensus/src/transaction/tests.rs +++ b/zebra-consensus/src/transaction/tests.rs @@ -7,7 +7,7 @@ use std::{collections::HashMap, sync::Arc}; use chrono::{DateTime, TimeZone, Utc}; use color_eyre::eyre::Report; use halo2::pasta::{group::ff::PrimeField, pallas}; -use tower::{service_fn, ServiceExt}; +use tower::{buffer::Buffer, service_fn, ServiceExt}; use zebra_chain::{ amount::{Amount, NonNegative}, @@ -1652,6 +1652,7 @@ async fn v5_coinbase_transaction_expiry_height() { let state_service = service_fn(|_| async { unreachable!("State service should not be called") }); let verifier = Verifier::new_for_tests(&network, state_service); + let verifier = Buffer::new(verifier, 10); let block_height = NetworkUpgrade::Nu5 .activation_height(&network) @@ -1701,7 +1702,11 @@ async fn v5_coinbase_transaction_expiry_height() { height: block_height, time: DateTime::::MAX_UTC, }) - .await; + .await + .map_err(|err| { + *err.downcast() + .expect("error type should be TransactionError") + }); assert_eq!( result, @@ -1726,7 +1731,11 @@ async fn v5_coinbase_transaction_expiry_height() { height: block_height, time: DateTime::::MAX_UTC, }) - .await; + .await + .map_err(|err| { + *err.downcast() + .expect("error type should be TransactionError") + }); assert_eq!( result, @@ -2059,7 +2068,6 @@ fn v4_with_signed_sprout_transfer_is_accepted() { // Test the transaction verifier let result = verifier - .clone() .oneshot(Request::Block { transaction, known_utxos: Arc::new(HashMap::new()), @@ -2136,6 +2144,7 @@ async fn v4_with_joinsplit_is_rejected_for_modification( let state_service = service_fn(|_| async { unreachable!("State service should not be called.") }); let verifier = Verifier::new_for_tests(&network, state_service); + let verifier = Buffer::new(verifier, 10); // Test the transaction verifier. // @@ -2154,7 +2163,11 @@ async fn v4_with_joinsplit_is_rejected_for_modification( height, time: DateTime::::MAX_UTC, }) - .await; + .await + .map_err(|err| { + *err.downcast() + .expect("error type should be TransactionError") + }); if result == expected_error || i >= 100 { break result; @@ -2190,7 +2203,6 @@ fn v4_with_sapling_spends() { // Test the transaction verifier let result = verifier - .clone() .oneshot(Request::Block { transaction, known_utxos: Arc::new(HashMap::new()), @@ -2233,7 +2245,6 @@ fn v4_with_duplicate_sapling_spends() { // Test the transaction verifier let result = verifier - .clone() .oneshot(Request::Block { transaction, known_utxos: Arc::new(HashMap::new()), @@ -2278,7 +2289,6 @@ fn v4_with_sapling_outputs_and_no_spends() { // Test the transaction verifier let result = verifier - .clone() .oneshot(Request::Block { transaction, known_utxos: Arc::new(HashMap::new()), @@ -2327,7 +2337,6 @@ fn v5_with_sapling_spends() { // Test the transaction verifier let result = verifier - .clone() .oneshot(Request::Block { transaction: Arc::new(transaction), known_utxos: Arc::new(HashMap::new()), @@ -2371,7 +2380,6 @@ fn v5_with_duplicate_sapling_spends() { // Test the transaction verifier let result = verifier - .clone() .oneshot(Request::Block { transaction: Arc::new(transaction), known_utxos: Arc::new(HashMap::new()), @@ -2434,7 +2442,6 @@ fn v5_with_duplicate_orchard_action() { // Test the transaction verifier let result = verifier - .clone() .oneshot(Request::Block { transaction: Arc::new(transaction), known_utxos: Arc::new(HashMap::new()), diff --git a/zebra-consensus/src/transaction/tests/prop.rs b/zebra-consensus/src/transaction/tests/prop.rs index 24b4cd2c982..856742e5d74 100644 --- a/zebra-consensus/src/transaction/tests/prop.rs +++ b/zebra-consensus/src/transaction/tests/prop.rs @@ -4,7 +4,7 @@ use std::{collections::HashMap, sync::Arc}; use chrono::{DateTime, Duration, Utc}; use proptest::{collection::vec, prelude::*}; -use tower::ServiceExt; +use tower::{buffer::Buffer, ServiceExt}; use zebra_chain::{ amount::Amount, @@ -451,6 +451,7 @@ fn validate( let state_service = tower::service_fn(|_| async { unreachable!("State service should not be called") }); let verifier = transaction::Verifier::new_for_tests(&network, state_service); + let verifier = Buffer::new(verifier, 10); // Test the transaction verifier verifier @@ -462,5 +463,9 @@ fn validate( time: block_time, }) .await + .map_err(|err| { + *err.downcast() + .expect("error type should be TransactionError") + }) }) } From 3c05b035a395eaa40073c82d26188ce85a2d7dc9 Mon Sep 17 00:00:00 2001 From: Arya Date: Tue, 3 Sep 2024 23:02:05 -0400 Subject: [PATCH 04/69] Adds TODOs --- zebra-consensus/src/transaction.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 8d6bcff63cb..c03b41c219a 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -68,6 +68,7 @@ pub type MempoolService = pub struct Verifier { network: Network, state: Timeout, + // TODO: Use an enum so that this can either be Pending(oneshot::Receiver) or Initialized(MempoolService) mempool: Option, script_verifier: script::Verifier, mempool_setup_rx: oneshot::Receiver, @@ -485,6 +486,9 @@ where legacy_sigop_count, }, Request::Mempool { transaction, .. } => { + // TODO: Poll the mempool so it sees the new verified result promptly / nearly-immediately + // (to solve concurrency issue with dependency chains of orphaned transactions) + let transaction = VerifiedUnminedTx::new( transaction, miner_fee.expect( From 77aa2e325efc0ec06d753aad30bbe1993009266d Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 6 Sep 2024 20:34:50 -0400 Subject: [PATCH 05/69] updates transaction verifier's poll_ready() method to setup the mempool service handle. --- zebra-consensus/src/transaction.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index c03b41c219a..9db03cd024a 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -307,6 +307,14 @@ where Pin> + Send + 'static>>; fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + // Note: The block verifier expects the transaction verifier to always be ready. + + if self.mempool.is_none() { + if let Ok(mempool) = self.mempool_setup_rx.try_recv() { + self.mempool = Some(mempool); + } + } + Poll::Ready(Ok(())) } From 5c7f45b8b8215bab265f29bc743925443041acc4 Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 6 Sep 2024 21:01:17 -0400 Subject: [PATCH 06/69] Updates VerifiedSet struct used in mempool storage --- .../mempool/storage/verified_set.rs | 97 +++++++++++-------- 1 file changed, 57 insertions(+), 40 deletions(-) diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index a9c850b4ef8..7d2232f7397 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -2,7 +2,7 @@ use std::{ borrow::Cow, - collections::{HashSet, VecDeque}, + collections::{HashMap, HashSet}, hash::Hash, }; @@ -18,11 +18,24 @@ use super::super::SameEffectsTipRejectionError; #[allow(unused_imports)] use zebra_chain::transaction::MEMPOOL_TRANSACTION_COST_THRESHOLD; +#[derive(Default)] +struct TransactionDependencies { + /// Lists of mempool transactions that create UTXOs spent + /// by a mempool transaction. + dependencies: HashMap>, + + /// Lists of mempool transactions that spend UTXOs created + /// by a mempool transaction. + dependants: HashMap>, +} + /// The set of verified transactions stored in the mempool. /// /// This also caches the all the spent outputs from the transactions in the mempool. The spent /// outputs include: /// +/// - the dependencies of transactions that spent the outputs of other transactions in the mempool +/// - the UTXOs of transactions in the mempool /// - the transparent outpoints spent by transactions in the mempool /// - the Sprout nullifiers revealed by transactions in the mempool /// - the Sapling nullifiers revealed by transactions in the mempool @@ -30,7 +43,20 @@ use zebra_chain::transaction::MEMPOOL_TRANSACTION_COST_THRESHOLD; #[derive(Default)] pub struct VerifiedSet { /// The set of verified transactions in the mempool. - transactions: VecDeque, + transactions: HashMap, + + /// A map of mempool transaction dependencies. + transaction_dependencies: TransactionDependencies, + + /// A map of mempool transaction dependants. + // TODO: Unify this field with `transaction_dependencies` + transaction_dependants: HashMap>, + + /// The [`transparent::Utxo`]s created by verified transactions in the mempool + /// + /// Note that these UTXOs may not be unspent. + /// Outputs can be spent by later transactions or blocks in the chain. + created_utxos: HashMap, /// The total size of the transactions in the mempool if they were /// serialized. @@ -60,20 +86,9 @@ impl Drop for VerifiedSet { } impl VerifiedSet { - /// Returns an iterator over the [`UnminedTx`] in the set. - // - // TODO: make the transactions() method return VerifiedUnminedTx, - // and remove the full_transactions() method - pub fn transactions(&self) -> impl Iterator + '_ { - self.transactions.iter().map(|tx| &tx.transaction) - } - - /// Returns an iterator over the [`VerifiedUnminedTx`] in the set. - /// - /// Each [`VerifiedUnminedTx`] contains an [`UnminedTx`], - /// and adds extra fields from the transaction verifier result. - pub fn full_transactions(&self) -> impl Iterator + '_ { - self.transactions.iter() + /// Returns a reference to the [`HashMap`] of [`VerifiedUnminedTx`]s in the set. + pub fn transactions(&self) -> &HashMap { + &self.transactions } /// Returns the number of verified transactions in the set. @@ -99,7 +114,7 @@ impl VerifiedSet { /// Returns `true` if the set of verified transactions contains the transaction with the /// specified [`UnminedTxId`]. pub fn contains(&self, id: &UnminedTxId) -> bool { - self.transactions.iter().any(|tx| &tx.transaction.id == id) + self.transactions.contains_key(id) } /// Clear the set of verified transactions. @@ -131,10 +146,13 @@ impl VerifiedSet { return Err(SameEffectsTipRejectionError::SpendConflict); } + // TODO: Update `created_utxos` and `transaction_dependencies` + self.cache_outputs_from(&transaction.transaction.transaction); self.transactions_serialized_size += transaction.transaction.size; self.total_cost += transaction.cost(); - self.transactions.push_front(transaction); + self.transactions + .insert(transaction.transaction.id, transaction); self.update_metrics(); @@ -168,16 +186,20 @@ impl VerifiedSet { use rand::distributions::{Distribution, WeightedIndex}; use rand::prelude::thread_rng; - let weights: Vec = self + let (keys, weights): (Vec, Vec) = self .transactions .iter() - .map(|tx| tx.clone().eviction_weight()) - .collect(); + .map(|(&tx_id, tx)| (tx_id, tx.eviction_weight())) + .unzip(); let dist = WeightedIndex::new(weights) .expect("there is at least one weight, all weights are non-negative, and the total is positive"); - Some(self.remove(dist.sample(&mut thread_rng()))) + let key_to_remove = keys + .get(dist.sample(&mut thread_rng())) + .expect("should have a key at every index in the distribution"); + + Some(self.remove(key_to_remove)) } } @@ -185,25 +207,16 @@ impl VerifiedSet { /// /// Returns the amount of transactions removed. pub fn remove_all_that(&mut self, predicate: impl Fn(&VerifiedUnminedTx) -> bool) -> usize { - // Clippy suggests to remove the `collect` and the `into_iter` further down. However, it is - // unable to detect that when that is done, there is a borrow conflict. What happens is the - // iterator borrows `self.transactions` immutably, but it also need to be borrowed mutably - // in order to remove the transactions while traversing the iterator. - #[allow(clippy::needless_collect)] - let indices_to_remove: Vec<_> = self + let keys_to_remove: Vec<_> = self .transactions .iter() - .enumerate() - .filter(|(_, tx)| predicate(tx)) - .map(|(index, _)| index) + .filter_map(|(&tx_id, tx)| predicate(tx).then_some(tx_id)) .collect(); - let removed_count = indices_to_remove.len(); + let removed_count = keys_to_remove.len(); - // Correctness: remove indexes in reverse order, - // so earlier indexes still correspond to the same transactions - for index_to_remove in indices_to_remove.into_iter().rev() { - self.remove(index_to_remove); + for key_to_remove in keys_to_remove { + self.remove(&key_to_remove); } removed_count @@ -212,11 +225,15 @@ impl VerifiedSet { /// Removes a transaction from the set. /// /// Also removes its outputs from the internal caches. - fn remove(&mut self, transaction_index: usize) -> VerifiedUnminedTx { + fn remove(&mut self, key_to_remove: &UnminedTxId) -> VerifiedUnminedTx { + // TODO: + // - Remove any dependant transactions as well + // - Update the `created_utxos` + let removed_tx = self .transactions - .remove(transaction_index) - .expect("invalid transaction index"); + .remove(key_to_remove) + .expect("invalid transaction key"); self.transactions_serialized_size -= removed_tx.transaction.size; self.total_cost -= removed_tx.cost(); @@ -308,7 +325,7 @@ impl VerifiedSet { let mut size_with_weight_gt2 = 0; let mut size_with_weight_gt3 = 0; - for entry in self.full_transactions() { + for entry in self.transactions().values() { paid_actions += entry.conventional_actions - entry.unpaid_actions; if entry.fee_weight_ratio > 3.0 { From ecc2dddf79902513074613c95f1ae575dfee858c Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 6 Sep 2024 21:11:15 -0400 Subject: [PATCH 07/69] Updates mempool service and its `Storage` to use the updated `VerifiedSet` `transactions()` return type. --- zebrad/src/components/mempool.rs | 7 +++- zebrad/src/components/mempool/storage.rs | 53 ++++++++++-------------- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index 05732ddaac2..e77d48c0bf6 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -132,7 +132,10 @@ impl ActiveState { } => { let mut transactions = Vec::new(); - let storage = storage.transactions().map(|tx| tx.clone().into()); + let storage = storage + .transactions() + .values() + .map(|tx| tx.transaction.clone().into()); transactions.extend(storage); let pending = tx_downloads.transaction_requests().cloned(); @@ -732,7 +735,7 @@ impl Service for Mempool { Request::FullTransactions => { trace!(?req, "got mempool request"); - let transactions: Vec<_> = storage.full_transactions().cloned().collect(); + let transactions: Vec<_> = storage.transactions().values().cloned().collect(); trace!(?req, transactions_count = ?transactions.len(), "answered mempool request"); diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index d380efb84aa..3e5423e5e67 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -327,23 +327,21 @@ impl Storage { let duplicate_spend_ids: HashSet<_> = self .verified .transactions() - .filter_map(|tx| { - (tx.transaction - .spent_outpoints() + .values() + .map(|tx| (tx.transaction.id, &tx.transaction.transaction)) + .filter_map(|(tx_id, tx)| { + (tx.spent_outpoints() .any(|outpoint| spent_outpoints.contains(&outpoint)) || tx - .transaction .sprout_nullifiers() .any(|nullifier| sprout_nullifiers.contains(nullifier)) || tx - .transaction .sapling_nullifiers() .any(|nullifier| sapling_nullifiers.contains(nullifier)) || tx - .transaction .orchard_nullifiers() .any(|nullifier| orchard_nullifiers.contains(nullifier))) - .then_some(tx.id) + .then_some(tx_id) }) .collect(); @@ -407,24 +405,15 @@ impl Storage { /// Returns the set of [`UnminedTxId`]s in the mempool. pub fn tx_ids(&self) -> impl Iterator + '_ { - self.verified.transactions().map(|tx| tx.id) + self.transactions().values().map(|tx| tx.transaction.id) } - /// Returns an iterator over the [`UnminedTx`]s in the mempool. - // - // TODO: make the transactions() method return VerifiedUnminedTx, - // and remove the full_transactions() method - pub fn transactions(&self) -> impl Iterator { - self.verified.transactions() - } - - /// Returns an iterator over the [`VerifiedUnminedTx`] in the set. + /// Returns a reference to the [`HashMap`] of [`VerifiedUnminedTx`]s in the verified set. /// /// Each [`VerifiedUnminedTx`] contains an [`UnminedTx`], /// and adds extra fields from the transaction verifier result. - #[allow(dead_code)] - pub fn full_transactions(&self) -> impl Iterator + '_ { - self.verified.full_transactions() + pub fn transactions(&self) -> &HashMap { + self.verified.transactions() } /// Returns the number of transactions in the mempool. @@ -455,9 +444,9 @@ impl Storage { &self, tx_ids: HashSet, ) -> impl Iterator { - self.verified - .transactions() - .filter(move |tx| tx_ids.contains(&tx.id)) + tx_ids + .into_iter() + .filter_map(|tx_id| self.transactions().get(&tx_id).map(|tx| &tx.transaction)) } /// Returns the set of [`UnminedTx`]es with matching [`transaction::Hash`]es @@ -471,7 +460,9 @@ impl Storage { ) -> impl Iterator { self.verified .transactions() - .filter(move |tx| tx_ids.contains(&tx.id.mined_id())) + .iter() + .filter(move |(tx_id, _)| tx_ids.contains(&tx_id.mined_id())) + .map(|(_, tx)| &tx.transaction) } /// Returns `true` if a transaction exactly matching an [`UnminedTxId`] is in @@ -480,7 +471,7 @@ impl Storage { /// This matches the exact transaction, with identical blockchain effects, /// signatures, and proofs. pub fn contains_transaction_exact(&self, txid: &UnminedTxId) -> bool { - self.verified.transactions().any(|tx| &tx.id == txid) + self.verified.contains(txid) } /// Returns the number of rejected [`UnminedTxId`]s or [`transaction::Hash`]es. @@ -611,11 +602,11 @@ impl Storage { // then extracts the mined ID out of it let mut unmined_id_set = HashSet::new(); - for t in self.transactions() { - if let Some(expiry_height) = t.transaction.expiry_height() { + for (&tx_id, tx) in self.transactions() { + if let Some(expiry_height) = tx.transaction.transaction.expiry_height() { if tip_height >= expiry_height { - txid_set.insert(t.id.mined_id()); - unmined_id_set.insert(t.id); + txid_set.insert(tx_id.mined_id()); + unmined_id_set.insert(tx_id); } } } @@ -625,8 +616,8 @@ impl Storage { .remove_all_that(|tx| txid_set.contains(&tx.transaction.id.mined_id())); // also reject it - for id in unmined_id_set.iter() { - self.reject(*id, SameEffectsChainRejectionError::Expired.into()); + for &id in &unmined_id_set { + self.reject(id, SameEffectsChainRejectionError::Expired.into()); } unmined_id_set From e60197c59cd46b5900a7bfbe30aaee98abef5612 Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 6 Sep 2024 21:44:18 -0400 Subject: [PATCH 08/69] updates `created_outputs` when inserting or removing a transaction from the mempool's verified set --- .../mempool/storage/verified_set.rs | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 7d2232f7397..14168ffb91c 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -48,15 +48,11 @@ pub struct VerifiedSet { /// A map of mempool transaction dependencies. transaction_dependencies: TransactionDependencies, - /// A map of mempool transaction dependants. - // TODO: Unify this field with `transaction_dependencies` - transaction_dependants: HashMap>, - /// The [`transparent::Utxo`]s created by verified transactions in the mempool /// /// Note that these UTXOs may not be unspent. /// Outputs can be spent by later transactions or blocks in the chain. - created_utxos: HashMap, + created_outputs: HashMap, /// The total size of the transactions in the mempool if they were /// serialized. @@ -146,13 +142,12 @@ impl VerifiedSet { return Err(SameEffectsTipRejectionError::SpendConflict); } - // TODO: Update `created_utxos` and `transaction_dependencies` - - self.cache_outputs_from(&transaction.transaction.transaction); + // TODO: Update `transaction_dependencies` + let tx_id = transaction.transaction.id; + self.cache_outputs_from(tx_id, &transaction.transaction.transaction); self.transactions_serialized_size += transaction.transaction.size; self.total_cost += transaction.cost(); - self.transactions - .insert(transaction.transaction.id, transaction); + self.transactions.insert(tx_id, transaction); self.update_metrics(); @@ -226,9 +221,7 @@ impl VerifiedSet { /// /// Also removes its outputs from the internal caches. fn remove(&mut self, key_to_remove: &UnminedTxId) -> VerifiedUnminedTx { - // TODO: - // - Remove any dependant transactions as well - // - Update the `created_utxos` + // TODO: Remove any dependant transactions as well let removed_tx = self .transactions @@ -259,7 +252,14 @@ impl VerifiedSet { } /// Inserts the transaction's outputs into the internal caches. - fn cache_outputs_from(&mut self, tx: &Transaction) { + fn cache_outputs_from(&mut self, tx_id: UnminedTxId, tx: &Transaction) { + for (index, output) in tx.outputs().iter().cloned().enumerate() { + self.created_outputs.insert( + transparent::OutPoint::from_usize(tx_id.mined_id(), index), + output, + ); + } + self.spent_outpoints.extend(tx.spent_outpoints()); self.sprout_nullifiers.extend(tx.sprout_nullifiers()); self.sapling_nullifiers.extend(tx.sapling_nullifiers()); @@ -270,6 +270,14 @@ impl VerifiedSet { fn remove_outputs(&mut self, unmined_tx: &UnminedTx) { let tx = &unmined_tx.transaction; + for index in 0..tx.outputs().len() { + self.created_outputs + .remove(&transparent::OutPoint::from_usize( + unmined_tx.id.mined_id(), + index, + )); + } + let spent_outpoints = tx.spent_outpoints().map(Cow::Owned); let sprout_nullifiers = tx.sprout_nullifiers().map(Cow::Borrowed); let sapling_nullifiers = tx.sapling_nullifiers().map(Cow::Borrowed); From 745e6a7836f2eb0a26ec07cbc0ff5aa2577d1c87 Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 6 Sep 2024 21:49:15 -0400 Subject: [PATCH 09/69] Adds a TODO, updates field docs --- zebra-consensus/src/transaction.rs | 2 ++ zebrad/src/components/mempool/storage/verified_set.rs | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 9db03cd024a..a758c7e2523 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -552,6 +552,8 @@ where /// /// Returns a tuple with a OutPoint -> Utxo map, and a vector of Outputs /// in the same order as the matching inputs in the transaction. + // TODO: Add `required_mempool_outputs` return value here so that the mempool can drop transactions that + // have been verified spending outputs that aren't in the `created_outputs` collection. async fn spent_utxos( tx: Arc, known_utxos: Arc>, diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 14168ffb91c..018054a9cac 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -45,7 +45,8 @@ pub struct VerifiedSet { /// The set of verified transactions in the mempool. transactions: HashMap, - /// A map of mempool transaction dependencies. + /// A map of dependencies between transactions in the mempool that + /// spend or create outputs of other transactions in the mempool. transaction_dependencies: TransactionDependencies, /// The [`transparent::Utxo`]s created by verified transactions in the mempool From c9c0a1102e21f56c544f5e0829b4bff0044866cc Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 6 Sep 2024 22:36:39 -0400 Subject: [PATCH 10/69] Updates `spent_utxos()` to query the mempool for unspent outputs --- zebra-consensus/src/router.rs | 15 ++- zebra-consensus/src/transaction.rs | 109 ++++++++++++++---- zebra-node-services/src/mempool.rs | 12 +- zebrad/src/commands/start.rs | 12 +- zebrad/src/components/mempool.rs | 12 ++ zebrad/src/components/mempool/storage.rs | 11 +- .../mempool/storage/verified_set.rs | 6 + 7 files changed, 145 insertions(+), 32 deletions(-) diff --git a/zebra-consensus/src/router.rs b/zebra-consensus/src/router.rs index 169fb32affb..38819d0b245 100644 --- a/zebra-consensus/src/router.rs +++ b/zebra-consensus/src/router.rs @@ -30,6 +30,7 @@ use zebra_chain::{ parameters::Network, }; +use zebra_node_services::mempool; use zebra_state as zs; use crate::{ @@ -231,11 +232,11 @@ where /// so that out-of-order and invalid requests do not hang indefinitely. /// See the [`router`](`crate::router`) module documentation for details. #[instrument(skip(state_service, mempool))] -pub async fn init( +pub async fn init( config: Config, network: &Network, mut state_service: S, - mempool: oneshot::Receiver, + mempool: oneshot::Receiver, ) -> ( Buffer, Request>, Buffer< @@ -248,6 +249,11 @@ pub async fn init( where S: Service + Send + Clone + 'static, S::Future: Send + 'static, + Mempool: Service + + Send + + Clone + + 'static, + Mempool::Future: Send + 'static, { // Give other tasks priority before spawning the checkpoint task. tokio::task::yield_now().await; @@ -424,7 +430,10 @@ where config.clone(), network, state_service.clone(), - oneshot::channel().1, + oneshot::channel::< + Buffer, mempool::Request>, + >() + .1, ) .await } diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index a758c7e2523..4785add87a4 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -54,10 +54,6 @@ mod tests; /// chain in the correct order.) const UTXO_LOOKUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(6 * 60); -/// Mempool service type used by the transaction verifier -pub type MempoolService = - Buffer, mempool::Request>; - /// Asynchronous transaction verification. /// /// # Correctness @@ -65,26 +61,27 @@ pub type MempoolService = /// Transaction verification requests should be wrapped in a timeout, so that /// out-of-order and invalid requests do not hang indefinitely. See the [`router`](`crate::router`) /// module documentation for details. -pub struct Verifier { +pub struct Verifier { network: Network, state: Timeout, // TODO: Use an enum so that this can either be Pending(oneshot::Receiver) or Initialized(MempoolService) - mempool: Option, + mempool: Option, script_verifier: script::Verifier, - mempool_setup_rx: oneshot::Receiver, + mempool_setup_rx: oneshot::Receiver, } -impl Verifier +impl Verifier where ZS: Service + Send + Clone + 'static, ZS::Future: Send + 'static, + Mempool: Service + + Send + + Clone + + 'static, + Mempool::Future: Send + 'static, { /// Create a new transaction verifier. - pub fn new( - network: &Network, - state: ZS, - mempool_setup_rx: oneshot::Receiver, - ) -> Self { + pub fn new(network: &Network, state: ZS, mempool_setup_rx: oneshot::Receiver) -> Self { Self { network: network.clone(), state: Timeout::new(state, UTXO_LOOKUP_TIMEOUT), @@ -93,11 +90,27 @@ where mempool_setup_rx, } } +} +impl + Verifier< + ZS, + Buffer, mempool::Request>, + > +where + ZS: Service + Send + Clone + 'static, + ZS::Future: Send + 'static, +{ /// Create a new transaction verifier with a closed channel receiver for mempool setup for tests. #[cfg(test)] pub fn new_for_tests(network: &Network, state: ZS) -> Self { - Self::new(network, state, oneshot::channel().1) + Self { + network: network.clone(), + state: Timeout::new(state, UTXO_LOOKUP_TIMEOUT), + mempool: None, + script_verifier: script::Verifier, + mempool_setup_rx: oneshot::channel().1, + } } } @@ -296,10 +309,15 @@ impl Response { } } -impl Service for Verifier +impl Service for Verifier where ZS: Service + Send + Clone + 'static, ZS::Future: Send + 'static, + Mempool: Service + + Send + + Clone + + 'static, + Mempool::Future: Send + 'static, { type Response = Response; type Error = TransactionError; @@ -323,6 +341,7 @@ where let script_verifier = self.script_verifier; let network = self.network.clone(); let state = self.state.clone(); + let mempool = self.mempool.clone(); let tx = req.transaction(); let tx_id = req.tx_id(); @@ -398,8 +417,8 @@ where // Load spent UTXOs from state. // The UTXOs are required for almost all the async checks. let load_spent_utxos_fut = - Self::spent_utxos(tx.clone(), req.known_utxos(), req.is_mempool(), state.clone()); - let (spent_utxos, spent_outputs) = load_spent_utxos_fut.await?; + Self::spent_utxos(tx.clone(), req.known_utxos(), req.is_mempool(), state.clone(), mempool); + let (spent_utxos, spent_outputs, spent_mempool_outpoints) = load_spent_utxos_fut.await?; // WONTFIX: Return an error for Request::Block as well to replace this check in // the state once #2336 has been implemented? @@ -519,10 +538,15 @@ where } } -impl Verifier +impl Verifier where ZS: Service + Send + Clone + 'static, ZS::Future: Send + 'static, + Mempool: Service + + Send + + Clone + + 'static, + Mempool::Future: Send + 'static, { /// Fetches the median-time-past of the *next* block after the best state tip. /// @@ -552,23 +576,25 @@ where /// /// Returns a tuple with a OutPoint -> Utxo map, and a vector of Outputs /// in the same order as the matching inputs in the transaction. - // TODO: Add `required_mempool_outputs` return value here so that the mempool can drop transactions that - // have been verified spending outputs that aren't in the `created_outputs` collection. async fn spent_utxos( tx: Arc, known_utxos: Arc>, is_mempool: bool, state: Timeout, + mempool: Option, ) -> Result< ( HashMap, Vec, + Vec, ), TransactionError, > { let inputs = tx.inputs(); let mut spent_utxos = HashMap::new(); let mut spent_outputs = Vec::new(); + let mut spent_mempool_outpoints = Vec::new(); + for input in inputs { if let transparent::Input::PrevOut { outpoint, .. } = input { tracing::trace!("awaiting outpoint lookup"); @@ -581,11 +607,17 @@ where let query = state .clone() .oneshot(zs::Request::UnspentBestChainUtxo(*outpoint)); - if let zebra_state::Response::UnspentBestChainUtxo(utxo) = query.await? { - utxo.ok_or(TransactionError::TransparentInputNotFound)? - } else { + + let zebra_state::Response::UnspentBestChainUtxo(utxo) = query.await? else { unreachable!("UnspentBestChainUtxo always responds with Option") - } + }; + + let Some(utxo) = utxo else { + spent_mempool_outpoints.push(*outpoint); + continue; + }; + + utxo } else { let query = state .clone() @@ -603,7 +635,34 @@ where continue; } } - Ok((spent_utxos, spent_outputs)) + + if let Some(mempool) = mempool { + for &spent_mempool_outpoint in &spent_mempool_outpoints { + let query = mempool + .clone() + .oneshot(mempool::Request::UnspentOutput(spent_mempool_outpoint)); + + let mempool::Response::UnspentOutput(output) = query.await? else { + unreachable!("UnspentOutput always responds with Option") + }; + + let Some(output) = output else { + // TODO: Add an `AwaitOutput` mempool Request to wait for another transaction + // to be added to the mempool that creates the required output and call + // here with a short timeout (everything will be invalidated by a chain + // tip change anyway, and the tx verifier should poll the mempool so it + // checks for new created outputs at the queued pending output outpoint + // almost immediately after a tx is verified). + return Err(TransactionError::TransparentInputNotFound); + }; + + spent_outputs.push(output.clone()); + } + } else if !spent_mempool_outpoints.is_empty() { + return Err(TransactionError::TransparentInputNotFound); + } + + Ok((spent_utxos, spent_outputs, spent_mempool_outpoints)) } /// Accepts `request`, a transaction verifier [`&Request`](Request), diff --git a/zebra-node-services/src/mempool.rs b/zebra-node-services/src/mempool.rs index fbaaf029c75..50875970302 100644 --- a/zebra-node-services/src/mempool.rs +++ b/zebra-node-services/src/mempool.rs @@ -5,7 +5,10 @@ use std::collections::HashSet; use tokio::sync::oneshot; -use zebra_chain::transaction::{self, UnminedTx, UnminedTxId}; +use zebra_chain::{ + transaction::{self, UnminedTx, UnminedTxId}, + transparent, +}; #[cfg(feature = "getblocktemplate-rpcs")] use zebra_chain::transaction::VerifiedUnminedTx; @@ -39,6 +42,10 @@ pub enum Request { /// the [`AuthDigest`](zebra_chain::transaction::AuthDigest). TransactionsByMinedId(HashSet), + /// Looks up a UTXO in the mempool transparent identified by the given [`OutPoint`](transparent::OutPoint), + /// returning `None` immediately if it is unknown. + UnspentOutput(transparent::OutPoint), + /// Get all the [`VerifiedUnminedTx`] in the mempool. /// /// Equivalent to `TransactionsById(TransactionIds)`, @@ -99,6 +106,9 @@ pub enum Response { /// different transactions with different mined IDs. Transactions(Vec), + /// Response to [`Request::UnspentOutput`] with the transparent output + UnspentOutput(Option), + /// Returns all [`VerifiedUnminedTx`] in the mempool. // // TODO: make the Transactions response return VerifiedUnminedTx, diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 13486ecd41f..89239f48a0f 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -185,7 +185,17 @@ impl StartCmd { &config.network.network, state.clone(), // TODO: Pass actual setup channel receiver - oneshot::channel().1, + oneshot::channel::< + tower::buffer::Buffer< + BoxService< + zebra_node_services::mempool::Request, + zebra_node_services::mempool::Response, + tower::BoxError, + >, + zebra_node_services::mempool::Request, + >, + >() + .1, ) .await; diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index e77d48c0bf6..fcc52cf4731 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -731,6 +731,16 @@ impl Service for Mempool { async move { Ok(Response::Transactions(res)) }.boxed() } + Request::UnspentOutput(outpoint) => { + trace!(?req, "got mempool request"); + + let res = storage.created_output(&outpoint); + + trace!(?res, "answered mempool request"); + + async move { Ok(Response::UnspentOutput(res)) }.boxed() + } + #[cfg(feature = "getblocktemplate-rpcs")] Request::FullTransactions => { trace!(?req, "got mempool request"); @@ -809,6 +819,8 @@ impl Service for Mempool { Request::TransactionsById(_) => Response::Transactions(Default::default()), Request::TransactionsByMinedId(_) => Response::Transactions(Default::default()), + Request::UnspentOutput(_) => Response::UnspentOutput(None), + #[cfg(feature = "getblocktemplate-rpcs")] Request::FullTransactions => { return async move { diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 3e5423e5e67..631d00e5112 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -16,8 +16,9 @@ use std::{ use thiserror::Error; -use zebra_chain::transaction::{ - self, Hash, Transaction, UnminedTx, UnminedTxId, VerifiedUnminedTx, +use zebra_chain::{ + transaction::{self, Hash, Transaction, UnminedTx, UnminedTxId, VerifiedUnminedTx}, + transparent, }; use self::{eviction_list::EvictionList, verified_set::VerifiedSet}; @@ -416,6 +417,12 @@ impl Storage { self.verified.transactions() } + /// Returns a [`transparent::Output`] created by a mempool transaction for the provided + /// [`transparent::OutPoint`] if one exists, or None otherwise. + pub fn created_output(&self, outpoint: &transparent::OutPoint) -> Option { + self.verified.created_output(outpoint) + } + /// Returns the number of transactions in the mempool. #[allow(dead_code)] pub fn transaction_count(&self) -> usize { diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 018054a9cac..ecab3f295af 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -88,6 +88,12 @@ impl VerifiedSet { &self.transactions } + /// Returns a [`transparent::Output`] created by a mempool transaction for the provided + /// [`transparent::OutPoint`] if one exists, or None otherwise. + pub fn created_output(&self, outpoint: &transparent::OutPoint) -> Option { + self.created_outputs.get(outpoint).cloned() + } + /// Returns the number of verified transactions in the set. pub fn transaction_count(&self) -> usize { self.transactions.len() From cbb34e646a7b3c7d3b607e79d8530a0ef096d77c Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 6 Sep 2024 22:50:01 -0400 Subject: [PATCH 11/69] Adds `spent_mempool_outpoints` as a field on tx verifier mempool response --- zebra-consensus/src/transaction.rs | 16 ++++++++++++++-- zebrad/src/components/mempool/downloads.rs | 1 + 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 4785add87a4..363d34e792f 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -189,12 +189,24 @@ pub enum Response { /// [`Response::Mempool`] responses are uniquely identified by the /// [`UnminedTxId`] variant for their transaction version. transaction: VerifiedUnminedTx, + + /// A list of spent [`transparent::OutPoint`]s that were found in + /// the mempool's list of `created_outputs`. + /// + /// Used by the mempool to determine dependencies between transactions + /// in the mempool and to avoid adding transactions with missing spends + /// to its verified set. + spent_mempool_outpoints: Vec, }, } +#[cfg(any(test, feature = "proptest-impl"))] impl From for Response { fn from(transaction: VerifiedUnminedTx) -> Self { - Response::Mempool { transaction } + Response::Mempool { + transaction, + spent_mempool_outpoints: Vec::new(), + } } } @@ -523,7 +535,7 @@ where ), legacy_sigop_count, )?; - Response::Mempool { transaction } + Response::Mempool { transaction, spent_mempool_outpoints } }, }; diff --git a/zebrad/src/components/mempool/downloads.rs b/zebrad/src/components/mempool/downloads.rs index b37f988dcc8..1a54c6bd07c 100644 --- a/zebrad/src/components/mempool/downloads.rs +++ b/zebrad/src/components/mempool/downloads.rs @@ -346,6 +346,7 @@ where height: next_height, }) .map_ok(|rsp| { + // TODO: Return the spent mempool outpoints as well (rsp.into_mempool_transaction() .expect("unexpected non-mempool response to mempool request"), tip_height) }) From be6fd7b556b7bc5a7674cd8fce2695c2f331c576 Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 6 Sep 2024 22:57:14 -0400 Subject: [PATCH 12/69] Updates mempool `Downloads` to return the spent_mempool_outpoints from the tx verifier response --- zebra-consensus/src/transaction.rs | 8 ------ zebrad/src/components/mempool.rs | 2 +- zebrad/src/components/mempool/downloads.rs | 33 +++++++++++++++------- zebrad/src/components/mempool/storage.rs | 6 +++- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 363d34e792f..6de11a3d852 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -273,14 +273,6 @@ impl Request { } impl Response { - /// The verified mempool transaction, if this is a mempool response. - pub fn into_mempool_transaction(self) -> Option { - match self { - Response::Block { .. } => None, - Response::Mempool { transaction, .. } => Some(transaction), - } - } - /// The unmined transaction ID for the transaction in this response. pub fn tx_id(&self) -> UnminedTxId { match self { diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index fcc52cf4731..37941452cb6 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -588,7 +588,7 @@ impl Service for Mempool { pin!(tx_downloads.timeout(RATE_LIMIT_DELAY)).poll_next(cx) { match r { - Ok(Ok((tx, expected_tip_height))) => { + Ok(Ok((tx, _spent_mempool_outpoints, expected_tip_height))) => { // # Correctness: // // It's okay to use tip height here instead of the tip hash since diff --git a/zebrad/src/components/mempool/downloads.rs b/zebrad/src/components/mempool/downloads.rs index 1a54c6bd07c..275346ce6e4 100644 --- a/zebrad/src/components/mempool/downloads.rs +++ b/zebrad/src/components/mempool/downloads.rs @@ -47,6 +47,7 @@ use tracing_futures::Instrument; use zebra_chain::{ block::Height, transaction::{self, UnminedTxId, VerifiedUnminedTx}, + transparent, }; use zebra_consensus::transaction as tx; use zebra_network as zn; @@ -153,7 +154,11 @@ where pending: FuturesUnordered< JoinHandle< Result< - (VerifiedUnminedTx, Option), + ( + VerifiedUnminedTx, + Vec, + Option, + ), (TransactionDownloadVerifyError, UnminedTxId), >, >, @@ -173,8 +178,14 @@ where ZS: Service + Send + Clone + 'static, ZS::Future: Send, { - type Item = - Result<(VerifiedUnminedTx, Option), (UnminedTxId, TransactionDownloadVerifyError)>; + type Item = Result< + ( + VerifiedUnminedTx, + Vec, + Option, + ), + (UnminedTxId, TransactionDownloadVerifyError), + >; fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { let this = self.project(); @@ -189,9 +200,9 @@ where // TODO: this would be cleaner with poll_map (#2693) if let Some(join_result) = ready!(this.pending.poll_next(cx)) { match join_result.expect("transaction download and verify tasks must not panic") { - Ok((tx, tip_height)) => { + Ok((tx, spent_mempool_outpoints, tip_height)) => { this.cancel_handles.remove(&tx.transaction.id); - Poll::Ready(Some(Ok((tx, tip_height)))) + Poll::Ready(Some(Ok((tx, spent_mempool_outpoints, tip_height)))) } Err((e, hash)) => { this.cancel_handles.remove(&hash); @@ -346,9 +357,11 @@ where height: next_height, }) .map_ok(|rsp| { - // TODO: Return the spent mempool outpoints as well - (rsp.into_mempool_transaction() - .expect("unexpected non-mempool response to mempool request"), tip_height) + let tx::Response::Mempool { transaction, spent_mempool_outpoints } = rsp else { + panic!("unexpected non-mempool response to mempool request") + }; + + (transaction, spent_mempool_outpoints, tip_height) }) .await; @@ -357,12 +370,12 @@ where result.map_err(|e| TransactionDownloadVerifyError::Invalid(e.into())) } - .map_ok(|(tx, tip_height)| { + .map_ok(|(tx, spent_mempool_outpoints, tip_height)| { metrics::counter!( "mempool.verified.transactions.total", "version" => format!("{}", tx.transaction.transaction.version()), ).increment(1); - (tx, tip_height) + (tx, spent_mempool_outpoints, tip_height) }) // Tack the hash onto the error so we can remove the cancel handle // on failure as well as on success. diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 631d00e5112..25e1df5274b 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -181,7 +181,11 @@ impl Storage { /// If inserting this transaction evicts other transactions, they will be tracked /// as [`SameEffectsChainRejectionError::RandomlyEvicted`]. #[allow(clippy::unwrap_in_result)] - pub fn insert(&mut self, tx: VerifiedUnminedTx) -> Result { + pub fn insert( + &mut self, + tx: VerifiedUnminedTx, + // _spent_mempool_outpoints: Vec, + ) -> Result { // # Security // // This method must call `reject`, rather than modifying the rejection lists directly. From 90812b1ff39662edc038d276fbe1d15f3744beed Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 6 Sep 2024 23:01:04 -0400 Subject: [PATCH 13/69] Updates `Storage.insert()` to accept a list of spent mempool transaction outputs --- .../components/inbound/tests/fake_peer_set.rs | 2 +- zebrad/src/components/mempool.rs | 4 ++-- zebrad/src/components/mempool/storage.rs | 2 +- .../components/mempool/storage/tests/prop.rs | 20 +++++++++---------- .../mempool/storage/tests/vectors.rs | 13 ++++++------ zebrad/src/components/mempool/tests/prop.rs | 6 +++--- zebrad/src/components/mempool/tests/vector.rs | 12 +++++++---- 7 files changed, 32 insertions(+), 27 deletions(-) diff --git a/zebrad/src/components/inbound/tests/fake_peer_set.rs b/zebrad/src/components/inbound/tests/fake_peer_set.rs index b85dc3f2cb0..dae03141762 100644 --- a/zebrad/src/components/inbound/tests/fake_peer_set.rs +++ b/zebrad/src/components/inbound/tests/fake_peer_set.rs @@ -1061,7 +1061,7 @@ fn add_some_stuff_to_mempool( // Insert the genesis block coinbase transaction into the mempool storage. mempool_service .storage() - .insert(genesis_transactions[0].clone()) + .insert(genesis_transactions[0].clone(), Vec::new()) .unwrap(); genesis_transactions diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index 37941452cb6..acab316062d 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -588,7 +588,7 @@ impl Service for Mempool { pin!(tx_downloads.timeout(RATE_LIMIT_DELAY)).poll_next(cx) { match r { - Ok(Ok((tx, _spent_mempool_outpoints, expected_tip_height))) => { + Ok(Ok((tx, spent_mempool_outpoints, expected_tip_height))) => { // # Correctness: // // It's okay to use tip height here instead of the tip hash since @@ -596,7 +596,7 @@ impl Service for Mempool { // the best chain changes (which is the only way to stay at the same height), and the // mempool re-verifies all pending tx_downloads when there's a `TipAction::Reset`. if best_tip_height == expected_tip_height { - let insert_result = storage.insert(tx.clone()); + let insert_result = storage.insert(tx.clone(), spent_mempool_outpoints); tracing::trace!( ?insert_result, diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 25e1df5274b..516aa732d8b 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -184,7 +184,7 @@ impl Storage { pub fn insert( &mut self, tx: VerifiedUnminedTx, - // _spent_mempool_outpoints: Vec, + _spent_mempool_outpoints: Vec, ) -> Result { // # Security // diff --git a/zebrad/src/components/mempool/storage/tests/prop.rs b/zebrad/src/components/mempool/storage/tests/prop.rs index eca65935acb..5c26b534783 100644 --- a/zebrad/src/components/mempool/storage/tests/prop.rs +++ b/zebrad/src/components/mempool/storage/tests/prop.rs @@ -72,7 +72,7 @@ proptest! { for (transaction_to_accept, transaction_to_reject) in input_permutations { let id_to_accept = transaction_to_accept.transaction.id; - prop_assert_eq!(storage.insert(transaction_to_accept), Ok(id_to_accept)); + prop_assert_eq!(storage.insert(transaction_to_accept, Vec::new()), Ok(id_to_accept)); // Make unique IDs by converting the index to bytes, and writing it to each ID let unique_ids = (0..MAX_EVICTION_MEMORY_ENTRIES as u32).map(move |index| { @@ -96,7 +96,7 @@ proptest! { // - transaction_to_accept, or // - a rejection from rejections prop_assert_eq!( - storage.insert(transaction_to_reject), + storage.insert(transaction_to_reject, Vec::new()), Err(MempoolError::StorageEffectsTip(SameEffectsTipRejectionError::SpendConflict)) ); @@ -147,13 +147,13 @@ proptest! { if i < transactions.len() - 1 { // The initial transactions should be successful prop_assert_eq!( - storage.insert(transaction.clone()), + storage.insert(transaction.clone(), Vec::new()), Ok(tx_id) ); } else { // The final transaction will cause a random eviction, // which might return an error if this transaction is chosen - let result = storage.insert(transaction.clone()); + let result = storage.insert(transaction.clone(), Vec::new()); if result.is_ok() { prop_assert_eq!( @@ -281,10 +281,10 @@ proptest! { let id_to_accept = transaction_to_accept.transaction.id; let id_to_reject = transaction_to_reject.transaction.id; - prop_assert_eq!(storage.insert(transaction_to_accept), Ok(id_to_accept)); + prop_assert_eq!(storage.insert(transaction_to_accept, Vec::new()), Ok(id_to_accept)); prop_assert_eq!( - storage.insert(transaction_to_reject), + storage.insert(transaction_to_reject, Vec::new()), Err(MempoolError::StorageEffectsTip(SameEffectsTipRejectionError::SpendConflict)) ); @@ -332,19 +332,19 @@ proptest! { let id_to_reject = transaction_to_reject.transaction.id; prop_assert_eq!( - storage.insert(first_transaction_to_accept), + storage.insert(first_transaction_to_accept, Vec::new()), Ok(first_id_to_accept) ); prop_assert_eq!( - storage.insert(transaction_to_reject), + storage.insert(transaction_to_reject, Vec::new()), Err(MempoolError::StorageEffectsTip(SameEffectsTipRejectionError::SpendConflict)) ); prop_assert!(storage.contains_rejected(&id_to_reject)); prop_assert_eq!( - storage.insert(second_transaction_to_accept), + storage.insert(second_transaction_to_accept, Vec::new()), Ok(second_id_to_accept) ); @@ -371,7 +371,7 @@ proptest! { .filter_map(|transaction| { let id = transaction.transaction.id; - storage.insert(transaction.clone()).ok().map(|_| id) + storage.insert(transaction.clone(), Vec::new()).ok().map(|_| id) }) .collect(); diff --git a/zebrad/src/components/mempool/storage/tests/vectors.rs b/zebrad/src/components/mempool/storage/tests/vectors.rs index 5b60c133e95..4d3a00297f6 100644 --- a/zebrad/src/components/mempool/storage/tests/vectors.rs +++ b/zebrad/src/components/mempool/storage/tests/vectors.rs @@ -40,7 +40,7 @@ fn mempool_storage_crud_exact_mainnet() { .expect("at least one unmined transaction"); // Insert unmined tx into the mempool. - let _ = storage.insert(unmined_tx.clone()); + let _ = storage.insert(unmined_tx.clone(), Vec::new()); // Check that it is in the mempool, and not rejected. assert!(storage.contains_transaction_exact(&unmined_tx.transaction.id)); @@ -94,7 +94,7 @@ fn mempool_storage_basic_for_network(network: Network) -> Result<()> { let mut maybe_inserted_transactions = Vec::new(); let mut some_rejected_transactions = Vec::new(); for unmined_transaction in unmined_transactions.clone() { - let result = storage.insert(unmined_transaction.clone()); + let result = storage.insert(unmined_transaction.clone(), Vec::new()); match result { Ok(_) => { // While the transaction was inserted here, it can be rejected later. @@ -167,7 +167,7 @@ fn mempool_storage_crud_same_effects_mainnet() { .expect("at least one unmined transaction"); // Insert unmined tx into the mempool. - let _ = storage.insert(unmined_tx_1.clone()); + let _ = storage.insert(unmined_tx_1.clone(), Vec::new()); // Check that it is in the mempool, and not rejected. assert!(storage.contains_transaction_exact(&unmined_tx_1.transaction.id)); @@ -188,7 +188,7 @@ fn mempool_storage_crud_same_effects_mainnet() { Some(SameEffectsChainRejectionError::Mined.into()) ); assert_eq!( - storage.insert(unmined_tx_1), + storage.insert(unmined_tx_1, Vec::new()), Err(SameEffectsChainRejectionError::Mined.into()) ); @@ -205,7 +205,7 @@ fn mempool_storage_crud_same_effects_mainnet() { // Insert unmined tx into the mempool. assert_eq!( - storage.insert(unmined_tx_2.clone()), + storage.insert(unmined_tx_2.clone(), Vec::new()), Ok(unmined_tx_2.transaction.id) ); @@ -228,7 +228,7 @@ fn mempool_storage_crud_same_effects_mainnet() { Some(SameEffectsChainRejectionError::DuplicateSpend.into()) ); assert_eq!( - storage.insert(unmined_tx_2), + storage.insert(unmined_tx_2, Vec::new()), Err(SameEffectsChainRejectionError::DuplicateSpend.into()) ); } @@ -269,6 +269,7 @@ fn mempool_expired_basic_for_network(network: Network) -> Result<()> { 0, ) .expect("verification should pass"), + Vec::new(), )?; assert_eq!(storage.transaction_count(), 1); diff --git a/zebrad/src/components/mempool/tests/prop.rs b/zebrad/src/components/mempool/tests/prop.rs index 9f05b79d567..e12b205e34c 100644 --- a/zebrad/src/components/mempool/tests/prop.rs +++ b/zebrad/src/components/mempool/tests/prop.rs @@ -74,7 +74,7 @@ proptest! { // Insert a dummy transaction. mempool .storage() - .insert(transaction.0) + .insert(transaction.0, Vec::new()) .expect("Inserting a transaction should succeed"); // The first call to `poll_ready` shouldn't clear the storage yet. @@ -148,7 +148,7 @@ proptest! { // Insert the dummy transaction into the mempool. mempool .storage() - .insert(transaction.0.clone()) + .insert(transaction.0.clone(), Vec::new()) .expect("Inserting a transaction should succeed"); // Set the new chain tip. @@ -205,7 +205,7 @@ proptest! { // Insert a dummy transaction. mempool .storage() - .insert(transaction) + .insert(transaction, Vec::new()) .expect("Inserting a transaction should succeed"); // The first call to `poll_ready` shouldn't clear the storage yet. diff --git a/zebrad/src/components/mempool/tests/vector.rs b/zebrad/src/components/mempool/tests/vector.rs index c285923fa7d..edd779e51c4 100644 --- a/zebrad/src/components/mempool/tests/vector.rs +++ b/zebrad/src/components/mempool/tests/vector.rs @@ -61,7 +61,9 @@ async fn mempool_service_basic_single() -> Result<(), Report> { // Insert the genesis block coinbase transaction into the mempool storage. let mut inserted_ids = HashSet::new(); - service.storage().insert(genesis_transaction.clone())?; + service + .storage() + .insert(genesis_transaction.clone(), Vec::new())?; inserted_ids.insert(genesis_transaction.transaction.id); // Test `Request::TransactionIds` @@ -131,7 +133,7 @@ async fn mempool_service_basic_single() -> Result<(), Report> { inserted_ids.insert(tx.transaction.id); // Error must be ignored because a insert can trigger an eviction and // an error is returned if the transaction being inserted in chosen. - let _ = service.storage().insert(tx.clone()); + let _ = service.storage().insert(tx.clone(), Vec::new()); } // Test `Request::RejectedTransactionIds` @@ -212,7 +214,7 @@ async fn mempool_queue_single() -> Result<(), Report> { for tx in transactions.iter() { // Error must be ignored because a insert can trigger an eviction and // an error is returned if the transaction being inserted in chosen. - let _ = service.storage().insert(tx.clone()); + let _ = service.storage().insert(tx.clone(), Vec::new()); } // Test `Request::Queue` for a new transaction @@ -293,7 +295,9 @@ async fn mempool_service_disabled() -> Result<(), Report> { assert!(service.is_enabled()); // Insert the genesis block coinbase transaction into the mempool storage. - service.storage().insert(genesis_transaction.clone())?; + service + .storage() + .insert(genesis_transaction.clone(), Vec::new())?; // Test if the mempool answers correctly (i.e. is enabled) let response = service From d0b2d41de1caaeb74b3d31f63f8541407b7447b2 Mon Sep 17 00:00:00 2001 From: Arya Date: Sat, 7 Sep 2024 00:06:54 -0400 Subject: [PATCH 14/69] Adds transaction dependencies when inserting a tx in `VerifiedSet` --- zebra-consensus/src/transaction.rs | 5 +- zebrad/src/components/mempool/storage.rs | 10 ++- .../mempool/storage/verified_set.rs | 75 ++++++++++++++++--- 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 6de11a3d852..34fb409cc52 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -517,8 +517,9 @@ where legacy_sigop_count, }, Request::Mempool { transaction, .. } => { - // TODO: Poll the mempool so it sees the new verified result promptly / nearly-immediately - // (to solve concurrency issue with dependency chains of orphaned transactions) + // TODO: If the mempool transaction has any transparent outputs, poll the mempool so + // it sees the new verified result promptly / nearly-immediately + // (to solve concurrency issue with dependency chains of orphaned transactions) let transaction = VerifiedUnminedTx::new( transaction, diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 516aa732d8b..3c15cb7086c 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -68,6 +68,12 @@ pub enum SameEffectsTipRejectionError { its inputs" )] SpendConflict, + + #[error( + "transaction rejected because it spends missing outputs from \ + another transaction in the mempool" + )] + MissingOutput, } /// Transactions rejected based only on their effects (spends, outputs, transaction header). @@ -184,7 +190,7 @@ impl Storage { pub fn insert( &mut self, tx: VerifiedUnminedTx, - _spent_mempool_outpoints: Vec, + spent_mempool_outpoints: Vec, ) -> Result { // # Security // @@ -219,7 +225,7 @@ impl Storage { // Then, we try to insert into the pool. If this fails the transaction is rejected. let mut result = Ok(tx_id); - if let Err(rejection_error) = self.verified.insert(tx) { + if let Err(rejection_error) = self.verified.insert(tx, spent_mempool_outpoints) { tracing::debug!( ?tx_id, ?rejection_error, diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index ecab3f295af..31d5b2f7595 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -8,7 +8,7 @@ use std::{ use zebra_chain::{ orchard, sapling, sprout, - transaction::{Transaction, UnminedTx, UnminedTxId, VerifiedUnminedTx}, + transaction::{self, Transaction, UnminedTx, UnminedTxId, VerifiedUnminedTx}, transparent, }; @@ -22,11 +22,50 @@ use zebra_chain::transaction::MEMPOOL_TRANSACTION_COST_THRESHOLD; struct TransactionDependencies { /// Lists of mempool transactions that create UTXOs spent /// by a mempool transaction. - dependencies: HashMap>, + dependencies: HashMap>, /// Lists of mempool transactions that spend UTXOs created /// by a mempool transaction. - dependants: HashMap>, + dependants: HashMap>, +} + +impl TransactionDependencies { + /// Adds a transaction that spends outputs created by other transactions in the mempool + /// as a dependant of those transactions, and adds the transactions that created the outputs + /// spent by the dependant transaction as dependencies of the dependant transaction. + // + // TODO: Order transactions in block templates based on their dependencies + fn add( + &mut self, + dependant: transaction::Hash, + spent_mempool_outpoints: Vec, + ) { + for &spent_mempool_outpoint in &spent_mempool_outpoints { + self.dependants + .entry(spent_mempool_outpoint.hash) + .or_default() + .insert(dependant); + } + + self.dependencies.entry(dependant).or_default().extend( + spent_mempool_outpoints + .into_iter() + .map(|outpoint| outpoint.hash), + ); + } + + /// Removes the hash of a transaction in the mempool and the hashes of any transactions + /// that are tracked as being directly dependant on that transaction from + /// this [`TransactionDependencies`]. + /// + /// Returns a list of transaction hashes that have been removed. + fn _remove(&mut self, tx_hash: &transaction::Hash) -> HashSet { + for dependencies in self.dependencies.values_mut() { + dependencies.remove(tx_hash); + } + + self.dependants.remove(tx_hash).unwrap_or_default() + } } /// The set of verified transactions stored in the mempool. @@ -144,14 +183,28 @@ impl VerifiedSet { pub fn insert( &mut self, transaction: VerifiedUnminedTx, + spent_mempool_outpoints: Vec, ) -> Result<(), SameEffectsTipRejectionError> { if self.has_spend_conflicts(&transaction.transaction) { return Err(SameEffectsTipRejectionError::SpendConflict); } - // TODO: Update `transaction_dependencies` + // This likely only needs to check that the transaction hash of the outpoint is still in the mempool, + // bu it's likely rare that a transaction spends multiple transparent outputs of + // a single transaction in practice. + for outpoint in &spent_mempool_outpoints { + if !self.created_outputs.contains_key(outpoint) { + return Err(SameEffectsTipRejectionError::MissingOutput); + } + } + let tx_id = transaction.transaction.id; - self.cache_outputs_from(tx_id, &transaction.transaction.transaction); + + // TODO: Update `transaction_dependencies` + self.transaction_dependencies + .add(tx_id.mined_id(), spent_mempool_outpoints); + + self.cache_outputs_from(tx_id.mined_id(), &transaction.transaction.transaction); self.transactions_serialized_size += transaction.transaction.size; self.total_cost += transaction.cost(); self.transactions.insert(tx_id, transaction); @@ -228,7 +281,9 @@ impl VerifiedSet { /// /// Also removes its outputs from the internal caches. fn remove(&mut self, key_to_remove: &UnminedTxId) -> VerifiedUnminedTx { - // TODO: Remove any dependant transactions as well + // TODO: + // - Remove any dependant transactions as well + // - Update `transaction_dependencies` let removed_tx = self .transactions @@ -259,12 +314,10 @@ impl VerifiedSet { } /// Inserts the transaction's outputs into the internal caches. - fn cache_outputs_from(&mut self, tx_id: UnminedTxId, tx: &Transaction) { + fn cache_outputs_from(&mut self, tx_hash: transaction::Hash, tx: &Transaction) { for (index, output) in tx.outputs().iter().cloned().enumerate() { - self.created_outputs.insert( - transparent::OutPoint::from_usize(tx_id.mined_id(), index), - output, - ); + self.created_outputs + .insert(transparent::OutPoint::from_usize(tx_hash, index), output); } self.spent_outpoints.extend(tx.spent_outpoints()); From 0a72b41db7c7f8b0a433e040e775ed24f75d5216 Mon Sep 17 00:00:00 2001 From: Arya Date: Sat, 7 Sep 2024 00:18:15 -0400 Subject: [PATCH 15/69] polls mempool svc from tx verifier when a mempool tx that creates transparent outputs has been verified. adds a TODO for adding a `pending_outputs` field to the mempool Storage --- zebra-consensus/src/transaction.rs | 17 ++++++++++++----- zebrad/src/components/mempool/storage.rs | 3 +++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 34fb409cc52..3a5293b9f00 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -6,6 +6,7 @@ use std::{ pin::Pin, sync::Arc, task::{Context, Poll}, + time::Duration, }; use chrono::{DateTime, Utc}; @@ -421,7 +422,7 @@ where // Load spent UTXOs from state. // The UTXOs are required for almost all the async checks. let load_spent_utxos_fut = - Self::spent_utxos(tx.clone(), req.known_utxos(), req.is_mempool(), state.clone(), mempool); + Self::spent_utxos(tx.clone(), req.known_utxos(), req.is_mempool(), state.clone(), mempool.clone()); let (spent_utxos, spent_outputs, spent_mempool_outpoints) = load_spent_utxos_fut.await?; // WONTFIX: Return an error for Request::Block as well to replace this check in @@ -517,10 +518,6 @@ where legacy_sigop_count, }, Request::Mempool { transaction, .. } => { - // TODO: If the mempool transaction has any transparent outputs, poll the mempool so - // it sees the new verified result promptly / nearly-immediately - // (to solve concurrency issue with dependency chains of orphaned transactions) - let transaction = VerifiedUnminedTx::new( transaction, miner_fee.expect( @@ -528,6 +525,16 @@ where ), legacy_sigop_count, )?; + + if let Some(mut mempool) = mempool { + if !transaction.transaction.transaction.outputs().is_empty() { + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + mempool.ready().await.expect("mempool poll_ready() method should not return an error"); + }); + } + } + Response::Mempool { transaction, spent_mempool_outpoints } }, }; diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 3c15cb7086c..3b37b049e22 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -119,6 +119,9 @@ pub enum RejectionError { } /// Hold mempool verified and rejected mempool transactions. +// +// Add a `pending_outputs` field similar to the `pending_utxos` field in the state service +// for queuing outpoint queries. pub struct Storage { /// The set of verified transactions in the mempool. verified: VerifiedSet, From 2fc7829f0d06a932315149339935e216d97e66fa Mon Sep 17 00:00:00 2001 From: Arya Date: Sat, 7 Sep 2024 00:53:08 -0400 Subject: [PATCH 16/69] Adds `pending_outputs` field on mempool Storage and responds to pending outputs requests when inserting new transactions into the mempool's verified set --- zebra-consensus/src/transaction.rs | 21 +++--- zebra-node-services/src/mempool.rs | 19 +++++- zebrad/src/components/mempool.rs | 12 ++++ .../src/components/mempool/pending_outputs.rs | 64 +++++++++++++++++++ zebrad/src/components/mempool/storage.rs | 17 +++-- .../mempool/storage/verified_set.rs | 26 ++++++-- 6 files changed, 137 insertions(+), 22 deletions(-) create mode 100644 zebrad/src/components/mempool/pending_outputs.rs diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 3a5293b9f00..bd78fc151e7 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -655,17 +655,20 @@ where .oneshot(mempool::Request::UnspentOutput(spent_mempool_outpoint)); let mempool::Response::UnspentOutput(output) = query.await? else { - unreachable!("UnspentOutput always responds with Option") + unreachable!("UnspentOutput always responds with UnspentOutput") }; - let Some(output) = output else { - // TODO: Add an `AwaitOutput` mempool Request to wait for another transaction - // to be added to the mempool that creates the required output and call - // here with a short timeout (everything will be invalidated by a chain - // tip change anyway, and the tx verifier should poll the mempool so it - // checks for new created outputs at the queued pending output outpoint - // almost immediately after a tx is verified). - return Err(TransactionError::TransparentInputNotFound); + let output = if let Some(output) = output { + output + } else { + let query = mempool + .clone() + .oneshot(mempool::Request::AwaitOutput(spent_mempool_outpoint)); + if let mempool::Response::UnspentOutput(output) = query.await? { + output.ok_or(TransactionError::TransparentInputNotFound)? + } else { + unreachable!("AwaitOutput always responds with UnspentOutput") + } }; spent_outputs.push(output.clone()); diff --git a/zebra-node-services/src/mempool.rs b/zebra-node-services/src/mempool.rs index 50875970302..09b70f8e258 100644 --- a/zebra-node-services/src/mempool.rs +++ b/zebra-node-services/src/mempool.rs @@ -42,10 +42,27 @@ pub enum Request { /// the [`AuthDigest`](zebra_chain::transaction::AuthDigest). TransactionsByMinedId(HashSet), - /// Looks up a UTXO in the mempool transparent identified by the given [`OutPoint`](transparent::OutPoint), + /// Looks up a [`transparent::Output`] in the mempool identified by the given [`OutPoint`](transparent::OutPoint), /// returning `None` immediately if it is unknown. + /// + /// Does not gaurantee that the output will remain in the mempool or that it is unspent. UnspentOutput(transparent::OutPoint), + /// Request a [`transparent::Output`] identified by the given [`OutPoint`](transparent::OutPoint), + /// waiting until it becomes available if it is unknown. + /// + /// This request is purely informational, and there are no guarantees about + /// whether the UTXO remains unspent or is on the best chain, or any chain. + /// Its purpose is to allow orphaned mempool transaction verification. + /// + /// # Correctness + /// + /// Output requests should be wrapped in a timeout, so that + /// out-of-order and invalid requests do not hang indefinitely. + /// + /// Outdated requests are pruned on a regular basis. + AwaitOutput(transparent::OutPoint), + /// Get all the [`VerifiedUnminedTx`] in the mempool. /// /// Equivalent to `TransactionsById(TransactionIds)`, diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index acab316062d..861be1ac857 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -50,6 +50,7 @@ mod crawler; pub mod downloads; mod error; pub mod gossip; +mod pending_outputs; mod queue_checker; mod storage; @@ -741,6 +742,16 @@ impl Service for Mempool { async move { Ok(Response::UnspentOutput(res)) }.boxed() } + Request::AwaitOutput(outpoint) => { + trace!(?req, "got mempool request"); + + let response_fut = storage.pending_outputs.queue(outpoint); + + trace!("answered mempool request"); + + response_fut.boxed() + } + #[cfg(feature = "getblocktemplate-rpcs")] Request::FullTransactions => { trace!(?req, "got mempool request"); @@ -820,6 +831,7 @@ impl Service for Mempool { Request::TransactionsById(_) => Response::Transactions(Default::default()), Request::TransactionsByMinedId(_) => Response::Transactions(Default::default()), Request::UnspentOutput(_) => Response::UnspentOutput(None), + Request::AwaitOutput(_) => Response::UnspentOutput(None), #[cfg(feature = "getblocktemplate-rpcs")] Request::FullTransactions => { diff --git a/zebrad/src/components/mempool/pending_outputs.rs b/zebrad/src/components/mempool/pending_outputs.rs new file mode 100644 index 00000000000..29a30b61e54 --- /dev/null +++ b/zebrad/src/components/mempool/pending_outputs.rs @@ -0,0 +1,64 @@ +//! Pending [`transparent::Output`] tracker for [`AwaitOutput` requests](zebra_node_services::Mempool::Request::AwaitOutput). + +use std::{collections::HashMap, future::Future}; + +use tokio::sync::broadcast; + +use tower::BoxError; +use zebra_chain::transparent; + +use zebra_node_services::mempool::Response; + +#[derive(Debug, Default)] +pub struct PendingOutputs(HashMap>); + +impl PendingOutputs { + /// Returns a future that will resolve to the `transparent::Output` pointed + /// to by the given `transparent::OutPoint` when it is available. + pub fn queue( + &mut self, + outpoint: transparent::OutPoint, + ) -> impl Future> { + let mut receiver = self + .0 + .entry(outpoint) + .or_insert_with(|| { + let (sender, _) = broadcast::channel(1); + sender + }) + .subscribe(); + + async move { + receiver + .recv() + .await + .map(Some) + .map(Response::UnspentOutput) + .map_err(BoxError::from) + } + } + + /// Notify all requests waiting for the [`transparent::Output`] pointed to by + /// the given [`transparent::OutPoint`] that the [`transparent::Output`] has + /// arrived. + #[inline] + pub fn respond(&mut self, outpoint: &transparent::OutPoint, output: transparent::Output) { + if let Some(sender) = self.0.remove(outpoint) { + // Adding the outpoint as a field lets us cross-reference + // with the trace of the verification that made the request. + tracing::trace!(?outpoint, "found pending mempool output"); + let _ = sender.send(output); + } + } + + /// Scan the set of waiting Output requests for channels where all receivers + /// have been dropped and remove the corresponding sender. + pub fn prune(&mut self) { + self.0.retain(|_, chan| chan.receiver_count() > 0); + } + + /// Returns the number of Outputs that are being waited on. + pub fn len(&self) -> usize { + self.0.len() + } +} diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 3b37b049e22..39866a97acb 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -22,7 +22,10 @@ use zebra_chain::{ }; use self::{eviction_list::EvictionList, verified_set::VerifiedSet}; -use super::{config, downloads::TransactionDownloadVerifyError, MempoolError}; +use super::{ + config, downloads::TransactionDownloadVerifyError, pending_outputs::PendingOutputs, + MempoolError, +}; #[cfg(any(test, feature = "proptest-impl"))] use proptest_derive::Arbitrary; @@ -119,13 +122,13 @@ pub enum RejectionError { } /// Hold mempool verified and rejected mempool transactions. -// -// Add a `pending_outputs` field similar to the `pending_utxos` field in the state service -// for queuing outpoint queries. pub struct Storage { /// The set of verified transactions in the mempool. verified: VerifiedSet, + /// The set of outpoints with pending requests for their associated transparent::Output. + pub(super) pending_outputs: PendingOutputs, + /// The set of transactions rejected due to bad authorizations, or for other /// reasons, and their rejection reasons. These rejections only apply to the /// current tip. @@ -175,6 +178,7 @@ impl Storage { tx_cost_limit: config.tx_cost_limit, eviction_memory_time: config.eviction_memory_time, verified: Default::default(), + pending_outputs: Default::default(), tip_rejected_exact: Default::default(), tip_rejected_same_effects: Default::default(), chain_rejected_same_effects: Default::default(), @@ -228,7 +232,10 @@ impl Storage { // Then, we try to insert into the pool. If this fails the transaction is rejected. let mut result = Ok(tx_id); - if let Err(rejection_error) = self.verified.insert(tx, spent_mempool_outpoints) { + if let Err(rejection_error) = + self.verified + .insert(tx, spent_mempool_outpoints, &mut self.pending_outputs) + { tracing::debug!( ?tx_id, ?rejection_error, diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 31d5b2f7595..01c961f5cc1 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -12,6 +12,8 @@ use zebra_chain::{ transparent, }; +use crate::components::mempool::pending_outputs::PendingOutputs; + use super::super::SameEffectsTipRejectionError; // Imports for doc links @@ -184,6 +186,7 @@ impl VerifiedSet { &mut self, transaction: VerifiedUnminedTx, spent_mempool_outpoints: Vec, + pending_outputs: &mut PendingOutputs, ) -> Result<(), SameEffectsTipRejectionError> { if self.has_spend_conflicts(&transaction.transaction) { return Err(SameEffectsTipRejectionError::SpendConflict); @@ -199,12 +202,15 @@ impl VerifiedSet { } let tx_id = transaction.transaction.id; - - // TODO: Update `transaction_dependencies` self.transaction_dependencies .add(tx_id.mined_id(), spent_mempool_outpoints); - self.cache_outputs_from(tx_id.mined_id(), &transaction.transaction.transaction); + self.cache_outputs_and_respond_to_pending_output_requests_from( + tx_id.mined_id(), + &transaction.transaction.transaction, + pending_outputs, + ); + self.transactions_serialized_size += transaction.transaction.size; self.total_cost += transaction.cost(); self.transactions.insert(tx_id, transaction); @@ -313,11 +319,17 @@ impl VerifiedSet { || Self::has_conflicts(&self.orchard_nullifiers, tx.orchard_nullifiers().copied()) } - /// Inserts the transaction's outputs into the internal caches. - fn cache_outputs_from(&mut self, tx_hash: transaction::Hash, tx: &Transaction) { + /// Inserts the transaction's outputs into the internal caches and responds to pending output requests. + fn cache_outputs_and_respond_to_pending_output_requests_from( + &mut self, + tx_hash: transaction::Hash, + tx: &Transaction, + pending_outputs: &mut PendingOutputs, + ) { for (index, output) in tx.outputs().iter().cloned().enumerate() { - self.created_outputs - .insert(transparent::OutPoint::from_usize(tx_hash, index), output); + let outpoint = transparent::OutPoint::from_usize(tx_hash, index); + self.created_outputs.insert(outpoint, output.clone()); + pending_outputs.respond(&outpoint, output) } self.spent_outpoints.extend(tx.spent_outpoints()); From ae87a2797419864d308b9fa18218619e5dfc8cf5 Mon Sep 17 00:00:00 2001 From: ar Date: Mon, 16 Sep 2024 16:07:33 -0400 Subject: [PATCH 17/69] replaces `UnminedTxId` type with `transaction::Hash` in mempool's verified set --- zebrad/src/components/mempool.rs | 10 ++-- zebrad/src/components/mempool/storage.rs | 60 ++++++++++--------- .../mempool/storage/verified_set.rs | 34 ++++++----- 3 files changed, 57 insertions(+), 47 deletions(-) diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index 861be1ac857..bc86cc91360 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -391,10 +391,11 @@ impl Mempool { /// Remove expired transaction ids from a given list of inserted ones. fn remove_expired_from_peer_list( send_to_peers_ids: &HashSet, - expired_transactions: &HashSet, + expired_transactions: &HashSet, ) -> HashSet { send_to_peers_ids - .difference(expired_transactions) + .iter() + .filter(|id| !expired_transactions.contains(&id.mined_id())) .copied() .collect() } @@ -620,7 +621,7 @@ impl Service for Mempool { tracing::debug!(?txid, ?error, "mempool transaction failed to verify"); metrics::counter!("mempool.failed.verify.tasks.total", "reason" => error.to_string()).increment(1); - storage.reject_if_needed(txid, error); + storage.reject_if_needed(txid.mined_id(), error); } Err(_elapsed) => { // A timeout happens when the stream hangs waiting for another service, @@ -791,7 +792,8 @@ impl Service for Mempool { MempoolError, > { let (rsp_tx, rsp_rx) = oneshot::channel(); - storage.should_download_or_verify(gossiped_tx.id())?; + storage + .should_download_or_verify(gossiped_tx.id().mined_id())?; tx_downloads .download_if_needed_and_verify(gossiped_tx, Some(rsp_tx))?; diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 39866a97acb..ccc557e12c9 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -134,7 +134,7 @@ pub struct Storage { /// current tip. /// /// Only transactions with the exact [`UnminedTxId`] are invalid. - tip_rejected_exact: HashMap, + tip_rejected_exact: HashMap, /// A set of transactions rejected for their effects, and their rejection /// reasons. These rejections only apply to the current tip. @@ -202,7 +202,8 @@ impl Storage { // # Security // // This method must call `reject`, rather than modifying the rejection lists directly. - let tx_id = tx.transaction.id; + let unmined_tx_id = tx.transaction.id; + let tx_id = unmined_tx_id.mined_id(); // First, check if we have a cached rejection for this transaction. if let Some(error) = self.rejection_error(&tx_id) { @@ -231,7 +232,7 @@ impl Storage { } // Then, we try to insert into the pool. If this fails the transaction is rejected. - let mut result = Ok(tx_id); + let mut result = Ok(unmined_tx_id); if let Err(rejection_error) = self.verified .insert(tx, spent_mempool_outpoints, &mut self.pending_outputs) @@ -272,13 +273,13 @@ impl Storage { // > Add the txid and the current time to RecentlyEvicted, dropping the oldest entry in // > RecentlyEvicted if necessary to keep it to at most `eviction_memory_entries entries`. self.reject( - victim_tx.transaction.id, + victim_tx.transaction.id.mined_id(), SameEffectsChainRejectionError::RandomlyEvicted.into(), ); // If this transaction gets evicted, set its result to the same error // (we could return here, but we still want to check the mempool size) - if victim_tx.transaction.id == tx_id { + if victim_tx.transaction.id == unmined_tx_id { result = Err(SameEffectsChainRejectionError::RandomlyEvicted.into()); } } @@ -374,14 +375,14 @@ impl Storage { self.reject( // the reject and rejection_error fns that store and check `SameEffectsChainRejectionError`s // only use the mined id, so using `Legacy` ids will apply to v5 transactions as well. - UnminedTxId::Legacy(mined_id), + mined_id, SameEffectsChainRejectionError::Mined.into(), ); } for duplicate_spend_id in duplicate_spend_ids { self.reject( - duplicate_spend_id, + duplicate_spend_id.mined_id(), SameEffectsChainRejectionError::DuplicateSpend.into(), ); } @@ -433,7 +434,7 @@ impl Storage { /// /// Each [`VerifiedUnminedTx`] contains an [`UnminedTx`], /// and adds extra fields from the transaction verifier result. - pub fn transactions(&self) -> &HashMap { + pub fn transactions(&self) -> &HashMap { self.verified.transactions() } @@ -471,9 +472,11 @@ impl Storage { &self, tx_ids: HashSet, ) -> impl Iterator { - tx_ids - .into_iter() - .filter_map(|tx_id| self.transactions().get(&tx_id).map(|tx| &tx.transaction)) + tx_ids.into_iter().filter_map(|tx_id| { + self.transactions() + .get(&tx_id.mined_id()) + .map(|tx| &tx.transaction) + }) } /// Returns the set of [`UnminedTx`]es with matching [`transaction::Hash`]es @@ -488,7 +491,7 @@ impl Storage { self.verified .transactions() .iter() - .filter(move |(tx_id, _)| tx_ids.contains(&tx_id.mined_id())) + .filter(move |(tx_id, _)| tx_ids.contains(&tx_id)) .map(|(_, tx)| &tx.transaction) } @@ -497,8 +500,8 @@ impl Storage { /// /// This matches the exact transaction, with identical blockchain effects, /// signatures, and proofs. - pub fn contains_transaction_exact(&self, txid: &UnminedTxId) -> bool { - self.verified.contains(txid) + pub fn contains_transaction_exact(&self, tx_id: &transaction::Hash) -> bool { + self.verified.contains(tx_id) } /// Returns the number of rejected [`UnminedTxId`]s or [`transaction::Hash`]es. @@ -516,13 +519,13 @@ impl Storage { } /// Add a transaction to the rejected list for the given reason. - pub fn reject(&mut self, txid: UnminedTxId, reason: RejectionError) { + pub fn reject(&mut self, tx_id: transaction::Hash, reason: RejectionError) { match reason { RejectionError::ExactTip(e) => { - self.tip_rejected_exact.insert(txid, e); + self.tip_rejected_exact.insert(tx_id, e); } RejectionError::SameEffectsTip(e) => { - self.tip_rejected_same_effects.insert(txid.mined_id(), e); + self.tip_rejected_same_effects.insert(tx_id, e); } RejectionError::SameEffectsChain(e) => { let eviction_memory_time = self.eviction_memory_time; @@ -531,7 +534,7 @@ impl Storage { .or_insert_with(|| { EvictionList::new(MAX_EVICTION_MEMORY_ENTRIES, eviction_memory_time) }) - .insert(txid.mined_id()); + .insert(tx_id); } } self.limit_rejection_list_memory(); @@ -543,17 +546,17 @@ impl Storage { /// This matches transactions based on each rejection list's matching rule. /// /// Returns an arbitrary error if the transaction is in multiple lists. - pub fn rejection_error(&self, txid: &UnminedTxId) -> Option { + pub fn rejection_error(&self, txid: &transaction::Hash) -> Option { if let Some(error) = self.tip_rejected_exact.get(txid) { return Some(error.clone().into()); } - if let Some(error) = self.tip_rejected_same_effects.get(&txid.mined_id()) { + if let Some(error) = self.tip_rejected_same_effects.get(&txid) { return Some(error.clone().into()); } for (error, set) in self.chain_rejected_same_effects.iter() { - if set.contains_key(&txid.mined_id()) { + if set.contains_key(&txid) { return Some(error.clone().into()); } } @@ -570,20 +573,20 @@ impl Storage { ) -> impl Iterator + '_ { tx_ids .into_iter() - .filter(move |txid| self.contains_rejected(txid)) + .filter(move |txid| self.contains_rejected(&txid.mined_id())) } /// Returns `true` if a transaction matching the supplied [`UnminedTxId`] is in /// the mempool rejected list. /// /// This matches transactions based on each rejection list's matching rule. - pub fn contains_rejected(&self, txid: &UnminedTxId) -> bool { + pub fn contains_rejected(&self, txid: &transaction::Hash) -> bool { self.rejection_error(txid).is_some() } /// Add a transaction that failed download and verification to the rejected list /// if needed, depending on the reason for the failure. - pub fn reject_if_needed(&mut self, txid: UnminedTxId, e: TransactionDownloadVerifyError) { + pub fn reject_if_needed(&mut self, txid: transaction::Hash, e: TransactionDownloadVerifyError) { match e { // Rejecting a transaction already in state would speed up further // download attempts without checking the state. However it would @@ -623,7 +626,7 @@ impl Storage { pub fn remove_expired_transactions( &mut self, tip_height: zebra_chain::block::Height, - ) -> HashSet { + ) -> HashSet { let mut txid_set = HashSet::new(); // we need a separate set, since reject() takes the original unmined ID, // then extracts the mined ID out of it @@ -632,7 +635,7 @@ impl Storage { for (&tx_id, tx) in self.transactions() { if let Some(expiry_height) = tx.transaction.transaction.expiry_height() { if tip_height >= expiry_height { - txid_set.insert(tx_id.mined_id()); + txid_set.insert(tx_id); unmined_id_set.insert(tx_id); } } @@ -654,7 +657,10 @@ impl Storage { /// /// If it is already in the mempool (or in its rejected list) /// then it shouldn't be downloaded/verified. - pub fn should_download_or_verify(&mut self, txid: UnminedTxId) -> Result<(), MempoolError> { + pub fn should_download_or_verify( + &mut self, + txid: transaction::Hash, + ) -> Result<(), MempoolError> { // Check if the transaction is already in the mempool. if self.contains_transaction_exact(&txid) { return Err(MempoolError::InMempool); diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 01c961f5cc1..982fa66800f 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -8,7 +8,7 @@ use std::{ use zebra_chain::{ orchard, sapling, sprout, - transaction::{self, Transaction, UnminedTx, UnminedTxId, VerifiedUnminedTx}, + transaction::{self, Transaction, UnminedTx, VerifiedUnminedTx}, transparent, }; @@ -28,7 +28,7 @@ struct TransactionDependencies { /// Lists of mempool transactions that spend UTXOs created /// by a mempool transaction. - dependants: HashMap>, + dependents: HashMap>, } impl TransactionDependencies { @@ -39,17 +39,17 @@ impl TransactionDependencies { // TODO: Order transactions in block templates based on their dependencies fn add( &mut self, - dependant: transaction::Hash, + dependent: transaction::Hash, spent_mempool_outpoints: Vec, ) { for &spent_mempool_outpoint in &spent_mempool_outpoints { - self.dependants + self.dependents .entry(spent_mempool_outpoint.hash) .or_default() - .insert(dependant); + .insert(dependent); } - self.dependencies.entry(dependant).or_default().extend( + self.dependencies.entry(dependent).or_default().extend( spent_mempool_outpoints .into_iter() .map(|outpoint| outpoint.hash), @@ -61,12 +61,12 @@ impl TransactionDependencies { /// this [`TransactionDependencies`]. /// /// Returns a list of transaction hashes that have been removed. - fn _remove(&mut self, tx_hash: &transaction::Hash) -> HashSet { + fn remove(&mut self, tx_hash: &transaction::Hash) -> HashSet { for dependencies in self.dependencies.values_mut() { dependencies.remove(tx_hash); } - self.dependants.remove(tx_hash).unwrap_or_default() + self.dependents.remove(tx_hash).unwrap_or_default() } } @@ -84,7 +84,7 @@ impl TransactionDependencies { #[derive(Default)] pub struct VerifiedSet { /// The set of verified transactions in the mempool. - transactions: HashMap, + transactions: HashMap, /// A map of dependencies between transactions in the mempool that /// spend or create outputs of other transactions in the mempool. @@ -125,7 +125,7 @@ impl Drop for VerifiedSet { impl VerifiedSet { /// Returns a reference to the [`HashMap`] of [`VerifiedUnminedTx`]s in the set. - pub fn transactions(&self) -> &HashMap { + pub fn transactions(&self) -> &HashMap { &self.transactions } @@ -157,7 +157,7 @@ impl VerifiedSet { /// Returns `true` if the set of verified transactions contains the transaction with the /// specified [`UnminedTxId`]. - pub fn contains(&self, id: &UnminedTxId) -> bool { + pub fn contains(&self, id: &transaction::Hash) -> bool { self.transactions.contains_key(id) } @@ -201,12 +201,12 @@ impl VerifiedSet { } } - let tx_id = transaction.transaction.id; + let tx_id = transaction.transaction.id.mined_id(); self.transaction_dependencies - .add(tx_id.mined_id(), spent_mempool_outpoints); + .add(tx_id, spent_mempool_outpoints); self.cache_outputs_and_respond_to_pending_output_requests_from( - tx_id.mined_id(), + tx_id, &transaction.transaction.transaction, pending_outputs, ); @@ -247,7 +247,7 @@ impl VerifiedSet { use rand::distributions::{Distribution, WeightedIndex}; use rand::prelude::thread_rng; - let (keys, weights): (Vec, Vec) = self + let (keys, weights): (Vec, Vec) = self .transactions .iter() .map(|(&tx_id, tx)| (tx_id, tx.eviction_weight())) @@ -286,7 +286,7 @@ impl VerifiedSet { /// Removes a transaction from the set. /// /// Also removes its outputs from the internal caches. - fn remove(&mut self, key_to_remove: &UnminedTxId) -> VerifiedUnminedTx { + fn remove(&mut self, key_to_remove: &transaction::Hash) -> VerifiedUnminedTx { // TODO: // - Remove any dependant transactions as well // - Update `transaction_dependencies` @@ -296,6 +296,8 @@ impl VerifiedSet { .remove(key_to_remove) .expect("invalid transaction key"); + self.transaction_dependencies.remove(key_to_remove); + self.transactions_serialized_size -= removed_tx.transaction.size; self.total_cost -= removed_tx.cost(); self.remove_outputs(&removed_tx.transaction); From 61e6e2127260b12d56615226fb412757f35ee8ef Mon Sep 17 00:00:00 2001 From: ar Date: Mon, 16 Sep 2024 16:18:38 -0400 Subject: [PATCH 18/69] prune pending outputs when rejecting and removing same effects. --- zebrad/src/components/mempool/storage.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index ccc557e12c9..445ae6a9479 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -315,6 +315,7 @@ impl Storage { /// - Returns the number of transactions which were removed. /// - Removes from the 'verified' set, if present. /// Maintains the order in which the other unmined transactions have been inserted into the mempool. + /// - Prunes `pending_outputs` of any closed channels. /// /// Reject and remove transactions from the mempool that contain any spent outpoints or revealed /// nullifiers from the passed in `transactions`. @@ -387,6 +388,8 @@ impl Storage { ); } + self.pending_outputs.prune(); + num_removed_mined + num_removed_duplicate_spend } From bebe50ef1d4c358f4ea8a96c81c1cc9645b3ddf2 Mon Sep 17 00:00:00 2001 From: ar Date: Mon, 16 Sep 2024 16:30:56 -0400 Subject: [PATCH 19/69] Remove dependent transactions from verified set when removing a tx --- zebrad/src/components/mempool/storage.rs | 30 +++++++++------- .../mempool/storage/verified_set.rs | 36 ++++++++++--------- 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 445ae6a9479..bafca85c74b 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -265,22 +265,26 @@ impl Storage { // > EvictTransaction MUST do the following: // > Select a random transaction to evict, with probability in direct proportion to // > eviction weight. (...) Remove it from the mempool. - let victim_tx = self - .verified - .evict_one() - .expect("mempool is empty, but was expected to be full"); + let victim_txs = self.verified.evict_one(); - // > Add the txid and the current time to RecentlyEvicted, dropping the oldest entry in - // > RecentlyEvicted if necessary to keep it to at most `eviction_memory_entries entries`. - self.reject( - victim_tx.transaction.id.mined_id(), - SameEffectsChainRejectionError::RandomlyEvicted.into(), + assert!( + !victim_txs.is_empty(), + "mempool is empty, but was expected to be full" ); - // If this transaction gets evicted, set its result to the same error - // (we could return here, but we still want to check the mempool size) - if victim_tx.transaction.id == unmined_tx_id { - result = Err(SameEffectsChainRejectionError::RandomlyEvicted.into()); + for victim_tx in victim_txs { + // > Add the txid and the current time to RecentlyEvicted, dropping the oldest entry in + // > RecentlyEvicted if necessary to keep it to at most `eviction_memory_entries entries`. + self.reject( + victim_tx.transaction.id.mined_id(), + SameEffectsChainRejectionError::RandomlyEvicted.into(), + ); + + // If this transaction gets evicted, set its result to the same error + // (we could return here, but we still want to check the mempool size) + if victim_tx.transaction.id == unmined_tx_id { + result = Err(SameEffectsChainRejectionError::RandomlyEvicted.into()); + } } } diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 982fa66800f..22d41881a32 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -240,9 +240,9 @@ impl VerifiedSet { /// /// [ZIP-401]: https://zips.z.cash/zip-0401 #[allow(clippy::unwrap_in_result)] - pub fn evict_one(&mut self) -> Option { + pub fn evict_one(&mut self) -> Vec { if self.transactions.is_empty() { - None + Vec::new() } else { use rand::distributions::{Distribution, WeightedIndex}; use rand::prelude::thread_rng; @@ -260,7 +260,7 @@ impl VerifiedSet { .get(dist.sample(&mut thread_rng())) .expect("should have a key at every index in the distribution"); - Some(self.remove(key_to_remove)) + self.remove(key_to_remove) } } @@ -274,10 +274,10 @@ impl VerifiedSet { .filter_map(|(&tx_id, tx)| predicate(tx).then_some(tx_id)) .collect(); - let removed_count = keys_to_remove.len(); + let mut removed_count = 0; for key_to_remove in keys_to_remove { - self.remove(&key_to_remove); + removed_count += self.remove(&key_to_remove).len(); } removed_count @@ -286,17 +286,12 @@ impl VerifiedSet { /// Removes a transaction from the set. /// /// Also removes its outputs from the internal caches. - fn remove(&mut self, key_to_remove: &transaction::Hash) -> VerifiedUnminedTx { - // TODO: - // - Remove any dependant transactions as well - // - Update `transaction_dependencies` - - let removed_tx = self - .transactions - .remove(key_to_remove) - .expect("invalid transaction key"); - - self.transaction_dependencies.remove(key_to_remove); + fn remove(&mut self, key_to_remove: &transaction::Hash) -> Vec { + let Some(removed_tx) = self.transactions.remove(key_to_remove) else { + // Transaction key not found, it may have been removed as a dependent + // of another transaction that was removed. + return Vec::new(); + }; self.transactions_serialized_size -= removed_tx.transaction.size; self.total_cost -= removed_tx.cost(); @@ -304,7 +299,14 @@ impl VerifiedSet { self.update_metrics(); - removed_tx + let mut removed_txs = vec![removed_tx]; + let dependent_transactions = self.transaction_dependencies.remove(key_to_remove); + + for dependent_tx in dependent_transactions { + removed_txs.extend(self.remove(&dependent_tx)); + } + + removed_txs } /// Returns `true` if the given `transaction` has any spend conflicts with transactions in the From 29654b42074be8ea3e2074c4cfdc77b4d724e3b2 Mon Sep 17 00:00:00 2001 From: ar Date: Mon, 16 Sep 2024 16:48:29 -0400 Subject: [PATCH 20/69] updates tests --- .../components/mempool/storage/tests/prop.rs | 22 +++++++++---------- .../mempool/storage/tests/vectors.rs | 20 ++++++++--------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/zebrad/src/components/mempool/storage/tests/prop.rs b/zebrad/src/components/mempool/storage/tests/prop.rs index 5c26b534783..3b398e20ebf 100644 --- a/zebrad/src/components/mempool/storage/tests/prop.rs +++ b/zebrad/src/components/mempool/storage/tests/prop.rs @@ -86,7 +86,7 @@ proptest! { }); for rejection in unique_ids { - storage.reject(rejection, SameEffectsTipRejectionError::SpendConflict.into()); + storage.reject(rejection.mined_id(), SameEffectsTipRejectionError::SpendConflict.into()); } // Make sure there were no duplicates @@ -135,7 +135,7 @@ proptest! { }); for rejection in unique_ids { - storage.reject(rejection, SameEffectsChainRejectionError::RandomlyEvicted.into()); + storage.reject(rejection.mined_id(), SameEffectsChainRejectionError::RandomlyEvicted.into()); } // Make sure there were no duplicates @@ -202,7 +202,7 @@ proptest! { }); for (index, rejection) in unique_ids.enumerate() { - storage.reject(rejection, rejection_error.clone()); + storage.reject(rejection.mined_id(), rejection_error.clone()); if index == MAX_EVICTION_MEMORY_ENTRIES - 1 { // Make sure there were no duplicates @@ -249,9 +249,9 @@ proptest! { rejection_template }).collect(); - storage.reject(unique_ids[0], SameEffectsChainRejectionError::RandomlyEvicted.into()); + storage.reject(unique_ids[0].mined_id(), SameEffectsChainRejectionError::RandomlyEvicted.into()); thread::sleep(Duration::from_millis(11)); - storage.reject(unique_ids[1], SameEffectsChainRejectionError::RandomlyEvicted.into()); + storage.reject(unique_ids[1].mined_id(), SameEffectsChainRejectionError::RandomlyEvicted.into()); prop_assert_eq!(storage.rejected_transaction_count(), 1); } @@ -288,7 +288,7 @@ proptest! { Err(MempoolError::StorageEffectsTip(SameEffectsTipRejectionError::SpendConflict)) ); - prop_assert!(storage.contains_rejected(&id_to_reject)); + prop_assert!(storage.contains_rejected(&id_to_reject.mined_id())); storage.clear(); } @@ -341,7 +341,7 @@ proptest! { Err(MempoolError::StorageEffectsTip(SameEffectsTipRejectionError::SpendConflict)) ); - prop_assert!(storage.contains_rejected(&id_to_reject)); + prop_assert!(storage.contains_rejected(&id_to_reject.mined_id())); prop_assert_eq!( storage.insert(second_transaction_to_accept, Vec::new()), @@ -377,7 +377,7 @@ proptest! { // Check that the inserted transactions are still there. for transaction_id in &inserted_transactions { - prop_assert!(storage.contains_transaction_exact(transaction_id)); + prop_assert!(storage.contains_transaction_exact(&transaction_id.mined_id())); } // Remove some transactions. @@ -387,7 +387,7 @@ proptest! { let num_removals = storage.reject_and_remove_same_effects(mined_ids_to_remove, vec![]); for &removed_transaction_id in mined_ids_to_remove.iter() { prop_assert_eq!( - storage.rejection_error(&UnminedTxId::Legacy(removed_transaction_id)), + storage.rejection_error(&removed_transaction_id), Some(SameEffectsChainRejectionError::Mined.into()) ); } @@ -399,14 +399,14 @@ proptest! { let removed_transactions = input.removed_transaction_ids(); for removed_transaction_id in &removed_transactions { - prop_assert!(!storage.contains_transaction_exact(removed_transaction_id)); + prop_assert!(!storage.contains_transaction_exact(&removed_transaction_id.mined_id())); } // Check that the remaining transactions are still in the storage. let remaining_transactions = inserted_transactions.difference(&removed_transactions); for remaining_transaction_id in remaining_transactions { - prop_assert!(storage.contains_transaction_exact(remaining_transaction_id)); + prop_assert!(storage.contains_transaction_exact(&remaining_transaction_id.mined_id())); } } } diff --git a/zebrad/src/components/mempool/storage/tests/vectors.rs b/zebrad/src/components/mempool/storage/tests/vectors.rs index 4d3a00297f6..5d12a8620c4 100644 --- a/zebrad/src/components/mempool/storage/tests/vectors.rs +++ b/zebrad/src/components/mempool/storage/tests/vectors.rs @@ -43,14 +43,14 @@ fn mempool_storage_crud_exact_mainnet() { let _ = storage.insert(unmined_tx.clone(), Vec::new()); // Check that it is in the mempool, and not rejected. - assert!(storage.contains_transaction_exact(&unmined_tx.transaction.id)); + assert!(storage.contains_transaction_exact(&unmined_tx.transaction.id.mined_id())); // Remove tx let removal_count = storage.remove_exact(&iter::once(unmined_tx.transaction.id).collect()); // Check that it is /not/ in the mempool. assert_eq!(removal_count, 1); - assert!(!storage.contains_transaction_exact(&unmined_tx.transaction.id)); + assert!(!storage.contains_transaction_exact(&unmined_tx.transaction.id.mined_id())); } #[test] @@ -124,7 +124,7 @@ fn mempool_storage_basic_for_network(network: Network) -> Result<()> { // Test if rejected transactions were actually rejected. for tx in some_rejected_transactions.iter() { - assert!(!storage.contains_transaction_exact(&tx.transaction.id)); + assert!(!storage.contains_transaction_exact(&tx.transaction.id.mined_id())); } // Query all the ids we have for rejected, get back `total - MEMPOOL_SIZE` @@ -170,7 +170,7 @@ fn mempool_storage_crud_same_effects_mainnet() { let _ = storage.insert(unmined_tx_1.clone(), Vec::new()); // Check that it is in the mempool, and not rejected. - assert!(storage.contains_transaction_exact(&unmined_tx_1.transaction.id)); + assert!(storage.contains_transaction_exact(&unmined_tx_1.transaction.id.mined_id())); // Reject and remove mined tx let removal_count = storage.reject_and_remove_same_effects( @@ -180,11 +180,11 @@ fn mempool_storage_crud_same_effects_mainnet() { // Check that it is /not/ in the mempool as a verified transaction. assert_eq!(removal_count, 1); - assert!(!storage.contains_transaction_exact(&unmined_tx_1.transaction.id)); + assert!(!storage.contains_transaction_exact(&unmined_tx_1.transaction.id.mined_id())); // Check that it's rejection is cached in the chain_rejected_same_effects' `Mined` eviction list. assert_eq!( - storage.rejection_error(&unmined_tx_1.transaction.id), + storage.rejection_error(&unmined_tx_1.transaction.id.mined_id()), Some(SameEffectsChainRejectionError::Mined.into()) ); assert_eq!( @@ -210,7 +210,7 @@ fn mempool_storage_crud_same_effects_mainnet() { ); // Check that it is in the mempool, and not rejected. - assert!(storage.contains_transaction_exact(&unmined_tx_2.transaction.id)); + assert!(storage.contains_transaction_exact(&unmined_tx_2.transaction.id.mined_id())); // Reject and remove duplicate spend tx let removal_count = storage.reject_and_remove_same_effects( @@ -220,11 +220,11 @@ fn mempool_storage_crud_same_effects_mainnet() { // Check that it is /not/ in the mempool as a verified transaction. assert_eq!(removal_count, 1); - assert!(!storage.contains_transaction_exact(&unmined_tx_2.transaction.id)); + assert!(!storage.contains_transaction_exact(&unmined_tx_2.transaction.id.mined_id())); // Check that it's rejection is cached in the chain_rejected_same_effects' `SpendConflict` eviction list. assert_eq!( - storage.rejection_error(&unmined_tx_2.transaction.id), + storage.rejection_error(&unmined_tx_2.transaction.id.mined_id()), Some(SameEffectsChainRejectionError::DuplicateSpend.into()) ); assert_eq!( @@ -281,7 +281,7 @@ fn mempool_expired_basic_for_network(network: Network) -> Result<()> { // remove_expired_transactions() will return what was removed let expired = storage.remove_expired_transactions(Height(1)); - assert!(expired.contains(&tx_id)); + assert!(expired.contains(&tx_id.mined_id())); let everything_in_mempool: HashSet = storage.tx_ids().collect(); assert_eq!(everything_in_mempool.len(), 0); From aed637ac0b96a2b533f8fc5fe34d32c67d8f6e28 Mon Sep 17 00:00:00 2001 From: ar Date: Mon, 16 Sep 2024 17:01:21 -0400 Subject: [PATCH 21/69] appeases clippy. --- zebrad/src/components/mempool/storage.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index bafca85c74b..3eb2fea688c 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -498,7 +498,7 @@ impl Storage { self.verified .transactions() .iter() - .filter(move |(tx_id, _)| tx_ids.contains(&tx_id)) + .filter(move |(tx_id, _)| tx_ids.contains(tx_id)) .map(|(_, tx)| &tx.transaction) } @@ -558,12 +558,12 @@ impl Storage { return Some(error.clone().into()); } - if let Some(error) = self.tip_rejected_same_effects.get(&txid) { + if let Some(error) = self.tip_rejected_same_effects.get(txid) { return Some(error.clone().into()); } for (error, set) in self.chain_rejected_same_effects.iter() { - if set.contains_key(&txid) { + if set.contains_key(txid) { return Some(error.clone().into()); } } From a8389344bb108e67d2edd05659c1c35f69e217b9 Mon Sep 17 00:00:00 2001 From: ar Date: Mon, 16 Sep 2024 17:47:52 -0400 Subject: [PATCH 22/69] removes unused `len()` method --- zebrad/src/components/mempool/pending_outputs.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/zebrad/src/components/mempool/pending_outputs.rs b/zebrad/src/components/mempool/pending_outputs.rs index 29a30b61e54..7e7737397b9 100644 --- a/zebrad/src/components/mempool/pending_outputs.rs +++ b/zebrad/src/components/mempool/pending_outputs.rs @@ -56,9 +56,4 @@ impl PendingOutputs { pub fn prune(&mut self) { self.0.retain(|_, chan| chan.receiver_count() > 0); } - - /// Returns the number of Outputs that are being waited on. - pub fn len(&self) -> usize { - self.0.len() - } } From 695a258093eccae6a8fb401599142b0cc7b9d43a Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 16 Sep 2024 18:34:52 -0400 Subject: [PATCH 23/69] fixes doc links --- zebrad/src/components/mempool/pending_outputs.rs | 2 +- zebrad/src/components/mempool/storage/verified_set.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zebrad/src/components/mempool/pending_outputs.rs b/zebrad/src/components/mempool/pending_outputs.rs index 7e7737397b9..234eb6ab235 100644 --- a/zebrad/src/components/mempool/pending_outputs.rs +++ b/zebrad/src/components/mempool/pending_outputs.rs @@ -1,4 +1,4 @@ -//! Pending [`transparent::Output`] tracker for [`AwaitOutput` requests](zebra_node_services::Mempool::Request::AwaitOutput). +//! Pending [`transparent::Output`] tracker for [`AwaitOutput` requests](zebra_node_services::mempool::Request::AwaitOutput). use std::{collections::HashMap, future::Future}; diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 22d41881a32..b42de90b55f 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -156,7 +156,7 @@ impl VerifiedSet { } /// Returns `true` if the set of verified transactions contains the transaction with the - /// specified [`UnminedTxId`]. + /// specified [`transaction::Hash`]. pub fn contains(&self, id: &transaction::Hash) -> bool { self.transactions.contains_key(id) } From bc981c0736e43ea9310802655bf0e8d5eb400bba Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 16 Sep 2024 19:47:40 -0400 Subject: [PATCH 24/69] Adds transaction dependencies to the `FullTransactions` response, let the caller handle it (required to avoid moving zip317 tx selection code to mempool) --- zebra-node-services/src/mempool.rs | 5 ++- zebra-rpc/src/methods.rs | 1 + zebrad/src/components/mempool.rs | 2 + zebrad/src/components/mempool/storage.rs | 7 +++ .../mempool/storage/verified_set.rs | 44 ++++++++++++------- 5 files changed, 42 insertions(+), 17 deletions(-) diff --git a/zebra-node-services/src/mempool.rs b/zebra-node-services/src/mempool.rs index 09b70f8e258..86e8c3f2a6c 100644 --- a/zebra-node-services/src/mempool.rs +++ b/zebra-node-services/src/mempool.rs @@ -2,7 +2,7 @@ //! //! A service that manages known unmined Zcash transactions. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use tokio::sync::oneshot; use zebra_chain::{ @@ -135,6 +135,9 @@ pub enum Response { /// All [`VerifiedUnminedTx`]s in the mempool transactions: Vec, + /// All transaction dependencies in the mempool + transaction_dependencies: HashMap>, + /// Last seen chain tip hash by mempool service last_seen_tip_hash: zebra_chain::block::Hash, }, diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index 471d542922c..9dc45e0c968 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -952,6 +952,7 @@ where #[cfg(feature = "getblocktemplate-rpcs")] mempool::Response::FullTransactions { mut transactions, + transaction_dependencies: _, last_seen_tip_hash: _, } => { // Sort transactions in descending order by fee/size, using hash in serialized byte order as a tie-breaker diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index bc86cc91360..28edbfc4173 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -758,11 +758,13 @@ impl Service for Mempool { trace!(?req, "got mempool request"); let transactions: Vec<_> = storage.transactions().values().cloned().collect(); + let transaction_dependencies = storage.transaction_dependencies().clone(); trace!(?req, transactions_count = ?transactions.len(), "answered mempool request"); let response = Response::FullTransactions { transactions, + transaction_dependencies, last_seen_tip_hash: *last_seen_tip_hash, }; diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 3eb2fea688c..10752a36e35 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -445,6 +445,13 @@ impl Storage { self.verified.transactions() } + /// Returns a reference to the [`HashMap`] of transaction dependencies in the verified set. + pub fn transaction_dependencies( + &self, + ) -> &HashMap> { + self.verified.transaction_dependencies() + } + /// Returns a [`transparent::Output`] created by a mempool transaction for the provided /// [`transparent::OutPoint`] if one exists, or None otherwise. pub fn created_output(&self, outpoint: &transparent::OutPoint) -> Option { diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index b42de90b55f..47bdcd33dba 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -26,17 +26,23 @@ struct TransactionDependencies { /// by a mempool transaction. dependencies: HashMap>, - /// Lists of mempool transactions that spend UTXOs created - /// by a mempool transaction. + /// Lists of transaction ids in the mempool that spend UTXOs created + /// by a transaction in the mempool, e.g. tx1 -> set(tx2, tx3, tx4) where + /// tx2, tx3, and tx4 spend outputs created by tx1. dependents: HashMap>, } impl TransactionDependencies { /// Adds a transaction that spends outputs created by other transactions in the mempool - /// as a dependant of those transactions, and adds the transactions that created the outputs - /// spent by the dependant transaction as dependencies of the dependant transaction. - // - // TODO: Order transactions in block templates based on their dependencies + /// as a dependent of those transactions, and adds the transactions that created the outputs + /// spent by the dependent transaction as dependencies of the dependent transaction. + /// + /// # Correctness + /// + /// It's the caller's responsibility to ensure that there are no cyclical dependencies. + /// + /// The transaction verifier will wait until the spent output of a transaction has been added to the verified set, + /// so its `AwaitOutput` requests will timeout if there is a cyclical dependency. fn add( &mut self, dependent: transaction::Hash, @@ -49,23 +55,22 @@ impl TransactionDependencies { .insert(dependent); } - self.dependencies.entry(dependent).or_default().extend( - spent_mempool_outpoints - .into_iter() - .map(|outpoint| outpoint.hash), - ); + if !spent_mempool_outpoints.is_empty() { + self.dependencies.entry(dependent).or_default().extend( + spent_mempool_outpoints + .into_iter() + .map(|outpoint| outpoint.hash), + ); + } } /// Removes the hash of a transaction in the mempool and the hashes of any transactions /// that are tracked as being directly dependant on that transaction from /// this [`TransactionDependencies`]. /// - /// Returns a list of transaction hashes that have been removed. + /// Returns a list of transaction hashes that depend on the transaction being removed. fn remove(&mut self, tx_hash: &transaction::Hash) -> HashSet { - for dependencies in self.dependencies.values_mut() { - dependencies.remove(tx_hash); - } - + self.dependencies.remove(tx_hash); self.dependents.remove(tx_hash).unwrap_or_default() } } @@ -129,6 +134,13 @@ impl VerifiedSet { &self.transactions } + /// Returns a reference to the [`HashMap`] of transaction dependencies in the verified set. + pub fn transaction_dependencies( + &self, + ) -> &HashMap> { + &self.transaction_dependencies.dependencies + } + /// Returns a [`transparent::Output`] created by a mempool transaction for the provided /// [`transparent::OutPoint`] if one exists, or None otherwise. pub fn created_output(&self, outpoint: &transparent::OutPoint) -> Option { From 520660f3dd244dcb70af4a038caff5a9bb4d345c Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 16 Sep 2024 20:03:29 -0400 Subject: [PATCH 25/69] updates block template construction to avoid including transactions unless their dependencies have already been added. --- .../src/methods/get_block_template_rpcs.rs | 26 ++++++++----- .../get_block_template.rs | 20 ++++++++-- .../types/long_poll.rs | 8 +++- .../methods/get_block_template_rpcs/zip317.rs | 37 ++++++++++++++++++- 4 files changed, 74 insertions(+), 17 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index c8c83e9315a..653b3345ad9 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -630,7 +630,13 @@ where // The loop returns the server long poll ID, // which should be different to the client long poll ID. - let (server_long_poll_id, chain_tip_and_local_time, mempool_txs, submit_old) = loop { + let ( + server_long_poll_id, + chain_tip_and_local_time, + mempool_txs, + mempool_tx_deps, + submit_old, + ) = loop { // Check if we are synced to the tip. // The result of this check can change during long polling. // @@ -670,12 +676,13 @@ where // // Optional TODO: // - add a `MempoolChange` type with an `async changed()` method (like `ChainTip`) - let Some(mempool_txs) = fetch_mempool_transactions(mempool.clone(), tip_hash) - .await? - // If the mempool and state responses are out of sync: - // - if we are not long polling, omit mempool transactions from the template, - // - if we are long polling, continue to the next iteration of the loop to make fresh state and mempool requests. - .or_else(|| client_long_poll_id.is_none().then(Vec::new)) + let Some((mempool_txs, mempool_tx_deps)) = + fetch_mempool_transactions(mempool.clone(), tip_hash) + .await? + // If the mempool and state responses are out of sync: + // - if we are not long polling, omit mempool transactions from the template, + // - if we are long polling, continue to the next iteration of the loop to make fresh state and mempool requests. + .or_else(|| client_long_poll_id.is_none().then(Default::default)) else { continue; }; @@ -710,6 +717,7 @@ where server_long_poll_id, chain_tip_and_local_time, mempool_txs, + mempool_tx_deps, submit_old, ); } @@ -870,10 +878,10 @@ where next_block_height, &miner_address, mempool_txs, + &mempool_tx_deps, debug_like_zcashd, extra_coinbase_data.clone(), - ) - .await; + ); tracing::debug!( selected_mempool_tx_hashes = ?mempool_txs diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs index 8e9578180be..e2bb702cef0 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs @@ -1,6 +1,10 @@ //! Support functions for the `get_block_template()` RPC. -use std::{collections::HashMap, iter, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + iter, + sync::Arc, +}; use jsonrpc_core::{Error, ErrorCode, Result}; use tower::{Service, ServiceExt}; @@ -16,7 +20,7 @@ use zebra_chain::{ chain_tip::ChainTip, parameters::{subsidy::FundingStreamReceiver, Network, NetworkUpgrade}, serialization::ZcashDeserializeInto, - transaction::{Transaction, UnminedTx, VerifiedUnminedTx}, + transaction::{self, Transaction, UnminedTx, VerifiedUnminedTx}, transparent, }; use zebra_consensus::{ @@ -253,7 +257,12 @@ where pub async fn fetch_mempool_transactions( mempool: Mempool, chain_tip_hash: block::Hash, -) -> Result>> +) -> Result< + Option<( + Vec, + HashMap>, + )>, +> where Mempool: Service< mempool::Request, @@ -271,8 +280,11 @@ where data: None, })?; + // TODO: Order transactions in block templates based on their dependencies + let mempool::Response::FullTransactions { transactions, + transaction_dependencies, last_seen_tip_hash, } = response else { @@ -280,7 +292,7 @@ where }; // Check that the mempool and state were in sync when we made the requests - Ok((last_seen_tip_hash == chain_tip_hash).then_some(transactions)) + Ok((last_seen_tip_hash == chain_tip_hash).then_some((transactions, transaction_dependencies))) } // - Response processing diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs index 8817a8c12c0..406f4994f4d 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs @@ -71,14 +71,18 @@ impl LongPollInput { max_time: DateTime32, mempool_tx_ids: impl IntoIterator, ) -> Self { - let mempool_transaction_mined_ids = + let mut tx_mined_ids: Vec = mempool_tx_ids.into_iter().map(|id| id.mined_id()).collect(); + // The mempool returns unordered transactions, we need to sort them here so + // that the longpollid doesn't change unexpectedly. + tx_mined_ids.sort(); + LongPollInput { tip_height, tip_hash, max_time, - mempool_transaction_mined_ids, + mempool_transaction_mined_ids: tx_mined_ids.into(), } } diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index 3f0979dc266..1319f261130 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -6,6 +6,8 @@ //! > when computing `size_target`, since there is no consensus requirement for this to be //! > exactly the same between implementations. +use std::collections::{HashMap, HashSet}; + use rand::{ distributions::{Distribution, WeightedIndex}, prelude::thread_rng, @@ -15,7 +17,7 @@ use zebra_chain::{ amount::NegativeOrZero, block::{Height, MAX_BLOCK_BYTES}, parameters::Network, - transaction::{zip317::BLOCK_UNPAID_ACTION_LIMIT, VerifiedUnminedTx}, + transaction::{self, zip317::BLOCK_UNPAID_ACTION_LIMIT, VerifiedUnminedTx}, transparent, }; use zebra_consensus::MAX_BLOCK_SIGOPS; @@ -36,11 +38,12 @@ use crate::methods::get_block_template_rpcs::{ /// Returns selected transactions from `mempool_txs`. /// /// [ZIP-317]: https://zips.z.cash/zip-0317#block-production -pub async fn select_mempool_transactions( +pub fn select_mempool_transactions( network: &Network, next_block_height: Height, miner_address: &transparent::Address, mempool_txs: Vec, + mempool_tx_deps: &HashMap>, like_zcashd: bool, extra_coinbase_data: Vec, ) -> Vec { @@ -79,6 +82,7 @@ pub async fn select_mempool_transactions( &mut conventional_fee_txs, tx_weights, &mut selected_txs, + mempool_tx_deps, &mut remaining_block_bytes, &mut remaining_block_sigops, // The number of unpaid actions is always zero for transactions that pay the @@ -95,6 +99,7 @@ pub async fn select_mempool_transactions( &mut low_fee_txs, tx_weights, &mut selected_txs, + mempool_tx_deps, &mut remaining_block_bytes, &mut remaining_block_sigops, &mut remaining_block_unpaid_actions, @@ -158,6 +163,22 @@ fn setup_fee_weighted_index(transactions: &[VerifiedUnminedTx]) -> Option, + deps: Option<&HashSet>, +) -> bool { + let Some(deps) = deps else { return true }; + + let mut num_available_deps = 0; + for tx in selected_txs { + if deps.contains(&tx.transaction.id.mined_id()) { + num_available_deps += 1; + } + } + + num_available_deps == deps.len() +} + /// Chooses a random transaction from `txs` using the weighted index `tx_weights`, /// and tries to add it to `selected_txs`. /// @@ -172,6 +193,7 @@ fn checked_add_transaction_weighted_random( candidate_txs: &mut Vec, tx_weights: WeightedIndex, selected_txs: &mut Vec, + mempool_tx_deps: &HashMap>, remaining_block_bytes: &mut usize, remaining_block_sigops: &mut u64, remaining_block_unpaid_actions: &mut u32, @@ -191,6 +213,17 @@ fn checked_add_transaction_weighted_random( if candidate_tx.transaction.size <= *remaining_block_bytes && candidate_tx.legacy_sigop_count <= *remaining_block_sigops && candidate_tx.unpaid_actions <= *remaining_block_unpaid_actions + // # Correctness + // + // Transactions that spend outputs created in the same block + // must appear after the transaction that created those outputs + // + // TODO: If it gets here but the dependencies aren't selected, add it to a list of transactions + // to be added immediately if there's room once their dependencies have been selected? + && has_dependencies( + selected_txs, + mempool_tx_deps.get(&candidate_tx.transaction.id.mined_id()), + ) { selected_txs.push(candidate_tx.clone()); From b520d7b83cc097f89840f2fc789f3e60035cba6c Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 16 Sep 2024 20:07:53 -0400 Subject: [PATCH 26/69] updates tests --- zebra-rpc/src/methods/tests/prop.rs | 1 + zebra-rpc/src/methods/tests/snapshot.rs | 1 + zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs | 1 + zebra-rpc/src/methods/tests/vectors.rs | 1 + zebrad/tests/acceptance.rs | 1 + 5 files changed, 5 insertions(+) diff --git a/zebra-rpc/src/methods/tests/prop.rs b/zebra-rpc/src/methods/tests/prop.rs index 409a6aefe52..726ddca159a 100644 --- a/zebra-rpc/src/methods/tests/prop.rs +++ b/zebra-rpc/src/methods/tests/prop.rs @@ -424,6 +424,7 @@ proptest! { .await? .respond(mempool::Response::FullTransactions { transactions, + transaction_dependencies: Default::default(), last_seen_tip_hash: [0; 32].into(), }); diff --git a/zebra-rpc/src/methods/tests/snapshot.rs b/zebra-rpc/src/methods/tests/snapshot.rs index f4d7804088e..c0cda974ede 100644 --- a/zebra-rpc/src/methods/tests/snapshot.rs +++ b/zebra-rpc/src/methods/tests/snapshot.rs @@ -356,6 +356,7 @@ async fn test_rpc_response_data_for_network(network: &Network) { .map(|responder| { responder.respond(mempool::Response::FullTransactions { transactions: vec![], + transaction_dependencies: Default::default(), last_seen_tip_hash: blocks[blocks.len() - 1].hash(), }); }); diff --git a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs index 2fbd11c3978..b2e012c7bcd 100644 --- a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs @@ -265,6 +265,7 @@ pub async fn test_responses( .await .respond(mempool::Response::FullTransactions { transactions: vec![], + transaction_dependencies: Default::default(), // tip hash needs to match chain info for long poll requests last_seen_tip_hash: fake_tip_hash, }); diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index f2a62e9e2bd..b82ac588d5c 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -1364,6 +1364,7 @@ async fn rpc_getblocktemplate_mining_address(use_p2pkh: bool) { .await .respond(mempool::Response::FullTransactions { transactions, + transaction_dependencies: Default::default(), last_seen_tip_hash, }); } diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 6c6594159f8..80133020828 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3349,6 +3349,7 @@ async fn nu6_funding_streams_and_coinbase_balance() -> Result<()> { .await .respond(mempool::Response::FullTransactions { transactions: vec![], + transaction_dependencies: Default::default(), // tip hash needs to match chain info for long poll requests last_seen_tip_hash: genesis_hash, }); From 6c233f4b37de674324c324d53f54fde0f3591450 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 16 Sep 2024 20:21:08 -0400 Subject: [PATCH 27/69] Replaces placeholder setup channel with one that sends the mempool svc to the tx verifier, adds a timeout layer, adds a TODO about a concurrency bug --- zebra-consensus/src/transaction.rs | 8 +++++--- zebrad/src/commands/start.rs | 17 ++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index bd78fc151e7..aa98821d093 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -66,7 +66,7 @@ pub struct Verifier { network: Network, state: Timeout, // TODO: Use an enum so that this can either be Pending(oneshot::Receiver) or Initialized(MempoolService) - mempool: Option, + mempool: Option>, script_verifier: script::Verifier, mempool_setup_rx: oneshot::Receiver, } @@ -334,7 +334,7 @@ where if self.mempool.is_none() { if let Ok(mempool) = self.mempool_setup_rx.try_recv() { - self.mempool = Some(mempool); + self.mempool = Some(Timeout::new(mempool, UTXO_LOOKUP_TIMEOUT)); } } @@ -593,7 +593,7 @@ where known_utxos: Arc>, is_mempool: bool, state: Timeout, - mempool: Option, + mempool: Option>, ) -> Result< ( HashMap, @@ -650,6 +650,8 @@ where if let Some(mempool) = mempool { for &spent_mempool_outpoint in &spent_mempool_outpoints { + // TODO: Respond to AwaitOutput requests immediately if the output is already available in the mempool + // instead of calling the mempool service twice (using 2 queries introduces a concurrency bug). let query = mempool .clone() .oneshot(mempool::Request::UnspentOutput(spent_mempool_outpoint)); diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 89239f48a0f..d5484e8dd3b 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -179,23 +179,14 @@ impl StartCmd { .await; info!("initializing verifiers"); + let (tx_verifier_setup_tx, tx_verifier_setup_rx) = oneshot::channel(); let (block_verifier_router, tx_verifier, consensus_task_handles, max_checkpoint_height) = zebra_consensus::router::init( config.consensus.clone(), &config.network.network, state.clone(), // TODO: Pass actual setup channel receiver - oneshot::channel::< - tower::buffer::Buffer< - BoxService< - zebra_node_services::mempool::Request, - zebra_node_services::mempool::Response, - tower::BoxError, - >, - zebra_node_services::mempool::Request, - >, - >() - .1, + tx_verifier_setup_rx, ) .await; @@ -224,6 +215,10 @@ impl StartCmd { .buffer(mempool::downloads::MAX_INBOUND_CONCURRENCY) .service(mempool); + if tx_verifier_setup_tx.send(mempool.clone()).is_err() { + warn!("error setting up the transaction verifier with a handle to the mempool service"); + }; + info!("fully initializing inbound peer request handler"); // Fully start the inbound service as soon as possible let setup_data = InboundSetupData { From 56ae48f635a2b4ef629d4c44fdc0d108462bced1 Mon Sep 17 00:00:00 2001 From: ar Date: Tue, 17 Sep 2024 16:20:43 -0400 Subject: [PATCH 28/69] Use a single query to check for unspent outputs in the mempool --- zebra-consensus/src/transaction.rs | 15 ++------------- zebra-node-services/src/mempool.rs | 6 ------ zebrad/src/components/mempool.rs | 15 ++++----------- 3 files changed, 6 insertions(+), 30 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index aa98821d093..d65e9f29454 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -654,24 +654,13 @@ where // instead of calling the mempool service twice (using 2 queries introduces a concurrency bug). let query = mempool .clone() - .oneshot(mempool::Request::UnspentOutput(spent_mempool_outpoint)); + .oneshot(mempool::Request::AwaitOutput(spent_mempool_outpoint)); let mempool::Response::UnspentOutput(output) = query.await? else { unreachable!("UnspentOutput always responds with UnspentOutput") }; - let output = if let Some(output) = output { - output - } else { - let query = mempool - .clone() - .oneshot(mempool::Request::AwaitOutput(spent_mempool_outpoint)); - if let mempool::Response::UnspentOutput(output) = query.await? { - output.ok_or(TransactionError::TransparentInputNotFound)? - } else { - unreachable!("AwaitOutput always responds with UnspentOutput") - } - }; + let output = output.ok_or(TransactionError::TransparentInputNotFound)?; spent_outputs.push(output.clone()); } diff --git a/zebra-node-services/src/mempool.rs b/zebra-node-services/src/mempool.rs index 86e8c3f2a6c..748dc5f6a83 100644 --- a/zebra-node-services/src/mempool.rs +++ b/zebra-node-services/src/mempool.rs @@ -42,12 +42,6 @@ pub enum Request { /// the [`AuthDigest`](zebra_chain::transaction::AuthDigest). TransactionsByMinedId(HashSet), - /// Looks up a [`transparent::Output`] in the mempool identified by the given [`OutPoint`](transparent::OutPoint), - /// returning `None` immediately if it is unknown. - /// - /// Does not gaurantee that the output will remain in the mempool or that it is unspent. - UnspentOutput(transparent::OutPoint), - /// Request a [`transparent::Output`] identified by the given [`OutPoint`](transparent::OutPoint), /// waiting until it becomes available if it is unknown. /// diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index 28edbfc4173..bbeb3e81ec9 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -733,21 +733,15 @@ impl Service for Mempool { async move { Ok(Response::Transactions(res)) }.boxed() } - Request::UnspentOutput(outpoint) => { - trace!(?req, "got mempool request"); - - let res = storage.created_output(&outpoint); - - trace!(?res, "answered mempool request"); - - async move { Ok(Response::UnspentOutput(res)) }.boxed() - } - Request::AwaitOutput(outpoint) => { trace!(?req, "got mempool request"); let response_fut = storage.pending_outputs.queue(outpoint); + if let Some(output) = storage.created_output(&outpoint) { + storage.pending_outputs.respond(&outpoint, output) + } + trace!("answered mempool request"); response_fut.boxed() @@ -834,7 +828,6 @@ impl Service for Mempool { Request::TransactionsById(_) => Response::Transactions(Default::default()), Request::TransactionsByMinedId(_) => Response::Transactions(Default::default()), - Request::UnspentOutput(_) => Response::UnspentOutput(None), Request::AwaitOutput(_) => Response::UnspentOutput(None), #[cfg(feature = "getblocktemplate-rpcs")] From 05724ae9b2df98ce8d951b88054460ac24d26761 Mon Sep 17 00:00:00 2001 From: ar Date: Fri, 20 Sep 2024 18:30:58 -0400 Subject: [PATCH 29/69] Updates `getblocktemplate` method to consider dependencies when sorting transactions for the final template --- .../src/methods/get_block_template_rpcs.rs | 2 +- .../types/get_block_template.rs | 30 +++++++++---- .../methods/get_block_template_rpcs/zip317.rs | 42 +++++++++++++++---- .../mempool/storage/verified_set.rs | 8 ++-- 4 files changed, 63 insertions(+), 19 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 653b3345ad9..3e92f6ae94a 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -886,7 +886,7 @@ where tracing::debug!( selected_mempool_tx_hashes = ?mempool_txs .iter() - .map(|tx| tx.transaction.id.mined_id()) + .map(|(_, tx)| tx.transaction.id.mined_id()) .collect::>(), "selected transactions for the template from the mempool" ); diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs index d7c31e11a81..671ba844ebd 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs @@ -35,6 +35,11 @@ pub mod proposal; pub use parameters::{GetBlockTemplateCapability, GetBlockTemplateRequestMode, JsonParameters}; pub use proposal::{proposal_block_from_template, ProposalResponse}; +/// An alias to indicate that a usize value is meant to be a minimum valid transaction index in block. +/// +/// See the `min_tx_index()` function in [`zip317`](super::super::zip317) for more details. +pub type MinimumTxIndex = usize; + /// A serialized `getblocktemplate` RPC response in template mode. #[derive(Clone, Eq, PartialEq, serde::Serialize, serde::Deserialize)] pub struct GetBlockTemplate { @@ -227,7 +232,7 @@ impl GetBlockTemplate { miner_address: &transparent::Address, chain_tip_and_local_time: &GetBlockTemplateChainInfo, long_poll_id: LongPollId, - mempool_txs: Vec, + mempool_txs: Vec<(MinimumTxIndex, VerifiedUnminedTx)>, submit_old: Option, like_zcashd: bool, extra_coinbase_data: Vec, @@ -238,27 +243,38 @@ impl GetBlockTemplate { // Convert transactions into TransactionTemplates let mut mempool_txs_with_templates: Vec<( + MinimumTxIndex, TransactionTemplate, VerifiedUnminedTx, )> = mempool_txs .into_iter() - .map(|tx| ((&tx).into(), tx)) + .map(|(min_tx_index, tx)| (min_tx_index, (&tx).into(), tx)) .collect(); // Transaction selection returns transactions in an arbitrary order, // but Zebra's snapshot tests expect the same order every time. + // + // # Correctness + // + // Transactions that spend outputs created in the same block must appear + // after the transactions that create those outputs. if like_zcashd { // Sort in serialized data order, excluding the length byte. // `zcashd` sometimes seems to do this, but other times the order is arbitrary. - mempool_txs_with_templates.sort_by_key(|(tx_template, _tx)| tx_template.data.clone()); + mempool_txs_with_templates.sort_by_key(|(min_tx_index, tx_template, _tx)| { + (*min_tx_index, tx_template.data.clone()) + }); } else { // Sort by hash, this is faster. - mempool_txs_with_templates - .sort_by_key(|(tx_template, _tx)| tx_template.hash.bytes_in_display_order()); + mempool_txs_with_templates.sort_by_key(|(min_tx_index, tx_template, _tx)| { + (*min_tx_index, tx_template.hash.bytes_in_display_order()) + }); } - let (mempool_tx_templates, mempool_txs): (Vec<_>, Vec<_>) = - mempool_txs_with_templates.into_iter().unzip(); + let (mempool_tx_templates, mempool_txs): (Vec<_>, Vec<_>) = mempool_txs_with_templates + .into_iter() + .map(|(_, template, tx)| (template, tx)) + .unzip(); // Generate the coinbase transaction and default roots // diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index 1319f261130..030406868c0 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -26,6 +26,8 @@ use crate::methods::get_block_template_rpcs::{ get_block_template::generate_coinbase_transaction, types::transaction::TransactionTemplate, }; +use super::get_block_template::MinimumTxIndex; + /// Selects mempool transactions for block production according to [ZIP-317], /// using a fake coinbase transaction and the mempool. /// @@ -46,7 +48,7 @@ pub fn select_mempool_transactions( mempool_tx_deps: &HashMap>, like_zcashd: bool, extra_coinbase_data: Vec, -) -> Vec { +) -> Vec<(MinimumTxIndex, VerifiedUnminedTx)> { // Use a fake coinbase transaction to break the dependency between transaction // selection, the miner fee, and the fee payment in the coinbase transaction. let fake_coinbase_tx = fake_coinbase_transaction( @@ -164,13 +166,15 @@ fn setup_fee_weighted_index(transactions: &[VerifiedUnminedTx]) -> Option, - deps: Option<&HashSet>, + candidate_tx_deps: Option<&HashSet>, + selected_txs: &Vec<(MinimumTxIndex, VerifiedUnminedTx)>, ) -> bool { - let Some(deps) = deps else { return true }; + let Some(deps) = candidate_tx_deps else { + return true; + }; let mut num_available_deps = 0; - for tx in selected_txs { + for (_, tx) in selected_txs { if deps.contains(&tx.transaction.id.mined_id()) { num_available_deps += 1; } @@ -179,6 +183,23 @@ fn has_dependencies( num_available_deps == deps.len() } +/// Returns the minimum valid transaction index in the block for a candidate +/// transaction with the provided dependencies. +fn min_tx_index( + candidate_tx_deps: Option<&HashSet>, + mempool_tx_deps: &HashMap>, +) -> MinimumTxIndex { + let Some(deps) = candidate_tx_deps else { + return 0; + }; + + 1 + deps + .iter() + .map(|dep| min_tx_index(mempool_tx_deps.get(dep), mempool_tx_deps)) + .max() + .unwrap_or_default() +} + /// Chooses a random transaction from `txs` using the weighted index `tx_weights`, /// and tries to add it to `selected_txs`. /// @@ -192,7 +213,7 @@ fn has_dependencies( fn checked_add_transaction_weighted_random( candidate_txs: &mut Vec, tx_weights: WeightedIndex, - selected_txs: &mut Vec, + selected_txs: &mut Vec<(MinimumTxIndex, VerifiedUnminedTx)>, mempool_tx_deps: &HashMap>, remaining_block_bytes: &mut usize, remaining_block_sigops: &mut u64, @@ -203,6 +224,8 @@ fn checked_add_transaction_weighted_random( let (new_tx_weights, candidate_tx) = choose_transaction_weighted_random(candidate_txs, tx_weights); + let candidate_tx_deps = mempool_tx_deps.get(&candidate_tx.transaction.id.mined_id()); + // > If the block template with this transaction included // > would be within the block size limit and block sigop limit, // > and block_unpaid_actions <= block_unpaid_action_limit, @@ -221,11 +244,14 @@ fn checked_add_transaction_weighted_random( // TODO: If it gets here but the dependencies aren't selected, add it to a list of transactions // to be added immediately if there's room once their dependencies have been selected? && has_dependencies( + candidate_tx_deps, selected_txs, - mempool_tx_deps.get(&candidate_tx.transaction.id.mined_id()), ) { - selected_txs.push(candidate_tx.clone()); + selected_txs.push(( + min_tx_index(candidate_tx_deps, mempool_tx_deps), + candidate_tx.clone(), + )); *remaining_block_bytes -= candidate_tx.transaction.size; *remaining_block_sigops -= candidate_tx.legacy_sigop_count; diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 47bdcd33dba..191c47bcc30 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -22,8 +22,10 @@ use zebra_chain::transaction::MEMPOOL_TRANSACTION_COST_THRESHOLD; #[derive(Default)] struct TransactionDependencies { - /// Lists of mempool transactions that create UTXOs spent - /// by a mempool transaction. + /// Lists of mempool transactions that create UTXOs spent by + /// a mempool transaction. Used during block template construction + /// to exclude transactions from block templates unless all of the + /// transactions they depend on have been included. dependencies: HashMap>, /// Lists of transaction ids in the mempool that spend UTXOs created @@ -65,7 +67,7 @@ impl TransactionDependencies { } /// Removes the hash of a transaction in the mempool and the hashes of any transactions - /// that are tracked as being directly dependant on that transaction from + /// that are tracked as being directly dependent on that transaction from /// this [`TransactionDependencies`]. /// /// Returns a list of transaction hashes that depend on the transaction being removed. From 71a9c1b32b4f8ceacda1e40693d049c1cb4bb229 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 23 Sep 2024 17:48:11 -0400 Subject: [PATCH 30/69] fixes clippy lints, removes unnecessary Option in UnspentOutput response variant --- zebra-consensus/src/transaction.rs | 20 ++++++++++++++----- zebra-node-services/src/mempool.rs | 8 +++++--- zebrad/src/components/mempool.rs | 7 ++++++- .../src/components/mempool/pending_outputs.rs | 1 - 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index d65e9f29454..85266f9be4d 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -15,7 +15,12 @@ use futures::{ FutureExt, }; use tokio::sync::oneshot; -use tower::{buffer::Buffer, timeout::Timeout, util::BoxService, Service, ServiceExt}; +use tower::{ + buffer::Buffer, + timeout::{error::Elapsed, Timeout}, + util::BoxService, + Service, ServiceExt, +}; use tracing::Instrument; use zebra_chain::{ @@ -656,12 +661,17 @@ where .clone() .oneshot(mempool::Request::AwaitOutput(spent_mempool_outpoint)); - let mempool::Response::UnspentOutput(output) = query.await? else { - unreachable!("UnspentOutput always responds with UnspentOutput") + let output = match query.await { + Ok(mempool::Response::UnspentOutput(output)) => output, + Ok(_) => unreachable!("UnspentOutput always responds with UnspentOutput"), + Err(err) => { + return match err.downcast::() { + Ok(_) => Err(TransactionError::TransparentInputNotFound), + Err(err) => Err(err.into()), + }; + } }; - let output = output.ok_or(TransactionError::TransparentInputNotFound)?; - spent_outputs.push(output.clone()); } } else if !spent_mempool_outpoints.is_empty() { diff --git a/zebra-node-services/src/mempool.rs b/zebra-node-services/src/mempool.rs index 748dc5f6a83..974a0d9dfa7 100644 --- a/zebra-node-services/src/mempool.rs +++ b/zebra-node-services/src/mempool.rs @@ -2,7 +2,7 @@ //! //! A service that manages known unmined Zcash transactions. -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use tokio::sync::oneshot; use zebra_chain::{ @@ -10,6 +10,8 @@ use zebra_chain::{ transparent, }; +#[cfg(feature = "getblocktemplate-rpcs")] +use std::collections::HashMap; #[cfg(feature = "getblocktemplate-rpcs")] use zebra_chain::transaction::VerifiedUnminedTx; @@ -117,8 +119,8 @@ pub enum Response { /// different transactions with different mined IDs. Transactions(Vec), - /// Response to [`Request::UnspentOutput`] with the transparent output - UnspentOutput(Option), + /// Response to [`Request::AwaitOutput`] with the transparent output + UnspentOutput(transparent::Output), /// Returns all [`VerifiedUnminedTx`] in the mempool. // diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index bbeb3e81ec9..ee230c7cbc2 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -828,7 +828,12 @@ impl Service for Mempool { Request::TransactionsById(_) => Response::Transactions(Default::default()), Request::TransactionsByMinedId(_) => Response::Transactions(Default::default()), - Request::AwaitOutput(_) => Response::UnspentOutput(None), + Request::AwaitOutput(_) => { + return async move { + Err("mempool is not active: wait for Zebra to sync to the tip".into()) + } + .boxed() + } #[cfg(feature = "getblocktemplate-rpcs")] Request::FullTransactions => { diff --git a/zebrad/src/components/mempool/pending_outputs.rs b/zebrad/src/components/mempool/pending_outputs.rs index 234eb6ab235..f0fdb09c721 100644 --- a/zebrad/src/components/mempool/pending_outputs.rs +++ b/zebrad/src/components/mempool/pending_outputs.rs @@ -32,7 +32,6 @@ impl PendingOutputs { receiver .recv() .await - .map(Some) .map(Response::UnspentOutput) .map_err(BoxError::from) } From 310c320147f286a0b64abb2fc06a6fd04c82c696 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 23 Sep 2024 17:59:48 -0400 Subject: [PATCH 31/69] renames type alias and method, adds a TODO to use iteration instead of recursion --- .../types/get_block_template.rs | 10 +++++----- .../methods/get_block_template_rpcs/zip317.rs | 17 +++++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs index 671ba844ebd..a8997da6c89 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs @@ -35,10 +35,10 @@ pub mod proposal; pub use parameters::{GetBlockTemplateCapability, GetBlockTemplateRequestMode, JsonParameters}; pub use proposal::{proposal_block_from_template, ProposalResponse}; -/// An alias to indicate that a usize value is meant to be a minimum valid transaction index in block. +/// An alias to indicate that a usize value represents the depth of in-block dependencies of a transaction. /// -/// See the `min_tx_index()` function in [`zip317`](super::super::zip317) for more details. -pub type MinimumTxIndex = usize; +/// See the `dependencies_depth()` function in [`zip317`](super::super::zip317) for more details. +pub type InBlockTxDependenciesDepth = usize; /// A serialized `getblocktemplate` RPC response in template mode. #[derive(Clone, Eq, PartialEq, serde::Serialize, serde::Deserialize)] @@ -232,7 +232,7 @@ impl GetBlockTemplate { miner_address: &transparent::Address, chain_tip_and_local_time: &GetBlockTemplateChainInfo, long_poll_id: LongPollId, - mempool_txs: Vec<(MinimumTxIndex, VerifiedUnminedTx)>, + mempool_txs: Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)>, submit_old: Option, like_zcashd: bool, extra_coinbase_data: Vec, @@ -243,7 +243,7 @@ impl GetBlockTemplate { // Convert transactions into TransactionTemplates let mut mempool_txs_with_templates: Vec<( - MinimumTxIndex, + InBlockTxDependenciesDepth, TransactionTemplate, VerifiedUnminedTx, )> = mempool_txs diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index 030406868c0..77bd5652154 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -26,7 +26,7 @@ use crate::methods::get_block_template_rpcs::{ get_block_template::generate_coinbase_transaction, types::transaction::TransactionTemplate, }; -use super::get_block_template::MinimumTxIndex; +use super::get_block_template::InBlockTxDependenciesDepth; /// Selects mempool transactions for block production according to [ZIP-317], /// using a fake coinbase transaction and the mempool. @@ -48,7 +48,7 @@ pub fn select_mempool_transactions( mempool_tx_deps: &HashMap>, like_zcashd: bool, extra_coinbase_data: Vec, -) -> Vec<(MinimumTxIndex, VerifiedUnminedTx)> { +) -> Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)> { // Use a fake coinbase transaction to break the dependency between transaction // selection, the miner fee, and the fee payment in the coinbase transaction. let fake_coinbase_tx = fake_coinbase_transaction( @@ -167,7 +167,7 @@ fn setup_fee_weighted_index(transactions: &[VerifiedUnminedTx]) -> Option>, - selected_txs: &Vec<(MinimumTxIndex, VerifiedUnminedTx)>, + selected_txs: &Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)>, ) -> bool { let Some(deps) = candidate_tx_deps else { return true; @@ -185,17 +185,18 @@ fn has_dependencies( /// Returns the minimum valid transaction index in the block for a candidate /// transaction with the provided dependencies. -fn min_tx_index( +fn dependencies_depth( candidate_tx_deps: Option<&HashSet>, mempool_tx_deps: &HashMap>, -) -> MinimumTxIndex { +) -> InBlockTxDependenciesDepth { let Some(deps) = candidate_tx_deps else { return 0; }; + // TODO: Use iteration instead of recursion to avoid potential stack overflow 1 + deps .iter() - .map(|dep| min_tx_index(mempool_tx_deps.get(dep), mempool_tx_deps)) + .map(|dep| dependencies_depth(mempool_tx_deps.get(dep), mempool_tx_deps)) .max() .unwrap_or_default() } @@ -213,7 +214,7 @@ fn min_tx_index( fn checked_add_transaction_weighted_random( candidate_txs: &mut Vec, tx_weights: WeightedIndex, - selected_txs: &mut Vec<(MinimumTxIndex, VerifiedUnminedTx)>, + selected_txs: &mut Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)>, mempool_tx_deps: &HashMap>, remaining_block_bytes: &mut usize, remaining_block_sigops: &mut u64, @@ -249,7 +250,7 @@ fn checked_add_transaction_weighted_random( ) { selected_txs.push(( - min_tx_index(candidate_tx_deps, mempool_tx_deps), + dependencies_depth(candidate_tx_deps, mempool_tx_deps), candidate_tx.clone(), )); From 40b66474c3275efb900e97bbaf3a87a7c51a4cc7 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 23 Sep 2024 19:16:49 -0400 Subject: [PATCH 32/69] Adds mempool_removes_dependent_transactions() test --- zebrad/src/components/mempool/storage.rs | 7 ++ .../src/components/mempool/storage/tests.rs | 4 +- .../mempool/storage/tests/vectors.rs | 92 +++++++++++++++++++ .../mempool/storage/verified_set.rs | 7 ++ 4 files changed, 108 insertions(+), 2 deletions(-) diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 10752a36e35..f181235f343 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -452,6 +452,13 @@ impl Storage { self.verified.transaction_dependencies() } + /// Returns a reference to the [`HashMap`] of transaction dependents in the verified set. + pub fn transaction_dependents( + &self, + ) -> &HashMap> { + self.verified.transaction_dependents() + } + /// Returns a [`transparent::Output`] created by a mempool transaction for the provided /// [`transparent::OutPoint`] if one exists, or None otherwise. pub fn created_output(&self, outpoint: &transparent::OutPoint) -> Option { diff --git a/zebrad/src/components/mempool/storage/tests.rs b/zebrad/src/components/mempool/storage/tests.rs index e47808a3860..ed9577c4d03 100644 --- a/zebrad/src/components/mempool/storage/tests.rs +++ b/zebrad/src/components/mempool/storage/tests.rs @@ -34,12 +34,12 @@ pub fn unmined_transactions_in_blocks( selected_blocks .flat_map(|block| block.transactions) .map(UnminedTx::from) - .map(|transaction| { + .filter_map(|transaction| { VerifiedUnminedTx::new( transaction, Amount::try_from(1_000_000).expect("invalid value"), 0, ) - .expect("verification should pass") + .ok() }) } diff --git a/zebrad/src/components/mempool/storage/tests/vectors.rs b/zebrad/src/components/mempool/storage/tests/vectors.rs index 5d12a8620c4..3061b0d0eee 100644 --- a/zebrad/src/components/mempool/storage/tests/vectors.rs +++ b/zebrad/src/components/mempool/storage/tests/vectors.rs @@ -4,6 +4,7 @@ use std::iter; use color_eyre::eyre::Result; +use transparent::OutPoint; use zebra_chain::{ amount::Amount, block::{Block, Height}, @@ -291,3 +292,94 @@ fn mempool_expired_basic_for_network(network: Network) -> Result<()> { Ok(()) } + +/// Check that the transaction dependencies are updated when transactions with spent mempool outputs +/// are inserted into storage, and that the `Storage.remove()` method also removes any transactions +/// that directly or indirectly spend outputs of a removed transaction. +#[test] +fn mempool_removes_dependent_transactions() -> Result<()> { + let network = Network::Mainnet; + + // Create an empty storage + let mut storage: Storage = Storage::new(&config::Config { + tx_cost_limit: 160_000_000, + eviction_memory_time: EVICTION_MEMORY_TIME, + ..Default::default() + }); + + let unmined_txs_with_transparent_outputs = || { + unmined_transactions_in_blocks(.., &network) + .filter(|tx| !tx.transaction.transaction.outputs().is_empty()) + }; + + let mut fake_spent_outpoints: Vec = Vec::new(); + let mut expected_transaction_dependencies = HashMap::new(); + let mut expected_transaction_dependents = HashMap::new(); + for unmined_tx in unmined_txs_with_transparent_outputs() { + let tx_id = unmined_tx.transaction.id.mined_id(); + let num_outputs = unmined_tx.transaction.transaction.outputs().len(); + + if let Some(&fake_spent_outpoint) = fake_spent_outpoints.first() { + expected_transaction_dependencies + .insert(tx_id, [fake_spent_outpoint.hash].into_iter().collect()); + expected_transaction_dependents + .insert(fake_spent_outpoint.hash, [tx_id].into_iter().collect()); + } + + storage + .insert(unmined_tx.clone(), fake_spent_outpoints) + .expect("should insert transaction"); + + // Add up to 5 of this transaction's outputs as fake spent outpoints for the next transaction + fake_spent_outpoints = (0..num_outputs.min(5)) + .map(|i| OutPoint::from_usize(tx_id, i)) + .collect(); + } + + assert_eq!( + storage.transaction_dependencies().len(), + unmined_txs_with_transparent_outputs() + .count() + .checked_sub(1) + .expect("at least one unmined transaction with transparent outputs"), + "should have an entry all inserted txns except the first one" + ); + + assert_eq!( + storage.transaction_dependencies(), + &expected_transaction_dependencies, + "should have expected transaction dependencies" + ); + + assert_eq!( + storage.transaction_dependents(), + &expected_transaction_dependents, + "should have expected transaction dependents" + ); + + // Remove the first transaction and check that everything in storage is emptied. + let first_tx = unmined_txs_with_transparent_outputs() + .next() + .expect("at least one unmined transaction with transparent outputs"); + + let expected_num_removed = storage.transaction_count(); + let num_removed = storage.remove_exact(&[first_tx.transaction.id].into_iter().collect()); + + assert_eq!( + num_removed, expected_num_removed, + "remove_exact should total storage transaction count" + ); + + assert!( + storage.transaction_dependencies().is_empty(), + "tx deps should be empty" + ); + + assert_eq!( + storage.transaction_count(), + 0, + "verified set should be empty" + ); + + Ok(()) +} diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 191c47bcc30..2712d1c3933 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -143,6 +143,13 @@ impl VerifiedSet { &self.transaction_dependencies.dependencies } + /// Returns a reference to the [`HashMap`] of transaction dependents in the verified set. + pub fn transaction_dependents( + &self, + ) -> &HashMap> { + &self.transaction_dependencies.dependents + } + /// Returns a [`transparent::Output`] created by a mempool transaction for the provided /// [`transparent::OutPoint`] if one exists, or None otherwise. pub fn created_output(&self, outpoint: &transparent::OutPoint) -> Option { From 319ea7a51174f6ff6f7faf7f89fc2d7c20ee28d2 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 23 Sep 2024 20:58:37 -0400 Subject: [PATCH 33/69] Updates Storage and VerifiedSet clear() methods to clear pending_outputs, created_outputs, and transaction_dependencies, adds TODO to use iteration instead of recursion. --- zebrad/src/components/mempool/pending_outputs.rs | 5 +++++ zebrad/src/components/mempool/storage.rs | 1 + zebrad/src/components/mempool/storage/verified_set.rs | 9 +++++++++ 3 files changed, 15 insertions(+) diff --git a/zebrad/src/components/mempool/pending_outputs.rs b/zebrad/src/components/mempool/pending_outputs.rs index f0fdb09c721..ad2cfec66b3 100644 --- a/zebrad/src/components/mempool/pending_outputs.rs +++ b/zebrad/src/components/mempool/pending_outputs.rs @@ -55,4 +55,9 @@ impl PendingOutputs { pub fn prune(&mut self) { self.0.retain(|_, chan| chan.receiver_count() > 0); } + + /// Clears the inner [`HashMap`] of queued pending output requests. + pub fn clear(&mut self) { + self.0.clear(); + } } diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index f181235f343..64213f32db8 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -402,6 +402,7 @@ impl Storage { pub fn clear(&mut self) { self.verified.clear(); self.tip_rejected_exact.clear(); + self.pending_outputs.clear(); self.tip_rejected_same_effects.clear(); self.chain_rejected_same_effects.clear(); self.update_rejected_metrics(); diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 2712d1c3933..da75222e237 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -75,6 +75,12 @@ impl TransactionDependencies { self.dependencies.remove(tx_hash); self.dependents.remove(tx_hash).unwrap_or_default() } + + /// Clear the maps of transaction dependencies. + fn clear(&mut self) { + self.dependencies.clear(); + self.dependents.clear(); + } } /// The set of verified transactions stored in the mempool. @@ -187,10 +193,12 @@ impl VerifiedSet { /// Also clears all internal caches. pub fn clear(&mut self) { self.transactions.clear(); + self.transaction_dependencies.clear(); self.spent_outpoints.clear(); self.sprout_nullifiers.clear(); self.sapling_nullifiers.clear(); self.orchard_nullifiers.clear(); + self.created_outputs.clear(); self.transactions_serialized_size = 0; self.total_cost = 0; self.update_metrics(); @@ -323,6 +331,7 @@ impl VerifiedSet { let mut removed_txs = vec![removed_tx]; let dependent_transactions = self.transaction_dependencies.remove(key_to_remove); + // TODO: Use iteration instead of recursion to avoid potential stack overflow for dependent_tx in dependent_transactions { removed_txs.extend(self.remove(&dependent_tx)); } From 475e2865ba153865139d51ca0624f912e9cbe20c Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 23 Sep 2024 21:33:15 -0400 Subject: [PATCH 34/69] removes outdated TODO --- zebrad/src/commands/start.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index d5484e8dd3b..2f8a1563b8a 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -185,7 +185,6 @@ impl StartCmd { config.consensus.clone(), &config.network.network, state.clone(), - // TODO: Pass actual setup channel receiver tx_verifier_setup_rx, ) .await; From 0ec768e922406e846d74a3ecc2157009ae5473fb Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 23 Sep 2024 22:25:38 -0400 Subject: [PATCH 35/69] Adds a TODO for reporting queued transaction verification results from the mempool from the poll_ready() method --- zebrad/src/components/mempool/downloads.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/zebrad/src/components/mempool/downloads.rs b/zebrad/src/components/mempool/downloads.rs index 275346ce6e4..c6f741cee40 100644 --- a/zebrad/src/components/mempool/downloads.rs +++ b/zebrad/src/components/mempool/downloads.rs @@ -400,6 +400,7 @@ where }; // Send the result to responder channel if one was provided. + // TODO: Wait until transactions are added to the verified set before sending an Ok to `rsp_tx`. if let Some(rsp_tx) = rsp_tx { let _ = rsp_tx.send( result From 2fb532ffd2fd6c843bd725e37adbbb86a05364ef Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 23 Sep 2024 22:26:11 -0400 Subject: [PATCH 36/69] Adds `mempool_responds_to_await_output` test --- zebrad/src/components/mempool/tests/vector.rs | 106 +++++++++++++++++- 1 file changed, 103 insertions(+), 3 deletions(-) diff --git a/zebrad/src/components/mempool/tests/vector.rs b/zebrad/src/components/mempool/tests/vector.rs index edd779e51c4..5aedc17e745 100644 --- a/zebrad/src/components/mempool/tests/vector.rs +++ b/zebrad/src/components/mempool/tests/vector.rs @@ -1,6 +1,6 @@ //! Fixed test vectors for the mempool. -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use color_eyre::Report; use tokio::time::{self, timeout}; @@ -8,7 +8,7 @@ use tower::{ServiceBuilder, ServiceExt}; use zebra_chain::{ amount::Amount, block::Block, fmt::humantime_seconds, parameters::Network, - serialization::ZcashDeserializeInto, transaction::VerifiedUnminedTx, + serialization::ZcashDeserializeInto, transaction::VerifiedUnminedTx, transparent::OutPoint, }; use zebra_consensus::transaction as tx; use zebra_state::{Config as StateConfig, CHAIN_TIP_UPDATE_WAIT_LIMIT}; @@ -922,6 +922,104 @@ async fn mempool_reverifies_after_tip_change() -> Result<(), Report> { Ok(()) } +/// Checks that the mempool service responds to AwaitOutput requests after verifying transactions +/// that create those outputs, or immediately if the outputs had been created by transaction that +/// are already in the mempool. +#[tokio::test(flavor = "multi_thread")] +async fn mempool_responds_to_await_output() -> Result<(), Report> { + let network = Network::Mainnet; + + let ( + mut mempool, + _peer_set, + _state_service, + _chain_tip_change, + mut tx_verifier, + mut recent_syncs, + ) = setup(&network, u64::MAX, true).await; + mempool.enable(&mut recent_syncs).await; + + let verified_unmined_tx = unmined_transactions_in_blocks(1..=10, &network) + .find(|tx| !tx.transaction.transaction.outputs().is_empty()) + .expect("should have at least 1 tx with transparent outputs"); + + let unmined_tx = verified_unmined_tx.transaction.clone(); + let output_index = 0; + let outpoint = OutPoint::from_usize(unmined_tx.id.mined_id(), output_index); + let expected_output = unmined_tx + .transaction + .outputs() + .get(output_index) + .expect("already checked that tx has outputs") + .clone(); + + // Call mempool with an AwaitOutput request + + let request = Request::AwaitOutput(outpoint); + let await_output_response_fut = mempool.ready().await.unwrap().call(request); + + // Queue the transaction with the pending output to be added to the mempool + + let request = Request::Queue(vec![Gossip::Tx(unmined_tx)]); + let queue_response_fut = mempool.ready().await.unwrap().call(request); + let mock_verify_tx_fut = tx_verifier.expect_request_that(|_| true).map(|responder| { + responder.respond(transaction::Response::Mempool { + transaction: verified_unmined_tx, + spent_mempool_outpoints: Vec::new(), + }); + }); + + let (response, _) = futures::join!(queue_response_fut, mock_verify_tx_fut); + let Response::Queued(mut results) = response.expect("response should be Ok") else { + panic!("wrong response from mempool to Queued request"); + }; + + let result_rx = results.remove(0).expect("should pass initial checks"); + assert!(results.is_empty(), "should have 1 result for 1 queued tx"); + + tokio::time::timeout(Duration::from_secs(10), result_rx) + .await + .expect("should not time out") + .expect("mempool tx verification result channel should not be closed") + .expect("mocked verification should be successful"); + + mempool + .ready() + .await + .expect("polling mempool should succeed"); + + assert_eq!( + mempool.storage().transaction_count(), + 1, + "should have 1 transaction in mempool's verified set" + ); + + assert_eq!( + mempool.storage().created_output(&outpoint), + Some(expected_output.clone()), + "created output should match expected output" + ); + + // Check that the AwaitOutput request has been responded to after the relevant tx was added to the verified set + + let response_fut = tokio::time::timeout(Duration::from_secs(30), await_output_response_fut); + let response = response_fut + .await + .expect("should not time out") + .expect("should not return RecvError"); + + let Response::UnspentOutput(response) = response else { + panic!("wrong response from mempool to AwaitOutput request"); + }; + + assert_eq!( + response, expected_output, + "AwaitOutput response should match expected output" + ); + + Ok(()) +} + /// Create a new [`Mempool`] instance using mocked services. async fn setup( network: &Network, @@ -947,7 +1045,7 @@ async fn setup( let (sync_status, recent_syncs) = SyncStatus::new(); - let (mempool, _mempool_transaction_receiver) = Mempool::new( + let (mempool, mut mempool_transaction_receiver) = Mempool::new( &mempool::Config { tx_cost_limit, ..Default::default() @@ -960,6 +1058,8 @@ async fn setup( chain_tip_change.clone(), ); + tokio::spawn(async move { while mempool_transaction_receiver.recv().await.is_ok() {} }); + if should_commit_genesis_block { let genesis_block: Arc = zebra_test::vectors::BLOCK_MAINNET_GENESIS_BYTES .zcash_deserialize_into() From c4ec52d8e5ef88bafc63925859995ed0f885ce71 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 23 Sep 2024 22:39:48 -0400 Subject: [PATCH 37/69] updates mempool_responds_to_await_output test --- zebrad/src/components/mempool/tests/vector.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/zebrad/src/components/mempool/tests/vector.rs b/zebrad/src/components/mempool/tests/vector.rs index 5aedc17e745..b2bf033dc41 100644 --- a/zebrad/src/components/mempool/tests/vector.rs +++ b/zebrad/src/components/mempool/tests/vector.rs @@ -1017,6 +1017,25 @@ async fn mempool_responds_to_await_output() -> Result<(), Report> { "AwaitOutput response should match expected output" ); + // Check that the mempool responds to AwaitOutput requests correctly when the outpoint is already in its `created_outputs` collection too. + + let request = Request::AwaitOutput(outpoint); + let await_output_response_fut = mempool.ready().await.unwrap().call(request); + let response_fut = tokio::time::timeout(Duration::from_secs(30), await_output_response_fut); + let response = response_fut + .await + .expect("should not time out") + .expect("should not return RecvError"); + + let Response::UnspentOutput(response) = response else { + panic!("wrong response from mempool to AwaitOutput request"); + }; + + assert_eq!( + response, expected_output, + "AwaitOutput response should match expected output" + ); + Ok(()) } From 8e118fc926e8407d6986f2943a2ee3773b971db6 Mon Sep 17 00:00:00 2001 From: ar Date: Tue, 24 Sep 2024 17:54:57 -0400 Subject: [PATCH 38/69] Uses iteration instead of recursion in verified set's remove() method and zip317 mod's dependencies_depth() method --- .../methods/get_block_template_rpcs/zip317.rs | 29 +++++---- .../src/components/mempool/storage/tests.rs | 1 + .../mempool/storage/verified_set.rs | 62 ++++++++++++------- 3 files changed, 56 insertions(+), 36 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index 77bd5652154..165f80d495b 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -165,7 +165,10 @@ fn setup_fee_weighted_index(transactions: &[VerifiedUnminedTx]) -> Option>, selected_txs: &Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)>, ) -> bool { @@ -183,22 +186,24 @@ fn has_dependencies( num_available_deps == deps.len() } -/// Returns the minimum valid transaction index in the block for a candidate +/// Returns the depth of a transaction's dependencies in the block for a candidate /// transaction with the provided dependencies. fn dependencies_depth( candidate_tx_deps: Option<&HashSet>, mempool_tx_deps: &HashMap>, ) -> InBlockTxDependenciesDepth { - let Some(deps) = candidate_tx_deps else { - return 0; - }; + let mut current_level_deps = candidate_tx_deps.cloned().unwrap_or_default(); + let mut current_level = 0; + + while !current_level_deps.is_empty() { + current_level += 1; + current_level_deps = current_level_deps + .iter() + .flat_map(|dep| mempool_tx_deps.get(dep).cloned().unwrap_or_default()) + .collect(); + } - // TODO: Use iteration instead of recursion to avoid potential stack overflow - 1 + deps - .iter() - .map(|dep| dependencies_depth(mempool_tx_deps.get(dep), mempool_tx_deps)) - .max() - .unwrap_or_default() + current_level } /// Chooses a random transaction from `txs` using the weighted index `tx_weights`, @@ -244,7 +249,7 @@ fn checked_add_transaction_weighted_random( // // TODO: If it gets here but the dependencies aren't selected, add it to a list of transactions // to be added immediately if there's room once their dependencies have been selected? - && has_dependencies( + && has_direct_dependencies( candidate_tx_deps, selected_txs, ) diff --git a/zebrad/src/components/mempool/storage/tests.rs b/zebrad/src/components/mempool/storage/tests.rs index ed9577c4d03..c134414f4a4 100644 --- a/zebrad/src/components/mempool/storage/tests.rs +++ b/zebrad/src/components/mempool/storage/tests.rs @@ -34,6 +34,7 @@ pub fn unmined_transactions_in_blocks( selected_blocks .flat_map(|block| block.transactions) .map(UnminedTx::from) + // Skip transactions that fail ZIP-317 mempool checks .filter_map(|transaction| { VerifiedUnminedTx::new( transaction, diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index da75222e237..b0178b4e16e 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -67,13 +67,28 @@ impl TransactionDependencies { } /// Removes the hash of a transaction in the mempool and the hashes of any transactions - /// that are tracked as being directly dependent on that transaction from + /// that are tracked as being directly or indirectly dependent on that transaction from /// this [`TransactionDependencies`]. /// - /// Returns a list of transaction hashes that depend on the transaction being removed. - fn remove(&mut self, tx_hash: &transaction::Hash) -> HashSet { - self.dependencies.remove(tx_hash); - self.dependents.remove(tx_hash).unwrap_or_default() + /// Returns a list of transaction hashes that have been removed if they were previously + /// in this [`TransactionDependencies`]. + fn remove(&mut self, &tx_hash: &transaction::Hash) -> HashSet { + let mut current_level_dependents: HashSet<_> = [tx_hash].into(); + let mut all_dependents = current_level_dependents.clone(); + + while !current_level_dependents.is_empty() { + current_level_dependents = current_level_dependents + .iter() + .flat_map(|dep| { + self.dependencies.remove(dep); + self.dependents.remove(dep).unwrap_or_default() + }) + .collect(); + + all_dependents.extend(¤t_level_dependents); + } + + all_dependents } /// Clear the maps of transaction dependencies. @@ -316,27 +331,26 @@ impl VerifiedSet { /// /// Also removes its outputs from the internal caches. fn remove(&mut self, key_to_remove: &transaction::Hash) -> Vec { - let Some(removed_tx) = self.transactions.remove(key_to_remove) else { - // Transaction key not found, it may have been removed as a dependent - // of another transaction that was removed. - return Vec::new(); - }; - - self.transactions_serialized_size -= removed_tx.transaction.size; - self.total_cost -= removed_tx.cost(); - self.remove_outputs(&removed_tx.transaction); + let removed_transactions: Vec<_> = self + .transaction_dependencies + .remove(key_to_remove) + .iter() + .map(|key_to_remove| { + let removed_tx = self + .transactions + .remove(key_to_remove) + .expect("invalid transaction key"); + + self.transactions_serialized_size -= removed_tx.transaction.size; + self.total_cost -= removed_tx.cost(); + self.remove_outputs(&removed_tx.transaction); + + removed_tx + }) + .collect(); self.update_metrics(); - - let mut removed_txs = vec![removed_tx]; - let dependent_transactions = self.transaction_dependencies.remove(key_to_remove); - - // TODO: Use iteration instead of recursion to avoid potential stack overflow - for dependent_tx in dependent_transactions { - removed_txs.extend(self.remove(&dependent_tx)); - } - - removed_txs + removed_transactions } /// Returns `true` if the given `transaction` has any spend conflicts with transactions in the From 58fd7b3c206f9553a9306066ce938414cdb7cd38 Mon Sep 17 00:00:00 2001 From: ar Date: Tue, 24 Sep 2024 18:42:23 -0400 Subject: [PATCH 39/69] Adds a mempool_request_with_mempool_output_is_accepted test for the transaction verifier --- zebra-consensus/src/transaction.rs | 22 +++-- zebra-consensus/src/transaction/tests.rs | 107 +++++++++++++++++++++++ 2 files changed, 123 insertions(+), 6 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 85266f9be4d..5d189a51bc3 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -33,7 +33,7 @@ use zebra_chain::{ transaction::{ self, HashType, SigHash, Transaction, UnminedTx, UnminedTxId, VerifiedUnminedTx, }, - transparent::{self, OrderedUtxo}, + transparent, }; use zebra_node_services::mempool; @@ -427,7 +427,7 @@ where // Load spent UTXOs from state. // The UTXOs are required for almost all the async checks. let load_spent_utxos_fut = - Self::spent_utxos(tx.clone(), req.known_utxos(), req.is_mempool(), state.clone(), mempool.clone()); + Self::spent_utxos(tx.clone(), req.clone(), state.clone(), mempool.clone(),); let (spent_utxos, spent_outputs, spent_mempool_outpoints) = load_spent_utxos_fut.await?; // WONTFIX: Return an error for Request::Block as well to replace this check in @@ -595,8 +595,7 @@ where /// in the same order as the matching inputs in the transaction. async fn spent_utxos( tx: Arc, - known_utxos: Arc>, - is_mempool: bool, + req: Request, state: Timeout, mempool: Option>, ) -> Result< @@ -607,6 +606,9 @@ where ), TransactionError, > { + let is_mempool = req.is_mempool(); + let known_utxos = req.known_utxos(); + let inputs = tx.inputs(); let mut spent_utxos = HashMap::new(); let mut spent_outputs = Vec::new(); @@ -655,8 +657,6 @@ where if let Some(mempool) = mempool { for &spent_mempool_outpoint in &spent_mempool_outpoints { - // TODO: Respond to AwaitOutput requests immediately if the output is already available in the mempool - // instead of calling the mempool service twice (using 2 queries introduces a concurrency bug). let query = mempool .clone() .oneshot(mempool::Request::AwaitOutput(spent_mempool_outpoint)); @@ -673,6 +673,16 @@ where }; spent_outputs.push(output.clone()); + spent_utxos.insert( + spent_mempool_outpoint, + // Assume the Utxo height will be next height after the best chain tip height + // + // # Correctness + // + // If the tip height changes while an umined transaction is being verified, + // the transaction must be re-verified before being added to the mempool. + transparent::Utxo::new(output, req.height(), false), + ); } } else if !spent_mempool_outpoints.is_empty() { return Err(TransactionError::TransparentInputNotFound); diff --git a/zebra-consensus/src/transaction/tests.rs b/zebra-consensus/src/transaction/tests.rs index a463565f252..7c0cc80dc1a 100644 --- a/zebra-consensus/src/transaction/tests.rs +++ b/zebra-consensus/src/transaction/tests.rs @@ -28,6 +28,7 @@ use zebra_chain::{ transparent::{self, CoinbaseData}, }; +use zebra_node_services::mempool; use zebra_state::ValidateContextError; use zebra_test::mock_service::MockService; @@ -585,6 +586,112 @@ async fn mempool_request_with_past_lock_time_is_accepted() { ); } +#[tokio::test] +async fn mempool_request_with_mempool_output_is_accepted() { + let mut state: MockService<_, _, _, _> = MockService::build().for_prop_tests(); + let mut mempool: MockService<_, _, _, _> = MockService::build().for_prop_tests(); + let (mempool_setup_tx, mempool_setup_rx) = tokio::sync::oneshot::channel(); + let verifier = Verifier::new(&Network::Mainnet, state.clone(), mempool_setup_rx); + mempool_setup_tx + .send(mempool.clone()) + .ok() + .expect("send should succeed"); + + let height = NetworkUpgrade::Canopy + .activation_height(&Network::Mainnet) + .expect("Canopy activation height is specified"); + let fund_height = (height - 1).expect("fake source fund block height is too small"); + let (input, output, known_utxos) = mock_transparent_transfer( + fund_height, + true, + 0, + Amount::try_from(10001).expect("invalid value"), + ); + + // Create a non-coinbase V4 tx with the last valid expiry height. + let tx = Transaction::V4 { + inputs: vec![input], + outputs: vec![output], + lock_time: LockTime::min_lock_time_timestamp(), + expiry_height: height, + joinsplit_data: None, + sapling_shielded_data: None, + }; + + let input_outpoint = match tx.inputs()[0] { + transparent::Input::PrevOut { outpoint, .. } => outpoint, + transparent::Input::Coinbase { .. } => panic!("requires a non-coinbase transaction"), + }; + + tokio::spawn(async move { + state + .expect_request(zebra_state::Request::BestChainNextMedianTimePast) + .await + .expect("verifier should call mock state service with correct request") + .respond(zebra_state::Response::BestChainNextMedianTimePast( + DateTime32::MAX, + )); + + state + .expect_request(zebra_state::Request::UnspentBestChainUtxo(input_outpoint)) + .await + .expect("verifier should call mock state service with correct request") + .respond(zebra_state::Response::UnspentBestChainUtxo(None)); + + state + .expect_request_that(|req| { + matches!( + req, + zebra_state::Request::CheckBestChainTipNullifiersAndAnchors(_) + ) + }) + .await + .expect("verifier should call mock state service with correct request") + .respond(zebra_state::Response::ValidBestChainTipNullifiersAndAnchors); + }); + + tokio::spawn(async move { + mempool + .expect_request(mempool::Request::AwaitOutput(input_outpoint)) + .await + .expect("verifier should call mock state service with correct request") + .respond(mempool::Response::UnspentOutput( + known_utxos + .get(&input_outpoint) + .expect("input outpoint should exist in known_utxos") + .utxo + .output + .clone(), + )); + }); + + let verifier_response = verifier + .oneshot(Request::Mempool { + transaction: tx.into(), + height, + }) + .await; + + assert!( + verifier_response.is_ok(), + "expected successful verification, got: {verifier_response:?}" + ); + + let crate::transaction::Response::Mempool { + transaction: _, + spent_mempool_outpoints, + } = verifier_response.expect("already checked that response is ok") + else { + panic!("unexpected response variant from transaction verifier for Mempool request") + }; + + assert_eq!( + spent_mempool_outpoints, + vec![input_outpoint], + "spent_mempool_outpoints in tx verifier response should match input_outpoint" + ); +} + /// Tests that calls to the transaction verifier with a mempool request that spends /// immature coinbase outputs will return an error. #[tokio::test] From 65fc0f8f5f50288039843698314e401c1e3cb88a Mon Sep 17 00:00:00 2001 From: ar Date: Tue, 24 Sep 2024 19:19:40 -0400 Subject: [PATCH 40/69] Moves delay duration before polling the mempool to a constant, uses a shorter timeout for mempool output lookups, adds a `poll_count` to MockService, and updates `mempool_request_with_unmined_output_spends_is_accepted` to check that the transaction verifier polls the mempool after verifying a mempool transaction with transparent outputs --- zebra-consensus/src/transaction.rs | 21 +++++++++++++++++++-- zebra-consensus/src/transaction/tests.rs | 19 +++++++++++++++---- zebra-test/src/mock_service.rs | 23 ++++++++++++++++++++++- 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 5d189a51bc3..b00854ee104 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -60,6 +60,23 @@ mod tests; /// chain in the correct order.) const UTXO_LOOKUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(6 * 60); +/// A timeout applied to output lookup requests sent to the mempool. This is shorter than the +/// timeout for the state UTXO lookups because a block is likely to be mined every 75 seconds +/// after Blossom is active, changing the best chain tip and requiring re-verification of transactions +/// in the mempool. +/// +/// This is how long Zebra will wait for an output to be added to the mempool before verification +/// of the transaction that spends it will fail. +const MEMPOOL_OUTPUT_LOOKUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); + +/// How long to wait after responding to a mempool request with a transaction that creates new +/// transparent outputs before polling the mempool service so that it will try adding the verified +/// transaction and responding to any potential `AwaitOutput` requests. +/// +/// This should be long enough for the mempool service's `Downloads` to finish processing the +/// response from the transaction verifier. +const POLL_MEMPOOL_DELAY: std::time::Duration = Duration::from_millis(50); + /// Asynchronous transaction verification. /// /// # Correctness @@ -339,7 +356,7 @@ where if self.mempool.is_none() { if let Ok(mempool) = self.mempool_setup_rx.try_recv() { - self.mempool = Some(Timeout::new(mempool, UTXO_LOOKUP_TIMEOUT)); + self.mempool = Some(Timeout::new(mempool, MEMPOOL_OUTPUT_LOOKUP_TIMEOUT)); } } @@ -534,7 +551,7 @@ where if let Some(mut mempool) = mempool { if !transaction.transaction.transaction.outputs().is_empty() { tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(50)).await; + tokio::time::sleep(POLL_MEMPOOL_DELAY).await; mempool.ready().await.expect("mempool poll_ready() method should not return an error"); }); } diff --git a/zebra-consensus/src/transaction/tests.rs b/zebra-consensus/src/transaction/tests.rs index 7c0cc80dc1a..d42bbb8594c 100644 --- a/zebra-consensus/src/transaction/tests.rs +++ b/zebra-consensus/src/transaction/tests.rs @@ -32,7 +32,7 @@ use zebra_node_services::mempool; use zebra_state::ValidateContextError; use zebra_test::mock_service::MockService; -use crate::error::TransactionError; +use crate::{error::TransactionError, transaction::POLL_MEMPOOL_DELAY}; use super::{check, Request, Verifier}; @@ -587,9 +587,9 @@ async fn mempool_request_with_past_lock_time_is_accepted() { } #[tokio::test] -async fn mempool_request_with_mempool_output_is_accepted() { +async fn mempool_request_with_unmined_output_spends_is_accepted() { let mut state: MockService<_, _, _, _> = MockService::build().for_prop_tests(); - let mut mempool: MockService<_, _, _, _> = MockService::build().for_prop_tests(); + let mempool: MockService<_, _, _, _> = MockService::build().for_prop_tests(); let (mempool_setup_tx, mempool_setup_rx) = tokio::sync::oneshot::channel(); let verifier = Verifier::new(&Network::Mainnet, state.clone(), mempool_setup_rx); mempool_setup_tx @@ -650,8 +650,9 @@ async fn mempool_request_with_mempool_output_is_accepted() { .respond(zebra_state::Response::ValidBestChainTipNullifiersAndAnchors); }); + let mut mempool_clone = mempool.clone(); tokio::spawn(async move { - mempool + mempool_clone .expect_request(mempool::Request::AwaitOutput(input_outpoint)) .await .expect("verifier should call mock state service with correct request") @@ -690,6 +691,16 @@ async fn mempool_request_with_mempool_output_is_accepted() { vec![input_outpoint], "spent_mempool_outpoints in tx verifier response should match input_outpoint" ); + + tokio::time::sleep(POLL_MEMPOOL_DELAY * 2).await; + assert_eq!( + mempool.poll_count(), + 2, + "the mempool service should have been polled twice, \ + first before being called with an AwaitOutput request, \ + then again shortly after a mempool transaction with transparent outputs \ + is successfully verified" + ); } /// Tests that calls to the transaction verifier with a mempool request that spends diff --git a/zebra-test/src/mock_service.rs b/zebra-test/src/mock_service.rs index 7ab0d1f613b..cf5c2da0db7 100644 --- a/zebra-test/src/mock_service.rs +++ b/zebra-test/src/mock_service.rs @@ -43,7 +43,10 @@ use std::{ fmt::Debug, marker::PhantomData, - sync::Arc, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, task::{Context, Poll}, time::Duration, }; @@ -111,6 +114,7 @@ type ProxyItem = pub struct MockService { receiver: broadcast::Receiver>, sender: broadcast::Sender>, + poll_count: Arc, max_request_delay: Duration, _assertion_type: PhantomData, } @@ -155,6 +159,7 @@ where type Future = BoxFuture<'static, Result>; fn poll_ready(&mut self, _context: &mut Context) -> Poll> { + self.poll_count.fetch_add(1, Ordering::SeqCst); Poll::Ready(Ok(())) } @@ -271,6 +276,7 @@ impl MockServiceBuilder { MockService { receiver, sender, + poll_count: Arc::new(AtomicUsize::new(0)), max_request_delay: self.max_request_delay.unwrap_or(DEFAULT_MAX_REQUEST_DELAY), _assertion_type: PhantomData, } @@ -454,6 +460,13 @@ impl MockService usize { + self.poll_count.load(Ordering::SeqCst) + } } /// Implementation of [`MockService`] methods that use [`mod@proptest`] assertions. @@ -667,6 +680,13 @@ impl MockService usize { + self.poll_count.load(Ordering::SeqCst) + } } /// Code that is independent of the assertions used in [`MockService`]. @@ -708,6 +728,7 @@ impl Clone MockService { receiver: self.sender.subscribe(), sender: self.sender.clone(), + poll_count: self.poll_count.clone(), max_request_delay: self.max_request_delay, _assertion_type: PhantomData, } From 5ff2da3c1e4e981e4224c3f9eb8b3e47f17573fc Mon Sep 17 00:00:00 2001 From: ar Date: Tue, 24 Sep 2024 20:08:49 -0400 Subject: [PATCH 41/69] adds long_poll_input_mempool_tx_ids_are_sorted test --- .../types/long_poll.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs index 406f4994f4d..f692f83f9ab 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs @@ -297,3 +297,28 @@ impl TryFrom for LongPollId { s.parse() } } + +/// Check that [`LongPollInput::new`] will sort mempool transaction ids. +/// +/// The mempool does not currently gaurantee the order in which it will return transactions and +/// may return the same items in a different order, while the long poll id should be the same if +/// its other components are equal and no transactions have been added or removed in the mempool. +#[test] +fn long_poll_input_mempool_tx_ids_are_sorted() { + let mempool_tx_ids = || { + (0..10) + .map(|i| transaction::Hash::from([i; 32])) + .map(UnminedTxId::Legacy) + }; + + assert_eq!( + LongPollInput::new(Height::MIN, Default::default(), 0.into(), mempool_tx_ids()), + LongPollInput::new( + Height::MIN, + Default::default(), + 0.into(), + mempool_tx_ids().rev() + ), + "long poll input should sort mempool tx ids" + ); +} From c72197c03d886e793918617efde0cfbcd4733f7b Mon Sep 17 00:00:00 2001 From: ar Date: Tue, 24 Sep 2024 20:23:26 -0400 Subject: [PATCH 42/69] Adds a `excludes_tx_with_unselected_dependencies` test --- .../methods/get_block_template_rpcs/zip317.rs | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index 165f80d495b..43cc4d4bc03 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -286,3 +286,70 @@ fn choose_transaction_weighted_random( // (setup_fee_weighted_index(candidate_txs), candidate_tx) } + +#[test] +fn excludes_tx_with_unselected_dependencies() { + use zebra_chain::{ + amount::Amount, block::Block, serialization::ZcashDeserializeInto, transaction::UnminedTx, + }; + + let network = Network::Mainnet; + let next_block_height = Height(1_000_000); + let miner_address = transparent::Address::from_pub_key_hash(network.kind(), [0; 20]); + let mut mempool_txns: Vec<_> = network + .block_iter() + .map(|(_, block)| { + block + .zcash_deserialize_into::() + .expect("block test vector is structurally valid") + }) + .flat_map(|block| block.transactions) + .map(UnminedTx::from) + // Skip transactions that fail ZIP-317 mempool checks + .filter_map(|transaction| { + VerifiedUnminedTx::new( + transaction, + Amount::try_from(1_000_000).expect("invalid value"), + 0, + ) + .ok() + }) + .take(2) + .collect(); + + let dep_tx_id = mempool_txns + .pop() + .expect("should not be empty") + .transaction + .id + .mined_id(); + + let mut mempool_tx_deps = HashMap::new(); + + mempool_tx_deps.insert( + mempool_txns + .first() + .expect("should not be empty") + .transaction + .id + .mined_id(), + [dep_tx_id].into(), + ); + + let like_zcashd = true; + let extra_coinbase_data = Vec::new(); + + assert!( + select_mempool_transactions( + &network, + next_block_height, + &miner_address, + mempool_txns, + &mempool_tx_deps, + like_zcashd, + extra_coinbase_data, + ) + .is_empty(), + "should not select any transactions when dependencies are unavailable" + ); +} From f93ca22197706295687e0dd008c17ed2ee019130 Mon Sep 17 00:00:00 2001 From: ar Date: Tue, 24 Sep 2024 20:29:10 -0400 Subject: [PATCH 43/69] Updates a TODO --- zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index 43cc4d4bc03..dbbc4dde0a9 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -248,7 +248,10 @@ fn checked_add_transaction_weighted_random( // must appear after the transaction that created those outputs // // TODO: If it gets here but the dependencies aren't selected, add it to a list of transactions - // to be added immediately if there's room once their dependencies have been selected? + // to be added immediately if there's room once their dependencies have been selected. + // Unlike the other checks in this if statement, candidate transactions that fail this + // check may pass it in the next round, but are currently removed from the candidate set + // and will not be re-considered. && has_direct_dependencies( candidate_tx_deps, selected_txs, From f44e640e8ced0c73637086e0acb21bca2db73493 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 26 Sep 2024 19:05:52 -0400 Subject: [PATCH 44/69] moves `TransactionDependencies` struct to `zebra-node-services` --- zebra-node-services/src/mempool.rs | 4 + .../src/mempool/transaction_dependencies.rs | 94 +++++++++++++++++++ zebrad/src/components/mempool.rs | 3 +- zebrad/src/components/mempool/storage.rs | 14 +-- .../mempool/storage/tests/vectors.rs | 8 +- .../mempool/storage/verified_set.rs | 94 +------------------ 6 files changed, 111 insertions(+), 106 deletions(-) create mode 100644 zebra-node-services/src/mempool/transaction_dependencies.rs diff --git a/zebra-node-services/src/mempool.rs b/zebra-node-services/src/mempool.rs index 974a0d9dfa7..ecbeb16e11b 100644 --- a/zebra-node-services/src/mempool.rs +++ b/zebra-node-services/src/mempool.rs @@ -18,6 +18,10 @@ use zebra_chain::transaction::VerifiedUnminedTx; use crate::BoxError; mod gossip; +mod transaction_dependencies; + +#[cfg(feature = "getblocktemplate-rpcs")] +pub use transaction_dependencies::TransactionDependencies; pub use self::gossip::Gossip; diff --git a/zebra-node-services/src/mempool/transaction_dependencies.rs b/zebra-node-services/src/mempool/transaction_dependencies.rs new file mode 100644 index 00000000000..d58aaad0b80 --- /dev/null +++ b/zebra-node-services/src/mempool/transaction_dependencies.rs @@ -0,0 +1,94 @@ +//! Representation of mempool transactions' dependencies on other transactions in the mempool. + +use std::collections::{HashMap, HashSet}; + +use zebra_chain::{transaction, transparent}; + +/// Representation of mempool transactions' dependencies on other transactions in the mempool. +#[derive(Default, Debug)] +pub struct TransactionDependencies { + /// Lists of mempool transactions that create UTXOs spent by + /// a mempool transaction. Used during block template construction + /// to exclude transactions from block templates unless all of the + /// transactions they depend on have been included. + dependencies: HashMap>, + + /// Lists of transaction ids in the mempool that spend UTXOs created + /// by a transaction in the mempool, e.g. tx1 -> set(tx2, tx3, tx4) where + /// tx2, tx3, and tx4 spend outputs created by tx1. + dependents: HashMap>, +} + +impl TransactionDependencies { + /// Adds a transaction that spends outputs created by other transactions in the mempool + /// as a dependent of those transactions, and adds the transactions that created the outputs + /// spent by the dependent transaction as dependencies of the dependent transaction. + /// + /// # Correctness + /// + /// It's the caller's responsibility to ensure that there are no cyclical dependencies. + /// + /// The transaction verifier will wait until the spent output of a transaction has been added to the verified set, + /// so its `AwaitOutput` requests will timeout if there is a cyclical dependency. + pub fn add( + &mut self, + dependent: transaction::Hash, + spent_mempool_outpoints: Vec, + ) { + for &spent_mempool_outpoint in &spent_mempool_outpoints { + self.dependents + .entry(spent_mempool_outpoint.hash) + .or_default() + .insert(dependent); + } + + if !spent_mempool_outpoints.is_empty() { + self.dependencies.entry(dependent).or_default().extend( + spent_mempool_outpoints + .into_iter() + .map(|outpoint| outpoint.hash), + ); + } + } + + /// Removes the hash of a transaction in the mempool and the hashes of any transactions + /// that are tracked as being directly or indirectly dependent on that transaction from + /// this [`TransactionDependencies`]. + /// + /// Returns a list of transaction hashes that have been removed if they were previously + /// in this [`TransactionDependencies`]. + pub fn remove(&mut self, &tx_hash: &transaction::Hash) -> HashSet { + let mut current_level_dependents: HashSet<_> = [tx_hash].into(); + let mut all_dependents = current_level_dependents.clone(); + + while !current_level_dependents.is_empty() { + current_level_dependents = current_level_dependents + .iter() + .flat_map(|dep| { + self.dependencies.remove(dep); + self.dependents.remove(dep).unwrap_or_default() + }) + .collect(); + + all_dependents.extend(¤t_level_dependents); + } + + all_dependents + } + + /// Clear the maps of transaction dependencies. + pub fn clear(&mut self) { + self.dependencies.clear(); + self.dependents.clear(); + } + + /// Returns the map of transaction's dependencies + pub fn dependencies(&self) -> &HashMap> { + &self.dependencies + } + + /// Returns the map of transaction's dependents + pub fn dependents(&self) -> &HashMap> { + &self.dependents + } +} diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index ee230c7cbc2..de59cfaacb0 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -752,7 +752,8 @@ impl Service for Mempool { trace!(?req, "got mempool request"); let transactions: Vec<_> = storage.transactions().values().cloned().collect(); - let transaction_dependencies = storage.transaction_dependencies().clone(); + let transaction_dependencies = + storage.transaction_dependencies().dependencies().clone(); trace!(?req, transactions_count = ?transactions.len(), "answered mempool request"); diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 64213f32db8..688b214f31c 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -20,6 +20,7 @@ use zebra_chain::{ transaction::{self, Hash, Transaction, UnminedTx, UnminedTxId, VerifiedUnminedTx}, transparent, }; +use zebra_node_services::mempool::TransactionDependencies; use self::{eviction_list::EvictionList, verified_set::VerifiedSet}; use super::{ @@ -446,20 +447,11 @@ impl Storage { self.verified.transactions() } - /// Returns a reference to the [`HashMap`] of transaction dependencies in the verified set. - pub fn transaction_dependencies( - &self, - ) -> &HashMap> { + /// Returns a reference to the [`TransactionDependencies`] in the verified set. + pub fn transaction_dependencies(&self) -> &TransactionDependencies { self.verified.transaction_dependencies() } - /// Returns a reference to the [`HashMap`] of transaction dependents in the verified set. - pub fn transaction_dependents( - &self, - ) -> &HashMap> { - self.verified.transaction_dependents() - } - /// Returns a [`transparent::Output`] created by a mempool transaction for the provided /// [`transparent::OutPoint`] if one exists, or None otherwise. pub fn created_output(&self, outpoint: &transparent::OutPoint) -> Option { diff --git a/zebrad/src/components/mempool/storage/tests/vectors.rs b/zebrad/src/components/mempool/storage/tests/vectors.rs index 3061b0d0eee..40db3b3c786 100644 --- a/zebrad/src/components/mempool/storage/tests/vectors.rs +++ b/zebrad/src/components/mempool/storage/tests/vectors.rs @@ -337,7 +337,7 @@ fn mempool_removes_dependent_transactions() -> Result<()> { } assert_eq!( - storage.transaction_dependencies().len(), + storage.transaction_dependencies().dependencies().len(), unmined_txs_with_transparent_outputs() .count() .checked_sub(1) @@ -346,13 +346,13 @@ fn mempool_removes_dependent_transactions() -> Result<()> { ); assert_eq!( - storage.transaction_dependencies(), + storage.transaction_dependencies().dependencies(), &expected_transaction_dependencies, "should have expected transaction dependencies" ); assert_eq!( - storage.transaction_dependents(), + storage.transaction_dependencies().dependents(), &expected_transaction_dependents, "should have expected transaction dependents" ); @@ -371,7 +371,7 @@ fn mempool_removes_dependent_transactions() -> Result<()> { ); assert!( - storage.transaction_dependencies().is_empty(), + storage.transaction_dependencies().dependencies().is_empty(), "tx deps should be empty" ); diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index b0178b4e16e..0fa7afadb57 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -11,6 +11,7 @@ use zebra_chain::{ transaction::{self, Transaction, UnminedTx, VerifiedUnminedTx}, transparent, }; +use zebra_node_services::mempool::TransactionDependencies; use crate::components::mempool::pending_outputs::PendingOutputs; @@ -20,84 +21,6 @@ use super::super::SameEffectsTipRejectionError; #[allow(unused_imports)] use zebra_chain::transaction::MEMPOOL_TRANSACTION_COST_THRESHOLD; -#[derive(Default)] -struct TransactionDependencies { - /// Lists of mempool transactions that create UTXOs spent by - /// a mempool transaction. Used during block template construction - /// to exclude transactions from block templates unless all of the - /// transactions they depend on have been included. - dependencies: HashMap>, - - /// Lists of transaction ids in the mempool that spend UTXOs created - /// by a transaction in the mempool, e.g. tx1 -> set(tx2, tx3, tx4) where - /// tx2, tx3, and tx4 spend outputs created by tx1. - dependents: HashMap>, -} - -impl TransactionDependencies { - /// Adds a transaction that spends outputs created by other transactions in the mempool - /// as a dependent of those transactions, and adds the transactions that created the outputs - /// spent by the dependent transaction as dependencies of the dependent transaction. - /// - /// # Correctness - /// - /// It's the caller's responsibility to ensure that there are no cyclical dependencies. - /// - /// The transaction verifier will wait until the spent output of a transaction has been added to the verified set, - /// so its `AwaitOutput` requests will timeout if there is a cyclical dependency. - fn add( - &mut self, - dependent: transaction::Hash, - spent_mempool_outpoints: Vec, - ) { - for &spent_mempool_outpoint in &spent_mempool_outpoints { - self.dependents - .entry(spent_mempool_outpoint.hash) - .or_default() - .insert(dependent); - } - - if !spent_mempool_outpoints.is_empty() { - self.dependencies.entry(dependent).or_default().extend( - spent_mempool_outpoints - .into_iter() - .map(|outpoint| outpoint.hash), - ); - } - } - - /// Removes the hash of a transaction in the mempool and the hashes of any transactions - /// that are tracked as being directly or indirectly dependent on that transaction from - /// this [`TransactionDependencies`]. - /// - /// Returns a list of transaction hashes that have been removed if they were previously - /// in this [`TransactionDependencies`]. - fn remove(&mut self, &tx_hash: &transaction::Hash) -> HashSet { - let mut current_level_dependents: HashSet<_> = [tx_hash].into(); - let mut all_dependents = current_level_dependents.clone(); - - while !current_level_dependents.is_empty() { - current_level_dependents = current_level_dependents - .iter() - .flat_map(|dep| { - self.dependencies.remove(dep); - self.dependents.remove(dep).unwrap_or_default() - }) - .collect(); - - all_dependents.extend(¤t_level_dependents); - } - - all_dependents - } - - /// Clear the maps of transaction dependencies. - fn clear(&mut self) { - self.dependencies.clear(); - self.dependents.clear(); - } -} - /// The set of verified transactions stored in the mempool. /// /// This also caches the all the spent outputs from the transactions in the mempool. The spent @@ -157,18 +80,9 @@ impl VerifiedSet { &self.transactions } - /// Returns a reference to the [`HashMap`] of transaction dependencies in the verified set. - pub fn transaction_dependencies( - &self, - ) -> &HashMap> { - &self.transaction_dependencies.dependencies - } - - /// Returns a reference to the [`HashMap`] of transaction dependents in the verified set. - pub fn transaction_dependents( - &self, - ) -> &HashMap> { - &self.transaction_dependencies.dependents + /// Returns a reference to the [`TransactionDependencies`] in the set. + pub fn transaction_dependencies(&self) -> &TransactionDependencies { + &self.transaction_dependencies } /// Returns a [`transparent::Output`] created by a mempool transaction for the provided From e195970a0fc323b80b90f66ecb591186ce36c9b6 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 26 Sep 2024 19:45:01 -0400 Subject: [PATCH 45/69] Updates `FullTransactions` response variant's `transaction_dependencies` type --- zebra-node-services/src/mempool.rs | 6 ++--- .../src/mempool/transaction_dependencies.rs | 2 +- .../src/methods/get_block_template_rpcs.rs | 2 +- .../get_block_template.rs | 17 ++++--------- .../methods/get_block_template_rpcs/zip317.rs | 24 +++++++++++-------- zebrad/src/components/mempool.rs | 3 +-- 6 files changed, 24 insertions(+), 30 deletions(-) diff --git a/zebra-node-services/src/mempool.rs b/zebra-node-services/src/mempool.rs index ecbeb16e11b..c986e7d701d 100644 --- a/zebra-node-services/src/mempool.rs +++ b/zebra-node-services/src/mempool.rs @@ -10,14 +10,14 @@ use zebra_chain::{ transparent, }; -#[cfg(feature = "getblocktemplate-rpcs")] -use std::collections::HashMap; #[cfg(feature = "getblocktemplate-rpcs")] use zebra_chain::transaction::VerifiedUnminedTx; use crate::BoxError; mod gossip; + +#[cfg(feature = "getblocktemplate-rpcs")] mod transaction_dependencies; #[cfg(feature = "getblocktemplate-rpcs")] @@ -136,7 +136,7 @@ pub enum Response { transactions: Vec, /// All transaction dependencies in the mempool - transaction_dependencies: HashMap>, + transaction_dependencies: TransactionDependencies, /// Last seen chain tip hash by mempool service last_seen_tip_hash: zebra_chain::block::Hash, diff --git a/zebra-node-services/src/mempool/transaction_dependencies.rs b/zebra-node-services/src/mempool/transaction_dependencies.rs index d58aaad0b80..02bdd65e0eb 100644 --- a/zebra-node-services/src/mempool/transaction_dependencies.rs +++ b/zebra-node-services/src/mempool/transaction_dependencies.rs @@ -5,7 +5,7 @@ use std::collections::{HashMap, HashSet}; use zebra_chain::{transaction, transparent}; /// Representation of mempool transactions' dependencies on other transactions in the mempool. -#[derive(Default, Debug)] +#[derive(Default, Debug, Clone)] pub struct TransactionDependencies { /// Lists of mempool transactions that create UTXOs spent by /// a mempool transaction. Used during block template construction diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 3e92f6ae94a..71d4d35e8f0 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -878,7 +878,7 @@ where next_block_height, &miner_address, mempool_txs, - &mempool_tx_deps, + mempool_tx_deps, debug_like_zcashd, extra_coinbase_data.clone(), ); diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs index e2bb702cef0..7ab1a48e20a 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs @@ -1,10 +1,6 @@ //! Support functions for the `get_block_template()` RPC. -use std::{ - collections::{HashMap, HashSet}, - iter, - sync::Arc, -}; +use std::{collections::HashMap, iter, sync::Arc}; use jsonrpc_core::{Error, ErrorCode, Result}; use tower::{Service, ServiceExt}; @@ -20,13 +16,13 @@ use zebra_chain::{ chain_tip::ChainTip, parameters::{subsidy::FundingStreamReceiver, Network, NetworkUpgrade}, serialization::ZcashDeserializeInto, - transaction::{self, Transaction, UnminedTx, VerifiedUnminedTx}, + transaction::{Transaction, UnminedTx, VerifiedUnminedTx}, transparent, }; use zebra_consensus::{ block_subsidy, funding_stream_address, funding_stream_values, miner_subsidy, }; -use zebra_node_services::mempool; +use zebra_node_services::mempool::{self, TransactionDependencies}; use zebra_state::GetBlockTemplateChainInfo; use crate::methods::{ @@ -257,12 +253,7 @@ where pub async fn fetch_mempool_transactions( mempool: Mempool, chain_tip_hash: block::Hash, -) -> Result< - Option<( - Vec, - HashMap>, - )>, -> +) -> Result, TransactionDependencies)>> where Mempool: Service< mempool::Request, diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index dbbc4dde0a9..c0c6c356cae 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -21,6 +21,7 @@ use zebra_chain::{ transparent, }; use zebra_consensus::MAX_BLOCK_SIGOPS; +use zebra_node_services::mempool::TransactionDependencies; use crate::methods::get_block_template_rpcs::{ get_block_template::generate_coinbase_transaction, types::transaction::TransactionTemplate, @@ -45,7 +46,7 @@ pub fn select_mempool_transactions( next_block_height: Height, miner_address: &transparent::Address, mempool_txs: Vec, - mempool_tx_deps: &HashMap>, + mut mempool_tx_deps: TransactionDependencies, like_zcashd: bool, extra_coinbase_data: Vec, ) -> Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)> { @@ -84,7 +85,7 @@ pub fn select_mempool_transactions( &mut conventional_fee_txs, tx_weights, &mut selected_txs, - mempool_tx_deps, + &mut mempool_tx_deps, &mut remaining_block_bytes, &mut remaining_block_sigops, // The number of unpaid actions is always zero for transactions that pay the @@ -101,7 +102,7 @@ pub fn select_mempool_transactions( &mut low_fee_txs, tx_weights, &mut selected_txs, - mempool_tx_deps, + &mut mempool_tx_deps, &mut remaining_block_bytes, &mut remaining_block_sigops, &mut remaining_block_unpaid_actions, @@ -220,7 +221,7 @@ fn checked_add_transaction_weighted_random( candidate_txs: &mut Vec, tx_weights: WeightedIndex, selected_txs: &mut Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)>, - mempool_tx_deps: &HashMap>, + mempool_tx_deps: &mut TransactionDependencies, remaining_block_bytes: &mut usize, remaining_block_sigops: &mut u64, remaining_block_unpaid_actions: &mut u32, @@ -230,7 +231,9 @@ fn checked_add_transaction_weighted_random( let (new_tx_weights, candidate_tx) = choose_transaction_weighted_random(candidate_txs, tx_weights); - let candidate_tx_deps = mempool_tx_deps.get(&candidate_tx.transaction.id.mined_id()); + let candidate_tx_deps = mempool_tx_deps + .dependencies() + .get(&candidate_tx.transaction.id.mined_id()); // > If the block template with this transaction included // > would be within the block size limit and block sigop limit, @@ -258,7 +261,7 @@ fn checked_add_transaction_weighted_random( ) { selected_txs.push(( - dependencies_depth(candidate_tx_deps, mempool_tx_deps), + dependencies_depth(candidate_tx_deps, mempool_tx_deps.dependencies()), candidate_tx.clone(), )); @@ -294,6 +297,7 @@ fn choose_transaction_weighted_random( fn excludes_tx_with_unselected_dependencies() { use zebra_chain::{ amount::Amount, block::Block, serialization::ZcashDeserializeInto, transaction::UnminedTx, + transparent::OutPoint, }; let network = Network::Mainnet; @@ -327,16 +331,16 @@ fn excludes_tx_with_unselected_dependencies() { .id .mined_id(); - let mut mempool_tx_deps = HashMap::new(); + let mut mempool_tx_deps = TransactionDependencies::default(); - mempool_tx_deps.insert( + mempool_tx_deps.add( mempool_txns .first() .expect("should not be empty") .transaction .id .mined_id(), - [dep_tx_id].into(), + vec![OutPoint::from_usize(dep_tx_id, 0)], ); let like_zcashd = true; @@ -348,7 +352,7 @@ fn excludes_tx_with_unselected_dependencies() { next_block_height, &miner_address, mempool_txns, - &mempool_tx_deps, + mempool_tx_deps, like_zcashd, extra_coinbase_data, ) diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index de59cfaacb0..ee230c7cbc2 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -752,8 +752,7 @@ impl Service for Mempool { trace!(?req, "got mempool request"); let transactions: Vec<_> = storage.transactions().values().cloned().collect(); - let transaction_dependencies = - storage.transaction_dependencies().dependencies().clone(); + let transaction_dependencies = storage.transaction_dependencies().clone(); trace!(?req, transactions_count = ?transactions.len(), "answered mempool request"); From ddea84885213ea7d59e57de83ce271091232ec6d Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 26 Sep 2024 20:32:13 -0400 Subject: [PATCH 46/69] updates zip317 transaction selection for block templates to include dependent transactions --- .../src/mempool/transaction_dependencies.rs | 22 +++ .../methods/get_block_template_rpcs/zip317.rs | 162 ++++++++++++------ 2 files changed, 129 insertions(+), 55 deletions(-) diff --git a/zebra-node-services/src/mempool/transaction_dependencies.rs b/zebra-node-services/src/mempool/transaction_dependencies.rs index 02bdd65e0eb..e829fafe970 100644 --- a/zebra-node-services/src/mempool/transaction_dependencies.rs +++ b/zebra-node-services/src/mempool/transaction_dependencies.rs @@ -76,6 +76,28 @@ impl TransactionDependencies { all_dependents } + /// Returns a list of lists of transaction hashes that directly on the transaction + /// with the provided transaction hash or one of the transactions in the prior list. + // TODO: Improve this method's documentation. + pub fn all_dependents_leveled( + &self, + &tx_hash: &transaction::Hash, + ) -> Vec> { + let mut current_level_dependents: HashSet<_> = [tx_hash].into(); + let mut all_dependents = Vec::new(); + + while !current_level_dependents.is_empty() { + current_level_dependents = current_level_dependents + .iter() + .flat_map(|dep| self.dependents.get(dep).cloned().unwrap_or_default()) + .collect(); + + all_dependents.push(current_level_dependents.clone()); + } + + all_dependents + } + /// Clear the maps of transaction dependencies. pub fn clear(&mut self) { self.dependencies.clear(); diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index c0c6c356cae..fe97b0e3483 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -46,7 +46,7 @@ pub fn select_mempool_transactions( next_block_height: Height, miner_address: &transparent::Address, mempool_txs: Vec, - mut mempool_tx_deps: TransactionDependencies, + mempool_tx_deps: TransactionDependencies, like_zcashd: bool, extra_coinbase_data: Vec, ) -> Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)> { @@ -60,9 +60,18 @@ pub fn select_mempool_transactions( extra_coinbase_data, ); + let dependent_tx_ids: HashSet = + mempool_tx_deps.dependencies().keys().copied().collect(); + + let (independent_mempool_txs, mut dependent_mempool_txs): (HashMap<_, _>, HashMap<_, _>) = + mempool_txs + .into_iter() + .map(|tx| (tx.transaction.id.mined_id(), tx)) + .partition(|(tx_id, _tx)| !dependent_tx_ids.contains(tx_id)); + // Setup the transaction lists. - let (mut conventional_fee_txs, mut low_fee_txs): (Vec<_>, Vec<_>) = mempool_txs - .into_iter() + let (mut conventional_fee_txs, mut low_fee_txs): (Vec<_>, Vec<_>) = independent_mempool_txs + .into_values() .partition(VerifiedUnminedTx::pays_conventional_fee); let mut selected_txs = Vec::new(); @@ -83,9 +92,10 @@ pub fn select_mempool_transactions( while let Some(tx_weights) = conventional_fee_tx_weights { conventional_fee_tx_weights = checked_add_transaction_weighted_random( &mut conventional_fee_txs, + &mut dependent_mempool_txs, tx_weights, &mut selected_txs, - &mut mempool_tx_deps, + &mempool_tx_deps, &mut remaining_block_bytes, &mut remaining_block_sigops, // The number of unpaid actions is always zero for transactions that pay the @@ -100,9 +110,10 @@ pub fn select_mempool_transactions( while let Some(tx_weights) = low_fee_tx_weights { low_fee_tx_weights = checked_add_transaction_weighted_random( &mut low_fee_txs, + &mut dependent_mempool_txs, tx_weights, &mut selected_txs, - &mut mempool_tx_deps, + &mempool_tx_deps, &mut remaining_block_bytes, &mut remaining_block_sigops, &mut remaining_block_unpaid_actions, @@ -189,6 +200,7 @@ fn has_direct_dependencies( /// Returns the depth of a transaction's dependencies in the block for a candidate /// transaction with the provided dependencies. +// TODO: Add a method for removing dependents from `TransactionDependencies` with their dependency depths fn dependencies_depth( candidate_tx_deps: Option<&HashSet>, mempool_tx_deps: &HashMap>, @@ -217,11 +229,13 @@ fn dependencies_depth( /// /// Returns the updated transaction weights. /// If all transactions have been chosen, returns `None`. +#[allow(clippy::too_many_arguments)] fn checked_add_transaction_weighted_random( candidate_txs: &mut Vec, + dependent_txs: &mut HashMap, tx_weights: WeightedIndex, selected_txs: &mut Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)>, - mempool_tx_deps: &mut TransactionDependencies, + mempool_tx_deps: &TransactionDependencies, remaining_block_bytes: &mut usize, remaining_block_sigops: &mut u64, remaining_block_unpaid_actions: &mut u32, @@ -231,10 +245,6 @@ fn checked_add_transaction_weighted_random( let (new_tx_weights, candidate_tx) = choose_transaction_weighted_random(candidate_txs, tx_weights); - let candidate_tx_deps = mempool_tx_deps - .dependencies() - .get(&candidate_tx.transaction.id.mined_id()); - // > If the block template with this transaction included // > would be within the block size limit and block sigop limit, // > and block_unpaid_actions <= block_unpaid_action_limit, @@ -242,40 +252,90 @@ fn checked_add_transaction_weighted_random( // // Unpaid actions are always zero for transactions that pay the conventional fee, // so the unpaid action check always passes for those transactions. - if candidate_tx.transaction.size <= *remaining_block_bytes - && candidate_tx.legacy_sigop_count <= *remaining_block_sigops - && candidate_tx.unpaid_actions <= *remaining_block_unpaid_actions - // # Correctness - // - // Transactions that spend outputs created in the same block - // must appear after the transaction that created those outputs - // - // TODO: If it gets here but the dependencies aren't selected, add it to a list of transactions - // to be added immediately if there's room once their dependencies have been selected. - // Unlike the other checks in this if statement, candidate transactions that fail this - // check may pass it in the next round, but are currently removed from the candidate set - // and will not be re-considered. - && has_direct_dependencies( - candidate_tx_deps, - selected_txs, - ) - { - selected_txs.push(( - dependencies_depth(candidate_tx_deps, mempool_tx_deps.dependencies()), - candidate_tx.clone(), - )); - - *remaining_block_bytes -= candidate_tx.transaction.size; - *remaining_block_sigops -= candidate_tx.legacy_sigop_count; - - // Unpaid actions are always zero for transactions that pay the conventional fee, - // so this limit always remains the same after they are added. - *remaining_block_unpaid_actions -= candidate_tx.unpaid_actions; + if candidate_tx.try_update_block_limits( + remaining_block_bytes, + remaining_block_sigops, + remaining_block_unpaid_actions, + ) { + let selected_tx_id = candidate_tx.transaction.id.mined_id(); + selected_txs.push((0, candidate_tx)); + + for dependent_candidate_tx_ids_by_level in + mempool_tx_deps.all_dependents_leveled(&selected_tx_id) + { + for dependent_candidate_tx_id in &dependent_candidate_tx_ids_by_level { + let mempool_tx_deps = mempool_tx_deps.dependencies(); + let candidate_tx_deps = mempool_tx_deps.get(dependent_candidate_tx_id); + + if has_direct_dependencies(candidate_tx_deps, selected_txs) { + let Some(candidate_tx) = dependent_txs.remove(dependent_candidate_tx_id) else { + continue; + }; + + // Transactions that don't pay the conventional fee should not have + // the same probability of being included as their dependencies. + if !candidate_tx.pays_conventional_fee() { + continue; + } + + if candidate_tx.try_update_block_limits( + remaining_block_bytes, + remaining_block_sigops, + remaining_block_unpaid_actions, + ) { + let candidate_tx_deps = mempool_tx_deps.get(dependent_candidate_tx_id); + selected_txs.push(( + dependencies_depth(candidate_tx_deps, mempool_tx_deps), + candidate_tx, + )); + } + } + } + } } new_tx_weights } +trait TryUpdateBlockLimits { + /// Checks if a transaction fits within the provided remaining block bytes, + /// sigops, and unpaid actions limits. + /// + /// Updates the limits and returns true if the transaction does fit, or + /// returns false otherwise. + fn try_update_block_limits( + &self, + remaining_block_bytes: &mut usize, + remaining_block_sigops: &mut u64, + remaining_block_unpaid_actions: &mut u32, + ) -> bool; +} + +impl TryUpdateBlockLimits for VerifiedUnminedTx { + fn try_update_block_limits( + &self, + remaining_block_bytes: &mut usize, + remaining_block_sigops: &mut u64, + remaining_block_unpaid_actions: &mut u32, + ) -> bool { + if self.transaction.size <= *remaining_block_bytes + && self.legacy_sigop_count <= *remaining_block_sigops + && self.unpaid_actions <= *remaining_block_unpaid_actions + { + *remaining_block_bytes -= self.transaction.size; + *remaining_block_sigops -= self.legacy_sigop_count; + + // Unpaid actions are always zero for transactions that pay the conventional fee, + // so this limit always remains the same after they are added. + *remaining_block_unpaid_actions -= self.unpaid_actions; + + true + } else { + false + } + } +} + /// Choose a transaction from `transactions`, using the previously set up `weighted_index`. /// /// If some transactions have not yet been chosen, returns the weighted index and the transaction. @@ -303,7 +363,7 @@ fn excludes_tx_with_unselected_dependencies() { let network = Network::Mainnet; let next_block_height = Height(1_000_000); let miner_address = transparent::Address::from_pub_key_hash(network.kind(), [0; 20]); - let mut mempool_txns: Vec<_> = network + let mempool_txns: Vec<_> = network .block_iter() .map(|(_, block)| { block @@ -311,26 +371,18 @@ fn excludes_tx_with_unselected_dependencies() { .expect("block test vector is structurally valid") }) .flat_map(|block| block.transactions) + .take(1) .map(UnminedTx::from) - // Skip transactions that fail ZIP-317 mempool checks - .filter_map(|transaction| { + .map(|transaction| { VerifiedUnminedTx::new( transaction, Amount::try_from(1_000_000).expect("invalid value"), 0, ) - .ok() + .unwrap() }) - .take(2) .collect(); - let dep_tx_id = mempool_txns - .pop() - .expect("should not be empty") - .transaction - .id - .mined_id(); - let mut mempool_tx_deps = TransactionDependencies::default(); mempool_tx_deps.add( @@ -340,13 +392,13 @@ fn excludes_tx_with_unselected_dependencies() { .transaction .id .mined_id(), - vec![OutPoint::from_usize(dep_tx_id, 0)], + vec![OutPoint::from_usize(transaction::Hash([0; 32]), 0)], ); let like_zcashd = true; let extra_coinbase_data = Vec::new(); - assert!( + assert_eq!( select_mempool_transactions( &network, next_block_height, @@ -355,8 +407,8 @@ fn excludes_tx_with_unselected_dependencies() { mempool_tx_deps, like_zcashd, extra_coinbase_data, - ) - .is_empty(), + ), + vec![], "should not select any transactions when dependencies are unavailable" ); } From dceaf87081a232f54e1c4985a7ce8b6a83598308 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 26 Sep 2024 20:44:55 -0400 Subject: [PATCH 47/69] Moves and refactors zip317 tx selection test to its own module, adds an `unmined_transactions_in_blocks()` method on network --- zebra-chain/src/tests/vectors.rs | 43 ++++++++++++- .../methods/get_block_template_rpcs/zip317.rs | 63 +------------------ .../get_block_template_rpcs/zip317/tests.rs | 45 +++++++++++++ 3 files changed, 89 insertions(+), 62 deletions(-) create mode 100644 zebra-rpc/src/methods/get_block_template_rpcs/zip317/tests.rs diff --git a/zebra-chain/src/tests/vectors.rs b/zebra-chain/src/tests/vectors.rs index deb2a507707..ef9d00f6d99 100644 --- a/zebra-chain/src/tests/vectors.rs +++ b/zebra-chain/src/tests/vectors.rs @@ -1,9 +1,15 @@ //! Network methods for fetching blockchain vectors. //! -use std::collections::BTreeMap; +use std::{collections::BTreeMap, ops::RangeBounds}; -use crate::{block::Block, parameters::Network, serialization::ZcashDeserializeInto}; +use crate::{ + amount::Amount, + block::Block, + parameters::Network, + serialization::ZcashDeserializeInto, + transaction::{UnminedTx, VerifiedUnminedTx}, +}; use zebra_test::vectors::{ BLOCK_MAINNET_1046400_BYTES, BLOCK_MAINNET_653599_BYTES, BLOCK_MAINNET_982681_BYTES, @@ -30,6 +36,39 @@ impl Network { } } + /// Returns iterator over verified unmined transactions in the provided block height range. + pub fn unmined_transactions_in_blocks( + &self, + block_height_range: impl RangeBounds, + ) -> impl DoubleEndedIterator { + let blocks = self.block_iter(); + + // Deserialize the blocks that are selected based on the specified `block_height_range`. + let selected_blocks = blocks + .filter(move |(&height, _)| block_height_range.contains(&height)) + .map(|(_, block)| { + block + .zcash_deserialize_into::() + .expect("block test vector is structurally valid") + }); + + // Extract the transactions from the blocks and wrap each one as an unmined transaction. + // Use a fake zero miner fee and sigops, because we don't have the UTXOs to calculate + // the correct fee. + selected_blocks + .flat_map(|block| block.transactions) + .map(UnminedTx::from) + // Skip transactions that fail ZIP-317 mempool checks + .filter_map(|transaction| { + VerifiedUnminedTx::new( + transaction, + Amount::try_from(1_000_000).expect("invalid value"), + 0, + ) + .ok() + }) + } + /// Returns blocks indexed by height in a [`BTreeMap`]. /// /// Returns Mainnet blocks if `self` is set to Mainnet, and Testnet blocks otherwise. diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index fe97b0e3483..065fa537050 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -29,6 +29,9 @@ use crate::methods::get_block_template_rpcs::{ use super::get_block_template::InBlockTxDependenciesDepth; +#[cfg(test)] +mod tests; + /// Selects mempool transactions for block production according to [ZIP-317], /// using a fake coinbase transaction and the mempool. /// @@ -352,63 +355,3 @@ fn choose_transaction_weighted_random( // (setup_fee_weighted_index(candidate_txs), candidate_tx) } - -#[test] -fn excludes_tx_with_unselected_dependencies() { - use zebra_chain::{ - amount::Amount, block::Block, serialization::ZcashDeserializeInto, transaction::UnminedTx, - transparent::OutPoint, - }; - - let network = Network::Mainnet; - let next_block_height = Height(1_000_000); - let miner_address = transparent::Address::from_pub_key_hash(network.kind(), [0; 20]); - let mempool_txns: Vec<_> = network - .block_iter() - .map(|(_, block)| { - block - .zcash_deserialize_into::() - .expect("block test vector is structurally valid") - }) - .flat_map(|block| block.transactions) - .take(1) - .map(UnminedTx::from) - .map(|transaction| { - VerifiedUnminedTx::new( - transaction, - Amount::try_from(1_000_000).expect("invalid value"), - 0, - ) - .unwrap() - }) - .collect(); - - let mut mempool_tx_deps = TransactionDependencies::default(); - - mempool_tx_deps.add( - mempool_txns - .first() - .expect("should not be empty") - .transaction - .id - .mined_id(), - vec![OutPoint::from_usize(transaction::Hash([0; 32]), 0)], - ); - - let like_zcashd = true; - let extra_coinbase_data = Vec::new(); - - assert_eq!( - select_mempool_transactions( - &network, - next_block_height, - &miner_address, - mempool_txns, - mempool_tx_deps, - like_zcashd, - extra_coinbase_data, - ), - vec![], - "should not select any transactions when dependencies are unavailable" - ); -} diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317/tests.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317/tests.rs new file mode 100644 index 00000000000..7f586e200ba --- /dev/null +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317/tests.rs @@ -0,0 +1,45 @@ +//! Tests for ZIP-317 transaction selection for block template production + +use zebra_chain::{ + block::Height, + parameters::Network, + transaction, + transparent::{self, OutPoint}, +}; +use zebra_node_services::mempool::TransactionDependencies; + +use super::select_mempool_transactions; + +#[test] +fn excludes_tx_with_unselected_dependencies() { + let network = Network::Mainnet; + let next_block_height = Height(1_000_000); + let miner_address = transparent::Address::from_pub_key_hash(network.kind(), [0; 20]); + let unmined_tx = network + .unmined_transactions_in_blocks(..) + .next() + .expect("should not be empty"); + + let mut mempool_tx_deps = TransactionDependencies::default(); + mempool_tx_deps.add( + unmined_tx.transaction.id.mined_id(), + vec![OutPoint::from_usize(transaction::Hash([0; 32]), 0)], + ); + + let like_zcashd = true; + let extra_coinbase_data = Vec::new(); + + assert_eq!( + select_mempool_transactions( + &network, + next_block_height, + &miner_address, + vec![unmined_tx], + mempool_tx_deps, + like_zcashd, + extra_coinbase_data, + ), + vec![], + "should not select any transactions when dependencies are unavailable" + ); +} From c2e58584f161ad71e391ac991de7ceebdb0291e6 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 26 Sep 2024 20:45:38 -0400 Subject: [PATCH 48/69] Removes `unmined_transactions_in_blocks()` test utility fn from mempool Storage test module and replaces calls to it with calls to the new test method on Network --- .../components/inbound/tests/fake_peer_set.rs | 7 +-- zebrad/src/components/mempool.rs | 2 +- .../src/components/mempool/storage/tests.rs | 44 +------------------ .../mempool/storage/tests/vectors.rs | 18 ++++---- zebrad/src/components/mempool/tests/vector.rs | 13 +++--- 5 files changed, 23 insertions(+), 61 deletions(-) diff --git a/zebrad/src/components/inbound/tests/fake_peer_set.rs b/zebrad/src/components/inbound/tests/fake_peer_set.rs index dae03141762..176ec8c1c57 100644 --- a/zebrad/src/components/inbound/tests/fake_peer_set.rs +++ b/zebrad/src/components/inbound/tests/fake_peer_set.rs @@ -31,8 +31,8 @@ use crate::{ components::{ inbound::{downloads::MAX_INBOUND_CONCURRENCY, Inbound, InboundSetupData}, mempool::{ - gossip_mempool_transaction_id, unmined_transactions_in_blocks, Config as MempoolConfig, - Mempool, MempoolError, SameEffectsChainRejectionError, UnboxMempoolError, + gossip_mempool_transaction_id, Config as MempoolConfig, Mempool, MempoolError, + SameEffectsChainRejectionError, UnboxMempoolError, }, sync::{self, BlockGossipError, SyncStatus, PEER_GOSSIP_DELAY}, }, @@ -1054,7 +1054,8 @@ fn add_some_stuff_to_mempool( network: Network, ) -> Vec { // get the genesis block coinbase transaction from the Zcash blockchain. - let genesis_transactions: Vec<_> = unmined_transactions_in_blocks(..=0, &network) + let genesis_transactions: Vec<_> = network + .unmined_transactions_in_blocks(..=0) .take(1) .collect(); diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index ee230c7cbc2..f08737ad9c5 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -69,7 +69,7 @@ pub use storage::{ }; #[cfg(test)] -pub use self::{storage::tests::unmined_transactions_in_blocks, tests::UnboxMempoolError}; +pub use self::tests::UnboxMempoolError; use downloads::{ Downloads as TxDownloads, TRANSACTION_DOWNLOAD_TIMEOUT, TRANSACTION_VERIFY_TIMEOUT, diff --git a/zebrad/src/components/mempool/storage/tests.rs b/zebrad/src/components/mempool/storage/tests.rs index c134414f4a4..197b706d2a4 100644 --- a/zebrad/src/components/mempool/storage/tests.rs +++ b/zebrad/src/components/mempool/storage/tests.rs @@ -1,46 +1,4 @@ -//! Tests and test utility functions for mempool storage. - -use std::ops::RangeBounds; - -use zebra_chain::{ - amount::Amount, - block::Block, - parameters::Network, - serialization::ZcashDeserializeInto, - transaction::{UnminedTx, VerifiedUnminedTx}, -}; +//! Tests for mempool storage. mod prop; mod vectors; - -pub fn unmined_transactions_in_blocks( - block_height_range: impl RangeBounds, - network: &Network, -) -> impl DoubleEndedIterator { - let blocks = network.block_iter(); - - // Deserialize the blocks that are selected based on the specified `block_height_range`. - let selected_blocks = blocks - .filter(move |(&height, _)| block_height_range.contains(&height)) - .map(|(_, block)| { - block - .zcash_deserialize_into::() - .expect("block test vector is structurally valid") - }); - - // Extract the transactions from the blocks and wrap each one as an unmined transaction. - // Use a fake zero miner fee and sigops, because we don't have the UTXOs to calculate - // the correct fee. - selected_blocks - .flat_map(|block| block.transactions) - .map(UnminedTx::from) - // Skip transactions that fail ZIP-317 mempool checks - .filter_map(|transaction| { - VerifiedUnminedTx::new( - transaction, - Amount::try_from(1_000_000).expect("invalid value"), - 0, - ) - .ok() - }) -} diff --git a/zebrad/src/components/mempool/storage/tests/vectors.rs b/zebrad/src/components/mempool/storage/tests/vectors.rs index 40db3b3c786..8bba8822d01 100644 --- a/zebrad/src/components/mempool/storage/tests/vectors.rs +++ b/zebrad/src/components/mempool/storage/tests/vectors.rs @@ -11,9 +11,7 @@ use zebra_chain::{ parameters::Network, }; -use crate::components::mempool::{ - storage::tests::unmined_transactions_in_blocks, storage::*, Mempool, -}; +use crate::components::mempool::{storage::*, Mempool}; /// Eviction memory time used for tests. Most tests won't care about this /// so we use a large enough value that will never be reached in the tests. @@ -36,7 +34,8 @@ fn mempool_storage_crud_exact_mainnet() { }); // Get one (1) unmined transaction - let unmined_tx = unmined_transactions_in_blocks(.., &network) + let unmined_tx = network + .unmined_transactions_in_blocks(..) .next() .expect("at least one unmined transaction"); @@ -70,7 +69,7 @@ fn mempool_storage_basic() -> Result<()> { fn mempool_storage_basic_for_network(network: Network) -> Result<()> { // Get transactions from the first 10 blocks of the Zcash blockchain - let unmined_transactions: Vec<_> = unmined_transactions_in_blocks(..=10, &network).collect(); + let unmined_transactions: Vec<_> = network.unmined_transactions_in_blocks(..=10).collect(); assert!( MEMPOOL_TX_COUNT < unmined_transactions.len(), @@ -163,7 +162,8 @@ fn mempool_storage_crud_same_effects_mainnet() { }); // Get one (1) unmined transaction - let unmined_tx_1 = unmined_transactions_in_blocks(.., &network) + let unmined_tx_1 = network + .unmined_transactions_in_blocks(..) .next() .expect("at least one unmined transaction"); @@ -194,7 +194,8 @@ fn mempool_storage_crud_same_effects_mainnet() { ); // Get a different unmined transaction - let unmined_tx_2 = unmined_transactions_in_blocks(1.., &network) + let unmined_tx_2 = network + .unmined_transactions_in_blocks(1..) .find(|tx| { tx.transaction .transaction @@ -308,7 +309,8 @@ fn mempool_removes_dependent_transactions() -> Result<()> { }); let unmined_txs_with_transparent_outputs = || { - unmined_transactions_in_blocks(.., &network) + network + .unmined_transactions_in_blocks(..) .filter(|tx| !tx.transaction.transaction.outputs().is_empty()) }; diff --git a/zebrad/src/components/mempool/tests/vector.rs b/zebrad/src/components/mempool/tests/vector.rs index b2bf033dc41..6b74d450d89 100644 --- a/zebrad/src/components/mempool/tests/vector.rs +++ b/zebrad/src/components/mempool/tests/vector.rs @@ -42,7 +42,7 @@ async fn mempool_service_basic_single() -> Result<(), Report> { let network = Network::Mainnet; // get the genesis block transactions from the Zcash blockchain. - let mut unmined_transactions = unmined_transactions_in_blocks(1..=10, &network); + let mut unmined_transactions = network.unmined_transactions_in_blocks(1..=10); let genesis_transaction = unmined_transactions .next() .expect("Missing genesis transaction"); @@ -187,7 +187,7 @@ async fn mempool_queue_single() -> Result<(), Report> { let network = Network::Mainnet; // Get transactions to use in the test - let unmined_transactions = unmined_transactions_in_blocks(1..=10, &network); + let unmined_transactions = network.unmined_transactions_in_blocks(1..=10); let mut transactions = unmined_transactions.collect::>(); // Split unmined_transactions into: // [transactions..., new_tx] @@ -280,7 +280,7 @@ async fn mempool_service_disabled() -> Result<(), Report> { setup(&network, u64::MAX, true).await; // get the genesis block transactions from the Zcash blockchain. - let mut unmined_transactions = unmined_transactions_in_blocks(1..=10, &network); + let mut unmined_transactions = network.unmined_transactions_in_blocks(1..=10); let genesis_transaction = unmined_transactions .next() .expect("Missing genesis transaction"); @@ -618,7 +618,7 @@ async fn mempool_failed_verification_is_rejected() -> Result<(), Report> { ) = setup(&network, u64::MAX, true).await; // Get transactions to use in the test - let mut unmined_transactions = unmined_transactions_in_blocks(1..=2, &network); + let mut unmined_transactions = network.unmined_transactions_in_blocks(1..=2); let rejected_tx = unmined_transactions.next().unwrap().clone(); // Enable the mempool @@ -693,7 +693,7 @@ async fn mempool_failed_download_is_not_rejected() -> Result<(), Report> { ) = setup(&network, u64::MAX, true).await; // Get transactions to use in the test - let mut unmined_transactions = unmined_transactions_in_blocks(1..=2, &network); + let mut unmined_transactions = network.unmined_transactions_in_blocks(1..=2); let rejected_valid_tx = unmined_transactions.next().unwrap().clone(); // Enable the mempool @@ -939,7 +939,8 @@ async fn mempool_responds_to_await_output() -> Result<(), Report> { ) = setup(&network, u64::MAX, true).await; mempool.enable(&mut recent_syncs).await; - let verified_unmined_tx = unmined_transactions_in_blocks(1..=10, &network) + let verified_unmined_tx = network + .unmined_transactions_in_blocks(1..=10) .find(|tx| !tx.transaction.transaction.outputs().is_empty()) .expect("should have at least 1 tx with transparent outputs"); From fa9d89b06f0ed34cfb574d84b4838d4ac483f3b4 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 26 Sep 2024 20:49:50 -0400 Subject: [PATCH 49/69] Fixes spelling mistake --- .../src/methods/get_block_template_rpcs/types/long_poll.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs index f692f83f9ab..08439df2fcf 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/long_poll.rs @@ -300,7 +300,7 @@ impl TryFrom for LongPollId { /// Check that [`LongPollInput::new`] will sort mempool transaction ids. /// -/// The mempool does not currently gaurantee the order in which it will return transactions and +/// The mempool does not currently guarantee the order in which it will return transactions and /// may return the same items in a different order, while the long poll id should be the same if /// its other components are equal and no transactions have been added or removed in the mempool. #[test] From dbcfafc65bfdf6c21ff1d8df15935ef60045088d Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 26 Sep 2024 21:08:52 -0400 Subject: [PATCH 50/69] Adds `includes_tx_with_selected_dependencies` test --- .../get_block_template_rpcs/zip317/tests.rs | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317/tests.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317/tests.rs index 7f586e200ba..a132d855937 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317/tests.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317/tests.rs @@ -43,3 +43,74 @@ fn excludes_tx_with_unselected_dependencies() { "should not select any transactions when dependencies are unavailable" ); } + +#[test] +fn includes_tx_with_selected_dependencies() { + let network = Network::Mainnet; + let next_block_height = Height(1_000_000); + let miner_address = transparent::Address::from_pub_key_hash(network.kind(), [0; 20]); + let unmined_txs: Vec<_> = network.unmined_transactions_in_blocks(..).take(3).collect(); + + let dependent_tx1 = unmined_txs.first().expect("should have 3 txns"); + let dependent_tx2 = unmined_txs.get(1).expect("should have 3 txns"); + let independent_tx_id = unmined_txs + .get(2) + .expect("should have 3 txns") + .transaction + .id + .mined_id(); + + let mut mempool_tx_deps = TransactionDependencies::default(); + mempool_tx_deps.add( + dependent_tx1.transaction.id.mined_id(), + vec![OutPoint::from_usize(independent_tx_id, 0)], + ); + mempool_tx_deps.add( + dependent_tx2.transaction.id.mined_id(), + vec![ + OutPoint::from_usize(independent_tx_id, 0), + OutPoint::from_usize(transaction::Hash([0; 32]), 0), + ], + ); + + let like_zcashd = true; + let extra_coinbase_data = Vec::new(); + + let selected_txs = select_mempool_transactions( + &network, + next_block_height, + &miner_address, + unmined_txs.clone(), + mempool_tx_deps.clone(), + like_zcashd, + extra_coinbase_data, + ); + + assert_eq!( + selected_txs.len(), + 2, + "should select the independent transaction and 1 of the dependent txs, selected: {selected_txs:?}" + ); + + let selected_tx_by_id = |id| { + selected_txs + .iter() + .find(|(_, tx)| tx.transaction.id.mined_id() == id) + }; + + let (dependency_depth, _) = + selected_tx_by_id(independent_tx_id).expect("should select the independent tx"); + + assert_eq!( + *dependency_depth, 0, + "should return a dependency depth of 0 for the independent tx" + ); + + let (dependency_depth, _) = selected_tx_by_id(dependent_tx1.transaction.id.mined_id()) + .expect("should select dependent_tx1"); + + assert_eq!( + *dependency_depth, 1, + "should return a dependency depth of 1 for the dependent tx" + ); +} From 66b841e91722b3e527d8cfec87bc179b8fe5b890 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 30 Sep 2024 15:13:47 -0400 Subject: [PATCH 51/69] fixes zip317 block construction issue --- .../src/mempool/transaction_dependencies.rs | 36 ++---- .../methods/get_block_template_rpcs/zip317.rs | 120 ++++++++++-------- .../mempool/storage/verified_set.rs | 2 +- 3 files changed, 83 insertions(+), 75 deletions(-) diff --git a/zebra-node-services/src/mempool/transaction_dependencies.rs b/zebra-node-services/src/mempool/transaction_dependencies.rs index e829fafe970..d27aec0a112 100644 --- a/zebra-node-services/src/mempool/transaction_dependencies.rs +++ b/zebra-node-services/src/mempool/transaction_dependencies.rs @@ -42,11 +42,15 @@ impl TransactionDependencies { .insert(dependent); } + // Only add an entries to `dependencies` for transactions that spend unmined outputs so it + // can be used to handle transactions with dependencies differently during block production. if !spent_mempool_outpoints.is_empty() { - self.dependencies.entry(dependent).or_default().extend( + self.dependencies.insert( + dependent, spent_mempool_outpoints .into_iter() - .map(|outpoint| outpoint.hash), + .map(|outpoint| outpoint.hash) + .collect(), ); } } @@ -57,7 +61,7 @@ impl TransactionDependencies { /// /// Returns a list of transaction hashes that have been removed if they were previously /// in this [`TransactionDependencies`]. - pub fn remove(&mut self, &tx_hash: &transaction::Hash) -> HashSet { + pub fn remove_all(&mut self, &tx_hash: &transaction::Hash) -> HashSet { let mut current_level_dependents: HashSet<_> = [tx_hash].into(); let mut all_dependents = current_level_dependents.clone(); @@ -76,26 +80,14 @@ impl TransactionDependencies { all_dependents } - /// Returns a list of lists of transaction hashes that directly on the transaction - /// with the provided transaction hash or one of the transactions in the prior list. - // TODO: Improve this method's documentation. - pub fn all_dependents_leveled( - &self, - &tx_hash: &transaction::Hash, - ) -> Vec> { - let mut current_level_dependents: HashSet<_> = [tx_hash].into(); - let mut all_dependents = Vec::new(); - - while !current_level_dependents.is_empty() { - current_level_dependents = current_level_dependents - .iter() - .flat_map(|dep| self.dependents.get(dep).cloned().unwrap_or_default()) - .collect(); - - all_dependents.push(current_level_dependents.clone()); - } + /// Returns a list of hashes of transactions that directly depend on the transaction for `tx_hash`. + pub fn direct_dependents(&self, tx_hash: &transaction::Hash) -> HashSet { + self.dependents.get(tx_hash).cloned().unwrap_or_default() + } - all_dependents + /// Returns a list of hashes of transactions that are direct dependencies of the transaction for `tx_hash`. + pub fn direct_dependencies(&self, tx_hash: &transaction::Hash) -> HashSet { + self.dependencies.get(tx_hash).cloned().unwrap_or_default() } /// Clear the maps of transaction dependencies. diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index 065fa537050..6aef284043b 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -63,14 +63,12 @@ pub fn select_mempool_transactions( extra_coinbase_data, ); - let dependent_tx_ids: HashSet = - mempool_tx_deps.dependencies().keys().copied().collect(); - + let tx_dependencies = mempool_tx_deps.dependencies(); let (independent_mempool_txs, mut dependent_mempool_txs): (HashMap<_, _>, HashMap<_, _>) = mempool_txs .into_iter() .map(|tx| (tx.transaction.id.mined_id(), tx)) - .partition(|(tx_id, _tx)| !dependent_tx_ids.contains(tx_id)); + .partition(|(tx_id, _tx)| !tx_dependencies.contains_key(tx_id)); // Setup the transaction lists. let (mut conventional_fee_txs, mut low_fee_txs): (Vec<_>, Vec<_>) = independent_mempool_txs @@ -195,27 +193,31 @@ fn has_direct_dependencies( for (_, tx) in selected_txs { if deps.contains(&tx.transaction.id.mined_id()) { num_available_deps += 1; + } else { + continue; + } + + if num_available_deps == deps.len() { + return true; } } - num_available_deps == deps.len() + false } /// Returns the depth of a transaction's dependencies in the block for a candidate /// transaction with the provided dependencies. -// TODO: Add a method for removing dependents from `TransactionDependencies` with their dependency depths fn dependencies_depth( - candidate_tx_deps: Option<&HashSet>, - mempool_tx_deps: &HashMap>, + dependent_tx_id: &transaction::Hash, + mempool_tx_deps: &TransactionDependencies, ) -> InBlockTxDependenciesDepth { - let mut current_level_deps = candidate_tx_deps.cloned().unwrap_or_default(); let mut current_level = 0; - + let mut current_level_deps = mempool_tx_deps.direct_dependencies(dependent_tx_id); while !current_level_deps.is_empty() { current_level += 1; current_level_deps = current_level_deps .iter() - .flat_map(|dep| mempool_tx_deps.get(dep).cloned().unwrap_or_default()) + .flat_map(|dep_id| mempool_tx_deps.direct_dependencies(dep_id)) .collect(); } @@ -232,6 +234,7 @@ fn dependencies_depth( /// /// Returns the updated transaction weights. /// If all transactions have been chosen, returns `None`. +// TODO: Refactor these arguments into a struct and this function into a method. #[allow(clippy::too_many_arguments)] fn checked_add_transaction_weighted_random( candidate_txs: &mut Vec, @@ -248,53 +251,59 @@ fn checked_add_transaction_weighted_random( let (new_tx_weights, candidate_tx) = choose_transaction_weighted_random(candidate_txs, tx_weights); - // > If the block template with this transaction included - // > would be within the block size limit and block sigop limit, - // > and block_unpaid_actions <= block_unpaid_action_limit, - // > add the transaction to the block template - // - // Unpaid actions are always zero for transactions that pay the conventional fee, - // so the unpaid action check always passes for those transactions. - if candidate_tx.try_update_block_limits( + if !candidate_tx.try_update_block_template_limits( remaining_block_bytes, remaining_block_sigops, remaining_block_unpaid_actions, ) { - let selected_tx_id = candidate_tx.transaction.id.mined_id(); - selected_txs.push((0, candidate_tx)); + return new_tx_weights; + } - for dependent_candidate_tx_ids_by_level in - mempool_tx_deps.all_dependents_leveled(&selected_tx_id) - { - for dependent_candidate_tx_id in &dependent_candidate_tx_ids_by_level { - let mempool_tx_deps = mempool_tx_deps.dependencies(); - let candidate_tx_deps = mempool_tx_deps.get(dependent_candidate_tx_id); - - if has_direct_dependencies(candidate_tx_deps, selected_txs) { - let Some(candidate_tx) = dependent_txs.remove(dependent_candidate_tx_id) else { - continue; - }; - - // Transactions that don't pay the conventional fee should not have - // the same probability of being included as their dependencies. - if !candidate_tx.pays_conventional_fee() { - continue; - } - - if candidate_tx.try_update_block_limits( - remaining_block_bytes, - remaining_block_sigops, - remaining_block_unpaid_actions, - ) { - let candidate_tx_deps = mempool_tx_deps.get(dependent_candidate_tx_id); - selected_txs.push(( - dependencies_depth(candidate_tx_deps, mempool_tx_deps), - candidate_tx, - )); - } + let tx_dependencies = mempool_tx_deps.dependencies(); + let selected_tx_id = &candidate_tx.transaction.id.mined_id(); + debug_assert!( + !tx_dependencies.contains_key(selected_tx_id), + "all candidate transactions should be independent" + ); + + selected_txs.push((0, candidate_tx)); + + // Try adding any dependent transactions if all of their dependencies have been selected. + + let mut current_level_dependents = mempool_tx_deps.direct_dependents(selected_tx_id); + while !current_level_dependents.is_empty() { + let mut next_level_dependents = HashSet::new(); + + for dependent_tx_id in ¤t_level_dependents { + if has_direct_dependencies(tx_dependencies.get(dependent_tx_id), selected_txs) { + let Some(candidate_tx) = dependent_txs.remove(dependent_tx_id) else { + continue; + }; + + // Transactions that don't pay the conventional fee should not have + // the same probability of being included as their dependencies. + if !candidate_tx.pays_conventional_fee() { + continue; } + + if !candidate_tx.try_update_block_template_limits( + remaining_block_bytes, + remaining_block_sigops, + remaining_block_unpaid_actions, + ) { + continue; + } + + selected_txs.push(( + dependencies_depth(dependent_tx_id, mempool_tx_deps), + candidate_tx, + )); + + next_level_dependents.extend(mempool_tx_deps.direct_dependents(dependent_tx_id)); } } + + current_level_dependents = next_level_dependents; } new_tx_weights @@ -306,7 +315,7 @@ trait TryUpdateBlockLimits { /// /// Updates the limits and returns true if the transaction does fit, or /// returns false otherwise. - fn try_update_block_limits( + fn try_update_block_template_limits( &self, remaining_block_bytes: &mut usize, remaining_block_sigops: &mut u64, @@ -315,12 +324,19 @@ trait TryUpdateBlockLimits { } impl TryUpdateBlockLimits for VerifiedUnminedTx { - fn try_update_block_limits( + fn try_update_block_template_limits( &self, remaining_block_bytes: &mut usize, remaining_block_sigops: &mut u64, remaining_block_unpaid_actions: &mut u32, ) -> bool { + // > If the block template with this transaction included + // > would be within the block size limit and block sigop limit, + // > and block_unpaid_actions <= block_unpaid_action_limit, + // > add the transaction to the block template + // + // Unpaid actions are always zero for transactions that pay the conventional fee, + // so the unpaid action check always passes for those transactions. if self.transaction.size <= *remaining_block_bytes && self.legacy_sigop_count <= *remaining_block_sigops && self.unpaid_actions <= *remaining_block_unpaid_actions diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 0fa7afadb57..5a39d938799 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -247,7 +247,7 @@ impl VerifiedSet { fn remove(&mut self, key_to_remove: &transaction::Hash) -> Vec { let removed_transactions: Vec<_> = self .transaction_dependencies - .remove(key_to_remove) + .remove_all(key_to_remove) .iter() .map(|key_to_remove| { let removed_tx = self From 2342d75d6d0f6aa007b2278c54704149e59f1857 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 30 Sep 2024 18:08:16 -0400 Subject: [PATCH 52/69] Fixes vectors test --- zebrad/src/components/mempool/tests/vector.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zebrad/src/components/mempool/tests/vector.rs b/zebrad/src/components/mempool/tests/vector.rs index 6b74d450d89..a49b4ebffec 100644 --- a/zebrad/src/components/mempool/tests/vector.rs +++ b/zebrad/src/components/mempool/tests/vector.rs @@ -984,6 +984,11 @@ async fn mempool_responds_to_await_output() -> Result<(), Report> { .expect("mempool tx verification result channel should not be closed") .expect("mocked verification should be successful"); + // Wait for next steps in mempool's Downloads to finish + // TODO: Move this and the `ready().await` below above waiting for the mempool verification result above after + // waiting to respond with a transaction's verification result until after it's been inserted into the mempool. + tokio::time::sleep(Duration::from_secs(1)).await; + mempool .ready() .await From 56f2ad411313461a5c4fd45556654c9ce235633d Mon Sep 17 00:00:00 2001 From: Arya Date: Tue, 1 Oct 2024 10:58:17 -0400 Subject: [PATCH 53/69] Update zebra-node-services/src/mempool.rs --- zebra-node-services/src/mempool.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/zebra-node-services/src/mempool.rs b/zebra-node-services/src/mempool.rs index c986e7d701d..10f51cf4a30 100644 --- a/zebra-node-services/src/mempool.rs +++ b/zebra-node-services/src/mempool.rs @@ -17,10 +17,8 @@ use crate::BoxError; mod gossip; -#[cfg(feature = "getblocktemplate-rpcs")] mod transaction_dependencies; -#[cfg(feature = "getblocktemplate-rpcs")] pub use transaction_dependencies::TransactionDependencies; pub use self::gossip::Gossip; From ce6d169a9a0a1848b1d50a9eb29b7710f48043da Mon Sep 17 00:00:00 2001 From: Arya Date: Tue, 15 Oct 2024 22:26:24 -0400 Subject: [PATCH 54/69] restores `tip_rejected_exact` type --- zebrad/src/components/mempool.rs | 9 ++-- zebrad/src/components/mempool/storage.rs | 55 +++++++++++------------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index f08737ad9c5..a971aff1c44 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -617,11 +617,11 @@ impl Service for Mempool { .download_if_needed_and_verify(tx.transaction.into(), None); } } - Ok(Err((txid, error))) => { - tracing::debug!(?txid, ?error, "mempool transaction failed to verify"); + Ok(Err((tx_id, error))) => { + tracing::debug!(?tx_id, ?error, "mempool transaction failed to verify"); metrics::counter!("mempool.failed.verify.tasks.total", "reason" => error.to_string()).increment(1); - storage.reject_if_needed(txid.mined_id(), error); + storage.reject_if_needed(tx_id, error); } Err(_elapsed) => { // A timeout happens when the stream hangs waiting for another service, @@ -788,8 +788,7 @@ impl Service for Mempool { MempoolError, > { let (rsp_tx, rsp_rx) = oneshot::channel(); - storage - .should_download_or_verify(gossiped_tx.id().mined_id())?; + storage.should_download_or_verify(gossiped_tx.id())?; tx_downloads .download_if_needed_and_verify(gossiped_tx, Some(rsp_tx))?; diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 688b214f31c..05499581d4c 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -135,7 +135,7 @@ pub struct Storage { /// current tip. /// /// Only transactions with the exact [`UnminedTxId`] are invalid. - tip_rejected_exact: HashMap, + tip_rejected_exact: HashMap, /// A set of transactions rejected for their effects, and their rejection /// reasons. These rejections only apply to the current tip. @@ -207,7 +207,7 @@ impl Storage { let tx_id = unmined_tx_id.mined_id(); // First, check if we have a cached rejection for this transaction. - if let Some(error) = self.rejection_error(&tx_id) { + if let Some(error) = self.rejection_error(&unmined_tx_id) { tracing::trace!( ?tx_id, ?error, @@ -246,7 +246,7 @@ impl Storage { ); // We could return here, but we still want to check the mempool size - self.reject(tx_id, rejection_error.clone().into()); + self.reject(unmined_tx_id, rejection_error.clone().into()); result = Err(rejection_error.into()); } @@ -277,7 +277,7 @@ impl Storage { // > Add the txid and the current time to RecentlyEvicted, dropping the oldest entry in // > RecentlyEvicted if necessary to keep it to at most `eviction_memory_entries entries`. self.reject( - victim_tx.transaction.id.mined_id(), + victim_tx.transaction.id, SameEffectsChainRejectionError::RandomlyEvicted.into(), ); @@ -381,14 +381,14 @@ impl Storage { self.reject( // the reject and rejection_error fns that store and check `SameEffectsChainRejectionError`s // only use the mined id, so using `Legacy` ids will apply to v5 transactions as well. - mined_id, + UnminedTxId::Legacy(mined_id), SameEffectsChainRejectionError::Mined.into(), ); } for duplicate_spend_id in duplicate_spend_ids { self.reject( - duplicate_spend_id.mined_id(), + duplicate_spend_id, SameEffectsChainRejectionError::DuplicateSpend.into(), ); } @@ -533,13 +533,13 @@ impl Storage { } /// Add a transaction to the rejected list for the given reason. - pub fn reject(&mut self, tx_id: transaction::Hash, reason: RejectionError) { + pub fn reject(&mut self, tx_id: UnminedTxId, reason: RejectionError) { match reason { RejectionError::ExactTip(e) => { self.tip_rejected_exact.insert(tx_id, e); } RejectionError::SameEffectsTip(e) => { - self.tip_rejected_same_effects.insert(tx_id, e); + self.tip_rejected_same_effects.insert(tx_id.mined_id(), e); } RejectionError::SameEffectsChain(e) => { let eviction_memory_time = self.eviction_memory_time; @@ -548,7 +548,7 @@ impl Storage { .or_insert_with(|| { EvictionList::new(MAX_EVICTION_MEMORY_ENTRIES, eviction_memory_time) }) - .insert(tx_id); + .insert(tx_id.mined_id()); } } self.limit_rejection_list_memory(); @@ -560,17 +560,17 @@ impl Storage { /// This matches transactions based on each rejection list's matching rule. /// /// Returns an arbitrary error if the transaction is in multiple lists. - pub fn rejection_error(&self, txid: &transaction::Hash) -> Option { + pub fn rejection_error(&self, txid: &UnminedTxId) -> Option { if let Some(error) = self.tip_rejected_exact.get(txid) { return Some(error.clone().into()); } - if let Some(error) = self.tip_rejected_same_effects.get(txid) { + if let Some(error) = self.tip_rejected_same_effects.get(&txid.mined_id()) { return Some(error.clone().into()); } for (error, set) in self.chain_rejected_same_effects.iter() { - if set.contains_key(txid) { + if set.contains_key(&txid.mined_id()) { return Some(error.clone().into()); } } @@ -587,20 +587,20 @@ impl Storage { ) -> impl Iterator + '_ { tx_ids .into_iter() - .filter(move |txid| self.contains_rejected(&txid.mined_id())) + .filter(move |txid| self.contains_rejected(txid)) } /// Returns `true` if a transaction matching the supplied [`UnminedTxId`] is in /// the mempool rejected list. /// /// This matches transactions based on each rejection list's matching rule. - pub fn contains_rejected(&self, txid: &transaction::Hash) -> bool { + pub fn contains_rejected(&self, txid: &UnminedTxId) -> bool { self.rejection_error(txid).is_some() } /// Add a transaction that failed download and verification to the rejected list /// if needed, depending on the reason for the failure. - pub fn reject_if_needed(&mut self, txid: transaction::Hash, e: TransactionDownloadVerifyError) { + pub fn reject_if_needed(&mut self, tx_id: UnminedTxId, e: TransactionDownloadVerifyError) { match e { // Rejecting a transaction already in state would speed up further // download attempts without checking the state. However it would @@ -623,7 +623,7 @@ impl Storage { // Consensus verification failed. Reject transaction to avoid // having to download and verify it again just for it to fail again. TransactionDownloadVerifyError::Invalid(e) => { - self.reject(txid, ExactTipRejectionError::FailedVerification(e).into()) + self.reject(tx_id, ExactTipRejectionError::FailedVerification(e).into()) } } } @@ -641,42 +641,39 @@ impl Storage { &mut self, tip_height: zebra_chain::block::Height, ) -> HashSet { - let mut txid_set = HashSet::new(); + let mut tx_ids = HashSet::new(); // we need a separate set, since reject() takes the original unmined ID, // then extracts the mined ID out of it - let mut unmined_id_set = HashSet::new(); + let mut mined_tx_ids = HashSet::new(); - for (&tx_id, tx) in self.transactions() { + for (tx_id, tx) in self.transactions() { if let Some(expiry_height) = tx.transaction.transaction.expiry_height() { if tip_height >= expiry_height { - txid_set.insert(tx_id); - unmined_id_set.insert(tx_id); + tx_ids.insert(tx.transaction.id); + mined_tx_ids.insert(*tx_id); } } } // expiry height is effecting data, so we match by non-malleable TXID self.verified - .remove_all_that(|tx| txid_set.contains(&tx.transaction.id.mined_id())); + .remove_all_that(|tx| tx_ids.contains(&tx.transaction.id)); // also reject it - for &id in &unmined_id_set { + for &id in &tx_ids { self.reject(id, SameEffectsChainRejectionError::Expired.into()); } - unmined_id_set + mined_tx_ids } /// Check if transaction should be downloaded and/or verified. /// /// If it is already in the mempool (or in its rejected list) /// then it shouldn't be downloaded/verified. - pub fn should_download_or_verify( - &mut self, - txid: transaction::Hash, - ) -> Result<(), MempoolError> { + pub fn should_download_or_verify(&mut self, txid: UnminedTxId) -> Result<(), MempoolError> { // Check if the transaction is already in the mempool. - if self.contains_transaction_exact(&txid) { + if self.contains_transaction_exact(&txid.mined_id()) { return Err(MempoolError::InMempool); } if let Some(error) = self.rejection_error(&txid) { From f897489f8b51299c86b06666c9be03d0423dfae2 Mon Sep 17 00:00:00 2001 From: Arya Date: Tue, 15 Oct 2024 22:30:03 -0400 Subject: [PATCH 55/69] updates affected tests --- .../src/components/mempool/storage/tests/prop.rs | 16 ++++++++-------- .../components/mempool/storage/tests/vectors.rs | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/zebrad/src/components/mempool/storage/tests/prop.rs b/zebrad/src/components/mempool/storage/tests/prop.rs index 3b398e20ebf..398ba0925f9 100644 --- a/zebrad/src/components/mempool/storage/tests/prop.rs +++ b/zebrad/src/components/mempool/storage/tests/prop.rs @@ -86,7 +86,7 @@ proptest! { }); for rejection in unique_ids { - storage.reject(rejection.mined_id(), SameEffectsTipRejectionError::SpendConflict.into()); + storage.reject(rejection, SameEffectsTipRejectionError::SpendConflict.into()); } // Make sure there were no duplicates @@ -135,7 +135,7 @@ proptest! { }); for rejection in unique_ids { - storage.reject(rejection.mined_id(), SameEffectsChainRejectionError::RandomlyEvicted.into()); + storage.reject(rejection, SameEffectsChainRejectionError::RandomlyEvicted.into()); } // Make sure there were no duplicates @@ -202,7 +202,7 @@ proptest! { }); for (index, rejection) in unique_ids.enumerate() { - storage.reject(rejection.mined_id(), rejection_error.clone()); + storage.reject(rejection, rejection_error.clone()); if index == MAX_EVICTION_MEMORY_ENTRIES - 1 { // Make sure there were no duplicates @@ -249,9 +249,9 @@ proptest! { rejection_template }).collect(); - storage.reject(unique_ids[0].mined_id(), SameEffectsChainRejectionError::RandomlyEvicted.into()); + storage.reject(unique_ids[0], SameEffectsChainRejectionError::RandomlyEvicted.into()); thread::sleep(Duration::from_millis(11)); - storage.reject(unique_ids[1].mined_id(), SameEffectsChainRejectionError::RandomlyEvicted.into()); + storage.reject(unique_ids[1], SameEffectsChainRejectionError::RandomlyEvicted.into()); prop_assert_eq!(storage.rejected_transaction_count(), 1); } @@ -288,7 +288,7 @@ proptest! { Err(MempoolError::StorageEffectsTip(SameEffectsTipRejectionError::SpendConflict)) ); - prop_assert!(storage.contains_rejected(&id_to_reject.mined_id())); + prop_assert!(storage.contains_rejected(&id_to_reject)); storage.clear(); } @@ -341,7 +341,7 @@ proptest! { Err(MempoolError::StorageEffectsTip(SameEffectsTipRejectionError::SpendConflict)) ); - prop_assert!(storage.contains_rejected(&id_to_reject.mined_id())); + prop_assert!(storage.contains_rejected(&id_to_reject)); prop_assert_eq!( storage.insert(second_transaction_to_accept, Vec::new()), @@ -387,7 +387,7 @@ proptest! { let num_removals = storage.reject_and_remove_same_effects(mined_ids_to_remove, vec![]); for &removed_transaction_id in mined_ids_to_remove.iter() { prop_assert_eq!( - storage.rejection_error(&removed_transaction_id), + storage.rejection_error(&UnminedTxId::Legacy(removed_transaction_id)), Some(SameEffectsChainRejectionError::Mined.into()) ); } diff --git a/zebrad/src/components/mempool/storage/tests/vectors.rs b/zebrad/src/components/mempool/storage/tests/vectors.rs index 8bba8822d01..30ce35bb832 100644 --- a/zebrad/src/components/mempool/storage/tests/vectors.rs +++ b/zebrad/src/components/mempool/storage/tests/vectors.rs @@ -185,7 +185,7 @@ fn mempool_storage_crud_same_effects_mainnet() { // Check that it's rejection is cached in the chain_rejected_same_effects' `Mined` eviction list. assert_eq!( - storage.rejection_error(&unmined_tx_1.transaction.id.mined_id()), + storage.rejection_error(&unmined_tx_1.transaction.id), Some(SameEffectsChainRejectionError::Mined.into()) ); assert_eq!( @@ -226,7 +226,7 @@ fn mempool_storage_crud_same_effects_mainnet() { // Check that it's rejection is cached in the chain_rejected_same_effects' `SpendConflict` eviction list. assert_eq!( - storage.rejection_error(&unmined_tx_2.transaction.id.mined_id()), + storage.rejection_error(&unmined_tx_2.transaction.id), Some(SameEffectsChainRejectionError::DuplicateSpend.into()) ); assert_eq!( From cc76b40885215fa1903ad26349583ac863f22f54 Mon Sep 17 00:00:00 2001 From: Arya Date: Tue, 15 Oct 2024 22:42:54 -0400 Subject: [PATCH 56/69] Documents the new argument in `Storage::insert()`, updates outdated comment --- zebrad/src/components/mempool/storage.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 05499581d4c..a7c98d66bad 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -188,6 +188,10 @@ impl Storage { /// Insert a [`VerifiedUnminedTx`] into the mempool, caching any rejections. /// + /// Accepts the [`VerifiedUnminedTx`] being inserted and `spent_mempool_outpoints`, + /// a list of transparent inputs of the provided [`VerifiedUnminedTx`] that were found + /// as newly created transparent outputs in the mempool during transaction verification. + /// /// Returns an error if the mempool's verified transactions or rejection caches /// prevent this transaction from being inserted. /// These errors should not be propagated to peers, because the transactions are valid. @@ -282,7 +286,6 @@ impl Storage { ); // If this transaction gets evicted, set its result to the same error - // (we could return here, but we still want to check the mempool size) if victim_tx.transaction.id == unmined_tx_id { result = Err(SameEffectsChainRejectionError::RandomlyEvicted.into()); } From 64edb3603f95cac080d4db0022e8fc22a89092a0 Mon Sep 17 00:00:00 2001 From: Arya Date: Tue, 15 Oct 2024 22:48:41 -0400 Subject: [PATCH 57/69] Update zebrad/src/components/mempool/storage/verified_set.rs --- zebrad/src/components/mempool/storage/verified_set.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 5a39d938799..d862fc2d841 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -151,7 +151,7 @@ impl VerifiedSet { } // This likely only needs to check that the transaction hash of the outpoint is still in the mempool, - // bu it's likely rare that a transaction spends multiple transparent outputs of + // but it's likely rare that a transaction spends multiple transparent outputs of // a single transaction in practice. for outpoint in &spent_mempool_outpoints { if !self.created_outputs.contains_key(outpoint) { From a6d963f4da7973dbfa49a0ca44a2c03837055f93 Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 18 Oct 2024 19:24:21 -0400 Subject: [PATCH 58/69] fixes potential issue with calling buffered mempool's poll_ready() method without calling it. --- zebra-consensus/src/transaction.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index b00854ee104..3d2c285ed90 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -552,7 +552,12 @@ where if !transaction.transaction.transaction.outputs().is_empty() { tokio::spawn(async move { tokio::time::sleep(POLL_MEMPOOL_DELAY).await; - mempool.ready().await.expect("mempool poll_ready() method should not return an error"); + let _ = mempool + .ready() + .await + .expect("mempool poll_ready() method should not return an error") + .call(mempool::Request::CheckForVerifiedTransactions) + .await; }); } } From 5793ada9db95cf172d5d7542b4d0725128bd6096 Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 18 Oct 2024 19:54:16 -0400 Subject: [PATCH 59/69] Avoids removing dependent transactions of transactions that have been mined onto the best chain. --- .../src/mempool/transaction_dependencies.rs | 16 ++++++++++++++++ zebrad/src/components/mempool.rs | 1 + zebrad/src/components/mempool/storage.rs | 5 +++++ .../components/mempool/storage/verified_set.rs | 7 +++++++ 4 files changed, 29 insertions(+) diff --git a/zebra-node-services/src/mempool/transaction_dependencies.rs b/zebra-node-services/src/mempool/transaction_dependencies.rs index d27aec0a112..6a6999ae44c 100644 --- a/zebra-node-services/src/mempool/transaction_dependencies.rs +++ b/zebra-node-services/src/mempool/transaction_dependencies.rs @@ -55,6 +55,22 @@ impl TransactionDependencies { } } + /// Removes all dependents for a list of mined transaction ids and removes the mined transaction ids + /// from the dependencies of their dependents. + pub fn clear_mined_dependencies(&mut self, mined_ids: &HashSet) { + for mined_tx_id in mined_ids { + for dependent_id in self.dependents.remove(mined_tx_id).unwrap_or_default() { + let Some(dependencies) = self.dependencies.get_mut(&dependent_id) else { + // TODO: Move this struct to zebra-chain and log a warning here. + continue; + }; + + // TODO: Move this struct to zebra-chain and log a warning here if the dependency was not found. + let _ = dependencies.remove(&dependent_id); + } + } + } + /// Removes the hash of a transaction in the mempool and the hashes of any transactions /// that are tracked as being directly or indirectly dependent on that transaction from /// this [`TransactionDependencies`]. diff --git a/zebrad/src/components/mempool.rs b/zebrad/src/components/mempool.rs index a971aff1c44..b94ad0b09b8 100644 --- a/zebrad/src/components/mempool.rs +++ b/zebrad/src/components/mempool.rs @@ -643,6 +643,7 @@ impl Service for Mempool { // with the same mined IDs as recently mined transactions. let mined_ids = block.transaction_hashes.iter().cloned().collect(); tx_downloads.cancel(&mined_ids); + storage.clear_mined_dependencies(&mined_ids); storage.reject_and_remove_same_effects(&mined_ids, block.transactions); // Clear any transaction rejections if they might have become valid after diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index a7c98d66bad..48650a8b444 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -315,6 +315,11 @@ impl Storage { .remove_all_that(|tx| exact_wtxids.contains(&tx.transaction.id)) } + /// Clears a list of mined transaction ids from the verified set's tracked transaction dependencies. + pub fn clear_mined_dependencies(&mut self, mined_ids: &HashSet) { + self.verified.clear_mined_dependencies(mined_ids); + } + /// Reject and remove transactions from the mempool via non-malleable [`transaction::Hash`]. /// - For v5 transactions, transactions are matched by TXID, /// using only the non-malleable transaction ID. diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index d862fc2d841..7d73bd043a8 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -222,6 +222,13 @@ impl VerifiedSet { } } + /// Clears a list of mined transaction ids from the lists of dependencies for + /// any other transactions in the mempool and removes their dependents. + pub fn clear_mined_dependencies(&mut self, mined_ids: &HashSet) { + self.transaction_dependencies + .clear_mined_dependencies(mined_ids); + } + /// Removes all transactions in the set that match the `predicate`. /// /// Returns the amount of transactions removed. From 257567b559dba0359519d61684e80daac52cc434 Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 4 Nov 2024 20:15:33 -0500 Subject: [PATCH 60/69] Updates `spent_utxos()` method documentation --- zebra-consensus/src/transaction.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 3d2c285ed90..8c52df76952 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -608,11 +608,15 @@ where } } - /// Wait for the UTXOs that are being spent by the given transaction. + /// Waits for the UTXOs that are being spent by the given transaction to arrive in + /// the state for [`Block`](Request::Block) requests. /// /// `known_utxos` are additional UTXOs known at the time of validation (i.e. /// from previous transactions in the block). /// + /// Looks up UTXOs that are being spent by the given transaction in the state or waits + /// for them to be added to the mempool for [`Mempool`](Request::Mempool) requests. + /// /// Returns a tuple with a OutPoint -> Utxo map, and a vector of Outputs /// in the same order as the matching inputs in the transaction. async fn spent_utxos( From 42dfc394501fa2a48dc2243682d94fadb37720b1 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 7 Nov 2024 18:23:27 -0500 Subject: [PATCH 61/69] Avoids sorting getblocktemplate transactions in non-test compilations --- .../src/methods/get_block_template_rpcs.rs | 2 +- .../types/get_block_template.rs | 59 +++++++++++-------- .../methods/get_block_template_rpcs/zip317.rs | 28 +++++++-- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index ef65d884bb5..aed926b3635 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -904,7 +904,7 @@ where tracing::debug!( selected_mempool_tx_hashes = ?mempool_txs .iter() - .map(|(_, tx)| tx.transaction.id.mined_id()) + .map(|#[cfg(not(test))] tx, #[cfg(test)] (_, tx)| tx.transaction.id.mined_id()) .collect::>(), "selected transactions for the template from the mempool" ); diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs index a8997da6c89..363d0523404 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs @@ -232,7 +232,8 @@ impl GetBlockTemplate { miner_address: &transparent::Address, chain_tip_and_local_time: &GetBlockTemplateChainInfo, long_poll_id: LongPollId, - mempool_txs: Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)>, + #[cfg(not(test))] mempool_txs: Vec, + #[cfg(test)] mempool_txs: Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)>, submit_old: Option, like_zcashd: bool, extra_coinbase_data: Vec, @@ -242,14 +243,9 @@ impl GetBlockTemplate { (chain_tip_and_local_time.tip_height + 1).expect("tip is far below Height::MAX"); // Convert transactions into TransactionTemplates - let mut mempool_txs_with_templates: Vec<( - InBlockTxDependenciesDepth, - TransactionTemplate, - VerifiedUnminedTx, - )> = mempool_txs - .into_iter() - .map(|(min_tx_index, tx)| (min_tx_index, (&tx).into(), tx)) - .collect(); + #[cfg(not(test))] + let (mempool_tx_templates, mempool_txs): (Vec<_>, Vec<_>) = + mempool_txs.into_iter().map(|tx| ((&tx).into(), tx)).unzip(); // Transaction selection returns transactions in an arbitrary order, // but Zebra's snapshot tests expect the same order every time. @@ -258,23 +254,34 @@ impl GetBlockTemplate { // // Transactions that spend outputs created in the same block must appear // after the transactions that create those outputs. - if like_zcashd { - // Sort in serialized data order, excluding the length byte. - // `zcashd` sometimes seems to do this, but other times the order is arbitrary. - mempool_txs_with_templates.sort_by_key(|(min_tx_index, tx_template, _tx)| { - (*min_tx_index, tx_template.data.clone()) - }); - } else { - // Sort by hash, this is faster. - mempool_txs_with_templates.sort_by_key(|(min_tx_index, tx_template, _tx)| { - (*min_tx_index, tx_template.hash.bytes_in_display_order()) - }); - } - - let (mempool_tx_templates, mempool_txs): (Vec<_>, Vec<_>) = mempool_txs_with_templates - .into_iter() - .map(|(_, template, tx)| (template, tx)) - .unzip(); + #[cfg(test)] + let (mempool_tx_templates, mempool_txs): (Vec<_>, Vec<_>) = { + let mut mempool_txs_with_templates: Vec<( + InBlockTxDependenciesDepth, + TransactionTemplate, + VerifiedUnminedTx, + )> = mempool_txs + .into_iter() + .map(|(min_tx_index, tx)| (min_tx_index, (&tx).into(), tx)) + .collect(); + #[cfg(test)] + if like_zcashd { + // Sort in serialized data order, excluding the length byte. + // `zcashd` sometimes seems to do this, but other times the order is arbitrary. + mempool_txs_with_templates.sort_by_key(|(min_tx_index, tx_template, _tx)| { + (*min_tx_index, tx_template.data.clone()) + }); + } else { + // Sort by hash, this is faster. + mempool_txs_with_templates.sort_by_key(|(min_tx_index, tx_template, _tx)| { + (*min_tx_index, tx_template.hash.bytes_in_display_order()) + }); + } + mempool_txs_with_templates + .into_iter() + .map(|(_, template, tx)| (template, tx)) + .unzip() + }; // Generate the coinbase transaction and default roots // diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index 6aef284043b..710d15f8958 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -27,11 +27,20 @@ use crate::methods::get_block_template_rpcs::{ get_block_template::generate_coinbase_transaction, types::transaction::TransactionTemplate, }; +#[cfg(test)] use super::get_block_template::InBlockTxDependenciesDepth; #[cfg(test)] mod tests; +/// Used in the return type of [`select_mempool_transactions()`] for test compilations. +#[cfg(test)] +type SelectedMempoolTx = (InBlockTxDependenciesDepth, VerifiedUnminedTx); + +/// Used in the return type of [`select_mempool_transactions()`] for non-test compilations. +#[cfg(not(test))] +type SelectedMempoolTx = VerifiedUnminedTx; + /// Selects mempool transactions for block production according to [ZIP-317], /// using a fake coinbase transaction and the mempool. /// @@ -52,7 +61,7 @@ pub fn select_mempool_transactions( mempool_tx_deps: TransactionDependencies, like_zcashd: bool, extra_coinbase_data: Vec, -) -> Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)> { +) -> Vec { // Use a fake coinbase transaction to break the dependency between transaction // selection, the miner fee, and the fee payment in the coinbase transaction. let fake_coinbase_tx = fake_coinbase_transaction( @@ -183,14 +192,16 @@ fn setup_fee_weighted_index(transactions: &[VerifiedUnminedTx]) -> Option>, - selected_txs: &Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)>, + selected_txs: &Vec, ) -> bool { let Some(deps) = candidate_tx_deps else { return true; }; let mut num_available_deps = 0; - for (_, tx) in selected_txs { + for tx in selected_txs { + #[cfg(test)] + let (_, tx) = tx; if deps.contains(&tx.transaction.id.mined_id()) { num_available_deps += 1; } else { @@ -207,6 +218,7 @@ fn has_direct_dependencies( /// Returns the depth of a transaction's dependencies in the block for a candidate /// transaction with the provided dependencies. +#[cfg(test)] fn dependencies_depth( dependent_tx_id: &transaction::Hash, mempool_tx_deps: &TransactionDependencies, @@ -240,7 +252,7 @@ fn checked_add_transaction_weighted_random( candidate_txs: &mut Vec, dependent_txs: &mut HashMap, tx_weights: WeightedIndex, - selected_txs: &mut Vec<(InBlockTxDependenciesDepth, VerifiedUnminedTx)>, + selected_txs: &mut Vec, mempool_tx_deps: &TransactionDependencies, remaining_block_bytes: &mut usize, remaining_block_sigops: &mut u64, @@ -266,6 +278,10 @@ fn checked_add_transaction_weighted_random( "all candidate transactions should be independent" ); + #[cfg(not(test))] + selected_txs.push(candidate_tx); + + #[cfg(test)] selected_txs.push((0, candidate_tx)); // Try adding any dependent transactions if all of their dependencies have been selected. @@ -294,6 +310,10 @@ fn checked_add_transaction_weighted_random( continue; } + #[cfg(not(test))] + selected_txs.push(candidate_tx); + + #[cfg(test)] selected_txs.push(( dependencies_depth(dependent_tx_id, mempool_tx_deps), candidate_tx, From 5043adf9e97e1e5e872ed77197b27e454f17efa5 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 7 Nov 2024 18:29:37 -0500 Subject: [PATCH 62/69] documents PendingOutputs struct --- zebrad/src/components/mempool/pending_outputs.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zebrad/src/components/mempool/pending_outputs.rs b/zebrad/src/components/mempool/pending_outputs.rs index ad2cfec66b3..495613019cc 100644 --- a/zebrad/src/components/mempool/pending_outputs.rs +++ b/zebrad/src/components/mempool/pending_outputs.rs @@ -9,6 +9,8 @@ use zebra_chain::transparent; use zebra_node_services::mempool::Response; +/// Pending [`transparent::Output`] tracker for handling the mempool's +/// [`AwaitOutput` requests](zebra_node_services::mempool::Request::AwaitOutput). #[derive(Debug, Default)] pub struct PendingOutputs(HashMap>); From 4713b98da1d9ff72e57d32ea7ba9b8366c07a709 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 7 Nov 2024 18:33:21 -0500 Subject: [PATCH 63/69] Apply suggestions from code review Co-authored-by: Marek --- zebra-consensus/src/transaction.rs | 11 ++++++----- .../src/mempool/transaction_dependencies.rs | 2 +- .../src/methods/get_block_template_rpcs/zip317.rs | 10 ++++++++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 8c52df76952..a50473d5cd5 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -611,14 +611,13 @@ where /// Waits for the UTXOs that are being spent by the given transaction to arrive in /// the state for [`Block`](Request::Block) requests. /// - /// `known_utxos` are additional UTXOs known at the time of validation (i.e. - /// from previous transactions in the block). - /// /// Looks up UTXOs that are being spent by the given transaction in the state or waits /// for them to be added to the mempool for [`Mempool`](Request::Mempool) requests. /// - /// Returns a tuple with a OutPoint -> Utxo map, and a vector of Outputs - /// in the same order as the matching inputs in the transaction. + /// Returns a triple containing: + /// - `OutPoint` -> `Utxo` map, + /// - vec of `Output`s in the same order as the matching inputs in the `tx`, + /// - vec of `Outpoint`s spent by a mempool `tx` that were not found in the best chain's utxo set. async fn spent_utxos( tx: Arc, req: Request, @@ -633,6 +632,8 @@ where TransactionError, > { let is_mempool = req.is_mempool(); + // Additional UTXOs known at the time of validation, + // i.e., from previous transactions in the block. let known_utxos = req.known_utxos(); let inputs = tx.inputs(); diff --git a/zebra-node-services/src/mempool/transaction_dependencies.rs b/zebra-node-services/src/mempool/transaction_dependencies.rs index 6a6999ae44c..4b5e747a58d 100644 --- a/zebra-node-services/src/mempool/transaction_dependencies.rs +++ b/zebra-node-services/src/mempool/transaction_dependencies.rs @@ -7,7 +7,7 @@ use zebra_chain::{transaction, transparent}; /// Representation of mempool transactions' dependencies on other transactions in the mempool. #[derive(Default, Debug, Clone)] pub struct TransactionDependencies { - /// Lists of mempool transactions that create UTXOs spent by + /// Lists of mempool transaction ids that create UTXOs spent by /// a mempool transaction. Used during block template construction /// to exclude transactions from block templates unless all of the /// transactions they depend on have been included. diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index 710d15f8958..fbb9283154e 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -198,6 +198,10 @@ fn has_direct_dependencies( return true; }; +if selected_txs.len() < deps.len() { + return false; +} + let mut num_available_deps = 0; for tx in selected_txs { #[cfg(test)] @@ -291,6 +295,12 @@ fn checked_add_transaction_weighted_random( let mut next_level_dependents = HashSet::new(); for dependent_tx_id in ¤t_level_dependents { + // ## Note + // + // A necessary condition for adding the dependent tx is that it spends unmined outputs coming only from + // the selected txs, which come from the mempool. If the tx also spends in-chain outputs, it won't + // be added. This behavior is not specified by consensus rules and can be changed at any time, + // meaning that such txs could be added. if has_direct_dependencies(tx_dependencies.get(dependent_tx_id), selected_txs) { let Some(candidate_tx) = dependent_txs.remove(dependent_tx_id) else { continue; From f24505a5785c149d0da52bd0dad357f5c28e5c50 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 7 Nov 2024 18:35:07 -0500 Subject: [PATCH 64/69] cargo fmt --- zebra-consensus/src/transaction.rs | 2 +- zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index a50473d5cd5..407618b2daa 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -615,7 +615,7 @@ where /// for them to be added to the mempool for [`Mempool`](Request::Mempool) requests. /// /// Returns a triple containing: - /// - `OutPoint` -> `Utxo` map, + /// - `OutPoint` -> `Utxo` map, /// - vec of `Output`s in the same order as the matching inputs in the `tx`, /// - vec of `Outpoint`s spent by a mempool `tx` that were not found in the best chain's utxo set. async fn spent_utxos( diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index fbb9283154e..75ae9575d62 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -198,9 +198,9 @@ fn has_direct_dependencies( return true; }; -if selected_txs.len() < deps.len() { - return false; -} + if selected_txs.len() < deps.len() { + return false; + } let mut num_available_deps = 0; for tx in selected_txs { From df9c1c1894885e16fb02194130f8dca40097074d Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 8 Nov 2024 19:21:13 -0500 Subject: [PATCH 65/69] Applies suggestions from code review Avoids unnecessarily rejecting dependent transactions of randomly evicted mempool transactions. Updates `TransactionDependencies::remove_all()` to omit provided transaction id from the list of removed transaction ids. --- .../src/mempool/transaction_dependencies.rs | 6 +- .../types/get_block_template.rs | 2 +- zebrad/src/components/mempool/storage.rs | 28 ++++----- .../mempool/storage/verified_set.rs | 59 +++++++++++-------- 4 files changed, 50 insertions(+), 45 deletions(-) diff --git a/zebra-node-services/src/mempool/transaction_dependencies.rs b/zebra-node-services/src/mempool/transaction_dependencies.rs index 4b5e747a58d..2b333060b77 100644 --- a/zebra-node-services/src/mempool/transaction_dependencies.rs +++ b/zebra-node-services/src/mempool/transaction_dependencies.rs @@ -75,11 +75,11 @@ impl TransactionDependencies { /// that are tracked as being directly or indirectly dependent on that transaction from /// this [`TransactionDependencies`]. /// - /// Returns a list of transaction hashes that have been removed if they were previously - /// in this [`TransactionDependencies`]. + /// Returns a list of transaction hashes that were being tracked as dependents of the + /// provided transaction hash. pub fn remove_all(&mut self, &tx_hash: &transaction::Hash) -> HashSet { + let mut all_dependents = HashSet::new(); let mut current_level_dependents: HashSet<_> = [tx_hash].into(); - let mut all_dependents = current_level_dependents.clone(); while !current_level_dependents.is_empty() { current_level_dependents = current_level_dependents diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs index 363d0523404..879425bb667 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs @@ -264,7 +264,7 @@ impl GetBlockTemplate { .into_iter() .map(|(min_tx_index, tx)| (min_tx_index, (&tx).into(), tx)) .collect(); - #[cfg(test)] + if like_zcashd { // Sort in serialized data order, excluding the length byte. // `zcashd` sometimes seems to do this, but other times the order is arbitrary. diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 48650a8b444..2459efc1149 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -270,25 +270,21 @@ impl Storage { // > EvictTransaction MUST do the following: // > Select a random transaction to evict, with probability in direct proportion to // > eviction weight. (...) Remove it from the mempool. - let victim_txs = self.verified.evict_one(); + let victim_tx = self + .verified + .evict_one() + .expect("mempool is empty, but was expected to be full"); - assert!( - !victim_txs.is_empty(), - "mempool is empty, but was expected to be full" + // > Add the txid and the current time to RecentlyEvicted, dropping the oldest entry in + // > RecentlyEvicted if necessary to keep it to at most `eviction_memory_entries entries`. + self.reject( + victim_tx.transaction.id, + SameEffectsChainRejectionError::RandomlyEvicted.into(), ); - for victim_tx in victim_txs { - // > Add the txid and the current time to RecentlyEvicted, dropping the oldest entry in - // > RecentlyEvicted if necessary to keep it to at most `eviction_memory_entries entries`. - self.reject( - victim_tx.transaction.id, - SameEffectsChainRejectionError::RandomlyEvicted.into(), - ); - - // If this transaction gets evicted, set its result to the same error - if victim_tx.transaction.id == unmined_tx_id { - result = Err(SameEffectsChainRejectionError::RandomlyEvicted.into()); - } + // If this transaction gets evicted, set its result to the same error + if victim_tx.transaction.id == unmined_tx_id { + result = Err(SameEffectsChainRejectionError::RandomlyEvicted.into()); } } diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 7d73bd043a8..bf6a17925fd 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -178,7 +178,8 @@ impl VerifiedSet { Ok(()) } - /// Evict one transaction from the set, returns the victim transaction. + /// Evict one transaction and any transactions that directly or indirectly depend on + /// its outputs from the set, returns the victim transaction and any dependent transactions. /// /// Removes a transaction with probability in direct proportion to the /// eviction weight, as per [ZIP-401]. @@ -196,30 +197,31 @@ impl VerifiedSet { /// to 20,000 (mempooltxcostlimit/min(cost)), so the actual cost shouldn't /// be too bad. /// + /// This function is equivalent to `EvictTransaction` in [ZIP-401]. + /// /// [ZIP-401]: https://zips.z.cash/zip-0401 #[allow(clippy::unwrap_in_result)] - pub fn evict_one(&mut self) -> Vec { - if self.transactions.is_empty() { - Vec::new() - } else { - use rand::distributions::{Distribution, WeightedIndex}; - use rand::prelude::thread_rng; - - let (keys, weights): (Vec, Vec) = self - .transactions - .iter() - .map(|(&tx_id, tx)| (tx_id, tx.eviction_weight())) - .unzip(); - - let dist = WeightedIndex::new(weights) - .expect("there is at least one weight, all weights are non-negative, and the total is positive"); - - let key_to_remove = keys - .get(dist.sample(&mut thread_rng())) - .expect("should have a key at every index in the distribution"); - - self.remove(key_to_remove) - } + pub fn evict_one(&mut self) -> Option { + use rand::distributions::{Distribution, WeightedIndex}; + use rand::prelude::thread_rng; + + let (keys, weights): (Vec, Vec) = self + .transactions + .iter() + .map(|(&tx_id, tx)| (tx_id, tx.eviction_weight())) + .unzip(); + + let dist = WeightedIndex::new(weights).expect( + "there is at least one weight, all weights are non-negative, and the total is positive", + ); + + let key_to_remove = keys + .get(dist.sample(&mut thread_rng())) + .expect("should have a key at every index in the distribution"); + + // Removes the randomly selected transaction and all of its dependents from the set, + // then returns just the randomly selected transaction + self.remove(key_to_remove).pop() } /// Clears a list of mined transaction ids from the lists of dependencies for @@ -248,14 +250,21 @@ impl VerifiedSet { removed_count } - /// Removes a transaction from the set. + /// Accepts a transaction id for a transaction to remove from the verified set. + /// + /// Removes the transaction and any transactions that directly or indirectly + /// depend on it from the set. + /// + /// Returns a list of transactions that have been removed with the target transaction + /// as the last item. /// - /// Also removes its outputs from the internal caches. + /// Also removes the outputs of any removed transactions from the internal caches. fn remove(&mut self, key_to_remove: &transaction::Hash) -> Vec { let removed_transactions: Vec<_> = self .transaction_dependencies .remove_all(key_to_remove) .iter() + .chain(std::iter::once(key_to_remove)) .map(|key_to_remove| { let removed_tx = self .transactions From 874372a8bfeabf08ce9aa903e62dfff8b97a566d Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 8 Nov 2024 19:39:18 -0500 Subject: [PATCH 66/69] Applies suggestions from code review. --- zebrad/src/components/mempool/storage.rs | 19 +++++---- .../mempool/storage/verified_set.rs | 42 +++++++------------ 2 files changed, 24 insertions(+), 37 deletions(-) diff --git a/zebrad/src/components/mempool/storage.rs b/zebrad/src/components/mempool/storage.rs index 2459efc1149..ce6f09cf1d6 100644 --- a/zebrad/src/components/mempool/storage.rs +++ b/zebrad/src/components/mempool/storage.rs @@ -646,29 +646,30 @@ impl Storage { tip_height: zebra_chain::block::Height, ) -> HashSet { let mut tx_ids = HashSet::new(); - // we need a separate set, since reject() takes the original unmined ID, - // then extracts the mined ID out of it - let mut mined_tx_ids = HashSet::new(); - for (tx_id, tx) in self.transactions() { + for (&tx_id, tx) in self.transactions() { if let Some(expiry_height) = tx.transaction.transaction.expiry_height() { if tip_height >= expiry_height { - tx_ids.insert(tx.transaction.id); - mined_tx_ids.insert(*tx_id); + tx_ids.insert(tx_id); } } } // expiry height is effecting data, so we match by non-malleable TXID self.verified - .remove_all_that(|tx| tx_ids.contains(&tx.transaction.id)); + .remove_all_that(|tx| tx_ids.contains(&tx.transaction.id.mined_id())); // also reject it for &id in &tx_ids { - self.reject(id, SameEffectsChainRejectionError::Expired.into()); + self.reject( + // It's okay to omit the auth digest here as we know that `reject()` will always + // use mined ids for `SameEffectsChainRejectionError`s. + UnminedTxId::Legacy(id), + SameEffectsChainRejectionError::Expired.into(), + ); } - mined_tx_ids + tx_ids } /// Check if transaction should be downloaded and/or verified. diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index bf6a17925fd..27d5c90f4d9 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -8,7 +8,7 @@ use std::{ use zebra_chain::{ orchard, sapling, sprout, - transaction::{self, Transaction, UnminedTx, VerifiedUnminedTx}, + transaction::{self, UnminedTx, VerifiedUnminedTx}, transparent, }; use zebra_node_services::mempool::TransactionDependencies; @@ -41,10 +41,9 @@ pub struct VerifiedSet { /// spend or create outputs of other transactions in the mempool. transaction_dependencies: TransactionDependencies, - /// The [`transparent::Utxo`]s created by verified transactions in the mempool + /// The [`transparent::Output`]s created by verified transactions in the mempool. /// - /// Note that these UTXOs may not be unspent. - /// Outputs can be spent by later transactions or blocks in the chain. + /// These outputs may be spent by other transactions in the mempool. created_outputs: HashMap, /// The total size of the transactions in the mempool if they were @@ -163,11 +162,17 @@ impl VerifiedSet { self.transaction_dependencies .add(tx_id, spent_mempool_outpoints); - self.cache_outputs_and_respond_to_pending_output_requests_from( - tx_id, - &transaction.transaction.transaction, - pending_outputs, - ); + let tx = &transaction.transaction.transaction; + for (index, output) in tx.outputs().iter().cloned().enumerate() { + let outpoint = transparent::OutPoint::from_usize(tx_id, index); + self.created_outputs.insert(outpoint, output.clone()); + pending_outputs.respond(&outpoint, output) + } + + self.spent_outpoints.extend(tx.spent_outpoints()); + self.sprout_nullifiers.extend(tx.sprout_nullifiers()); + self.sapling_nullifiers.extend(tx.sapling_nullifiers()); + self.orchard_nullifiers.extend(tx.orchard_nullifiers()); self.transactions_serialized_size += transaction.transaction.size; self.total_cost += transaction.cost(); @@ -297,25 +302,6 @@ impl VerifiedSet { || Self::has_conflicts(&self.orchard_nullifiers, tx.orchard_nullifiers().copied()) } - /// Inserts the transaction's outputs into the internal caches and responds to pending output requests. - fn cache_outputs_and_respond_to_pending_output_requests_from( - &mut self, - tx_hash: transaction::Hash, - tx: &Transaction, - pending_outputs: &mut PendingOutputs, - ) { - for (index, output) in tx.outputs().iter().cloned().enumerate() { - let outpoint = transparent::OutPoint::from_usize(tx_hash, index); - self.created_outputs.insert(outpoint, output.clone()); - pending_outputs.respond(&outpoint, output) - } - - self.spent_outpoints.extend(tx.spent_outpoints()); - self.sprout_nullifiers.extend(tx.sprout_nullifiers()); - self.sapling_nullifiers.extend(tx.sapling_nullifiers()); - self.orchard_nullifiers.extend(tx.orchard_nullifiers()); - } - /// Removes the tracked transaction outputs from the mempool. fn remove_outputs(&mut self, unmined_tx: &UnminedTx) { let tx = &unmined_tx.transaction; From 5829ef3a33305babefae159f84723f3354432726 Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 8 Nov 2024 19:46:47 -0500 Subject: [PATCH 67/69] Adds minor comments --- zebrad/src/components/mempool/storage/verified_set.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 27d5c90f4d9..54790477a49 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -162,13 +162,13 @@ impl VerifiedSet { self.transaction_dependencies .add(tx_id, spent_mempool_outpoints); + // Inserts the transaction's outputs into the internal caches and responds to pending output requests. let tx = &transaction.transaction.transaction; for (index, output) in tx.outputs().iter().cloned().enumerate() { let outpoint = transparent::OutPoint::from_usize(tx_id, index); self.created_outputs.insert(outpoint, output.clone()); pending_outputs.respond(&outpoint, output) } - self.spent_outpoints.extend(tx.spent_outpoints()); self.sprout_nullifiers.extend(tx.sprout_nullifiers()); self.sapling_nullifiers.extend(tx.sapling_nullifiers()); From 813eeca272ec2dc296575e93798ae475946f27aa Mon Sep 17 00:00:00 2001 From: Arya Date: Mon, 11 Nov 2024 15:23:10 -0500 Subject: [PATCH 68/69] Update zebrad/src/components/mempool/storage/verified_set.rs Co-authored-by: Marek --- zebrad/src/components/mempool/storage/verified_set.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 54790477a49..7cd82fb0be4 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -27,7 +27,7 @@ use zebra_chain::transaction::MEMPOOL_TRANSACTION_COST_THRESHOLD; /// outputs include: /// /// - the dependencies of transactions that spent the outputs of other transactions in the mempool -/// - the UTXOs of transactions in the mempool +/// - the outputs of transactions in the mempool /// - the transparent outpoints spent by transactions in the mempool /// - the Sprout nullifiers revealed by transactions in the mempool /// - the Sapling nullifiers revealed by transactions in the mempool From e5fbe48711a6f216eabfb48590b5a70b272844ce Mon Sep 17 00:00:00 2001 From: Marek Date: Tue, 12 Nov 2024 16:21:07 +0100 Subject: [PATCH 69/69] Remove an outdated comment (#9013) --- zebra-consensus/src/transaction.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 407618b2daa..aac77a055d6 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -644,8 +644,6 @@ where for input in inputs { if let transparent::Input::PrevOut { outpoint, .. } = input { tracing::trace!("awaiting outpoint lookup"); - // Currently, Zebra only supports known UTXOs in block transactions. - // But it might support them in the mempool in future. let utxo = if let Some(output) = known_utxos.get(outpoint) { tracing::trace!("UXTO in known_utxos, discarding query"); output.utxo.clone()