From 09a38c5267354f9f3a35d7515f47db261c8afe7d Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 29 Sep 2022 18:13:02 +1000 Subject: [PATCH 01/20] Add RPC timing to zcash-rpc-diff --- zebra-utils/zcash-rpc-diff | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zebra-utils/zcash-rpc-diff b/zebra-utils/zcash-rpc-diff index 885c773ee69..f134f0a43e1 100755 --- a/zebra-utils/zcash-rpc-diff +++ b/zebra-utils/zcash-rpc-diff @@ -89,10 +89,12 @@ echo "$@" echo echo "Querying $ZEBRAD $ZEBRAD_NET chain at height >=$ZEBRAD_HEIGHT..." -$ZCASH_CLI -rpcport="$ZEBRAD_RPC_PORT" "$@" > "$ZEBRAD_RESPONSE" +time $ZCASH_CLI -rpcport="$ZEBRAD_RPC_PORT" "$@" > "$ZEBRAD_RESPONSE" +echo echo "Querying $ZCASHD $ZCASHD_NET chain at height >=$ZCASHD_HEIGHT..." -$ZCASH_CLI "$@" > "$ZCASHD_RESPONSE" +time $ZCASH_CLI "$@" > "$ZCASHD_RESPONSE" +echo echo From 9f55229ef2dd871d418335621893b648abb82d38 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 29 Sep 2022 19:49:46 +1000 Subject: [PATCH 02/20] Use transaction hash index for verbose block requests, rather than block data --- zebra-rpc/src/methods.rs | 81 ++++++++++++------- zebra-state/src/request.rs | 13 +++ zebra-state/src/response.rs | 8 +- zebra-state/src/service.rs | 35 +++++++- .../service/finalized_state/zebra_db/block.rs | 33 ++++++++ .../src/service/non_finalized_state/chain.rs | 15 ++++ zebra-state/src/service/read.rs | 2 +- zebra-state/src/service/read/block.rs | 25 ++++++ 8 files changed, 178 insertions(+), 34 deletions(-) diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index 4f9e4bfb40e..79f2d34f66e 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -540,46 +540,65 @@ where let mut state = self.state.clone(); async move { - let height = height.parse().map_err(|error: SerializationError| Error { + let height: Height = height.parse().map_err(|error: SerializationError| Error { code: ErrorCode::ServerError(0), message: error.to_string(), data: None, })?; - let request = - zebra_state::ReadRequest::Block(zebra_state::HashOrHeight::Height(height)); - let response = state - .ready() - .and_then(|service| service.call(request)) - .await - .map_err(|error| Error { - code: ErrorCode::ServerError(0), - message: error.to_string(), - data: None, - })?; + if verbosity == 0 { + let request = zebra_state::ReadRequest::Block(height.into()); + let response = state + .ready() + .and_then(|service| service.call(request)) + .await + .map_err(|error| Error { + code: ErrorCode::ServerError(0), + message: error.to_string(), + data: None, + })?; - match response { - zebra_state::ReadResponse::Block(Some(block)) => match verbosity { - 0 => Ok(GetBlock::Raw(block.into())), - 1 => Ok(GetBlock::Object { - tx: block - .transactions - .iter() - .map(|tx| tx.hash().encode_hex()) - .collect(), + match response { + zebra_state::ReadResponse::Block(Some(block)) => { + Ok(GetBlock::Raw(block.into())) + } + zebra_state::ReadResponse::Block(None) => Err(Error { + code: MISSING_BLOCK_ERROR_CODE, + message: "Block not found".to_string(), + data: None, }), - _ => Err(Error { - code: ErrorCode::InvalidParams, - message: "Invalid verbosity value".to_string(), + _ => unreachable!("unmatched response to a block request"), + } + } else if verbosity == 1 { + let request = zebra_state::ReadRequest::TransactionIdsForBlock(height.into()); + let response = state + .ready() + .and_then(|service| service.call(request)) + .await + .map_err(|error| Error { + code: ErrorCode::ServerError(0), + message: error.to_string(), + data: None, + })?; + + match response { + zebra_state::ReadResponse::TransactionIdsForBlock(Some(tx_ids)) => { + let tx_ids = tx_ids.iter().map(|tx_id| tx_id.encode_hex()).collect(); + Ok(GetBlock::Object { tx: tx_ids }) + } + zebra_state::ReadResponse::TransactionIdsForBlock(None) => Err(Error { + code: MISSING_BLOCK_ERROR_CODE, + message: "Block not found".to_string(), data: None, }), - }, - zebra_state::ReadResponse::Block(None) => Err(Error { - code: MISSING_BLOCK_ERROR_CODE, - message: "Block not found".to_string(), + _ => unreachable!("unmatched response to a transaction_ids_for_block request"), + } + } else { + Err(Error { + code: ErrorCode::InvalidParams, + message: "Invalid verbosity value".to_string(), data: None, - }), - _ => unreachable!("unmatched response to a block request"), + }) } } .boxed() @@ -1111,7 +1130,7 @@ pub enum GetBlock { Raw(#[serde(with = "hex")] SerializedBlock), /// The block object. Object { - /// Vector of hex-encoded TXIDs of the transactions of the block + /// List of transaction IDs in block order, hex-encoded. tx: Vec, }, } diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index 044c6a9a9c0..bca3f01409b 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -597,6 +597,18 @@ pub enum ReadRequest { /// * [`ReadResponse::Transaction(None)`](ReadResponse::Transaction) otherwise. Transaction(transaction::Hash), + /// Looks up the transaction IDs for a block, using a block hash or height. + /// + /// Returns + /// + /// * An ordered list of transaction hashes, or + /// * `None` if the block was not found. + /// + /// Note: Each block has at least one transaction: the coinbase transaction. + /// + /// Returned txids are in the order they appear in the block. + TransactionIdsForBlock(HashOrHeight), + /// Looks up a UTXO identified by the given [`OutPoint`](transparent::OutPoint), /// returning `None` immediately if it is unknown. /// @@ -728,6 +740,7 @@ impl ReadRequest { ReadRequest::Depth(_) => "depth", ReadRequest::Block(_) => "block", ReadRequest::Transaction(_) => "transaction", + ReadRequest::TransactionIdsForBlock(_) => "transaction_ids_for_block", ReadRequest::BestChainUtxo { .. } => "best_chain_utxo", ReadRequest::AnyChainUtxo { .. } => "any_chain_utxo", ReadRequest::BlockLocator => "block_locator", diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs index ca66938b20f..7ca08b3fd60 100644 --- a/zebra-state/src/response.rs +++ b/zebra-state/src/response.rs @@ -67,6 +67,11 @@ pub enum ReadResponse { /// Response to [`ReadRequest::Transaction`] with the specified transaction. Transaction(Option<(Arc, block::Height)>), + /// Response to [`ReadRequest::TransactionIdsForBlock`], + /// with an list of transaction hashes in block order, + /// or `None` if the block was not found. + TransactionIdsForBlock(Option>), + /// Response to [`ReadRequest::BlockLocator`] with a block locator object. BlockLocator(Vec), @@ -130,7 +135,8 @@ impl TryFrom for Response { ReadResponse::BlockHashes(hashes) => Ok(Response::BlockHashes(hashes)), ReadResponse::BlockHeaders(headers) => Ok(Response::BlockHeaders(headers)), - ReadResponse::BestChainUtxo(_) + ReadResponse::TransactionIdsForBlock(_) + | ReadResponse::BestChainUtxo(_) | ReadResponse::SaplingTree(_) | ReadResponse::OrchardTree(_) | ReadResponse::AddressBalance(_) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index de881879171..b3ee4d392dd 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1173,7 +1173,7 @@ impl Service for ReadStateService { .boxed() } - // Used by get_block RPC and the StateService. + // Used by the get_block (raw) RPC and the StateService. ReadRequest::Block(hash_or_height) => { let timer = CodeTimer::start(); @@ -1227,6 +1227,39 @@ impl Service for ReadStateService { .boxed() } + // Used by the getblock (verbose) RPC. + ReadRequest::TransactionIdsForBlock(hash_or_height) => { + let timer = CodeTimer::start(); + + let state = self.clone(); + + let span = Span::current(); + tokio::task::spawn_blocking(move || { + span.in_scope(move || { + let transaction_ids = state.non_finalized_state_receiver.with_watch_data( + |non_finalized_state| { + read::transaction_hashes_for_block( + non_finalized_state.best_chain(), + &state.db, + hash_or_height, + ) + }, + ); + + // The work is done in the future. + timer.finish( + module_path!(), + line!(), + "ReadRequest::TransactionIdsForBlock", + ); + + Ok(ReadResponse::TransactionIdsForBlock(transaction_ids)) + }) + }) + .map(|join_result| join_result.expect("panic in ReadRequest::Block")) + .boxed() + } + // Currently unused. ReadRequest::BestChainUtxo(outpoint) => { let timer = CodeTimer::start(); diff --git a/zebra-state/src/service/finalized_state/zebra_db/block.rs b/zebra-state/src/service/finalized_state/zebra_db/block.rs index 6e092758ab5..cc707f51770 100644 --- a/zebra-state/src/service/finalized_state/zebra_db/block.rs +++ b/zebra-state/src/service/finalized_state/zebra_db/block.rs @@ -226,6 +226,39 @@ impl ZebraDb { .map(|tx| (tx, transaction_location.height)) } + /// Returns the [`transaction::Hash`]es in the block with `hash_or_height`, + /// if it exists in this chain. + /// + /// Hashes are returned in block order. + /// + /// Returns `None` if the block is not found. + #[allow(clippy::unwrap_in_result)] + pub fn transaction_hashes_for_block( + &self, + hash_or_height: HashOrHeight, + ) -> Option> { + // Block + let height = hash_or_height.height_or_else(|hash| self.height(hash))?; + + // Transaction hashes + let hash_by_tx_loc = self.db.cf_handle("hash_by_tx_loc").unwrap(); + + // Manually fetch the entire block's transaction hashes + let mut transaction_hashes = Vec::new(); + + for tx_index in 0..=Transaction::max_allocation() { + let tx_loc = TransactionLocation::from_u64(height, tx_index); + + if let Some(tx_hash) = self.db.zs_get(&hash_by_tx_loc, &tx_loc) { + transaction_hashes.push(tx_hash); + } else { + break; + } + } + + Some(transaction_hashes.into()) + } + // Write block methods /// Write `finalized` to the finalized state. diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs index 0d707b197e0..1c91f9c42e3 100644 --- a/zebra-state/src/service/non_finalized_state/chain.rs +++ b/zebra-state/src/service/non_finalized_state/chain.rs @@ -471,6 +471,21 @@ impl Chain { .get(tx_loc.index.as_usize()) } + /// Returns the [`transaction::Hash`]es in the block with `hash_or_height`, + /// if it exists in this chain. + /// + /// Hashes are returned in block order. + /// + /// Returns `None` if the block is not found. + pub fn transaction_hashes_for_block( + &self, + hash_or_height: HashOrHeight, + ) -> Option> { + let transaction_hashes = self.block(hash_or_height)?.transaction_hashes.clone(); + + Some(transaction_hashes) + } + /// Returns the [`block::Hash`] for `height`, if it exists in this chain. pub fn hash_by_height(&self, height: Height) -> Option { let hash = self.blocks.get(&height)?.hash; diff --git a/zebra-state/src/service/read.rs b/zebra-state/src/service/read.rs index e2a0c01695d..d1f7f4c0a81 100644 --- a/zebra-state/src/service/read.rs +++ b/zebra-state/src/service/read.rs @@ -27,7 +27,7 @@ pub use address::{ tx_id::transparent_tx_ids, utxo::{address_utxos, AddressUtxos, ADDRESS_HEIGHTS_FULL_RANGE}, }; -pub use block::{any_utxo, block, block_header, transaction, utxo}; +pub use block::{any_utxo, block, block_header, transaction, transaction_hashes_for_block, utxo}; pub use find::{ block_locator, chain_contains_hash, depth, find_chain_hashes, find_chain_headers, hash_by_height, height_by_hash, tip, tip_height, diff --git a/zebra-state/src/service/read/block.rs b/zebra-state/src/service/read/block.rs index 985c44eaf94..54eb485c367 100644 --- a/zebra-state/src/service/read/block.rs +++ b/zebra-state/src/service/read/block.rs @@ -93,6 +93,31 @@ where .or_else(|| db.transaction(hash)) } +/// Returns the [`transaction::Hash`]es for the block with `hash_or_height`, +/// if it exists in the non-finalized `chain` or finalized `db`. +/// +/// The returned hashes are in block order. +/// +/// Returns `None` if the block is not found. +pub fn transaction_hashes_for_block( + chain: Option, + db: &ZebraDb, + hash_or_height: HashOrHeight, +) -> Option> +where + C: AsRef, +{ + // # Correctness + // + // Since blocks are the same in the finalized and non-finalized state, we + // check the most efficient alternative first. (`chain` is always in memory, + // but `db` stores blocks on disk, with a memory cache.) + chain + .as_ref() + .and_then(|chain| chain.as_ref().transaction_hashes_for_block(hash_or_height)) + .or_else(|| db.transaction_hashes_for_block(hash_or_height)) +} + /// Returns the [`Utxo`] for [`transparent::OutPoint`], if it exists in the /// non-finalized `chain` or finalized `db`. /// From a588a6534fd0551ad5b27c2adc942a1f6d5294ef Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 21 Sep 2022 11:15:01 +1000 Subject: [PATCH 03/20] check if we are at tip for lightwallet wallet tests --- zebrad/tests/acceptance.rs | 108 ++++++++---------- zebrad/tests/common/launch.rs | 15 ++- .../lightwalletd/send_transaction_test.rs | 40 +++++-- .../tests/common/lightwalletd/wallet_grpc.rs | 53 ++++++++- .../common/lightwalletd/wallet_grpc_test.rs | 55 ++++++--- 5 files changed, 174 insertions(+), 97 deletions(-) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index cc99d535658..a86df1ba5f6 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -136,19 +136,16 @@ use common::{ config::{ config_file_full_path, configs_dir, default_test_config, persistent_test_config, testdir, }, - launch::{ - spawn_zebrad_for_rpc_without_initial_peers, ZebradTestDirExt, BETWEEN_NODES_DELAY, - LAUNCH_DELAY, - }, + launch::{spawn_zebrad_for_rpc, ZebradTestDirExt, BETWEEN_NODES_DELAY, LAUNCH_DELAY}, lightwalletd::{ random_known_rpc_port_config, zebra_skip_lightwalletd_tests, LightWalletdTestDirExt, LightwalletdTestType::{self, *}, }, sync::{ create_cached_database_height, sync_until, MempoolBehavior, LARGE_CHECKPOINT_TEST_HEIGHT, - LARGE_CHECKPOINT_TIMEOUT, LIGHTWALLETD_SYNC_FINISHED_REGEX, MEDIUM_CHECKPOINT_TEST_HEIGHT, - STOP_AT_HEIGHT_REGEX, STOP_ON_LOAD_TIMEOUT, SYNC_FINISHED_REGEX, - TINY_CHECKPOINT_TEST_HEIGHT, TINY_CHECKPOINT_TIMEOUT, + LARGE_CHECKPOINT_TIMEOUT, MEDIUM_CHECKPOINT_TEST_HEIGHT, STOP_AT_HEIGHT_REGEX, + STOP_ON_LOAD_TIMEOUT, SYNC_FINISHED_REGEX, TINY_CHECKPOINT_TEST_HEIGHT, + TINY_CHECKPOINT_TIMEOUT, }, }; @@ -1476,21 +1473,21 @@ async fn lightwalletd_test_suite() -> Result<()> { // Only runs when ZEBRA_CACHED_STATE_DIR is set. lightwalletd_integration_test(UpdateZebraCachedStateNoRpc)?; - // Only runs when ZEBRA_CACHED_STATE_DIR is set. - // When manually running the test suite, allow cached state in the full sync test. - lightwalletd_integration_test(FullSyncFromGenesis { - allow_lightwalletd_cached_state: true, - })?; - - // Only runs when LIGHTWALLETD_DATA_DIR and ZEBRA_CACHED_STATE_DIR are set - lightwalletd_integration_test(UpdateCachedState)?; - - // Only runs when LIGHTWALLETD_DATA_DIR and ZEBRA_CACHED_STATE_DIR are set, - // and the compile-time gRPC feature is on. + // These tests need the compile-time gRPC feature #[cfg(feature = "lightwalletd-grpc-tests")] { - common::lightwalletd::send_transaction_test::run().await?; + // Only runs when ZEBRA_CACHED_STATE_DIR is set. + // When manually running the test suite, allow cached state in the full sync test. + lightwalletd_integration_test(FullSyncFromGenesis { + allow_lightwalletd_cached_state: true, + })?; + + // Only runs when LIGHTWALLETD_DATA_DIR and ZEBRA_CACHED_STATE_DIR are set + lightwalletd_integration_test(UpdateCachedState)?; + + // Only runs when LIGHTWALLETD_DATA_DIR and ZEBRA_CACHED_STATE_DIR are set common::lightwalletd::wallet_grpc_test::run().await?; + common::lightwalletd::send_transaction_test::run().await?; } Ok(()) @@ -1498,9 +1495,19 @@ async fn lightwalletd_test_suite() -> Result<()> { /// Run a lightwalletd integration test with a configuration for `test_type`. /// +/// Tests that sync `lightwalletd` to the chain tip require the `lightwalletd-grpc-tests` feature`: +/// - [`FullSyncFromGenesis`] +/// - [`UpdateCachedState`] +/// /// Set `FullSyncFromGenesis { allow_lightwalletd_cached_state: true }` to speed up manual full sync tests. /// +/// # Relibility +/// /// The random ports in this test can cause [rare port conflicts.](#Note on port conflict) +/// +/// # Panics +/// +/// If the `test_type` is UpdateZebraCachedStateNoRpc #[tracing::instrument] fn lightwalletd_integration_test(test_type: LightwalletdTestType) -> Result<()> { let _init_guard = zebra_test::init(); @@ -1655,43 +1662,24 @@ fn lightwalletd_integration_test(test_type: LightwalletdTestType) -> Result<()> }; let (mut zebrad, lightwalletd) = if test_type.needs_zebra_cached_state() { - if let Some(mut lightwalletd) = lightwalletd { - // Wait for lightwalletd to sync to Zebra's tip. - // - // "Adding block" and "Waiting for block" logs stop when `lightwalletd` reaches the tip. - // But if the logs just stop, we can't tell the difference between a hang and fully synced. - // So we assume `lightwalletd` will sync and log large groups of blocks, - // and check for logs with heights near the mainnet tip height. - let lightwalletd_thread = std::thread::spawn(move || -> Result<_> { - tracing::info!(?test_type, "waiting for lightwalletd to sync to the tip"); - - lightwalletd.expect_stdout_line_matches(LIGHTWALLETD_SYNC_FINISHED_REGEX)?; - - Ok(lightwalletd) - }); - - // `lightwalletd` syncs can take a long time, - // so we need to check that `zebrad` has synced to the tip in parallel. - let lightwalletd_thread_and_zebrad = std::thread::spawn(move || -> Result<_> { - tracing::info!(?test_type, "waiting for zebrad to sync to the tip"); - - while !lightwalletd_thread.is_finished() { - zebrad.expect_stdout_line_matches(SYNC_FINISHED_REGEX)?; - } - - Ok((lightwalletd_thread, zebrad)) - }); - - // Retrieve the child process handles from the threads - let (lightwalletd_thread, zebrad) = lightwalletd_thread_and_zebrad - .join() - .unwrap_or_else(|panic_object| panic::resume_unwind(panic_object))?; - - let lightwalletd = lightwalletd_thread - .join() - .unwrap_or_else(|panic_object| panic::resume_unwind(panic_object))?; - - (zebrad, Some(lightwalletd)) + if let Some(lightwalletd) = lightwalletd { + #[cfg(feature = "lightwalletd-grpc-tests")] + { + let (zebrad, lightwalletd) = + common::lightwalletd::wallet_grpc::wait_for_zebrad_and_lightwalletd_tip( + lightwalletd, + zebrad, + test_type, + )?; + (zebrad, Some(lightwalletd)) + } + + #[cfg(not(feature = "lightwalletd-grpc-tests"))] + panic!( + "the {test_type:?} test requires `cargo test --feature lightwalletd-grpc-tests` \n\ + zebrad: {zebrad:?} \n\ + lightwalletd: {lightwalletd:?}" + ); } else { // We're just syncing Zebra, so there's no lightwalletd to check tracing::info!(?test_type, "waiting for zebrad to sync to the tip"); @@ -1995,12 +1983,8 @@ async fn fully_synced_rpc_test() -> Result<()> { let network = Network::Mainnet; - let (_zebrad, zebra_rpc_address) = spawn_zebrad_for_rpc_without_initial_peers( - network, - cached_state_path.unwrap(), - test_type, - true, - )?; + let (_zebrad, zebra_rpc_address) = + spawn_zebrad_for_rpc(network, cached_state_path.unwrap(), test_type, true, false)?; // Make a getblock test that works only on synced node (high block number). // The block is before the mandatory checkpoint, so the checkpoint cached state can be used diff --git a/zebrad/tests/common/launch.rs b/zebrad/tests/common/launch.rs index dcbe2e4df19..2ec75a44e63 100644 --- a/zebrad/tests/common/launch.rs +++ b/zebrad/tests/common/launch.rs @@ -195,26 +195,32 @@ where } } -/// Spawns a zebrad instance to interact with lightwalletd, but without an internet connection. +/// Spawns a zebrad instance to interact with lightwalletd. /// +/// If `internet_connection` is not true then spawn but without any peers. /// This prevents it from downloading blocks. Instead, the `zebra_directory` parameter allows /// providing an initial state to the zebrad instance. #[tracing::instrument] -pub fn spawn_zebrad_for_rpc_without_initial_peers( +pub fn spawn_zebrad_for_rpc( network: Network, zebra_directory: P, test_type: LightwalletdTestType, debug_skip_parameter_preload: bool, + internet_connection: bool, ) -> Result<(TestChild

, SocketAddr)> { // This is what we recommend our users configure. let mut config = random_known_rpc_port_config(true) .expect("Failed to create a config file with a known RPC listener port"); config.state.ephemeral = false; - config.network.initial_mainnet_peers = IndexSet::new(); + if !internet_connection { + config.network.initial_mainnet_peers = IndexSet::new(); + } config.network.initial_testnet_peers = IndexSet::new(); config.network.network = network; - config.mempool.debug_enable_at_height = Some(0); + if !internet_connection { + config.mempool.debug_enable_at_height = Some(0); + } config.consensus.debug_skip_parameter_preload = debug_skip_parameter_preload; let (zebrad_failure_messages, zebrad_ignore_messages) = test_type.zebrad_failure_messages(); @@ -228,7 +234,6 @@ pub fn spawn_zebrad_for_rpc_without_initial_peers Result<()> { "got transactions to send", ); - // TODO: change debug_skip_parameter_preload to true if we do the mempool test in the wallet gRPC test - let (mut zebrad, zebra_rpc_address) = spawn_zebrad_for_rpc_without_initial_peers( - Network::Mainnet, - zebrad_state_path, - test_type, - false, - )?; + // Start zebrad with no peers, we want to send transactions without blocks coming in. If `wallet_grpc_test` + // runs before this test (as it does in `lightwalletd_test_suite`), then we are the most up to date with tip we can. + let (mut zebrad, zebra_rpc_address) = + spawn_zebrad_for_rpc(network, zebrad_state_path, test_type, false, false)?; tracing::info!( ?zebra_rpc_address, @@ -136,10 +133,6 @@ pub async fn run() -> Result<()> { zebrad.expect_stdout_line_matches("activating mempool")?; - // TODO: check that lightwalletd is at the tip using gRPC (#4894) - // - // If this takes a long time, we might need to check zebrad logs for failures in a separate thread. - tracing::info!( ?lightwalletd_rpc_port, "connecting gRPC client to lightwalletd...", @@ -147,6 +140,29 @@ pub async fn run() -> Result<()> { let mut rpc_client = connect_to_lightwalletd(lightwalletd_rpc_port).await?; + // Check if zebrad and lightwalletd are both in the same height. + + // Get the block tip from lightwalletd + let block_tip = rpc_client + .get_latest_block(wallet_grpc::ChainSpec {}) + .await? + .into_inner(); + + // Get the block tip from zebrad + let client = reqwest::Client::new(); + let res = client + .post(format!("http://{}", &zebra_rpc_address.to_string())) + .body(r#"{"jsonrpc": "2.0", "method": "getblockchaininfo", "params": [], "id":123 }"#) + .header("Content-Type", "application/json") + .send() + .await? + .text() + .await?; + + // Make sure tip from lightwalletd is the same as zebrad + let parsed: serde_json::Value = serde_json::from_str(&res)?; + assert_eq!(block_tip.height, parsed["result"]["blocks"]); + // To avoid filling the mempool queue, limit the transactions to be sent to the RPC and mempool queue limits transactions.truncate(max_sent_transactions()); diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc.rs b/zebrad/tests/common/lightwalletd/wallet_grpc.rs index edfed7c5b3e..5226a838791 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc.rs @@ -7,7 +7,8 @@ use tempfile::TempDir; use zebra_test::{args, net::random_known_port, prelude::*}; use crate::common::{ - config::testdir, lightwalletd::LightWalletdTestDirExt, sync::LIGHTWALLETD_SYNC_FINISHED_REGEX, + config::testdir, launch::ZebradTestDirExt, lightwalletd::LightWalletdTestDirExt, + sync::LIGHTWALLETD_SYNC_FINISHED_REGEX, }; use super::LightwalletdTestType; @@ -65,3 +66,53 @@ pub async fn connect_to_lightwalletd(lightwalletd_rpc_port: u16) -> Result( + mut lightwalletd: TestChild, + mut zebrad: TestChild

, + test_type: LightwalletdTestType, +) -> Result<(TestChild, TestChild

)> { + let lightwalletd_thread = std::thread::spawn(move || -> Result<_> { + tracing::info!(?test_type, "waiting for lightwalletd to sync to the tip"); + + lightwalletd + .expect_stdout_line_matches(crate::common::sync::LIGHTWALLETD_SYNC_FINISHED_REGEX)?; + + Ok(lightwalletd) + }); + + // `lightwalletd` syncs can take a long time, + // so we need to check that `zebrad` has synced to the tip in parallel. + let lightwalletd_thread_and_zebrad = std::thread::spawn(move || -> Result<_> { + tracing::info!(?test_type, "waiting for zebrad to sync to the tip"); + + while !lightwalletd_thread.is_finished() { + zebrad.expect_stdout_line_matches(crate::common::sync::SYNC_FINISHED_REGEX)?; + } + + Ok((lightwalletd_thread, zebrad)) + }); + + // Retrieve the child process handles from the threads + let (lightwalletd_thread, mut zebrad) = lightwalletd_thread_and_zebrad + .join() + .unwrap_or_else(|panic_object| std::panic::resume_unwind(panic_object))?; + + let lightwalletd = lightwalletd_thread + .join() + .unwrap_or_else(|panic_object| std::panic::resume_unwind(panic_object))?; + + // If we are in sync mempool should activate + zebrad.expect_stdout_line_matches("activating mempool")?; + + Ok((lightwalletd, zebrad)) +} diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs index 8b4bade02e4..9e55e4083ac 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs @@ -46,12 +46,12 @@ use zebra_chain::{ use zebra_network::constants::USER_AGENT; use crate::common::{ - launch::spawn_zebrad_for_rpc_without_initial_peers, + launch::spawn_zebrad_for_rpc, lightwalletd::{ wallet_grpc::{ - connect_to_lightwalletd, spawn_lightwalletd_with_rpc_server, Address, AddressList, - BlockId, BlockRange, ChainSpec, Empty, GetAddressUtxosArg, - TransparentAddressBlockFilter, TxFilter, + connect_to_lightwalletd, spawn_lightwalletd_with_rpc_server, + wait_for_zebrad_and_lightwalletd_tip, Address, AddressList, BlockId, BlockRange, + ChainSpec, Empty, GetAddressUtxosArg, TransparentAddressBlockFilter, TxFilter, }, zebra_skip_lightwalletd_tests, LightwalletdTestType::UpdateCachedState, @@ -100,14 +100,14 @@ pub async fn run() -> Result<()> { launching disconnected zebrad...", ); - // Launch zebra using a predefined zebrad state path - // - // TODO: change debug_skip_parameter_preload to true if we do the mempool test in the send transaction test - let (mut zebrad, zebra_rpc_address) = spawn_zebrad_for_rpc_without_initial_peers( + // Launch zebra with peers and using a predefined zebrad state path. + // As this tests are just queries we can have a live chain where blocks are coming. + let (zebrad, zebra_rpc_address) = spawn_zebrad_for_rpc( network, - zebrad_state_path.unwrap(), + zebrad_state_path.clone().unwrap(), test_type, - false, + true, + true, )?; tracing::info!( @@ -116,9 +116,9 @@ pub async fn run() -> Result<()> { ); // Launch lightwalletd - let (_lightwalletd, lightwalletd_rpc_port) = spawn_lightwalletd_with_rpc_server( + let (lightwalletd, lightwalletd_rpc_port) = spawn_lightwalletd_with_rpc_server( zebra_rpc_address, - lightwalletd_state_path, + lightwalletd_state_path.clone(), test_type, false, )?; @@ -128,13 +128,11 @@ pub async fn run() -> Result<()> { "spawned lightwalletd connected to zebrad, waiting for zebrad mempool activation...", ); - zebrad.expect_stdout_line_matches("activating mempool")?; + // Make sure we are in sync + let (_zebrad, _lightwalletd) = + wait_for_zebrad_and_lightwalletd_tip(lightwalletd, zebrad, test_type)?; // Give lightwalletd a few seconds to sync to the tip before connecting to it - // - // TODO: check that lightwalletd is at the tip using gRPC (#4894) - // - // If this takes a long time, we might need to check zebrad logs for failures in a separate thread. tokio::time::sleep(std::time::Duration::from_secs(60)).await; tracing::info!( @@ -145,6 +143,29 @@ pub async fn run() -> Result<()> { // Connect to the lightwalletd instance let mut rpc_client = connect_to_lightwalletd(lightwalletd_rpc_port).await?; + // Check if zebrad and lightwalletd are both in the same height. + + // Get the block tip from lightwalletd + let block_tip = rpc_client + .get_latest_block(ChainSpec {}) + .await? + .into_inner(); + + // Get the block tip from zebrad + let client = reqwest::Client::new(); + let res = client + .post(format!("http://{}", &zebra_rpc_address.to_string())) + .body(r#"{"jsonrpc": "2.0", "method": "getblockchaininfo", "params": [], "id":123 }"#) + .header("Content-Type", "application/json") + .send() + .await? + .text() + .await?; + + // Make sure tip from lightwalletd and from zebrad are the same + let parsed: serde_json::Value = serde_json::from_str(&res)?; + assert_eq!(block_tip.height, parsed["result"]["blocks"]); + // End of the setup and start the tests tracing::info!(?lightwalletd_rpc_port, "sending gRPC queries..."); From b50e3c310d8abebd1ff1cc732ecb1704838ded1c Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 14 Sep 2022 19:36:20 -0300 Subject: [PATCH 04/20] move function --- zebrad/tests/acceptance.rs | 5 +- zebrad/tests/common/launch.rs | 51 ++++++++++++++++++ .../tests/common/lightwalletd/wallet_grpc.rs | 53 +------------------ .../common/lightwalletd/wallet_grpc_test.rs | 8 +-- 4 files changed, 60 insertions(+), 57 deletions(-) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index a86df1ba5f6..b80e29d14f8 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -136,7 +136,10 @@ use common::{ config::{ config_file_full_path, configs_dir, default_test_config, persistent_test_config, testdir, }, - launch::{spawn_zebrad_for_rpc, ZebradTestDirExt, BETWEEN_NODES_DELAY, LAUNCH_DELAY}, + launch::{ + spawn_zebrad_for_rpc, wait_for_zebrad_and_lightwalletd_tip, ZebradTestDirExt, + BETWEEN_NODES_DELAY, LAUNCH_DELAY, + }, lightwalletd::{ random_known_rpc_port_config, zebra_skip_lightwalletd_tests, LightWalletdTestDirExt, LightwalletdTestType::{self, *}, diff --git a/zebrad/tests/common/launch.rs b/zebrad/tests/common/launch.rs index 2ec75a44e63..081dc21b2c8 100644 --- a/zebrad/tests/common/launch.rs +++ b/zebrad/tests/common/launch.rs @@ -14,6 +14,7 @@ use std::{ use color_eyre::eyre::Result; use indexmap::IndexSet; +use tempfile::TempDir; use zebra_chain::parameters::Network; use zebra_test::{ @@ -239,6 +240,56 @@ pub fn spawn_zebrad_for_rpc( Ok((zebrad, rpc_address)) } +/// Wait for lightwalletd to sync to Zebra's tip. +/// +/// "Adding block" and "Waiting for block" logs stop when `lightwalletd` reaches the tip. +/// But if the logs just stop, we can't tell the difference between a hang and fully synced. +/// So we assume `lightwalletd` will sync and log large groups of blocks, +/// and check for logs with heights near the mainnet tip height. +#[tracing::instrument] +pub fn wait_for_zebrad_and_lightwalletd_tip< + P: ZebradTestDirExt + std::fmt::Debug + std::marker::Send + 'static, +>( + mut lightwalletd: TestChild, + mut zebrad: TestChild

, + test_type: LightwalletdTestType, +) -> Result<(TestChild, TestChild

)> { + let lightwalletd_thread = std::thread::spawn(move || -> Result<_> { + tracing::info!(?test_type, "waiting for lightwalletd to sync to the tip"); + + lightwalletd + .expect_stdout_line_matches(crate::common::sync::LIGHTWALLETD_SYNC_FINISHED_REGEX)?; + + Ok(lightwalletd) + }); + + // `lightwalletd` syncs can take a long time, + // so we need to check that `zebrad` has synced to the tip in parallel. + let lightwalletd_thread_and_zebrad = std::thread::spawn(move || -> Result<_> { + tracing::info!(?test_type, "waiting for zebrad to sync to the tip"); + + while !lightwalletd_thread.is_finished() { + zebrad.expect_stdout_line_matches(crate::common::sync::SYNC_FINISHED_REGEX)?; + } + + Ok((lightwalletd_thread, zebrad)) + }); + + // Retrieve the child process handles from the threads + let (lightwalletd_thread, mut zebrad) = lightwalletd_thread_and_zebrad + .join() + .unwrap_or_else(|panic_object| std::panic::resume_unwind(panic_object))?; + + let lightwalletd = lightwalletd_thread + .join() + .unwrap_or_else(|panic_object| std::panic::resume_unwind(panic_object))?; + + // If we are in sync mempool should activate + zebrad.expect_stdout_line_matches("activating mempool")?; + + Ok((lightwalletd, zebrad)) +} + /// Panics if `$pred` is false, with an error report containing: /// * context from `$source`, and /// * an optional wrapper error, using `$fmt_arg`+ as a format string and diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc.rs b/zebrad/tests/common/lightwalletd/wallet_grpc.rs index 5226a838791..edfed7c5b3e 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc.rs @@ -7,8 +7,7 @@ use tempfile::TempDir; use zebra_test::{args, net::random_known_port, prelude::*}; use crate::common::{ - config::testdir, launch::ZebradTestDirExt, lightwalletd::LightWalletdTestDirExt, - sync::LIGHTWALLETD_SYNC_FINISHED_REGEX, + config::testdir, lightwalletd::LightWalletdTestDirExt, sync::LIGHTWALLETD_SYNC_FINISHED_REGEX, }; use super::LightwalletdTestType; @@ -66,53 +65,3 @@ pub async fn connect_to_lightwalletd(lightwalletd_rpc_port: u16) -> Result( - mut lightwalletd: TestChild, - mut zebrad: TestChild

, - test_type: LightwalletdTestType, -) -> Result<(TestChild, TestChild

)> { - let lightwalletd_thread = std::thread::spawn(move || -> Result<_> { - tracing::info!(?test_type, "waiting for lightwalletd to sync to the tip"); - - lightwalletd - .expect_stdout_line_matches(crate::common::sync::LIGHTWALLETD_SYNC_FINISHED_REGEX)?; - - Ok(lightwalletd) - }); - - // `lightwalletd` syncs can take a long time, - // so we need to check that `zebrad` has synced to the tip in parallel. - let lightwalletd_thread_and_zebrad = std::thread::spawn(move || -> Result<_> { - tracing::info!(?test_type, "waiting for zebrad to sync to the tip"); - - while !lightwalletd_thread.is_finished() { - zebrad.expect_stdout_line_matches(crate::common::sync::SYNC_FINISHED_REGEX)?; - } - - Ok((lightwalletd_thread, zebrad)) - }); - - // Retrieve the child process handles from the threads - let (lightwalletd_thread, mut zebrad) = lightwalletd_thread_and_zebrad - .join() - .unwrap_or_else(|panic_object| std::panic::resume_unwind(panic_object))?; - - let lightwalletd = lightwalletd_thread - .join() - .unwrap_or_else(|panic_object| std::panic::resume_unwind(panic_object))?; - - // If we are in sync mempool should activate - zebrad.expect_stdout_line_matches("activating mempool")?; - - Ok((lightwalletd, zebrad)) -} diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs index 9e55e4083ac..f41beb57330 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs @@ -46,12 +46,12 @@ use zebra_chain::{ use zebra_network::constants::USER_AGENT; use crate::common::{ - launch::spawn_zebrad_for_rpc, + launch::{spawn_zebrad_for_rpc, wait_for_zebrad_and_lightwalletd_tip}, lightwalletd::{ wallet_grpc::{ - connect_to_lightwalletd, spawn_lightwalletd_with_rpc_server, - wait_for_zebrad_and_lightwalletd_tip, Address, AddressList, BlockId, BlockRange, - ChainSpec, Empty, GetAddressUtxosArg, TransparentAddressBlockFilter, TxFilter, + connect_to_lightwalletd, spawn_lightwalletd_with_rpc_server, Address, AddressList, + BlockId, BlockRange, ChainSpec, Empty, GetAddressUtxosArg, + TransparentAddressBlockFilter, TxFilter, }, zebra_skip_lightwalletd_tests, LightwalletdTestType::UpdateCachedState, From e4d3ed76c2b88c4ac2c791377d2649d060472983 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 15 Sep 2022 17:38:42 -0400 Subject: [PATCH 05/20] Apply suggestions from code review Co-authored-by: teor --- zebrad/tests/common/launch.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/zebrad/tests/common/launch.rs b/zebrad/tests/common/launch.rs index 081dc21b2c8..c6e0ca2a635 100644 --- a/zebrad/tests/common/launch.rs +++ b/zebrad/tests/common/launch.rs @@ -198,7 +198,7 @@ where /// Spawns a zebrad instance to interact with lightwalletd. /// -/// If `internet_connection` is not true then spawn but without any peers. +/// If `internet_connection` is `false` then spawn, but without any peers. /// This prevents it from downloading blocks. Instead, the `zebra_directory` parameter allows /// providing an initial state to the zebrad instance. #[tracing::instrument] @@ -216,8 +216,8 @@ pub fn spawn_zebrad_for_rpc( config.state.ephemeral = false; if !internet_connection { config.network.initial_mainnet_peers = IndexSet::new(); + config.network.initial_testnet_peers = IndexSet::new(); } - config.network.initial_testnet_peers = IndexSet::new(); config.network.network = network; if !internet_connection { config.mempool.debug_enable_at_height = Some(0); @@ -247,12 +247,13 @@ pub fn spawn_zebrad_for_rpc( /// So we assume `lightwalletd` will sync and log large groups of blocks, /// and check for logs with heights near the mainnet tip height. #[tracing::instrument] -pub fn wait_for_zebrad_and_lightwalletd_tip< +pub fn wait_for_zebrad_and_lightwalletd_sync< P: ZebradTestDirExt + std::fmt::Debug + std::marker::Send + 'static, >( mut lightwalletd: TestChild, mut zebrad: TestChild

, test_type: LightwalletdTestType, + wait_for_zebrad_tip: bool, ) -> Result<(TestChild, TestChild

)> { let lightwalletd_thread = std::thread::spawn(move || -> Result<_> { tracing::info!(?test_type, "waiting for lightwalletd to sync to the tip"); From e04e2886515a273a247354b18914a565f6db880f Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 21 Sep 2022 18:20:20 +1000 Subject: [PATCH 06/20] Combine the lightwalletd sync and gRPC test APIs --- zebrad/tests/common/launch.rs | 119 ++++------ zebrad/tests/common/lightwalletd.rs | 100 +++++++- .../tests/common/lightwalletd/wallet_grpc.rs | 213 ++++++++++++++---- zebrad/tests/common/sync.rs | 8 - 4 files changed, 314 insertions(+), 126 deletions(-) diff --git a/zebrad/tests/common/launch.rs b/zebrad/tests/common/launch.rs index c6e0ca2a635..17f564f01f8 100644 --- a/zebrad/tests/common/launch.rs +++ b/zebrad/tests/common/launch.rs @@ -25,7 +25,8 @@ use zebra_test::{ use zebrad::config::ZebradConfig; use crate::common::{ - lightwalletd::{random_known_rpc_port_config, LightwalletdTestType}, + config::testdir, + lightwalletd::{zebra_skip_lightwalletd_tests, LightwalletdTestType}, sync::FINISH_PARTIAL_SYNC_TIMEOUT, }; @@ -196,37 +197,49 @@ where } } -/// Spawns a zebrad instance to interact with lightwalletd. +/// Spawns a zebrad instance on `network` to test lightwalletd with `test_type`. /// -/// If `internet_connection` is `false` then spawn, but without any peers. -/// This prevents it from downloading blocks. Instead, the `zebra_directory` parameter allows -/// providing an initial state to the zebrad instance. +/// If `use_internet_connection` is `false` then spawn, but without any peers. +/// This prevents it from downloading blocks. Instead, use the `ZEBRA_CACHED_STATE_DIR` +/// environmental variable to provide an initial state to the zebrad instance. +/// +/// Returns: +/// - `Ok(Some(zebrad, zebra_rpc_address))` on success, +/// - `Ok(None)` if the test doesn't have the required network or cached state, and +/// - `Err(_)` if spawning zebrad fails. #[tracing::instrument] -pub fn spawn_zebrad_for_rpc( +pub fn spawn_zebrad_for_rpc + std::fmt::Debug>( network: Network, - zebra_directory: P, + test_name: S, test_type: LightwalletdTestType, - debug_skip_parameter_preload: bool, - internet_connection: bool, -) -> Result<(TestChild

, SocketAddr)> { - // This is what we recommend our users configure. - let mut config = random_known_rpc_port_config(true) - .expect("Failed to create a config file with a known RPC listener port"); - - config.state.ephemeral = false; - if !internet_connection { - config.network.initial_mainnet_peers = IndexSet::new(); - config.network.initial_testnet_peers = IndexSet::new(); + use_internet_connection: bool, +) -> Result, SocketAddr)>> { + let test_name = test_name.as_ref(); + + // Skip the test unless the user specifically asked for it + if !can_spawn_zebrad_for_rpc(test_name, test_type) { + return Ok(None); } + + // Get the zebrad config + let mut config = test_type + .zebrad_config(test_name) + .expect("already checked config")?; + + // TODO: move this into zebrad_config() config.network.network = network; - if !internet_connection { + if !use_internet_connection { + config.network.initial_mainnet_peers = IndexSet::new(); + config.network.initial_testnet_peers = IndexSet::new(); + config.mempool.debug_enable_at_height = Some(0); } - config.consensus.debug_skip_parameter_preload = debug_skip_parameter_preload; let (zebrad_failure_messages, zebrad_ignore_messages) = test_type.zebrad_failure_messages(); - let mut zebrad = zebra_directory + // Writes a configuration that has RPC listen_addr set (if needed). + // If the state path env var is set, uses it in the config. + let zebrad = testdir()? .with_config(&mut config)? .spawn_child(args!["start"])? .bypass_test_capture(true) @@ -235,60 +248,28 @@ pub fn spawn_zebrad_for_rpc( let rpc_address = config.rpc.listen_addr.unwrap(); - zebrad.expect_stdout_line_matches(&format!("Opened RPC endpoint at {}", rpc_address))?; - - Ok((zebrad, rpc_address)) + Ok(Some((zebrad, rpc_address))) } -/// Wait for lightwalletd to sync to Zebra's tip. -/// -/// "Adding block" and "Waiting for block" logs stop when `lightwalletd` reaches the tip. -/// But if the logs just stop, we can't tell the difference between a hang and fully synced. -/// So we assume `lightwalletd` will sync and log large groups of blocks, -/// and check for logs with heights near the mainnet tip height. +/// Returns `true` if a zebrad test for `test_type` has everything it needs to run. #[tracing::instrument] -pub fn wait_for_zebrad_and_lightwalletd_sync< - P: ZebradTestDirExt + std::fmt::Debug + std::marker::Send + 'static, ->( - mut lightwalletd: TestChild, - mut zebrad: TestChild

, +pub fn can_spawn_zebrad_for_rpc + std::fmt::Debug>( + test_name: S, test_type: LightwalletdTestType, - wait_for_zebrad_tip: bool, -) -> Result<(TestChild, TestChild

)> { - let lightwalletd_thread = std::thread::spawn(move || -> Result<_> { - tracing::info!(?test_type, "waiting for lightwalletd to sync to the tip"); - - lightwalletd - .expect_stdout_line_matches(crate::common::sync::LIGHTWALLETD_SYNC_FINISHED_REGEX)?; - - Ok(lightwalletd) - }); - - // `lightwalletd` syncs can take a long time, - // so we need to check that `zebrad` has synced to the tip in parallel. - let lightwalletd_thread_and_zebrad = std::thread::spawn(move || -> Result<_> { - tracing::info!(?test_type, "waiting for zebrad to sync to the tip"); - - while !lightwalletd_thread.is_finished() { - zebrad.expect_stdout_line_matches(crate::common::sync::SYNC_FINISHED_REGEX)?; - } - - Ok((lightwalletd_thread, zebrad)) - }); - - // Retrieve the child process handles from the threads - let (lightwalletd_thread, mut zebrad) = lightwalletd_thread_and_zebrad - .join() - .unwrap_or_else(|panic_object| std::panic::resume_unwind(panic_object))?; - - let lightwalletd = lightwalletd_thread - .join() - .unwrap_or_else(|panic_object| std::panic::resume_unwind(panic_object))?; +) -> bool { + if zebra_test::net::zebra_skip_network_tests() { + return false; + } - // If we are in sync mempool should activate - zebrad.expect_stdout_line_matches("activating mempool")?; + // Skip the test unless the user specifically asked for it + // + // TODO: pass test_type to zebra_skip_lightwalletd_tests() and check for lightwalletd launch in there + if test_type.launches_lightwalletd() && zebra_skip_lightwalletd_tests() { + return false; + } - Ok((lightwalletd, zebrad)) + // Check if we have any necessary cached states for the zebrad config + test_type.zebrad_config(test_name).is_some() } /// Panics if `$pred` is false, with an error report containing: diff --git a/zebrad/tests/common/lightwalletd.rs b/zebrad/tests/common/lightwalletd.rs index 1b71d8d681d..cb0454624ca 100644 --- a/zebrad/tests/common/lightwalletd.rs +++ b/zebrad/tests/common/lightwalletd.rs @@ -12,7 +12,11 @@ use std::{ time::Duration, }; +use tempfile::TempDir; + +use zebra_chain::parameters::Network::{self, *}; use zebra_test::{ + args, command::{Arguments, TestChild, TestDirExt, NO_MATCHES_REGEX_ITER}, net::random_known_port, prelude::*, @@ -21,7 +25,7 @@ use zebrad::config::ZebradConfig; use super::{ cached_state::ZEBRA_CACHED_STATE_DIR, - config::default_test_config, + config::{default_test_config, testdir}, failure_messages::{ LIGHTWALLETD_EMPTY_ZEBRA_STATE_IGNORE_MESSAGES, LIGHTWALLETD_FAILURE_MESSAGES, PROCESS_FAILURE_MESSAGES, ZEBRA_FAILURE_MESSAGES, @@ -106,6 +110,80 @@ pub fn random_known_rpc_port_config(parallel_cpu_threads: bool) -> Result + std::fmt::Debug>( + network: Network, + test_name: S, + test_type: LightwalletdTestType, + zebrad_rpc_address: SocketAddr, +) -> Result, u16)>> { + assert_eq!(network, Mainnet, "this test only supports Mainnet for now"); + + let test_name = test_name.as_ref(); + + // Skip the test unless the user specifically asked for it + if !can_spawn_lightwalletd_for_rpc(test_name, test_type) { + return Ok(None); + } + + let lightwalletd_state_path = test_type.lightwalletd_state_path(test_name); + let lightwalletd_dir = testdir()?.with_lightwalletd_config(zebrad_rpc_address)?; + + let lightwalletd_rpc_port = random_known_port(); + let lightwalletd_rpc_address = format!("127.0.0.1:{lightwalletd_rpc_port}"); + + let arguments = args!["--grpc-bind-addr": lightwalletd_rpc_address]; + + let (lightwalletd_failure_messages, lightwalletd_ignore_messages) = + test_type.lightwalletd_failure_messages(); + + let mut lightwalletd = lightwalletd_dir + .spawn_lightwalletd_child(lightwalletd_state_path, arguments)? + .with_timeout(test_type.lightwalletd_timeout()) + .with_failure_regex_iter(lightwalletd_failure_messages, lightwalletd_ignore_messages); + + // Wait until `lightwalletd` has launched. + // This log happens very quickly, so it is ok to block for a short while here. + lightwalletd.expect_stdout_line_matches(regex::escape("Starting gRPC server"))?; + + Ok(Some((lightwalletd, lightwalletd_rpc_port))) +} + +/// Returns `true` if a lightwalletd test for `test_type` has everything it needs to run. +#[tracing::instrument] +pub fn can_spawn_lightwalletd_for_rpc + std::fmt::Debug>( + test_name: S, + test_type: LightwalletdTestType, +) -> bool { + if zebra_test::net::zebra_skip_network_tests() { + return false; + } + + // Skip the test unless the user specifically asked for it + // + // TODO: pass test_type to zebra_skip_lightwalletd_tests() and check for lightwalletd launch in there + if test_type.launches_lightwalletd() && zebra_skip_lightwalletd_tests() { + return false; + } + + let lightwalletd_state_path = test_type.lightwalletd_state_path(test_name); + if test_type.needs_lightwalletd_cached_state() && lightwalletd_state_path.is_none() { + return false; + } + + true +} + /// Extension trait for methods on `tempfile::TempDir` for using it as a test /// directory for `zebrad`. pub trait LightWalletdTestDirExt: ZebradTestDirExt @@ -234,6 +312,9 @@ pub enum LightwalletdTestType { /// Do a full sync from an empty lightwalletd state. /// /// This test requires a cached Zebra state. + // + // Only used with `--features=lightwalletd-grpc-tests`. + #[allow(dead_code)] FullSyncFromGenesis { /// Should the test allow a cached lightwalletd state? /// @@ -258,6 +339,10 @@ pub enum LightwalletdTestType { impl LightwalletdTestType { /// Does this test need a Zebra cached state? pub fn needs_zebra_cached_state(&self) -> bool { + // Handle the Zebra state directory based on the test type: + // - LaunchWithEmptyState: ignore the state directory + // - FullSyncFromGenesis, UpdateCachedState, UpdateZebraCachedStateNoRpc: + // skip the test if it is not available match self { LaunchWithEmptyState => false, FullSyncFromGenesis { .. } | UpdateCachedState | UpdateZebraCachedStateNoRpc => true, @@ -274,6 +359,10 @@ impl LightwalletdTestType { /// Does this test need a `lightwalletd` cached state? pub fn needs_lightwalletd_cached_state(&self) -> bool { + // Handle the lightwalletd state directory based on the test type: + // - LaunchWithEmptyState, UpdateZebraCachedStateNoRpc: ignore the state directory + // - FullSyncFromGenesis: use it if available, timeout if it is already populated + // - UpdateCachedState: skip the test if it is not available match self { LaunchWithEmptyState | FullSyncFromGenesis { .. } | UpdateZebraCachedStateNoRpc => { false @@ -295,10 +384,11 @@ impl LightwalletdTestType { /// Returns the Zebra state path for this test, if set. #[allow(clippy::print_stderr)] - pub fn zebrad_state_path(&self, test_name: String) -> Option { + pub fn zebrad_state_path>(&self, test_name: S) -> Option { match env::var_os(ZEBRA_CACHED_STATE_DIR) { Some(path) => Some(path.into()), None => { + let test_name = test_name.as_ref(); eprintln!( "skipped {test_name:?} {self:?} lightwalletd test, \ set the {ZEBRA_CACHED_STATE_DIR:?} environment variable to run the test", @@ -313,7 +403,7 @@ impl LightwalletdTestType { /// /// Returns `None` if the test should be skipped, /// and `Some(Err(_))` if the config could not be created. - pub fn zebrad_config(&self, test_name: String) -> Option> { + pub fn zebrad_config>(&self, test_name: S) -> Option> { let config = if self.launches_lightwalletd() { // This is what we recommend our users configure. random_known_rpc_port_config(true) @@ -346,7 +436,9 @@ impl LightwalletdTestType { } /// Returns the `lightwalletd` state path for this test, if set, and if allowed for this test. - pub fn lightwalletd_state_path(&self, test_name: String) -> Option { + pub fn lightwalletd_state_path>(&self, test_name: S) -> Option { + let test_name = test_name.as_ref(); + if !self.launches_lightwalletd() { tracing::info!( "running {test_name:?} {self:?} lightwalletd test, \ diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc.rs b/zebrad/tests/common/lightwalletd/wallet_grpc.rs index edfed7c5b3e..1473294fe09 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc.rs @@ -1,16 +1,16 @@ //! Lightwalletd gRPC interface and utility functions. -use std::{env, net::SocketAddr, path::PathBuf}; +use std::{ + env, + net::SocketAddr, + sync::atomic::{AtomicBool, Ordering}, +}; use tempfile::TempDir; -use zebra_test::{args, net::random_known_port, prelude::*}; - -use crate::common::{ - config::testdir, lightwalletd::LightWalletdTestDirExt, sync::LIGHTWALLETD_SYNC_FINISHED_REGEX, -}; +use zebra_test::prelude::*; -use super::LightwalletdTestType; +use crate::common::{launch::ZebradTestDirExt, lightwalletd::LightwalletdTestType}; tonic::include_proto!("cash.z.wallet.sdk.rpc"); @@ -18,44 +18,6 @@ tonic::include_proto!("cash.z.wallet.sdk.rpc"); pub type LightwalletdRpcClient = compact_tx_streamer_client::CompactTxStreamerClient; -/// Start a lightwalletd instance connected to `zebrad_rpc_address`, -/// using the `lightwalletd_state_path`, with its gRPC server functionality enabled. -/// -/// Expects cached state based on the `test_type`. -/// Waits for `lightwalletd` to sync to near the tip, if `wait_for_sync` is true. -/// -/// Returns the lightwalletd instance and the port number that it is listening for RPC connections. -#[tracing::instrument] -pub fn spawn_lightwalletd_with_rpc_server( - zebrad_rpc_address: SocketAddr, - lightwalletd_state_path: Option, - test_type: LightwalletdTestType, - wait_for_sync: bool, -) -> Result<(TestChild, u16)> { - let lightwalletd_dir = testdir()?.with_lightwalletd_config(zebrad_rpc_address)?; - - let lightwalletd_rpc_port = random_known_port(); - let lightwalletd_rpc_address = format!("127.0.0.1:{lightwalletd_rpc_port}"); - - let arguments = args!["--grpc-bind-addr": lightwalletd_rpc_address]; - - let (lightwalletd_failure_messages, lightwalletd_ignore_messages) = - test_type.lightwalletd_failure_messages(); - - let mut lightwalletd = lightwalletd_dir - .spawn_lightwalletd_child(lightwalletd_state_path, arguments)? - .with_timeout(test_type.lightwalletd_timeout()) - .with_failure_regex_iter(lightwalletd_failure_messages, lightwalletd_ignore_messages); - - lightwalletd.expect_stdout_line_matches("Starting gRPC server")?; - - if wait_for_sync { - lightwalletd.expect_stdout_line_matches(LIGHTWALLETD_SYNC_FINISHED_REGEX)?; - } - - Ok((lightwalletd, lightwalletd_rpc_port)) -} - /// Connect to a lightwalletd RPC instance. #[tracing::instrument] pub async fn connect_to_lightwalletd(lightwalletd_rpc_port: u16) -> Result { @@ -65,3 +27,164 @@ pub async fn connect_to_lightwalletd(lightwalletd_rpc_port: u16) -> Result( + mut lightwalletd: TestChild, + lightwalletd_rpc_port: u16, + mut zebrad: TestChild

, + zebra_rpc_address: SocketAddr, + test_type: LightwalletdTestType, + wait_for_zebrad_mempool: bool, + wait_for_zebrad_tip: bool, +) -> Result<(TestChild, TestChild

)> { + let is_zebrad_finished = AtomicBool::new(false); + let is_lightwalletd_finished = AtomicBool::new(false); + + let is_zebrad_finished = &is_zebrad_finished; + let is_lightwalletd_finished = &is_lightwalletd_finished; + + // TODO: split these closures out into their own functions + + // Check Zebra's logs for errors. + // Optionally waits until Zebra has synced to the tip, based on `wait_for_zebrad_tip`. + // + // `lightwalletd` syncs can take a long time, + // so we need to check that `zebrad` has synced to the tip in parallel. + let zebrad_mut = &mut zebrad; + let zebrad_wait_fn = || -> Result<_> { + tracing::info!( + ?test_type, + ?zebra_rpc_address, + "waiting for zebrad to open its RPC port..." + ); + zebrad_mut + .expect_stdout_line_matches(&format!("Opened RPC endpoint at {}", zebra_rpc_address))?; + + // When we are near the tip, the mempool should activate at least once + if wait_for_zebrad_mempool { + tracing::info!( + ?test_type, + "waiting for zebrad to activate the mempool when it gets near the tip..." + ); + zebrad_mut.expect_stdout_line_matches("activating mempool")?; + } + + // When we are near the tip, this message is logged multiple times + if wait_for_zebrad_tip { + tracing::info!(?test_type, "waiting for zebrad to sync to the tip..."); + zebrad_mut.expect_stdout_line_matches(crate::common::sync::SYNC_FINISHED_REGEX)?; + } + + // Tell the other thread that Zebra has finished + is_zebrad_finished.store(true, Ordering::SeqCst); + + tracing::info!( + ?test_type, + "zebrad is waiting for lightwalletd to sync to the tip..." + ); + while !is_lightwalletd_finished.load(Ordering::SeqCst) { + // Just keep checking the Zebra logs for errors, + // using the sync finished regex, which is logged every minute or so. + zebrad_mut.expect_stdout_line_matches(crate::common::sync::SYNC_FINISHED_REGEX)?; + } + + Ok(zebrad_mut) + }; + + // Wait until `lightwalletd` has synced to Zebra's tip. + // Calls `lightwalletd`'s gRPCs and Zebra's JSON-RPCs. + // Also checks `lightwalletd`'s logs for errors. + // + // `zebrad` syncs can take a long time, + // so we need to check that `lightwalletd` has synced to the tip in parallel. + let lightwalletd_mut = &mut lightwalletd; + let lightwalletd_wait_fn = || -> Result<_> { + tracing::info!( + ?test_type, + "lightwalletd is waiting for zebrad to sync to the tip..." + ); + while !is_zebrad_finished.load(Ordering::SeqCst) { + // Just keep checking the `lightwalletd` logs for errors. + // It usually logs something every 30-90 seconds, + // but there's no specific message we need to wait for here. + assert!( + lightwalletd_mut.wait_for_stdout_line(None), + "lightwalletd output unexpectedly finished early", + ); + } + + tracing::info!(?test_type, "waiting for lightwalletd to sync to the tip..."); + while !are_zebrad_and_lightwalletd_tips_synced(lightwalletd_rpc_port, zebra_rpc_address)? { + assert!( + lightwalletd_mut.wait_for_stdout_line(None), + "lightwalletd output unexpectedly finished early", + ); + } + + // Tell the other thread that `lightwalletd` has finished + is_lightwalletd_finished.store(true, Ordering::SeqCst); + + Ok(lightwalletd_mut) + }; + + // Run both threads in parallel, automatically propagating any panics to this thread. + std::thread::scope(|s| { + s.spawn(|| { + let zebrad_result = zebrad_wait_fn(); + is_zebrad_finished.store(true, Ordering::SeqCst); + zebrad_result.expect("test failed while waiting for zebrad to sync"); + }); + s.spawn(|| { + let lightwalletd_result = lightwalletd_wait_fn(); + is_lightwalletd_finished.store(true, Ordering::SeqCst); + lightwalletd_result.expect("test failed while waiting for lightwalletd to sync"); + }); + }); + + Ok((lightwalletd, zebrad)) +} + +/// Returns `Ok(true)` if zebrad and lightwalletd are both at the same height. +#[tracing::instrument] +pub fn are_zebrad_and_lightwalletd_tips_synced( + lightwalletd_rpc_port: u16, + zebra_rpc_address: SocketAddr, +) -> Result { + let rt = tokio::runtime::Builder::new_multi_thread().build()?; + + rt.block_on(async { + let mut lightwalletd_grpc_client = connect_to_lightwalletd(lightwalletd_rpc_port).await?; + + // Get the block tip from lightwalletd + let lightwalletd_tip_block = lightwalletd_grpc_client + .get_latest_block(ChainSpec {}) + .await? + .into_inner(); + let lightwalletd_tip_height = lightwalletd_tip_block.height; + + // Get the block tip from zebrad + let zebrad_json_rpc_client = reqwest::Client::new(); + let zebrad_blockchain_info = zebrad_json_rpc_client + .post(format!("http://{}", &zebra_rpc_address.to_string())) + .body(r#"{"jsonrpc": "2.0", "method": "getblockchaininfo", "params": [], "id":123 }"#) + .header("Content-Type", "application/json") + .send() + .await? + .text() + .await?; + let zebrad_blockchain_info: serde_json::Value = + serde_json::from_str(&zebrad_blockchain_info)?; + let zebrad_tip_height = zebrad_blockchain_info["result"]["blocks"] + .as_u64() + .expect("unexpected block height: doesn't fit in u64"); + + Ok(lightwalletd_tip_height == zebrad_tip_height) + }) +} diff --git a/zebrad/tests/common/sync.rs b/zebrad/tests/common/sync.rs index 8bd3ca5d900..1b9ac172fad 100644 --- a/zebrad/tests/common/sync.rs +++ b/zebrad/tests/common/sync.rs @@ -45,14 +45,6 @@ pub const STOP_AT_HEIGHT_REGEX: &str = "stopping at configured height"; pub const SYNC_FINISHED_REGEX: &str = r"finished initial sync to chain tip, using gossiped blocks .*sync_percent.*=.*100\."; -/// The text that should be logged when `lightwalletd`'s initial sync is near the chain tip. -/// -/// We can't guarantee a "Waiting for block" log, so we just check for a block near the tip height. -/// -/// TODO: update the regex to `1[8-9][0-9]{5}` when mainnet reaches block 1_800_000 -pub const LIGHTWALLETD_SYNC_FINISHED_REGEX: &str = - r"([Aa]dding block to cache 1[7-9][0-9]{5})|([Ww]aiting for block: 1[7-9][0-9]{5})"; - /// The maximum amount of time Zebra should take to reload after shutting down. /// /// This should only take a second, but sometimes CI VMs or RocksDB can be slow. From d7824705337776c80e810d1ae24ec8fe83fbc3a7 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 21 Sep 2022 18:21:13 +1000 Subject: [PATCH 07/20] Rewrite the gRPC and full sync tests for the new APIs --- zebrad/tests/acceptance.rs | 187 ++++++++---------- .../lightwalletd/send_transaction_test.rs | 98 ++++----- .../common/lightwalletd/wallet_grpc_test.rs | 112 ++++------- 3 files changed, 162 insertions(+), 235 deletions(-) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index b80e29d14f8..8f592f85b84 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -136,12 +136,9 @@ use common::{ config::{ config_file_full_path, configs_dir, default_test_config, persistent_test_config, testdir, }, - launch::{ - spawn_zebrad_for_rpc, wait_for_zebrad_and_lightwalletd_tip, ZebradTestDirExt, - BETWEEN_NODES_DELAY, LAUNCH_DELAY, - }, + launch::{spawn_zebrad_for_rpc, ZebradTestDirExt, BETWEEN_NODES_DELAY, LAUNCH_DELAY}, lightwalletd::{ - random_known_rpc_port_config, zebra_skip_lightwalletd_tests, LightWalletdTestDirExt, + can_spawn_lightwalletd_for_rpc, random_known_rpc_port_config, spawn_lightwalletd_for_rpc, LightwalletdTestType::{self, *}, }, sync::{ @@ -1428,25 +1425,31 @@ fn zebrad_update_sync() -> Result<()> { /// Make sure `lightwalletd` can sync from Zebra, in update sync mode. /// -/// This test only runs when the `ZEBRA_TEST_LIGHTWALLETD`, -/// `ZEBRA_CACHED_STATE_DIR`, and `LIGHTWALLETD_DATA_DIR` env vars are set. +/// This test only runs when: +/// - the `ZEBRA_TEST_LIGHTWALLETD`, `ZEBRA_CACHED_STATE_DIR`, and +/// `LIGHTWALLETD_DATA_DIR` env vars are set, and +/// - Zebra is compiled with `--features=lightwalletd-grpc-tests`. /// /// This test doesn't work on Windows, so it is always skipped on that platform. #[test] #[cfg(not(target_os = "windows"))] +#[cfg(feature = "lightwalletd-grpc-tests")] fn lightwalletd_update_sync() -> Result<()> { lightwalletd_integration_test(UpdateCachedState) } /// Make sure `lightwalletd` can fully sync from genesis using Zebra. /// -/// This test only runs when the `ZEBRA_TEST_LIGHTWALLETD` and -/// `ZEBRA_CACHED_STATE_DIR` env vars are set. +/// This test only runs when: +/// - the `ZEBRA_TEST_LIGHTWALLETD` and `ZEBRA_CACHED_STATE_DIR` env vars are set, and +/// - Zebra is compiled with `--features=lightwalletd-grpc-tests`. +/// /// /// This test doesn't work on Windows, so it is always skipped on that platform. #[test] #[ignore] #[cfg(not(target_os = "windows"))] +#[cfg(feature = "lightwalletd-grpc-tests")] fn lightwalletd_full_sync() -> Result<()> { lightwalletd_integration_test(FullSyncFromGenesis { allow_lightwalletd_cached_state: false, @@ -1464,7 +1467,7 @@ fn lightwalletd_full_sync() -> Result<()> { /// - run a send transaction gRPC test, /// - run read-only gRPC tests. /// -/// The gRPC tests only run when the `lightwalletd-grpc-tests` is on. +/// The lightwalletd full, update, and gRPC tests only run with `--features=lightwalletd-grpc-tests`. /// /// These tests don't work on Windows, so they are always skipped on that platform. #[tokio::test] @@ -1510,69 +1513,37 @@ async fn lightwalletd_test_suite() -> Result<()> { /// /// # Panics /// -/// If the `test_type` is UpdateZebraCachedStateNoRpc +/// If the `test_type` requires `--features=lightwalletd-grpc-tests`, +/// but Zebra was not compiled with that feature. #[tracing::instrument] fn lightwalletd_integration_test(test_type: LightwalletdTestType) -> Result<()> { let _init_guard = zebra_test::init(); - if zebra_test::net::zebra_skip_network_tests() { - return Ok(()); - } + // We run these sync tests with a network connection, for better test coverage. + let use_internet_connection = true; + let network = Mainnet; + let test_name = "lightwalletd_integration_test"; - // Skip the test unless the user specifically asked for it - // - // TODO: pass test_type to zebra_skip_lightwalletd_tests() and check for lightwalletd launch in there - if test_type.launches_lightwalletd() && zebra_skip_lightwalletd_tests() { + if test_type.launches_lightwalletd() && !can_spawn_lightwalletd_for_rpc(test_name, test_type) { + tracing::info!("skipping test due to missing lightwalletd network or cached state"); return Ok(()); } - // TODO: split the zebrad and lightwalletd launches and checks into separate functions? - - // Get the zebrad config - - // Handle the Zebra state directory based on the test type: - // - LaunchWithEmptyState: ignore the state directory - // - FullSyncFromGenesis, UpdateCachedState, UpdateZebraCachedStateNoRpc: - // skip the test if it is not available, timeout if it is not populated - - // Write a configuration that has RPC listen_addr set (if needed). - // If the state path env var is set, use it in the config. - let config = if let Some(config) = - test_type.zebrad_config("lightwalletd_integration_test".to_string()) + // Launch zebra with peers and using a predefined zebrad state path. + let (mut zebrad, zebra_rpc_address) = if let Some(zebrad_and_address) = + spawn_zebrad_for_rpc(network, test_name, test_type, use_internet_connection)? { - config? + tracing::info!( + ?test_type, + "running lightwalletd & zebrad integration test, launching zebrad...", + ); + + zebrad_and_address } else { + // Skip the test, we don't have the required cached state return Ok(()); }; - // Handle the lightwalletd state directory based on the test type: - // - LaunchWithEmptyState, UpdateZebraCachedStateNoRpc: ignore the state directory - // - FullSyncFromGenesis: use it if available, timeout if it is already populated - // - UpdateCachedState: skip the test if it is not available, timeout if it is not populated - let lightwalletd_state_path = - test_type.lightwalletd_state_path("lightwalletd_integration_test".to_string()); - - if test_type.needs_lightwalletd_cached_state() && lightwalletd_state_path.is_none() { - return Ok(()); - } - - tracing::info!( - ?test_type, - ?config, - ?lightwalletd_state_path, - "running lightwalletd & zebrad integration test", - ); - - // Get the lists of process failure logs - let (zebrad_failure_messages, zebrad_ignore_messages) = test_type.zebrad_failure_messages(); - - // Launch zebrad - let zdir = testdir()?.with_exact_config(&config)?; - let mut zebrad = zdir - .spawn_child(args!["start"])? - .with_timeout(test_type.zebrad_timeout()) - .with_failure_regex_iter(zebrad_failure_messages, zebrad_ignore_messages); - if test_type.needs_zebra_cached_state() { zebrad .expect_stdout_line_matches(r"loaded Zebra state cache .*tip.*=.*Height\([0-9]{7}\)")?; @@ -1582,32 +1553,21 @@ fn lightwalletd_integration_test(test_type: LightwalletdTestType) -> Result<()> } // Launch lightwalletd, if needed - let lightwalletd = if test_type.launches_lightwalletd() { - // Wait until `zebrad` has opened the RPC endpoint - zebrad.expect_stdout_line_matches(regex::escape( - format!("Opened RPC endpoint at {}", config.rpc.listen_addr.unwrap()).as_str(), - ))?; - - // Write a fake zcashd configuration that has the rpcbind and rpcport options set - let ldir = testdir()?; - let ldir = ldir.with_lightwalletd_config(config.rpc.listen_addr.unwrap())?; - - let (lightwalletd_failure_messages, lightwalletd_ignore_messages) = - test_type.lightwalletd_failure_messages(); - - // Launch the lightwalletd process - let lightwalletd = if test_type == LaunchWithEmptyState { - ldir.spawn_lightwalletd_child(None, args![])? - } else { - ldir.spawn_lightwalletd_child(lightwalletd_state_path, args![])? - }; + let lightwalletd_and_port = if test_type.launches_lightwalletd() { + tracing::info!( + ?zebra_rpc_address, + "launching lightwalletd connected to zebrad", + ); - let mut lightwalletd = lightwalletd - .with_timeout(test_type.lightwalletd_timeout()) - .with_failure_regex_iter(lightwalletd_failure_messages, lightwalletd_ignore_messages); + // Launch lightwalletd + let (mut lightwalletd, lightwalletd_rpc_port) = + spawn_lightwalletd_for_rpc(network, test_name, test_type, zebra_rpc_address)? + .expect("already checked for lightwalletd cached state and network"); - // Wait until `lightwalletd` has launched - lightwalletd.expect_stdout_line_matches(regex::escape("Starting gRPC server"))?; + tracing::info!( + ?lightwalletd_rpc_port, + "spawned lightwalletd connected to zebrad", + ); // Check that `lightwalletd` is calling the expected Zebra RPCs @@ -1659,29 +1619,42 @@ fn lightwalletd_integration_test(test_type: LightwalletdTestType) -> Result<()> } } - Some(lightwalletd) + Some((lightwalletd, lightwalletd_rpc_port)) } else { None }; let (mut zebrad, lightwalletd) = if test_type.needs_zebra_cached_state() { - if let Some(lightwalletd) = lightwalletd { + if let Some((lightwalletd, lightwalletd_rpc_port)) = lightwalletd_and_port { #[cfg(feature = "lightwalletd-grpc-tests")] { - let (zebrad, lightwalletd) = - common::lightwalletd::wallet_grpc::wait_for_zebrad_and_lightwalletd_tip( - lightwalletd, - zebrad, - test_type, - )?; + use common::lightwalletd::wallet_grpc::wait_for_zebrad_and_lightwalletd_sync; + + tracing::info!( + ?lightwalletd_rpc_port, + "waiting for zebrad and lightwalletd to sync...", + ); + + let (lightwalletd, zebrad) = wait_for_zebrad_and_lightwalletd_sync( + lightwalletd, + lightwalletd_rpc_port, + zebrad, + zebra_rpc_address, + test_type, + // We want to wait for the mempool and network for better coverage + true, + use_internet_connection, + )?; + (zebrad, Some(lightwalletd)) } #[cfg(not(feature = "lightwalletd-grpc-tests"))] panic!( - "the {test_type:?} test requires `cargo test --feature lightwalletd-grpc-tests` \n\ - zebrad: {zebrad:?} \n\ - lightwalletd: {lightwalletd:?}" + "the {test_type:?} test requires `cargo test --feature lightwalletd-grpc-tests`\n\ + zebrad: {zebrad:?}\n\ + lightwalletd: {lightwalletd:?}\n\ + lightwalletd_rpc_port: {lightwalletd_rpc_port:?}" ); } else { // We're just syncing Zebra, so there's no lightwalletd to check @@ -1691,6 +1664,8 @@ fn lightwalletd_integration_test(test_type: LightwalletdTestType) -> Result<()> (zebrad, None) } } else { + let lightwalletd = lightwalletd_and_port.map(|(lightwalletd, _port)| lightwalletd); + // We don't have a cached state, so we don't do any tip checks for Zebra or lightwalletd (zebrad, lightwalletd) }; @@ -1971,23 +1946,21 @@ async fn fully_synced_rpc_test() -> Result<()> { let _init_guard = zebra_test::init(); // We're only using cached Zebra state here, so this test type is the most similar - let test_type = LightwalletdTestType::FullSyncFromGenesis { - allow_lightwalletd_cached_state: false, - }; + let test_type = LightwalletdTestType::UpdateCachedState; + let network = Network::Mainnet; - // Handle the Zebra state directory - let cached_state_path = test_type.zebrad_state_path("fully_synced_rpc_test".to_string()); + let (mut zebrad, zebra_rpc_address) = if let Some(zebrad_and_address) = + spawn_zebrad_for_rpc(network, "fully_synced_rpc_test", test_type, false)? + { + tracing::info!("running fully synced zebrad RPC test"); - if cached_state_path.is_none() { + zebrad_and_address + } else { + // Skip the test, we don't have the required cached state return Ok(()); }; - tracing::info!("running fully synced zebrad RPC test"); - - let network = Network::Mainnet; - - let (_zebrad, zebra_rpc_address) = - spawn_zebrad_for_rpc(network, cached_state_path.unwrap(), test_type, true, false)?; + zebrad.expect_stdout_line_matches(&format!("Opened RPC endpoint at {}", zebra_rpc_address))?; // Make a getblock test that works only on synced node (high block number). // The block is before the mandatory checkpoint, so the checkpoint cached state can be used diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index 5495eabebb4..494580e09f4 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -26,7 +26,7 @@ use tower::{Service, ServiceExt}; use zebra_chain::{ block, chain_tip::ChainTip, - parameters::Network, + parameters::Network::{self, *}, serialization::ZcashSerialize, transaction::{self, Transaction}, }; @@ -36,12 +36,12 @@ use zebrad::components::mempool::downloads::MAX_INBOUND_CONCURRENCY; use crate::common::{ cached_state::{load_tip_height_from_state_directory, start_state_service_with_cache_dir}, - launch::spawn_zebrad_for_rpc, + launch::{can_spawn_zebrad_for_rpc, spawn_zebrad_for_rpc}, lightwalletd::{ + can_spawn_lightwalletd_for_rpc, spawn_lightwalletd_for_rpc, wallet_grpc::{ - self, connect_to_lightwalletd, spawn_lightwalletd_with_rpc_server, Empty, Exclude, + self, connect_to_lightwalletd, wait_for_zebrad_and_lightwalletd_sync, Empty, Exclude, }, - zebra_skip_lightwalletd_tests, LightwalletdTestType::*, }, sync::copy_state_and_perform_full_sync, @@ -65,38 +65,32 @@ fn max_sent_transactions() -> usize { pub async fn run() -> Result<()> { let _init_guard = zebra_test::init(); - if zebra_test::net::zebra_skip_network_tests() { - return Ok(()); - } + // We want a zebra state dir and a lightwalletd data dir in place, + // so `UpdateCachedState` can be used as our test type + let test_type = UpdateCachedState; + let test_name = "send_transaction_test"; + let network = Mainnet; // Skip the test unless the user specifically asked for it - if zebra_skip_lightwalletd_tests() { + if !can_spawn_zebrad_for_rpc(test_name, test_type) { return Ok(()); } - // We want a zebra state dir and a lightwalletd data dir in place, - // so `UpdateCachedState` can be used as our test type - let test_type = UpdateCachedState; + if test_type.launches_lightwalletd() && !can_spawn_lightwalletd_for_rpc(test_name, test_type) { + tracing::info!("skipping test due to missing lightwalletd network or cached state"); + return Ok(()); + } - let zebrad_state_path = test_type.zebrad_state_path("send_transaction_tests".to_string()); + let zebrad_state_path = test_type.zebrad_state_path(test_name); let zebrad_state_path = match zebrad_state_path { Some(zebrad_state_path) => zebrad_state_path, None => return Ok(()), }; - let lightwalletd_state_path = - test_type.lightwalletd_state_path("send_transaction_tests".to_string()); - if lightwalletd_state_path.is_none() { - return Ok(()); - } - - let network = Network::Mainnet; - tracing::info!( ?network, ?test_type, ?zebrad_state_path, - ?lightwalletd_state_path, "running gRPC send transaction test using lightwalletd & zebrad", ); @@ -106,32 +100,47 @@ pub async fn run() -> Result<()> { tracing::info!( transaction_count = ?transactions.len(), partial_sync_path = ?zebrad_state_path, - "got transactions to send", + "got transactions to send, spawning isolated zebrad...", ); + // We run these gRPC tests without a network connection. + let use_internet_connection = false; + // Start zebrad with no peers, we want to send transactions without blocks coming in. If `wallet_grpc_test` // runs before this test (as it does in `lightwalletd_test_suite`), then we are the most up to date with tip we can. - let (mut zebrad, zebra_rpc_address) = - spawn_zebrad_for_rpc(network, zebrad_state_path, test_type, false, false)?; + let (zebrad, zebra_rpc_address) = if let Some(zebrad_and_address) = + spawn_zebrad_for_rpc(network, test_name, test_type, use_internet_connection)? + { + zebrad_and_address + } else { + // Skip the test, we don't have the required cached state + return Ok(()); + }; tracing::info!( ?zebra_rpc_address, - "spawned disconnected zebrad with shorter chain, waiting for mempool activation...", + "spawned isolated zebrad with shorter chain, spawning lightwalletd...", ); - let (_lightwalletd, lightwalletd_rpc_port) = spawn_lightwalletd_with_rpc_server( - zebra_rpc_address, - lightwalletd_state_path, - test_type, - true, - )?; + let (lightwalletd, lightwalletd_rpc_port) = + spawn_lightwalletd_for_rpc(network, test_name, test_type, zebra_rpc_address)? + .expect("already checked cached state and network requirements"); tracing::info!( ?lightwalletd_rpc_port, - "spawned lightwalletd connected to zebrad, waiting for zebrad mempool activation...", + "spawned lightwalletd connected to zebrad, waiting for them both to sync...", ); - zebrad.expect_stdout_line_matches("activating mempool")?; + let (_lightwalletd, mut zebrad) = wait_for_zebrad_and_lightwalletd_sync( + lightwalletd, + lightwalletd_rpc_port, + zebrad, + zebra_rpc_address, + test_type, + // We want to send transactions to the mempool, but we aren't syncing with the network + true, + use_internet_connection, + )?; tracing::info!( ?lightwalletd_rpc_port, @@ -140,29 +149,6 @@ pub async fn run() -> Result<()> { let mut rpc_client = connect_to_lightwalletd(lightwalletd_rpc_port).await?; - // Check if zebrad and lightwalletd are both in the same height. - - // Get the block tip from lightwalletd - let block_tip = rpc_client - .get_latest_block(wallet_grpc::ChainSpec {}) - .await? - .into_inner(); - - // Get the block tip from zebrad - let client = reqwest::Client::new(); - let res = client - .post(format!("http://{}", &zebra_rpc_address.to_string())) - .body(r#"{"jsonrpc": "2.0", "method": "getblockchaininfo", "params": [], "id":123 }"#) - .header("Content-Type", "application/json") - .send() - .await? - .text() - .await?; - - // Make sure tip from lightwalletd is the same as zebrad - let parsed: serde_json::Value = serde_json::from_str(&res)?; - assert_eq!(block_tip.height, parsed["result"]["blocks"]); - // To avoid filling the mempool queue, limit the transactions to be sent to the RPC and mempool queue limits transactions.truncate(max_sent_transactions()); diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs index f41beb57330..24a889d65f0 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs @@ -46,14 +46,14 @@ use zebra_chain::{ use zebra_network::constants::USER_AGENT; use crate::common::{ - launch::{spawn_zebrad_for_rpc, wait_for_zebrad_and_lightwalletd_tip}, + launch::spawn_zebrad_for_rpc, lightwalletd::{ + can_spawn_lightwalletd_for_rpc, spawn_lightwalletd_for_rpc, wallet_grpc::{ - connect_to_lightwalletd, spawn_lightwalletd_with_rpc_server, Address, AddressList, + connect_to_lightwalletd, wait_for_zebrad_and_lightwalletd_sync, Address, AddressList, BlockId, BlockRange, ChainSpec, Empty, GetAddressUtxosArg, TransparentAddressBlockFilter, TxFilter, }, - zebra_skip_lightwalletd_tests, LightwalletdTestType::UpdateCachedState, }, }; @@ -67,73 +67,64 @@ use crate::common::{ pub async fn run() -> Result<()> { let _init_guard = zebra_test::init(); - // Skip the test unless the user specifically asked for it - if zebra_skip_lightwalletd_tests() { - return Ok(()); - } - // We want a zebra state dir and a lightwalletd data dir in place, // so `UpdateCachedState` can be used as our test type let test_type = UpdateCachedState; - // Require to have a `ZEBRA_CACHED_STATE_DIR` in place - let zebrad_state_path = test_type.zebrad_state_path("wallet_grpc_test".to_string()); - if zebrad_state_path.is_none() { - return Ok(()); - } - - // Require to have a `LIGHTWALLETD_DATA_DIR` in place - let lightwalletd_state_path = test_type.lightwalletd_state_path("wallet_grpc_test".to_string()); - if lightwalletd_state_path.is_none() { - return Ok(()); - } - // This test is only for the mainnet let network = Network::Mainnet; + let test_name = "wallet_grpc_test"; - tracing::info!( - ?network, - ?test_type, - ?zebrad_state_path, - ?lightwalletd_state_path, - "running gRPC query tests using lightwalletd & zebrad, \ - launching disconnected zebrad...", - ); + // We run these gRPC tests with a network connection, for better test coverage. + let use_internet_connection = true; + + if test_type.launches_lightwalletd() && !can_spawn_lightwalletd_for_rpc(test_name, test_type) { + tracing::info!("skipping test due to missing lightwalletd network or cached state"); + return Ok(()); + } // Launch zebra with peers and using a predefined zebrad state path. // As this tests are just queries we can have a live chain where blocks are coming. - let (zebrad, zebra_rpc_address) = spawn_zebrad_for_rpc( - network, - zebrad_state_path.clone().unwrap(), - test_type, - true, - true, - )?; + let (zebrad, zebra_rpc_address) = if let Some(zebrad_and_address) = + spawn_zebrad_for_rpc(network, test_name, test_type, use_internet_connection)? + { + tracing::info!( + ?network, + ?test_type, + "running gRPC query tests using lightwalletd & zebrad...", + ); + + zebrad_and_address + } else { + // Skip the test, we don't have the required cached state + return Ok(()); + }; tracing::info!( ?zebra_rpc_address, - "launching lightwalletd connected to zebrad, waiting for the mempool to activate...", + "launched zebrad, launching lightwalletd connected to zebrad...", ); // Launch lightwalletd - let (lightwalletd, lightwalletd_rpc_port) = spawn_lightwalletd_with_rpc_server( - zebra_rpc_address, - lightwalletd_state_path.clone(), - test_type, - false, - )?; + let (lightwalletd, lightwalletd_rpc_port) = + spawn_lightwalletd_for_rpc(network, test_name, test_type, zebra_rpc_address)? + .expect("already checked cached state and network requirements"); tracing::info!( ?lightwalletd_rpc_port, - "spawned lightwalletd connected to zebrad, waiting for zebrad mempool activation...", + "spawned lightwalletd connected to zebrad, waiting for them both to sync...", ); - // Make sure we are in sync - let (_zebrad, _lightwalletd) = - wait_for_zebrad_and_lightwalletd_tip(lightwalletd, zebrad, test_type)?; - - // Give lightwalletd a few seconds to sync to the tip before connecting to it - tokio::time::sleep(std::time::Duration::from_secs(60)).await; + let (_lightwalletd, _zebrad) = wait_for_zebrad_and_lightwalletd_sync( + lightwalletd, + lightwalletd_rpc_port, + zebrad, + zebra_rpc_address, + test_type, + // We want our queries to include the mempool and network for better coverage + true, + use_internet_connection, + )?; tracing::info!( ?lightwalletd_rpc_port, @@ -143,29 +134,6 @@ pub async fn run() -> Result<()> { // Connect to the lightwalletd instance let mut rpc_client = connect_to_lightwalletd(lightwalletd_rpc_port).await?; - // Check if zebrad and lightwalletd are both in the same height. - - // Get the block tip from lightwalletd - let block_tip = rpc_client - .get_latest_block(ChainSpec {}) - .await? - .into_inner(); - - // Get the block tip from zebrad - let client = reqwest::Client::new(); - let res = client - .post(format!("http://{}", &zebra_rpc_address.to_string())) - .body(r#"{"jsonrpc": "2.0", "method": "getblockchaininfo", "params": [], "id":123 }"#) - .header("Content-Type", "application/json") - .send() - .await? - .text() - .await?; - - // Make sure tip from lightwalletd and from zebrad are the same - let parsed: serde_json::Value = serde_json::from_str(&res)?; - assert_eq!(block_tip.height, parsed["result"]["blocks"]); - // End of the setup and start the tests tracing::info!(?lightwalletd_rpc_port, "sending gRPC queries..."); From 06cbf9582c559d6c97a19c538ef4f85b3d45be94 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 23 Sep 2022 11:05:47 +1000 Subject: [PATCH 08/20] Make zebra_rpc_address optional because only some tests need it --- zebrad/tests/acceptance.rs | 24 ++++++++++++++----- zebrad/tests/common/launch.rs | 8 +++---- .../lightwalletd/send_transaction_test.rs | 12 ++++++---- .../common/lightwalletd/wallet_grpc_test.rs | 12 ++++++---- 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 8f592f85b84..9f207f6acd0 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -1560,9 +1560,13 @@ fn lightwalletd_integration_test(test_type: LightwalletdTestType) -> Result<()> ); // Launch lightwalletd - let (mut lightwalletd, lightwalletd_rpc_port) = - spawn_lightwalletd_for_rpc(network, test_name, test_type, zebra_rpc_address)? - .expect("already checked for lightwalletd cached state and network"); + let (mut lightwalletd, lightwalletd_rpc_port) = spawn_lightwalletd_for_rpc( + network, + test_name, + test_type, + zebra_rpc_address.expect("lightwalletd test must have RPC port"), + )? + .expect("already checked for lightwalletd cached state and network"); tracing::info!( ?lightwalletd_rpc_port, @@ -1639,7 +1643,7 @@ fn lightwalletd_integration_test(test_type: LightwalletdTestType) -> Result<()> lightwalletd, lightwalletd_rpc_port, zebrad, - zebra_rpc_address, + zebra_rpc_address.expect("lightwalletd test must have RPC port"), test_type, // We want to wait for the mempool and network for better coverage true, @@ -1960,14 +1964,22 @@ async fn fully_synced_rpc_test() -> Result<()> { return Ok(()); }; - zebrad.expect_stdout_line_matches(&format!("Opened RPC endpoint at {}", zebra_rpc_address))?; + zebrad.expect_stdout_line_matches(&format!( + "Opened RPC endpoint at {}", + zebra_rpc_address.expect("lightwalletd test must have RPC port"), + ))?; // Make a getblock test that works only on synced node (high block number). // The block is before the mandatory checkpoint, so the checkpoint cached state can be used // if desired. let client = reqwest::Client::new(); let res = client - .post(format!("http://{}", &zebra_rpc_address.to_string())) + .post(format!( + "http://{}", + &zebra_rpc_address + .expect("lightwalletd test must have RPC port") + .to_string() + )) // Manually constructed request to avoid encoding it, for simplicity .body(r#"{"jsonrpc": "2.0", "method": "getblock", "params": ["1180900", 0], "id":123 }"#) .header("Content-Type", "application/json") diff --git a/zebrad/tests/common/launch.rs b/zebrad/tests/common/launch.rs index 17f564f01f8..65854fc0dc4 100644 --- a/zebrad/tests/common/launch.rs +++ b/zebrad/tests/common/launch.rs @@ -207,13 +207,15 @@ where /// - `Ok(Some(zebrad, zebra_rpc_address))` on success, /// - `Ok(None)` if the test doesn't have the required network or cached state, and /// - `Err(_)` if spawning zebrad fails. +/// +/// `zebra_rpc_address` is `None` if the test type doesn't need an RPC port. #[tracing::instrument] pub fn spawn_zebrad_for_rpc + std::fmt::Debug>( network: Network, test_name: S, test_type: LightwalletdTestType, use_internet_connection: bool, -) -> Result, SocketAddr)>> { +) -> Result, Option)>> { let test_name = test_name.as_ref(); // Skip the test unless the user specifically asked for it @@ -246,9 +248,7 @@ pub fn spawn_zebrad_for_rpc + std::fmt::Debug>( .with_timeout(test_type.zebrad_timeout()) .with_failure_regex_iter(zebrad_failure_messages, zebrad_ignore_messages); - let rpc_address = config.rpc.listen_addr.unwrap(); - - Ok(Some((zebrad, rpc_address))) + Ok(Some((zebrad, config.rpc.listen_addr))) } /// Returns `true` if a zebrad test for `test_type` has everything it needs to run. diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index 494580e09f4..838042ffe2b 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -122,9 +122,13 @@ pub async fn run() -> Result<()> { "spawned isolated zebrad with shorter chain, spawning lightwalletd...", ); - let (lightwalletd, lightwalletd_rpc_port) = - spawn_lightwalletd_for_rpc(network, test_name, test_type, zebra_rpc_address)? - .expect("already checked cached state and network requirements"); + let (lightwalletd, lightwalletd_rpc_port) = spawn_lightwalletd_for_rpc( + network, + test_name, + test_type, + zebra_rpc_address.expect("lightwalletd test must have RPC port"), + )? + .expect("already checked cached state and network requirements"); tracing::info!( ?lightwalletd_rpc_port, @@ -135,7 +139,7 @@ pub async fn run() -> Result<()> { lightwalletd, lightwalletd_rpc_port, zebrad, - zebra_rpc_address, + zebra_rpc_address.expect("lightwalletd test must have RPC port"), test_type, // We want to send transactions to the mempool, but we aren't syncing with the network true, diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs index 24a889d65f0..a0ae3afcfe3 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs @@ -106,9 +106,13 @@ pub async fn run() -> Result<()> { ); // Launch lightwalletd - let (lightwalletd, lightwalletd_rpc_port) = - spawn_lightwalletd_for_rpc(network, test_name, test_type, zebra_rpc_address)? - .expect("already checked cached state and network requirements"); + let (lightwalletd, lightwalletd_rpc_port) = spawn_lightwalletd_for_rpc( + network, + test_name, + test_type, + zebra_rpc_address.expect("lightwalletd test must have RPC port"), + )? + .expect("already checked cached state and network requirements"); tracing::info!( ?lightwalletd_rpc_port, @@ -119,7 +123,7 @@ pub async fn run() -> Result<()> { lightwalletd, lightwalletd_rpc_port, zebrad, - zebra_rpc_address, + zebra_rpc_address.expect("lightwalletd test must have RPC port"), test_type, // We want our queries to include the mempool and network for better coverage true, From 8aa7633c648677e1cf4473906d40ceeca8b5477e Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 23 Sep 2022 11:14:46 +1000 Subject: [PATCH 09/20] Check for the zebrad RPC port to open in the right place --- zebrad/tests/acceptance.rs | 10 ++++++++ .../lightwalletd/send_transaction_test.rs | 25 +++++++++++-------- .../tests/common/lightwalletd/wallet_grpc.rs | 8 ------ .../common/lightwalletd/wallet_grpc_test.rs | 25 +++++++++++-------- 4 files changed, 40 insertions(+), 28 deletions(-) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 9f207f6acd0..d41829c5460 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -1554,6 +1554,16 @@ fn lightwalletd_integration_test(test_type: LightwalletdTestType) -> Result<()> // Launch lightwalletd, if needed let lightwalletd_and_port = if test_type.launches_lightwalletd() { + tracing::info!( + ?test_type, + ?zebra_rpc_address, + "waiting for zebrad to open its RPC port..." + ); + zebrad.expect_stdout_line_matches(&format!( + "Opened RPC endpoint at {}", + zebra_rpc_address.expect("lightwalletd test must have RPC port") + ))?; + tracing::info!( ?zebra_rpc_address, "launching lightwalletd connected to zebrad", diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index 838042ffe2b..a9970eced18 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -108,7 +108,7 @@ pub async fn run() -> Result<()> { // Start zebrad with no peers, we want to send transactions without blocks coming in. If `wallet_grpc_test` // runs before this test (as it does in `lightwalletd_test_suite`), then we are the most up to date with tip we can. - let (zebrad, zebra_rpc_address) = if let Some(zebrad_and_address) = + let (mut zebrad, zebra_rpc_address) = if let Some(zebrad_and_address) = spawn_zebrad_for_rpc(network, test_name, test_type, use_internet_connection)? { zebrad_and_address @@ -117,18 +117,23 @@ pub async fn run() -> Result<()> { return Ok(()); }; + let zebra_rpc_address = zebra_rpc_address.expect("lightwalletd test must have RPC port"); + tracing::info!( + ?test_type, ?zebra_rpc_address, - "spawned isolated zebrad with shorter chain, spawning lightwalletd...", + "spawned isolated zebrad with shorter chain, waiting for zebrad to open its RPC port..." ); + zebrad.expect_stdout_line_matches(&format!("Opened RPC endpoint at {}", zebra_rpc_address))?; - let (lightwalletd, lightwalletd_rpc_port) = spawn_lightwalletd_for_rpc( - network, - test_name, - test_type, - zebra_rpc_address.expect("lightwalletd test must have RPC port"), - )? - .expect("already checked cached state and network requirements"); + tracing::info!( + ?zebra_rpc_address, + "zebrad opened its RPC port, spawning lightwalletd...", + ); + + let (lightwalletd, lightwalletd_rpc_port) = + spawn_lightwalletd_for_rpc(network, test_name, test_type, zebra_rpc_address)? + .expect("already checked cached state and network requirements"); tracing::info!( ?lightwalletd_rpc_port, @@ -139,7 +144,7 @@ pub async fn run() -> Result<()> { lightwalletd, lightwalletd_rpc_port, zebrad, - zebra_rpc_address.expect("lightwalletd test must have RPC port"), + zebra_rpc_address, test_type, // We want to send transactions to the mempool, but we aren't syncing with the network true, diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc.rs b/zebrad/tests/common/lightwalletd/wallet_grpc.rs index 1473294fe09..925cb767482 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc.rs @@ -59,14 +59,6 @@ pub fn wait_for_zebrad_and_lightwalletd_sync< // so we need to check that `zebrad` has synced to the tip in parallel. let zebrad_mut = &mut zebrad; let zebrad_wait_fn = || -> Result<_> { - tracing::info!( - ?test_type, - ?zebra_rpc_address, - "waiting for zebrad to open its RPC port..." - ); - zebrad_mut - .expect_stdout_line_matches(&format!("Opened RPC endpoint at {}", zebra_rpc_address))?; - // When we are near the tip, the mempool should activate at least once if wait_for_zebrad_mempool { tracing::info!( diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs index a0ae3afcfe3..a96ca6e11d5 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs @@ -85,7 +85,7 @@ pub async fn run() -> Result<()> { // Launch zebra with peers and using a predefined zebrad state path. // As this tests are just queries we can have a live chain where blocks are coming. - let (zebrad, zebra_rpc_address) = if let Some(zebrad_and_address) = + let (mut zebrad, zebra_rpc_address) = if let Some(zebrad_and_address) = spawn_zebrad_for_rpc(network, test_name, test_type, use_internet_connection)? { tracing::info!( @@ -100,19 +100,24 @@ pub async fn run() -> Result<()> { return Ok(()); }; + let zebra_rpc_address = zebra_rpc_address.expect("lightwalletd test must have RPC port"); + + tracing::info!( + ?test_type, + ?zebra_rpc_address, + "launched zebrad, waiting for zebrad to open its RPC port..." + ); + zebrad.expect_stdout_line_matches(&format!("Opened RPC endpoint at {}", zebra_rpc_address))?; + tracing::info!( ?zebra_rpc_address, - "launched zebrad, launching lightwalletd connected to zebrad...", + "zebrad opened its RPC port, spawning lightwalletd...", ); // Launch lightwalletd - let (lightwalletd, lightwalletd_rpc_port) = spawn_lightwalletd_for_rpc( - network, - test_name, - test_type, - zebra_rpc_address.expect("lightwalletd test must have RPC port"), - )? - .expect("already checked cached state and network requirements"); + let (lightwalletd, lightwalletd_rpc_port) = + spawn_lightwalletd_for_rpc(network, test_name, test_type, zebra_rpc_address)? + .expect("already checked cached state and network requirements"); tracing::info!( ?lightwalletd_rpc_port, @@ -123,7 +128,7 @@ pub async fn run() -> Result<()> { lightwalletd, lightwalletd_rpc_port, zebrad, - zebra_rpc_address.expect("lightwalletd test must have RPC port"), + zebra_rpc_address, test_type, // We want our queries to include the mempool and network for better coverage true, From a55d4861663a9c37556ecc14350137f8aa176a9c Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 23 Sep 2022 11:24:12 +1000 Subject: [PATCH 10/20] Do the quick lightwalletd integration tests first in the sequential test function --- zebrad/tests/acceptance.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index d41829c5460..b7526d1bdbd 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -1482,6 +1482,16 @@ async fn lightwalletd_test_suite() -> Result<()> { // These tests need the compile-time gRPC feature #[cfg(feature = "lightwalletd-grpc-tests")] { + // Do the quick tests first + + // Only runs when LIGHTWALLETD_DATA_DIR and ZEBRA_CACHED_STATE_DIR are set + lightwalletd_integration_test(UpdateCachedState)?; + + // Only runs when LIGHTWALLETD_DATA_DIR and ZEBRA_CACHED_STATE_DIR are set + common::lightwalletd::wallet_grpc_test::run().await?; + + // Then do the slow tests + // Only runs when ZEBRA_CACHED_STATE_DIR is set. // When manually running the test suite, allow cached state in the full sync test. lightwalletd_integration_test(FullSyncFromGenesis { @@ -1489,10 +1499,6 @@ async fn lightwalletd_test_suite() -> Result<()> { })?; // Only runs when LIGHTWALLETD_DATA_DIR and ZEBRA_CACHED_STATE_DIR are set - lightwalletd_integration_test(UpdateCachedState)?; - - // Only runs when LIGHTWALLETD_DATA_DIR and ZEBRA_CACHED_STATE_DIR are set - common::lightwalletd::wallet_grpc_test::run().await?; common::lightwalletd::send_transaction_test::run().await?; } From df07bd3a0fdf1dde1c8a164e658598ea4c5341ec Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 23 Sep 2022 11:35:24 +1000 Subject: [PATCH 11/20] Ignore the lightwalletd cached state env var in tests that don't want it --- zebrad/tests/common/lightwalletd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebrad/tests/common/lightwalletd.rs b/zebrad/tests/common/lightwalletd.rs index cb0454624ca..4f179fa1d08 100644 --- a/zebrad/tests/common/lightwalletd.rs +++ b/zebrad/tests/common/lightwalletd.rs @@ -439,7 +439,7 @@ impl LightwalletdTestType { pub fn lightwalletd_state_path>(&self, test_name: S) -> Option { let test_name = test_name.as_ref(); - if !self.launches_lightwalletd() { + if !self.launches_lightwalletd() || !self.allow_lightwalletd_cached_state() { tracing::info!( "running {test_name:?} {self:?} lightwalletd test, \ ignoring any cached state in the {LIGHTWALLETD_DATA_DIR:?} environment variable", From 02f56e57e67a0543ec813e9ef2f57d5426b05dba Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 23 Sep 2022 11:45:21 +1000 Subject: [PATCH 12/20] Don't replace the state path in RPC tests --- zebrad/tests/common/launch.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebrad/tests/common/launch.rs b/zebrad/tests/common/launch.rs index 65854fc0dc4..3bf9dcd14d5 100644 --- a/zebrad/tests/common/launch.rs +++ b/zebrad/tests/common/launch.rs @@ -242,7 +242,7 @@ pub fn spawn_zebrad_for_rpc + std::fmt::Debug>( // Writes a configuration that has RPC listen_addr set (if needed). // If the state path env var is set, uses it in the config. let zebrad = testdir()? - .with_config(&mut config)? + .with_exact_config(&config)? .spawn_child(args!["start"])? .bypass_test_capture(true) .with_timeout(test_type.zebrad_timeout()) From e27a416e5ec1315fe647b28c12812a00d9674431 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 23 Sep 2022 11:59:57 +1000 Subject: [PATCH 13/20] Enable IO (and timers) on the tip check tokio runtime --- zebrad/tests/common/lightwalletd/wallet_grpc.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc.rs b/zebrad/tests/common/lightwalletd/wallet_grpc.rs index 925cb767482..a384de1458d 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc.rs @@ -149,7 +149,9 @@ pub fn are_zebrad_and_lightwalletd_tips_synced( lightwalletd_rpc_port: u16, zebra_rpc_address: SocketAddr, ) -> Result { - let rt = tokio::runtime::Builder::new_multi_thread().build()?; + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; rt.block_on(async { let mut lightwalletd_grpc_client = connect_to_lightwalletd(lightwalletd_rpc_port).await?; From 95d98c28946b9998683a66f306d81536909d8db2 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 23 Sep 2022 12:19:10 +1000 Subject: [PATCH 14/20] Stop waiting for sync if either waiter thread errors or panics --- .../tests/common/lightwalletd/wallet_grpc.rs | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc.rs b/zebrad/tests/common/lightwalletd/wallet_grpc.rs index a384de1458d..d373212112d 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc.rs @@ -128,15 +128,34 @@ pub fn wait_for_zebrad_and_lightwalletd_sync< // Run both threads in parallel, automatically propagating any panics to this thread. std::thread::scope(|s| { + // Launch the sync-waiting threads + let zebrad_thread = s.spawn(|| { + zebrad_wait_fn().expect("test failed while waiting for zebrad to sync"); + }); + + let lightwalletd_thread = s.spawn(|| { + let lightwalletd_result = lightwalletd_wait_fn(); + is_lightwalletd_finished.store(true, Ordering::SeqCst); + lightwalletd_result.expect("test failed while waiting for lightwalletd to sync"); + }); + + // Mark the sync-waiting threads as finished if they fail or panic. + // This tells the other thread that it can exit. + // + // TODO: use `panic::catch_unwind()` instead, + // when `&mut zebra_test::command::TestChild` is unwind-safe s.spawn(|| { - let zebrad_result = zebrad_wait_fn(); + let zebrad_result = zebrad_thread.join(); is_zebrad_finished.store(true, Ordering::SeqCst); - zebrad_result.expect("test failed while waiting for zebrad to sync"); + + zebrad_result.expect("test panicked or failed while waiting for zebrad to sync"); }); s.spawn(|| { - let lightwalletd_result = lightwalletd_wait_fn(); + let lightwalletd_result = lightwalletd_thread.join(); is_lightwalletd_finished.store(true, Ordering::SeqCst); - lightwalletd_result.expect("test failed while waiting for lightwalletd to sync"); + + lightwalletd_result + .expect("test panicked or failed while waiting for lightwalletd to sync"); }); }); From 3e2d5dfc0f2bbc31e577e00171f0732ed0d98032 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 26 Sep 2022 13:11:36 +1000 Subject: [PATCH 15/20] Try to speed up slow lightwalletd full syncs --- zebrad/tests/common/lightwalletd.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zebrad/tests/common/lightwalletd.rs b/zebrad/tests/common/lightwalletd.rs index 4f179fa1d08..594aa421e43 100644 --- a/zebrad/tests/common/lightwalletd.rs +++ b/zebrad/tests/common/lightwalletd.rs @@ -420,6 +420,12 @@ impl LightwalletdTestType { // except when we're doing the quick empty state test config.consensus.debug_skip_parameter_preload = !self.needs_zebra_cached_state(); + // We want to run multi-threaded RPCs, if we're using them + if self.launches_lightwalletd() { + // Automatically runs one thread per available CPU core + config.rpc.parallel_cpu_threads = 0; + } + if !self.needs_zebra_cached_state() { return Some(Ok(config)); } From f61ed5d755045aa0c8e1d9f04f64358839f02484 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 26 Sep 2022 13:30:11 +1000 Subject: [PATCH 16/20] Don't wait for the tip in send transaction tests, and try to speed up full lightwalletd syncs --- zebrad/tests/acceptance.rs | 2 +- zebrad/tests/common/lightwalletd.rs | 2 + .../lightwalletd/send_transaction_test.rs | 5 +- zebrad/tests/common/lightwalletd/sync.rs | 206 ++++++++++++++++++ .../tests/common/lightwalletd/wallet_grpc.rs | 184 +--------------- .../common/lightwalletd/wallet_grpc_test.rs | 6 +- zebrad/tests/common/sync.rs | 6 + 7 files changed, 221 insertions(+), 190 deletions(-) create mode 100644 zebrad/tests/common/lightwalletd/sync.rs diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index b7526d1bdbd..8eae9b988a0 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -1648,7 +1648,7 @@ fn lightwalletd_integration_test(test_type: LightwalletdTestType) -> Result<()> if let Some((lightwalletd, lightwalletd_rpc_port)) = lightwalletd_and_port { #[cfg(feature = "lightwalletd-grpc-tests")] { - use common::lightwalletd::wallet_grpc::wait_for_zebrad_and_lightwalletd_sync; + use common::lightwalletd::sync::wait_for_zebrad_and_lightwalletd_sync; tracing::info!( ?lightwalletd_rpc_port, diff --git a/zebrad/tests/common/lightwalletd.rs b/zebrad/tests/common/lightwalletd.rs index 594aa421e43..9178eb7fc00 100644 --- a/zebrad/tests/common/lightwalletd.rs +++ b/zebrad/tests/common/lightwalletd.rs @@ -41,6 +41,8 @@ use LightwalletdTestType::*; #[cfg(feature = "lightwalletd-grpc-tests")] pub mod send_transaction_test; #[cfg(feature = "lightwalletd-grpc-tests")] +pub mod sync; +#[cfg(feature = "lightwalletd-grpc-tests")] pub mod wallet_grpc; #[cfg(feature = "lightwalletd-grpc-tests")] pub mod wallet_grpc_test; diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index a9970eced18..c60368f7a99 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -39,9 +39,8 @@ use crate::common::{ launch::{can_spawn_zebrad_for_rpc, spawn_zebrad_for_rpc}, lightwalletd::{ can_spawn_lightwalletd_for_rpc, spawn_lightwalletd_for_rpc, - wallet_grpc::{ - self, connect_to_lightwalletd, wait_for_zebrad_and_lightwalletd_sync, Empty, Exclude, - }, + sync::wait_for_zebrad_and_lightwalletd_sync, + wallet_grpc::{self, connect_to_lightwalletd, Empty, Exclude}, LightwalletdTestType::*, }, sync::copy_state_and_perform_full_sync, diff --git a/zebrad/tests/common/lightwalletd/sync.rs b/zebrad/tests/common/lightwalletd/sync.rs new file mode 100644 index 00000000000..aee01779367 --- /dev/null +++ b/zebrad/tests/common/lightwalletd/sync.rs @@ -0,0 +1,206 @@ +//! Lightwalletd sync functions. + +use std::{ + net::SocketAddr, + sync::atomic::{AtomicBool, Ordering}, + time::{Duration, Instant}, +}; + +use tempfile::TempDir; + +use zebra_test::prelude::*; + +use crate::common::{ + launch::ZebradTestDirExt, + lightwalletd::{ + wallet_grpc::{connect_to_lightwalletd, ChainSpec}, + LightwalletdTestType, + }, +}; + +/// The amount of time we wait between each tip check. +pub const TIP_CHECK_RATE_LIMIT: Duration = Duration::from_secs(60); + +/// Wait for lightwalletd to sync to Zebra's tip. +/// +/// If `wait_for_zebrad_mempool` is `true`, wait for Zebra to activate its mempool. +/// If `wait_for_zebrad_tip` is `true`, also wait for Zebra to sync to the network chain tip. +#[tracing::instrument] +pub fn wait_for_zebrad_and_lightwalletd_sync< + P: ZebradTestDirExt + std::fmt::Debug + std::marker::Send + 'static, +>( + mut lightwalletd: TestChild, + lightwalletd_rpc_port: u16, + mut zebrad: TestChild

, + zebra_rpc_address: SocketAddr, + test_type: LightwalletdTestType, + wait_for_zebrad_mempool: bool, + wait_for_zebrad_tip: bool, +) -> Result<(TestChild, TestChild

)> { + let is_zebrad_finished = AtomicBool::new(false); + let is_lightwalletd_finished = AtomicBool::new(false); + + let is_zebrad_finished = &is_zebrad_finished; + let is_lightwalletd_finished = &is_lightwalletd_finished; + + // TODO: split these closures out into their own functions + + // Check Zebra's logs for errors. + // Optionally waits until Zebra has synced to the tip, based on `wait_for_zebrad_tip`. + // + // `lightwalletd` syncs can take a long time, + // so we need to check that `zebrad` has synced to the tip in parallel. + let zebrad_mut = &mut zebrad; + let zebrad_wait_fn = || -> Result<_> { + // When we are near the tip, the mempool should activate at least once + if wait_for_zebrad_mempool { + tracing::info!( + ?test_type, + "waiting for zebrad to activate the mempool when it gets near the tip..." + ); + zebrad_mut.expect_stdout_line_matches("activating mempool")?; + } + + // When we are near the tip, this message is logged multiple times + if wait_for_zebrad_tip { + tracing::info!(?test_type, "waiting for zebrad to sync to the tip..."); + zebrad_mut.expect_stdout_line_matches(crate::common::sync::SYNC_FINISHED_REGEX)?; + } + + // Tell the other thread that Zebra has finished + is_zebrad_finished.store(true, Ordering::SeqCst); + + tracing::info!( + ?test_type, + "zebrad is waiting for lightwalletd to sync to the tip..." + ); + while !is_lightwalletd_finished.load(Ordering::SeqCst) { + // Just keep checking the Zebra logs for errors... + if wait_for_zebrad_tip { + // Make sure the sync is still finished, this is logged every minute or so. + zebrad_mut.expect_stdout_line_matches(crate::common::sync::SYNC_FINISHED_REGEX)?; + } else { + // Use sync progress logs, which are logged every minute or so. + zebrad_mut.expect_stdout_line_matches(crate::common::sync::SYNC_PROGRESS_REGEX)?; + } + } + + Ok(zebrad_mut) + }; + + // Wait until `lightwalletd` has synced to Zebra's tip. + // Calls `lightwalletd`'s gRPCs and Zebra's JSON-RPCs. + // Also checks `lightwalletd`'s logs for errors. + // + // `zebrad` syncs can take a long time, + // so we need to check that `lightwalletd` has synced to the tip in parallel. + let lightwalletd_mut = &mut lightwalletd; + let lightwalletd_wait_fn = || -> Result<_> { + tracing::info!( + ?test_type, + "lightwalletd is waiting for zebrad to sync to the tip..." + ); + while !is_zebrad_finished.load(Ordering::SeqCst) { + // Just keep checking the `lightwalletd` logs for errors. + // It usually logs something every 30-90 seconds, + // but there's no specific message we need to wait for here. + assert!( + lightwalletd_mut.wait_for_stdout_line(None), + "lightwalletd output unexpectedly finished early", + ); + } + + tracing::info!(?test_type, "waiting for lightwalletd to sync to the tip..."); + while !are_zebrad_and_lightwalletd_tips_synced(lightwalletd_rpc_port, zebra_rpc_address)? { + let previous_check = Instant::now(); + + // To improve performance, only check the tips occasionally + while previous_check.elapsed() < TIP_CHECK_RATE_LIMIT { + assert!( + lightwalletd_mut.wait_for_stdout_line(None), + "lightwalletd output unexpectedly finished early", + ); + } + } + + // Tell the other thread that `lightwalletd` has finished + is_lightwalletd_finished.store(true, Ordering::SeqCst); + + Ok(lightwalletd_mut) + }; + + // Run both threads in parallel, automatically propagating any panics to this thread. + std::thread::scope(|s| { + // Launch the sync-waiting threads + let zebrad_thread = s.spawn(|| { + zebrad_wait_fn().expect("test failed while waiting for zebrad to sync"); + }); + + let lightwalletd_thread = s.spawn(|| { + let lightwalletd_result = lightwalletd_wait_fn(); + is_lightwalletd_finished.store(true, Ordering::SeqCst); + lightwalletd_result.expect("test failed while waiting for lightwalletd to sync"); + }); + + // Mark the sync-waiting threads as finished if they fail or panic. + // This tells the other thread that it can exit. + // + // TODO: use `panic::catch_unwind()` instead, + // when `&mut zebra_test::command::TestChild` is unwind-safe + s.spawn(|| { + let zebrad_result = zebrad_thread.join(); + is_zebrad_finished.store(true, Ordering::SeqCst); + + zebrad_result.expect("test panicked or failed while waiting for zebrad to sync"); + }); + s.spawn(|| { + let lightwalletd_result = lightwalletd_thread.join(); + is_lightwalletd_finished.store(true, Ordering::SeqCst); + + lightwalletd_result + .expect("test panicked or failed while waiting for lightwalletd to sync"); + }); + }); + + Ok((lightwalletd, zebrad)) +} + +/// Returns `Ok(true)` if zebrad and lightwalletd are both at the same height. +#[tracing::instrument] +pub fn are_zebrad_and_lightwalletd_tips_synced( + lightwalletd_rpc_port: u16, + zebra_rpc_address: SocketAddr, +) -> Result { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + + rt.block_on(async { + let mut lightwalletd_grpc_client = connect_to_lightwalletd(lightwalletd_rpc_port).await?; + + // Get the block tip from lightwalletd + let lightwalletd_tip_block = lightwalletd_grpc_client + .get_latest_block(ChainSpec {}) + .await? + .into_inner(); + let lightwalletd_tip_height = lightwalletd_tip_block.height; + + // Get the block tip from zebrad + let zebrad_json_rpc_client = reqwest::Client::new(); + let zebrad_blockchain_info = zebrad_json_rpc_client + .post(format!("http://{}", &zebra_rpc_address.to_string())) + .body(r#"{"jsonrpc": "2.0", "method": "getblockchaininfo", "params": [], "id":123 }"#) + .header("Content-Type", "application/json") + .send() + .await? + .text() + .await?; + let zebrad_blockchain_info: serde_json::Value = + serde_json::from_str(&zebrad_blockchain_info)?; + let zebrad_tip_height = zebrad_blockchain_info["result"]["blocks"] + .as_u64() + .expect("unexpected block height: doesn't fit in u64"); + + Ok(lightwalletd_tip_height == zebrad_tip_height) + }) +} diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc.rs b/zebrad/tests/common/lightwalletd/wallet_grpc.rs index d373212112d..fc80fd6f65f 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc.rs @@ -1,17 +1,9 @@ //! Lightwalletd gRPC interface and utility functions. -use std::{ - env, - net::SocketAddr, - sync::atomic::{AtomicBool, Ordering}, -}; - -use tempfile::TempDir; +use std::env; use zebra_test::prelude::*; -use crate::common::{launch::ZebradTestDirExt, lightwalletd::LightwalletdTestType}; - tonic::include_proto!("cash.z.wallet.sdk.rpc"); /// Type alias for the RPC client to communicate with a lightwalletd instance. @@ -27,177 +19,3 @@ pub async fn connect_to_lightwalletd(lightwalletd_rpc_port: u16) -> Result( - mut lightwalletd: TestChild, - lightwalletd_rpc_port: u16, - mut zebrad: TestChild

, - zebra_rpc_address: SocketAddr, - test_type: LightwalletdTestType, - wait_for_zebrad_mempool: bool, - wait_for_zebrad_tip: bool, -) -> Result<(TestChild, TestChild

)> { - let is_zebrad_finished = AtomicBool::new(false); - let is_lightwalletd_finished = AtomicBool::new(false); - - let is_zebrad_finished = &is_zebrad_finished; - let is_lightwalletd_finished = &is_lightwalletd_finished; - - // TODO: split these closures out into their own functions - - // Check Zebra's logs for errors. - // Optionally waits until Zebra has synced to the tip, based on `wait_for_zebrad_tip`. - // - // `lightwalletd` syncs can take a long time, - // so we need to check that `zebrad` has synced to the tip in parallel. - let zebrad_mut = &mut zebrad; - let zebrad_wait_fn = || -> Result<_> { - // When we are near the tip, the mempool should activate at least once - if wait_for_zebrad_mempool { - tracing::info!( - ?test_type, - "waiting for zebrad to activate the mempool when it gets near the tip..." - ); - zebrad_mut.expect_stdout_line_matches("activating mempool")?; - } - - // When we are near the tip, this message is logged multiple times - if wait_for_zebrad_tip { - tracing::info!(?test_type, "waiting for zebrad to sync to the tip..."); - zebrad_mut.expect_stdout_line_matches(crate::common::sync::SYNC_FINISHED_REGEX)?; - } - - // Tell the other thread that Zebra has finished - is_zebrad_finished.store(true, Ordering::SeqCst); - - tracing::info!( - ?test_type, - "zebrad is waiting for lightwalletd to sync to the tip..." - ); - while !is_lightwalletd_finished.load(Ordering::SeqCst) { - // Just keep checking the Zebra logs for errors, - // using the sync finished regex, which is logged every minute or so. - zebrad_mut.expect_stdout_line_matches(crate::common::sync::SYNC_FINISHED_REGEX)?; - } - - Ok(zebrad_mut) - }; - - // Wait until `lightwalletd` has synced to Zebra's tip. - // Calls `lightwalletd`'s gRPCs and Zebra's JSON-RPCs. - // Also checks `lightwalletd`'s logs for errors. - // - // `zebrad` syncs can take a long time, - // so we need to check that `lightwalletd` has synced to the tip in parallel. - let lightwalletd_mut = &mut lightwalletd; - let lightwalletd_wait_fn = || -> Result<_> { - tracing::info!( - ?test_type, - "lightwalletd is waiting for zebrad to sync to the tip..." - ); - while !is_zebrad_finished.load(Ordering::SeqCst) { - // Just keep checking the `lightwalletd` logs for errors. - // It usually logs something every 30-90 seconds, - // but there's no specific message we need to wait for here. - assert!( - lightwalletd_mut.wait_for_stdout_line(None), - "lightwalletd output unexpectedly finished early", - ); - } - - tracing::info!(?test_type, "waiting for lightwalletd to sync to the tip..."); - while !are_zebrad_and_lightwalletd_tips_synced(lightwalletd_rpc_port, zebra_rpc_address)? { - assert!( - lightwalletd_mut.wait_for_stdout_line(None), - "lightwalletd output unexpectedly finished early", - ); - } - - // Tell the other thread that `lightwalletd` has finished - is_lightwalletd_finished.store(true, Ordering::SeqCst); - - Ok(lightwalletd_mut) - }; - - // Run both threads in parallel, automatically propagating any panics to this thread. - std::thread::scope(|s| { - // Launch the sync-waiting threads - let zebrad_thread = s.spawn(|| { - zebrad_wait_fn().expect("test failed while waiting for zebrad to sync"); - }); - - let lightwalletd_thread = s.spawn(|| { - let lightwalletd_result = lightwalletd_wait_fn(); - is_lightwalletd_finished.store(true, Ordering::SeqCst); - lightwalletd_result.expect("test failed while waiting for lightwalletd to sync"); - }); - - // Mark the sync-waiting threads as finished if they fail or panic. - // This tells the other thread that it can exit. - // - // TODO: use `panic::catch_unwind()` instead, - // when `&mut zebra_test::command::TestChild` is unwind-safe - s.spawn(|| { - let zebrad_result = zebrad_thread.join(); - is_zebrad_finished.store(true, Ordering::SeqCst); - - zebrad_result.expect("test panicked or failed while waiting for zebrad to sync"); - }); - s.spawn(|| { - let lightwalletd_result = lightwalletd_thread.join(); - is_lightwalletd_finished.store(true, Ordering::SeqCst); - - lightwalletd_result - .expect("test panicked or failed while waiting for lightwalletd to sync"); - }); - }); - - Ok((lightwalletd, zebrad)) -} - -/// Returns `Ok(true)` if zebrad and lightwalletd are both at the same height. -#[tracing::instrument] -pub fn are_zebrad_and_lightwalletd_tips_synced( - lightwalletd_rpc_port: u16, - zebra_rpc_address: SocketAddr, -) -> Result { - let rt = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build()?; - - rt.block_on(async { - let mut lightwalletd_grpc_client = connect_to_lightwalletd(lightwalletd_rpc_port).await?; - - // Get the block tip from lightwalletd - let lightwalletd_tip_block = lightwalletd_grpc_client - .get_latest_block(ChainSpec {}) - .await? - .into_inner(); - let lightwalletd_tip_height = lightwalletd_tip_block.height; - - // Get the block tip from zebrad - let zebrad_json_rpc_client = reqwest::Client::new(); - let zebrad_blockchain_info = zebrad_json_rpc_client - .post(format!("http://{}", &zebra_rpc_address.to_string())) - .body(r#"{"jsonrpc": "2.0", "method": "getblockchaininfo", "params": [], "id":123 }"#) - .header("Content-Type", "application/json") - .send() - .await? - .text() - .await?; - let zebrad_blockchain_info: serde_json::Value = - serde_json::from_str(&zebrad_blockchain_info)?; - let zebrad_tip_height = zebrad_blockchain_info["result"]["blocks"] - .as_u64() - .expect("unexpected block height: doesn't fit in u64"); - - Ok(lightwalletd_tip_height == zebrad_tip_height) - }) -} diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs index a96ca6e11d5..23ec3d776ae 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs @@ -49,10 +49,10 @@ use crate::common::{ launch::spawn_zebrad_for_rpc, lightwalletd::{ can_spawn_lightwalletd_for_rpc, spawn_lightwalletd_for_rpc, + sync::wait_for_zebrad_and_lightwalletd_sync, wallet_grpc::{ - connect_to_lightwalletd, wait_for_zebrad_and_lightwalletd_sync, Address, AddressList, - BlockId, BlockRange, ChainSpec, Empty, GetAddressUtxosArg, - TransparentAddressBlockFilter, TxFilter, + connect_to_lightwalletd, Address, AddressList, BlockId, BlockRange, ChainSpec, Empty, + GetAddressUtxosArg, TransparentAddressBlockFilter, TxFilter, }, LightwalletdTestType::UpdateCachedState, }, diff --git a/zebrad/tests/common/sync.rs b/zebrad/tests/common/sync.rs index 1b9ac172fad..0886a3e5c9b 100644 --- a/zebrad/tests/common/sync.rs +++ b/zebrad/tests/common/sync.rs @@ -45,6 +45,12 @@ pub const STOP_AT_HEIGHT_REGEX: &str = "stopping at configured height"; pub const SYNC_FINISHED_REGEX: &str = r"finished initial sync to chain tip, using gossiped blocks .*sync_percent.*=.*100\."; +/// The text that should be logged every time Zebra checks the sync progress. +// +// This is only used with `--feature lightwalletd-grpc-tests` +#[allow(dead_code)] +pub const SYNC_PROGRESS_REGEX: &str = r"sync_percent"; + /// The maximum amount of time Zebra should take to reload after shutting down. /// /// This should only take a second, but sometimes CI VMs or RocksDB can be slow. From 67cfd6fea2293eebd4e0c77051ae60ba60a694d6 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 27 Sep 2022 06:50:39 +1000 Subject: [PATCH 17/20] Remove redundant is_lightwalletd_finished store Co-authored-by: Arya --- zebrad/tests/common/lightwalletd/sync.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/zebrad/tests/common/lightwalletd/sync.rs b/zebrad/tests/common/lightwalletd/sync.rs index aee01779367..5808ed2be14 100644 --- a/zebrad/tests/common/lightwalletd/sync.rs +++ b/zebrad/tests/common/lightwalletd/sync.rs @@ -137,9 +137,7 @@ pub fn wait_for_zebrad_and_lightwalletd_sync< }); let lightwalletd_thread = s.spawn(|| { - let lightwalletd_result = lightwalletd_wait_fn(); - is_lightwalletd_finished.store(true, Ordering::SeqCst); - lightwalletd_result.expect("test failed while waiting for lightwalletd to sync"); + lightwalletd_wait_fn().expect("test failed while waiting for lightwalletd to sync."); }); // Mark the sync-waiting threads as finished if they fail or panic. From 3bd2c412ac171c2da3ce66457398919f2671ae35 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 29 Sep 2022 12:34:15 +1000 Subject: [PATCH 18/20] Fix unused variable error --- zebrad/tests/common/lightwalletd/send_transaction_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index c60368f7a99..18165a20139 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -139,7 +139,7 @@ pub async fn run() -> Result<()> { "spawned lightwalletd connected to zebrad, waiting for them both to sync...", ); - let (_lightwalletd, mut zebrad) = wait_for_zebrad_and_lightwalletd_sync( + let (_lightwalletd, _zebrad) = wait_for_zebrad_and_lightwalletd_sync( lightwalletd, lightwalletd_rpc_port, zebrad, From 0e2d62f049c41babb3ce93aedafec5a7b9465795 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 3 Oct 2022 07:46:04 +1000 Subject: [PATCH 19/20] Actually create the lightwalletd cached state --- zebrad/tests/common/lightwalletd.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/zebrad/tests/common/lightwalletd.rs b/zebrad/tests/common/lightwalletd.rs index 9178eb7fc00..13a33fc9beb 100644 --- a/zebrad/tests/common/lightwalletd.rs +++ b/zebrad/tests/common/lightwalletd.rs @@ -384,6 +384,15 @@ impl LightwalletdTestType { } } + /// Can this test create a new `LIGHTWALLETD_DATA_DIR` cached state? + pub fn can_create_lightwalletd_cached_state(&self) -> bool { + match self { + LaunchWithEmptyState => false, + FullSyncFromGenesis { .. } | UpdateCachedState => true, + UpdateZebraCachedStateNoRpc => false, + } + } + /// Returns the Zebra state path for this test, if set. #[allow(clippy::print_stderr)] pub fn zebrad_state_path>(&self, test_name: S) -> Option { @@ -447,7 +456,7 @@ impl LightwalletdTestType { pub fn lightwalletd_state_path>(&self, test_name: S) -> Option { let test_name = test_name.as_ref(); - if !self.launches_lightwalletd() || !self.allow_lightwalletd_cached_state() { + if !self.launches_lightwalletd() || !self.can_create_lightwalletd_cached_state() { tracing::info!( "running {test_name:?} {self:?} lightwalletd test, \ ignoring any cached state in the {LIGHTWALLETD_DATA_DIR:?} environment variable", From ef6f7230b990a4ca78b426dbb89b2930e8fd1055 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 3 Oct 2022 07:54:23 +1000 Subject: [PATCH 20/20] Fix lwd cache check logic --- zebrad/tests/common/lightwalletd.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zebrad/tests/common/lightwalletd.rs b/zebrad/tests/common/lightwalletd.rs index 13a33fc9beb..d1301aa9742 100644 --- a/zebrad/tests/common/lightwalletd.rs +++ b/zebrad/tests/common/lightwalletd.rs @@ -456,7 +456,11 @@ impl LightwalletdTestType { pub fn lightwalletd_state_path>(&self, test_name: S) -> Option { let test_name = test_name.as_ref(); - if !self.launches_lightwalletd() || !self.can_create_lightwalletd_cached_state() { + // Can this test type use a lwd cached state, or create/update one? + let use_or_create_lwd_cache = + self.allow_lightwalletd_cached_state() || self.can_create_lightwalletd_cached_state(); + + if !self.launches_lightwalletd() || !use_or_create_lwd_cache { tracing::info!( "running {test_name:?} {self:?} lightwalletd test, \ ignoring any cached state in the {LIGHTWALLETD_DATA_DIR:?} environment variable",