From 7cbfec29d67df6005165aac4e68a668c3194c15d Mon Sep 17 00:00:00 2001 From: Brian Cho Date: Thu, 22 Sep 2022 15:34:41 -0700 Subject: [PATCH 1/3] [smoke tests] local swarm build retries 3 times --- .../smoke-test/src/smoke_test_environment.rs | 58 +++++++++---------- testsuite/smoke-test/src/storage.rs | 8 +-- testsuite/smoke-test/src/txn_broadcast.rs | 23 +++----- 3 files changed, 36 insertions(+), 53 deletions(-) diff --git a/testsuite/smoke-test/src/smoke_test_environment.rs b/testsuite/smoke-test/src/smoke_test_environment.rs index 0b03a1c5592cf..ac9b8917ecc7a 100644 --- a/testsuite/smoke-test/src/smoke_test_environment.rs +++ b/testsuite/smoke-test/src/smoke_test_environment.rs @@ -14,10 +14,12 @@ use forge::{Factory, LocalFactory, LocalSwarm}; use framework::ReleaseBundle; use once_cell::sync::Lazy; use rand::rngs::OsRng; -use std::pin::Pin; use std::{num::NonZeroUsize, path::PathBuf, sync::Arc}; use tokio::task::JoinHandle; +const SWARM_BUILD_NUM_RETRIES: u8 = 3; + +#[derive(Clone)] pub struct SwarmBuilder { local: bool, num_validators: NonZeroUsize, @@ -64,7 +66,7 @@ impl SwarmBuilder { } // Gas is not enabled with this setup, it's enabled via forge instance. - pub async fn build_wrapped(self) -> anyhow::Result { + pub async fn build_inner(self) -> anyhow::Result { ::aptos_logger::Logger::new().init(); info!("Preparing to finish compiling"); // TODO change to return Swarm trait @@ -79,16 +81,16 @@ impl SwarmBuilder { static ACTIVE_NODES: Lazy>> = Lazy::new(|| Arc::new(Mutex::new(0))); let guard = ActiveNodesGuard::grab(slots, ACTIVE_NODES.clone()).await; - let init_genesis_config = self.init_genesis_config; - + let builder = self.clone(); + let init_genesis_config = builder.init_genesis_config; FACTORY .new_swarm_with_version( OsRng, - self.num_validators, - self.num_fullnodes, + builder.num_validators, + builder.num_fullnodes, &version, - self.genesis_framework, - self.init_config, + builder.genesis_framework, + builder.init_config, Some(Arc::new(move |genesis_config| { if let Some(init_genesis_config) = &init_genesis_config { (init_genesis_config)(genesis_config); @@ -100,12 +102,25 @@ impl SwarmBuilder { } // Gas is not enabled with this setup, it's enabled via forge instance. - pub async fn build(self) -> LocalSwarm { - self.build_wrapped().await.unwrap() + pub async fn build(&mut self) -> LocalSwarm { + let num_retries = SWARM_BUILD_NUM_RETRIES; + let mut attempt = 0; + loop { + if attempt > num_retries { + panic!("Exhausted retries: {} / {}", attempt, num_retries); + } + match self.build_inner().await { + Ok(swarm) => { + return swarm; + } + Err(err) => warn!("Attempt {} / {} failed with: {}", attempt, num_retries, err), + } + attempt += 1; + } } pub async fn build_with_cli( - self, + &mut self, num_cli_accounts: usize, ) -> (LocalSwarm, CliTestFramework, JoinHandle<()>) { let swarm = self.build().await; @@ -136,27 +151,6 @@ impl SwarmBuilder { } } -pub async fn with_retry< - F: Fn() -> Pin>>>, ->( - build: F, - num_retries: u8, -) -> LocalSwarm { - let mut attempt = 0; - loop { - if attempt >= num_retries { - panic!("Exhausted retries: {} / {}", attempt, num_retries); - } - match build().await { - Ok(swarm) => { - return swarm; - } - Err(err) => warn!("Attempt {} / {} failed with: {}", attempt, num_retries, err), - } - attempt += 1; - } -} - // Gas is not enabled with this setup, it's enabled via forge instance. pub async fn new_local_swarm_with_aptos(num_validators: usize) -> LocalSwarm { SwarmBuilder::new_local(num_validators) diff --git a/testsuite/smoke-test/src/storage.rs b/testsuite/smoke-test/src/storage.rs index e231d715a91cb..4d373bc88c4a4 100644 --- a/testsuite/smoke-test/src/storage.rs +++ b/testsuite/smoke-test/src/storage.rs @@ -1,7 +1,7 @@ // Copyright (c) Aptos // SPDX-License-Identifier: Apache-2.0 -use crate::smoke_test_environment::{with_retry, SwarmBuilder}; +use crate::smoke_test_environment::SwarmBuilder; use crate::{ test_utils::{ assert_balance, create_and_fund_account, swarm_utils::insert_waypoint, @@ -35,11 +35,7 @@ async fn test_db_restore() { workspace_builder::get_bin("db-backup-verify"); info!("---------- 1. pre-building finished."); - let mut swarm = with_retry( - || Box::pin(SwarmBuilder::new_local(4).with_aptos().build_wrapped()), - 3, - ) - .await; + let mut swarm = SwarmBuilder::new_local(4).with_aptos().build().await; info!("---------- 1.1 swarm built, sending some transactions."); let validator_peer_ids = swarm.validators().map(|v| v.peer_id()).collect::>(); let client_1 = swarm diff --git a/testsuite/smoke-test/src/txn_broadcast.rs b/testsuite/smoke-test/src/txn_broadcast.rs index f3e33f3363cca..911633b7e49ae 100644 --- a/testsuite/smoke-test/src/txn_broadcast.rs +++ b/testsuite/smoke-test/src/txn_broadcast.rs @@ -1,7 +1,7 @@ // Copyright (c) Aptos // SPDX-License-Identifier: Apache-2.0 -use crate::smoke_test_environment::{with_retry, SwarmBuilder}; +use crate::smoke_test_environment::SwarmBuilder; use crate::test_utils::{assert_balance, create_and_fund_account, transfer_coins}; use aptos_config::config::NodeConfig; use forge::{NodeExt, Swarm, SwarmExt}; @@ -14,20 +14,13 @@ const MAX_WAIT_SECS: u64 = 60; /// This behavior should be true with both mempool and quorum store. #[tokio::test] async fn test_txn_broadcast() { - let mut swarm = with_retry( - || { - Box::pin( - SwarmBuilder::new_local(4) - .with_aptos() - .with_init_config(Arc::new(|_, conf, _| { - conf.api.failpoints_enabled = true; - })) - .build_wrapped(), - ) - }, - 3, - ) - .await; + let mut swarm = SwarmBuilder::new_local(4) + .with_aptos() + .with_init_config(Arc::new(|_, conf, _| { + conf.api.failpoints_enabled = true; + })) + .build() + .await; let transaction_factory = swarm.chain_info().transaction_factory(); let version = swarm.versions().max().unwrap(); let validator_peer_ids = swarm.validators().map(|v| v.peer_id()).collect::>(); From d63cfbf10aaecc3ba2345e476c795df41509b092 Mon Sep 17 00:00:00 2001 From: Brian Cho Date: Thu, 22 Sep 2022 15:40:28 -0700 Subject: [PATCH 2/3] comment --- testsuite/smoke-test/src/smoke_test_environment.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/testsuite/smoke-test/src/smoke_test_environment.rs b/testsuite/smoke-test/src/smoke_test_environment.rs index ac9b8917ecc7a..ab0aade96aebc 100644 --- a/testsuite/smoke-test/src/smoke_test_environment.rs +++ b/testsuite/smoke-test/src/smoke_test_environment.rs @@ -102,6 +102,7 @@ impl SwarmBuilder { } // Gas is not enabled with this setup, it's enabled via forge instance. + // Local swarm spin-up can fail due to port issues. So we retry SWARM_BUILD_NUM_RETRIES times. pub async fn build(&mut self) -> LocalSwarm { let num_retries = SWARM_BUILD_NUM_RETRIES; let mut attempt = 0; From bcc3379c2a31be62bd87768624f3a186cc4b8ed0 Mon Sep 17 00:00:00 2001 From: Brian Cho Date: Fri, 23 Sep 2022 09:12:52 -0700 Subject: [PATCH 3/3] fix --- testsuite/smoke-test/src/smoke_test_environment.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/smoke-test/src/smoke_test_environment.rs b/testsuite/smoke-test/src/smoke_test_environment.rs index ab0aade96aebc..1c98068fd351c 100644 --- a/testsuite/smoke-test/src/smoke_test_environment.rs +++ b/testsuite/smoke-test/src/smoke_test_environment.rs @@ -66,7 +66,7 @@ impl SwarmBuilder { } // Gas is not enabled with this setup, it's enabled via forge instance. - pub async fn build_inner(self) -> anyhow::Result { + pub async fn build_inner(&mut self) -> anyhow::Result { ::aptos_logger::Logger::new().init(); info!("Preparing to finish compiling"); // TODO change to return Swarm trait