[smoke tests] local swarm build retries 3 times

testsuite/smoke-test/src/smoke_test_environment.rs

@@ -14,10 +14,12 @@ use forge::{Factory, LocalFactory, LocalSwarm};
 use framework::ReleaseBundle;
 use once_cell::sync::Lazy;
 use rand::rngs::OsRng;
-use std::pin::Pin;
 use std::{num::NonZeroUsize, path::PathBuf, sync::Arc};
 use tokio::task::JoinHandle;
 
+const SWARM_BUILD_NUM_RETRIES: u8 = 3;
+
+#[derive(Clone)]
 pub struct SwarmBuilder {
     local: bool,
     num_validators: NonZeroUsize,
@@ -64,7 +66,7 @@ impl SwarmBuilder {
     }
 
     // Gas is not enabled with this setup, it's enabled via forge instance.
-    pub async fn build_wrapped(self) -> anyhow::Result<LocalSwarm> {
+    pub async fn build_inner(&mut self) -> anyhow::Result<LocalSwarm> {
         ::aptos_logger::Logger::new().init();
         info!("Preparing to finish compiling");
         // TODO change to return Swarm trait
@@ -79,16 +81,16 @@ impl SwarmBuilder {
         static ACTIVE_NODES: Lazy<Arc<Mutex<usize>>> = Lazy::new(|| Arc::new(Mutex::new(0)));
         let guard = ActiveNodesGuard::grab(slots, ACTIVE_NODES.clone()).await;
 
-        let init_genesis_config = self.init_genesis_config;
-
+        let builder = self.clone();
+        let init_genesis_config = builder.init_genesis_config;
         FACTORY
             .new_swarm_with_version(
                 OsRng,
-                self.num_validators,
-                self.num_fullnodes,
+                builder.num_validators,
+                builder.num_fullnodes,
                 &version,
-                self.genesis_framework,
-                self.init_config,
+                builder.genesis_framework,
+                builder.init_config,
                 Some(Arc::new(move |genesis_config| {
                     if let Some(init_genesis_config) = &init_genesis_config {
                         (init_genesis_config)(genesis_config);
@@ -100,12 +102,26 @@ impl SwarmBuilder {
     }
 
     // Gas is not enabled with this setup, it's enabled via forge instance.
-    pub async fn build(self) -> LocalSwarm {
-        self.build_wrapped().await.unwrap()
+    // Local swarm spin-up can fail due to port issues. So we retry SWARM_BUILD_NUM_RETRIES times.
+    pub async fn build(&mut self) -> LocalSwarm {
+        let num_retries = SWARM_BUILD_NUM_RETRIES;
+        let mut attempt = 0;
+        loop {
+            if attempt > num_retries {
+                panic!("Exhausted retries: {} / {}", attempt, num_retries);
+            }
+            match self.build_inner().await {
+                Ok(swarm) => {
+                    return swarm;
+                }
+                Err(err) => warn!("Attempt {} / {} failed with: {}", attempt, num_retries, err),
+            }
+            attempt += 1;
+        }
     }
 
     pub async fn build_with_cli(
-        self,
+        &mut self,
         num_cli_accounts: usize,
     ) -> (LocalSwarm, CliTestFramework, JoinHandle<()>) {
         let swarm = self.build().await;
@@ -136,27 +152,6 @@ impl SwarmBuilder {
     }
 }
 
-pub async fn with_retry<
-    F: Fn() -> Pin<Box<dyn std::future::Future<Output = anyhow::Result<LocalSwarm>>>>,
->(
-    build: F,
-    num_retries: u8,
-) -> LocalSwarm {
-    let mut attempt = 0;
-    loop {
-        if attempt >= num_retries {
-            panic!("Exhausted retries: {} / {}", attempt, num_retries);
-        }
-        match build().await {
-            Ok(swarm) => {
-                return swarm;
-            }
-            Err(err) => warn!("Attempt {} / {} failed with: {}", attempt, num_retries, err),
-        }
-        attempt += 1;
-    }
-}
-
 // Gas is not enabled with this setup, it's enabled via forge instance.
 pub async fn new_local_swarm_with_aptos(num_validators: usize) -> LocalSwarm {
     SwarmBuilder::new_local(num_validators)

testsuite/smoke-test/src/storage.rs

@@ -1,7 +1,7 @@
 // Copyright (c) Aptos
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::smoke_test_environment::{with_retry, SwarmBuilder};
+use crate::smoke_test_environment::SwarmBuilder;
 use crate::{
     test_utils::{
         assert_balance, create_and_fund_account, swarm_utils::insert_waypoint,
@@ -35,11 +35,7 @@ async fn test_db_restore() {
     workspace_builder::get_bin("db-backup-verify");
     info!("---------- 1. pre-building finished.");
 
-    let mut swarm = with_retry(
-        || Box::pin(SwarmBuilder::new_local(4).with_aptos().build_wrapped()),
-        3,
-    )
-    .await;
+    let mut swarm = SwarmBuilder::new_local(4).with_aptos().build().await;
     info!("---------- 1.1 swarm built, sending some transactions.");
     let validator_peer_ids = swarm.validators().map(|v| v.peer_id()).collect::<Vec<_>>();
     let client_1 = swarm

testsuite/smoke-test/src/txn_broadcast.rs

@@ -1,7 +1,7 @@
 // Copyright (c) Aptos
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::smoke_test_environment::{with_retry, SwarmBuilder};
+use crate::smoke_test_environment::SwarmBuilder;
 use crate::test_utils::{assert_balance, create_and_fund_account, transfer_coins};
 use aptos_config::config::NodeConfig;
 use forge::{NodeExt, Swarm, SwarmExt};
@@ -14,20 +14,13 @@ const MAX_WAIT_SECS: u64 = 60;
 /// This behavior should be true with both mempool and quorum store.
 #[tokio::test]
 async fn test_txn_broadcast() {
-    let mut swarm = with_retry(
-        || {
-            Box::pin(
-                SwarmBuilder::new_local(4)
-                    .with_aptos()
-                    .with_init_config(Arc::new(|_, conf, _| {
-                        conf.api.failpoints_enabled = true;
-                    }))
-                    .build_wrapped(),
-            )
-        },
-        3,
-    )
-    .await;
+    let mut swarm = SwarmBuilder::new_local(4)
+        .with_aptos()
+        .with_init_config(Arc::new(|_, conf, _| {
+            conf.api.failpoints_enabled = true;
+        }))
+        .build()
+        .await;
     let transaction_factory = swarm.chain_info().transaction_factory();
     let version = swarm.versions().max().unwrap();
     let validator_peer_ids = swarm.validators().map(|v| v.peer_id()).collect::<Vec<_>>();