From 2d55848502d4b30c281ba453f823715b03caadb8 Mon Sep 17 00:00:00 2001 From: Perry Randall Date: Mon, 21 Oct 2024 09:37:18 -0700 Subject: [PATCH] [forge] Add junit support Add junit support for forge This will allow us to onboard forge onto trunk flaky test detection Also make suite choice more clean Test Plan: running on PR --- testsuite/forge-cli/Cargo.toml | 1 + testsuite/forge-cli/src/main.rs | 129 +++++++++++++++----------------- testsuite/forge/src/runner.rs | 8 ++ 3 files changed, 71 insertions(+), 67 deletions(-) diff --git a/testsuite/forge-cli/Cargo.toml b/testsuite/forge-cli/Cargo.toml index 4006dbc32c6f73..e6b3000ee9b305 100644 --- a/testsuite/forge-cli/Cargo.toml +++ b/testsuite/forge-cli/Cargo.toml @@ -32,6 +32,7 @@ random_word = { workspace = true } reqwest = { workspace = true } serde_json = { workspace = true } serde_yaml = { workspace = true } +sugars = { workspace = true } tokio = { workspace = true } url = { workspace = true } diff --git a/testsuite/forge-cli/src/main.rs b/testsuite/forge-cli/src/main.rs index 269247405b1626..963930eb9dd783 100644 --- a/testsuite/forge-cli/src/main.rs +++ b/testsuite/forge-cli/src/main.rs @@ -12,6 +12,7 @@ use futures::{future, FutureExt}; use rand::{rngs::ThreadRng, seq::SliceRandom, Rng}; use serde_json::{json, Value}; use std::{self, env, num::NonZeroUsize, process, time::Duration}; +use sugars::hmap; use suites::{ dag::get_dag_test, indexer::get_indexer_test, @@ -277,13 +278,13 @@ fn main() -> Result<()> { mempool_backlog: 5000, })); let swarm_dir = local_cfg.swarmdir.clone(); - run_forge( - duration, + let forge = Forge::new( + &args.options, test_suite, + duration, LocalFactory::from_workspace(swarm_dir)?, - &args.options, - args.changelog.clone(), - ) + ); + run_forge_with_changelog(forge, &args.options, args.changelog.clone()) }, TestCommand::K8sSwarm(k8s) => { if let Some(move_modules_dir) = &k8s.move_modules_dir { @@ -308,9 +309,10 @@ fn main() -> Result<()> { }; let forge_runner_mode = ForgeRunnerMode::try_from_env().unwrap_or(ForgeRunnerMode::K8s); - run_forge( - duration, + let forge = Forge::new( + &args.options, test_suite, + duration, K8sFactory::new( namespace, k8s.image_tag.clone(), @@ -322,12 +324,9 @@ fn main() -> Result<()> { k8s.enable_haproxy, k8s.enable_indexer, k8s.deployer_profile.clone(), - ) - .unwrap(), - &args.options, - args.changelog, - )?; - Ok(()) + )?, + ); + run_forge_with_changelog(forge, &args.options, args.changelog) }, } }, @@ -413,39 +412,33 @@ fn main() -> Result<()> { } } -pub fn run_forge( - global_duration: Duration, - tests: ForgeConfig, - factory: F, +pub fn run_forge_with_changelog( + forge: Forge, options: &Options, - logs: Option>, + changelog: Option>, ) -> Result<()> { - let forge = Forge::new(options, tests, global_duration, factory); - if options.list { forge.list()?; return Ok(()); } - match forge.run() { - Ok(report) => { - if let Some(mut changelog) = logs { - if changelog.len() != 2 { - println!("Use: changelog "); - process::exit(1); - } - let to_commit = changelog.remove(1); - let from_commit = Some(changelog.remove(0)); - send_changelog_message(&report.to_string(), &from_commit, &to_commit); - } - Ok(()) - }, - Err(e) => { - eprintln!("Failed to run tests:\n{}", e); - Err(e) - }, - } + let forge_result = forge.run(); + let report = forge_result.map_err(|e| { + eprintln!("Failed to run tests:\n{}", e); + anyhow::anyhow!(e) + })?; + + changelog.map(|mut changelog| { + if changelog.len() != 2 { + println!("Use: changelog "); + process::exit(1); + } + let to_commit = changelog.remove(1); + let from_commit = Some(changelog.remove(0)); + send_changelog_message(&report.to_string(), &from_commit, &to_commit); + }); + Ok(()) } pub fn send_changelog_message(perf_msg: &str, from_commit: &Option, to_commit: &str) { @@ -503,39 +496,41 @@ fn get_test_suite( duration: Duration, test_cmd: &TestCommand, ) -> Result { - // Check the test name against the multi-test suites - match test_name { - "local_test_suite" => return Ok(local_test_suite()), - "pre_release" => return Ok(pre_release_suite()), - "run_forever" => return Ok(run_forever()), - // TODO(rustielin): verify each test suite - "k8s_suite" => return Ok(k8s_test_suite()), - "chaos" => return Ok(chaos_test_suite(duration)), - _ => {}, // No multi-test suite matches! + // These are high level suite aliases that express an intent + let suite_aliases = hmap! { + "local_test_suite" => || local_test_suite(), + "pre_release" => || pre_release_suite(), + "run_forever" => || run_forever(), + "k8s_suite" => || k8s_test_suite(), + "chaos" => || chaos_test_suite(duration), }; + if suite_aliases.contains_key(test_name) { + return suite_aliases[test_name](); + } + // Otherwise, check the test name against the grouped test suites - if let Some(test_suite) = get_land_blocking_test(test_name, duration, test_cmd) { - Ok(test_suite) - } else if let Some(test_suite) = get_multi_region_test(test_name) { - return Ok(test_suite); - } else if let Some(test_suite) = get_netbench_test(test_name) { - return Ok(test_suite); - } else if let Some(test_suite) = get_pfn_test(test_name, duration) { - return Ok(test_suite); - } else if let Some(test_suite) = get_realistic_env_test(test_name, duration, test_cmd) { - return Ok(test_suite); - } else if let Some(test_suite) = get_state_sync_test(test_name) { - return Ok(test_suite); - } else if let Some(test_suite) = get_dag_test(test_name, duration, test_cmd) { - return Ok(test_suite); - } else if let Some(test_suite) = get_indexer_test(test_name) { - return Ok(test_suite); - } else if let Some(test_suite) = get_ungrouped_test(test_name) { - return Ok(test_suite); - } else { - bail!(format_err!("Invalid --suite given: {:?}", test_name)) + // This is done in order of priority + // A match higher up in the list will take precedence + let named_test_suites = vec![ + || get_land_blocking_test(test_name, duration, test_cmd), + || get_multi_region_test(test_name), + || get_netbench_test(test_name), + || get_pfn_test(test_name, duration), + || get_realistic_env_test(test_name, duration, test_cmd), + || get_state_sync_test(test_name), + || get_dag_test(test_name, duration, test_cmd), + || get_indexer_test(test_name), + || get_ungrouped_test(test_name), + ]; + + for named_suite in named_test_suites.iter() { + if let Some(suite) = named_suite() { + return Ok(suite); + } } + + bail!(format_err!("Invalid --suite given: {:?}", test_name)) } #[cfg(test)] mod test { diff --git a/testsuite/forge/src/runner.rs b/testsuite/forge/src/runner.rs index 46407a4f8cc657..377fb08312d076 100644 --- a/testsuite/forge/src/runner.rs +++ b/testsuite/forge/src/runner.rs @@ -131,6 +131,8 @@ pub struct NodeResourceOverride { } pub struct ForgeConfig { + suite_name: Option, + aptos_tests: Vec>, admin_tests: Vec>, network_tests: Vec>, @@ -185,6 +187,11 @@ impl ForgeConfig { self } + pub fn with_suite_name(mut self, suite_name: String) -> Self { + self.suite_name = Some(suite_name); + self + } + pub fn with_aptos_tests(mut self, aptos_tests: Vec>) -> Self { self.aptos_tests = aptos_tests; self @@ -484,6 +491,7 @@ impl Default for ForgeConfig { )) }; Self { + suite_name: None, aptos_tests: vec![], admin_tests: vec![], network_tests: vec![],