Skip to content

Commit

Permalink
[forge] Add junit support
Browse files Browse the repository at this point in the history 
Add junit support for forge

This will allow us to onboard forge onto trunk flaky test detection

Also make suite choice more clean

Test Plan: running on PR
  • Loading branch information
@perryjrandall
perryjrandall committed Oct 21, 2024
1 parent fbf09ce commit 2d55848
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 67 deletions.
1 change: 1 addition & 0 deletions testsuite/forge-cli/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ random_word = { workspace = true }
reqwest = { workspace = true }
serde_json = { workspace = true }
serde_yaml = { workspace = true }
sugars = { workspace = true }
tokio = { workspace = true }
url = { workspace = true }

Expand Down
129 changes: 62 additions & 67 deletions testsuite/forge-cli/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ use futures::{future, FutureExt};
use rand::{rngs::ThreadRng, seq::SliceRandom, Rng};
use serde_json::{json, Value};
use std::{self, env, num::NonZeroUsize, process, time::Duration};
use sugars::hmap;
use suites::{
dag::get_dag_test,
indexer::get_indexer_test,
Expand Down Expand Up @@ -277,13 +278,13 @@ fn main() -> Result<()> {
mempool_backlog: 5000,
}));
let swarm_dir = local_cfg.swarmdir.clone();
run_forge(
duration,
let forge = Forge::new(
&args.options,
test_suite,
duration,
LocalFactory::from_workspace(swarm_dir)?,
&args.options,
args.changelog.clone(),
)
);
run_forge_with_changelog(forge, &args.options, args.changelog.clone())
},
TestCommand::K8sSwarm(k8s) => {
if let Some(move_modules_dir) = &k8s.move_modules_dir {
Expand All @@ -308,9 +309,10 @@ fn main() -> Result<()> {
};
let forge_runner_mode =
ForgeRunnerMode::try_from_env().unwrap_or(ForgeRunnerMode::K8s);
run_forge(
duration,
let forge = Forge::new(
&args.options,
test_suite,
duration,
K8sFactory::new(
namespace,
k8s.image_tag.clone(),
Expand All @@ -322,12 +324,9 @@ fn main() -> Result<()> {
k8s.enable_haproxy,
k8s.enable_indexer,
k8s.deployer_profile.clone(),
)
.unwrap(),
&args.options,
args.changelog,
)?;
Ok(())
)?,
);
run_forge_with_changelog(forge, &args.options, args.changelog)
},
}
},
Expand Down Expand Up @@ -413,39 +412,33 @@ fn main() -> Result<()> {
}
}

pub fn run_forge<F: Factory>(
global_duration: Duration,
tests: ForgeConfig,
factory: F,
pub fn run_forge_with_changelog<F: Factory>(
forge: Forge<F>,
options: &Options,
logs: Option<Vec<String>>,
changelog: Option<Vec<String>>,
) -> Result<()> {
let forge = Forge::new(options, tests, global_duration, factory);

if options.list {
forge.list()?;

return Ok(());
}

match forge.run() {
Ok(report) => {
if let Some(mut changelog) = logs {
if changelog.len() != 2 {
println!("Use: changelog <from> <to>");
process::exit(1);
}
let to_commit = changelog.remove(1);
let from_commit = Some(changelog.remove(0));
send_changelog_message(&report.to_string(), &from_commit, &to_commit);
}
Ok(())
},
Err(e) => {
eprintln!("Failed to run tests:\n{}", e);
Err(e)
},
}
let forge_result = forge.run();
let report = forge_result.map_err(|e| {
eprintln!("Failed to run tests:\n{}", e);
anyhow::anyhow!(e)
})?;

changelog.map(|mut changelog| {
if changelog.len() != 2 {
println!("Use: changelog <from> <to>");
process::exit(1);
}
let to_commit = changelog.remove(1);
let from_commit = Some(changelog.remove(0));
send_changelog_message(&report.to_string(), &from_commit, &to_commit);
});
Ok(())
}

pub fn send_changelog_message(perf_msg: &str, from_commit: &Option<String>, to_commit: &str) {
Expand Down Expand Up @@ -503,39 +496,41 @@ fn get_test_suite(
duration: Duration,
test_cmd: &TestCommand,
) -> Result<ForgeConfig> {
// Check the test name against the multi-test suites
match test_name {
"local_test_suite" => return Ok(local_test_suite()),
"pre_release" => return Ok(pre_release_suite()),
"run_forever" => return Ok(run_forever()),
// TODO(rustielin): verify each test suite
"k8s_suite" => return Ok(k8s_test_suite()),
"chaos" => return Ok(chaos_test_suite(duration)),
_ => {}, // No multi-test suite matches!
// These are high level suite aliases that express an intent
let suite_aliases = hmap! {
"local_test_suite" => || local_test_suite(),
"pre_release" => || pre_release_suite(),
"run_forever" => || run_forever(),
"k8s_suite" => || k8s_test_suite(),
"chaos" => || chaos_test_suite(duration),
};

if suite_aliases.contains_key(test_name) {
return suite_aliases[test_name]();
}

// Otherwise, check the test name against the grouped test suites
if let Some(test_suite) = get_land_blocking_test(test_name, duration, test_cmd) {
Ok(test_suite)
} else if let Some(test_suite) = get_multi_region_test(test_name) {
return Ok(test_suite);
} else if let Some(test_suite) = get_netbench_test(test_name) {
return Ok(test_suite);
} else if let Some(test_suite) = get_pfn_test(test_name, duration) {
return Ok(test_suite);
} else if let Some(test_suite) = get_realistic_env_test(test_name, duration, test_cmd) {
return Ok(test_suite);
} else if let Some(test_suite) = get_state_sync_test(test_name) {
return Ok(test_suite);
} else if let Some(test_suite) = get_dag_test(test_name, duration, test_cmd) {
return Ok(test_suite);
} else if let Some(test_suite) = get_indexer_test(test_name) {
return Ok(test_suite);
} else if let Some(test_suite) = get_ungrouped_test(test_name) {
return Ok(test_suite);
} else {
bail!(format_err!("Invalid --suite given: {:?}", test_name))
// This is done in order of priority
// A match higher up in the list will take precedence
let named_test_suites = vec![
|| get_land_blocking_test(test_name, duration, test_cmd),
|| get_multi_region_test(test_name),
|| get_netbench_test(test_name),
|| get_pfn_test(test_name, duration),
|| get_realistic_env_test(test_name, duration, test_cmd),
|| get_state_sync_test(test_name),
|| get_dag_test(test_name, duration, test_cmd),
|| get_indexer_test(test_name),
|| get_ungrouped_test(test_name),
];

for named_suite in named_test_suites.iter() {
if let Some(suite) = named_suite() {
return Ok(suite);
}
}

bail!(format_err!("Invalid --suite given: {:?}", test_name))
}
#[cfg(test)]
mod test {
Expand Down
8 changes: 8 additions & 0 deletions testsuite/forge/src/runner.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,8 @@ pub struct NodeResourceOverride {
}

pub struct ForgeConfig {
suite_name: Option<String>,

aptos_tests: Vec<Box<dyn AptosTest>>,
admin_tests: Vec<Box<dyn AdminTest>>,
network_tests: Vec<Box<dyn NetworkTest>>,
Expand Down Expand Up @@ -185,6 +187,11 @@ impl ForgeConfig {
self
}

pub fn with_suite_name(mut self, suite_name: String) -> Self {
self.suite_name = Some(suite_name);
self
}

pub fn with_aptos_tests(mut self, aptos_tests: Vec<Box<dyn AptosTest>>) -> Self {
self.aptos_tests = aptos_tests;
self
Expand Down Expand Up @@ -484,6 +491,7 @@ impl Default for ForgeConfig {
))
};
Self {
suite_name: None,
aptos_tests: vec![],
admin_tests: vec![],
network_tests: vec![],
Expand Down

0 comments on commit 2d55848

Please sign in to comment.