From 0c463ca6d27c508eea97e7dd177ca26c3dafa0a4 Mon Sep 17 00:00:00 2001 From: Oliver Nordbjerg Date: Wed, 3 Aug 2022 18:54:07 +0200 Subject: [PATCH] feat: snapshot fuzz tests using determin. seed --- cli/src/cmd/forge/coverage.rs | 2 +- cli/src/cmd/forge/snapshot.rs | 20 ++++++++++++++------ cli/src/cmd/forge/test/mod.rs | 29 +++++++++++------------------ forge/src/multi_runner.rs | 31 +++++++++++-------------------- forge/src/runner.rs | 15 +++++---------- 5 files changed, 42 insertions(+), 55 deletions(-) diff --git a/cli/src/cmd/forge/coverage.rs b/cli/src/cmd/forge/coverage.rs index 6d7cb5393d2a..f01fad17dce9 100644 --- a/cli/src/cmd/forge/coverage.rs +++ b/cli/src/cmd/forge/coverage.rs @@ -283,7 +283,7 @@ impl CoverageArgs { let local_identifier = LocalTraceIdentifier::new(&runner.known_contracts); // TODO: Coverage for fuzz tests - let handle = thread::spawn(move || runner.test(&self.filter, Some(tx), false).unwrap()); + let handle = thread::spawn(move || runner.test(&self.filter, Some(tx)).unwrap()); for mut result in rx.into_iter().flat_map(|(_, suite)| suite.test_results.into_values()) { if let Some(hit_map) = result.coverage.take() { for (_, trace) in &mut result.traces { diff --git a/cli/src/cmd/forge/snapshot.rs b/cli/src/cmd/forge/snapshot.rs index 1d536727e7fc..b25e5b0ca01a 100644 --- a/cli/src/cmd/forge/snapshot.rs +++ b/cli/src/cmd/forge/snapshot.rs @@ -8,6 +8,7 @@ use crate::cmd::{ Cmd, }; use clap::{Parser, ValueHint}; +use ethers::types::U256; use eyre::Context; use forge::result::TestKindGas; use once_cell::sync::Lazy; @@ -30,6 +31,14 @@ pub static RE_BASIC_SNAPSHOT_ENTRY: Lazy = Lazy::new(|| { Regex::new(r"(?P(.*?)):(?P(\w+)\s*\((.*?)\))\s*\(((gas:)?\s*(?P\d+)|(runs:\s*(?P\d+),\s*μ:\s*(?P\d+),\s*~:\s*(?P\d+)))\)").unwrap() }); +/// Deterministic fuzzer seed used for gas snapshots. +/// +/// The keccak256 hash of "foundry rulez" +pub static SNAPSHOT_FUZZ_SEED: [u8; 32] = [ + 0x01, 0x00, 0xfa, 0x69, 0xa5, 0xf1, 0x71, 0x0a, 0x95, 0xcd, 0xef, 0x94, 0x88, 0x9b, 0x02, 0x84, + 0x5d, 0x64, 0x0b, 0x19, 0xad, 0xf0, 0xe3, 0x57, 0xb8, 0xd4, 0xbe, 0x7d, 0x49, 0xee, 0x70, 0xe6, +]; + #[derive(Debug, Clone, Parser)] pub struct SnapshotArgs { /// All test arguments are supported @@ -75,10 +84,6 @@ pub struct SnapshotArgs { value_name = "SNAPSHOT_FILE" )] snap: PathBuf, - - /// Include the mean and median gas use of fuzz tests in the snapshot. - #[clap(long, env = "FORGE_INCLUDE_FUZZ_TESTS")] - pub include_fuzz_tests: bool, } impl SnapshotArgs { @@ -102,8 +107,11 @@ impl SnapshotArgs { impl Cmd for SnapshotArgs { type Output = (); - fn run(self) -> eyre::Result<()> { - let outcome = custom_run(self.test, self.include_fuzz_tests)?; + fn run(mut self) -> eyre::Result<()> { + // Set fuzz seed so gas snapshots are deterministic + self.test.fuzz_seed = Some(U256::from_big_endian(&SNAPSHOT_FUZZ_SEED)); + + let outcome = custom_run(self.test)?; outcome.ensure_ok()?; let tests = self.config.apply(outcome); diff --git a/cli/src/cmd/forge/test/mod.rs b/cli/src/cmd/forge/test/mod.rs index 644cb355da1d..dd16982d300b 100644 --- a/cli/src/cmd/forge/test/mod.rs +++ b/cli/src/cmd/forge/test/mod.rs @@ -96,7 +96,11 @@ pub struct TestArgs { #[clap(long, short, help_heading = "DISPLAY OPTIONS")] list: bool, - #[clap(long, help = "Set seed used to generate randomness during your fuzz runs", parse(try_from_str = utils::parse_u256))] + #[clap( + long, + help = "Set seed used to generate randomness during your fuzz runs", + parse(try_from_str = utils::parse_u256) + )] pub fuzz_seed: Option, } @@ -159,7 +163,7 @@ impl Cmd for TestArgs { fn run(self) -> eyre::Result { trace!(target: "forge::test", "executing test command"); - custom_run(self, true) + custom_run(self) } } @@ -295,7 +299,7 @@ fn short_test_result(name: &str, result: &TestResult) { println!("{} {} {}", status, name, result.kind.gas_used()); } -pub fn custom_run(args: TestArgs, include_fuzz_tests: bool) -> eyre::Result { +pub fn custom_run(args: TestArgs) -> eyre::Result { // Merge all configs let (config, mut evm_opts) = args.config_and_evm_opts()?; @@ -359,7 +363,7 @@ pub fn custom_run(args: TestArgs, include_fuzz_tests: bool) -> eyre::Result { // Run the test - let results = runner.test(&filter, None, true)?; + let results = runner.test(&filter, None)?; // Get the result of the single test let (id, sig, test_kind, counterexample) = results.iter().map(|(id, SuiteResult{ test_results, .. })| { @@ -403,16 +407,7 @@ pub fn custom_run(args: TestArgs, include_fuzz_tests: bool) -> eyre::Result eyre::Result { trace!(target: "forge::test", "running all tests"); @@ -468,7 +462,7 @@ fn test( } if json { - let results = runner.test(&filter, None, include_fuzz_tests)?; + let results = runner.test(&filter, None)?; println!("{}", serde_json::to_string(&results)?); Ok(TestOutcome::new(results, allow_failure)) } else { @@ -489,8 +483,7 @@ fn test( let (tx, rx) = channel::<(String, SuiteResult)>(); // Run tests - let handle = - thread::spawn(move || runner.test(&filter, Some(tx), include_fuzz_tests).unwrap()); + let handle = thread::spawn(move || runner.test(&filter, Some(tx)).unwrap()); let mut results: BTreeMap = BTreeMap::new(); let mut gas_report = GasReport::new(config.gas_reports); diff --git a/forge/src/multi_runner.rs b/forge/src/multi_runner.rs index 22e1515bf7c0..326865a2336b 100644 --- a/forge/src/multi_runner.rs +++ b/forge/src/multi_runner.rs @@ -119,10 +119,7 @@ impl MultiContractRunner { &mut self, filter: &impl TestFilter, stream_result: Option>, - include_fuzz_tests: bool, ) -> Result> { - tracing::info!(include_fuzz_tests= ?include_fuzz_tests, "running all tests"); - let db = Backend::spawn(self.fork.take()); let results = @@ -156,7 +153,7 @@ impl MultiContractRunner { executor, deploy_code.clone(), libs, - (filter, include_fuzz_tests), + filter )?; tracing::trace!(contract= ?identifier, "executed all tests in contract"); @@ -190,7 +187,7 @@ impl MultiContractRunner { executor: Executor, deploy_code: Bytes, libs: &[Bytes], - (filter, include_fuzz_tests): (&impl TestFilter, bool), + filter: &impl TestFilter, ) -> Result { let runner = ContractRunner::new( executor, @@ -201,7 +198,7 @@ impl MultiContractRunner { self.errors.as_ref(), libs, ); - runner.run_tests(filter, self.fuzzer.clone(), include_fuzz_tests) + runner.run_tests(filter, self.fuzzer.clone()) } } @@ -519,7 +516,7 @@ mod tests { #[test] fn test_core() { let mut runner = runner(); - let results = runner.test(&Filter::new(".*", ".*", ".*core"), None, true).unwrap(); + let results = runner.test(&Filter::new(".*", ".*", ".*core"), None).unwrap(); assert_multiple( &results, @@ -594,7 +591,7 @@ mod tests { #[test] fn test_logs() { let mut runner = runner(); - let results = runner.test(&Filter::new(".*", ".*", ".*logs"), None, true).unwrap(); + let results = runner.test(&Filter::new(".*", ".*", ".*logs"), None).unwrap(); assert_multiple( &results, @@ -1157,7 +1154,7 @@ mod tests { // test `setEnv` first, and confirm that it can correctly set environment variables, // so that we can use it in subsequent `env*` tests - runner.test(&Filter::new("testSetEnv", ".*", ".*"), None, true).unwrap(); + runner.test(&Filter::new("testSetEnv", ".*", ".*"), None).unwrap(); let env_var_key = "_foundryCheatcodeSetEnvTestKey"; let env_var_val = "_foundryCheatcodeSetEnvTestVal"; let res = env::var(env_var_key); @@ -1182,7 +1179,6 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", &format!(".*cheats{}Fork", RE_PATH_SEPARATOR), ), None, - true, ) .unwrap(); assert_eq!(suite_result.len(), 1); @@ -1206,7 +1202,6 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", &Filter::new(".*", ".*", &format!(".*cheats{}Fork", RE_PATH_SEPARATOR)) .exclude_tests(".*Revert"), None, - true, ) .unwrap(); assert!(!suite_result.is_empty()); @@ -1230,11 +1225,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", fn test_cheats_local() { let mut runner = runner(); let suite_result = runner - .test( - &Filter::new(".*", ".*", &format!(".*cheats{}[^Fork]", RE_PATH_SEPARATOR)), - None, - true, - ) + .test(&Filter::new(".*", ".*", &format!(".*cheats{}[^Fork]", RE_PATH_SEPARATOR)), None) .unwrap(); assert!(!suite_result.is_empty()); @@ -1258,7 +1249,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", let cfg = proptest::test_runner::Config { failure_persistence: None, ..Default::default() }; runner.fuzzer = Some(proptest::test_runner::TestRunner::new(cfg)); - let suite_result = runner.test(&Filter::new(".*", ".*", ".*fuzz"), None, true).unwrap(); + let suite_result = runner.test(&Filter::new(".*", ".*", ".*fuzz"), None).unwrap(); assert!(!suite_result.is_empty()); @@ -1292,7 +1283,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", #[test] fn test_trace() { let mut runner = tracing_runner(); - let suite_result = runner.test(&Filter::new(".*", ".*", ".*trace"), None, true).unwrap(); + let suite_result = runner.test(&Filter::new(".*", ".*", ".*trace"), None).unwrap(); // TODO: This trace test is very basic - it is probably a good candidate for snapshot // testing. @@ -1330,7 +1321,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", fn test_fork() { let rpc_url = foundry_utils::rpc::next_http_archive_rpc_endpoint(); let mut runner = forked_runner(&rpc_url); - let suite_result = runner.test(&Filter::new(".*", ".*", ".*fork"), None, true).unwrap(); + let suite_result = runner.test(&Filter::new(".*", ".*", ".*fork"), None).unwrap(); for (_, SuiteResult { test_results, .. }) in suite_result { for (test_name, result) in test_results { @@ -1351,7 +1342,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`", fn test_doesnt_run_abstract_contract() { let mut runner = runner(); let results = runner - .test(&Filter::new(".*", ".*", ".*Abstract.t.sol".to_string().as_str()), None, true) + .test(&Filter::new(".*", ".*", ".*Abstract.t.sol".to_string().as_str()), None) .unwrap(); assert!(results.get("core/Abstract.t.sol:AbstractTestBase").is_none()); assert!(results.get("core/Abstract.t.sol:AbstractTest").is_some()); diff --git a/forge/src/runner.rs b/forge/src/runner.rs index 78da2fb0b04f..a677af68a846 100644 --- a/forge/src/runner.rs +++ b/forge/src/runner.rs @@ -176,7 +176,6 @@ impl<'a> ContractRunner<'a> { mut self, filter: &impl TestFilter, fuzzer: Option, - include_fuzz_tests: bool, ) -> Result { tracing::info!("starting tests"); let start = Instant::now(); @@ -247,23 +246,19 @@ impl<'a> ContractRunner<'a> { .contract .functions() .into_iter() - .filter(|func| { - func.name.is_test() && - filter.matches_test(func.signature()) && - (include_fuzz_tests || func.inputs.is_empty()) - }) - .map(|func| (func, func.name.is_test_fail())) + .filter(|func| func.is_test() && filter.matches_test(func.signature())) + .map(|func| (func, func.is_test_fail())) .collect(); let test_results = tests .par_iter() .filter_map(|(func, should_fail)| { - let result = if func.inputs.is_empty() { - Some(self.clone().run_test(func, *should_fail, setup.clone())) - } else { + let result = if func.is_fuzz_test() { fuzzer.as_ref().map(|fuzzer| { self.run_fuzz_test(func, *should_fail, fuzzer.clone(), setup.clone()) }) + } else { + Some(self.clone().run_test(func, *should_fail, setup.clone())) }; result.map(|result| Ok((func.signature(), result?)))