Skip to content

Commit

Permalink
feat: snapshot fuzz tests using determin. seed (foundry-rs#2591)
Browse files Browse the repository at this point in the history
  • Loading branch information
@onbjerg @iFrostizz
onbjerg authored and iFrostizz committed Nov 9, 2022
1 parent e683241 commit cd8606a
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 55 deletions.
2 changes: 1 addition & 1 deletion cli/src/cmd/forge/coverage.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ impl CoverageArgs {
let local_identifier = LocalTraceIdentifier::new(&runner.known_contracts);

// TODO: Coverage for fuzz tests
let handle = thread::spawn(move || runner.test(&self.filter, Some(tx), false).unwrap());
let handle = thread::spawn(move || runner.test(&self.filter, Some(tx)).unwrap());
for mut result in rx.into_iter().flat_map(|(_, suite)| suite.test_results.into_values()) {
if let Some(hit_map) = result.coverage.take() {
for (_, trace) in &mut result.traces {
Expand Down
20 changes: 14 additions & 6 deletions cli/src/cmd/forge/snapshot.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use crate::cmd::{
Cmd,
};
use clap::{Parser, ValueHint};
use ethers::types::U256;
use eyre::Context;
use forge::result::TestKindGas;
use once_cell::sync::Lazy;
Expand All @@ -30,6 +31,14 @@ pub static RE_BASIC_SNAPSHOT_ENTRY: Lazy<Regex> = Lazy::new(|| {
Regex::new(r"(?P<file>(.*?)):(?P<sig>(\w+)\s*\((.*?)\))\s*\(((gas:)?\s*(?P<gas>\d+)|(runs:\s*(?P<runs>\d+),\s*μ:\s*(?P<avg>\d+),\s*~:\s*(?P<med>\d+)))\)").unwrap()
});

/// Deterministic fuzzer seed used for gas snapshots.
///
/// The keccak256 hash of "foundry rulez"
pub static SNAPSHOT_FUZZ_SEED: [u8; 32] = [
0x01, 0x00, 0xfa, 0x69, 0xa5, 0xf1, 0x71, 0x0a, 0x95, 0xcd, 0xef, 0x94, 0x88, 0x9b, 0x02, 0x84,
0x5d, 0x64, 0x0b, 0x19, 0xad, 0xf0, 0xe3, 0x57, 0xb8, 0xd4, 0xbe, 0x7d, 0x49, 0xee, 0x70, 0xe6,
];

#[derive(Debug, Clone, Parser)]
pub struct SnapshotArgs {
/// All test arguments are supported
Expand Down Expand Up @@ -75,10 +84,6 @@ pub struct SnapshotArgs {
value_name = "SNAPSHOT_FILE"
)]
snap: PathBuf,

/// Include the mean and median gas use of fuzz tests in the snapshot.
#[clap(long, env = "FORGE_INCLUDE_FUZZ_TESTS")]
pub include_fuzz_tests: bool,
}

impl SnapshotArgs {
Expand All @@ -102,8 +107,11 @@ impl SnapshotArgs {
impl Cmd for SnapshotArgs {
type Output = ();

fn run(self) -> eyre::Result<()> {
let outcome = custom_run(self.test, self.include_fuzz_tests)?;
fn run(mut self) -> eyre::Result<()> {
// Set fuzz seed so gas snapshots are deterministic
self.test.fuzz_seed = Some(U256::from_big_endian(&SNAPSHOT_FUZZ_SEED));

let outcome = custom_run(self.test)?;
outcome.ensure_ok()?;
let tests = self.config.apply(outcome);

Expand Down
29 changes: 11 additions & 18 deletions cli/src/cmd/forge/test/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,11 @@ pub struct TestArgs {
#[clap(long, short, help_heading = "DISPLAY OPTIONS")]
list: bool,

#[clap(long, help = "Set seed used to generate randomness during your fuzz runs", parse(try_from_str = utils::parse_u256))]
#[clap(
long,
help = "Set seed used to generate randomness during your fuzz runs",
parse(try_from_str = utils::parse_u256)
)]
pub fuzz_seed: Option<U256>,
}

Expand Down Expand Up @@ -159,7 +163,7 @@ impl Cmd for TestArgs {

fn run(self) -> eyre::Result<Self::Output> {
trace!(target: "forge::test", "executing test command");
custom_run(self, true)
custom_run(self)
}
}

Expand Down Expand Up @@ -295,7 +299,7 @@ fn short_test_result(name: &str, result: &TestResult) {
println!("{} {} {}", status, name, result.kind.gas_used());
}

pub fn custom_run(args: TestArgs, include_fuzz_tests: bool) -> eyre::Result<TestOutcome> {
pub fn custom_run(args: TestArgs) -> eyre::Result<TestOutcome> {
// Merge all configs
let (config, mut evm_opts) = args.config_and_evm_opts()?;

Expand Down Expand Up @@ -359,7 +363,7 @@ pub fn custom_run(args: TestArgs, include_fuzz_tests: bool) -> eyre::Result<Test
match runner.count_filtered_tests(&filter) {
1 => {
// Run the test
let results = runner.test(&filter, None, true)?;
let results = runner.test(&filter, None)?;

// Get the result of the single test
let (id, sig, test_kind, counterexample) = results.iter().map(|(id, SuiteResult{ test_results, .. })| {
Expand Down Expand Up @@ -403,16 +407,7 @@ pub fn custom_run(args: TestArgs, include_fuzz_tests: bool) -> eyre::Result<Test
} else if args.list {
list(runner, filter, args.json)
} else {
test(
config,
runner,
verbosity,
filter,
args.json,
args.allow_failure,
include_fuzz_tests,
args.gas_report,
)
test(config, runner, verbosity, filter, args.json, args.allow_failure, args.gas_report)
}
}

Expand Down Expand Up @@ -443,7 +438,6 @@ fn test(
filter: Filter,
json: bool,
allow_failure: bool,
include_fuzz_tests: bool,
gas_reporting: bool,
) -> eyre::Result<TestOutcome> {
trace!(target: "forge::test", "running all tests");
Expand All @@ -468,7 +462,7 @@ fn test(
}

if json {
let results = runner.test(&filter, None, include_fuzz_tests)?;
let results = runner.test(&filter, None)?;
println!("{}", serde_json::to_string(&results)?);
Ok(TestOutcome::new(results, allow_failure))
} else {
Expand All @@ -489,8 +483,7 @@ fn test(
let (tx, rx) = channel::<(String, SuiteResult)>();

// Run tests
let handle =
thread::spawn(move || runner.test(&filter, Some(tx), include_fuzz_tests).unwrap());
let handle = thread::spawn(move || runner.test(&filter, Some(tx)).unwrap());

let mut results: BTreeMap<String, SuiteResult> = BTreeMap::new();
let mut gas_report = GasReport::new(config.gas_reports);
Expand Down
31 changes: 11 additions & 20 deletions forge/src/multi_runner.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,10 +119,7 @@ impl MultiContractRunner {
&mut self,
filter: &impl TestFilter,
stream_result: Option<Sender<(String, SuiteResult)>>,
include_fuzz_tests: bool,
) -> Result<BTreeMap<String, SuiteResult>> {
tracing::info!(include_fuzz_tests= ?include_fuzz_tests, "running all tests");

let db = Backend::spawn(self.fork.take());

let results =
Expand Down Expand Up @@ -156,7 +153,7 @@ impl MultiContractRunner {
executor,
deploy_code.clone(),
libs,
(filter, include_fuzz_tests),
filter
)?;

tracing::trace!(contract= ?identifier, "executed all tests in contract");
Expand Down Expand Up @@ -190,7 +187,7 @@ impl MultiContractRunner {
executor: Executor,
deploy_code: Bytes,
libs: &[Bytes],
(filter, include_fuzz_tests): (&impl TestFilter, bool),
filter: &impl TestFilter,
) -> Result<SuiteResult> {
let runner = ContractRunner::new(
executor,
Expand All @@ -201,7 +198,7 @@ impl MultiContractRunner {
self.errors.as_ref(),
libs,
);
runner.run_tests(filter, self.fuzzer.clone(), include_fuzz_tests)
runner.run_tests(filter, self.fuzzer.clone())
}
}

Expand Down Expand Up @@ -519,7 +516,7 @@ mod tests {
#[test]
fn test_core() {
let mut runner = runner();
let results = runner.test(&Filter::new(".*", ".*", ".*core"), None, true).unwrap();
let results = runner.test(&Filter::new(".*", ".*", ".*core"), None).unwrap();

assert_multiple(
&results,
Expand Down Expand Up @@ -594,7 +591,7 @@ mod tests {
#[test]
fn test_logs() {
let mut runner = runner();
let results = runner.test(&Filter::new(".*", ".*", ".*logs"), None, true).unwrap();
let results = runner.test(&Filter::new(".*", ".*", ".*logs"), None).unwrap();

assert_multiple(
&results,
Expand Down Expand Up @@ -1157,7 +1154,7 @@ mod tests {

// test `setEnv` first, and confirm that it can correctly set environment variables,
// so that we can use it in subsequent `env*` tests
runner.test(&Filter::new("testSetEnv", ".*", ".*"), None, true).unwrap();
runner.test(&Filter::new("testSetEnv", ".*", ".*"), None).unwrap();
let env_var_key = "_foundryCheatcodeSetEnvTestKey";
let env_var_val = "_foundryCheatcodeSetEnvTestVal";
let res = env::var(env_var_key);
Expand All @@ -1182,7 +1179,6 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
&format!(".*cheats{}Fork", RE_PATH_SEPARATOR),
),
None,
true,
)
.unwrap();
assert_eq!(suite_result.len(), 1);
Expand All @@ -1206,7 +1202,6 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
&Filter::new(".*", ".*", &format!(".*cheats{}Fork", RE_PATH_SEPARATOR))
.exclude_tests(".*Revert"),
None,
true,
)
.unwrap();
assert!(!suite_result.is_empty());
Expand All @@ -1230,11 +1225,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
fn test_cheats_local() {
let mut runner = runner();
let suite_result = runner
.test(
&Filter::new(".*", ".*", &format!(".*cheats{}[^Fork]", RE_PATH_SEPARATOR)),
None,
true,
)
.test(&Filter::new(".*", ".*", &format!(".*cheats{}[^Fork]", RE_PATH_SEPARATOR)), None)
.unwrap();
assert!(!suite_result.is_empty());

Expand All @@ -1258,7 +1249,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
let cfg = proptest::test_runner::Config { failure_persistence: None, ..Default::default() };
runner.fuzzer = Some(proptest::test_runner::TestRunner::new(cfg));

let suite_result = runner.test(&Filter::new(".*", ".*", ".*fuzz"), None, true).unwrap();
let suite_result = runner.test(&Filter::new(".*", ".*", ".*fuzz"), None).unwrap();

assert!(!suite_result.is_empty());

Expand Down Expand Up @@ -1292,7 +1283,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
#[test]
fn test_trace() {
let mut runner = tracing_runner();
let suite_result = runner.test(&Filter::new(".*", ".*", ".*trace"), None, true).unwrap();
let suite_result = runner.test(&Filter::new(".*", ".*", ".*trace"), None).unwrap();

// TODO: This trace test is very basic - it is probably a good candidate for snapshot
// testing.
Expand Down Expand Up @@ -1330,7 +1321,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
fn test_fork() {
let rpc_url = foundry_utils::rpc::next_http_archive_rpc_endpoint();
let mut runner = forked_runner(&rpc_url);
let suite_result = runner.test(&Filter::new(".*", ".*", ".*fork"), None, true).unwrap();
let suite_result = runner.test(&Filter::new(".*", ".*", ".*fork"), None).unwrap();

for (_, SuiteResult { test_results, .. }) in suite_result {
for (test_name, result) in test_results {
Expand All @@ -1351,7 +1342,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
fn test_doesnt_run_abstract_contract() {
let mut runner = runner();
let results = runner
.test(&Filter::new(".*", ".*", ".*Abstract.t.sol".to_string().as_str()), None, true)
.test(&Filter::new(".*", ".*", ".*Abstract.t.sol".to_string().as_str()), None)
.unwrap();
assert!(results.get("core/Abstract.t.sol:AbstractTestBase").is_none());
assert!(results.get("core/Abstract.t.sol:AbstractTest").is_some());
Expand Down
15 changes: 5 additions & 10 deletions forge/src/runner.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,6 @@ impl<'a> ContractRunner<'a> {
mut self,
filter: &impl TestFilter,
fuzzer: Option<TestRunner>,
include_fuzz_tests: bool,
) -> Result<SuiteResult> {
tracing::info!("starting tests");
let start = Instant::now();
Expand Down Expand Up @@ -247,23 +246,19 @@ impl<'a> ContractRunner<'a> {
.contract
.functions()
.into_iter()
.filter(|func| {
func.name.is_test() &&
filter.matches_test(func.signature()) &&
(include_fuzz_tests || func.inputs.is_empty())
})
.map(|func| (func, func.name.is_test_fail()))
.filter(|func| func.is_test() && filter.matches_test(func.signature()))
.map(|func| (func, func.is_test_fail()))
.collect();

let test_results = tests
.par_iter()
.filter_map(|(func, should_fail)| {
let result = if func.inputs.is_empty() {
Some(self.clone().run_test(func, *should_fail, setup.clone()))
} else {
let result = if func.is_fuzz_test() {
fuzzer.as_ref().map(|fuzzer| {
self.run_fuzz_test(func, *should_fail, fuzzer.clone(), setup.clone())
})
} else {
Some(self.clone().run_test(func, *should_fail, setup.clone()))
};

result.map(|result| Ok((func.signature(), result?)))
Expand Down

0 comments on commit cd8606a

Please sign in to comment.