Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: snapshot fuzz tests using deterministic seed #2591

Merged
merged 1 commit into from
Aug 3, 2022
Merged

feat: snapshot fuzz tests using deterministic seed #2591

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cli/src/cmd/forge/coverage.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ impl CoverageArgs {
let local_identifier = LocalTraceIdentifier::new(&runner.known_contracts);

// TODO: Coverage for fuzz tests
let handle = thread::spawn(move || runner.test(&self.filter, Some(tx), false).unwrap());
let handle = thread::spawn(move || runner.test(&self.filter, Some(tx)).unwrap());
for mut result in rx.into_iter().flat_map(|(_, suite)| suite.test_results.into_values()) {
if let Some(hit_map) = result.coverage.take() {
for (_, trace) in &mut result.traces {
Expand Down
20 changes: 14 additions & 6 deletions cli/src/cmd/forge/snapshot.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use crate::cmd::{
Cmd,
};
use clap::{Parser, ValueHint};
use ethers::types::U256;
use eyre::Context;
use forge::result::TestKindGas;
use once_cell::sync::Lazy;
Expand All @@ -30,6 +31,14 @@ pub static RE_BASIC_SNAPSHOT_ENTRY: Lazy<Regex> = Lazy::new(|| {
Regex::new(r"(?P<file>(.*?)):(?P<sig>(\w+)\s*\((.*?)\))\s*\(((gas:)?\s*(?P<gas>\d+)|(runs:\s*(?P<runs>\d+),\s*μ:\s*(?P<avg>\d+),\s*~:\s*(?P<med>\d+)))\)").unwrap()
});

/// Deterministic fuzzer seed used for gas snapshots.
///
/// The keccak256 hash of "foundry rulez"
pub static SNAPSHOT_FUZZ_SEED: [u8; 32] = [
0x01, 0x00, 0xfa, 0x69, 0xa5, 0xf1, 0x71, 0x0a, 0x95, 0xcd, 0xef, 0x94, 0x88, 0x9b, 0x02, 0x84,
0x5d, 0x64, 0x0b, 0x19, 0xad, 0xf0, 0xe3, 0x57, 0xb8, 0xd4, 0xbe, 0x7d, 0x49, 0xee, 0x70, 0xe6,
];

#[derive(Debug, Clone, Parser)]
pub struct SnapshotArgs {
/// All test arguments are supported
Expand Down Expand Up @@ -75,10 +84,6 @@ pub struct SnapshotArgs {
value_name = "SNAPSHOT_FILE"
)]
snap: PathBuf,

/// Include the mean and median gas use of fuzz tests in the snapshot.
#[clap(long, env = "FORGE_INCLUDE_FUZZ_TESTS")]
pub include_fuzz_tests: bool,
}

impl SnapshotArgs {
Expand All @@ -102,8 +107,11 @@ impl SnapshotArgs {
impl Cmd for SnapshotArgs {
type Output = ();

fn run(self) -> eyre::Result<()> {
let outcome = custom_run(self.test, self.include_fuzz_tests)?;
fn run(mut self) -> eyre::Result<()> {
// Set fuzz seed so gas snapshots are deterministic
self.test.fuzz_seed = Some(U256::from_big_endian(&SNAPSHOT_FUZZ_SEED));

let outcome = custom_run(self.test)?;
outcome.ensure_ok()?;
let tests = self.config.apply(outcome);

Expand Down
29 changes: 11 additions & 18 deletions cli/src/cmd/forge/test/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,11 @@ pub struct TestArgs {
#[clap(long, short, help_heading = "DISPLAY OPTIONS")]
list: bool,

#[clap(long, help = "Set seed used to generate randomness during your fuzz runs", parse(try_from_str = utils::parse_u256))]
#[clap(
long,
help = "Set seed used to generate randomness during your fuzz runs",
parse(try_from_str = utils::parse_u256)
)]
pub fuzz_seed: Option<U256>,
}

Expand Down Expand Up @@ -159,7 +163,7 @@ impl Cmd for TestArgs {

fn run(self) -> eyre::Result<Self::Output> {
trace!(target: "forge::test", "executing test command");
custom_run(self, true)
custom_run(self)
}
}

Expand Down Expand Up @@ -295,7 +299,7 @@ fn short_test_result(name: &str, result: &TestResult) {
println!("{} {} {}", status, name, result.kind.gas_used());
}

pub fn custom_run(args: TestArgs, include_fuzz_tests: bool) -> eyre::Result<TestOutcome> {
pub fn custom_run(args: TestArgs) -> eyre::Result<TestOutcome> {
// Merge all configs
let (config, mut evm_opts) = args.config_and_evm_opts()?;

Expand Down Expand Up @@ -359,7 +363,7 @@ pub fn custom_run(args: TestArgs, include_fuzz_tests: bool) -> eyre::Result<Test
match runner.count_filtered_tests(&filter) {
1 => {
// Run the test
let results = runner.test(&filter, None, true)?;
let results = runner.test(&filter, None)?;

// Get the result of the single test
let (id, sig, test_kind, counterexample) = results.iter().map(|(id, SuiteResult{ test_results, .. })| {
Expand Down Expand Up @@ -403,16 +407,7 @@ pub fn custom_run(args: TestArgs, include_fuzz_tests: bool) -> eyre::Result<Test
} else if args.list {
list(runner, filter, args.json)
} else {
test(
config,
runner,
verbosity,
filter,
args.json,
args.allow_failure,
include_fuzz_tests,
args.gas_report,
)
test(config, runner, verbosity, filter, args.json, args.allow_failure, args.gas_report)
}
}

Expand Down Expand Up @@ -443,7 +438,6 @@ fn test(
filter: Filter,
json: bool,
allow_failure: bool,
include_fuzz_tests: bool,
gas_reporting: bool,
) -> eyre::Result<TestOutcome> {
trace!(target: "forge::test", "running all tests");
Expand All @@ -468,7 +462,7 @@ fn test(
}

if json {
let results = runner.test(&filter, None, include_fuzz_tests)?;
let results = runner.test(&filter, None)?;
println!("{}", serde_json::to_string(&results)?);
Ok(TestOutcome::new(results, allow_failure))
} else {
Expand All @@ -489,8 +483,7 @@ fn test(
let (tx, rx) = channel::<(String, SuiteResult)>();

// Run tests
let handle =
thread::spawn(move || runner.test(&filter, Some(tx), include_fuzz_tests).unwrap());
let handle = thread::spawn(move || runner.test(&filter, Some(tx)).unwrap());

let mut results: BTreeMap<String, SuiteResult> = BTreeMap::new();
let mut gas_report = GasReport::new(config.gas_reports);
Expand Down
31 changes: 11 additions & 20 deletions forge/src/multi_runner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,10 +119,7 @@ impl MultiContractRunner {
&mut self,
filter: &impl TestFilter,
stream_result: Option<Sender<(String, SuiteResult)>>,
include_fuzz_tests: bool,
) -> Result<BTreeMap<String, SuiteResult>> {
tracing::info!(include_fuzz_tests= ?include_fuzz_tests, "running all tests");

let db = Backend::spawn(self.fork.take());

let results =
Expand Down Expand Up @@ -156,7 +153,7 @@ impl MultiContractRunner {
executor,
deploy_code.clone(),
libs,
(filter, include_fuzz_tests),
filter
)?;

tracing::trace!(contract= ?identifier, "executed all tests in contract");
Expand Down Expand Up @@ -190,7 +187,7 @@ impl MultiContractRunner {
executor: Executor,
deploy_code: Bytes,
libs: &[Bytes],
(filter, include_fuzz_tests): (&impl TestFilter, bool),
filter: &impl TestFilter,
) -> Result<SuiteResult> {
let runner = ContractRunner::new(
executor,
Expand All @@ -201,7 +198,7 @@ impl MultiContractRunner {
self.errors.as_ref(),
libs,
);
runner.run_tests(filter, self.fuzzer.clone(), include_fuzz_tests)
runner.run_tests(filter, self.fuzzer.clone())
}
}

Expand Down Expand Up @@ -519,7 +516,7 @@ mod tests {
#[test]
fn test_core() {
let mut runner = runner();
let results = runner.test(&Filter::new(".*", ".*", ".*core"), None, true).unwrap();
let results = runner.test(&Filter::new(".*", ".*", ".*core"), None).unwrap();

assert_multiple(
&results,
Expand Down Expand Up @@ -594,7 +591,7 @@ mod tests {
#[test]
fn test_logs() {
let mut runner = runner();
let results = runner.test(&Filter::new(".*", ".*", ".*logs"), None, true).unwrap();
let results = runner.test(&Filter::new(".*", ".*", ".*logs"), None).unwrap();

assert_multiple(
&results,
Expand Down Expand Up @@ -1157,7 +1154,7 @@ mod tests {

// test `setEnv` first, and confirm that it can correctly set environment variables,
// so that we can use it in subsequent `env*` tests
runner.test(&Filter::new("testSetEnv", ".*", ".*"), None, true).unwrap();
runner.test(&Filter::new("testSetEnv", ".*", ".*"), None).unwrap();
let env_var_key = "_foundryCheatcodeSetEnvTestKey";
let env_var_val = "_foundryCheatcodeSetEnvTestVal";
let res = env::var(env_var_key);
Expand All @@ -1182,7 +1179,6 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
&format!(".*cheats{}Fork", RE_PATH_SEPARATOR),
),
None,
true,
)
.unwrap();
assert_eq!(suite_result.len(), 1);
Expand All @@ -1206,7 +1202,6 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
&Filter::new(".*", ".*", &format!(".*cheats{}Fork", RE_PATH_SEPARATOR))
.exclude_tests(".*Revert"),
None,
true,
)
.unwrap();
assert!(!suite_result.is_empty());
Expand All @@ -1230,11 +1225,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
fn test_cheats_local() {
let mut runner = runner();
let suite_result = runner
.test(
&Filter::new(".*", ".*", &format!(".*cheats{}[^Fork]", RE_PATH_SEPARATOR)),
None,
true,
)
.test(&Filter::new(".*", ".*", &format!(".*cheats{}[^Fork]", RE_PATH_SEPARATOR)), None)
.unwrap();
assert!(!suite_result.is_empty());

Expand All @@ -1258,7 +1249,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
let cfg = proptest::test_runner::Config { failure_persistence: None, ..Default::default() };
runner.fuzzer = Some(proptest::test_runner::TestRunner::new(cfg));

let suite_result = runner.test(&Filter::new(".*", ".*", ".*fuzz"), None, true).unwrap();
let suite_result = runner.test(&Filter::new(".*", ".*", ".*fuzz"), None).unwrap();

assert!(!suite_result.is_empty());

Expand Down Expand Up @@ -1292,7 +1283,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
#[test]
fn test_trace() {
let mut runner = tracing_runner();
let suite_result = runner.test(&Filter::new(".*", ".*", ".*trace"), None, true).unwrap();
let suite_result = runner.test(&Filter::new(".*", ".*", ".*trace"), None).unwrap();

// TODO: This trace test is very basic - it is probably a good candidate for snapshot
// testing.
Expand Down Expand Up @@ -1330,7 +1321,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
fn test_fork() {
let rpc_url = foundry_utils::rpc::next_http_archive_rpc_endpoint();
let mut runner = forked_runner(&rpc_url);
let suite_result = runner.test(&Filter::new(".*", ".*", ".*fork"), None, true).unwrap();
let suite_result = runner.test(&Filter::new(".*", ".*", ".*fork"), None).unwrap();

for (_, SuiteResult { test_results, .. }) in suite_result {
for (test_name, result) in test_results {
Expand All @@ -1351,7 +1342,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
fn test_doesnt_run_abstract_contract() {
let mut runner = runner();
let results = runner
.test(&Filter::new(".*", ".*", ".*Abstract.t.sol".to_string().as_str()), None, true)
.test(&Filter::new(".*", ".*", ".*Abstract.t.sol".to_string().as_str()), None)
.unwrap();
assert!(results.get("core/Abstract.t.sol:AbstractTestBase").is_none());
assert!(results.get("core/Abstract.t.sol:AbstractTest").is_some());
Expand Down
15 changes: 5 additions & 10 deletions forge/src/runner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,6 @@ impl<'a> ContractRunner<'a> {
mut self,
filter: &impl TestFilter,
fuzzer: Option<TestRunner>,
include_fuzz_tests: bool,
) -> Result<SuiteResult> {
tracing::info!("starting tests");
let start = Instant::now();
Expand Down Expand Up @@ -247,23 +246,19 @@ impl<'a> ContractRunner<'a> {
.contract
.functions()
.into_iter()
.filter(|func| {
func.name.is_test() &&
filter.matches_test(func.signature()) &&
(include_fuzz_tests || func.inputs.is_empty())
})
.map(|func| (func, func.name.is_test_fail()))
.filter(|func| func.is_test() && filter.matches_test(func.signature()))
.map(|func| (func, func.is_test_fail()))
.collect();

let test_results = tests
.par_iter()
.filter_map(|(func, should_fail)| {
let result = if func.inputs.is_empty() {
Some(self.clone().run_test(func, *should_fail, setup.clone()))
} else {
let result = if func.is_fuzz_test() {
fuzzer.as_ref().map(|fuzzer| {
self.run_fuzz_test(func, *should_fail, fuzzer.clone(), setup.clone())
})
} else {
Some(self.clone().run_test(func, *should_fail, setup.clone()))
};

result.map(|result| Ok((func.signature(), result?)))
Expand Down