Skip to content

Commit

Permalink
Parallel EVM Tests (gakonst#444)
Browse files Browse the repository at this point in the history 
* feat(fuzz): expose function to get internal evm

* refactor(evm): move EvmOpts from cli to evm-adapters

* feat(evm): add helper for creating sputnik backend

* feat(forge): add base evm opts for test usage

* feat(evm): derive default for EvmOpts

* test(forge): add utils for instantiating backend

* feat(forge): instantiate runner with EvmOpts instead of an EVM

This allows us to instantiate as many EVMs as we want inside of the runner,
which in turn will enable running tests in parallel

* feat(forge): pass evm by reference instead of using self.evm

* feat(forge): run unit tests with unique evm instantiation

previously we'd reuse the same EVM, now, we use a different EVM
per test, allowing us to get rid of the mutable reference on self

* feat(forge): run fuzz tests with unique evm instantiations

* test(forge): adjust tests to new instantiation style

* feat(forge): run tests in parallel with rayon

* feat(evm-adapters): put backend behind enum to avoid trait object

* chore(forge): move fuzzer instead of ref

* feat(forge): make multi contract runner compatible with new runner

* feat(forge): parallelize multi contract runner by file

* chore(cli): remove unused helper functions

* fix(cli/run): use new contract runner initialization

There's a TODO here around how we should do the evm.debug_calls check which we should figure out

* fix(cli/test): use evm_opts instead of directly passing evm

* chore: formatting fixes

* chore: update lockfile

* fix(evm-adapters): correctly init test caller and origin

fixes foundry-rs/foundry#249
fixes foundry-rs/foundry#253

* chore: clippy lint on unreachable code w disabled features

* fix: instantiate evm cfg without contract size limit

* fix debugging (gakonst#445)

* merge cleanup

Co-authored-by: brockelmore <[email protected]>
Co-authored-by: Brock <[email protected]>
  • Loading branch information
@gakonst @brockelmore
3 people authored Jan 13, 2022
1 parent d87c516 commit 764c69c
Show file tree
Hide file tree
Showing 16 changed files with 609 additions and 556 deletions.
3 changes: 3 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion cli/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,4 +76,4 @@ doc = false
[[bin]]
name = "forge"
path = "src/forge.rs"
doc = false
doc = false
120 changes: 4 additions & 116 deletions cli/src/cmd/build.rs
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
//! build command
use ethers::{
solc::{
artifacts::{Optimizer, Settings},
remappings::Remapping,
MinimalCombinedArtifacts, Project, ProjectCompileOutput, ProjectPathsConfig, SolcConfig,
},
types::Address,
use ethers::solc::{
artifacts::{Optimizer, Settings},
remappings::Remapping,
MinimalCombinedArtifacts, Project, ProjectCompileOutput, ProjectPathsConfig, SolcConfig,
};
use std::{
collections::BTreeMap,
Expand All @@ -17,10 +14,6 @@ use std::{
use crate::{cmd::Cmd, opts::forge::CompilerArgs, utils};

use clap::{Parser, ValueHint};
#[cfg(feature = "evmodin-evm")]
use evmodin::util::mocked_host::MockedHost;
#[cfg(feature = "sputnik-evm")]
use sputnik::backend::MemoryVicinity;

#[derive(Debug, Clone, Parser)]
pub struct BuildArgs {
Expand Down Expand Up @@ -250,108 +243,3 @@ impl BuildArgs {
Ok(project)
}
}

#[derive(Clone, Debug)]
pub enum EvmType {
#[cfg(feature = "sputnik-evm")]
Sputnik,
#[cfg(feature = "evmodin-evm")]
EvmOdin,
}

impl FromStr for EvmType {
type Err = eyre::Error;

fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(match s.to_lowercase().as_str() {
#[cfg(feature = "sputnik-evm")]
"sputnik" => EvmType::Sputnik,
#[cfg(feature = "evmodin-evm")]
"evmodin" => EvmType::EvmOdin,
other => eyre::bail!("unknown EVM type {}", other),
})
}
}

#[derive(Debug, Clone, Parser)]
pub struct Env {
#[clap(help = "the block gas limit", long, default_value_t = u64::MAX)]
pub gas_limit: u64,

#[clap(help = "the chainid opcode value", long, default_value = "1")]
pub chain_id: u64,

#[clap(help = "the tx.gasprice value during EVM execution", long, default_value = "0")]
pub gas_price: u64,

#[clap(help = "the base fee in a block", long, default_value = "0")]
pub block_base_fee_per_gas: u64,

#[clap(
help = "the tx.origin value during EVM execution",
long,
default_value = "0x0000000000000000000000000000000000000000"
)]
pub tx_origin: Address,

#[clap(
help = "the block.coinbase value during EVM execution",
long,
default_value = "0x0000000000000000000000000000000000000000"
)]
pub block_coinbase: Address,
#[clap(
help = "the block.timestamp value during EVM execution",
long,
default_value = "0",
env = "DAPP_TEST_TIMESTAMP"
)]
pub block_timestamp: u64,

#[clap(help = "the block.number value during EVM execution", long, default_value = "0")]
#[clap(env = "DAPP_TEST_NUMBER")]
pub block_number: u64,

#[clap(help = "the block.difficulty value during EVM execution", long, default_value = "0")]
pub block_difficulty: u64,

#[clap(help = "the block.gaslimit value during EVM execution", long)]
pub block_gas_limit: Option<u64>,
// TODO: Add configuration option for base fee.
}

impl Env {
#[cfg(feature = "sputnik-evm")]
pub fn sputnik_state(&self) -> MemoryVicinity {
MemoryVicinity {
chain_id: self.chain_id.into(),

gas_price: self.gas_price.into(),
origin: self.tx_origin,

block_coinbase: self.block_coinbase,
block_number: self.block_number.into(),
block_timestamp: self.block_timestamp.into(),
block_difficulty: self.block_difficulty.into(),
block_base_fee_per_gas: self.block_base_fee_per_gas.into(),
block_gas_limit: self.block_gas_limit.unwrap_or(self.gas_limit).into(),
block_hashes: Vec::new(),
}
}

#[cfg(feature = "evmodin-evm")]
pub fn evmodin_state(&self) -> MockedHost {
let mut host = MockedHost::default();

host.tx_context.chain_id = self.chain_id.into();
host.tx_context.tx_gas_price = self.gas_price.into();
host.tx_context.tx_origin = self.tx_origin;
host.tx_context.block_coinbase = self.block_coinbase;
host.tx_context.block_number = self.block_number;
host.tx_context.block_timestamp = self.block_timestamp;
host.tx_context.block_difficulty = self.block_difficulty.into();
host.tx_context.block_gas_limit = self.block_gas_limit.unwrap_or(self.gas_limit);

host
}
}
68 changes: 41 additions & 27 deletions cli/src/cmd/run.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
use crate::{
cmd::{build::BuildArgs, compile, manual_compile, Cmd},
opts::forge::EvmOpts,
};
use crate::cmd::{build::BuildArgs, compile, manual_compile, Cmd};
use clap::{Parser, ValueHint};
use ethers::abi::Abi;
use forge::ContractRunner;
Expand All @@ -14,13 +11,15 @@ use ethers::solc::{
MinimalCombinedArtifacts, Project, ProjectPathsConfig, SolcConfig,
};

use evm_adapters::Evm;

use ansi_term::Colour;
use ethers::{
prelude::{artifacts::ContractBytecode, Artifact},
solc::artifacts::{CompactContractSome, ContractBytecodeSome},
};
use evm_adapters::{
evm_opts::{BackendKind, EvmOpts},
sputnik::{cheatcodes::debugger::DebugArena, helpers::vm},
};

#[derive(Debug, Clone, Parser)]
pub struct RunArgs {
Expand Down Expand Up @@ -79,25 +78,40 @@ impl Cmd for RunArgs {
let CompactContractSome { abi, bin, .. } = contract;
// this should never fail if compilation was successful
let bytecode = bin.into_bytes().unwrap();

// 2. instantiate the EVM w forked backend if needed / pre-funded account(s)
let mut cfg = crate::utils::sputnik_cfg(self.opts.compiler.evm_version);
let vicinity = self.evm_opts.vicinity()?;
let mut evm = crate::utils::sputnik_helpers::evm(&evm_opts, &mut cfg, &vicinity)?;

// 3. deploy the contract
let (addr, _, _, logs) = evm.deploy(self.evm_opts.sender, bytecode, 0u32.into())?;

// 4. set up the runner
let mut runner =
ContractRunner::new(&mut evm, &abi, addr, Some(self.evm_opts.sender), &logs);

// 5. run the test function & potentially the setup
let needs_setup = abi.functions().any(|func| func.name == "setUp");
let result = runner.run_test(&func, needs_setup, Some(&known_contracts))?;

if self.evm_opts.debug {
// 6. Boot up debugger
let cfg = crate::utils::sputnik_cfg(&self.opts.compiler.evm_version);
let vicinity = evm_opts.vicinity()?;
let backend = evm_opts.backend(&vicinity)?;

// need to match on the backend type
let result = match backend {
BackendKind::Simple(ref backend) => {
let runner = ContractRunner::new(
&evm_opts,
&cfg,
backend,
&abi,
bytecode,
Some(evm_opts.sender),
);
runner.run_test(&func, needs_setup, Some(&known_contracts))?
}
BackendKind::Shared(ref backend) => {
let runner = ContractRunner::new(
&evm_opts,
&cfg,
backend,
&abi,
bytecode,
Some(evm_opts.sender),
);
runner.run_test(&func, needs_setup, Some(&known_contracts))?
}
};

if evm_opts.debug {
// 4. Boot up debugger
let source_code: BTreeMap<u32, String> = sources
.iter()
.map(|(id, path)| {
Expand All @@ -123,7 +137,7 @@ impl Cmd for RunArgs {
})
.collect();

let calls = evm.debug_calls();
let calls: Vec<DebugArena> = result.debug_calls.expect("Debug must be enabled by now");
println!("debugging");
let index = if needs_setup && calls.len() > 1 { 1 } else { 0 };
let mut flattened = Vec::new();
Expand All @@ -149,14 +163,14 @@ impl Cmd for RunArgs {
if evm_opts.verbosity > 4 || !result.success {
// print setup calls as well
traces.iter().for_each(|trace| {
trace.pretty_print(0, &known_contracts, &mut ident, runner.evm, "");
trace.pretty_print(0, &known_contracts, &mut ident, &vm(), "");
});
} else if !traces.is_empty() {
traces.last().expect("no last but not empty").pretty_print(
0,
&known_contracts,
&mut ident,
runner.evm,
&vm(),
"",
);
}
Expand All @@ -165,7 +179,7 @@ impl Cmd for RunArgs {
println!();
}
} else {
// 6. print the result nicely
// 5. print the result nicely
if result.success {
println!("{}", Colour::Green.paint("Script ran successfully."));
} else {
Expand Down
51 changes: 13 additions & 38 deletions cli/src/cmd/test.rs
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,10 @@
//! Test command
use crate::{
cmd::{
build::{BuildArgs, EvmType},
Cmd,
},
opts::forge::EvmOpts,
utils,
};
use crate::cmd::{build::BuildArgs, Cmd};
use ansi_term::Colour;
use clap::{AppSettings, Parser};
use ethers::solc::{ArtifactOutput, Project};
use evm_adapters::{evm_opts::EvmOpts, sputnik::helpers::vm};
use forge::{MultiContractRunnerBuilder, TestFilter};
use std::collections::BTreeMap;

Expand Down Expand Up @@ -118,35 +112,16 @@ impl Cmd for TestArgs {
let project = opts.project()?;

// prepare the test builder
let mut evm_cfg = crate::utils::sputnik_cfg(&opts.compiler.evm_version);
evm_cfg.create_contract_limit = None;

let builder = MultiContractRunnerBuilder::default()
.fuzzer(fuzzer)
.initial_balance(evm_opts.initial_balance)
.evm_cfg(evm_cfg)
.sender(evm_opts.sender);

// run the tests depending on the chosen EVM
match evm_opts.evm_type {
#[cfg(feature = "sputnik-evm")]
EvmType::Sputnik => {
let mut cfg = utils::sputnik_cfg(opts.compiler.evm_version);
let vicinity = evm_opts.vicinity()?;
let evm = utils::sputnik_helpers::evm(&evm_opts, &mut cfg, &vicinity)?;
test(builder, project, evm, filter, json, evm_opts.verbosity, allow_failure)
}
#[cfg(feature = "evmodin-evm")]
EvmType::EvmOdin => {
use evm_adapters::evmodin::EvmOdin;
use evmodin::tracing::NoopTracer;

let revision = utils::evmodin_cfg(opts.compiler.evm_version);

// TODO: Replace this with a proper host. We'll want this to also be
// provided generically when we add the Forking host(s).
let host = evm_opts.env.evmodin_state();

let evm = EvmOdin::new(host, evm_opts.env.gas_limit, revision, NoopTracer);
test(builder, project, evm, filter, json, evm_opts.verbosity, allow_failure)
}
}
test(builder, project, evm_opts, filter, json, allow_failure)
}
}

Expand Down Expand Up @@ -221,16 +196,16 @@ impl TestOutcome {
}

/// Runs all the tests
fn test<A: ArtifactOutput + 'static, S: Clone, E: evm_adapters::Evm<S>>(
fn test<A: ArtifactOutput + 'static>(
builder: MultiContractRunnerBuilder,
project: Project<A>,
evm: E,
evm_opts: EvmOpts,
filter: Filter,
json: bool,
verbosity: u8,
allow_failure: bool,
) -> eyre::Result<TestOutcome> {
let mut runner = builder.build(project, evm)?;
let verbosity = evm_opts.verbosity;
let mut runner = builder.build(project, evm_opts)?;

let results = runner.test(&filter)?;

Expand Down Expand Up @@ -301,7 +276,7 @@ fn test<A: ArtifactOutput + 'static, S: Clone, E: evm_adapters::Evm<S>>(
0,
&runner.known_contracts,
&mut ident,
&runner.evm,
&vm(),
"",
);
});
Expand All @@ -310,7 +285,7 @@ fn test<A: ArtifactOutput + 'static, S: Clone, E: evm_adapters::Evm<S>>(
0,
&runner.known_contracts,
&mut ident,
&runner.evm,
&vm(),
"",
);
}
Expand Down
Loading

0 comments on commit 764c69c

Please sign in to comment.