diff --git a/Cargo.lock b/Cargo.lock index 098f4361be14..7603c83aaba7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2526,6 +2526,7 @@ dependencies = [ "reqwest", "semver", "serde", + "serde_json", "serde_regex", "tempfile", "thiserror", diff --git a/cli/src/cmd/forge/test/mod.rs b/cli/src/cmd/forge/test/mod.rs index 03dbfcf57f69..6b84fc5cdfbf 100644 --- a/cli/src/cmd/forge/test/mod.rs +++ b/cli/src/cmd/forge/test/mod.rs @@ -18,14 +18,14 @@ use forge::{ identifier::{EtherscanIdentifier, LocalTraceIdentifier, SignaturesIdentifier}, CallTraceDecoderBuilder, TraceKind, }, - MultiContractRunner, MultiContractRunnerBuilder, TestOptions, + MultiContractRunner, MultiContractRunnerBuilder, TestOptions, TestOptionsBuilder, }; use foundry_common::{ compile::{self, ProjectCompiler}, evm::EvmArgs, get_contract_name, get_file_name, }; -use foundry_config::{figment, Config}; +use foundry_config::{figment, get_available_profiles, Config}; use regex::Regex; use std::{collections::BTreeMap, path::PathBuf, sync::mpsc::channel, thread, time::Duration}; use tracing::trace; @@ -123,8 +123,6 @@ impl TestArgs { // Merge all configs let (mut config, mut evm_opts) = self.load_config_and_evm_opts_emit_warnings()?; - let test_options = TestOptions { fuzz: config.fuzz, invariant: config.invariant }; - let mut filter = self.filter(&config); trace!(target: "forge::test", ?filter, "using filter"); @@ -150,6 +148,19 @@ impl TestArgs { compiler.compile(&project) }?; + // Create test options from general project settings + // and compiler output + let project_root = &project.paths.root; + let toml = config.get_config_path(); + let profiles = get_available_profiles(toml)?; + + let test_options: TestOptions = TestOptionsBuilder::default() + .fuzz(config.fuzz) + .invariant(config.invariant) + .compile_output(&output) + .profiles(profiles) + .build(project_root)?; + // Determine print verbosity and executor verbosity let verbosity = evm_opts.verbosity; if self.gas_report && evm_opts.verbosity < 3 { @@ -167,8 +178,8 @@ impl TestArgs { .sender(evm_opts.sender) .with_fork(evm_opts.get_fork(&config, env.clone())) .with_cheats_config(CheatsConfig::new(&config, &evm_opts)) - .with_test_options(test_options) - .build(project.paths.root, output, env, evm_opts)?; + .with_test_options(test_options.clone()) + .build(project_root, output, env, evm_opts)?; if self.debug.is_some() { filter.args_mut().test_pattern = self.debug; diff --git a/config/Cargo.toml b/config/Cargo.toml index f71f98b6f698..1bc6b324d93f 100644 --- a/config/Cargo.toml +++ b/config/Cargo.toml @@ -19,6 +19,7 @@ figment = { version = "0.10", features = ["toml", "env"] } number_prefix = "0.4.0" serde = { version = "1.0", features = ["derive"] } serde_regex = "1.1.0" +serde_json = "1.0.95" toml = { version = "0.7", features = ["preserve_order"] } toml_edit = "0.19" diff --git a/config/src/fuzz.rs b/config/src/fuzz.rs index 02937b64f4a0..b43ae10e156c 100644 --- a/config/src/fuzz.rs +++ b/config/src/fuzz.rs @@ -3,6 +3,10 @@ use ethers_core::types::U256; use serde::{Deserialize, Serialize}; +use crate::inline::{ + parse_config_u32, InlineConfigParser, InlineConfigParserError, INLINE_CONFIG_FUZZ_KEY, +}; + /// Contains for fuzz testing #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct FuzzConfig { @@ -35,6 +39,34 @@ impl Default for FuzzConfig { } } +impl InlineConfigParser for FuzzConfig { + fn config_key() -> String { + INLINE_CONFIG_FUZZ_KEY.into() + } + + fn try_merge(&self, configs: &[String]) -> Result, InlineConfigParserError> { + let overrides: Vec<(String, String)> = Self::get_config_overrides(configs); + + if overrides.is_empty() { + return Ok(None) + } + + // self is Copy. We clone it with dereference. + let mut conf_clone = *self; + + for pair in overrides { + let key = pair.0; + let value = pair.1; + match key.as_str() { + "runs" => conf_clone.runs = parse_config_u32(key, value)?, + "max-test-rejects" => conf_clone.max_test_rejects = parse_config_u32(key, value)?, + _ => Err(InlineConfigParserError::InvalidConfigProperty(key))?, + } + } + Ok(Some(conf_clone)) + } +} + /// Contains for fuzz testing #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct FuzzDictionaryConfig { @@ -70,3 +102,57 @@ impl Default for FuzzDictionaryConfig { } } } + +#[cfg(test)] +mod tests { + use crate::{inline::InlineConfigParser, FuzzConfig}; + + #[test] + fn unrecognized_property() { + let configs = &["forge-config: default.fuzz.unknownprop = 200".to_string()]; + let base_config = FuzzConfig::default(); + if let Err(e) = base_config.try_merge(configs) { + assert_eq!(e.to_string(), "'unknownprop' is an invalid config property"); + } else { + assert!(false) + } + } + + #[test] + fn successful_merge() { + let configs = &["forge-config: default.fuzz.runs = 42424242".to_string()]; + let base_config = FuzzConfig::default(); + let merged: FuzzConfig = base_config.try_merge(configs).expect("No errors").unwrap(); + assert_eq!(merged.runs, 42424242); + } + + #[test] + fn merge_is_none() { + let empty_config = &[]; + let base_config = FuzzConfig::default(); + let merged = base_config.try_merge(empty_config).expect("No errors"); + assert!(merged.is_none()); + } + + #[test] + fn merge_is_none_unrelated_property() { + let unrelated_configs = &["forge-config: default.invariant.runs = 2".to_string()]; + let base_config = FuzzConfig::default(); + let merged = base_config.try_merge(unrelated_configs).expect("No errors"); + assert!(merged.is_none()); + } + + #[test] + fn override_detection() { + let configs = &[ + "forge-config: default.fuzz.runs = 42424242".to_string(), + "forge-config: ci.fuzz.runs = 666666".to_string(), + "forge-config: default.invariant.runs = 2".to_string(), + ]; + let variables = FuzzConfig::get_config_overrides(configs); + assert_eq!( + variables, + vec![("runs".into(), "42424242".into()), ("runs".into(), "666666".into())] + ); + } +} diff --git a/config/src/inline/conf_parser.rs b/config/src/inline/conf_parser.rs new file mode 100644 index 000000000000..a64cd6729918 --- /dev/null +++ b/config/src/inline/conf_parser.rs @@ -0,0 +1,206 @@ +use regex::Regex; + +use crate::{InlineConfigError, NatSpec}; + +use super::{remove_whitespaces, INLINE_CONFIG_PREFIX}; + +/// Errors returned by the [`InlineConfigParser`] trait. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum InlineConfigParserError { + /// An invalid configuration property has been provided. + /// The property cannot be mapped to the configuration object + #[error("'{0}' is an invalid config property")] + InvalidConfigProperty(String), + /// An invalid profile has been provided + #[error("'{0}' specifies an invalid profile. Available profiles are: {1}")] + InvalidProfile(String, String), + /// An error occurred while trying to parse an integer configuration value + #[error("Invalid config value for key '{0}'. Unable to parse '{1}' into an integer value")] + ParseInt(String, String), + /// An error occurred while trying to parse a boolean configuration value + #[error("Invalid config value for key '{0}'. Unable to parse '{1}' into a boolean value")] + ParseBool(String, String), +} + +/// This trait is intended to parse configurations from +/// structured text. Foundry users can annotate Solidity test functions, +/// providing special configs just for the execution of a specific test. +/// +/// An example: +/// +/// ```solidity +/// contract MyTest is Test { +/// /// forge-config: default.fuzz.runs = 100 +/// /// forge-config: ci.fuzz.runs = 500 +/// function test_SimpleFuzzTest(uint256 x) public {...} +/// +/// /// forge-config: default.fuzz.runs = 500 +/// /// forge-config: ci.fuzz.runs = 10000 +/// function test_ImportantFuzzTest(uint256 x) public {...} +/// } +/// ``` +pub trait InlineConfigParser +where + Self: Clone + Default + Sized + 'static, +{ + /// Returns a config key that is common to all valid configuration lines + /// for the current impl. This helps to extract correct values out of a text. + /// + /// An example key would be `fuzz` of `invariant`. + fn config_key() -> String; + + /// Tries to override `self` properties with values specified in the `configs` parameter. + /// + /// Returns + /// - `Some(Self)` in case some configurations are merged into self. + /// - `None` in case there are no configurations that can be applied to self. + /// - `Err(InlineConfigParserError)` in case of wrong configuration. + fn try_merge(&self, configs: &[String]) -> Result, InlineConfigParserError>; + + /// Validates all configurations contained in a natspec that apply + /// to the current configuration key. + /// + /// i.e. Given the `invariant` config key and a natspec comment of the form, + /// ```solidity + /// /// forge-config: default.invariant.runs = 500 + /// /// forge-config: default.invariant.depth = 500 + /// /// forge-config: ci.invariant.depth = 500 + /// /// forge-config: ci.fuzz.runs = 10 + /// ``` + /// would validate the whole `invariant` configuration. + fn validate_configs(natspec: &NatSpec) -> Result<(), InlineConfigError> { + let config_key = Self::config_key(); + + let configs = natspec + .config_lines() + .into_iter() + .filter(|l| l.contains(&config_key)) + .collect::>(); + + Self::default().try_merge(&configs).map_err(|e| { + let line = natspec.debug_context(); + InlineConfigError { line, source: e } + })?; + + Ok(()) + } + + /// Given a list of `config_lines, returns all available pairs (key, value) + /// matching the current config key + /// + /// i.e. Given the `invariant` config key and a vector of config lines + /// ```rust + /// let _config_lines = vec![ + /// "forge-config: default.invariant.runs = 500", + /// "forge-config: default.invariant.depth = 500", + /// "forge-config: ci.invariant.depth = 500", + /// "forge-config: ci.fuzz.runs = 10" + /// ]; + /// ``` + /// would return the whole set of `invariant` configs. + /// ```rust + /// let _result = vec![ + /// ("runs", "500"), + /// ("depth", "500"), + /// ("depth", "500"), + /// ]; + /// ``` + fn get_config_overrides(config_lines: &[String]) -> Vec<(String, String)> { + let mut result: Vec<(String, String)> = vec![]; + let config_key = Self::config_key(); + let profile = ".*"; + let prefix = format!("^{INLINE_CONFIG_PREFIX}:{profile}{config_key}\\."); + let re = Regex::new(&prefix).unwrap(); + + config_lines + .iter() + .map(|l| remove_whitespaces(l)) + .filter(|l| re.is_match(l)) + .map(|l| re.replace(&l, "").to_string()) + .for_each(|line| { + let key_value = line.split('=').collect::>(); // i.e. "['runs', '500']" + if let Some(key) = key_value.first() { + if let Some(value) = key_value.last() { + result.push((key.to_string(), value.to_string())); + } + } + }); + + result + } +} + +/// Checks if all configuration lines specified in `natspec` use a valid profile. +/// +/// i.e. Given available profiles +/// ```rust +/// let _profiles = vec!["ci", "default"]; +/// ``` +/// A configuration like `forge-config: ciii.invariant.depth = 1` would result +/// in an error. +pub fn validate_profiles(natspec: &NatSpec, profiles: &[String]) -> Result<(), InlineConfigError> { + for config in natspec.config_lines() { + if !profiles.iter().any(|p| config.starts_with(&format!("{INLINE_CONFIG_PREFIX}:{p}."))) { + let err_line: String = natspec.debug_context(); + let profiles = format!("{profiles:?}"); + Err(InlineConfigError { + source: InlineConfigParserError::InvalidProfile(config, profiles), + line: err_line, + })? + } + } + Ok(()) +} + +/// Tries to parse a `u32` from `value`. The `key` argument is used to give details +/// in the case of an error. +pub fn parse_config_u32(key: String, value: String) -> Result { + value.parse().map_err(|_| InlineConfigParserError::ParseInt(key, value)) +} + +/// Tries to parse a `bool` from `value`. The `key` argument is used to give details +/// in the case of an error. +pub fn parse_config_bool(key: String, value: String) -> Result { + value.parse().map_err(|_| InlineConfigParserError::ParseBool(key, value)) +} + +#[cfg(test)] +mod tests { + use crate::{inline::conf_parser::validate_profiles, NatSpec}; + + #[test] + fn can_reject_invalid_profiles() { + let profiles = ["ci".to_string(), "default".to_string()]; + let natspec = NatSpec { + contract: Default::default(), + function: Default::default(), + line: Default::default(), + docs: r#" + forge-config: ciii.invariant.depth = 1 + forge-config: default.invariant.depth = 1 + "# + .into(), + }; + + let result = validate_profiles(&natspec, &profiles); + assert!(result.is_err()); + } + + #[test] + fn can_accept_valid_profiles() { + let profiles = ["ci".to_string(), "default".to_string()]; + let natspec = NatSpec { + contract: Default::default(), + function: Default::default(), + line: Default::default(), + docs: r#" + forge-config: ci.invariant.depth = 1 + forge-config: default.invariant.depth = 1 + "# + .into(), + }; + + let result = validate_profiles(&natspec, &profiles); + assert!(result.is_ok()); + } +} diff --git a/config/src/inline/mod.rs b/config/src/inline/mod.rs new file mode 100644 index 000000000000..a96f816ade6f --- /dev/null +++ b/config/src/inline/mod.rs @@ -0,0 +1,81 @@ +mod conf_parser; +pub use conf_parser::{ + parse_config_bool, parse_config_u32, validate_profiles, InlineConfigParser, + InlineConfigParserError, +}; +use once_cell::sync::Lazy; +use std::collections::HashMap; + +mod natspec; +pub use natspec::NatSpec; + +use crate::Config; + +pub const INLINE_CONFIG_FUZZ_KEY: &str = "fuzz"; +pub const INLINE_CONFIG_INVARIANT_KEY: &str = "invariant"; +const INLINE_CONFIG_PREFIX: &str = "forge-config"; + +static INLINE_CONFIG_PREFIX_SELECTED_PROFILE: Lazy = Lazy::new(|| { + let selected_profile = Config::selected_profile().to_string(); + format!("{INLINE_CONFIG_PREFIX}:{selected_profile}.") +}); + +/// Wrapper error struct that catches config parsing +/// errors [`InlineConfigParserError`], enriching them with context information +/// reporting the misconfigured line. +#[derive(thiserror::Error, Debug)] +#[error("Inline config error detected at {line}")] +pub struct InlineConfigError { + /// Specifies the misconfigured line. This is something of the form + /// `dir/TestContract.t.sol:FuzzContract:10:12:111` + pub line: String, + /// The inner error + pub source: InlineConfigParserError, +} + +/// Represents a (test-contract, test-function) pair +type InlineConfigKey = (String, String); + +/// Represents per-test configurations, declared inline +/// as structured comments in Solidity test files. This allows +/// to create configs directly bound to a solidity test. +#[derive(Default, Debug, Clone)] +pub struct InlineConfig { + /// Maps a (test-contract, test-function) pair + /// to a specific configuration provided by the user. + configs: HashMap, +} + +impl InlineConfig { + /// Returns an inline configuration, if any, for a test function. + /// Configuration is identified by the pair "contract", "function". + pub fn get>(&self, contract_id: S, fn_name: S) -> Option<&T> { + self.configs.get(&(contract_id.into(), fn_name.into())) + } + + /// Inserts an inline configuration, for a test function. + /// Configuration is identified by the pair "contract", "function". + pub fn insert>(&mut self, contract_id: S, fn_name: S, config: T) { + self.configs.insert((contract_id.into(), fn_name.into()), config); + } +} + +fn remove_whitespaces(s: &str) -> String { + s.chars().filter(|c| !c.is_whitespace()).collect() +} + +#[cfg(test)] +mod tests { + use super::InlineConfigParserError; + use crate::InlineConfigError; + + #[test] + fn can_format_inline_config_errors() { + let source = InlineConfigParserError::ParseBool("key".into(), "invalid-bool-value".into()); + let line = "dir/TestContract.t.sol:FuzzContract".to_string(); + let error = InlineConfigError { line: line.clone(), source: source.clone() }; + + let expected = format!("Inline config error detected at {line}"); + assert_eq!(error.to_string(), expected); + } +} diff --git a/config/src/inline/natspec.rs b/config/src/inline/natspec.rs new file mode 100644 index 000000000000..ae85089c1aa9 --- /dev/null +++ b/config/src/inline/natspec.rs @@ -0,0 +1,238 @@ +use std::{collections::BTreeMap, path::Path}; + +use ethers_solc::{ + artifacts::{ast::NodeType, Node}, + ProjectCompileOutput, +}; +use serde_json::Value; + +use super::{remove_whitespaces, INLINE_CONFIG_PREFIX, INLINE_CONFIG_PREFIX_SELECTED_PROFILE}; + +/// Convenient struct to hold in-line per-test configurations +pub struct NatSpec { + /// The parent contract of the natspec + pub contract: String, + /// The function annotated with the natspec + pub function: String, + /// The line the natspec appears, in the form + /// `row:col:length` i.e. `10:21:122` + pub line: String, + /// The actual natspec comment, without slashes or block + /// punctuation + pub docs: String, +} + +impl NatSpec { + /// Factory function that extracts a vector of [`NatSpec`] instances from + /// a solc compiler output. The root path is to express contract base dirs. + /// That is essential to match per-test configs at runtime. + pub fn parse

(output: &ProjectCompileOutput, root: &P) -> Vec + where + P: AsRef, + { + let mut natspecs: Vec = vec![]; + + let output = output.clone(); + for artifact in output.with_stripped_file_prefixes(root).into_artifacts() { + if let Some(ast) = artifact.1.ast.as_ref() { + let contract: String = artifact.0.identifier(); + if let Some(node) = contract_root_node(&ast.nodes, &contract) { + apply(&mut natspecs, &contract, node) + } + } + } + natspecs + } + + /// Returns a string describing the natspec + /// context, for debugging purposes 🐞 + /// i.e. `test/Counter.t.sol:CounterTest:testSetNumber` + pub fn debug_context(&self) -> String { + format!("{}:{}", self.contract, self.function) + } + + /// Returns a list of configuration lines that match the current profile + pub fn current_profile_configs(&self) -> Vec { + let prefix: &str = INLINE_CONFIG_PREFIX_SELECTED_PROFILE.as_ref(); + self.config_lines_with_prefix(prefix) + } + + /// Returns a list of configuration lines that match a specific string prefix + pub fn config_lines_with_prefix>(&self, prefix: S) -> Vec { + let prefix: String = prefix.into(); + self.config_lines().into_iter().filter(|l| l.starts_with(&prefix)).collect() + } + + /// Returns a list of all the configuration lines available in the natspec + pub fn config_lines(&self) -> Vec { + self.docs + .split('\n') + .map(remove_whitespaces) + .filter(|line| line.contains(INLINE_CONFIG_PREFIX)) + .collect::>() + } +} + +/// Given a list of nodes, find a "ContractDefinition" node that matches +/// the provided contract_id. +fn contract_root_node<'a>(nodes: &'a [Node], contract_id: &'a str) -> Option<&'a Node> { + for n in nodes.iter() { + if let NodeType::ContractDefinition = n.node_type { + let contract_data = &n.other; + if let Value::String(contract_name) = contract_data.get("name")? { + if contract_id.ends_with(contract_name) { + return Some(n) + } + } + } + } + None +} + +/// Implements a DFS over a compiler output node and its children. +/// If a natspec is found it is added to `natspecs` +fn apply(natspecs: &mut Vec, contract: &str, node: &Node) { + for n in node.nodes.iter() { + if let Some((function, docs, line)) = get_fn_data(n) { + natspecs.push(NatSpec { contract: contract.into(), function, line, docs }) + } + apply(natspecs, contract, n); + } +} + +/// Given a compilation output node, if it is a function definition +/// that also contains a natspec then return a tuple of: +/// - Function name +/// - Natspec text +/// - Natspec position with format "row:col:length" +/// +/// Return None otherwise. +fn get_fn_data(node: &Node) -> Option<(String, String, String)> { + if let NodeType::FunctionDefinition = node.node_type { + let fn_data = &node.other; + let fn_name: String = get_fn_name(fn_data)?; + let (fn_docs, docs_src_line): (String, String) = get_fn_docs(fn_data)?; + return Some((fn_name, fn_docs, docs_src_line)) + } + + None +} + +/// Given a dictionary of function data returns the name of the function. +fn get_fn_name(fn_data: &BTreeMap) -> Option { + match fn_data.get("name")? { + Value::String(fn_name) => Some(fn_name.into()), + _ => None, + } +} + +/// Inspects Solc compiler output for documentation comments. Returns: +/// - `Some((String, String))` in case the function has natspec comments. First item is a textual +/// natspec representation, the second item is the natspec src line, in the form "raw:col:length". +/// - `None` in case the function has not natspec comments. +fn get_fn_docs(fn_data: &BTreeMap) -> Option<(String, String)> { + if let Value::Object(fn_docs) = fn_data.get("documentation")? { + if let Value::String(comment) = fn_docs.get("text")? { + if comment.contains(INLINE_CONFIG_PREFIX) { + let mut src_line = fn_docs + .get("src") + .map(|src| src.to_string()) + .unwrap_or(String::from("")); + + src_line.retain(|c| c != '"'); + return Some((comment.into(), src_line)) + } + } + } + None +} + +#[cfg(test)] +mod tests { + use crate::{inline::natspec::get_fn_docs, NatSpec}; + use serde_json::{json, Value}; + use std::collections::BTreeMap; + + #[test] + fn config_lines() { + let natspec = natspec(); + let config_lines = natspec.config_lines(); + assert_eq!( + config_lines, + vec![ + "forge-config:default.fuzz.runs=600".to_string(), + "forge-config:ci.fuzz.runs=500".to_string(), + "forge-config:default.invariant.runs=1".to_string() + ] + ) + } + + #[test] + fn current_profile_configs() { + let natspec = natspec(); + let config_lines = natspec.current_profile_configs(); + + assert_eq!( + config_lines, + vec![ + "forge-config:default.fuzz.runs=600".to_string(), + "forge-config:default.invariant.runs=1".to_string() + ] + ); + } + + #[test] + fn config_lines_with_prefix() { + use super::INLINE_CONFIG_PREFIX; + let natspec = natspec(); + let prefix = format!("{INLINE_CONFIG_PREFIX}:default"); + let config_lines = natspec.config_lines_with_prefix(prefix); + assert_eq!( + config_lines, + vec![ + "forge-config:default.fuzz.runs=600".to_string(), + "forge-config:default.invariant.runs=1".to_string() + ] + ) + } + + #[test] + fn can_handle_unavailable_src_line_with_fallback() { + let mut fn_data: BTreeMap = BTreeMap::new(); + let doc_withouth_src_field = json!({ "text": "forge-config:default.fuzz.runs=600" }); + fn_data.insert("documentation".into(), doc_withouth_src_field); + let (_, src_line) = get_fn_docs(&fn_data).expect("Some docs"); + assert_eq!(src_line, "".to_string()); + } + + #[test] + fn can_handle_available_src_line() { + let mut fn_data: BTreeMap = BTreeMap::new(); + let doc_withouth_src_field = + json!({ "text": "forge-config:default.fuzz.runs=600", "src": "73:21:12" }); + fn_data.insert("documentation".into(), doc_withouth_src_field); + let (_, src_line) = get_fn_docs(&fn_data).expect("Some docs"); + assert_eq!(src_line, "73:21:12".to_string()); + } + + fn natspec() -> NatSpec { + let conf = r#" + forge-config: default.fuzz.runs = 600 + forge-config: ci.fuzz.runs = 500 + ========= SOME NOISY TEXT ============= + 䩹𧀫Jx닧Ʀ̳盅K擷􅟽Ɂw첊}ꏻk86ᖪk-檻ܴ렝[Dz𐤬oᘓƤ + ꣖ۻ%Ƅ㪕ς:(饁΍av/烲ڻ̛߉橞㗡𥺃̹M봓䀖ؿ̄󵼁)𯖛d􂽰񮍃 + ϊ&»ϿЏ񊈞2򕄬񠪁鞷砕eߥH󶑶J粊񁼯머?槿ᴴጅ𙏑ϖ뀓򨙺򷃅Ӽ츙4󍔹 + 醤㭊r􎜕󷾸𶚏 ܖ̹灱녗V*竅􋹲⒪苏贗񾦼=숽ؓ򗋲бݧ󫥛𛲍ʹ園Ьi + ======================================= + forge-config: default.invariant.runs = 1 + "#; + + NatSpec { + contract: "dir/TestContract.t.sol:FuzzContract".to_string(), + function: "test_myFunction".to_string(), + line: "10:12:111".to_string(), + docs: conf.to_string(), + } + } +} diff --git a/config/src/invariant.rs b/config/src/invariant.rs index b71b88d525e6..ad49e40fffb8 100644 --- a/config/src/invariant.rs +++ b/config/src/invariant.rs @@ -1,6 +1,12 @@ //! Configuration for invariant testing -use crate::fuzz::FuzzDictionaryConfig; +use crate::{ + fuzz::FuzzDictionaryConfig, + inline::{ + parse_config_bool, parse_config_u32, InlineConfigParser, InlineConfigParserError, + INLINE_CONFIG_INVARIANT_KEY, + }, +}; use serde::{Deserialize, Serialize}; /// Contains for invariant testing @@ -31,3 +37,84 @@ impl Default for InvariantConfig { } } } + +impl InlineConfigParser for InvariantConfig { + fn config_key() -> String { + INLINE_CONFIG_INVARIANT_KEY.into() + } + + fn try_merge(&self, configs: &[String]) -> Result, InlineConfigParserError> { + let overrides: Vec<(String, String)> = Self::get_config_overrides(configs); + + if overrides.is_empty() { + return Ok(None) + } + + // self is Copy. We clone it with dereference. + let mut conf_clone = *self; + + for pair in overrides { + let key = pair.0; + let value = pair.1; + match key.as_str() { + "runs" => conf_clone.runs = parse_config_u32(key, value)?, + "depth" => conf_clone.depth = parse_config_u32(key, value)?, + "fail-on-revert" => conf_clone.fail_on_revert = parse_config_bool(key, value)?, + "call-override" => conf_clone.call_override = parse_config_bool(key, value)?, + _ => Err(InlineConfigParserError::InvalidConfigProperty(key.to_string()))?, + } + } + Ok(Some(conf_clone)) + } +} + +#[cfg(test)] +mod tests { + use crate::{inline::InlineConfigParser, InvariantConfig}; + + #[test] + fn unrecognized_property() { + let configs = &["forge-config: default.invariant.unknownprop = 200".to_string()]; + let base_config = InvariantConfig::default(); + if let Err(e) = base_config.try_merge(configs) { + assert_eq!(e.to_string(), "'unknownprop' is an invalid config property"); + } else { + assert!(false) + } + } + + #[test] + fn successful_merge() { + let configs = &["forge-config: default.invariant.runs = 42424242".to_string()]; + let base_config = InvariantConfig::default(); + let merged: InvariantConfig = base_config.try_merge(configs).expect("No errors").unwrap(); + assert_eq!(merged.runs, 42424242); + } + + #[test] + fn merge_is_none() { + let empty_config = &[]; + let base_config = InvariantConfig::default(); + let merged = base_config.try_merge(empty_config).expect("No errors"); + assert!(merged.is_none()); + } + + #[test] + fn can_merge_unrelated_properties_into_config() { + let unrelated_configs = &["forge-config: default.fuzz.runs = 2".to_string()]; + let base_config = InvariantConfig::default(); + let merged = base_config.try_merge(unrelated_configs).expect("No errors"); + assert!(merged.is_none()); + } + + #[test] + fn override_detection() { + let configs = &[ + "forge-config: default.fuzz.runs = 42424242".to_string(), + "forge-config: ci.fuzz.runs = 666666".to_string(), + "forge-config: default.invariant.runs = 2".to_string(), + ]; + let variables = InvariantConfig::get_config_overrides(configs); + assert_eq!(variables, vec![("runs".into(), "2".into())]); + } +} diff --git a/config/src/lib.rs b/config/src/lib.rs index b6dac70e17cf..328869408c78 100644 --- a/config/src/lib.rs +++ b/config/src/lib.rs @@ -92,6 +92,9 @@ use crate::fs_permissions::PathPermission; pub use invariant::InvariantConfig; use providers::remappings::RemappingsProvider; +mod inline; +pub use inline::{validate_profiles, InlineConfig, InlineConfigError, InlineConfigParser, NatSpec}; + /// Foundry configuration /// /// # Defaults diff --git a/config/src/utils.rs b/config/src/utils.rs index d51a4d20dc15..abbf3f727cca 100644 --- a/config/src/utils.rs +++ b/config/src/utils.rs @@ -9,6 +9,7 @@ use std::{ path::{Path, PathBuf}, str::FromStr, }; +use toml_edit::{Document, Item}; /// Loads the config for the current project workspace pub fn load_config() -> Config { @@ -168,6 +169,45 @@ pub(crate) fn get_dir_remapping(dir: impl AsRef) -> Option { } } +/// Returns all available `profile` keys in a given `.toml` file +/// +/// i.e. The toml below would return would return `["default", "ci", "local"]` +/// ```toml +/// [profile.default] +/// ... +/// [profile.ci] +/// ... +/// [profile.local] +/// ``` +pub fn get_available_profiles(toml_path: impl AsRef) -> eyre::Result> { + let mut result = vec![Config::DEFAULT_PROFILE.to_string()]; + + if !toml_path.as_ref().exists() { + return Ok(result) + } + + let doc = read_toml(toml_path)?; + + if let Some(Item::Table(profiles)) = doc.as_table().get(Config::PROFILE_SECTION) { + for (_, (profile, _)) in profiles.iter().enumerate() { + let p = profile.to_string(); + if !result.contains(&p) { + result.push(p); + } + } + } + + Ok(result) +} + +/// Returns a [`toml_edit::Document`] loaded from the provided `path`. +/// Can raise an error in case of I/O or parsing errors. +fn read_toml(path: impl AsRef) -> eyre::Result { + let path = path.as_ref().to_owned(); + let doc: Document = std::fs::read_to_string(path)?.parse()?; + Ok(doc) +} + /// Deserialize stringified percent. The value must be between 0 and 100 inclusive. pub(crate) fn deserialize_stringified_percent<'de, D>(deserializer: D) -> Result where @@ -209,3 +249,41 @@ where }; Ok(num) } + +#[cfg(test)] +mod tests { + use crate::get_available_profiles; + use std::path::Path; + + #[test] + fn get_profiles_from_toml() { + figment::Jail::expect_with(|jail| { + jail.create_file( + "foundry.toml", + r#" + [foo.baz] + libs = ['node_modules', 'lib'] + + [profile.default] + libs = ['node_modules', 'lib'] + + [profile.ci] + libs = ['node_modules', 'lib'] + + [profile.local] + libs = ['node_modules', 'lib'] + "#, + )?; + + let path = Path::new("./foundry.toml"); + let profiles = get_available_profiles(path).unwrap(); + + assert_eq!( + profiles, + vec!["default".to_string(), "ci".to_string(), "local".to_string()] + ); + + Ok(()) + }); + } +} diff --git a/forge/src/lib.rs b/forge/src/lib.rs index 591752ac80c4..c6b86d8d1d57 100644 --- a/forge/src/lib.rs +++ b/forge/src/lib.rs @@ -1,3 +1,10 @@ +use std::path::Path; + +use ethers::solc::ProjectCompileOutput; +use foundry_config::{ + validate_profiles, Config, FuzzConfig, InlineConfig, InlineConfigError, InlineConfigParser, + InvariantConfig, NatSpec, +}; use proptest::test_runner::{RngAlgorithm, TestRng, TestRunner}; use tracing::trace; @@ -24,21 +31,77 @@ pub mod result; pub use foundry_evm::*; /// Metadata on how to run fuzz/invariant tests -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Default)] pub struct TestOptions { - /// The fuzz test configuration - pub fuzz: foundry_config::FuzzConfig, - /// The invariant test configuration - pub invariant: foundry_config::InvariantConfig, + /// The base "fuzz" test configuration. To be used as a fallback in case + /// no more specific configs are found for a given run. + pub fuzz: FuzzConfig, + /// The base "invariant" test configuration. To be used as a fallback in case + /// no more specific configs are found for a given run. + pub invariant: InvariantConfig, + /// Contains per-test specific "fuzz" configurations. + pub inline_fuzz: InlineConfig, + /// Contains per-test specific "invariant" configurations. + pub inline_invariant: InlineConfig, } impl TestOptions { - pub fn invariant_fuzzer(&self) -> TestRunner { - self.fuzzer_with_cases(self.invariant.runs) + /// Returns a "fuzz" test runner instance. Parameters are used to select tight scoped fuzz + /// configs that apply for a contract-function pair. A fallback configuration is applied + /// if no specific setup is found for a given input. + /// + /// - `contract_id` is the id of the test contract, expressed as a relative path from the + /// project root. + /// - `test_fn` is the name of the test function declared inside the test contract. + pub fn fuzz_runner(&self, contract_id: S, test_fn: S) -> TestRunner + where + S: Into, + { + let fuzz = self.fuzz_config(contract_id, test_fn); + self.fuzzer_with_cases(fuzz.runs) + } + + /// Returns an "invariant" test runner instance. Parameters are used to select tight scoped fuzz + /// configs that apply for a contract-function pair. A fallback configuration is applied + /// if no specific setup is found for a given input. + /// + /// - `contract_id` is the id of the test contract, expressed as a relative path from the + /// project root. + /// - `test_fn` is the name of the test function declared inside the test contract. + pub fn invariant_runner(&self, contract_id: S, test_fn: S) -> TestRunner + where + S: Into, + { + let invariant = self.invariant_config(contract_id, test_fn); + self.fuzzer_with_cases(invariant.runs) + } + + /// Returns a "fuzz" configuration setup. Parameters are used to select tight scoped fuzz + /// configs that apply for a contract-function pair. A fallback configuration is applied + /// if no specific setup is found for a given input. + /// + /// - `contract_id` is the id of the test contract, expressed as a relative path from the + /// project root. + /// - `test_fn` is the name of the test function declared inside the test contract. + pub fn fuzz_config(&self, contract_id: S, test_fn: S) -> &FuzzConfig + where + S: Into, + { + self.inline_fuzz.get(contract_id, test_fn).unwrap_or(&self.fuzz) } - pub fn fuzzer(&self) -> TestRunner { - self.fuzzer_with_cases(self.fuzz.runs) + /// Returns an "invariant" configuration setup. Parameters are used to select tight scoped + /// invariant configs that apply for a contract-function pair. A fallback configuration is + /// applied if no specific setup is found for a given input. + /// + /// - `contract_id` is the id of the test contract, expressed as a relative path from the + /// project root. + /// - `test_fn` is the name of the test function declared inside the test contract. + pub fn invariant_config(&self, contract_id: S, test_fn: S) -> &InvariantConfig + where + S: Into, + { + self.inline_invariant.get(contract_id, test_fn).unwrap_or(&self.invariant) } pub fn fuzzer_with_cases(&self, cases: u32) -> TestRunner { @@ -62,3 +125,130 @@ impl TestOptions { } } } + +impl<'a, P> TryFrom<(&'a ProjectCompileOutput, &'a P, Vec, FuzzConfig, InvariantConfig)> + for TestOptions +where + P: AsRef, +{ + type Error = InlineConfigError; + + /// Tries to create an instance of `Self`, detecting inline configurations from the project + /// compile output. + /// + /// Param is a tuple, whose elements are: + /// 1. Solidity compiler output, essential to extract natspec test configs. + /// 2. Root path to express contract base dirs. This is essential to match inline configs at + /// runtime. 3. List of available configuration profiles + /// 4. Reference to a fuzz base configuration. + /// 5. Reference to an invariant base configuration. + fn try_from( + value: (&'a ProjectCompileOutput, &'a P, Vec, FuzzConfig, InvariantConfig), + ) -> Result { + let output = value.0; + let root = value.1; + let profiles = &value.2; + let base_fuzz: FuzzConfig = value.3; + let base_invariant: InvariantConfig = value.4; + + let natspecs: Vec = NatSpec::parse(output, root); + let mut inline_invariant = InlineConfig::::default(); + let mut inline_fuzz = InlineConfig::::default(); + + for natspec in natspecs { + // Perform general validation + validate_profiles(&natspec, profiles)?; + FuzzConfig::validate_configs(&natspec)?; + InvariantConfig::validate_configs(&natspec)?; + + // Apply in-line configurations for the current profile + let configs: Vec = natspec.current_profile_configs(); + let c: &str = &natspec.contract; + let f: &str = &natspec.function; + let line: String = natspec.debug_context(); + + match base_fuzz.try_merge(&configs) { + Ok(Some(conf)) => inline_fuzz.insert(c, f, conf), + Err(e) => Err(InlineConfigError { line: line.clone(), source: e })?, + _ => { /* No inline config found, do nothing */ } + } + + match base_invariant.try_merge(&configs) { + Ok(Some(conf)) => inline_invariant.insert(c, f, conf), + Err(e) => Err(InlineConfigError { line: line.clone(), source: e })?, + _ => { /* No inline config found, do nothing */ } + } + } + + Ok(Self { fuzz: base_fuzz, invariant: base_invariant, inline_fuzz, inline_invariant }) + } +} + +/// Builder utility to create a [`TestOptions`] instance. +#[derive(Default)] +pub struct TestOptionsBuilder { + fuzz: Option, + invariant: Option, + profiles: Option>, + output: Option, +} + +impl TestOptionsBuilder { + /// Sets a [`FuzzConfig`] to be used as base "fuzz" configuration. + #[must_use = "A base 'fuzz' config must be provided"] + pub fn fuzz(mut self, conf: FuzzConfig) -> Self { + self.fuzz = Some(conf); + self + } + + /// Sets a [`InvariantConfig`] to be used as base "invariant" configuration. + #[must_use = "A base 'invariant' config must be provided"] + pub fn invariant(mut self, conf: InvariantConfig) -> Self { + self.invariant = Some(conf); + self + } + + /// Sets available configuration profiles. Profiles are useful to validate existing in-line + /// configurations. This argument is necessary in case a `compile_output`is provided. + pub fn profiles(mut self, p: Vec) -> Self { + self.profiles = Some(p); + self + } + + /// Sets a project compiler output instance. This is used to extract + /// inline test configurations that override `self.fuzz` and `self.invariant` + /// specs when necessary. + pub fn compile_output(mut self, output: &ProjectCompileOutput) -> Self { + self.output = Some(output.clone()); + self + } + + /// Creates an instance of [`TestOptions`]. This takes care of creating "fuzz" and + /// "invariant" fallbacks, and extracting all inline test configs, if available. + /// + /// `root` is a reference to the user's project root dir. This is essential + /// to determine the base path of generated contract identifiers. This is to provide correct + /// matchers for inline test configs. + pub fn build(self, root: impl AsRef) -> Result { + let default_profiles = vec![Config::selected_profile().into()]; + let profiles: Vec = self.profiles.unwrap_or(default_profiles); + let base_fuzz = self.fuzz.unwrap_or_default(); + let base_invariant = self.invariant.unwrap_or_default(); + + match self.output { + Some(compile_output) => Ok(TestOptions::try_from(( + &compile_output, + &root, + profiles, + base_fuzz, + base_invariant, + ))?), + None => Ok(TestOptions { + fuzz: base_fuzz, + invariant: base_invariant, + inline_fuzz: InlineConfig::default(), + inline_invariant: InlineConfig::default(), + }), + } + } +} diff --git a/forge/src/multi_runner.rs b/forge/src/multi_runner.rs index a4402033d233..c323c4bd88c4 100644 --- a/forge/src/multi_runner.rs +++ b/forge/src/multi_runner.rs @@ -152,7 +152,7 @@ impl MultiContractRunner { executor, deploy_code.clone(), libs, - (filter, test_options), + (filter, test_options.clone()), )?; tracing::trace!(contract= ?identifier, "executed all tests in contract"); @@ -171,16 +171,15 @@ impl MultiContractRunner { Ok(results) } - // The _name field is unused because we only want it for tracing #[tracing::instrument( name = "contract", skip_all, err, - fields(name = %_name) + fields(name = %name) )] fn run_tests( &self, - _name: &str, + name: &str, contract: &Abi, executor: Executor, deploy_code: Bytes, @@ -188,6 +187,7 @@ impl MultiContractRunner { (filter, test_options): (&impl TestFilter, TestOptions), ) -> Result { let runner = ContractRunner::new( + name, executor, contract, deploy_code, diff --git a/forge/src/runner.rs b/forge/src/runner.rs index f27b1983cc1d..228ac5979c29 100644 --- a/forge/src/runner.rs +++ b/forge/src/runner.rs @@ -11,7 +11,7 @@ use foundry_common::{ contracts::{ContractsByAddress, ContractsByArtifact}, TestFunctionExt, }; -use foundry_config::FuzzConfig; +use foundry_config::{FuzzConfig, InvariantConfig}; use foundry_evm::{ decode::decode_console_logs, executor::{CallResult, DeployResult, EvmError, ExecutionErr, Executor}, @@ -35,9 +35,9 @@ use tracing::{error, trace}; /// A type that executes all tests of a contract #[derive(Debug, Clone)] pub struct ContractRunner<'a> { + pub name: &'a str, /// The executor used by the runner. pub executor: Executor, - /// Library contracts to be deployed before the test contract pub predeploy_libs: &'a [Bytes], /// The deployed contract's code @@ -56,6 +56,7 @@ pub struct ContractRunner<'a> { impl<'a> ContractRunner<'a> { #[allow(clippy::too_many_arguments)] pub fn new( + name: &'a str, executor: Executor, contract: &'a Abi, code: Bytes, @@ -65,6 +66,7 @@ impl<'a> ContractRunner<'a> { predeploy_libs: &'a [Bytes], ) -> Self { Self { + name, executor, contract, code, @@ -288,12 +290,16 @@ impl<'a> ContractRunner<'a> { .par_iter() .flat_map(|(func, should_fail)| { if func.is_fuzz_test() { + let fn_name = &func.name; + let runner = test_options.fuzz_runner(self.name, fn_name); + let fuzz_config = test_options.fuzz_config(self.name, fn_name); + self.run_fuzz_test( func, *should_fail, - test_options.fuzzer(), + runner, setup.clone(), - test_options.fuzz, + *fuzz_config, ) } else { self.clone().run_test(func, *should_fail, setup.clone()) @@ -306,6 +312,7 @@ impl<'a> ContractRunner<'a> { if has_invariants { let identified_contracts = load_contracts(setup.traces.clone(), known_contracts); + let functions: Vec<&Function> = self .contract .functions() @@ -314,14 +321,26 @@ impl<'a> ContractRunner<'a> { }) .collect(); - let results = self.run_invariant_test( - test_options.invariant_fuzzer(), - setup, - test_options, - functions.clone(), - known_contracts, - identified_contracts, - )?; + let mut results: Vec = vec![]; + + for func in functions.iter() { + let fn_name = &func.name; + let runner = test_options.invariant_runner(self.name, fn_name); + let invariant_config = test_options.invariant_config(self.name, fn_name); + + let invariant_results = self.run_invariant_test( + runner, + setup.clone(), + *invariant_config, + vec![func], + known_contracts, + identified_contracts.clone(), + )?; + + for test_result in invariant_results { + results.push(test_result); + } + } results.into_iter().zip(functions.iter()).for_each(|(result, function)| { match result.kind { @@ -471,7 +490,7 @@ impl<'a> ContractRunner<'a> { &mut self, runner: TestRunner, setup: TestSetup, - test_options: TestOptions, + invariant_config: InvariantConfig, functions: Vec<&Function>, known_contracts: Option<&ContractsByArtifact>, identified_contracts: ContractsByAddress, @@ -484,7 +503,7 @@ impl<'a> ContractRunner<'a> { let mut evm = InvariantExecutor::new( &mut self.executor, runner, - test_options.invariant, + invariant_config, &identified_contracts, project_contracts, ); diff --git a/forge/tests/it/config.rs b/forge/tests/it/config.rs index 53deca42e0e4..8a0034a7364f 100644 --- a/forge/tests/it/config.rs +++ b/forge/tests/it/config.rs @@ -32,7 +32,7 @@ impl TestConfig { } pub fn with_filter(runner: MultiContractRunner, filter: Filter) -> Self { - Self { runner, should_fail: false, filter, opts: TEST_OPTS } + Self { runner, should_fail: false, filter, opts: test_opts() } } pub fn filter(filter: Filter) -> Self { @@ -55,7 +55,7 @@ impl TestConfig { /// Executes the test runner pub fn test(&mut self) -> BTreeMap { - self.runner.test(&self.filter, None, self.opts).unwrap() + self.runner.test(&self.filter, None, self.opts.clone()).unwrap() } #[track_caller] @@ -69,7 +69,7 @@ impl TestConfig { /// * filter matched 0 test cases /// * a test results deviates from the configured `should_fail` setting pub fn try_run(&mut self) -> eyre::Result<()> { - let suite_result = self.runner.test(&self.filter, None, self.opts).unwrap(); + let suite_result = self.runner.test(&self.filter, None, self.opts.clone()).unwrap(); if suite_result.is_empty() { eyre::bail!("empty test result"); } @@ -100,33 +100,37 @@ impl Default for TestConfig { } } -pub static TEST_OPTS: TestOptions = TestOptions { - fuzz: FuzzConfig { - runs: 256, - max_test_rejects: 65536, - seed: None, - dictionary: FuzzDictionaryConfig { - include_storage: true, - include_push_bytes: true, - dictionary_weight: 40, - max_fuzz_dictionary_addresses: 10_000, - max_fuzz_dictionary_values: 10_000, +pub fn test_opts() -> TestOptions { + TestOptions { + fuzz: FuzzConfig { + runs: 256, + max_test_rejects: 65536, + seed: None, + dictionary: FuzzDictionaryConfig { + include_storage: true, + include_push_bytes: true, + dictionary_weight: 40, + max_fuzz_dictionary_addresses: 10_000, + max_fuzz_dictionary_values: 10_000, + }, }, - }, - invariant: InvariantConfig { - runs: 256, - depth: 15, - fail_on_revert: false, - call_override: false, - dictionary: FuzzDictionaryConfig { - dictionary_weight: 80, - include_storage: true, - include_push_bytes: true, - max_fuzz_dictionary_addresses: 10_000, - max_fuzz_dictionary_values: 10_000, + invariant: InvariantConfig { + runs: 256, + depth: 15, + fail_on_revert: false, + call_override: false, + dictionary: FuzzDictionaryConfig { + dictionary_weight: 80, + include_storage: true, + include_push_bytes: true, + max_fuzz_dictionary_addresses: 10_000, + max_fuzz_dictionary_values: 10_000, + }, }, - }, -}; + inline_fuzz: Default::default(), + inline_invariant: Default::default(), + } +} pub fn manifest_root() -> PathBuf { let mut root = Path::new(env!("CARGO_MANIFEST_DIR")); diff --git a/forge/tests/it/core.rs b/forge/tests/it/core.rs index ddb042e41606..1d88cc4d8b23 100644 --- a/forge/tests/it/core.rs +++ b/forge/tests/it/core.rs @@ -10,7 +10,7 @@ use std::{collections::BTreeMap, env}; #[test] fn test_core() { let mut runner = runner(); - let results = runner.test(&Filter::new(".*", ".*", ".*core"), None, TEST_OPTS).unwrap(); + let results = runner.test(&Filter::new(".*", ".*", ".*core"), None, test_opts()).unwrap(); assert_multiple( &results, @@ -86,7 +86,7 @@ fn test_core() { #[test] fn test_logs() { let mut runner = runner(); - let results = runner.test(&Filter::new(".*", ".*", ".*logs"), None, TEST_OPTS).unwrap(); + let results = runner.test(&Filter::new(".*", ".*", ".*logs"), None, test_opts()).unwrap(); assert_multiple( &results, @@ -649,7 +649,7 @@ fn test_env_vars() { // test `setEnv` first, and confirm that it can correctly set environment variables, // so that we can use it in subsequent `env*` tests - runner.test(&Filter::new("testSetEnv", ".*", ".*"), None, TEST_OPTS).unwrap(); + runner.test(&Filter::new("testSetEnv", ".*", ".*"), None, test_opts()).unwrap(); let env_var_key = "_foundryCheatcodeSetEnvTestKey"; let env_var_val = "_foundryCheatcodeSetEnvTestVal"; let res = env::var(env_var_key); @@ -664,7 +664,7 @@ Reason: `setEnv` failed to set an environment variable `{env_var_key}={env_var_v fn test_doesnt_run_abstract_contract() { let mut runner = runner(); let results = runner - .test(&Filter::new(".*", ".*", ".*Abstract.t.sol".to_string().as_str()), None, TEST_OPTS) + .test(&Filter::new(".*", ".*", ".*Abstract.t.sol".to_string().as_str()), None, test_opts()) .unwrap(); assert!(results.get("core/Abstract.t.sol:AbstractTestBase").is_none()); assert!(results.get("core/Abstract.t.sol:AbstractTest").is_some()); @@ -673,7 +673,7 @@ fn test_doesnt_run_abstract_contract() { #[test] fn test_trace() { let mut runner = tracing_runner(); - let suite_result = runner.test(&Filter::new(".*", ".*", ".*trace"), None, TEST_OPTS).unwrap(); + let suite_result = runner.test(&Filter::new(".*", ".*", ".*trace"), None, test_opts()).unwrap(); // TODO: This trace test is very basic - it is probably a good candidate for snapshot // testing. diff --git a/forge/tests/it/fork.rs b/forge/tests/it/fork.rs index 340854920fce..337e0530585c 100644 --- a/forge/tests/it/fork.rs +++ b/forge/tests/it/fork.rs @@ -18,7 +18,7 @@ fn test_cheats_fork_revert() { &format!(".*cheats{RE_PATH_SEPARATOR}Fork"), ), None, - TEST_OPTS, + test_opts(), ) .unwrap(); assert_eq!(suite_result.len(), 1); diff --git a/forge/tests/it/fuzz.rs b/forge/tests/it/fuzz.rs index acc4ae027e9c..54439aaad206 100644 --- a/forge/tests/it/fuzz.rs +++ b/forge/tests/it/fuzz.rs @@ -15,7 +15,7 @@ fn test_fuzz() { .exclude_tests(r#"invariantCounter|testIncrement\(address\)|testNeedle\(uint256\)"#) .exclude_paths("invariant"), None, - TEST_OPTS, + test_opts(), ) .unwrap(); @@ -53,12 +53,12 @@ fn test_fuzz() { fn test_fuzz_collection() { let mut runner = runner(); - let mut opts = TEST_OPTS; + let mut opts = test_opts(); opts.invariant.depth = 100; opts.invariant.runs = 1000; opts.fuzz.runs = 1000; opts.fuzz.seed = Some(U256::from(6u32)); - runner.test_options = opts; + runner.test_options = opts.clone(); let results = runner.test(&Filter::new(".*", ".*", ".*fuzz/FuzzCollection.t.sol"), None, opts).unwrap(); diff --git a/forge/tests/it/inline.rs b/forge/tests/it/inline.rs new file mode 100644 index 000000000000..c6c1437a0e69 --- /dev/null +++ b/forge/tests/it/inline.rs @@ -0,0 +1,114 @@ +#[cfg(test)] +mod tests { + use crate::{ + config::runner, + test_helpers::{filter::Filter, COMPILED, PROJECT}, + }; + use forge::{ + result::{SuiteResult, TestKind, TestResult}, + TestOptions, TestOptionsBuilder, + }; + use foundry_config::{FuzzConfig, InvariantConfig}; + + #[test] + fn inline_config_run_fuzz() { + let opts = test_options(); + + let filter = Filter::new(".*", ".*", ".*inline/FuzzInlineConf.t.sol"); + + let mut runner = runner(); + runner.test_options = opts.clone(); + + let result = runner.test(&filter, None, opts).expect("Test ran"); + let suite_result: &SuiteResult = + result.get("inline/FuzzInlineConf.t.sol:FuzzInlineConf").unwrap(); + let test_result: &TestResult = + suite_result.test_results.get("testInlineConfFuzz(uint8)").unwrap(); + match &test_result.kind { + TestKind::Fuzz { runs, .. } => { + assert_eq!(runs, &1024); + } + _ => { + assert!(false); // Force test to fail + } + } + } + + #[test] + fn inline_config_run_invariant() { + const ROOT: &str = "inline/InvariantInlineConf.t.sol"; + + let opts = test_options(); + let filter = Filter::new(".*", ".*", ".*inline/InvariantInlineConf.t.sol"); + let mut runner = runner(); + runner.test_options = opts.clone(); + + let result = runner.test(&filter, None, opts).expect("Test ran"); + + let suite_result_1 = + result.get(&format!("{ROOT}:InvariantInlineConf")).expect("Result exists"); + let suite_result_2 = + result.get(&format!("{ROOT}:InvariantInlineConf2")).expect("Result exists"); + + let test_result_1 = suite_result_1.test_results.get("invariant_neverFalse()").unwrap(); + let test_result_2 = suite_result_2.test_results.get("invariant_neverFalse()").unwrap(); + + match &test_result_1.kind { + TestKind::Invariant { runs, .. } => { + assert_eq!(runs, &333); + } + _ => { + assert!(false); // Force test to fail + } + } + + match &test_result_2.kind { + TestKind::Invariant { runs, .. } => { + assert_eq!(runs, &42); + } + _ => { + assert!(false); // Force test to fail + } + } + } + + #[test] + fn build_test_options() { + let root = &PROJECT.paths.root; + let profiles = vec!["default".to_string(), "ci".to_string()]; + let build_result = TestOptionsBuilder::default() + .fuzz(FuzzConfig::default()) + .invariant(InvariantConfig::default()) + .compile_output(&COMPILED) + .profiles(profiles) + .build(root); + + assert!(build_result.is_ok()); + } + + #[test] + fn build_test_options_just_one_valid_profile() { + let root = &PROJECT.paths.root; + let valid_profiles = vec!["profile-sheldon-cooper".to_string()]; + let build_result = TestOptionsBuilder::default() + .fuzz(FuzzConfig::default()) + .invariant(InvariantConfig::default()) + .compile_output(&COMPILED) + .profiles(valid_profiles) + .build(root); + + // We expect an error, since COMPILED contains in-line + // per-test configs for "default" and "ci" profiles + assert!(build_result.is_err()); + } + + fn test_options() -> TestOptions { + let root = &PROJECT.paths.root; + TestOptionsBuilder::default() + .fuzz(FuzzConfig::default()) + .invariant(InvariantConfig::default()) + .compile_output(&COMPILED) + .build(root) + .expect("Config loaded") + } +} diff --git a/forge/tests/it/invariant.rs b/forge/tests/it/invariant.rs index 211a88de1b70..e49d4f3cffa6 100644 --- a/forge/tests/it/invariant.rs +++ b/forge/tests/it/invariant.rs @@ -13,7 +13,7 @@ fn test_invariant() { .test( &Filter::new(".*", ".*", ".*fuzz/invariant/(target|targetAbi|common)"), None, - TEST_OPTS, + test_opts(), ) .unwrap(); @@ -79,9 +79,9 @@ fn test_invariant() { fn test_invariant_override() { let mut runner = runner(); - let mut opts = TEST_OPTS; + let mut opts = test_opts(); opts.invariant.call_override = true; - runner.test_options = opts; + runner.test_options = opts.clone(); let results = runner .test( @@ -104,10 +104,10 @@ fn test_invariant_override() { fn test_invariant_storage() { let mut runner = runner(); - let mut opts = TEST_OPTS; + let mut opts = test_opts(); opts.invariant.depth = 100; opts.fuzz.seed = Some(U256::from(6u32)); - runner.test_options = opts; + runner.test_options = opts.clone(); let results = runner .test( @@ -137,9 +137,9 @@ fn test_invariant_storage() { fn test_invariant_shrink() { let mut runner = runner(); - let mut opts = TEST_OPTS; + let mut opts = test_opts(); opts.fuzz.seed = Some(U256::from(102u32)); - runner.test_options = opts; + runner.test_options = opts.clone(); let results = runner .test( diff --git a/forge/tests/it/main.rs b/forge/tests/it/main.rs index 31c0cf748672..0ed7348c0bef 100644 --- a/forge/tests/it/main.rs +++ b/forge/tests/it/main.rs @@ -4,6 +4,7 @@ mod core; mod fork; mod fs; mod fuzz; +mod inline; mod invariant; mod repros; mod spec; diff --git a/forge/tests/it/test_helpers.rs b/forge/tests/it/test_helpers.rs index 1b9465f6e8f2..a423b01d223d 100644 --- a/forge/tests/it/test_helpers.rs +++ b/forge/tests/it/test_helpers.rs @@ -83,7 +83,7 @@ pub fn fuzz_executor(executor: &Executor) -> FuzzedExecutor { executor, proptest::test_runner::TestRunner::new(cfg), CALLER, - config::TEST_OPTS.fuzz, + config::test_opts().fuzz, ) } diff --git a/testdata/inline/FuzzInlineConf.t.sol b/testdata/inline/FuzzInlineConf.t.sol new file mode 100644 index 000000000000..8ab3a16d310b --- /dev/null +++ b/testdata/inline/FuzzInlineConf.t.sol @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Unlicense +pragma solidity >=0.8.0; + +import "ds-test/test.sol"; + +contract FuzzInlineConf is DSTest { + /** + * forge-config: default.fuzz.runs = 1024 + * forge-config: default.fuzz.max-test-rejects = 500 + */ + function testInlineConfFuzz(uint8 x) public { + require(true, "this is not going to revert"); + } +} diff --git a/testdata/inline/InvariantInlineConf.t.sol b/testdata/inline/InvariantInlineConf.t.sol new file mode 100644 index 000000000000..41e534f44a90 --- /dev/null +++ b/testdata/inline/InvariantInlineConf.t.sol @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Unlicense +pragma solidity >=0.8.0; + +import "ds-test/test.sol"; + +contract InvariantBreaker { + bool public flag0 = true; + bool public flag1 = true; + + function set0(int256 val) public returns (bool) { + if (val % 100 == 0) { + flag0 = false; + } + return flag0; + } + + function set1(int256 val) public returns (bool) { + if (val % 10 == 0 && !flag0) { + flag1 = false; + } + return flag1; + } +} + +contract InvariantInlineConf is DSTest { + InvariantBreaker inv; + + function setUp() public { + inv = new InvariantBreaker(); + } + + /// forge-config: default.invariant.runs = 333 + /// forge-config: default.invariant.depth = 32 + /// forge-config: default.invariant.fail-on-revert = false + /// forge-config: default.invariant.call-override = true + function invariant_neverFalse() public { + require(true, "this is not going to revert"); + } +} + +contract InvariantInlineConf2 is DSTest { + InvariantBreaker inv; + + function setUp() public { + inv = new InvariantBreaker(); + } + + /// forge-config: default.invariant.runs = 42 + function invariant_neverFalse() public { + require(true, "this is not going to revert"); + } +}