feat(forge test): stream test results

cli/README.md

@@ -265,7 +265,7 @@ You can optionally specify a regular expression, to only run matching functions:
 ```bash
 $ forge test -m Cannot
 $HOME/oss/foundry/target/release/forge test -m Cannot
-no files changed, compilation skipped.
+no files changed, compilation skipped
 Running 1 test for "Greet.json":Greet
 [PASS] testCannotGm (gas: 6819)
 
@@ -279,7 +279,7 @@ the `--json` flag
 
 ```bash
 $ forge test --json
-no files changed, compilation skipped.
+no files changed, compilation skipped
 {"\"Gm.json\":Gm":{"testNonOwnerCannotGm":{"success":true,"reason":null,"gas_used":3782,"counterexample":null,"logs":[]},"testOwnerCannotGmOnBadBlocks":{"success":true,"reason":null,"gas_used":7771,"counterexample":null,"logs":[]},"testOwnerCanGmOnGoodBlocks":{"success":true,"reason":null,"gas_used":31696,"counterexample":null,"logs":[]}},"\"Greet.json\":Greet":{"testWorksForAllGreetings":{"success":true,"reason":null,"gas_used":null,"counterexample":null,"logs":[]},"testCannotGm":{"success":true,"reason":null,"gas_used":6819,"counterexample":null,"logs":[]},"testCanSetGreeting":{"success":true,"reason":null,"gas_used":31070,"counterexample":null,"logs":[]}}}
 ```
 

cli/src/cmd/mod.rs

@@ -81,27 +81,27 @@ If you are in a subdirectory in a Git repository, try adding `--root .`"#,
         );
     }
 
-    println!("compiling...");
+    println!("Compiling...");
     let output = project.compile()?;
     if output.has_compiler_errors() {
         eyre::bail!(output.to_string())
     } else if output.is_unchanged() {
-        println!("no files changed, compilation skipped.");
+        println!("No files changed, compilation skipped");
     } else {
         println!("{}", output);
-        println!("success.");
+        println!("Success");
     }
     Ok(output)
 }
 
 /// Compile a set of files not necessarily included in the `project`'s source dir
 pub fn compile_files(project: &Project, files: Vec<PathBuf>) -> eyre::Result<ProjectCompileOutput> {
-    println!("compiling...");
+    println!("Compiling...");
     let output = project.compile_files(files)?;
     if output.has_compiler_errors() {
         eyre::bail!(output.to_string())
     }
-    println!("success.");
+    println!("Success");
     Ok(output)
 }
 

cli/src/cmd/test.rs

@@ -10,10 +10,14 @@ use ethers::solc::{ArtifactOutput, Project};
 use evm_adapters::{
     call_tracing::ExecutionInfo, evm_opts::EvmOpts, gas_report::GasReport, sputnik::helpers::vm,
 };
-use forge::{MultiContractRunnerBuilder, TestFilter};
+use forge::{MultiContractRunnerBuilder, TestFilter, TestResult, TestResultStreamFn};
 use foundry_config::{figment::Figment, Config};
 use regex::Regex;
-use std::{collections::BTreeMap, str::FromStr};
+use std::{
+    collections::BTreeMap,
+    str::FromStr,
+    sync::{Arc, Mutex},
+};
 
 #[derive(Debug, Clone, Parser)]
 pub struct Filter {
@@ -311,124 +315,123 @@ fn test<A: ArtifactOutput + 'static>(
 ) -> eyre::Result<TestOutcome> {
     let verbosity = evm_opts.verbosity;
     let gas_reporting = gas_reports.0;
-
     if gas_reporting && evm_opts.verbosity < 3 {
         // force evm to do tracing, but dont hit the verbosity print path
         evm_opts.verbosity = 3;
     }
-
     let mut runner = builder.build(project, evm_opts)?;
 
-    let results = runner.test(&filter)?;
-
-    let mut gas_report = GasReport::new(gas_reports.1);
-
-    let (funcs, events, errors) = runner.execution_info;
     if json {
-        let res = serde_json::to_string(&results)?;
+        let results = runner.test(&filter, None)?;
+        let res = serde_json::to_string(&results)?; // TODO: Make this work normally
         println!("{}", res);
+        Ok(TestOutcome::new(results, allow_failure))
     } else {
         // Dapptools-style printing of test results
-        for (i, (contract_name, tests)) in results.iter().enumerate() {
-            if i > 0 {
-                println!()
-            }
-            if !tests.is_empty() {
-                let term = if tests.len() > 1 { "tests" } else { "test" };
-                println!("Running {} {} for {}", tests.len(), term, contract_name);
-            }
-
-            for (name, result) in tests {
-                // build up gas report
-                if gas_reporting {
-                    if let (Some(traces), Some(identified_contracts)) =
-                        (&result.traces, &result.identified_contracts)
-                    {
-                        gas_report.analyze(traces, identified_contracts);
-                    }
+        let mut gas_report = GasReport::new(gas_reports.1);
+        let execution_info = runner.execution_info.clone();
+        let known_contracts = runner.known_contracts.clone();
+        let index = Arc::new(Mutex::new(0));
+
+        let results = runner.test(
+            &filter,
+            Some(Box::new(move |contract_name: String, tests: BTreeMap<String, TestResult>| {
+                // This mutex syncs stdout per contract, do not remove even if the newline below is
+                let mut index = index.lock().unwrap();
+                if *index > 0 {
+                    println!()
                 }
-
-                short_test_result(name, result);
-
-                // adds a linebreak only if there were any traces or logs, so that the
-                // output does not look like 1 big block.
-                let mut add_newline = false;
-                if verbosity > 1 && !result.logs.is_empty() {
-                    add_newline = true;
-                    println!("Logs:");
-                    for log in &result.logs {
-                        println!("  {}", log);
+                *index += 1;
+                if !tests.is_empty() {
+                    let term = if tests.len() > 1 { "tests" } else { "test" };
+                    println!("Running {} {} for {}", tests.len(), term, contract_name);
+                }
+                for (name, result) in tests {
+                    short_test_result(&name, &result);
+                    // adds a linebreak only if there were any traces or logs, so that the
+                    // output does not look like 1 big block.
+                    let mut add_newline = false;
+                    if verbosity > 1 && !result.logs.is_empty() {
+                        add_newline = true;
+                        println!("Logs:");
+                        for log in &result.logs {
+                            println!("  {}", log);
+                        }
+                    }
+                    if verbosity > 2 {
+                        if let (Some(traces), Some(identified_contracts)) =
+                            (&result.traces, &result.identified_contracts)
+                        {
+                            if !result.success && verbosity == 3 || verbosity > 3 {
+                                // add a new line if any logs were printed & to separate them from
+                                // the traces to be printed
+                                if !result.logs.is_empty() {
+                                    println!();
+                                }
+                                let mut ident = identified_contracts.clone();
+                                let mut exec_info = ExecutionInfo::new(
+                                    &known_contracts,
+                                    &mut ident,
+                                    &result.labeled_addresses,
+                                    &execution_info.0,
+                                    &execution_info.1,
+                                    &execution_info.2,
+                                );
+                                let vm = vm();
+                                let mut trace_string = "".to_string();
+                                if verbosity > 4 || !result.success {
+                                    add_newline = true;
+                                    println!("Traces:");
+                                    // print setup calls as well
+                                    traces.iter().for_each(|trace| {
+                                        trace.construct_trace_string(
+                                            0,
+                                            &mut exec_info,
+                                            &vm,
+                                            "  ",
+                                            &mut trace_string,
+                                        );
+                                    });
+                                } else if !traces.is_empty() {
+                                    add_newline = true;
+                                    println!("Traces:");
+                                    traces
+                                        .last()
+                                        .expect("no last but not empty")
+                                        .construct_trace_string(
+                                            0,
+                                            &mut exec_info,
+                                            &vm,
+                                            "  ",
+                                            &mut trace_string,
+                                        );
+                                }
+                                if !trace_string.is_empty() {
+                                    println!("{}", trace_string);
+                                }
+                            }
+                        }
+                    }
+                    if add_newline {
+                        println!();
                     }
                 }
+            }) as TestResultStreamFn),
+        )?;
 
-                if verbosity > 2 {
+        if gas_reporting {
+            for tests in results.values() {
+                for result in tests.values() {
                     if let (Some(traces), Some(identified_contracts)) =
                         (&result.traces, &result.identified_contracts)
                     {
-                        if !result.success && verbosity == 3 || verbosity > 3 {
-                            // add a new line if any logs were printed & to separate them from
-                            // the traces to be printed
-                            if !result.logs.is_empty() {
-                                println!();
-                            }
-
-                            let mut ident = identified_contracts.clone();
-                            let mut exec_info = ExecutionInfo::new(
-                                &runner.known_contracts,
-                                &mut ident,
-                                &result.labeled_addresses,
-                                &funcs,
-                                &events,
-                                &errors,
-                            );
-                            let vm = vm();
-                            let mut trace_string = "".to_string();
-                            if verbosity > 4 || !result.success {
-                                add_newline = true;
-                                println!("Traces:");
-
-                                // print setup calls as well
-                                traces.iter().for_each(|trace| {
-                                    trace.construct_trace_string(
-                                        0,
-                                        &mut exec_info,
-                                        &vm,
-                                        "  ",
-                                        &mut trace_string,
-                                    );
-                                });
-                            } else if !traces.is_empty() {
-                                add_newline = true;
-                                println!("Traces:");
-                                traces
-                                    .last()
-                                    .expect("no last but not empty")
-                                    .construct_trace_string(
-                                        0,
-                                        &mut exec_info,
-                                        &vm,
-                                        "  ",
-                                        &mut trace_string,
-                                    );
-                            }
-                            if !trace_string.is_empty() {
-                                println!("{}", trace_string);
-                            }
-                        }
+                        gas_report.analyze(traces, identified_contracts);
                     }
                 }
-
-                if add_newline {
-                    println!();
-                }
             }
+            gas_report.finalize();
+            println!("{}", gas_report);
         }
+        Ok(TestOutcome::new(results, allow_failure))
     }
-
-    if gas_reporting {
-        gas_report.finalize();
-        println!("{}", gas_report);
-    }
-
-    Ok(TestOutcome::new(results, allow_failure))
 }

cli/tests/cmd.rs

@@ -282,11 +282,11 @@ contract Greeter {}
     cmd.arg("build");
 
     assert!(cmd.stdout_lossy().ends_with(
-        "compiling...
+        "Compiling...
 Compiling 1 files with 0.8.10
 Compilation finished successfully
 Compiler run successful
-success.
+Success
 ",
     ));
 });
@@ -335,11 +335,11 @@ library FooLib {
     cmd.arg("build");
 
     assert_eq!(
-        "compiling...
+        "Compiling...
 Compiling 2 files with 0.8.10
 Compilation finished successfully
 Compiler run successful
-success.
+Success
 ",
         cmd.stdout_lossy()
     );
@@ -380,11 +380,11 @@ contract Foo {
     prj.write_config(config);
 
     assert!(cmd.stdout_lossy().ends_with(
-        "compiling...
+        "Compiling...
 Compiling 1 files with 0.8.10
 Compilation finished successfully
 Compiler run successful
-success.
+Success
 ",
     ));
 });
@@ -412,10 +412,10 @@ contract Demo {
     let output = cmd.stdout_lossy();
     assert_eq!(
         format!(
-            "compiling...
+            "Compiling...
 Compiling 1 files with 0.8.10
 Compilation finished successfully
-success.
+Success
 {}
 Gas Used: 1751
 == Logs ==

forge/src/lib.rs

@@ -2,7 +2,7 @@ mod runner;
 pub use runner::{ContractRunner, TestKind, TestKindGas, TestResult};
 
 mod multi_runner;
-pub use multi_runner::{MultiContractRunner, MultiContractRunnerBuilder};
+pub use multi_runner::{MultiContractRunner, MultiContractRunnerBuilder, TestResultStreamFn};
 
 pub trait TestFilter {
     fn matches_test(&self, test_name: impl AsRef<str>) -> bool;