Auto merge of #3008 - oli-obk:ui_test_progress_bars, r=RalfJung

bump ui test crate

The recommended way to run tests locally is `./miri bless -- -- --quiet`, which will show

* progress bars
* the currently running tests (allowing you to see which ones are still running towards the end of the test suite)
* the output of the currently running tests (if they are slow). This means slow running tests can output lines to `stderr` and the last line will be shown after the test name and updated every few hundred milliseconds.

As a side effect this PR also fixes #2998 and only builds dependencies if any tests actually need them (this means that with the next ui_test update we'll be able to merge all our test suites).

Also fixes #3052.

Cargo.toml

@@ -36,7 +36,7 @@ libloading = "0.7"
 
 [dev-dependencies]
 colored = "2"
-ui_test = "0.11.7"
+ui_test = "0.21.1"
 rustc_version = "0.4"
 # Features chosen to match those required by env_logger, to avoid rebuilds
 regex = { version = "1.5.5", default-features = false, features = ["perf", "std"] }

README.md

@@ -478,6 +478,8 @@ Moreover, Miri recognizes some environment variables:
 * `MIRI_TEST_TARGET` (recognized by the test suite and the `./miri` script) indicates which target
   architecture to test against.  `miri` and `cargo miri` accept the `--target` flag for the same
   purpose.
+* `MIRI_TEST_THREADS` (recognized by the test suite): set the number of threads to use for running tests.
+  By default the number of cores is used.
 * `MIRI_NO_STD` (recognized by `cargo miri` and the test suite) makes sure that the target's
   sysroot is built without libstd. This allows testing and running no_std programs.
 * `RUSTC_BLESS` (recognized by the test suite and `cargo-miri-test/run-test.py`): overwrite all

tests/compiletest.rs

@@ -1,10 +1,11 @@
 use colored::*;
 use regex::bytes::Regex;
 use std::ffi::OsString;
+use std::num::NonZeroUsize;
 use std::path::{Path, PathBuf};
 use std::{env, process::Command};
 use ui_test::{color_eyre::Result, Config, Match, Mode, OutputConflictHandling};
-use ui_test::{status_emitter, CommandBuilder};
+use ui_test::{status_emitter, CommandBuilder, Format, RustfixMode};
 
 fn miri_path() -> PathBuf {
     PathBuf::from(option_env!("MIRI").unwrap_or(env!("CARGO_BIN_EXE_miri")))
@@ -78,26 +79,18 @@ fn test_config(target: &str, path: &str, mode: Mode, with_dependencies: bool) ->
         program.args.push(flag);
     }
 
-    let bless = env::var_os("RUSTC_BLESS").is_some_and(|v| v != "0");
-    let skip_ui_checks = env::var_os("MIRI_SKIP_UI_CHECKS").is_some();
-
-    let output_conflict_handling = match (bless, skip_ui_checks) {
-        (false, false) => OutputConflictHandling::Error("./miri test --bless".into()),
-        (true, false) => OutputConflictHandling::Bless,
-        (false, true) => OutputConflictHandling::Ignore,
-        (true, true) => panic!("cannot use RUSTC_BLESS and MIRI_SKIP_UI_CHECKS at the same time"),
-    };
-
     let mut config = Config {
         target: Some(target.to_owned()),
         stderr_filters: STDERR.clone(),
         stdout_filters: STDOUT.clone(),
         mode,
         program,
-        output_conflict_handling,
         out_dir: PathBuf::from(std::env::var_os("CARGO_TARGET_DIR").unwrap()).join("ui"),
         edition: Some("2021".into()),
-        ..Config::rustc(path.into())
+        threads: std::env::var("MIRI_TEST_THREADS")
+            .ok()
+            .map(|threads| NonZeroUsize::new(threads.parse().unwrap()).unwrap()),
+        ..Config::rustc(path)
     };
 
     let use_std = env::var_os("MIRI_NO_STD").is_none();
@@ -120,51 +113,32 @@ fn test_config(target: &str, path: &str, mode: Mode, with_dependencies: bool) ->
 }
 
 fn run_tests(mode: Mode, path: &str, target: &str, with_dependencies: bool) -> Result<()> {
-    let config = test_config(target, path, mode, with_dependencies);
+    let mut config = test_config(target, path, mode, with_dependencies);
 
     // Handle command-line arguments.
-    let mut after_dashdash = false;
-    let mut quiet = false;
-    let filters = std::env::args()
-        .skip(1)
-        .filter(|arg| {
-            if after_dashdash {
-                // Just propagate everything.
-                return true;
-            }
-            match &**arg {
-                "--quiet" => {
-                    quiet = true;
-                    false
-                }
-                "--" => {
-                    after_dashdash = true;
-                    false
-                }
-                s if s.starts_with('-') => {
-                    panic!("unknown compiletest flag `{s}`");
-                }
-                _ => true,
-            }
-        })
-        .collect::<Vec<_>>();
+    let args = ui_test::Args::test()?;
+    let default_bless = env::var_os("RUSTC_BLESS").is_some_and(|v| v != "0");
+    config.with_args(&args, default_bless);
+    if let OutputConflictHandling::Error(msg) = &mut config.output_conflict_handling {
+        *msg = "./miri test --bless".into();
+    }
+    if env::var_os("MIRI_SKIP_UI_CHECKS").is_some() {
+        assert!(!default_bless, "cannot use RUSTC_BLESS and MIRI_SKIP_UI_CHECKS at the same time");
+        config.output_conflict_handling = OutputConflictHandling::Ignore;
+    }
     eprintln!("   Compiler: {}", config.program.display());
     ui_test::run_tests_generic(
-        config,
+        // Only run one test suite. In the future we can add all test suites to one `Vec` and run
+        // them all at once, making best use of systems with high parallelism.
+        vec![config],
         // The files we're actually interested in (all `.rs` files).
-        |path| {
-            path.extension().is_some_and(|ext| ext == "rs")
-                && (filters.is_empty()
-                    || filters.iter().any(|f| path.display().to_string().contains(f)))
-        },
+        ui_test::default_file_filter,
         // This could be used to overwrite the `Config` on a per-test basis.
-        |_, _| None,
+        |_, _, _| {},
         (
-            if quiet {
-                Box::<status_emitter::Quiet>::default()
-                    as Box<dyn status_emitter::StatusEmitter + Send>
-            } else {
-                Box::new(status_emitter::Text)
+            match args.format {
+                Format::Terse => status_emitter::Text::quiet(),
+                Format::Pretty => status_emitter::Text::verbose(),
             },
             status_emitter::Gha::</* GHA Actions groups*/ false> {
                 name: format!("{mode:?} {path} ({target})"),
@@ -269,11 +243,22 @@ fn main() -> Result<()> {
     ui(Mode::Pass, "tests/pass", &target, WithoutDependencies)?;
     ui(Mode::Pass, "tests/pass-dep", &target, WithDependencies)?;
     ui(Mode::Panic, "tests/panic", &target, WithDependencies)?;
-    ui(Mode::Fail { require_patterns: true }, "tests/fail", &target, WithDependencies)?;
+    ui(
+        Mode::Fail { require_patterns: true, rustfix: RustfixMode::Disabled },
+        "tests/fail",
+        &target,
+        WithoutDependencies,
+    )?;
+    ui(
+        Mode::Fail { require_patterns: true, rustfix: RustfixMode::Disabled },
+        "tests/fail-dep",
+        &target,
+        WithDependencies,
+    )?;
     if cfg!(target_os = "linux") {
         ui(Mode::Pass, "tests/extern-so/pass", &target, WithoutDependencies)?;
         ui(
-            Mode::Fail { require_patterns: true },
+            Mode::Fail { require_patterns: true, rustfix: RustfixMode::Disabled },
             "tests/extern-so/fail",
             &target,
             WithoutDependencies,
@@ -285,11 +270,17 @@ fn main() -> Result<()> {
 
 fn run_dep_mode(target: String, mut args: impl Iterator<Item = OsString>) -> Result<()> {
     let path = args.next().expect("./miri run-dep must be followed by a file name");
-    let mut config = test_config(&target, "", Mode::Yolo, /* with dependencies */ true);
+    let mut config = test_config(
+        &target,
+        "",
+        Mode::Yolo { rustfix: RustfixMode::Disabled },
+        /* with dependencies */ true,
+    );
     config.program.args.clear(); // We want to give the user full control over flags
-    config.build_dependencies_and_link_them()?;
+    let dep_args = config.build_dependencies()?;
 
     let mut cmd = config.program.build(&config.out_dir);
+    cmd.args(dep_args);
 
     cmd.arg(path);
 

tests/pass/backtrace/backtrace-api-v0.stdout

@@ -1,5 +1,5 @@
 $DIR/backtrace-api-v0.rs:24:14 (func_d)
 $DIR/backtrace-api-v0.rs:20:5 (func_c)
-$DIR/backtrace-api-v0.rs:9:5 (func_b)
+$DIR/backtrace-api-v0.rs:9:5 (func_b::<u8>)
 $DIR/backtrace-api-v0.rs:5:5 (func_a)
 $DIR/backtrace-api-v0.rs:29:18 (main)