Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add assert redactions #14054

Merged
merged 6 commits into from
Jun 12, 2024
Merged

Add assert redactions #14054

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6914c29
test: Merge common redactions
Muscraft Jun 12, 2024
d7de937
test: Only compile regex replacements once
Muscraft Jun 12, 2024
6a7b15a
test: Auto-redact host target and alt target
Muscraft Jun 10, 2024
02459b7
test: Auto-redact bench timing results
Muscraft Jun 10, 2024
875a25f
test: Auto-redact exit status
Muscraft Jun 12, 2024
190ef86
test: Auto-redact file hash
Muscraft Jun 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 54 additions & 41 deletions crates/cargo-test-support/src/compare.rs
View file
Open in desktop
Muscraft marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,25 @@
//! a problem.
//! - Carriage returns are removed, which can help when running on Windows.

use crate::diff;
use crate::cross_compile::try_alternate;
use crate::paths;
use crate::{diff, rustc_host};
use anyhow::{bail, Context, Result};
use serde_json::Value;
use std::fmt;
use std::path::Path;
use std::str;
use url::Url;

/// This makes it easier to write regex replacements that are guaranteed to only
/// get compiled once
macro_rules! regex {
($re:literal $(,)?) => {{
static RE: std::sync::OnceLock<regex::Regex> = std::sync::OnceLock::new();
RE.get_or_init(|| regex::Regex::new($re).unwrap())
}};
}

/// Assertion policy for UI tests
///
/// This emphasizes showing as much content as possible at the cost of more brittleness
Expand Down Expand Up @@ -77,37 +87,8 @@ use url::Url;
/// a problem.
/// - Carriage returns are removed, which can help when running on Windows.
pub fn assert_ui() -> snapbox::Assert {
let root = paths::root();
// Use `from_file_path` instead of `from_dir_path` so the trailing slash is
// put in the users output, rather than hidden in the variable
let root_url = url::Url::from_file_path(&root).unwrap().to_string();

let mut subs = snapbox::Redactions::new();
subs.extend(MIN_LITERAL_REDACTIONS.into_iter().cloned())
.unwrap();
subs.insert("[ROOT]", root).unwrap();
subs.insert("[ROOTURL]", root_url).unwrap();
subs.insert(
"[ELAPSED]",
regex::Regex::new("Finished.*in (?<redacted>[0-9]+(\\.[0-9]+))s").unwrap(),
)
.unwrap();
// output from libtest
subs.insert(
"[ELAPSED]",
regex::Regex::new("; finished in (?<redacted>[0-9]+(\\.[0-9]+))s").unwrap(),
)
.unwrap();
subs.insert(
"[FILE_SIZE]",
regex::Regex::new("(?<redacted>[0-9]+(\\.[0-9]+)([a-zA-Z]i)?)B").unwrap(),
)
.unwrap();
subs.insert(
"[HASH]",
regex::Regex::new("home/\\.cargo/registry/src/-(?<redacted>[a-z0-9]+)").unwrap(),
)
.unwrap();
add_common_redactions(&mut subs);
snapbox::Assert::new()
.action_env(snapbox::assert::DEFAULT_ACTION_ENV)
.redact_with(subs)
Expand Down Expand Up @@ -145,48 +126,80 @@ pub fn assert_ui() -> snapbox::Assert {
/// a problem.
/// - Carriage returns are removed, which can help when running on Windows.
pub fn assert_e2e() -> snapbox::Assert {
let mut subs = snapbox::Redactions::new();
add_common_redactions(&mut subs);
subs.extend(E2E_LITERAL_REDACTIONS.into_iter().cloned())
.unwrap();

snapbox::Assert::new()
.action_env(snapbox::assert::DEFAULT_ACTION_ENV)
.redact_with(subs)
}

fn add_common_redactions(subs: &mut snapbox::Redactions) {
let root = paths::root();
// Use `from_file_path` instead of `from_dir_path` so the trailing slash is
// put in the users output, rather than hidden in the variable
let root_url = url::Url::from_file_path(&root).unwrap().to_string();

let mut subs = snapbox::Redactions::new();
subs.extend(MIN_LITERAL_REDACTIONS.into_iter().cloned())
.unwrap();
subs.extend(E2E_LITERAL_REDACTIONS.into_iter().cloned())
.unwrap();
subs.insert("[ROOT]", root).unwrap();
subs.insert("[ROOTURL]", root_url).unwrap();
// For e2e tests
subs.insert(
"[ELAPSED]",
regex::Regex::new("[FINISHED].*in (?<redacted>[0-9]+(\\.[0-9]+))s").unwrap(),
regex!("[FINISHED].*in (?<redacted>[0-9]+(\\.[0-9]+))s"),
)
.unwrap();
// for UI tests
subs.insert(
"[ELAPSED]",
regex!("Finished.*in (?<redacted>[0-9]+(\\.[0-9]+))s"),
)
.unwrap();
// output from libtest
subs.insert(
"[ELAPSED]",
regex::Regex::new("; finished in (?<redacted>[0-9]+(\\.[0-9]+))s").unwrap(),
regex!("; finished in (?<redacted>[0-9]+(\\.[0-9]+))s"),
)
.unwrap();
subs.insert(
"[FILE_SIZE]",
regex::Regex::new("(?<redacted>[0-9]+(\\.[0-9]+)([a-zA-Z]i)?)B").unwrap(),
regex!("(?<redacted>[0-9]+(\\.[0-9]+)([a-zA-Z]i)?)B"),
)
.unwrap();
subs.insert(
"[HASH]",
regex::Regex::new("home/\\.cargo/registry/src/-(?<redacted>[a-z0-9]+)").unwrap(),
regex!("home/\\.cargo/registry/src/-(?<redacted>[a-z0-9]+)"),
)
.unwrap();
subs.insert("[HASH]", regex!("/[a-z0-9\\-_]+-(?<redacted>[0-9a-f]{16})"))
.unwrap();
subs.insert("[HOST_TARGET]", rustc_host()).unwrap();
if let Some(alt_target) = try_alternate() {
subs.insert("[ALT_TARGET]", alt_target).unwrap();
}
subs.insert(
"[AVG_ELAPSED]",
regex!("(?<redacted>[0-9]+(\\.[0-9]+)?) ns/iter"),
)
.unwrap();
subs.insert(
"[JITTER]",
regex!("ns/iter \\(\\+/- (?<redacted>[0-9]+(\\.[0-9]+)?)\\)"),
)
.unwrap();
snapbox::Assert::new()
.action_env(snapbox::assert::DEFAULT_ACTION_ENV)
.redact_with(subs)
}

static MIN_LITERAL_REDACTIONS: &[(&str, &str)] = &[
("[EXE]", std::env::consts::EXE_SUFFIX),
("[BROKEN_PIPE]", "Broken pipe (os error 32)"),
("[BROKEN_PIPE]", "The pipe is being closed. (os error 232)"),
// Unix message for exit status
("[EXIT_STATUS]", "exit status"),
// Windows message for exit status
("[EXIT_STATUS]", "exit code"),
];
static E2E_LITERAL_REDACTIONS: &[(&str, &str)] = &[
("[RUNNING]", " Running"),
Expand Down
17 changes: 11 additions & 6 deletions crates/cargo-test-support/src/cross_compile.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,18 +209,23 @@ pub fn native_arch() -> &'static str {
///
/// Only use this function on tests that check `cross_compile::disabled`.
pub fn alternate() -> &'static str {
try_alternate().expect("This test should be gated on cross_compile::disabled.")
}

/// A possible alternate target-triple to build with.
pub(crate) fn try_alternate() -> Option<&'static str> {
if cfg!(all(target_os = "macos", target_arch = "aarch64")) {
"x86_64-apple-darwin"
Some("x86_64-apple-darwin")
} else if cfg!(target_os = "macos") {
"x86_64-apple-ios"
Some("x86_64-apple-ios")
} else if cfg!(target_os = "linux") {
"i686-unknown-linux-gnu"
Some("i686-unknown-linux-gnu")
} else if cfg!(all(target_os = "windows", target_env = "msvc")) {
"i686-pc-windows-msvc"
Some("i686-pc-windows-msvc")
} else if cfg!(all(target_os = "windows", target_env = "gnu")) {
"i686-pc-windows-gnu"
Some("i686-pc-windows-gnu")
} else {
panic!("This test should be gated on cross_compile::disabled.");
None
}
}

Expand Down
2 changes: 1 addition & 1 deletion tests/testsuite/cargo_add/target/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ fn case() {

snapbox::cmd::Command::cargo_ui()
.arg("add")
.arg_line("my-package1 my-package2 --target i686-unknown-linux-gnu")
.arg_line("my-package1 my-package2 --target wasm32-unknown-unknown")
.current_dir(cwd)
.assert()
.success()
Expand Down
2 changes: 1 addition & 1 deletion tests/testsuite/cargo_add/target/out/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@ name = "cargo-list-test-fixture"
version = "0.0.0"
edition = "2015"

[target.i686-unknown-linux-gnu.dependencies]
[target.wasm32-unknown-unknown.dependencies]
my-package1 = "99999.0.0"
my-package2 = "99999.0.0"
4 changes: 2 additions & 2 deletions tests/testsuite/cargo_add/target/stderr.term.svg
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
6 changes: 3 additions & 3 deletions tests/testsuite/cargo_remove/invalid_target/out/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ path = "src/main.rs"
[target.x86_64-unknown-freebsd.build-dependencies]
semver = "0.1.0"

[target.x86_64-unknown-linux-gnu.build-dependencies]
[target.wasm32-unknown-unknown.build-dependencies]
semver = "0.1.0"

[dependencies]
Expand All @@ -20,14 +20,14 @@ semver = "0.1"
toml = "0.1"
clippy = "0.4"

[target.x86_64-unknown-linux-gnu.dependencies]
[target.wasm32-unknown-unknown.dependencies]
dbus = "0.6.2"

[dev-dependencies]
regex = "0.1.1"
serde = "1.0.90"

[target.x86_64-unknown-linux-gnu.dev-dependencies]
[target.wasm32-unknown-unknown.dev-dependencies]
ncurses = "20.0"

[features]
Expand Down
2 changes: 1 addition & 1 deletion tests/testsuite/cargo_remove/invalid_target/stderr.term.svg
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
2 changes: 1 addition & 1 deletion tests/testsuite/cargo_remove/invalid_target_dep/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ fn case() {

snapbox::cmd::Command::cargo_ui()
.arg("remove")
.args(["--target", "x86_64-unknown-linux-gnu", "toml"])
.args(["--target", "wasm32-unknown-unknown", "toml"])
.current_dir(cwd)
.assert()
.code(101)
Expand Down
6 changes: 3 additions & 3 deletions tests/testsuite/cargo_remove/invalid_target_dep/out/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ path = "src/main.rs"
[target.x86_64-unknown-freebsd.build-dependencies]
semver = "0.1.0"

[target.x86_64-unknown-linux-gnu.build-dependencies]
[target.wasm32-unknown-unknown.build-dependencies]
semver = "0.1.0"

[dependencies]
Expand All @@ -20,14 +20,14 @@ semver = "0.1"
toml = "0.1"
clippy = "0.4"

[target.x86_64-unknown-linux-gnu.dependencies]
[target.wasm32-unknown-unknown.dependencies]
dbus = "0.6.2"

[dev-dependencies]
regex = "0.1.1"
serde = "1.0.90"

[target.x86_64-unknown-linux-gnu.dev-dependencies]
[target.wasm32-unknown-unknown.dev-dependencies]
ncurses = "20.0"

[features]
Expand Down
4 changes: 2 additions & 2 deletions tests/testsuite/cargo_remove/invalid_target_dep/stderr.term.svg
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
6 changes: 3 additions & 3 deletions tests/testsuite/cargo_remove/remove-target.in/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ path = "src/main.rs"
[target.x86_64-unknown-freebsd.build-dependencies]
semver = "0.1.0"

[target.x86_64-unknown-linux-gnu.build-dependencies]
[target.wasm32-unknown-unknown.build-dependencies]
semver = "0.1.0"

[dependencies]
Expand All @@ -20,14 +20,14 @@ semver = "0.1"
toml = "0.1"
clippy = "0.4"

[target.x86_64-unknown-linux-gnu.dependencies]
[target.wasm32-unknown-unknown.dependencies]
dbus = "0.6.2"

[dev-dependencies]
regex = "0.1.1"
serde = "1.0.90"

[target.x86_64-unknown-linux-gnu.dev-dependencies]
[target.wasm32-unknown-unknown.dev-dependencies]
ncurses = "20.0"

[features]
Expand Down
2 changes: 1 addition & 1 deletion tests/testsuite/cargo_remove/target/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ fn case() {

snapbox::cmd::Command::cargo_ui()
.arg("remove")
.args(["--target", "x86_64-unknown-linux-gnu", "dbus"])
.args(["--target", "wasm32-unknown-unknown", "dbus"])
.current_dir(cwd)
.assert()
.success()
Expand Down
4 changes: 2 additions & 2 deletions tests/testsuite/cargo_remove/target/out/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ path = "src/main.rs"
[target.x86_64-unknown-freebsd.build-dependencies]
semver = "0.1.0"

[target.x86_64-unknown-linux-gnu.build-dependencies]
[target.wasm32-unknown-unknown.build-dependencies]
semver = "0.1.0"

[dependencies]
Expand All @@ -24,7 +24,7 @@ clippy = "0.4"
regex = "0.1.1"
serde = "1.0.90"

[target.x86_64-unknown-linux-gnu.dev-dependencies]
[target.wasm32-unknown-unknown.dev-dependencies]
ncurses = "20.0"

[features]
Expand Down
2 changes: 1 addition & 1 deletion tests/testsuite/cargo_remove/target/stderr.term.svg
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
2 changes: 1 addition & 1 deletion tests/testsuite/cargo_remove/target_build/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ fn case() {

snapbox::cmd::Command::cargo_ui()
.arg("remove")
.args(["--build", "--target", "x86_64-unknown-linux-gnu", "semver"])
.args(["--build", "--target", "wasm32-unknown-unknown", "semver"])
.current_dir(cwd)
.assert()
.success()
Expand Down
4 changes: 2 additions & 2 deletions tests/testsuite/cargo_remove/target_build/out/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ semver = "0.1"
toml = "0.1"
clippy = "0.4"

[target.x86_64-unknown-linux-gnu.dependencies]
[target.wasm32-unknown-unknown.dependencies]
dbus = "0.6.2"

[dev-dependencies]
regex = "0.1.1"
serde = "1.0.90"

[target.x86_64-unknown-linux-gnu.dev-dependencies]
[target.wasm32-unknown-unknown.dev-dependencies]
ncurses = "20.0"

[features]
Expand Down
2 changes: 1 addition & 1 deletion tests/testsuite/cargo_remove/target_build/stderr.term.svg
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading