diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 93e5980f1f5fb..edfdae435974e 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use std::{env, iter, str}; use anyhow::Context; +use build_helper::ci::CiEnv; use colored::Colorize; use regex::{Captures, Regex}; use tracing::*; @@ -22,7 +23,7 @@ use crate::common::{ UI_STDERR, UI_STDOUT, UI_SVG, UI_WINDOWS_SVG, Ui, expected_output_path, incremental_dir, output_base_dir, output_base_name, output_testname_unique, }; -use crate::compute_diff::{write_diff, write_filtered_diff}; +use crate::compute_diff::{DiffLine, make_diff, write_diff, write_filtered_diff}; use crate::errors::{self, Error, ErrorKind}; use crate::header::TestProps; use crate::read2::{Truncated, read2_abbreviated}; @@ -2292,17 +2293,31 @@ impl<'test> TestCx<'test> { match output_kind { TestOutput::Compile => { if !self.props.dont_check_compiler_stdout { - errors += - self.compare_output(stdout_kind, &normalized_stdout, &expected_stdout); + errors += self.compare_output( + stdout_kind, + &normalized_stdout, + &proc_res.stdout, + &expected_stdout, + ); } if !self.props.dont_check_compiler_stderr { - errors += - self.compare_output(stderr_kind, &normalized_stderr, &expected_stderr); + errors += self.compare_output( + stderr_kind, + &normalized_stderr, + &stderr, + &expected_stderr, + ); } } TestOutput::Run => { - errors += self.compare_output(stdout_kind, &normalized_stdout, &expected_stdout); - errors += self.compare_output(stderr_kind, &normalized_stderr, &expected_stderr); + errors += self.compare_output( + stdout_kind, + &normalized_stdout, + &proc_res.stdout, + &expected_stdout, + ); + errors += + self.compare_output(stderr_kind, &normalized_stderr, &stderr, &expected_stderr); } } errors @@ -2530,7 +2545,13 @@ impl<'test> TestCx<'test> { } } - fn compare_output(&self, stream: &str, actual: &str, expected: &str) -> usize { + fn compare_output( + &self, + stream: &str, + actual: &str, + actual_unnormalized: &str, + expected: &str, + ) -> usize { let are_different = match (self.force_color_svg(), expected.find('\n'), actual.find('\n')) { // FIXME: We ignore the first line of SVG files // because the width parameter is non-deterministic. @@ -2590,27 +2611,16 @@ impl<'test> TestCx<'test> { if expected.is_empty() { println!("normalized {}:\n{}\n", stream, actual); } else { - println!("diff of {stream}:\n"); - if let Some(diff_command) = self.config.diff_command.as_deref() { - let mut args = diff_command.split_whitespace(); - let name = args.next().unwrap(); - match Command::new(name) - .args(args) - .args([&expected_path, &actual_path]) - .output() - { - Err(err) => { - self.fatal(&format!( - "failed to call custom diff command `{diff_command}`: {err}" - )); - } - Ok(output) => { - let output = String::from_utf8_lossy(&output.stdout); - print!("{output}"); - } - } - } else { - print!("{}", write_diff(expected, actual, 3)); + let show_full = self.show_diff( + stream, + &expected_path, + &actual_path, + expected, + actual, + actual_unnormalized, + ); + if show_full || self.config.verbose { + println!("{}", ProcRes::render(stream, actual)); } } } else { @@ -2633,6 +2643,84 @@ impl<'test> TestCx<'test> { if self.config.bless { 0 } else { 1 } } + /// Returns whether to show the full stderr/stdout. + fn show_diff( + &self, + stream: &str, + expected_path: &Path, + actual_path: &Path, + expected: &str, + actual: &str, + actual_unnormalized: &str, + ) -> bool { + let context_size = 3; + // NOTE: argument order is important, we need `actual` to be on the left so the line number match up when we compare it to `actual_unnormalized` below. + let diff_results = make_diff(actual, expected, context_size); + let diff_lines: usize = diff_results.iter().map(|hunk| hunk.lines.len()).sum(); + let show_diff = diff_lines < 10 || diff_lines < expected.lines().count() / 10; + if !show_diff && !CiEnv::is_ci() { + // don't even bother showing a diff if too many lines are changed, it won't be helpful. + let diff_cmd = self.config.diff_command.as_deref().unwrap_or("diff"); + println!( + "too many lines changed; hiding diff. use `{diff_cmd} {expected_path:?} {actual_path:?}` to view anyway." + ); + return true; + } + + println!("diff of {stream}:\n"); + if let Some(diff_command) = self.config.diff_command.as_deref() { + let mut args = diff_command.split_whitespace(); + let name = args.next().unwrap(); + match Command::new(name).args(args).args([expected_path, actual_path]).output() { + Err(err) => { + self.fatal(&format!( + "failed to call custom diff command `{diff_command}`: {err}" + )); + } + Ok(output) => { + let output = String::from_utf8_lossy(&output.stdout); + print!("{output}"); + } + } + } else { + print!("{}", write_diff(expected, actual, context_size)); + } + + let (mut mismatches_normalized, mut mismatch_line_nos) = (String::new(), vec![]); + for hunk in diff_results { + let mut line_no = hunk.line_number; + for line in hunk.lines { + if let DiffLine::Resulting(normalized) = line { + mismatches_normalized += &normalized; + mismatches_normalized += "\n"; + mismatch_line_nos.push(line_no); + line_no += 1; + } + } + } + let mut mismatches_unnormalized = String::new(); + let diff_normalized = make_diff(actual, actual_unnormalized, 0); + for hunk in diff_normalized { + if mismatch_line_nos.contains(&hunk.line_number) { + for line in hunk.lines { + if let DiffLine::Resulting(unnormalized) = line { + mismatches_unnormalized += &unnormalized; + mismatches_unnormalized += "\n"; + } + } + } + } + + let normalized_diff = make_diff(&mismatches_normalized, &mismatches_unnormalized, 0); + if !normalized_diff.is_empty() { + println!("Note: some mismatched output was normalized before being compared"); + // FIXME: respect diff_command + print!("{}", write_diff(&mismatches_unnormalized, &mismatches_normalized, 0)); + } + + CiEnv::is_ci() + } + fn check_and_prune_duplicate_outputs( &self, proc_res: &ProcRes, @@ -2749,28 +2837,28 @@ pub struct ProcRes { } impl ProcRes { - pub fn print_info(&self) { - fn render(name: &str, contents: &str) -> String { - let contents = json::extract_rendered(contents); - let contents = contents.trim_end(); - if contents.is_empty() { - format!("{name}: none") - } else { - format!( - "\ - --- {name} -------------------------------\n\ - {contents}\n\ - ------------------------------------------", - ) - } + pub fn render(name: &str, contents: &str) -> String { + let contents = json::extract_rendered(contents); + let contents = contents.trim_end(); + if contents.is_empty() { + format!("{name}: none") + } else { + format!( + "\ + --- {name} -------------------------------\n\ + {contents}\n\ + ------------------------------------------", + ) } + } + pub fn print_info(&self) { println!( "status: {}\ncommand: {}\n{}\n{}\n", self.status, self.cmdline, - render("stdout", &self.stdout), - render("stderr", &self.stderr), + Self::render("stdout", &self.stdout), + Self::render("stderr", &self.stderr), ); } diff --git a/src/tools/compiletest/src/runtest/coverage.rs b/src/tools/compiletest/src/runtest/coverage.rs index 961a160298631..030ca5ebb2474 100644 --- a/src/tools/compiletest/src/runtest/coverage.rs +++ b/src/tools/compiletest/src/runtest/coverage.rs @@ -39,8 +39,12 @@ impl<'test> TestCx<'test> { let expected_coverage_dump = self.load_expected_output(kind); let actual_coverage_dump = self.normalize_output(&proc_res.stdout, &[]); - let coverage_dump_errors = - self.compare_output(kind, &actual_coverage_dump, &expected_coverage_dump); + let coverage_dump_errors = self.compare_output( + kind, + &actual_coverage_dump, + &proc_res.stdout, + &expected_coverage_dump, + ); if coverage_dump_errors > 0 { self.fatal_proc_rec( @@ -135,8 +139,12 @@ impl<'test> TestCx<'test> { self.fatal_proc_rec(&err, &proc_res); }); - let coverage_errors = - self.compare_output(kind, &normalized_actual_coverage, &expected_coverage); + let coverage_errors = self.compare_output( + kind, + &normalized_actual_coverage, + &proc_res.stdout, + &expected_coverage, + ); if coverage_errors > 0 { self.fatal_proc_rec( diff --git a/src/tools/compiletest/src/runtest/ui.rs b/src/tools/compiletest/src/runtest/ui.rs index bb747c6802926..a6c8201a6cbdd 100644 --- a/src/tools/compiletest/src/runtest/ui.rs +++ b/src/tools/compiletest/src/runtest/ui.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; use std::fs::OpenOptions; use std::io::Write; +use build_helper::ci::CiEnv; use rustfix::{Filter, apply_suggestions, get_suggestions_from_json}; use tracing::debug; @@ -100,7 +101,7 @@ impl TestCx<'_> { ) }); - errors += self.compare_output("fixed", &fixed_code, &expected_fixed); + errors += self.compare_output("fixed", &fixed_code, &fixed_code, &expected_fixed); } else if !expected_fixed.is_empty() { panic!( "the `//@ run-rustfix` directive wasn't found but a `*.fixed` \ @@ -116,10 +117,11 @@ impl TestCx<'_> { "To only update this specific test, also pass `--test-args {}`", relative_path_to_file.display(), ); - self.fatal_proc_rec( - &format!("{} errors occurred comparing output.", errors), - &proc_res, - ); + self.error(&format!("{} errors occurred comparing output.", errors)); + if self.config.verbose || CiEnv::is_ci() { + println!("status: {}\ncommand: {}", proc_res.status, proc_res.cmdline); + } + std::panic::resume_unwind(Box::new(())); } let expected_errors = errors::load_errors(&self.testpaths.file, self.revision);