diff --git a/src/tools/run-make-support/src/rustc.rs b/src/tools/run-make-support/src/rustc.rs
index 32fa5018d800f..13cec0cb6bede 100644
--- a/src/tools/run-make-support/src/rustc.rs
+++ b/src/tools/run-make-support/src/rustc.rs
@@ -105,6 +105,12 @@ impl Rustc {
         self
     }
 
+    //Adjust the backtrace level, displaying more detailed information at higher levels.
+    pub fn set_backtrace_level<R: AsRef<OsStr>>(&mut self, level: R) -> &mut Self {
+        self.cmd.env("RUST_BACKTRACE", level);
+        self
+    }
+
     /// Specify path to the output file. Equivalent to `-o`` in rustc.
     pub fn output<P: AsRef<Path>>(&mut self, path: P) -> &mut Self {
         self.cmd.arg("-o");
diff --git a/src/tools/tidy/src/allowed_run_make_makefiles.txt b/src/tools/tidy/src/allowed_run_make_makefiles.txt
index ac89a30f35342..87185eef95236 100644
--- a/src/tools/tidy/src/allowed_run_make_makefiles.txt
+++ b/src/tools/tidy/src/allowed_run_make_makefiles.txt
@@ -214,7 +214,6 @@ run-make/sepcomp-cci-copies/Makefile
 run-make/sepcomp-inlining/Makefile
 run-make/sepcomp-separate/Makefile
 run-make/share-generics-dylib/Makefile
-run-make/short-ice/Makefile
 run-make/silly-file-names/Makefile
 run-make/simd-ffi/Makefile
 run-make/split-debuginfo/Makefile
diff --git a/tests/run-make/short-ice/Makefile b/tests/run-make/short-ice/Makefile
deleted file mode 100644
index 23006fc09e2f3..0000000000000
--- a/tests/run-make/short-ice/Makefile
+++ /dev/null
@@ -1,10 +0,0 @@
-include ../tools.mk
-
-# ignore-windows
-
-export RUSTC := $(RUSTC_ORIGINAL)
-export LD_LIBRARY_PATH := $(HOST_RPATH_DIR)
-export TMPDIR := $(TMPDIR)
-
-all:
-	bash check.sh
diff --git a/tests/run-make/short-ice/check.sh b/tests/run-make/short-ice/check.sh
deleted file mode 100644
index 56babd2142f8c..0000000000000
--- a/tests/run-make/short-ice/check.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh
-export RUSTC_ICE=0
-RUST_BACKTRACE=1 $RUSTC src/lib.rs -Z treat-err-as-bug=1 1>$TMPDIR/rust-test-1.log 2>&1
-RUST_BACKTRACE=full $RUSTC src/lib.rs -Z treat-err-as-bug=1 1>$TMPDIR/rust-test-2.log 2>&1
-
-short=$(cat $TMPDIR/rust-test-1.log | wc -l)
-full=$(cat $TMPDIR/rust-test-2.log | wc -l)
-rustc_query_count=$(cat $TMPDIR/rust-test-1.log | grep rustc_query_ | wc -l)
-rustc_query_count_full=$(cat $TMPDIR/rust-test-2.log | grep rustc_query_ | wc -l)
-
-begin_count=$(cat $TMPDIR/rust-test-2.log | grep __rust_begin_short_backtrace | wc -l)
-end_count=$(cat $TMPDIR/rust-test-2.log | grep __rust_end_short_backtrace | wc -l)
-
-cat $TMPDIR/rust-test-1.log
-echo "====================="
-cat $TMPDIR/rust-test-2.log
-echo "====================="
-
-echo "short backtrace: $short"
-echo "full  backtrace: $full"
-echo "begin_count: $begin_count"
-echo "end_count  : $end_count"
-echo "rustc_query_count: $rustc_query_count"
-echo "rustc_query_count_full: $rustc_query_count_full"
-
-## backtraces to vary a bit depending on platform and configuration options,
-## here we make sure that the short backtrace of rustc_query is shorter than the full,
-## and marks are in pairs.
-if [ $short -lt $full ] &&
-    [ $begin_count -eq $end_count ] &&
-    [ $(($rustc_query_count + 5)) -lt $rustc_query_count_full ] &&
-    [ $rustc_query_count_full -gt 5 ]; then
-    exit 0
-else
-    exit 1
-fi
diff --git a/tests/run-make/short-ice/rmake.rs b/tests/run-make/short-ice/rmake.rs
new file mode 100644
index 0000000000000..81403931c783e
--- /dev/null
+++ b/tests/run-make/short-ice/rmake.rs
@@ -0,0 +1,42 @@
+// Backtraces in internal compiler errors used to be unbearably long, spanning
+// multiple hundreds of lines. A fix was pushed in #108938, and this test gathers
+// varied metrics on level 1 and full-level backtraces to check that the output
+// was shortened down to an appropriate length.
+// See https://github.com/rust-lang/rust/issues/107910
+
+//@ ignore-windows
+// Reason: the assert_eq! on line 32 fails, as error output on Windows is different.
+
+use run_make_support::rustc;
+
+fn main() {
+    let rust_test_1 =
+        rustc().set_backtrace_level("1").input("src/lib.rs").arg("-Ztreat-err-as-bug=1").run_fail();
+    let rust_test_2 = rustc()
+        .set_backtrace_level("full")
+        .input("src/lib.rs")
+        .arg("-Ztreat-err-as-bug=1")
+        .run_fail();
+
+    let mut rust_test_log_1 = rust_test_1.stderr_utf8();
+    rust_test_log_1.push_str(&rust_test_1.stdout_utf8());
+    let rust_test_log_1 = rust_test_log_1.as_str();
+
+    let mut rust_test_log_2 = rust_test_2.stderr_utf8();
+    rust_test_log_2.push_str(&rust_test_2.stdout_utf8());
+    let rust_test_log_2 = rust_test_log_2.as_str();
+
+    let rustc_query_count_full = count_lines_with(rust_test_log_2, "rustc_query_");
+
+    assert!(rust_test_log_1.lines().count() < rust_test_log_2.lines().count());
+    assert_eq!(
+        count_lines_with(rust_test_log_2, "__rust_begin_short_backtrace"),
+        count_lines_with(rust_test_log_2, "__rust_end_short_backtrace")
+    );
+    assert!(count_lines_with(rust_test_log_1, "rustc_query_") + 5 < rustc_query_count_full);
+    assert!(rustc_query_count_full > 5);
+}
+
+fn count_lines_with(s: &str, search: &str) -> usize {
+    s.lines().filter(|l| l.contains(search)).count()
+}