From b368dcb246c5aedb087034e0f62941d038799f38 Mon Sep 17 00:00:00 2001 From: Ding Xiang Fei Date: Mon, 29 Jul 2024 23:10:34 +0800 Subject: [PATCH 01/16] unconditionally allow shadow call-stack for AArch64 whenever fixed-x18 is applied --- compiler/rustc_session/src/session.rs | 7 ++++++- src/doc/rustc/src/platform-support/android.md | 5 +++++ .../src/compiler-flags/fixed-x18.md | 7 ++++++- .../src/compiler-flags/sanitizer.md | 4 ++++ ...arch64-shadow-call-stack-with-fixed-x18.rs | 19 +++++++++++++++++++ .../shadow-call-stack-without-fixed-x18.rs | 15 +++++++++++++++ 6 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 tests/codegen/sanitizer/aarch64-shadow-call-stack-with-fixed-x18.rs create mode 100644 tests/ui/abi/shadow-call-stack-without-fixed-x18.rs diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs index be67baf57f6dc..8cce665677eb9 100644 --- a/compiler/rustc_session/src/session.rs +++ b/compiler/rustc_session/src/session.rs @@ -1187,7 +1187,12 @@ fn validate_commandline_args_with_session_available(sess: &Session) { // Sanitizers can only be used on platforms that we know have working sanitizer codegen. let supported_sanitizers = sess.target.options.supported_sanitizers; - let unsupported_sanitizers = sess.opts.unstable_opts.sanitizer - supported_sanitizers; + let mut unsupported_sanitizers = sess.opts.unstable_opts.sanitizer - supported_sanitizers; + // Niche: if `fixed-x18`, or effectively switching on `reserved-x18` flag, is enabled + // we should allow Shadow Call Stack sanitizer. + if sess.opts.unstable_opts.fixed_x18 && sess.target.arch == "aarch64" { + unsupported_sanitizers -= SanitizerSet::SHADOWCALLSTACK; + } match unsupported_sanitizers.into_iter().count() { 0 => {} 1 => { diff --git a/src/doc/rustc/src/platform-support/android.md b/src/doc/rustc/src/platform-support/android.md index 9ddf00e3a5053..96499b0d80148 100644 --- a/src/doc/rustc/src/platform-support/android.md +++ b/src/doc/rustc/src/platform-support/android.md @@ -61,3 +61,8 @@ Currently the `riscv64-linux-android` target requires the following architecture * `Zba` (address calculation instructions) * `Zbb` (base instructions) * `Zbs` (single-bit instructions) + +### aarch64-linux-android on Nightly compilers + +As soon as `-Zfixed-x18` compiler flag is supplied, the [`ShadowCallStack` sanitizer](https://releases.llvm.org/7.0.1/tools/clang/docs/ShadowCallStack.html) +instrumentation is also made avaiable by supplying the second compiler flag `-Zsanitizer=shadow-call-stack`. diff --git a/src/doc/unstable-book/src/compiler-flags/fixed-x18.md b/src/doc/unstable-book/src/compiler-flags/fixed-x18.md index 8c8bff5fa296d..b215bc84fbb71 100644 --- a/src/doc/unstable-book/src/compiler-flags/fixed-x18.md +++ b/src/doc/unstable-book/src/compiler-flags/fixed-x18.md @@ -1,7 +1,7 @@ # `fixed-x18` This option prevents the compiler from using the x18 register. It is only -supported on aarch64. +supported on `aarch64`. From the [ABI spec][arm-abi]: @@ -23,6 +23,11 @@ Currently, the `-Zsanitizer=shadow-call-stack` flag is only supported on platforms that always treat x18 as a reserved register, and the `-Zfixed-x18` flag is not required to use the sanitizer on such platforms. However, the sanitizer may be supported on targets where this is not the case in the future. +One way to do so now on Nightly compilers is to explicitly supply this `-Zfixed-x18` +flag with `aarch64` targets, so that the sanitizer is available for instrumentation +on targets like `aarch64-unknown-none`, for instance. However, discretion is still +required to make sure that the runtime support is in place for this sanitizer +to be effective. It is undefined behavior for `-Zsanitizer=shadow-call-stack` code to call into code where x18 is a temporary register. On the other hand, when you are *not* diff --git a/src/doc/unstable-book/src/compiler-flags/sanitizer.md b/src/doc/unstable-book/src/compiler-flags/sanitizer.md index 72b44e002b4d4..edc63a25ac1be 100644 --- a/src/doc/unstable-book/src/compiler-flags/sanitizer.md +++ b/src/doc/unstable-book/src/compiler-flags/sanitizer.md @@ -787,6 +787,10 @@ A runtime must be provided by the application or operating system. See the [Clang ShadowCallStack documentation][clang-scs] for more details. +* `aarch64-unknown-none` + +In addition to support from a runtime by the application or operating system, the `-Zfixed-x18` flag is also mandatory. + # ThreadSanitizer ThreadSanitizer is a data race detection tool. It is supported on the following diff --git a/tests/codegen/sanitizer/aarch64-shadow-call-stack-with-fixed-x18.rs b/tests/codegen/sanitizer/aarch64-shadow-call-stack-with-fixed-x18.rs new file mode 100644 index 0000000000000..e2e14ab14a801 --- /dev/null +++ b/tests/codegen/sanitizer/aarch64-shadow-call-stack-with-fixed-x18.rs @@ -0,0 +1,19 @@ +//@ revisions: aarch64 android +//@[aarch64] compile-flags: --target aarch64-unknown-none -Zfixed-x18 -Zsanitizer=shadow-call-stack +//@[aarch64] needs-llvm-components: aarch64 +//@[android] compile-flags: --target aarch64-linux-android -Zsanitizer=shadow-call-stack +//@[android] needs-llvm-components: aarch64 + +#![allow(internal_features)] +#![crate_type = "rlib"] +#![feature(no_core, lang_items)] +#![no_core] + +#[lang = "sized"] +trait Sized {} + +// CHECK: ; Function Attrs:{{.*}}shadowcallstack +#[no_mangle] +pub fn foo() {} + +// CHECK: attributes #0 = {{.*}}shadowcallstack{{.*}} diff --git a/tests/ui/abi/shadow-call-stack-without-fixed-x18.rs b/tests/ui/abi/shadow-call-stack-without-fixed-x18.rs new file mode 100644 index 0000000000000..d758c903087b6 --- /dev/null +++ b/tests/ui/abi/shadow-call-stack-without-fixed-x18.rs @@ -0,0 +1,15 @@ +//@ compile-flags: --target aarch64-unknown-none -Zsanitizer=shadow-call-stack +//@ error-pattern: shadow-call-stack sanitizer is not supported for this target +//@ dont-check-compiler-stderr +//@ needs-llvm-components: aarch64 + +#![allow(internal_features)] +#![crate_type = "rlib"] +#![feature(no_core, lang_items)] +#![no_core] + +#[lang = "sized"] +trait Sized {} + +#[no_mangle] +pub fn foo() {} From 55e981070326751eef8508272ca2694427198381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sat, 10 Aug 2024 15:02:13 +0000 Subject: [PATCH 02/16] compiletest: implement `needs-lvm-zstd` directive --- src/tools/compiletest/src/command-list.rs | 1 + src/tools/compiletest/src/header.rs | 101 ++++++++++++++++++++++ src/tools/compiletest/src/header/needs.rs | 10 ++- 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/src/tools/compiletest/src/command-list.rs b/src/tools/compiletest/src/command-list.rs index 50c909793f5e1..c95dd2fcf9f94 100644 --- a/src/tools/compiletest/src/command-list.rs +++ b/src/tools/compiletest/src/command-list.rs @@ -138,6 +138,7 @@ const KNOWN_DIRECTIVE_NAMES: &[&str] = &[ "needs-force-clang-based-tests", "needs-git-hash", "needs-llvm-components", + "needs-llvm-zstd", "needs-profiler-support", "needs-relocation-model-pic", "needs-run-enabled", diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs index 1fc24301c85e6..277a88584ba60 100644 --- a/src/tools/compiletest/src/header.rs +++ b/src/tools/compiletest/src/header.rs @@ -1203,6 +1203,107 @@ pub fn extract_llvm_version_from_binary(binary_path: &str) -> Option { None } +/// For tests using the `needs-llvm-zstd` directive: +/// - for local LLVM builds, try to find the static zstd library in the llvm-config system libs. +/// - for `download-ci-llvm`, see if `lld` was built with zstd support. +pub fn llvm_has_libzstd(config: &Config) -> bool { + // Strategy 1: works for local builds but not with `download-ci-llvm`. + // + // We check whether `llvm-config` returns the zstd library. Bootstrap's `llvm.libzstd` will only + // ask to statically link it when building LLVM, so we only check if the list of system libs + // contains a path to that static lib, and that it exists. + // + // See compiler/rustc_llvm/build.rs for more details and similar expectations. + fn is_zstd_in_config(llvm_bin_dir: &Path) -> Option<()> { + let llvm_config_path = llvm_bin_dir.join("llvm-config"); + let output = Command::new(llvm_config_path).arg("--system-libs").output().ok()?; + assert!(output.status.success(), "running llvm-config --system-libs failed"); + + let libs = String::from_utf8(output.stdout).ok()?; + for lib in libs.split_whitespace() { + if lib.ends_with("libzstd.a") && Path::new(lib).exists() { + return Some(()); + } + } + + None + } + + // Strategy 2: `download-ci-llvm`'s `llvm-config --system-libs` will not return any libs to + // use. + // + // The CI artifacts also don't contain the bootstrap config used to build them: otherwise we + // could have looked at the `llvm.libzstd` config. + // + // We infer whether `LLVM_ENABLE_ZSTD` was used to build LLVM as a byproduct of testing whether + // `lld` supports it. If not, an error will be emitted: "LLVM was not built with + // LLVM_ENABLE_ZSTD or did not find zstd at build time". + #[cfg(unix)] + fn is_lld_built_with_zstd(llvm_bin_dir: &Path) -> Option<()> { + let lld_path = llvm_bin_dir.join("lld"); + if lld_path.exists() { + // We can't call `lld` as-is, it expects to be invoked by a compiler driver using a + // different name. Prepare a temporary symlink to do that. + let lld_symlink_path = llvm_bin_dir.join("ld.lld"); + if !lld_symlink_path.exists() { + std::os::unix::fs::symlink(lld_path, &lld_symlink_path).ok()?; + } + + // Run `lld` with a zstd flag. We expect this command to always error here, we don't + // want to link actual files and don't pass any. + let output = Command::new(&lld_symlink_path) + .arg("--compress-debug-sections=zstd") + .output() + .ok()?; + assert!(!output.status.success()); + + // Look for a specific error caused by LLVM not being built with zstd support. We could + // also look for the "no input files" message, indicating the zstd flag was accepted. + let stderr = String::from_utf8(output.stderr).ok()?; + let zstd_available = !stderr.contains("LLVM was not built with LLVM_ENABLE_ZSTD"); + + // We don't particularly need to clean the link up (so the previous commands could fail + // in theory but won't in practice), but we can try. + std::fs::remove_file(lld_symlink_path).ok()?; + + if zstd_available { + return Some(()); + } + } + + None + } + + #[cfg(not(unix))] + fn is_lld_built_with_zstd(llvm_bin_dir: &Path) -> Option<()> { + None + } + + if let Some(llvm_bin_dir) = &config.llvm_bin_dir { + // Strategy 1: for local LLVM builds. + if is_zstd_in_config(llvm_bin_dir).is_some() { + return true; + } + + // Strategy 2: for LLVM artifacts built on CI via `download-ci-llvm`. + // + // It doesn't work for cases where the artifacts don't contain the linker, but it's + // best-effort: CI has `llvm.libzstd` and `lld` enabled on the x64 linux artifacts, so it + // will at least work there. + // + // If this can be improved and expanded to less common cases in the future, it should. + if config.target == "x86_64-unknown-linux-gnu" + && config.host == config.target + && is_lld_built_with_zstd(llvm_bin_dir).is_some() + { + return true; + } + } + + // Otherwise, all hope is lost. + false +} + /// Takes a directive of the form " [- ]", /// returns the numeric representation of and as /// tuple: ( as u32, as u32) diff --git a/src/tools/compiletest/src/header/needs.rs b/src/tools/compiletest/src/header/needs.rs index 5b2665f7d0ba7..3a24adaa2a3b7 100644 --- a/src/tools/compiletest/src/header/needs.rs +++ b/src/tools/compiletest/src/header/needs.rs @@ -1,5 +1,5 @@ use crate::common::{Config, Debugger, Sanitizer}; -use crate::header::IgnoreDecision; +use crate::header::{llvm_has_libzstd, IgnoreDecision}; pub(super) fn handle_needs( cache: &CachedNeedsConditions, @@ -149,6 +149,11 @@ pub(super) fn handle_needs( condition: cache.symlinks, ignore_reason: "ignored if symlinks are unavailable", }, + Need { + name: "needs-llvm-zstd", + condition: cache.llvm_zstd, + ignore_reason: "ignored if LLVM wasn't build with zstd for ELF section compression", + }, ]; let (name, comment) = match ln.split_once([':', ' ']) { @@ -215,6 +220,8 @@ pub(super) struct CachedNeedsConditions { rust_lld: bool, dlltool: bool, symlinks: bool, + /// Whether LLVM built with zstd, for the `needs-llvm-zstd` directive. + llvm_zstd: bool, } impl CachedNeedsConditions { @@ -258,6 +265,7 @@ impl CachedNeedsConditions { .join(if config.host.contains("windows") { "rust-lld.exe" } else { "rust-lld" }) .exists(), + llvm_zstd: llvm_has_libzstd(&config), dlltool: find_dlltool(&config), symlinks: has_symlinks(), } From c5d911033d7be2660950e43020993fed161b6f30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sat, 10 Aug 2024 15:09:20 +0000 Subject: [PATCH 03/16] mark `rust-lld-compress-debug-sections` test as needing zstd also make it fail if there's a compression issue --- .../rust-lld-compress-debug-sections/rmake.rs | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/tests/run-make/rust-lld-compress-debug-sections/rmake.rs b/tests/run-make/rust-lld-compress-debug-sections/rmake.rs index ea4997fab8099..9889659046ea7 100644 --- a/tests/run-make/rust-lld-compress-debug-sections/rmake.rs +++ b/tests/run-make/rust-lld-compress-debug-sections/rmake.rs @@ -1,12 +1,13 @@ // Checks the `compress-debug-sections` option on rust-lld. //@ needs-rust-lld +//@ needs-llvm-zstd //@ only-linux //@ ignore-cross-compile // FIXME: This test isn't comprehensive and isn't covering all possible combinations. -use run_make_support::{assert_contains, llvm_readobj, run_in_tmpdir, rustc}; +use run_make_support::{llvm_readobj, run_in_tmpdir, rustc}; fn check_compression(compression: &str, to_find: &str) { run_in_tmpdir(|| { @@ -17,19 +18,8 @@ fn check_compression(compression: &str, to_find: &str) { .arg("-Cdebuginfo=full") .link_arg(&format!("-Wl,--compress-debug-sections={compression}")) .input("main.rs") - .run_unchecked(); - let stderr = out.stderr_utf8(); - if stderr.is_empty() { - llvm_readobj().arg("-t").arg("main").run().assert_stdout_contains(to_find); - } else { - assert_contains( - stderr, - format!( - "LLVM was not built with LLVM_ENABLE_{to_find} \ - or did not find {compression} at build time" - ), - ); - } + .run(); + llvm_readobj().arg("-t").arg("main").run().assert_stdout_contains(to_find); }); } From 2d4af9fed4a0203011c121fe2f9f91bb105bc561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sat, 10 Aug 2024 15:52:58 +0000 Subject: [PATCH 04/16] make `compressed-debuginfo` test about zlib only zlib is seemingly always enabled, so we can test it unconditionally --- tests/run-make/compressed-debuginfo/rmake.rs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/tests/run-make/compressed-debuginfo/rmake.rs b/tests/run-make/compressed-debuginfo/rmake.rs index 5ba1a1852d51b..362b2e2e1e449 100644 --- a/tests/run-make/compressed-debuginfo/rmake.rs +++ b/tests/run-make/compressed-debuginfo/rmake.rs @@ -1,11 +1,9 @@ -// Checks the `debuginfo-compression` option. +// Checks the always enabled `debuginfo-compression` option: zlib. //@ only-linux //@ ignore-cross-compile -// FIXME: This test isn't comprehensive and isn't covering all possible combinations. - -use run_make_support::{assert_contains, llvm_readobj, run_in_tmpdir, rustc}; +use run_make_support::{llvm_readobj, run_in_tmpdir, rustc}; fn check_compression(compression: &str, to_find: &str) { run_in_tmpdir(|| { @@ -17,19 +15,10 @@ fn check_compression(compression: &str, to_find: &str) { .arg(&format!("-Zdebuginfo-compression={compression}")) .input("foo.rs") .run(); - let stderr = out.stderr_utf8(); - if stderr.is_empty() { - llvm_readobj().arg("-t").arg("foo.o").run().assert_stdout_contains(to_find); - } else { - assert_contains( - stderr, - format!("unknown debuginfo compression algorithm {compression}"), - ); - } + llvm_readobj().arg("-t").arg("foo.o").run().assert_stdout_contains(to_find); }); } fn main() { check_compression("zlib", "ZLIB"); - check_compression("zstd", "ZSTD"); } From 2bcbfdb91e3914a69be6691c08cf1aef1dcf174e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sat, 10 Aug 2024 16:22:04 +0000 Subject: [PATCH 05/16] prepare test for expanding scope --- .../main.rs | 0 .../rmake.rs | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/run-make/{rust-lld-compress-debug-sections => compressed-debuginfo-zstd}/main.rs (100%) rename tests/run-make/{rust-lld-compress-debug-sections => compressed-debuginfo-zstd}/rmake.rs (100%) diff --git a/tests/run-make/rust-lld-compress-debug-sections/main.rs b/tests/run-make/compressed-debuginfo-zstd/main.rs similarity index 100% rename from tests/run-make/rust-lld-compress-debug-sections/main.rs rename to tests/run-make/compressed-debuginfo-zstd/main.rs diff --git a/tests/run-make/rust-lld-compress-debug-sections/rmake.rs b/tests/run-make/compressed-debuginfo-zstd/rmake.rs similarity index 100% rename from tests/run-make/rust-lld-compress-debug-sections/rmake.rs rename to tests/run-make/compressed-debuginfo-zstd/rmake.rs From 0728264546a790daefffef3ff7bb0fc9a103b20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sat, 10 Aug 2024 16:59:00 +0000 Subject: [PATCH 06/16] expand zstd debuginfo compression test it now checks zlib and zstd, via rustc and rust-lld --- .../compressed-debuginfo-zstd/rmake.rs | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/run-make/compressed-debuginfo-zstd/rmake.rs b/tests/run-make/compressed-debuginfo-zstd/rmake.rs index 9889659046ea7..8356373e949aa 100644 --- a/tests/run-make/compressed-debuginfo-zstd/rmake.rs +++ b/tests/run-make/compressed-debuginfo-zstd/rmake.rs @@ -1,24 +1,37 @@ -// Checks the `compress-debug-sections` option on rust-lld. +// Checks debuginfo compression both for the always-enabled zlib, and when the optional zstd is +// enabled: +// - via rustc's `debuginfo-compression`, +// - and via rust-lld's `compress-debug-sections` -//@ needs-rust-lld -//@ needs-llvm-zstd +//@ needs-llvm-zstd: we want LLVM/LLD to be built with zstd support +//@ needs-rust-lld: the system linker will most likely not support zstd //@ only-linux //@ ignore-cross-compile -// FIXME: This test isn't comprehensive and isn't covering all possible combinations. - -use run_make_support::{llvm_readobj, run_in_tmpdir, rustc}; +use run_make_support::{llvm_readobj, run_in_tmpdir, Rustc}; fn check_compression(compression: &str, to_find: &str) { + // check compressed debug sections via rustc flag + prepare_and_check(to_find, |rustc| { + rustc.arg(&format!("-Zdebuginfo-compression={compression}")) + }); + + // check compressed debug sections via rust-lld flag + prepare_and_check(to_find, |rustc| { + rustc.link_arg(&format!("-Wl,--compress-debug-sections={compression}")) + }); +} + +fn prepare_and_check &mut Rustc>(to_find: &str, prepare_rustc: F) { run_in_tmpdir(|| { - let out = rustc() + let mut rustc = Rustc::new(); + rustc .arg("-Zlinker-features=+lld") .arg("-Clink-self-contained=+linker") .arg("-Zunstable-options") .arg("-Cdebuginfo=full") - .link_arg(&format!("-Wl,--compress-debug-sections={compression}")) - .input("main.rs") - .run(); + .input("main.rs"); + prepare_rustc(&mut rustc).run(); llvm_readobj().arg("-t").arg("main").run().assert_stdout_contains(to_find); }); } From e42e5c2d7e2dfc4ee12262e1cb5bb7a4a0d055ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sat, 10 Aug 2024 22:25:22 +0000 Subject: [PATCH 07/16] move and rename zstd script move it where it's used, and name it like the other scripts --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 6 +++--- .../zstd.sh => host-x86_64/dist-x86_64-linux/build-zstd.sh} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename src/ci/docker/{scripts/zstd.sh => host-x86_64/dist-x86_64-linux/build-zstd.sh} (100%) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 61e9694f1e2ae..e857f38e68a85 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -62,9 +62,9 @@ COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/ RUN ./build-clang.sh ENV CC=clang CXX=clang++ -# rustc's LLVM needs zstd. -COPY scripts/zstd.sh /tmp/ -RUN ./zstd.sh +# Build zstd to enable `llvm.libzstd`. +COPY host-x86_64/dist-x86_64-linux/build-zstd.sh /tmp/ +RUN ./build-zstd.sh COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/scripts/zstd.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh similarity index 100% rename from src/ci/docker/scripts/zstd.sh rename to src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh From 323bcdc5d7e3ee2174a1c801982fcd682cfc63f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sun, 11 Aug 2024 15:02:13 +0000 Subject: [PATCH 08/16] strip whitespace for ignored tests reason comments --- src/tools/compiletest/src/header/needs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/compiletest/src/header/needs.rs b/src/tools/compiletest/src/header/needs.rs index 3a24adaa2a3b7..3c5a391007bed 100644 --- a/src/tools/compiletest/src/header/needs.rs +++ b/src/tools/compiletest/src/header/needs.rs @@ -179,7 +179,7 @@ pub(super) fn handle_needs( } else { return IgnoreDecision::Ignore { reason: if let Some(comment) = comment { - format!("{} ({comment})", need.ignore_reason) + format!("{} ({})", need.ignore_reason, comment.trim()) } else { need.ignore_reason.into() }, From f27f3a3345e302db7229cfbc23b6a3c8f175c9c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sun, 11 Aug 2024 19:05:30 +0000 Subject: [PATCH 09/16] enable `llvm.libzstd` on test x64 linux builder --- src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile index 19683317126ab..83c2aa8cfb3b7 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile @@ -28,5 +28,6 @@ ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --enable-sanitizers \ --enable-profiler \ - --enable-compiler-docs + --enable-compiler-docs \ + --set llvm.libzstd=true ENV SCRIPT python3 ../x.py --stage 2 test From 1e1d8393881f60781ac6b13e2d7667afaee2d764 Mon Sep 17 00:00:00 2001 From: Michael Goulet Date: Wed, 14 Aug 2024 12:55:11 -0400 Subject: [PATCH 10/16] Fix projections when parent capture is by-ref --- .../src/coroutine/by_move_body.rs | 54 ++++++++++++++----- .../async-closures/move-out-of-ref.rs | 16 ++++++ .../async-closures/move-out-of-ref.stderr | 18 +++++++ 3 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 tests/ui/async-await/async-closures/move-out-of-ref.rs create mode 100644 tests/ui/async-await/async-closures/move-out-of-ref.stderr diff --git a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs index 69d21a63f557b..7ea36f08232e2 100644 --- a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs +++ b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs @@ -78,6 +78,8 @@ use rustc_middle::mir::{self, dump_mir, MirPass}; use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt, TypeVisitableExt}; use rustc_target::abi::{FieldIdx, VariantIdx}; +use crate::pass_manager::validate_body; + pub struct ByMoveBody; impl<'tcx> MirPass<'tcx> for ByMoveBody { @@ -131,20 +133,40 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody { |(parent_field_idx, parent_capture), (child_field_idx, child_capture)| { // Store this set of additional projections (fields and derefs). // We need to re-apply them later. - let child_precise_captures = - &child_capture.place.projections[parent_capture.place.projections.len()..]; + let mut child_precise_captures = child_capture.place.projections + [parent_capture.place.projections.len()..] + .to_vec(); - // If the parent captures by-move, and the child captures by-ref, then we - // need to peel an additional `deref` off of the body of the child. - let needs_deref = child_capture.is_by_ref() && !parent_capture.is_by_ref(); - if needs_deref { - assert_ne!( - coroutine_kind, - ty::ClosureKind::FnOnce, + // If the parent capture is by-ref, then we need to apply an additional + // deref before applying any further projections to this place. + if parent_capture.is_by_ref() { + child_precise_captures.insert( + 0, + Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref }, + ); + } + // If the child capture is by-ref, then we need to apply a "ref" + // projection (i.e. `&`) at the end. But wait! We don't have that + // as a projection kind. So instead, we can apply its dual and + // *peel* a deref off of the place when it shows up in the MIR body. + // Luckily, by construction this is always possible. + let peel_deref = if child_capture.is_by_ref() { + assert!( + parent_capture.is_by_ref() || coroutine_kind != ty::ClosureKind::FnOnce, "`FnOnce` coroutine-closures return coroutines that capture from \ their body; it will always result in a borrowck error!" ); - } + true + } else { + false + }; + + // Regarding the behavior above, you may think that it's redundant to both + // insert a deref and then peel a deref if the parent and child are both + // captured by-ref. This would be correct, except for the case where we have + // precise capturing projections, since the inserted deref is to the *beginning* + // and the peeled deref is at the *end*. I cannot seem to actually find a + // case where this happens, though, but let's keep this code flexible. // Finally, store the type of the parent's captured place. We need // this when building the field projection in the MIR body later on. @@ -164,7 +186,7 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody { ( FieldIdx::from_usize(parent_field_idx + num_args), parent_capture_ty, - needs_deref, + peel_deref, child_precise_captures, ), ) @@ -192,6 +214,10 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody { let mut by_move_body = body.clone(); MakeByMoveBody { tcx, field_remapping, by_move_coroutine_ty }.visit_body(&mut by_move_body); dump_mir(tcx, false, "coroutine_by_move", &0, &by_move_body, |_, _| Ok(())); + + // Let's just always validate this body. + validate_body(tcx, &mut by_move_body, "Initial coroutine_by_move body".to_string()); + // FIXME: use query feeding to generate the body right here and then only store the `DefId` of the new body. by_move_body.source = mir::MirSource::from_instance(InstanceKind::CoroutineKindShim { coroutine_def_id: coroutine_def_id.to_def_id(), @@ -202,7 +228,7 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody { struct MakeByMoveBody<'tcx> { tcx: TyCtxt<'tcx>, - field_remapping: UnordMap, bool, &'tcx [Projection<'tcx>])>, + field_remapping: UnordMap, bool, Vec>)>, by_move_coroutine_ty: Ty<'tcx>, } @@ -223,14 +249,14 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> { if place.local == ty::CAPTURE_STRUCT_LOCAL && let Some((&mir::ProjectionElem::Field(idx, _), projection)) = place.projection.split_first() - && let Some(&(remapped_idx, remapped_ty, needs_deref, bridging_projections)) = + && let Some(&(remapped_idx, remapped_ty, peel_deref, ref bridging_projections)) = self.field_remapping.get(&idx) { // As noted before, if the parent closure captures a field by value, and // the child captures a field by ref, then for the by-move body we're // generating, we also are taking that field by value. Peel off a deref, // since a layer of ref'ing has now become redundant. - let final_projections = if needs_deref { + let final_projections = if peel_deref { let Some((mir::ProjectionElem::Deref, projection)) = projection.split_first() else { bug!( diff --git a/tests/ui/async-await/async-closures/move-out-of-ref.rs b/tests/ui/async-await/async-closures/move-out-of-ref.rs new file mode 100644 index 0000000000000..a05447232f62b --- /dev/null +++ b/tests/ui/async-await/async-closures/move-out-of-ref.rs @@ -0,0 +1,16 @@ +//@ compile-flags: -Zvalidate-mir +//@ edition: 2021 + +#![feature(async_closure)] + +// NOT copy. +struct Ty; + +fn hello(x: &Ty) { + let c = async || { + *x; + //~^ ERROR cannot move out of `*x` which is behind a shared reference + }; +} + +fn main() {} diff --git a/tests/ui/async-await/async-closures/move-out-of-ref.stderr b/tests/ui/async-await/async-closures/move-out-of-ref.stderr new file mode 100644 index 0000000000000..294905a481d65 --- /dev/null +++ b/tests/ui/async-await/async-closures/move-out-of-ref.stderr @@ -0,0 +1,18 @@ +error[E0507]: cannot move out of `*x` which is behind a shared reference + --> $DIR/move-out-of-ref.rs:11:9 + | +LL | *x; + | ^^ move occurs because `*x` has type `Ty`, which does not implement the `Copy` trait + | +note: if `Ty` implemented `Clone`, you could clone the value + --> $DIR/move-out-of-ref.rs:7:1 + | +LL | struct Ty; + | ^^^^^^^^^ consider implementing `Clone` for this type +... +LL | *x; + | -- you could clone this value + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0507`. From f264e5d0112f45e00c548d04d2ead8a072f34f5c Mon Sep 17 00:00:00 2001 From: Michael Goulet Date: Wed, 14 Aug 2024 14:12:44 -0400 Subject: [PATCH 11/16] Remove redundant type ops --- compiler/rustc_middle/src/query/mod.rs | 27 ++------------- .../src/traits/query/type_op/eq.rs | 33 ------------------- .../src/traits/query/type_op/mod.rs | 2 -- .../src/traits/query/type_op/subtype.rs | 30 ----------------- compiler/rustc_traits/src/type_op.rs | 24 -------------- 5 files changed, 3 insertions(+), 113 deletions(-) delete mode 100644 compiler/rustc_trait_selection/src/traits/query/type_op/eq.rs delete mode 100644 compiler/rustc_trait_selection/src/traits/query/type_op/subtype.rs diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 075eae029041e..92c8d265c09ec 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -65,10 +65,9 @@ use crate::query::plumbing::{ }; use crate::traits::query::{ CanonicalAliasGoal, CanonicalPredicateGoal, CanonicalTyGoal, - CanonicalTypeOpAscribeUserTypeGoal, CanonicalTypeOpEqGoal, CanonicalTypeOpNormalizeGoal, - CanonicalTypeOpProvePredicateGoal, CanonicalTypeOpSubtypeGoal, DropckConstraint, - DropckOutlivesResult, MethodAutoderefStepsResult, NoSolution, NormalizationResult, - OutlivesBound, + CanonicalTypeOpAscribeUserTypeGoal, CanonicalTypeOpNormalizeGoal, + CanonicalTypeOpProvePredicateGoal, DropckConstraint, DropckOutlivesResult, + MethodAutoderefStepsResult, NoSolution, NormalizationResult, OutlivesBound, }; use crate::traits::{ specialization_graph, CodegenObligationError, EvaluationResult, ImplSource, @@ -2090,26 +2089,6 @@ rustc_queries! { desc { "evaluating `type_op_ascribe_user_type` `{:?}`", goal.value.value } } - /// Do not call this query directly: part of the `Eq` type-op - query type_op_eq( - goal: CanonicalTypeOpEqGoal<'tcx> - ) -> Result< - &'tcx Canonical<'tcx, canonical::QueryResponse<'tcx, ()>>, - NoSolution, - > { - desc { "evaluating `type_op_eq` `{:?}`", goal.value.value } - } - - /// Do not call this query directly: part of the `Subtype` type-op - query type_op_subtype( - goal: CanonicalTypeOpSubtypeGoal<'tcx> - ) -> Result< - &'tcx Canonical<'tcx, canonical::QueryResponse<'tcx, ()>>, - NoSolution, - > { - desc { "evaluating `type_op_subtype` `{:?}`", goal.value.value } - } - /// Do not call this query directly: part of the `ProvePredicate` type-op query type_op_prove_predicate( goal: CanonicalTypeOpProvePredicateGoal<'tcx> diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/eq.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/eq.rs deleted file mode 100644 index 656130cda19eb..0000000000000 --- a/compiler/rustc_trait_selection/src/traits/query/type_op/eq.rs +++ /dev/null @@ -1,33 +0,0 @@ -pub use rustc_middle::traits::query::type_op::Eq; -use rustc_middle::traits::query::NoSolution; -use rustc_middle::traits::ObligationCause; -use rustc_middle::ty::{ParamEnvAnd, TyCtxt}; - -use crate::infer::canonical::{Canonical, CanonicalQueryResponse}; -use crate::traits::ObligationCtxt; - -impl<'tcx> super::QueryTypeOp<'tcx> for Eq<'tcx> { - type QueryResponse = (); - - fn try_fast_path( - _tcx: TyCtxt<'tcx>, - key: &ParamEnvAnd<'tcx, Eq<'tcx>>, - ) -> Option { - if key.value.a == key.value.b { Some(()) } else { None } - } - - fn perform_query( - tcx: TyCtxt<'tcx>, - canonicalized: Canonical<'tcx, ParamEnvAnd<'tcx, Self>>, - ) -> Result, NoSolution> { - tcx.type_op_eq(canonicalized) - } - - fn perform_locally_with_next_solver( - ocx: &ObligationCtxt<'_, 'tcx>, - key: ParamEnvAnd<'tcx, Self>, - ) -> Result { - ocx.eq(&ObligationCause::dummy(), key.param_env, key.value.a, key.value.b)?; - Ok(()) - } -} diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs index 2f64ed963f967..791424065d1c9 100644 --- a/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs @@ -16,12 +16,10 @@ use crate::traits::{ObligationCause, ObligationCtxt}; pub mod ascribe_user_type; pub mod custom; -pub mod eq; pub mod implied_outlives_bounds; pub mod normalize; pub mod outlives; pub mod prove_predicate; -pub mod subtype; pub use rustc_middle::traits::query::type_op::*; diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/subtype.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/subtype.rs deleted file mode 100644 index 892c2a1f11309..0000000000000 --- a/compiler/rustc_trait_selection/src/traits/query/type_op/subtype.rs +++ /dev/null @@ -1,30 +0,0 @@ -pub use rustc_middle::traits::query::type_op::Subtype; -use rustc_middle::traits::query::NoSolution; -use rustc_middle::traits::ObligationCause; -use rustc_middle::ty::{ParamEnvAnd, TyCtxt}; - -use crate::infer::canonical::{Canonical, CanonicalQueryResponse}; -use crate::traits::ObligationCtxt; - -impl<'tcx> super::QueryTypeOp<'tcx> for Subtype<'tcx> { - type QueryResponse = (); - - fn try_fast_path(_tcx: TyCtxt<'tcx>, key: &ParamEnvAnd<'tcx, Self>) -> Option<()> { - if key.value.sub == key.value.sup { Some(()) } else { None } - } - - fn perform_query( - tcx: TyCtxt<'tcx>, - canonicalized: Canonical<'tcx, ParamEnvAnd<'tcx, Self>>, - ) -> Result, NoSolution> { - tcx.type_op_subtype(canonicalized) - } - - fn perform_locally_with_next_solver( - ocx: &ObligationCtxt<'_, 'tcx>, - key: ParamEnvAnd<'tcx, Self>, - ) -> Result { - ocx.sub(&ObligationCause::dummy(), key.param_env, key.value.sub, key.value.sup)?; - Ok(()) - } -} diff --git a/compiler/rustc_traits/src/type_op.rs b/compiler/rustc_traits/src/type_op.rs index 5affadaac38c3..f34adf8575565 100644 --- a/compiler/rustc_traits/src/type_op.rs +++ b/compiler/rustc_traits/src/type_op.rs @@ -10,18 +10,14 @@ use rustc_trait_selection::traits::query::normalize::QueryNormalizeExt; use rustc_trait_selection::traits::query::type_op::ascribe_user_type::{ type_op_ascribe_user_type_with_span, AscribeUserType, }; -use rustc_trait_selection::traits::query::type_op::eq::Eq; use rustc_trait_selection::traits::query::type_op::normalize::Normalize; use rustc_trait_selection::traits::query::type_op::prove_predicate::ProvePredicate; -use rustc_trait_selection::traits::query::type_op::subtype::Subtype; use rustc_trait_selection::traits::{Normalized, Obligation, ObligationCause, ObligationCtxt}; pub(crate) fn provide(p: &mut Providers) { *p = Providers { type_op_ascribe_user_type, - type_op_eq, type_op_prove_predicate, - type_op_subtype, type_op_normalize_ty, type_op_normalize_clause, type_op_normalize_fn_sig, @@ -39,16 +35,6 @@ fn type_op_ascribe_user_type<'tcx>( }) } -fn type_op_eq<'tcx>( - tcx: TyCtxt<'tcx>, - canonicalized: Canonical<'tcx, ParamEnvAnd<'tcx, Eq<'tcx>>>, -) -> Result<&'tcx Canonical<'tcx, QueryResponse<'tcx, ()>>, NoSolution> { - tcx.infer_ctxt().enter_canonical_trait_query(&canonicalized, |ocx, key| { - let (param_env, Eq { a, b }) = key.into_parts(); - Ok(ocx.eq(&ObligationCause::dummy(), param_env, a, b)?) - }) -} - fn type_op_normalize<'tcx, T>( ocx: &ObligationCtxt<'_, 'tcx>, key: ParamEnvAnd<'tcx, Normalize>, @@ -91,16 +77,6 @@ fn type_op_normalize_poly_fn_sig<'tcx>( tcx.infer_ctxt().enter_canonical_trait_query(&canonicalized, type_op_normalize) } -fn type_op_subtype<'tcx>( - tcx: TyCtxt<'tcx>, - canonicalized: Canonical<'tcx, ParamEnvAnd<'tcx, Subtype<'tcx>>>, -) -> Result<&'tcx Canonical<'tcx, QueryResponse<'tcx, ()>>, NoSolution> { - tcx.infer_ctxt().enter_canonical_trait_query(&canonicalized, |ocx, key| { - let (param_env, Subtype { sub, sup }) = key.into_parts(); - Ok(ocx.sup(&ObligationCause::dummy(), param_env, sup, sub)?) - }) -} - fn type_op_prove_predicate<'tcx>( tcx: TyCtxt<'tcx>, canonicalized: Canonical<'tcx, ParamEnvAnd<'tcx, ProvePredicate<'tcx>>>, From bb84372e436dfeba61ad1b4deef408a3cf9a12c5 Mon Sep 17 00:00:00 2001 From: Nadrieril Date: Sat, 10 Aug 2024 14:49:26 +0200 Subject: [PATCH 12/16] rust-analyzer: use in-tree pattern_analysis crate --- src/tools/rust-analyzer/crates/hir-ty/src/lib.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tools/rust-analyzer/crates/hir-ty/src/lib.rs b/src/tools/rust-analyzer/crates/hir-ty/src/lib.rs index 4c9e0a1e11836..21c84511dc35e 100644 --- a/src/tools/rust-analyzer/crates/hir-ty/src/lib.rs +++ b/src/tools/rust-analyzer/crates/hir-ty/src/lib.rs @@ -15,8 +15,10 @@ extern crate rustc_abi; #[cfg(not(feature = "in-rust-tree"))] extern crate ra_ap_rustc_abi as rustc_abi; -// Use the crates.io version unconditionally until the API settles enough that we can switch to -// using the in-tree one. +#[cfg(feature = "in-rust-tree")] +extern crate rustc_pattern_analysis; + +#[cfg(not(feature = "in-rust-tree"))] extern crate ra_ap_rustc_pattern_analysis as rustc_pattern_analysis; mod builder; From 4290943fb35b1e0472d6ee4888c5ab106abe1311 Mon Sep 17 00:00:00 2001 From: Michael Goulet Date: Tue, 13 Aug 2024 23:28:08 -0400 Subject: [PATCH 13/16] Infer async closure args from Fn bound even if there is no corresponding Future bound --- compiler/rustc_hir_typeck/src/closure.rs | 45 ++++++++++++----- .../async-closures/sig-from-bare-fn.rs | 49 +++++++++++++++++++ 2 files changed, 83 insertions(+), 11 deletions(-) create mode 100644 tests/ui/async-await/async-closures/sig-from-bare-fn.rs diff --git a/compiler/rustc_hir_typeck/src/closure.rs b/compiler/rustc_hir_typeck/src/closure.rs index a7953acc95c80..3d176b7819307 100644 --- a/compiler/rustc_hir_typeck/src/closure.rs +++ b/compiler/rustc_hir_typeck/src/closure.rs @@ -14,7 +14,7 @@ use rustc_middle::span_bug; use rustc_middle::ty::visit::{TypeVisitable, TypeVisitableExt}; use rustc_middle::ty::{self, GenericArgs, Ty, TyCtxt, TypeSuperVisitable, TypeVisitor}; use rustc_span::def_id::LocalDefId; -use rustc_span::Span; +use rustc_span::{Span, DUMMY_SP}; use rustc_target::spec::abi::Abi; use rustc_trait_selection::error_reporting::traits::ArgKind; use rustc_trait_selection::traits; @@ -539,6 +539,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { /// we identify the `FnOnce` bound, and if the output type is /// an inference variable `?Fut`, we check if that is bounded by a `Future` /// projection. + /// + /// This function is actually best-effort with the return type; if we don't find a + /// `Future` projection, we still will return arguments that we extracted from the `FnOnce` + /// projection, and the output will be an unconstrained type variable instead. fn extract_sig_from_projection_and_future_bound( &self, cause_span: Option, @@ -564,24 +568,43 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { }; // FIXME: We may want to elaborate here, though I assume this will be exceedingly rare. + let mut return_ty = None; for bound in self.obligations_for_self_ty(return_vid) { if let Some(ret_projection) = bound.predicate.as_projection_clause() && let Some(ret_projection) = ret_projection.no_bound_vars() && self.tcx.is_lang_item(ret_projection.def_id(), LangItem::FutureOutput) { - let sig = projection.rebind(self.tcx.mk_fn_sig( - input_tys, - ret_projection.term.expect_type(), - false, - hir::Safety::Safe, - Abi::Rust, - )); - - return Some(ExpectedSig { cause_span, sig }); + return_ty = Some(ret_projection.term.expect_type()); + break; } } - None + // SUBTLE: If we didn't find a `Future` bound for the return + // vid, we still want to attempt to provide inference guidance for the async + // closure's arguments. Instantiate a new vid to plug into the output type. + // + // You may be wondering, what if it's higher-ranked? Well, given that we + // found a type variable for the `FnOnce::Output` projection above, we know + // that the output can't mention any of the vars. + // + // Also note that we use a fresh var here for the signature since the signature + // records the output of the *future*, and `return_vid` above is the type + // variable of the future, not its output. + // + // FIXME: We probably should store this signature inference output in a way + // that does not misuse a `FnSig` type, but that can be done separately. + let return_ty = + return_ty.unwrap_or_else(|| self.next_ty_var(cause_span.unwrap_or(DUMMY_SP))); + + let sig = projection.rebind(self.tcx.mk_fn_sig( + input_tys, + return_ty, + false, + hir::Safety::Safe, + Abi::Rust, + )); + + return Some(ExpectedSig { cause_span, sig }); } fn sig_of_closure( diff --git a/tests/ui/async-await/async-closures/sig-from-bare-fn.rs b/tests/ui/async-await/async-closures/sig-from-bare-fn.rs new file mode 100644 index 0000000000000..a679471a3b3d0 --- /dev/null +++ b/tests/ui/async-await/async-closures/sig-from-bare-fn.rs @@ -0,0 +1,49 @@ +//@ check-pass +//@ edition: 2021 + +// Make sure that we infer the args of an async closure even if it's passed to +// a function that requires the async closure implement `Fn*` but does *not* have +// a `Future` bound on the return type. + +#![feature(async_closure)] + +use std::future::Future; + +trait TryStream { + type Ok; + type Err; +} + +trait TryFuture { + type Ok; + type Err; +} + +impl TryFuture for F where F: Future> { + type Ok = T; + type Err = E; +} + +trait TryStreamExt: TryStream { + fn try_for_each(&self, f: F) + where + F: FnMut(Self::Ok) -> Fut, + Fut: TryFuture; +} + +impl TryStreamExt for S where S: TryStream { + fn try_for_each(&self, f: F) + where + F: FnMut(Self::Ok) -> Fut, + Fut: TryFuture, + { } +} + +fn test(stream: impl TryStream) { + stream.try_for_each(async |s| { + s.trim(); // Make sure we know the type of `s` at this point. + Ok(()) + }); +} + +fn main() {} From 9028b5381b2867bcd4d9cd0ba95ff97607deaaf3 Mon Sep 17 00:00:00 2001 From: Alona Enraght-Moony Date: Thu, 15 Aug 2024 13:12:11 +0000 Subject: [PATCH 14/16] rustdoc-json: Use FxHashMap from rustdoc_json_types --- src/librustdoc/json/mod.rs | 10 +++++----- src/rustdoc-json-types/lib.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/librustdoc/json/mod.rs b/src/librustdoc/json/mod.rs index 860672443f2e8..a424faaf99986 100644 --- a/src/librustdoc/json/mod.rs +++ b/src/librustdoc/json/mod.rs @@ -13,12 +13,15 @@ use std::io::{stdout, BufWriter, Write}; use std::path::PathBuf; use std::rc::Rc; -use rustc_data_structures::fx::FxHashMap; use rustc_hir::def_id::{DefId, DefIdSet}; use rustc_middle::ty::TyCtxt; use rustc_session::Session; use rustc_span::def_id::LOCAL_CRATE; use rustdoc_json_types as types; +// It's important to use the FxHashMap from rustdoc_json_types here, instead of +// the one from rustc_data_structures, as they're different types due to sysroots. +// See #110051 and #127456 for details +use rustdoc_json_types::FxHashMap; use crate::clean::types::{ExternalCrate, ExternalLocation}; use crate::clean::ItemKind; @@ -234,14 +237,11 @@ impl<'tcx> FormatRenderer<'tcx> for JsonRenderer<'tcx> { let index = (*self.index).clone().into_inner(); debug!("Constructing Output"); - // This needs to be the default HashMap for compatibility with the public interface for - // rustdoc-json-types - #[allow(rustc::default_hash_types)] let output = types::Crate { root: types::Id(format!("0:0:{}", e.name(self.tcx).as_u32())), crate_version: self.cache.crate_version.clone(), includes_private: self.cache.document_private, - index: index.into_iter().collect(), + index, paths: self .cache .paths diff --git a/src/rustdoc-json-types/lib.rs b/src/rustdoc-json-types/lib.rs index 999134a40909d..40a90c1a56504 100644 --- a/src/rustdoc-json-types/lib.rs +++ b/src/rustdoc-json-types/lib.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; -use rustc_hash::FxHashMap; +pub use rustc_hash::FxHashMap; use serde::{Deserialize, Serialize}; /// The version of JSON output that this crate represents. From a19a8f8e52364b4ce5ade2899b57260dd347c93c Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 15 Aug 2024 14:44:38 +0200 Subject: [PATCH 15/16] Remove duplicated `Rustdoc::output` method from `run-make-support` lib --- src/tools/run-make-support/src/external_deps/rustdoc.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/tools/run-make-support/src/external_deps/rustdoc.rs b/src/tools/run-make-support/src/external_deps/rustdoc.rs index 96c2218a563e6..96b1c719e2e3e 100644 --- a/src/tools/run-make-support/src/external_deps/rustdoc.rs +++ b/src/tools/run-make-support/src/external_deps/rustdoc.rs @@ -71,14 +71,8 @@ impl Rustdoc { self } - /// Specify path to the output folder. - pub fn output>(&mut self, path: P) -> &mut Self { - self.cmd.arg("-o"); - self.cmd.arg(path.as_ref()); - self - } - /// Specify output directory. + #[doc(alias = "output")] pub fn out_dir>(&mut self, path: P) -> &mut Self { self.cmd.arg("--out-dir").arg(path.as_ref()); self From d562a4a5108d0b67ba2a648530aab2b15224a615 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 15 Aug 2024 14:44:48 +0200 Subject: [PATCH 16/16] About rmake tests --- tests/run-make/deref-impl-rustdoc-ice/rmake.rs | 2 +- tests/run-make/emit-shared-files/rmake.rs | 6 +++--- tests/run-make/exit-code/rmake.rs | 2 +- tests/run-make/rustdoc-determinism/rmake.rs | 8 ++++---- tests/run-make/rustdoc-io-error/rmake.rs | 2 +- tests/run-make/rustdoc-map-file/rmake.rs | 2 +- tests/run-make/rustdoc-output-path/rmake.rs | 2 +- tests/run-make/rustdoc-output-stdout/rmake.rs | 2 +- tests/run-make/rustdoc-scrape-examples-macros/rmake.rs | 4 ++-- tests/run-make/rustdoc-scrape-examples-remap/scrape.rs | 4 ++-- tests/run-make/rustdoc-target-spec-json-path/rmake.rs | 2 +- tests/run-make/rustdoc-themes/rmake.rs | 2 +- tests/run-make/rustdoc-with-out-dir-option/rmake.rs | 2 +- 13 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/run-make/deref-impl-rustdoc-ice/rmake.rs b/tests/run-make/deref-impl-rustdoc-ice/rmake.rs index 91fc0a9025fad..0ad5934f62e18 100644 --- a/tests/run-make/deref-impl-rustdoc-ice/rmake.rs +++ b/tests/run-make/deref-impl-rustdoc-ice/rmake.rs @@ -12,5 +12,5 @@ use run_make_support::{cwd, rustc, rustdoc}; fn main() { rustc().input("foo.rs").run(); rustc().input("bar.rs").run(); - rustdoc().input("baz.rs").library_search_path(cwd()).output(cwd()).run(); + rustdoc().input("baz.rs").library_search_path(cwd()).out_dir(cwd()).run(); } diff --git a/tests/run-make/emit-shared-files/rmake.rs b/tests/run-make/emit-shared-files/rmake.rs index 8ac9073e993ba..e5482af10bb94 100644 --- a/tests/run-make/emit-shared-files/rmake.rs +++ b/tests/run-make/emit-shared-files/rmake.rs @@ -13,7 +13,7 @@ fn main() { rustdoc() .arg("-Zunstable-options") .arg("--emit=invocation-specific") - .output("invocation-only") + .out_dir("invocation-only") .arg("--resource-suffix=-xxx") .args(&["--theme", "y.css"]) .args(&["--extend-css", "z.css"]) @@ -34,7 +34,7 @@ fn main() { rustdoc() .arg("-Zunstable-options") .arg("--emit=toolchain-shared-resources") - .output("toolchain-only") + .out_dir("toolchain-only") .arg("--resource-suffix=-xxx") .args(&["--extend-css", "z.css"]) .input("x.rs") @@ -68,7 +68,7 @@ fn main() { rustdoc() .arg("-Zunstable-options") .arg("--emit=toolchain-shared-resources,unversioned-shared-resources") - .output("all-shared") + .out_dir("all-shared") .arg("--resource-suffix=-xxx") .args(&["--extend-css", "z.css"]) .input("x.rs") diff --git a/tests/run-make/exit-code/rmake.rs b/tests/run-make/exit-code/rmake.rs index f290554831db7..d3dcc04428cb6 100644 --- a/tests/run-make/exit-code/rmake.rs +++ b/tests/run-make/exit-code/rmake.rs @@ -16,7 +16,7 @@ fn main() { .run_fail() .assert_exit_code(101); - rustdoc().arg("success.rs").output("exit-code").run(); + rustdoc().arg("success.rs").out_dir("exit-code").run(); rustdoc().arg("--invalid-arg-foo").run_fail().assert_exit_code(1); diff --git a/tests/run-make/rustdoc-determinism/rmake.rs b/tests/run-make/rustdoc-determinism/rmake.rs index aa8090174d9c3..ce0885081783b 100644 --- a/tests/run-make/rustdoc-determinism/rmake.rs +++ b/tests/run-make/rustdoc-determinism/rmake.rs @@ -7,12 +7,12 @@ use run_make_support::{diff, rustdoc}; fn main() { let foo_first = Path::new("foo_first"); - rustdoc().input("foo.rs").output(&foo_first).run(); - rustdoc().input("bar.rs").output(&foo_first).run(); + rustdoc().input("foo.rs").out_dir(&foo_first).run(); + rustdoc().input("bar.rs").out_dir(&foo_first).run(); let bar_first = Path::new("bar_first"); - rustdoc().input("bar.rs").output(&bar_first).run(); - rustdoc().input("foo.rs").output(&bar_first).run(); + rustdoc().input("bar.rs").out_dir(&bar_first).run(); + rustdoc().input("foo.rs").out_dir(&bar_first).run(); diff() .expected_file(foo_first.join("search-index.js")) diff --git a/tests/run-make/rustdoc-io-error/rmake.rs b/tests/run-make/rustdoc-io-error/rmake.rs index a5fae36e733da..31441d7ebc5c7 100644 --- a/tests/run-make/rustdoc-io-error/rmake.rs +++ b/tests/run-make/rustdoc-io-error/rmake.rs @@ -25,7 +25,7 @@ fn main() { permissions.set_readonly(true); rfs::set_permissions(&out_dir, permissions); - let output = rustdoc().input("foo.rs").output(&out_dir).env("RUST_BACKTRACE", "1").run_fail(); + let output = rustdoc().input("foo.rs").out_dir(&out_dir).env("RUST_BACKTRACE", "1").run_fail(); rfs::set_permissions(&out_dir, original_permissions); diff --git a/tests/run-make/rustdoc-map-file/rmake.rs b/tests/run-make/rustdoc-map-file/rmake.rs index 08f9595ef9f6a..a4485165fd277 100644 --- a/tests/run-make/rustdoc-map-file/rmake.rs +++ b/tests/run-make/rustdoc-map-file/rmake.rs @@ -6,7 +6,7 @@ fn main() { .input("foo.rs") .arg("-Zunstable-options") .arg("--generate-redirect-map") - .output(&out_dir) + .out_dir(&out_dir) .run(); // FIXME (GuillaumeGomez): Port the python script to Rust as well. python_command().arg("validate_json.py").arg(&out_dir).run(); diff --git a/tests/run-make/rustdoc-output-path/rmake.rs b/tests/run-make/rustdoc-output-path/rmake.rs index 3c1ccd3a06936..181239eac4dd3 100644 --- a/tests/run-make/rustdoc-output-path/rmake.rs +++ b/tests/run-make/rustdoc-output-path/rmake.rs @@ -6,6 +6,6 @@ use run_make_support::rustdoc; fn main() { let out_dir = Path::new("foo/bar/doc"); - rustdoc().input("foo.rs").output(&out_dir).run(); + rustdoc().input("foo.rs").out_dir(&out_dir).run(); assert!(out_dir.exists()); } diff --git a/tests/run-make/rustdoc-output-stdout/rmake.rs b/tests/run-make/rustdoc-output-stdout/rmake.rs index e7dfb66602cf3..dbc9892f3f59a 100644 --- a/tests/run-make/rustdoc-output-stdout/rmake.rs +++ b/tests/run-make/rustdoc-output-stdout/rmake.rs @@ -10,7 +10,7 @@ fn main() { // First we check that we generate the JSON in the stdout. rustdoc() .input("foo.rs") - .output("-") + .out_dir("-") .arg("-Zunstable-options") .output_format("json") .run() diff --git a/tests/run-make/rustdoc-scrape-examples-macros/rmake.rs b/tests/run-make/rustdoc-scrape-examples-macros/rmake.rs index b77df7adc8d9e..546a0685b4ee5 100644 --- a/tests/run-make/rustdoc-scrape-examples-macros/rmake.rs +++ b/tests/run-make/rustdoc-scrape-examples-macros/rmake.rs @@ -35,7 +35,7 @@ fn main() { .input("examples/ex.rs") .crate_name("ex") .crate_type("bin") - .output(&out_dir) + .out_dir(&out_dir) .extern_(crate_name, rust_lib_name(crate_name)) .extern_(proc_crate_name, dylib_name.trim()) .arg("-Zunstable-options") @@ -49,7 +49,7 @@ fn main() { .input("src/lib.rs") .crate_name(crate_name) .crate_type("lib") - .output(&out_dir) + .out_dir(&out_dir) .arg("-Zunstable-options") .arg("--with-examples") .arg(&ex_dir) diff --git a/tests/run-make/rustdoc-scrape-examples-remap/scrape.rs b/tests/run-make/rustdoc-scrape-examples-remap/scrape.rs index eca07043b557c..c4d7814c3c831 100644 --- a/tests/run-make/rustdoc-scrape-examples-remap/scrape.rs +++ b/tests/run-make/rustdoc-scrape-examples-remap/scrape.rs @@ -20,7 +20,7 @@ pub fn scrape(extra_args: &[&str]) { .input(&dep) .crate_name(&dep_stem) .crate_type("bin") - .output(&out_dir) + .out_dir(&out_dir) .extern_(crate_name, format!("lib{crate_name}.rmeta")) .arg("-Zunstable-options") .arg("--scrape-examples-output-path") @@ -35,7 +35,7 @@ pub fn scrape(extra_args: &[&str]) { let mut rustdoc = rustdoc(); rustdoc .input("src/lib.rs") - .output(&out_dir) + .out_dir(&out_dir) .crate_name(crate_name) .crate_type("lib") .arg("-Zunstable-options"); diff --git a/tests/run-make/rustdoc-target-spec-json-path/rmake.rs b/tests/run-make/rustdoc-target-spec-json-path/rmake.rs index 3246fc5650646..fe9587f502228 100644 --- a/tests/run-make/rustdoc-target-spec-json-path/rmake.rs +++ b/tests/run-make/rustdoc-target-spec-json-path/rmake.rs @@ -7,7 +7,7 @@ fn main() { rustc().crate_type("lib").input("dummy_core.rs").target("target.json").run(); rustdoc() .input("my_crate.rs") - .output(out_dir) + .out_dir(out_dir) .library_search_path(cwd()) .target("target.json") .run(); diff --git a/tests/run-make/rustdoc-themes/rmake.rs b/tests/run-make/rustdoc-themes/rmake.rs index 8a961beb9f742..4577e47d47e72 100644 --- a/tests/run-make/rustdoc-themes/rmake.rs +++ b/tests/run-make/rustdoc-themes/rmake.rs @@ -27,6 +27,6 @@ fn main() { rfs::create_dir_all(&out_dir); rfs::write(&test_css, test_content); - rustdoc().output(&out_dir).input("foo.rs").arg("--theme").arg(&test_css).run(); + rustdoc().out_dir(&out_dir).input("foo.rs").arg("--theme").arg(&test_css).run(); htmldocck().arg(out_dir).arg("foo.rs").run(); } diff --git a/tests/run-make/rustdoc-with-out-dir-option/rmake.rs b/tests/run-make/rustdoc-with-out-dir-option/rmake.rs index ded89c9ae7913..a82a1965a9c9d 100644 --- a/tests/run-make/rustdoc-with-out-dir-option/rmake.rs +++ b/tests/run-make/rustdoc-with-out-dir-option/rmake.rs @@ -2,6 +2,6 @@ use run_make_support::{htmldocck, rustdoc}; fn main() { let out_dir = "rustdoc"; - rustdoc().input("src/lib.rs").crate_name("foobar").crate_type("lib").output(&out_dir).run(); + rustdoc().input("src/lib.rs").crate_name("foobar").crate_type("lib").out_dir(&out_dir).run(); htmldocck().arg(out_dir).arg("src/lib.rs").run(); }