Auto merge of #123502 - bjorn3:sync_cg_clif-2024-04-05, r=bjorn3

Subtree sync for rustc_codegen_cranelift

This fixes an ICE when compiling unchecked_shl/unchecked_shr.

r? `@ghost`

`@rustbot` label +A-codegen +A-cranelift +T-compiler

compiler/rustc_codegen_cranelift/.cirrus.yml

@@ -13,4 +13,7 @@ task:
     - ./y.sh prepare
   test_script:
     - . $HOME/.cargo/env
+    # Disabling incr comp reduces cache size and incr comp doesn't save as much
+    # on CI anyway.
+    - export CARGO_BUILD_INCREMENTAL=false
     - ./y.sh test

compiler/rustc_codegen_cranelift/.github/workflows/abi-cafe.yml

@@ -3,6 +3,8 @@ name: Abi-cafe
 on:
   - push
 
+permissions: {}
+
 jobs:
   abi_cafe:
     runs-on: ${{ matrix.os }}

compiler/rustc_codegen_cranelift/.github/workflows/main.yml

@@ -4,6 +4,20 @@ on:
   - push
   - pull_request
 
+defaults:
+  run:
+    shell: bash
+
+permissions: {}
+
+env:
+  # Disabling incr comp reduces cache size and incr comp doesn't save as much
+  # on CI anyway.
+  CARGO_BUILD_INCREMENTAL: false
+  # Rust's CI denies warnings. Deny them here too to ensure subtree syncs don't
+  # fail because of warnings.
+  RUSTFLAGS: "-Dwarnings"
+
 jobs:
   rustfmt:
     runs-on: ubuntu-latest
@@ -23,15 +37,15 @@ jobs:
         cargo fmt --check
         rustfmt --check build_system/main.rs
         rustfmt --check example/*
+        rustfmt --check scripts/*.rs
 
 
   test:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
 
-    defaults:
-      run:
-        shell: bash
+    env:
+      CG_CLIF_EXPENSIVE_CHECKS: 1
 
     strategy:
       fail-fast: false
@@ -47,15 +61,19 @@ jobs:
           - os: ubuntu-latest
             env:
               TARGET_TRIPLE: x86_64-pc-windows-gnu
+            apt_deps: gcc-mingw-w64-x86-64 wine-stable
           - os: ubuntu-latest
             env:
               TARGET_TRIPLE: aarch64-unknown-linux-gnu
+            apt_deps: gcc-aarch64-linux-gnu qemu-user
           - os: ubuntu-latest
             env:
               TARGET_TRIPLE: s390x-unknown-linux-gnu
+            apt_deps: gcc-s390x-linux-gnu qemu-user
           - os: ubuntu-latest
             env:
               TARGET_TRIPLE: riscv64gc-unknown-linux-gnu
+            apt_deps: gcc-riscv64-linux-gnu qemu-user
           - os: windows-latest
             env:
               TARGET_TRIPLE: x86_64-pc-windows-msvc
@@ -80,29 +98,11 @@ jobs:
       if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
       run: rustup set default-host x86_64-pc-windows-gnu
 
-    - name: Install MinGW toolchain and wine
-      if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
-
-    - name: Install AArch64 toolchain and qemu
-      if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'aarch64-unknown-linux-gnu'
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y gcc-aarch64-linux-gnu qemu-user
-
-    - name: Install s390x toolchain and qemu
-      if: matrix.env.TARGET_TRIPLE == 's390x-unknown-linux-gnu'
+    - name: Install toolchain and emulator
+      if: matrix.apt_deps != null
       run: |
         sudo apt-get update
-        sudo apt-get install -y gcc-s390x-linux-gnu qemu-user
-
-    - name: Install riscv64gc toolchain and qemu
-      if: matrix.env.TARGET_TRIPLE == 'riscv64gc-unknown-linux-gnu'
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y gcc-riscv64-linux-gnu qemu-user
+        sudo apt-get install -y ${{ matrix.apt_deps }}
 
     - name: Prepare dependencies
       run: ./y.sh prepare
@@ -142,10 +142,6 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 60
 
-    defaults:
-      run:
-        shell: bash
-
     steps:
     - uses: actions/checkout@v4
 
@@ -168,10 +164,6 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 60
 
-    defaults:
-      run:
-        shell: bash
-
     steps:
     - uses: actions/checkout@v4
 
@@ -193,20 +185,16 @@ jobs:
       run: ./y.sh prepare
 
     - name: Build
-      run: CI_OPT=1 ./y.sh build --sysroot none
+      run: ./y.sh build --sysroot none
 
     - name: Benchmark
-      run: CI_OPT=1 ./y.sh bench
+      run: ./y.sh bench
 
 
   dist:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
 
-    defaults:
-      run:
-        shell: bash
-
     strategy:
       fail-fast: false
       matrix:
@@ -252,10 +240,10 @@ jobs:
       run: ./y.sh prepare
 
     - name: Build backend
-      run: CI_OPT=1 ./y.sh build --sysroot none
+      run: ./y.sh build --sysroot none
 
     - name: Build sysroot
-      run: CI_OPT=1 ./y.sh build
+      run: ./y.sh build
 
     - name: Package prebuilt cg_clif
       run: tar cvfJ cg_clif.tar.xz dist

compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml

@@ -3,6 +3,8 @@ name: Various rustc tests
 on:
   - push
 
+permissions: {}
+
 jobs:
   bootstrap_rustc:
     runs-on: ubuntu-latest

compiler/rustc_codegen_cranelift/Readme.md

@@ -101,27 +101,7 @@ For additional ways to use rustc_codegen_cranelift like the JIT mode see [usage.
 
 ## Building and testing with changes in rustc code
 
-This is useful when changing code in `rustc_codegen_cranelift` as part of changing [main Rust repository](https://github.com/rust-lang/rust/).
-This can happen, for example, when you are implementing a new compiler intrinsic.
-
-Instruction below uses `$RustCheckoutDir` as substitute for any folder where you cloned Rust repository.
-
-You need to do this steps to successfully compile and use the cranelift backend with your changes in rustc code:
-
-1. `cd $RustCheckoutDir`
-2. Run `python x.py setup` and choose option for compiler (`b`).
-3. Build compiler and necessary tools: `python x.py build --stage=2 compiler library/std src/tools/rustdoc src/tools/rustfmt`
-   * (Optional) You can also build cargo by adding `src/tools/cargo` to previous command.
-4. Copy cargo from a nightly toolchain: `cp $(rustup +nightly which cargo) ./build/host/stage2/bin/cargo`. Note that you would need to do this every time you rebuilt `rust` repository.
-5. Link your new `rustc` to toolchain: `rustup toolchain link stage2 ./build/host/stage2/`.
-6. (Windows only) compile the build system: `rustc +stage2 -O build_system/main.rs -o y.exe`.
-7. You need to prefix every `./y.sh` (or `y` if you built `build_system/main.rs` as `y`) command by `rustup run stage2` to make cg_clif use your local changes in rustc.
-  * `rustup run stage2 ./y.sh prepare`
-  * `rustup run stage2 ./y.sh build`
-  * (Optional) run tests: `rustup run stage2 ./y.sh test`
-8. Now you can use your cg_clif build to compile other Rust programs, e.g. you can open any Rust crate and run commands like `$RustCheckoutDir/compiler/rustc_codegen_cranelift/dist/cargo-clif build --release`.
-
-You can also set `rust-analyzer.rustc.source` to your rust workspace to get rust-analyzer to understand your changes.
+See [rustc_testing.md](docs/rustc_testing.md).
 
 ## Not yet supported
 

compiler/rustc_codegen_cranelift/build_system/build_backend.rs

@@ -1,9 +1,10 @@
+use std::env;
 use std::path::PathBuf;
 
 use crate::path::{Dirs, RelPath};
 use crate::rustc_info::get_file_name;
 use crate::shared_utils::{rustflags_from_env, rustflags_to_cmd_env};
-use crate::utils::{is_ci, is_ci_opt, maybe_incremental, CargoProject, Compiler, LogGroup};
+use crate::utils::{CargoProject, Compiler, LogGroup};
 
 pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
 
@@ -16,20 +17,15 @@ pub(crate) fn build_backend(
     let _group = LogGroup::guard("Build backend");
 
     let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs);
-    maybe_incremental(&mut cmd);
 
     let mut rustflags = rustflags_from_env("RUSTFLAGS");
 
     rustflags.push("-Zallow-features=rustc_private".to_owned());
 
-    if is_ci() {
-        // Deny warnings on CI
-        rustflags.push("-Dwarnings".to_owned());
-
-        if !is_ci_opt() {
-            cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true");
-            cmd.env("CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS", "true");
-        }
+    if env::var("CG_CLIF_EXPENSIVE_CHECKS").is_ok() {
+        // Enabling debug assertions implicitly enables the clif ir verifier
+        cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true");
+        cmd.env("CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS", "true");
     }
 
     if use_unstable_features {

compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs

@@ -6,8 +6,7 @@ use std::process::Command;
 use crate::path::{Dirs, RelPath};
 use crate::rustc_info::get_file_name;
 use crate::utils::{
-    maybe_incremental, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler,
-    LogGroup,
+    remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler, LogGroup,
 };
 use crate::{config, CodegenBackend, SysrootKind};
 
@@ -270,7 +269,6 @@ fn build_clif_sysroot_for_triple(
     }
     compiler.rustflags.extend(rustflags);
     let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs);
-    maybe_incremental(&mut build_cmd);
     if channel == "release" {
         build_cmd.arg("--release");
     }

compiler/rustc_codegen_cranelift/build_system/main.rs

@@ -6,7 +6,7 @@ use std::env;
 use std::path::PathBuf;
 use std::process;
 
-use self::utils::{is_ci, is_ci_opt, Compiler};
+use self::utils::Compiler;
 
 mod abi_cafe;
 mod bench;
@@ -60,14 +60,9 @@ fn main() {
     }
     env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1");
 
-    if is_ci() {
-        // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
-        env::set_var("CARGO_BUILD_INCREMENTAL", "false");
-
-        if !is_ci_opt() {
-            // Enable the Cranelift verifier
-            env::set_var("CG_CLIF_ENABLE_VERIFIER", "1");
-        }
+    // Force incr comp even in release mode unless in CI or incremental builds are explicitly disabled
+    if env::var_os("CARGO_BUILD_INCREMENTAL").is_none() {
+        env::set_var("CARGO_BUILD_INCREMENTAL", "true");
     }
 
     let mut args = env::args().skip(1);

compiler/rustc_codegen_cranelift/build_system/prepare.rs

@@ -15,7 +15,6 @@ pub(crate) fn prepare(dirs: &Dirs) {
     RelPath::DOWNLOAD.ensure_exists(dirs);
     crate::tests::RAND_REPO.fetch(dirs);
     crate::tests::REGEX_REPO.fetch(dirs);
-    crate::tests::PORTABLE_SIMD_REPO.fetch(dirs);
 }
 
 pub(crate) fn prepare_stdlib(dirs: &Dirs, rustc: &Path) {

compiler/rustc_codegen_cranelift/build_system/tests.rs

@@ -130,16 +130,10 @@ pub(crate) static REGEX_REPO: GitRepo = GitRepo::github(
 
 pub(crate) static REGEX: CargoProject = CargoProject::new(&REGEX_REPO.source_dir(), "regex_target");
 
-pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
-    "rust-lang",
-    "portable-simd",
-    "5794c837bc605c4cd9dbb884285976dfdb293cce",
-    "a64d8fdd0ed0d9c4",
-    "portable-simd",
-);
+pub(crate) static PORTABLE_SIMD_SRC: RelPath = RelPath::BUILD.join("coretests");
 
 pub(crate) static PORTABLE_SIMD: CargoProject =
-    CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable-simd_target");
+    CargoProject::new(&PORTABLE_SIMD_SRC, "portable-simd_target");
 
 static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests");
 
@@ -221,7 +215,12 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
         }
     }),
     TestCase::custom("test.portable-simd", &|runner| {
-        PORTABLE_SIMD_REPO.patch(&runner.dirs);
+        apply_patches(
+            &runner.dirs,
+            "portable-simd",
+            &runner.stdlib_source.join("library/portable-simd"),
+            &PORTABLE_SIMD_SRC.to_path(&runner.dirs),
+        );
 
         PORTABLE_SIMD.clean(&runner.dirs);
 

compiler/rustc_codegen_cranelift/build_system/utils.rs

@@ -254,14 +254,6 @@ pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) {
     }
 }
 
-pub(crate) fn is_ci() -> bool {
-    env::var("CI").is_ok()
-}
-
-pub(crate) fn is_ci_opt() -> bool {
-    env::var("CI_OPT").is_ok()
-}
-
 static IN_GROUP: AtomicBool = AtomicBool::new(false);
 pub(crate) struct LogGroup {
     is_gha: bool,
@@ -288,13 +280,3 @@ impl Drop for LogGroup {
         IN_GROUP.store(false, Ordering::SeqCst);
     }
 }
-
-pub(crate) fn maybe_incremental(cmd: &mut Command) {
-    if is_ci() || std::env::var("CARGO_BUILD_INCREMENTAL").map_or(false, |val| val == "false") {
-        // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
-        cmd.env("CARGO_BUILD_INCREMENTAL", "false");
-    } else {
-        // Force incr comp even in release mode unless in CI or incremental builds are explicitly disabled
-        cmd.env("CARGO_BUILD_INCREMENTAL", "true");
-    }
-}

compiler/rustc_codegen_cranelift/docs/rustc_testing.md

@@ -0,0 +1,23 @@
+# Building and testing with changes in rustc code
+
+This is useful when changing code in `rustc_codegen_cranelift` as part of changing [main Rust repository](https://github.com/rust-lang/rust/).
+This can happen, for example, when you are implementing a new compiler intrinsic.
+
+Instruction below uses `$RustCheckoutDir` as substitute for any folder where you cloned Rust repository.
+
+You need to do this steps to successfully compile and use the cranelift backend with your changes in rustc code:
+
+1. `cd $RustCheckoutDir`
+2. Run `python x.py setup` and choose option for compiler (`b`).
+3. Build compiler and necessary tools: `python x.py build --stage=2 compiler library/std src/tools/rustdoc src/tools/rustfmt`
+   * (Optional) You can also build cargo by adding `src/tools/cargo` to previous command.
+4. Copy cargo from a nightly toolchain: `cp $(rustup +nightly which cargo) ./build/host/stage2/bin/cargo`. Note that you would need to do this every time you rebuilt `rust` repository.
+5. Link your new `rustc` to toolchain: `rustup toolchain link stage2 ./build/host/stage2/`.
+6. (Windows only) compile the build system: `rustc +stage2 -O build_system/main.rs -o y.exe`.
+7. You need to prefix every `./y.sh` (or `y` if you built `build_system/main.rs` as `y`) command by `rustup run stage2` to make cg_clif use your local changes in rustc.
+  * `rustup run stage2 ./y.sh prepare`
+  * `rustup run stage2 ./y.sh build`
+  * (Optional) run tests: `rustup run stage2 ./y.sh test`
+8. Now you can use your cg_clif build to compile other Rust programs, e.g. you can open any Rust crate and run commands like `$RustCheckoutDir/compiler/rustc_codegen_cranelift/dist/cargo-clif build --release`.
+
+You can also set `rust-analyzer.rustc.source` to your rust workspace to get rust-analyzer to understand your changes.