Auto merge of rust-lang#125419 - GuillaumeGomez:add-gcc-to-dist, r=<try>

[bootstrap] Add gcc to dist generation

As `@eholk` summarized below:

> From my understanding, this change would add libgccjit as an optional component to the Rust distribution. This library is licensed under GPLv2 and currently we do not have any other components under that license so it would be a new license, and one that is generally more restrictive than the other licenses we use.

It'll greatly improve the experience for anyone wanting to work on the GCC backend from the compiler.

Should help with rust-lang#124172.
Will unblock rust-lang#124353.

r? `@Kobzol`

.gitmodules

@@ -47,3 +47,6 @@
 	path = src/tools/rustc-perf
 	url = https://github.com/rust-lang/rustc-perf.git
 	shallow = true
+[submodule "src/gcc"]
+	path = src/gcc
+	url = https://github.com/rust-lang/gcc

LICENSES/GCC-exception-3.1.txt

@@ -0,0 +1,30 @@
+GCC RUNTIME LIBRARY EXCEPTION
+
+Version 3.1, 31 March 2009
+
+Copyright © 2009 Free Software Foundation, Inc. <https://fsf.org/>
+
+Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
+
+This GCC Runtime Library Exception ("Exception") is an additional permission under section 7 of the GNU General Public License, version 3 ("GPLv3"). It applies to a given file (the "Runtime Library") that bears a notice placed by the copyright holder of the file stating that the file is governed by GPLv3 along with this Exception.
+
+When you use GCC to compile a program, GCC may combine portions of certain GCC header files and runtime libraries with the compiled program. The purpose of this Exception is to allow compilation of non-GPL (including proprietary) programs to use, in this way, the header files and runtime libraries covered by this Exception.
+0. Definitions.
+
+A file is an "Independent Module" if it either requires the Runtime Library for execution after a Compilation Process, or makes use of an interface provided by the Runtime Library, but is not otherwise based on the Runtime Library.
+
+"GCC" means a version of the GNU Compiler Collection, with or without modifications, governed by version 3 (or a specified later version) of the GNU General Public License (GPL) with the option of using any subsequent versions published by the FSF.
+
+"GPL-compatible Software" is software whose conditions of propagation, modification and use would permit combination with GCC in accord with the license of GCC.
+
+"Target Code" refers to output from any compiler for a real or virtual target processor architecture, in executable form or suitable for input to an assembler, loader, linker and/or execution phase. Notwithstanding that, Target Code does not include data in any format that is used as a compiler intermediate representation, or used for producing a compiler intermediate representation.
+
+The "Compilation Process" transforms code entirely represented in non-intermediate languages designed for human-written code, and/or in Java Virtual Machine byte code, into Target Code. Thus, for example, use of source code generators and preprocessors need not be considered part of the Compilation Process, since the Compilation Process can be understood as starting with the output of the generators or preprocessors.
+
+A Compilation Process is "Eligible" if it is done using GCC, alone or with other GPL-compatible software, or if it is done without using any work based on GCC. For example, using non-GPL-compatible Software to optimize any GCC intermediate representations would not qualify as an Eligible Compilation Process.
+1. Grant of Additional Permission.
+
+You have permission to propagate a work of Target Code formed by combining the Runtime Library with Independent Modules, even if such propagation would otherwise violate the terms of GPLv3, provided that all Target Code was generated by Eligible Compilation Processes. You may then convey such a combination under terms of your choice, consistent with the licensing of the Independent Modules.
+2. No Weakening of GCC Copyleft.
+
+The availability of this Exception does not imply any general presumption that third-party software is unaffected by the copyleft requirements of the license of GCC.

LICENSES/ISC.txt

@@ -0,0 +1,7 @@
+ISC License
+
+<copyright notice>
+
+Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

src/bootstrap/src/core/build_steps/dist.rs

@@ -2332,6 +2332,7 @@ impl Step for RustDev {
         tarball.permit_symlinks(true);
 
         builder.ensure(crate::core::build_steps::llvm::Llvm { target });
+        builder.ensure(crate::core::build_steps::gcc::Gcc { target });
 
         // We want to package `lld` to use it with `download-ci-llvm`.
         builder.ensure(crate::core::build_steps::llvm::Lld { target });
@@ -2365,6 +2366,10 @@ impl Step for RustDev {
         // just broadly useful to be able to link against the bundled LLVM.
         tarball.add_dir(builder.llvm_out(target).join("include"), "include");
 
+        tarball.add_dir(builder.gcc_out(target).join("install/lib/libgccjit.so"), "libgccjit.so");
+        tarball
+            .add_dir(builder.gcc_out(target).join("install/lib/libgccjit.so.0"), "libgccjit.so.0");
+
         // Copy libLLVM.so to the target lib dir as well, so the RPATH like
         // `$ORIGIN/../lib` can find it. It may also be used as a dependency
         // of `rustc-dev` to support the inherited `-lLLVM` when using the

src/bootstrap/src/core/build_steps/gcc.rs

@@ -0,0 +1,227 @@
+//! Compilation of native dependencies like LLVM.
+//!
+//! Native projects like LLVM unfortunately aren't suited just yet for
+//! compilation in build scripts that Cargo has. This is because the
+//! compilation takes a *very* long time but also because we don't want to
+//! compile LLVM 3 times as part of a normal bootstrap (we want it cached).
+//!
+//! LLVM and compiler-rt are essentially just wired up to everything else to
+//! ensure that they're always in place if needed.
+
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::sync::OnceLock;
+
+use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::core::config::TargetSelection;
+use crate::utils::helpers::{self, t};
+use crate::{generate_smart_stamp_hash, Kind};
+
+use super::llvm::HashStamp;
+
+pub struct Meta {
+    stamp: HashStamp,
+    out_dir: PathBuf,
+    install_dir: PathBuf,
+    root: PathBuf,
+}
+
+pub enum GccBuildStatus {
+    AlreadyBuilt,
+    ShouldBuild(Meta),
+}
+
+/// This returns whether we've already previously built GCC.
+///
+/// It's used to avoid busting caches during x.py check -- if we've already built
+/// GCC, it's fine for us to not try to avoid doing so.
+pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> GccBuildStatus {
+    // If we have gcc submodule initialized already, sync it.
+    builder.update_existing_submodule(&Path::new("src").join("gcc"));
+
+    // FIXME (GuillaumeGomez): To be done once gccjit has been built in the CI.
+    // builder.config.maybe_download_ci_gcc();
+
+    // Initialize the llvm submodule if not initialized already.
+    builder.update_submodule(&Path::new("src").join("gcc"));
+
+    let root = "src/gcc";
+    let out_dir = builder.gcc_out(target).join("build");
+    let install_dir = builder.gcc_out(target).join("install");
+
+    static STAMP_HASH_MEMO: OnceLock<String> = OnceLock::new();
+    let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| {
+        generate_smart_stamp_hash(
+            &builder.config.src.join("src/llvm-project"),
+            builder.in_tree_llvm_info.sha().unwrap_or_default(),
+        )
+    });
+
+    let stamp = out_dir.join("gcc-finished-building");
+    let stamp = HashStamp::new(stamp, Some(smart_stamp_hash));
+
+    if stamp.is_done() {
+        if stamp.hash.is_none() {
+            builder.info(
+                "Could not determine the GCC submodule commit hash. \
+                     Assuming that an GCC rebuild is not necessary.",
+            );
+            builder.info(&format!(
+                "To force GCC to rebuild, remove the file `{}`",
+                stamp.path.display()
+            ));
+        }
+        return GccBuildStatus::AlreadyBuilt;
+    }
+
+    GccBuildStatus::ShouldBuild(Meta { stamp, out_dir, install_dir, root: root.into() })
+}
+
+// FIXME (GuillaumeGomez): When gcc-ci-download option is added, uncomment this code.
+// /// This retrieves the GCC sha we *want* to use, according to git history.
+// pub(crate) fn detect_gcc_sha(config: &Config, is_git: bool) -> String {
+//     let gcc_sha = if is_git {
+//         // We proceed in 2 steps. First we get the closest commit that is actually upstream. Then we
+//         // walk back further to the last bors merge commit that actually changed GCC. The first
+//         // step will fail on CI because only the `auto` branch exists; we just fall back to `HEAD`
+//         // in that case.
+//         let closest_upstream = get_git_merge_base(&config.git_config(), Some(&config.src))
+//             .unwrap_or_else(|_| "HEAD".into());
+//         let mut rev_list = config.git();
+//         rev_list.args(&[
+//             PathBuf::from("rev-list"),
+//             format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(),
+//             "-n1".into(),
+//             "--first-parent".into(),
+//             closest_upstream.into(),
+//             "--".into(),
+//             config.src.join("src/gcc"),
+//             config.src.join("src/bootstrap/download-ci-gcc-stamp"),
+//             // the GCC shared object file is named `gcc-12-rust-{version}-nightly`
+//             config.src.join("src/version"),
+//         ]);
+//         output(&mut rev_list).trim().to_owned()
+//     } else if let Some(info) = channel::read_commit_info_file(&config.src) {
+//         info.sha.trim().to_owned()
+//     } else {
+//         "".to_owned()
+//     };
+
+//     if gcc_sha.is_empty() {
+//         eprintln!("error: could not find commit hash for downloading LLVM");
+//         eprintln!("HELP: maybe your repository history is too shallow?");
+//         eprintln!("HELP: consider disabling `download-ci-gcc`");
+//         eprintln!("HELP: or fetch enough history to include one upstream commit");
+//         panic!();
+//     }
+
+//     gcc_sha
+// }
+
+// /// Returns whether the CI-found GCC is currently usable.
+// ///
+// /// This checks both the build triple platform to confirm we're usable at all,
+// /// and then verifies if the current HEAD matches the detected GCC SHA head,
+// /// in which case GCC is indicated as not available.
+// pub(crate) fn is_ci_gcc_available(config: &Config, asserts: bool) -> bool {
+//     // This is currently all tier 1 targets and tier 2 targets with host tools
+//     // (since others may not have CI artifacts)
+//     // https://doc.rust-lang.org/rustc/platform-support.html#tier-1
+//     let supported_platforms = [
+//         // tier 1
+//         ("x86_64-unknown-linux-gnu", true),
+//     ];
+
+//     if !supported_platforms.contains(&(&*config.build.triple, asserts))
+//         && (asserts || !supported_platforms.contains(&(&*config.build.triple, true)))
+//     {
+//         return false;
+//     }
+
+//     if is_ci_gcc_modified(config) {
+//         eprintln!("Detected GCC as non-available: running in CI and modified GCC in this change");
+//         return false;
+//     }
+
+//     true
+// }
+
+// /// Returns true if we're running in CI with modified GCC (and thus can't download it)
+// pub(crate) fn is_ci_gcc_modified(config: &Config) -> bool {
+//     CiEnv::is_ci() && config.rust_info.is_managed_git_subrepository() && {
+//         // We assume we have access to git, so it's okay to unconditionally pass
+//         // `true` here.
+//         let gcc_sha = detect_gcc_sha(config, true);
+//         let head_sha = output(config.git().arg("rev-parse").arg("HEAD"));
+//         let head_sha = head_sha.trim();
+//         gcc_sha == head_sha
+//     }
+// }
+
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub struct Gcc {
+    pub target: TargetSelection,
+}
+
+impl Step for Gcc {
+    type Output = bool;
+
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/gcc")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Gcc { target: run.target });
+    }
+
+    /// Compile GCC for `target`.
+    fn run(self, builder: &Builder<'_>) -> bool {
+        let target = self.target;
+        if !target.contains("linux") || !target.contains("x86_64") {
+            return false;
+        }
+
+        // If GCC has already been built or been downloaded through download-ci-gcc, we avoid
+        // building it again.
+        let Meta { stamp, out_dir, install_dir, root } = match prebuilt_gcc_config(builder, target)
+        {
+            GccBuildStatus::AlreadyBuilt => return true,
+            GccBuildStatus::ShouldBuild(m) => m,
+        };
+
+        let _guard = builder.msg_unstaged(Kind::Build, "GCC", target);
+        t!(stamp.remove());
+        let _time = helpers::timeit(builder);
+        t!(fs::create_dir_all(&out_dir));
+
+        if builder.config.dry_run() {
+            return true;
+        }
+
+        builder.run(
+            Command::new(root.join("configure"))
+                .current_dir(&out_dir)
+                .arg("--enable-host-shared")
+                .arg("--enable-languages=jit")
+                .arg("--enable-checking=release")
+                .arg("--disable-bootstrap")
+                .arg("--disable-multilib")
+                .arg(format!("--prefix={}", install_dir.display())),
+        );
+        builder
+            .run(Command::new("make").current_dir(&out_dir).arg(format!("-j{}", builder.jobs())));
+        builder.run(Command::new("make").current_dir(&out_dir).arg("install"));
+
+        t!(builder.symlink_file(
+            install_dir.join("lib/libgccjit.so"),
+            install_dir.join("lib/libgccjit.so.0")
+        ));
+
+        t!(stamp.write());
+
+        true
+    }
+}

src/bootstrap/src/core/build_steps/llvm.rs

@@ -1144,17 +1144,17 @@ fn supported_sanitizers(
     }
 }
 
-struct HashStamp {
-    path: PathBuf,
-    hash: Option<Vec<u8>>,
+pub(super) struct HashStamp {
+    pub(super) path: PathBuf,
+    pub(super) hash: Option<Vec<u8>>,
 }
 
 impl HashStamp {
-    fn new(path: PathBuf, hash: Option<&str>) -> Self {
+    pub(super) fn new(path: PathBuf, hash: Option<&str>) -> Self {
         HashStamp { path, hash: hash.map(|s| s.as_bytes().to_owned()) }
     }
 
-    fn is_done(&self) -> bool {
+    pub(super) fn is_done(&self) -> bool {
         match fs::read(&self.path) {
             Ok(h) => self.hash.as_deref().unwrap_or(b"") == h.as_slice(),
             Err(e) if e.kind() == io::ErrorKind::NotFound => false,
@@ -1164,7 +1164,7 @@ impl HashStamp {
         }
     }
 
-    fn remove(&self) -> io::Result<()> {
+    pub(super) fn remove(&self) -> io::Result<()> {
         match fs::remove_file(&self.path) {
             Ok(()) => Ok(()),
             Err(e) => {
@@ -1177,7 +1177,7 @@ impl HashStamp {
         }
     }
 
-    fn write(&self) -> io::Result<()> {
+    pub(super) fn write(&self) -> io::Result<()> {
         fs::write(&self.path, self.hash.as_deref().unwrap_or(b""))
     }
 }

src/bootstrap/src/core/build_steps/mod.rs

@@ -5,6 +5,7 @@ pub(crate) mod compile;
 pub(crate) mod dist;
 pub(crate) mod doc;
 pub(crate) mod format;
+pub(crate) mod gcc;
 pub(crate) mod install;
 pub(crate) mod llvm;
 pub(crate) mod run;

src/bootstrap/src/lib.rs

@@ -835,6 +835,10 @@ impl Build {
         }
     }
 
+    fn gcc_out(&self, target: TargetSelection) -> PathBuf {
+        self.out.join(&*target.triple).join("gcc")
+    }
+
     fn lld_out(&self, target: TargetSelection) -> PathBuf {
         self.out.join(&*target.triple).join("lld")
     }

src/tools/tidy/config/black.toml

@@ -12,5 +12,6 @@ extend-exclude = """(\
     src/llvm-project/|\
     src/doc/embedded-book/|\
     src/tools/rustc-perf/|\
-    library/backtrace/
+    library/backtrace/|\
+    src/gcc/
     )"""

src/tools/tidy/config/ruff.toml

@@ -27,6 +27,7 @@ extend-exclude = [
     "src/doc/embedded-book/",
     "library/backtrace/",
     "src/tools/rustc-perf/",
+    "src/gcc/",
     # Hack: CI runs from a subdirectory under the main checkout
     "../src/doc/nomicon/",
     "../src/tools/cargo/",
@@ -40,4 +41,5 @@ extend-exclude = [
     "../src/doc/embedded-book/",
     "../library/backtrace/",
     "../src/tools/rustc-perf/",
+    "../src/gcc/",
 ]

src/tools/tidy/src/walk.rs

@@ -28,6 +28,7 @@ pub fn filter_dirs(path: &Path) -> bool {
         "src/doc/rust-by-example",
         "src/doc/rustc-dev-guide",
         "src/doc/reference",
+        "src/gcc",
         // Filter RLS output directories
         "target/rls",
         "src/bootstrap/target",