Auto merge of rust-lang#119899 - onur-ozkan:redesign-stage0-std, r=<try>

redesign stage 0 std

This is intended to update bootstrap to use the beta standard library on stage 0, rather than compiling it from source (see the motivation at rust-lang/compiler-team#619).

The only drawback encountered was the requirement of using the stage 1 compiler to build the standard library from source. This issue has been resolved by adding the `--build-std-on-stage0` flag. Therefore if the goal is to only build/test the compiled standard library without spending time on compiling rustc, it can be achieved by running `x build --stage 0 std --build-std-on-stage0`.

src/bootstrap/src/core/build_steps/compile.rs

@@ -137,6 +137,16 @@ impl Step for Std {
         let target = self.target;
         let compiler = self.compiler;
 
+        // We already have std ready to be used for stage 0.
+        if compiler.stage == 0 && !builder.config.build_std_on_stage0 {
+            builder.ensure(StdLink::from_std(
+                self,
+                builder.compiler(compiler.stage, builder.config.build),
+            ));
+
+            return;
+        }
+
         // When using `download-rustc`, we already have artifacts for the host available. Don't
         // recompile them.
         if builder.download_rustc() && target == builder.build.build
@@ -175,8 +185,20 @@ impl Step for Std {
 
         let mut target_deps = builder.ensure(StartupObjects { compiler, target });
 
-        let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
-        if compiler_to_use != compiler {
+        let mut compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
+
+        // We only need to build std twice; therefore,
+        // - If stage 2 std is built, the artifacts are fully up to date. For the next stages, uplift the artifacts instead of compiling std again.
+        // - If `--build-std-on-stage0` was used it means std was already built on stage 0. So, if stage 1 is built, that means std is fully up to date.
+        //      For the next stages, uplift the artifacts instead of compiling std again.
+        if (compiler.stage > 2 || builder.config.build_std_on_stage0) && compiler_to_use != compiler
+        {
+            // If stage 2 std was built (not uplifted), uplift the artifacts from stage 2.
+            // (This is necessary; without this, they will be uplifted from stage 1).
+            if !builder.config.build_std_on_stage0 && compiler.stage > 2 {
+                compiler_to_use.stage = 2;
+            }
+
             builder.ensure(Std::new(compiler_to_use, target));
             let msg = if compiler_to_use.host == target {
                 format!(
@@ -585,18 +607,16 @@ impl Step for StdLink {
             (libdir, hostdir)
         };
 
-        add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target));
+        let is_downloaded_beta_stage0 = builder
+            .build
+            .config
+            .initial_rustc
+            .starts_with(builder.out.join(&compiler.host.triple).join("stage0/bin"));
 
         // Special case for stage0, to make `rustup toolchain link` and `x dist --stage 0`
         // work for stage0-sysroot. We only do this if the stage0 compiler comes from beta,
         // and is not set to a custom path.
-        if compiler.stage == 0
-            && builder
-                .build
-                .config
-                .initial_rustc
-                .starts_with(builder.out.join(&compiler.host.triple).join("stage0/bin"))
-        {
+        if compiler.stage == 0 && is_downloaded_beta_stage0 && !builder.config.build_std_on_stage0 {
             // Copy bin files from stage0/bin to stage0-sysroot/bin
             let sysroot = builder.out.join(&compiler.host.triple).join("stage0-sysroot");
 
@@ -608,7 +628,10 @@ impl Step for StdLink {
 
             // Copy all *.so files from stage0/lib to stage0-sysroot/lib
             let stage0_lib_dir = builder.out.join(&host).join("stage0/lib");
-            if let Ok(files) = fs::read_dir(&stage0_lib_dir) {
+
+            if !builder.config.build_std_on_stage0 {
+                builder.cp_r(&stage0_lib_dir, &sysroot.join("lib"));
+            } else if let Ok(files) = fs::read_dir(&stage0_lib_dir) {
                 for file in files {
                     let file = t!(file);
                     let path = file.path();
@@ -630,6 +653,11 @@ impl Step for StdLink {
             if stage0_codegen_backends.exists() {
                 builder.cp_r(&stage0_codegen_backends, &sysroot_codegen_backends);
             }
+        } else if compiler.stage == 0 && !builder.config.build_std_on_stage0 {
+            let sysroot = builder.out.join(&compiler.host.triple).join("stage0-sysroot");
+            builder.cp_r(&builder.initial_sysroot.join("lib"), &sysroot.join("lib"));
+        } else {
+            add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target));
         }
     }
 }

src/bootstrap/src/core/build_steps/test.rs

@@ -1121,7 +1121,7 @@ impl Step for Tidy {
         if builder.config.channel == "dev" || builder.config.channel == "nightly" {
             builder.info("fmt check");
             if builder.initial_rustfmt().is_none() {
-                let inferred_rustfmt_dir = builder.initial_rustc.parent().unwrap();
+                let inferred_rustfmt_dir = builder.initial_sysroot.join("bin");
                 eprintln!(
                     "\
 ERROR: no `rustfmt` binary found in {PATH}

src/bootstrap/src/core/builder.rs

@@ -1167,7 +1167,7 @@ impl<'a> Builder<'a> {
     }
 
     pub fn cargo_clippy_cmd(&self, run_compiler: Compiler) -> Command {
-        let initial_sysroot_bin = self.initial_rustc.parent().unwrap();
+        let initial_sysroot_bin = self.initial_sysroot.join("bin");
         // Set PATH to include the sysroot bin dir so clippy can find cargo.
         // FIXME: once rust-clippy#11944 lands on beta, set `CARGO` directly instead.
         let path = t!(env::join_paths(

src/bootstrap/src/core/config/config.rs

@@ -186,6 +186,7 @@ pub struct Config {
 
     pub on_fail: Option<String>,
     pub stage: u32,
+    pub build_std_on_stage0: bool,
     pub keep_stage: Vec<u32>,
     pub keep_stage_std: Vec<u32>,
     pub src: PathBuf,
@@ -1219,6 +1220,7 @@ impl Config {
         config.incremental = flags.incremental;
         config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled };
         config.dump_bootstrap_shims = flags.dump_bootstrap_shims;
+        config.build_std_on_stage0 = flags.build_std_on_stage0;
         config.keep_stage = flags.keep_stage;
         config.keep_stage_std = flags.keep_stage_std;
         config.color = flags.color;

src/bootstrap/src/core/config/flags.rs

@@ -92,6 +92,9 @@ pub struct Flags {
     /// stage to build (indicates compiler to use/test, e.g., stage 0 uses the
     /// bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)
     pub stage: Option<u32>,
+    /// Useful for library testing with the beta compiler (without building the stage 1 compiler)
+    #[arg(global(true), long)]
+    pub build_std_on_stage0: bool,
 
     #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")]
     /// stage(s) to keep without recompiling

src/bootstrap/src/utils/change_tracker.rs

@@ -111,4 +111,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[
         severity: ChangeSeverity::Warning,
         summary: "A new `optimized-compiler-builtins` option has been introduced. Whether to build llvm's `compiler-rt` from source is no longer implicitly controlled by git state. See the PR for more details.",
     },
+    ChangeInfo {
+        change_id: 119899,
+        severity: ChangeSeverity::Info,
+        summary: "Stage 0 std is now same as beta std rather than the compiled version from source (which is now stage 1). If you want to test/build std on stage 0, use `--build-std-on-stage0` flag (e.g., `x test --stage0 --build-std-on-stage0 alloc`).",
+    },
 ];

src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile

@@ -27,5 +27,5 @@ RUN echo "[rust]" > /config/nopt-std-config.toml
 RUN echo "optimize = false" >> /config/nopt-std-config.toml
 
 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu --disable-optimize-tests
-ENV SCRIPT python3 ../x.py test --stage 0 --config /config/nopt-std-config.toml library/std \
+ENV SCRIPT python3 ../x.py test --build-std-on-stage0 --stage 0 --config /config/nopt-std-config.toml library/std \
   && python3 ../x.py --stage 2 test

src/ci/docker/host-x86_64/mingw-check/Dockerfile

@@ -49,7 +49,7 @@ ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \
            python3 ../x.py clippy --stage 0 -Awarnings && \
            python3 ../x.py build --stage 0 src/tools/build-manifest && \
            python3 ../x.py test --stage 0 src/tools/compiletest && \
-           python3 ../x.py test --stage 0 core alloc std test proc_macro && \
+           python3 ../x.py test --build-std-on-stage0 --stage 0 core alloc std test proc_macro && \
            # Build both public and internal documentation.
            RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 library && \
            mkdir -p /checkout/obj/staging/doc && \

src/ci/docker/host-x86_64/x86_64-gnu-nopt/Dockerfile

@@ -29,5 +29,5 @@ RUN echo "optimize = false" >> /config/nopt-std-config.toml
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu \
   --disable-optimize-tests \
   --set rust.test-compare-mode
-ENV SCRIPT python3 ../x.py test --stage 0 --config /config/nopt-std-config.toml library/std \
+ENV SCRIPT python3 ../x.py test --build-std-on-stage0 --stage 0 --config /config/nopt-std-config.toml library/std \
   && python3 ../x.py --stage 2 test