Auto merge of #52036 - collin5:b50509-2, r=<try>

Clean up dependency tracking in Rustbuild [2/2]

Make `clear_if_dirty` calls in `Builder::cargo` with stamp dependencies for the given Mode.

Continuation of #50904
Ref issue #50509
r? @Mark-Simulacrum

src/bootstrap/builder.rs

@@ -708,6 +708,71 @@ impl<'a> Builder<'a> {
     ) -> Command {
         let mut cargo = Command::new(&self.initial_cargo);
         let out_dir = self.stage_out(compiler, mode);
+
+        // command specific path, we call clear_if_dirty with this
+        let mut my_out = match cmd {
+            "build" => self.cargo_out(compiler, mode, target),
+
+            // This is the intended out directory for crate documentation.
+            "doc" =>  self.crate_doc_out(target),
+
+            _ => self.stage_out(compiler, mode),
+        };
+
+        let libstd_stamp = match cmd {
+            "check" => check::libstd_stamp(self, compiler, target),
+            _ => compile::libstd_stamp(self, compiler, target),
+        };
+
+        let libtest_stamp = match cmd {
+            "check" => check::libtest_stamp(self, compiler, target),
+            _ => compile::libstd_stamp(self, compiler, target),
+        };
+
+        let librustc_stamp = match cmd {
+            "check" => check::librustc_stamp(self, compiler, target),
+            _ => compile::librustc_stamp(self, compiler, target),
+        };
+
+        if cmd == "doc" {
+            if mode == Mode::Rustc || mode == Mode::ToolRustc || mode == Mode::Codegen {
+                // This is the intended out directory for compiler documentation.
+                my_out = self.compiler_doc_out(target);
+            }
+            let rustdoc = self.rustdoc(compiler.host);
+            self.clear_if_dirty(&my_out, &rustdoc);
+        } else {
+            match mode {
+                Mode::Std => {
+                    self.clear_if_dirty(&my_out, &self.rustc(compiler));
+                },
+                Mode::Test => {
+                    self.clear_if_dirty(&my_out, &libstd_stamp);
+                },
+                Mode::Rustc => {
+                    self.clear_if_dirty(&my_out, &self.rustc(compiler));
+                    self.clear_if_dirty(&my_out, &libstd_stamp);
+                    self.clear_if_dirty(&my_out, &libtest_stamp);
+                },
+                Mode::Codegen => {
+                    self.clear_if_dirty(&my_out, &librustc_stamp);
+                },
+                Mode::ToolBootstrap => { },
+                Mode::ToolStd => {
+                    self.clear_if_dirty(&my_out, &libstd_stamp);
+                },
+                Mode::ToolTest => {
+                    self.clear_if_dirty(&my_out, &libstd_stamp);
+                    self.clear_if_dirty(&my_out, &libtest_stamp);
+                },
+                Mode::ToolRustc => {
+                    self.clear_if_dirty(&my_out, &libstd_stamp);
+                    self.clear_if_dirty(&my_out, &libtest_stamp);
+                    self.clear_if_dirty(&my_out, &librustc_stamp);
+                },
+            }
+        }
+
         cargo
             .env("CARGO_TARGET_DIR", out_dir)
             .arg(cmd);

src/bootstrap/check.rs

@@ -12,7 +12,7 @@
 
 use compile::{run_cargo, std_cargo, test_cargo, rustc_cargo, rustc_cargo_env, add_to_sysroot};
 use builder::{RunConfig, Builder, ShouldRun, Step};
-use tool::{self, prepare_tool_cargo, SourceType};
+use tool::{prepare_tool_cargo, SourceType};
 use {Compiler, Mode};
 use cache::{INTERNER, Interned};
 use std::path::PathBuf;
@@ -40,9 +40,6 @@ impl Step for Std {
         let target = self.target;
         let compiler = builder.compiler(0, builder.config.build);
 
-        let out_dir = builder.stage_out(compiler, Mode::Std);
-        builder.clear_if_dirty(&out_dir, &builder.rustc(compiler));
-
         let mut cargo = builder.cargo(compiler, Mode::Std, target, "check");
         std_cargo(builder, &compiler, target, &mut cargo);
 
@@ -88,10 +85,6 @@ impl Step for Rustc {
         let compiler = builder.compiler(0, builder.config.build);
         let target = self.target;
 
-        let stage_out = builder.stage_out(compiler, Mode::Rustc);
-        builder.clear_if_dirty(&stage_out, &libstd_stamp(builder, compiler, target));
-        builder.clear_if_dirty(&stage_out, &libtest_stamp(builder, compiler, target));
-
         let mut cargo = builder.cargo(compiler, Mode::Rustc, target, "check");
         rustc_cargo(builder, &mut cargo);
 
@@ -139,9 +132,6 @@ impl Step for CodegenBackend {
         let target = self.target;
         let backend = self.backend;
 
-        let out_dir = builder.cargo_out(compiler, Mode::Codegen, target);
-        builder.clear_if_dirty(&out_dir, &librustc_stamp(builder, compiler, target));
-
         let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "check");
         cargo.arg("--manifest-path").arg(builder.src.join("src/librustc_codegen_llvm/Cargo.toml"));
         rustc_cargo_env(builder, &mut cargo);
@@ -180,9 +170,6 @@ impl Step for Test {
         let compiler = builder.compiler(0, builder.config.build);
         let target = self.target;
 
-        let out_dir = builder.stage_out(compiler, Mode::Test);
-        builder.clear_if_dirty(&out_dir, &libstd_stamp(builder, compiler, target));
-
         let mut cargo = builder.cargo(compiler, Mode::Test, target, "check");
         test_cargo(builder, &compiler, target, &mut cargo);
 
@@ -220,14 +207,9 @@ impl Step for Rustdoc {
     }
 
     fn run(self, builder: &Builder) {
-        let compiler = builder.compiler(0, builder.config.build);
+        let mut compiler = builder.compiler(0, builder.config.build);
         let target = self.target;
 
-        let stage_out = builder.stage_out(compiler, Mode::ToolRustc);
-        builder.clear_if_dirty(&stage_out, &libstd_stamp(builder, compiler, target));
-        builder.clear_if_dirty(&stage_out, &libtest_stamp(builder, compiler, target));
-        builder.clear_if_dirty(&stage_out, &librustc_stamp(builder, compiler, target));
-
         let mut cargo = prepare_tool_cargo(builder,
                                            compiler,
                                            Mode::ToolRustc,
@@ -247,11 +229,14 @@ impl Step for Rustdoc {
         let libdir = builder.sysroot_libdir(compiler, target);
         add_to_sysroot(&builder, &libdir, &rustdoc_stamp(builder, compiler, target));
 
-        builder.ensure(tool::CleanTools {
-            compiler,
-            target,
-            cause: Mode::Rustc,
-        });
+        // This is for the original compiler, but if we're forced to use stage 1, then
+        // std/test/rustc stamps won't exist in stage 2, so we need to get those from stage 1, since
+        // we copy the libs forward.
+        if builder.force_use_stage1(compiler, target) {
+            compiler = builder.compiler(1, compiler.host)
+        };
+
+        builder.cargo(compiler, Mode::ToolRustc, target, "clean");
     }
 }
 

src/bootstrap/compile.rs

@@ -32,7 +32,6 @@ use serde_json;
 use util::{exe, libdir, is_dylib, CiEnv};
 use {Compiler, Mode};
 use native;
-use tool;
 
 use cache::{INTERNER, Interned};
 use builder::{Step, RunConfig, ShouldRun, Builder};
@@ -107,8 +106,6 @@ impl Step for Std {
             copy_musl_third_party_objects(builder, target, &libdir);
         }
 
-        let out_dir = builder.cargo_out(compiler, Mode::Std, target);
-        builder.clear_if_dirty(&out_dir, &builder.rustc(compiler));
         let mut cargo = builder.cargo(compiler, Mode::Std, target, "build");
         std_cargo(builder, &compiler, target, &mut cargo);
 
@@ -228,7 +225,7 @@ impl Step for StdLink {
     /// output directory.
     fn run(self, builder: &Builder) {
         let compiler = self.compiler;
-        let target_compiler = self.target_compiler;
+        let mut target_compiler = self.target_compiler;
         let target = self.target;
         builder.info(&format!("Copying stage{} std from stage{} ({} -> {} / {})",
                 target_compiler.stage,
@@ -246,11 +243,14 @@ impl Step for StdLink {
             copy_apple_sanitizer_dylibs(builder, &builder.native_dir(target), "osx", &libdir);
         }
 
-        builder.ensure(tool::CleanTools {
-            compiler: target_compiler,
-            target,
-            cause: Mode::Std,
-        });
+        // This is for the original compiler, but if we're forced to use stage 1, then
+        // std/test/rustc stamps won't exist in stage 2, so we need to get those from stage 1, since
+        // we copy the libs forward.
+        if builder.force_use_stage1(target_compiler, target) {
+            target_compiler = builder.compiler(1, target_compiler.host)
+        };
+
+        builder.cargo(target_compiler, Mode::ToolStd, target, "clean");
     }
 }
 
@@ -387,8 +387,6 @@ impl Step for Test {
             return;
         }
 
-        let out_dir = builder.cargo_out(compiler, Mode::Test, target);
-        builder.clear_if_dirty(&out_dir, &libstd_stamp(builder, compiler, target));
         let mut cargo = builder.cargo(compiler, Mode::Test, target, "build");
         test_cargo(builder, &compiler, target, &mut cargo);
 
@@ -438,7 +436,7 @@ impl Step for TestLink {
     /// Same as `std_link`, only for libtest
     fn run(self, builder: &Builder) {
         let compiler = self.compiler;
-        let target_compiler = self.target_compiler;
+        let mut target_compiler = self.target_compiler;
         let target = self.target;
         builder.info(&format!("Copying stage{} test from stage{} ({} -> {} / {})",
                 target_compiler.stage,
@@ -448,11 +446,15 @@ impl Step for TestLink {
                 target));
         add_to_sysroot(builder, &builder.sysroot_libdir(target_compiler, target),
                     &libtest_stamp(builder, compiler, target));
-        builder.ensure(tool::CleanTools {
-            compiler: target_compiler,
-            target,
-            cause: Mode::Test,
-        });
+
+        // This is for the original compiler, but if we're forced to use stage 1, then
+        // std/test/rustc stamps won't exist in stage 2, so we need to get those from stage 1, since
+        // we copy the libs forward.
+        if builder.force_use_stage1(target_compiler, target) {
+            target_compiler = builder.compiler(1, target_compiler.host)
+        };
+
+        builder.cargo(target_compiler, Mode::ToolTest, target, "clean");
     }
 }
 
@@ -519,9 +521,6 @@ impl Step for Rustc {
             compiler: builder.compiler(self.compiler.stage, builder.config.build),
             target: builder.config.build,
         });
-        let cargo_out = builder.cargo_out(compiler, Mode::Rustc, target);
-        builder.clear_if_dirty(&cargo_out, &libstd_stamp(builder, compiler, target));
-        builder.clear_if_dirty(&cargo_out, &libtest_stamp(builder, compiler, target));
 
         let mut cargo = builder.cargo(compiler, Mode::Rustc, target, "build");
         rustc_cargo(builder, &mut cargo);
@@ -603,7 +602,7 @@ impl Step for RustcLink {
     /// Same as `std_link`, only for librustc
     fn run(self, builder: &Builder) {
         let compiler = self.compiler;
-        let target_compiler = self.target_compiler;
+        let mut target_compiler = self.target_compiler;
         let target = self.target;
         builder.info(&format!("Copying stage{} rustc from stage{} ({} -> {} / {})",
                  target_compiler.stage,
@@ -613,11 +612,15 @@ impl Step for RustcLink {
                  target));
         add_to_sysroot(builder, &builder.sysroot_libdir(target_compiler, target),
                        &librustc_stamp(builder, compiler, target));
-        builder.ensure(tool::CleanTools {
-            compiler: target_compiler,
-            target,
-            cause: Mode::Rustc,
-        });
+
+        // This is for the original compiler, but if we're forced to use stage 1, then
+        // std/test/rustc stamps won't exist in stage 2, so we need to get those from stage 1, since
+        // we copy the libs forward.
+        if builder.force_use_stage1(target_compiler, target) {
+            target_compiler = builder.compiler(1, target_compiler.host)
+        };
+
+        builder.cargo(target_compiler, Mode::ToolRustc, target, "clean");
     }
 }
 
@@ -674,7 +677,6 @@ impl Step for CodegenBackend {
         }
 
         let out_dir = builder.cargo_out(compiler, Mode::Codegen, target);
-        builder.clear_if_dirty(&out_dir, &librustc_stamp(builder, compiler, target));
 
         let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "rustc");
         cargo.arg("--manifest-path")

src/bootstrap/doc.rs

@@ -455,7 +455,6 @@ impl Step for Std {
         let out = builder.doc_out(target);
         t!(fs::create_dir_all(&out));
         let compiler = builder.compiler(stage, builder.config.build);
-        let rustdoc = builder.rustdoc(compiler.host);
         let compiler = if builder.force_use_stage1(compiler, target) {
             builder.compiler(1, compiler.host)
         } else {
@@ -480,7 +479,6 @@ impl Step for Std {
         // This way rustdoc generates output directly into the output, and rustdoc
         // will also directly handle merging.
         let my_out = builder.crate_doc_out(target);
-        builder.clear_if_dirty(&my_out, &rustdoc);
         t!(symlink_dir_force(&builder.config, &my_out, &out_dir));
 
         let mut cargo = builder.cargo(compiler, Mode::Std, target, "doc");
@@ -535,7 +533,6 @@ impl Step for Test {
         let out = builder.doc_out(target);
         t!(fs::create_dir_all(&out));
         let compiler = builder.compiler(stage, builder.config.build);
-        let rustdoc = builder.rustdoc(compiler.host);
         let compiler = if builder.force_use_stage1(compiler, target) {
             builder.compiler(1, compiler.host)
         } else {
@@ -551,7 +548,6 @@ impl Step for Test {
 
         // See docs in std above for why we symlink
         let my_out = builder.crate_doc_out(target);
-        builder.clear_if_dirty(&my_out, &rustdoc);
         t!(symlink_dir_force(&builder.config, &my_out, &out_dir));
 
         let mut cargo = builder.cargo(compiler, Mode::Test, target, "doc");
@@ -603,7 +599,6 @@ impl Step for WhitelistedRustc {
         let out = builder.doc_out(target);
         t!(fs::create_dir_all(&out));
         let compiler = builder.compiler(stage, builder.config.build);
-        let rustdoc = builder.rustdoc(compiler.host);
         let compiler = if builder.force_use_stage1(compiler, target) {
             builder.compiler(1, compiler.host)
         } else {
@@ -619,7 +614,6 @@ impl Step for WhitelistedRustc {
 
         // See docs in std above for why we symlink
         let my_out = builder.crate_doc_out(target);
-        builder.clear_if_dirty(&my_out, &rustdoc);
         t!(symlink_dir_force(&builder.config, &my_out, &out_dir));
 
         let mut cargo = builder.cargo(compiler, Mode::Rustc, target, "doc");
@@ -678,7 +672,6 @@ impl Step for Rustc {
 
         // Get the correct compiler for this stage.
         let compiler = builder.compiler(stage, builder.config.build);
-        let rustdoc = builder.rustdoc(compiler.host);
         let compiler = if builder.force_use_stage1(compiler, target) {
             builder.compiler(1, compiler.host)
         } else {
@@ -699,7 +692,6 @@ impl Step for Rustc {
         // We do not symlink to the same shared folder that already contains std library
         // documentation from previous steps as we do not want to include that.
         let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target).join("doc");
-        builder.clear_if_dirty(&out, &rustdoc);
         t!(symlink_dir_force(&builder.config, &out, &out_dir));
 
         // Build cargo command.
@@ -780,7 +772,6 @@ impl Step for Rustdoc {
 
         // Get the correct compiler for this stage.
         let compiler = builder.compiler(stage, builder.config.build);
-        let rustdoc = builder.rustdoc(compiler.host);
         let compiler = if builder.force_use_stage1(compiler, target) {
             builder.compiler(1, compiler.host)
         } else {
@@ -803,7 +794,6 @@ impl Step for Rustdoc {
             .join(target)
             .join("doc");
         t!(fs::create_dir_all(&out_dir));
-        builder.clear_if_dirty(&out, &rustdoc);
         t!(symlink_dir_force(&builder.config, &out, &out_dir));
 
         // Build cargo command.

src/bootstrap/lib.rs

@@ -347,6 +347,7 @@ pub enum Mode {
     /// Compile a tool which uses all libraries we compile (up to rustc).
     /// Doesn't use the stage0 compiler libraries like "other", and includes
     /// tools like rustdoc, cargo, rls, etc.
+    ToolTest,
     ToolStd,
     ToolRustc,
 }
@@ -567,6 +568,7 @@ impl Build {
             Mode::Codegen => "-codegen",
             Mode::ToolBootstrap => "-bootstrap-tools",
             Mode::ToolStd => "-tools",
+            Mode::ToolTest => "-tools",
             Mode::ToolRustc => "-tools",
         };
         self.out.join(&*compiler.host)