Auto merge of rust-lang#98483 - dvtkrlbs:bootstrap-dist, r=jyn514

Distribute bootstrap in CI

This pre-compiles bootstrap from source and adds it to the existing `rust-dev` component. There are two main goals here:
1. Make it faster to build rust from source, both the first time and incrementally
2. Make it easier to add non-python entrypoints, since they can call out to bootstrap directly rather than having to figure out the right flags to pre-compile it. This second part is still in a bit of flux, see the tracking issue below for more information.

There are also several changes to make bootstrap able to run on a machine other than the one it was built (particularly around `config.src` and `config.out` detection). I (`@jyn514)` am slightly concerned these will regress unless tested - maybe we should add an automated test that runs bootstrap in a chroot or something? Unclear whether the effort is worth the test coverage.

Helps with rust-lang#94829.

.github/workflows/ci.yml

@@ -297,7 +297,7 @@ jobs:
             os: ubuntu-20.04-xl
           - name: dist-x86_64-apple
             env:
-              SCRIPT: "./x.py dist --host=x86_64-apple-darwin --target=x86_64-apple-darwin"
+              SCRIPT: "./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin"
               RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
@@ -308,7 +308,7 @@ jobs:
             os: macos-latest
           - name: dist-apple-various
             env:
-              SCRIPT: "./x.py dist --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim"
+              SCRIPT: "./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim"
               RUST_CONFIGURE_ARGS: "--enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
@@ -318,7 +318,7 @@ jobs:
             os: macos-latest
           - name: dist-x86_64-apple-alt
             env:
-              SCRIPT: "./x.py dist"
+              SCRIPT: "./x.py dist bootstrap --include-default-paths"
               RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
@@ -350,7 +350,7 @@ jobs:
             os: macos-latest
           - name: dist-aarch64-apple
             env:
-              SCRIPT: "./x.py dist --stage 2"
+              SCRIPT: "./x.py dist bootstrap --include-default-paths --stage 2"
               RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --host=aarch64-apple-darwin --target=aarch64-apple-darwin --enable-full-tools --enable-sanitizers --enable-profiler --disable-docs --set rust.jemalloc --set llvm.ninja=false"
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               USE_XCODE_CLANG: 1
@@ -424,33 +424,33 @@ jobs:
           - name: dist-x86_64-msvc
             env:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler"
-              SCRIPT: PGO_HOST=x86_64-pc-windows-msvc src/ci/pgo.sh python x.py dist
+              SCRIPT: PGO_HOST=x86_64-pc-windows-msvc src/ci/pgo.sh python x.py dist bootstrap --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-latest-xl
           - name: dist-i686-msvc
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler"
-              SCRIPT: python x.py dist
+              SCRIPT: python x.py dist bootstrap --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-latest-xl
           - name: dist-aarch64-msvc
             env:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler"
-              SCRIPT: python x.py dist
+              SCRIPT: python x.py dist bootstrap --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
               WINDOWS_SDK_20348_HACK: 1
             os: windows-latest-xl
           - name: dist-i686-mingw
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain"
               NO_DOWNLOAD_CI_LLVM: 1
-              SCRIPT: python x.py dist
+              SCRIPT: python x.py dist bootstrap --include-default-paths
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-latest-xl
           - name: dist-x86_64-mingw
             env:
-              SCRIPT: python x.py dist
+              SCRIPT: python x.py dist bootstrap --include-default-paths
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain"
               NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
@@ -459,7 +459,7 @@ jobs:
           - name: dist-x86_64-msvc-alt
             env:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler"
-              SCRIPT: python x.py dist
+              SCRIPT: python x.py dist bootstrap --include-default-paths
             os: windows-latest-xl
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"

src/bootstrap/builder.rs

@@ -724,6 +724,7 @@ impl<'a> Builder<'a> {
                 dist::Miri,
                 dist::LlvmTools,
                 dist::RustDev,
+                dist::Bootstrap,
                 dist::Extended,
                 // It seems that PlainSourceTarball somehow changes how some of the tools
                 // perceive their dependencies (see #93033) which would invalidate fingerprints

src/bootstrap/config.rs

@@ -772,21 +772,20 @@ impl Config {
 
         // set by build.rs
         config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE"));
+
         let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
         // Undo `src/bootstrap`
         config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned();
         config.out = PathBuf::from("build");
 
-        config.initial_cargo = PathBuf::from(env!("CARGO"));
-        config.initial_rustc = PathBuf::from(env!("RUSTC"));
-
         config
     }
 
     pub fn parse(args: &[String]) -> Config {
         let flags = Flags::parse(&args);
-
         let mut config = Config::default_opts();
+
+        // Set flags.
         config.exclude = flags.exclude.into_iter().map(|path| TaskPath::parse(path)).collect();
         config.include_default_paths = flags.include_default_paths;
         config.rustc_error_format = flags.rustc_error_format;
@@ -805,7 +804,49 @@ impl Config {
         config.llvm_profile_use = flags.llvm_profile_use;
         config.llvm_profile_generate = flags.llvm_profile_generate;
 
+        // Infer the rest of the configuration.
+
+        // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary,
+        // running on a completely machine from where it was compiled.
+        let mut cmd = Command::new("git");
+        // NOTE: we cannot support running from outside the repository because the only path we have available
+        // is set at compile time, which can be wrong if bootstrap was downloaded from source.
+        // We still support running outside the repository if we find we aren't in a git directory.
+        cmd.arg("rev-parse").arg("--show-toplevel");
+        // Discard stderr because we expect this to fail when building from a tarball.
+        let output = cmd
+            .stderr(std::process::Stdio::null())
+            .output()
+            .ok()
+            .and_then(|output| if output.status.success() { Some(output) } else { None });
+        if let Some(output) = output {
+            let git_root = String::from_utf8(output.stdout).unwrap();
+            // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes.
+            let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap();
+            let s = git_root.to_str().unwrap();
+
+            // Bootstrap is quite bad at handling /? in front of paths
+            config.src = match s.strip_prefix("\\\\?\\") {
+                Some(p) => PathBuf::from(p),
+                None => PathBuf::from(git_root),
+            };
+        } else {
+            // We're building from a tarball, not git sources.
+            // We don't support pre-downloaded bootstrap in this case.
+        }
+
+        if cfg!(test) {
+            // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly.
+            config.out = Path::new(
+                &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"),
+            )
+            .parent()
+            .unwrap()
+            .to_path_buf();
+        }
+
         let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json")));
+
         config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json));
 
         #[cfg(test)]
@@ -860,6 +901,7 @@ impl Config {
         config.config = toml_path;
 
         let build = toml.build.unwrap_or_default();
+        let has_custom_rustc = build.rustc.is_some();
 
         set(&mut config.initial_rustc, build.rustc.map(PathBuf::from));
         set(&mut config.out, flags.build_dir.or_else(|| build.build_dir.map(PathBuf::from)));
@@ -870,6 +912,12 @@ impl Config {
             config.out = crate::util::absolute(&config.out);
         }
 
+        if !has_custom_rustc && !config.initial_rustc.starts_with(&config.out) {
+            config.initial_rustc = config.out.join(config.build.triple).join("stage0/bin/rustc");
+            config.initial_cargo = config.out.join(config.build.triple).join("stage0/bin/cargo");
+        }
+
+        // NOTE: it's important this comes *after* we set `initial_rustc` just above.
         if config.dry_run {
             let dir = config.out.join("tmp-dry-run");
             t!(fs::create_dir_all(&dir));

src/bootstrap/dist.rs

@@ -1879,7 +1879,7 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir
         let mut cmd = Command::new(llvm_config);
         cmd.arg("--libfiles");
         builder.verbose(&format!("running {:?}", cmd));
-        let files = output(&mut cmd);
+        let files = if builder.config.dry_run { "".into() } else { output(&mut cmd) };
         let build_llvm_out = &builder.llvm_out(builder.config.build);
         let target_llvm_out = &builder.llvm_out(target);
         for file in files.trim_end().split(' ') {
@@ -2057,6 +2057,41 @@ impl Step for RustDev {
     }
 }
 
+// Tarball intended for internal consumption to ease rustc/std development.
+//
+// Should not be considered stable by end users.
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct Bootstrap {
+    pub target: TargetSelection,
+}
+
+impl Step for Bootstrap {
+    type Output = Option<GeneratedTarball>;
+    const DEFAULT: bool = false;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.alias("bootstrap")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Bootstrap { target: run.target });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+        let target = self.target;
+
+        let tarball = Tarball::new(builder, "bootstrap", &target.triple);
+
+        let bootstrap_outdir = &builder.bootstrap_out;
+        for file in &["bootstrap", "llvm-config-wrapper", "rustc", "rustdoc", "sccache-plus-cl"] {
+            tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755);
+        }
+
+        Some(tarball.generate())
+    }
+}
+
 /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the
 /// release process to avoid cloning the monorepo and building stuff.
 ///

src/bootstrap/lib.rs

@@ -458,19 +458,18 @@ impl Build {
             .expect("failed to read src/version");
         let version = version.trim();
 
-        let bootstrap_out = if std::env::var("BOOTSTRAP_PYTHON").is_ok() {
-            out.join("bootstrap").join("debug")
-        } else {
-            let workspace_target_dir = std::env::var("CARGO_TARGET_DIR")
-                .map(PathBuf::from)
-                .unwrap_or_else(|_| src.join("target"));
-            let bootstrap_out = workspace_target_dir.join("debug");
-            if !bootstrap_out.join("rustc").exists() && !cfg!(test) {
-                // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented
-                panic!("run `cargo build --bins` before `cargo run`")
-            }
-            bootstrap_out
-        };
+        let bootstrap_out = std::env::current_exe()
+            .expect("could not determine path to running process")
+            .parent()
+            .unwrap()
+            .to_path_buf();
+        if !bootstrap_out.join(exe("rustc", config.build)).exists() && !cfg!(test) {
+            // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented
+            panic!(
+                "`rustc` not found in {}, run `cargo build --bins` before `cargo run`",
+                bootstrap_out.display()
+            )
+        }
 
         let mut build = Build {
             initial_rustc: config.initial_rustc.clone(),

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -82,7 +82,7 @@ ENV RUST_CONFIGURE_ARGS \
 ENV SCRIPT ../src/ci/pgo.sh python3 ../x.py dist \
     --host $HOSTS --target $HOSTS \
     --include-default-paths \
-    build-manifest
+    build-manifest bootstrap
 ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang
 
 # This is the only builder which will create source tarballs

src/ci/github-actions/ci.yml

@@ -462,7 +462,7 @@ jobs:
 
           - name: dist-x86_64-apple
             env:
-              SCRIPT: ./x.py dist --host=x86_64-apple-darwin --target=x86_64-apple-darwin
+              SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin
               RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
@@ -474,7 +474,7 @@ jobs:
 
           - name: dist-apple-various
             env:
-              SCRIPT: ./x.py dist --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim
+              SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim
               RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
@@ -485,7 +485,7 @@ jobs:
 
           - name: dist-x86_64-apple-alt
             env:
-              SCRIPT: ./x.py dist
+              SCRIPT: ./x.py dist bootstrap --include-default-paths
               RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false
               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
               MACOSX_DEPLOYMENT_TARGET: 10.7
@@ -515,7 +515,7 @@ jobs:
           # This target only needs to support 11.0 and up as nothing else supports the hardware
           - name: dist-aarch64-apple
             env:
-              SCRIPT: ./x.py dist --stage 2
+              SCRIPT: ./x.py dist bootstrap --include-default-paths --stage 2
               RUST_CONFIGURE_ARGS: >-
                 --build=x86_64-apple-darwin
                 --host=aarch64-apple-darwin
@@ -659,7 +659,7 @@ jobs:
                 --target=x86_64-pc-windows-msvc
                 --enable-full-tools
                 --enable-profiler
-              SCRIPT: PGO_HOST=x86_64-pc-windows-msvc src/ci/pgo.sh python x.py dist
+              SCRIPT: PGO_HOST=x86_64-pc-windows-msvc src/ci/pgo.sh python x.py dist bootstrap --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
             <<: *job-windows-xl
 
@@ -671,7 +671,7 @@ jobs:
                 --target=i686-pc-windows-msvc,i586-pc-windows-msvc
                 --enable-full-tools
                 --enable-profiler
-              SCRIPT: python x.py dist
+              SCRIPT: python x.py dist bootstrap --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
             <<: *job-windows-xl
 
@@ -682,7 +682,7 @@ jobs:
                 --host=aarch64-pc-windows-msvc
                 --enable-full-tools
                 --enable-profiler
-              SCRIPT: python x.py dist
+              SCRIPT: python x.py dist bootstrap --include-default-paths
               DIST_REQUIRE_ALL_TOOLS: 1
               # Hack around this SDK version, because it doesn't work with clang.
               # See https://github.com/rust-lang/rust/issues/88796
@@ -699,14 +699,14 @@ jobs:
               # We are intentionally allowing an old toolchain on this builder (and that's
               # incompatible with LLVM downloads today).
               NO_DOWNLOAD_CI_LLVM: 1
-              SCRIPT: python x.py dist
+              SCRIPT: python x.py dist bootstrap --include-default-paths
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
             <<: *job-windows-xl
 
           - name: dist-x86_64-mingw
             env:
-              SCRIPT: python x.py dist
+              SCRIPT: python x.py dist bootstrap --include-default-paths
               RUST_CONFIGURE_ARGS: >-
                 --build=x86_64-pc-windows-gnu
                 --enable-full-tools
@@ -722,7 +722,7 @@ jobs:
           - name: dist-x86_64-msvc-alt
             env:
               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
-              SCRIPT: python x.py dist
+              SCRIPT: python x.py dist bootstrap --include-default-paths
             <<: *job-windows-xl
 
   try: