Auto merge of #76544 - Mark-Simulacrum:less-python, r=alexcrichton

De-couple Python and bootstrap slightly

This revises rustbuild's entry points from Python to rely less on magic environment variables, preferring to use Cargo-provided environment variables where feasible.

Notably, BUILD_DIR and BOOTSTRAP_CONFIG are *not* moved, because both more-or-less have some non-trivial discovery logic and replicating it in rustbuild seems unfortunate; if it moved to Cargo that would be a different story.

Best reviewed by-commit.

src/bootstrap/Cargo.toml

@@ -3,6 +3,7 @@ authors = ["The Rust Project Developers"]
 name = "bootstrap"
 version = "0.0.0"
 edition = "2018"
+build = "build.rs"
 
 [lib]
 path = "lib.rs"

src/bootstrap/bootstrap.py

@@ -966,7 +966,6 @@ def bootstrap(help_triggered):
     parser = argparse.ArgumentParser(description='Build rust')
     parser.add_argument('--config')
     parser.add_argument('--build')
-    parser.add_argument('--src')
     parser.add_argument('--clean', action='store_true')
     parser.add_argument('-v', '--verbose', action='count', default=0)
 
@@ -975,7 +974,7 @@ def bootstrap(help_triggered):
 
     # Configure initial bootstrap
     build = RustBuild()
-    build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
+    build.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
     build.verbose = args.verbose
     build.clean = args.clean
 
@@ -1032,18 +1031,12 @@ def bootstrap(help_triggered):
     args = [build.bootstrap_binary()]
     args.extend(sys.argv[1:])
     env = os.environ.copy()
-    env["BUILD"] = build.build
-    env["SRC"] = build.rust_root
     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
     env["BOOTSTRAP_PYTHON"] = sys.executable
     env["BUILD_DIR"] = build.build_dir
     env["RUSTC_BOOTSTRAP"] = '1'
-    env["CARGO"] = build.cargo()
-    env["RUSTC"] = build.rustc()
     if toml_path:
         env["BOOTSTRAP_CONFIG"] = toml_path
-    if build.rustfmt():
-        env["RUSTFMT"] = build.rustfmt()
     run(args, env=env, verbose=build.verbose)
 
 

src/bootstrap/build.rs

@@ -0,0 +1,26 @@
+use std::env;
+use std::path::PathBuf;
+
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rustc-env=BUILD_TRIPLE={}", env::var("HOST").unwrap());
+
+    // This may not be a canonicalized path.
+    let mut rustc = PathBuf::from(env::var_os("RUSTC").unwrap());
+
+    if rustc.is_relative() {
+        for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) {
+            let absolute = dir.join(&rustc);
+            if absolute.exists() {
+                rustc = absolute;
+                break;
+            }
+        }
+    }
+    assert!(rustc.is_absolute());
+
+    // FIXME: if the path is not utf-8, this is going to break. Unfortunately
+    // Cargo doesn't have a way for us to specify non-utf-8 paths easily, so
+    // we'll need to invent some encoding scheme if this becomes a problem.
+    println!("cargo:rustc-env=RUSTC={}", rustc.to_str().unwrap());
+}

src/bootstrap/builder/tests.rs

@@ -10,6 +10,9 @@ fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config {
     config.dry_run = true;
     config.ninja_in_file = false;
     // try to avoid spurious failures in dist where we create/delete each others file
+    config.out = PathBuf::from(env::var_os("BOOTSTRAP_OUTPUT_DIRECTORY").unwrap());
+    config.initial_rustc = PathBuf::from(env::var_os("RUSTC").unwrap());
+    config.initial_cargo = PathBuf::from(env::var_os("BOOTSTRAP_INITIAL_CARGO").unwrap());
     let dir = config
         .out
         .join("tmp-rustbuild-tests")

src/bootstrap/config.rs

@@ -291,7 +291,7 @@ struct Build {
     build_dir: Option<String>,
     cargo: Option<String>,
     rustc: Option<String>,
-    rustfmt: Option<String>, /* allow bootstrap.py to use rustfmt key */
+    rustfmt: Option<PathBuf>,
     docs: Option<bool>,
     compiler_docs: Option<bool>,
     submodules: Option<bool>,
@@ -487,13 +487,14 @@ impl Config {
         config.missing_tools = false;
 
         // set by bootstrap.py
-        config.build = TargetSelection::from_user(&env::var("BUILD").expect("'BUILD' to be set"));
-        config.src = Config::path_from_python("SRC");
+        config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE"));
+        let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        // Undo `src/bootstrap`
+        config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned();
         config.out = Config::path_from_python("BUILD_DIR");
 
-        config.initial_rustc = Config::path_from_python("RUSTC");
-        config.initial_cargo = Config::path_from_python("CARGO");
-        config.initial_rustfmt = env::var_os("RUSTFMT").map(Config::normalize_python_path);
+        config.initial_cargo = PathBuf::from(env!("CARGO"));
+        config.initial_rustc = PathBuf::from(env!("RUSTC"));
 
         config
     }
@@ -582,6 +583,9 @@ impl Config {
         set(&mut config.full_bootstrap, build.full_bootstrap);
         set(&mut config.extended, build.extended);
         config.tools = build.tools;
+        if build.rustfmt.is_some() {
+            config.initial_rustfmt = build.rustfmt;
+        }
         set(&mut config.verbose, build.verbose);
         set(&mut config.sanitizers, build.sanitizers);
         set(&mut config.profiler, build.profiler);
@@ -836,12 +840,15 @@ impl Config {
             set(&mut config.missing_tools, t.missing_tools);
         }
 
+        // Cargo does not provide a RUSTFMT environment variable, so we
+        // synthesize it manually. Note that we also later check the config.toml
+        // and set this to that path if necessary.
+        let rustfmt = config.initial_rustc.with_file_name(exe("rustfmt", config.build));
+        config.initial_rustfmt = if rustfmt.exists() { Some(rustfmt) } else { None };
+
         // Now that we've reached the end of our configuration, infer the
         // default values for all options that we haven't otherwise stored yet.
 
-        set(&mut config.initial_rustc, build.rustc.map(PathBuf::from));
-        set(&mut config.initial_cargo, build.cargo.map(PathBuf::from));
-
         config.llvm_skip_rebuild = llvm_skip_rebuild.unwrap_or(false);
 
         let default = false;

src/bootstrap/test.rs

@@ -1281,6 +1281,8 @@ impl Step for Compiletest {
             cmd.arg("--rustfix-coverage");
         }
 
+        cmd.env("BOOTSTRAP_CARGO", &builder.initial_cargo);
+
         builder.ci_env.force_coloring_in_ci(&mut cmd);
 
         builder.info(&format!(
@@ -2022,6 +2024,8 @@ impl Step for Bootstrap {
             .current_dir(builder.src.join("src/bootstrap"))
             .env("RUSTFLAGS", "-Cdebuginfo=2")
             .env("CARGO_TARGET_DIR", builder.out.join("bootstrap"))
+            .env("BOOTSTRAP_OUTPUT_DIRECTORY", &builder.config.out)
+            .env("BOOTSTRAP_INITIAL_CARGO", &builder.config.initial_cargo)
             .env("RUSTC_BOOTSTRAP", "1")
             .env("RUSTC", &builder.initial_rustc);
         if let Some(flags) = option_env!("RUSTFLAGS") {

src/bootstrap/tool.rs

@@ -700,6 +700,10 @@ impl<'a> Builder<'a> {
         }
 
         add_dylib_path(lib_paths, &mut cmd);
+
+        // Provide a RUSTC for this command to use.
+        cmd.env("RUSTC", &self.initial_rustc);
+
         cmd
     }
 }

src/test/run-make/thumb-none-cortex-m/Makefile

@@ -35,4 +35,4 @@ all:
 	# HACK(eddyb) sets `RUSTC_BOOTSTRAP=1` so Cargo can accept nightly features.
 	# These come from the top-level Rust workspace, that this crate is not a
 	# member of, but Cargo tries to load the workspace `Cargo.toml` anyway.
-	cd $(WORK_DIR) && cd $(CRATE) && env RUSTC_BOOTSTRAP=1 $(CARGO) build --target $(TARGET) -v
+	cd $(WORK_DIR) && cd $(CRATE) && env RUSTC_BOOTSTRAP=1 $(BOOTSTRAP_CARGO) build --target $(TARGET) -v

src/test/run-make/thumb-none-qemu/script.sh

@@ -12,8 +12,8 @@ pushd $WORK_DIR
         # These come from the top-level Rust workspace, that this crate is not a
         # member of, but Cargo tries to load the workspace `Cargo.toml` anyway.
         env RUSTC_BOOTSTRAP=1 RUSTFLAGS="-C linker=arm-none-eabi-ld -C link-arg=-Tlink.x" \
-            $CARGO run --target $TARGET           | grep "x = 42"
+            $BOOTSTRAP_CARGO run --target $TARGET           | grep "x = 42"
         env RUSTC_BOOTSTRAP=1 RUSTFLAGS="-C linker=arm-none-eabi-ld -C link-arg=-Tlink.x" \
-            $CARGO run --target $TARGET --release | grep "x = 42"
+            $BOOTSTRAP_CARGO run --target $TARGET --release | grep "x = 42"
     popd
 popd