From 6d195615872440798ac47ad7a4d8fd090fcbc24a Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sun, 6 Sep 2020 19:57:07 -0400 Subject: [PATCH 1/5] Set BUILD_TRIPLE via build script This moves build triple discovery for rustbuild from bootstrap.py into a build script, meaning it will "just work" if building rustbuild via Cargo rather than Python. --- src/bootstrap/Cargo.toml | 1 + src/bootstrap/bootstrap.py | 1 - src/bootstrap/build.rs | 3 +++ src/bootstrap/config.rs | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 src/bootstrap/build.rs diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index faec2c53742ec..890315a744e56 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -3,6 +3,7 @@ authors = ["The Rust Project Developers"] name = "bootstrap" version = "0.0.0" edition = "2018" +build = "build.rs" [lib] path = "lib.rs" diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 5f78031e1c7cb..c9a9a484ee338 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -1032,7 +1032,6 @@ def bootstrap(help_triggered): args = [build.bootstrap_binary()] args.extend(sys.argv[1:]) env = os.environ.copy() - env["BUILD"] = build.build env["SRC"] = build.rust_root env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) env["BOOTSTRAP_PYTHON"] = sys.executable diff --git a/src/bootstrap/build.rs b/src/bootstrap/build.rs new file mode 100644 index 0000000000000..5e5c31de5b6dc --- /dev/null +++ b/src/bootstrap/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:rustc-env=BUILD_TRIPLE={}", std::env::var("HOST").unwrap()); +} diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 6de4388495baf..1ec99ef30ffa5 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -487,7 +487,7 @@ impl Config { config.missing_tools = false; // set by bootstrap.py - config.build = TargetSelection::from_user(&env::var("BUILD").expect("'BUILD' to be set")); + config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); config.src = Config::path_from_python("SRC"); config.out = Config::path_from_python("BUILD_DIR"); From e21eb613e09a0ecb4eaa141712a389d05c40ffad Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sun, 6 Sep 2020 21:32:55 -0400 Subject: [PATCH 2/5] Remove support for different src directory This requires that bootstrap is run from the same worktree as the sources it'll build, but this is basically required for the build to work anyway. You can still run it from a different directory, just that the files it builds must be beside it. --- src/bootstrap/bootstrap.py | 4 +--- src/bootstrap/config.rs | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index c9a9a484ee338..c8cb2595d32f4 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -966,7 +966,6 @@ def bootstrap(help_triggered): parser = argparse.ArgumentParser(description='Build rust') parser.add_argument('--config') parser.add_argument('--build') - parser.add_argument('--src') parser.add_argument('--clean', action='store_true') parser.add_argument('-v', '--verbose', action='count', default=0) @@ -975,7 +974,7 @@ def bootstrap(help_triggered): # Configure initial bootstrap build = RustBuild() - build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..')) + build.rust_root = os.path.abspath(os.path.join(__file__, '../../..')) build.verbose = args.verbose build.clean = args.clean @@ -1032,7 +1031,6 @@ def bootstrap(help_triggered): args = [build.bootstrap_binary()] args.extend(sys.argv[1:]) env = os.environ.copy() - env["SRC"] = build.rust_root env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) env["BOOTSTRAP_PYTHON"] = sys.executable env["BUILD_DIR"] = build.build_dir diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 1ec99ef30ffa5..3ecfac835642f 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -488,7 +488,9 @@ impl Config { // set by bootstrap.py config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); - config.src = Config::path_from_python("SRC"); + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // Undo `src/bootstrap` + config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); config.out = Config::path_from_python("BUILD_DIR"); config.initial_rustc = Config::path_from_python("RUSTC"); From a625ab77e87b31ac6689e97f2f10cb09773bed38 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Mon, 7 Sep 2020 12:02:50 -0400 Subject: [PATCH 3/5] Discover Rust toolchain without Python --- src/bootstrap/bootstrap.py | 4 ---- src/bootstrap/build.rs | 25 ++++++++++++++++++++++++- src/bootstrap/config.rs | 19 ++++++++++++------- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index c8cb2595d32f4..e58cf0d164197 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -1035,12 +1035,8 @@ def bootstrap(help_triggered): env["BOOTSTRAP_PYTHON"] = sys.executable env["BUILD_DIR"] = build.build_dir env["RUSTC_BOOTSTRAP"] = '1' - env["CARGO"] = build.cargo() - env["RUSTC"] = build.rustc() if toml_path: env["BOOTSTRAP_CONFIG"] = toml_path - if build.rustfmt(): - env["RUSTFMT"] = build.rustfmt() run(args, env=env, verbose=build.verbose) diff --git a/src/bootstrap/build.rs b/src/bootstrap/build.rs index 5e5c31de5b6dc..d40b924e0ff5f 100644 --- a/src/bootstrap/build.rs +++ b/src/bootstrap/build.rs @@ -1,3 +1,26 @@ +use std::env; +use std::path::PathBuf; + fn main() { - println!("cargo:rustc-env=BUILD_TRIPLE={}", std::env::var("HOST").unwrap()); + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rustc-env=BUILD_TRIPLE={}", env::var("HOST").unwrap()); + + // This may not be a canonicalized path. + let mut rustc = PathBuf::from(env::var_os("RUSTC").unwrap()); + + if rustc.is_relative() { + for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) { + let absolute = dir.join(&rustc); + if absolute.exists() { + rustc = absolute; + break; + } + } + } + assert!(rustc.is_absolute()); + + // FIXME: if the path is not utf-8, this is going to break. Unfortunately + // Cargo doesn't have a way for us to specify non-utf-8 paths easily, so + // we'll need to invent some encoding scheme if this becomes a problem. + println!("cargo:rustc-env=RUSTC={}", rustc.to_str().unwrap()); } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 3ecfac835642f..d4f6ce64decc7 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -291,7 +291,7 @@ struct Build { build_dir: Option, cargo: Option, rustc: Option, - rustfmt: Option, /* allow bootstrap.py to use rustfmt key */ + rustfmt: Option, docs: Option, compiler_docs: Option, submodules: Option, @@ -493,9 +493,8 @@ impl Config { config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); config.out = Config::path_from_python("BUILD_DIR"); - config.initial_rustc = Config::path_from_python("RUSTC"); - config.initial_cargo = Config::path_from_python("CARGO"); - config.initial_rustfmt = env::var_os("RUSTFMT").map(Config::normalize_python_path); + config.initial_cargo = PathBuf::from(env!("CARGO")); + config.initial_rustc = PathBuf::from(env!("RUSTC")); config } @@ -584,6 +583,9 @@ impl Config { set(&mut config.full_bootstrap, build.full_bootstrap); set(&mut config.extended, build.extended); config.tools = build.tools; + if build.rustfmt.is_some() { + config.initial_rustfmt = build.rustfmt; + } set(&mut config.verbose, build.verbose); set(&mut config.sanitizers, build.sanitizers); set(&mut config.profiler, build.profiler); @@ -838,12 +840,15 @@ impl Config { set(&mut config.missing_tools, t.missing_tools); } + // Cargo does not provide a RUSTFMT environment variable, so we + // synthesize it manually. Note that we also later check the config.toml + // and set this to that path if necessary. + let rustfmt = config.initial_rustc.with_file_name(exe("rustfmt", config.build)); + config.initial_rustfmt = if rustfmt.exists() { Some(rustfmt) } else { None }; + // Now that we've reached the end of our configuration, infer the // default values for all options that we haven't otherwise stored yet. - set(&mut config.initial_rustc, build.rustc.map(PathBuf::from)); - set(&mut config.initial_cargo, build.cargo.map(PathBuf::from)); - config.llvm_skip_rebuild = llvm_skip_rebuild.unwrap_or(false); let default = false; From 1aac99de25d3e28e4b2a16e8fbe6737d5202f27a Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Thu, 10 Sep 2020 10:17:32 -0400 Subject: [PATCH 4/5] Provide bootstrap tools with RUSTC in environment --- src/bootstrap/test.rs | 2 ++ src/bootstrap/tool.rs | 4 ++++ src/test/run-make/thumb-none-cortex-m/Makefile | 2 +- src/test/run-make/thumb-none-qemu/script.sh | 4 ++-- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index dc28b8ece2452..a9a8d3783fd1e 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1281,6 +1281,8 @@ impl Step for Compiletest { cmd.arg("--rustfix-coverage"); } + cmd.env("BOOTSTRAP_CARGO", &builder.initial_cargo); + builder.ci_env.force_coloring_in_ci(&mut cmd); builder.info(&format!( diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 5d66632d92ceb..460dffb5c8a57 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -700,6 +700,10 @@ impl<'a> Builder<'a> { } add_dylib_path(lib_paths, &mut cmd); + + // Provide a RUSTC for this command to use. + cmd.env("RUSTC", &self.initial_rustc); + cmd } } diff --git a/src/test/run-make/thumb-none-cortex-m/Makefile b/src/test/run-make/thumb-none-cortex-m/Makefile index 36e51bcab6de2..13385369e4451 100644 --- a/src/test/run-make/thumb-none-cortex-m/Makefile +++ b/src/test/run-make/thumb-none-cortex-m/Makefile @@ -35,4 +35,4 @@ all: # HACK(eddyb) sets `RUSTC_BOOTSTRAP=1` so Cargo can accept nightly features. # These come from the top-level Rust workspace, that this crate is not a # member of, but Cargo tries to load the workspace `Cargo.toml` anyway. - cd $(WORK_DIR) && cd $(CRATE) && env RUSTC_BOOTSTRAP=1 $(CARGO) build --target $(TARGET) -v + cd $(WORK_DIR) && cd $(CRATE) && env RUSTC_BOOTSTRAP=1 $(BOOTSTRAP_CARGO) build --target $(TARGET) -v diff --git a/src/test/run-make/thumb-none-qemu/script.sh b/src/test/run-make/thumb-none-qemu/script.sh index c5cbff5c3c36d..045d02a8ed25d 100644 --- a/src/test/run-make/thumb-none-qemu/script.sh +++ b/src/test/run-make/thumb-none-qemu/script.sh @@ -12,8 +12,8 @@ pushd $WORK_DIR # These come from the top-level Rust workspace, that this crate is not a # member of, but Cargo tries to load the workspace `Cargo.toml` anyway. env RUSTC_BOOTSTRAP=1 RUSTFLAGS="-C linker=arm-none-eabi-ld -C link-arg=-Tlink.x" \ - $CARGO run --target $TARGET | grep "x = 42" + $BOOTSTRAP_CARGO run --target $TARGET | grep "x = 42" env RUSTC_BOOTSTRAP=1 RUSTFLAGS="-C linker=arm-none-eabi-ld -C link-arg=-Tlink.x" \ - $CARGO run --target $TARGET --release | grep "x = 42" + $BOOTSTRAP_CARGO run --target $TARGET --release | grep "x = 42" popd popd From cf33aad8fdd3885fbc26518aaccff0788d1a21a7 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Thu, 10 Sep 2020 10:52:27 -0400 Subject: [PATCH 5/5] Specify output directory for bootstrap tests --- src/bootstrap/builder/tests.rs | 3 +++ src/bootstrap/test.rs | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs index f96925f927086..cd90021507ec7 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/builder/tests.rs @@ -10,6 +10,9 @@ fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config { config.dry_run = true; config.ninja_in_file = false; // try to avoid spurious failures in dist where we create/delete each others file + config.out = PathBuf::from(env::var_os("BOOTSTRAP_OUTPUT_DIRECTORY").unwrap()); + config.initial_rustc = PathBuf::from(env::var_os("RUSTC").unwrap()); + config.initial_cargo = PathBuf::from(env::var_os("BOOTSTRAP_INITIAL_CARGO").unwrap()); let dir = config .out .join("tmp-rustbuild-tests") diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index a9a8d3783fd1e..00522ee6b673c 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -2024,6 +2024,8 @@ impl Step for Bootstrap { .current_dir(builder.src.join("src/bootstrap")) .env("RUSTFLAGS", "-Cdebuginfo=2") .env("CARGO_TARGET_DIR", builder.out.join("bootstrap")) + .env("BOOTSTRAP_OUTPUT_DIRECTORY", &builder.config.out) + .env("BOOTSTRAP_INITIAL_CARGO", &builder.config.initial_cargo) .env("RUSTC_BOOTSTRAP", "1") .env("RUSTC", &builder.initial_rustc); if let Some(flags) = option_env!("RUSTFLAGS") {