From 9307bb2a8820b80d073ab42e64b55d2d3bb27725 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Wed, 1 Feb 2023 21:47:30 -0600 Subject: [PATCH 1/4] Download beta compiler toolchain in bootstrap if it doesn't yet exist This is needed for when the shell scripts bypass python altogether and run the downloaded bootstrap directly --- src/bootstrap/config.rs | 35 ++++++++++---------- src/bootstrap/download.rs | 67 ++++++++++++++++++++++++++++++++------- src/bootstrap/flags.rs | 3 +- src/bootstrap/test.rs | 2 +- 4 files changed, 76 insertions(+), 31 deletions(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 165502b0a41d8..7bd5f33b58c23 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -55,8 +55,7 @@ pub enum DryRun { /// filled out from the decoded forms of the structs below. For documentation /// each field, see the corresponding fields in /// `config.toml.example`. -#[derive(Default)] -#[cfg_attr(test, derive(Clone))] +#[derive(Clone, Default)] pub struct Config { pub changelog_seen: Option, pub ccache: Option, @@ -217,27 +216,33 @@ pub struct Config { pub reuse: Option, pub cargo_native_static: bool, pub configure_args: Vec, + pub out: PathBuf, + pub rust_info: channel::GitInfo, // These are either the stage0 downloaded binaries or the locally installed ones. pub initial_cargo: PathBuf, pub initial_rustc: PathBuf, + #[cfg(not(test))] initial_rustfmt: RefCell, #[cfg(test)] pub initial_rustfmt: RefCell, - pub out: PathBuf, - pub rust_info: channel::GitInfo, } -#[derive(Default, Deserialize)] -#[cfg_attr(test, derive(Clone))] +#[derive(Clone, Default, Deserialize)] pub struct Stage0Metadata { + pub compiler: CompilerMetadata, pub config: Stage0Config, pub checksums_sha256: HashMap, pub rustfmt: Option, } -#[derive(Default, Deserialize)] -#[cfg_attr(test, derive(Clone))] +#[derive(Clone, Default, Deserialize)] +pub struct CompilerMetadata { + pub date: String, + pub version: String, +} + +#[derive(Clone, Default, Deserialize)] pub struct Stage0Config { pub dist_server: String, pub artifacts_server: String, @@ -245,8 +250,7 @@ pub struct Stage0Config { pub git_merge_commit_email: String, pub nightly_branch: String, } -#[derive(Default, Deserialize)] -#[cfg_attr(test, derive(Clone))] +#[derive(Clone, Default, Deserialize)] pub struct RustfmtMetadata { pub date: String, pub version: String, @@ -422,8 +426,7 @@ impl PartialEq<&str> for TargetSelection { } /// Per-target configuration stored in the global configuration structure. -#[derive(Default)] -#[cfg_attr(test, derive(Clone))] +#[derive(Clone, Default)] pub struct Target { /// Some(path to llvm-config) if using an external LLVM. pub llvm_config: Option, @@ -979,10 +982,10 @@ impl Config { config.out = crate::util::absolute(&config.out); } - config.initial_rustc = build - .rustc - .map(PathBuf::from) - .unwrap_or_else(|| config.out.join(config.build.triple).join("stage0/bin/rustc")); + config.initial_rustc = build.rustc.map(PathBuf::from).unwrap_or_else(|| { + config.download_beta_toolchain(); + config.out.join(config.build.triple).join("stage0/bin/rustc") + }); config.initial_cargo = build .cargo .map(PathBuf::from) diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs index bd67978a7662e..2b135675f32c6 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/download.rs @@ -361,26 +361,69 @@ impl Config { pub(crate) fn download_ci_rustc(&self, commit: &str) { self.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); + let version = self.artifact_version_part(commit); + // download-rustc doesn't need its own cargo, it can just use beta's. But it does need the + // `rustc_private` crates for tools. + let extra_components = ["rustc-dev"]; + + self.download_toolchain( + &version, + "ci-rustc", + commit, + &extra_components, + Self::download_ci_component, + ); + } + + pub(crate) fn download_beta_toolchain(&self) { + self.verbose(&format!("downloading stage0 beta artifacts")); + + let date = &self.stage0_metadata.compiler.date; + let version = &self.stage0_metadata.compiler.version; + let extra_components = ["cargo"]; + + let download_beta_component = |config: &Config, filename, prefix: &_, date: &_| { + config.download_component(DownloadSource::Dist, filename, prefix, date, "stage0") + }; + + self.download_toolchain( + version, + "stage0", + date, + &extra_components, + download_beta_component, + ); + } + + fn download_toolchain( + &self, + channel: &str, + sysroot: &str, + stamp_key: &str, + extra_components: &[&str], + download_component: fn(&Config, String, &str, &str), + ) { let host = self.build.triple; - let bin_root = self.out.join(host).join("ci-rustc"); + let bin_root = self.out.join(host).join(sysroot); let rustc_stamp = bin_root.join(".rustc-stamp"); - if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit) + if !bin_root.join("bin").join("rustc").exists() + || program_out_of_date(&rustc_stamp, stamp_key) { if bin_root.exists() { t!(fs::remove_dir_all(&bin_root)); } - let filename = format!("rust-std-{version}-{host}.tar.xz"); + let filename = format!("rust-std-{channel}-{host}.tar.xz"); let pattern = format!("rust-std-{host}"); - self.download_ci_component(filename, &pattern, commit); - let filename = format!("rustc-{version}-{host}.tar.xz"); - self.download_ci_component(filename, "rustc", commit); - // download-rustc doesn't need its own cargo, it can just use beta's. - let filename = format!("rustc-dev-{version}-{host}.tar.xz"); - self.download_ci_component(filename, "rustc-dev", commit); - let filename = format!("rust-src-{version}.tar.xz"); - self.download_ci_component(filename, "rust-src", commit); + download_component(self, filename, &pattern, stamp_key); + let filename = format!("rustc-{channel}-{host}.tar.xz"); + download_component(self, filename, "rustc", stamp_key); + + for component in extra_components { + let filename = format!("{component}-{channel}-{host}.tar.xz"); + download_component(self, filename, component, stamp_key); + } if self.should_fix_bins_and_dylibs() { self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); @@ -397,7 +440,7 @@ impl Config { } } - t!(fs::write(rustc_stamp, commit)); + t!(fs::write(rustc_stamp, stamp_key)); } } diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 52c3dc0bf7591..af2faa72344cc 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -82,8 +82,7 @@ pub struct Flags { pub llvm_bolt_profile_use: Option, } -#[derive(Debug)] -#[cfg_attr(test, derive(Clone))] +#[derive(Clone, Debug)] pub enum Subcommand { Build { paths: Vec, diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 6078e39ac9d3b..41a65296ad9b7 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1120,7 +1120,7 @@ impl Step for Tidy { if builder.config.channel == "dev" || builder.config.channel == "nightly" { builder.info("fmt check"); if builder.initial_rustfmt().is_none() { - let inferred_rustfmt_dir = builder.config.initial_rustc.parent().unwrap(); + let inferred_rustfmt_dir = builder.initial_rustc.parent().unwrap(); eprintln!( "\ error: no `rustfmt` binary found in {PATH} From bc8498394f24bb70fe2daa8795ccea8b8af9bd92 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Thu, 17 Nov 2022 00:18:56 -0500 Subject: [PATCH 2/4] [wip] Add bootstrap-shim and package it on nightly - Pass `dist bootstrap-shim` explicitly in CI This makes it possible to run `dist bootstrap` without also building bootstrap-shim, and more importantly works around a bug where currently two steps aren't allowed to have the same path. - Add `check::BootstrapShim` - [wip] start unifying parsing for Config and MinimalConfig --- src/bootstrap/Cargo.toml | 5 + src/bootstrap/bin/bootstrap-shim.rs | 29 ++ src/bootstrap/builder.rs | 4 +- src/bootstrap/check.rs | 2 + src/bootstrap/config.rs | 273 ++----------- src/bootstrap/dist.rs | 35 ++ src/bootstrap/download.rs | 202 +++++---- src/bootstrap/lib.rs | 2 + src/bootstrap/min_config.rs | 382 ++++++++++++++++++ .../host-x86_64/dist-x86_64-linux/Dockerfile | 2 +- src/ci/github-actions/ci.yml | 20 +- src/tools/build-manifest/src/main.rs | 6 +- src/tools/build-manifest/src/versions.rs | 3 + src/tools/x/src/main.rs | 2 +- 14 files changed, 627 insertions(+), 340 deletions(-) create mode 100644 src/bootstrap/bin/bootstrap-shim.rs create mode 100644 src/bootstrap/min_config.rs diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 22ceeca941e93..15a48447e0645 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -14,6 +14,11 @@ name = "bootstrap" path = "bin/main.rs" test = false +[[bin]] +name = "bootstrap-shim" +path = "bin/bootstrap-shim.rs" +test = false + [[bin]] name = "rustc" path = "bin/rustc.rs" diff --git a/src/bootstrap/bin/bootstrap-shim.rs b/src/bootstrap/bin/bootstrap-shim.rs new file mode 100644 index 0000000000000..d0d044d2373a8 --- /dev/null +++ b/src/bootstrap/bin/bootstrap-shim.rs @@ -0,0 +1,29 @@ +use std::{env, process::Command}; + +use bootstrap::{t, MinimalConfig}; + +#[path = "../../../src/tools/x/src/main.rs"] +mod run_python; + +fn main() { + let args = env::args().skip(1).collect::>(); + let mut opts = getopts::Options::new(); + opts.optopt("", "config", "TOML configuration file for build", "FILE"); + let matches = t!(opts.parse(args)); + + // If there are no untracked changes to bootstrap, download it from CI. + // Otherwise, build it from source. Use python to build to avoid duplicating the code between python and rust. + let config = MinimalConfig::parse(t!(matches.opt_get("config"))); + let bootstrap_bin = if let Some(commit) = last_modified_bootstrap_commit(&config) { + config.download_bootstrap(&commit) + } else { + return run_python::main(); + }; + + let args: Vec<_> = std::env::args().skip(1).collect(); + Command::new(bootstrap_bin).args(args).status().expect("failed to spawn bootstrap binairy"); +} + +fn last_modified_bootstrap_commit(config: &MinimalConfig) -> Option { + config.last_modified_commit(&["src/bootstrap"], "download-bootstrap", true) +} diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index b4fc1d4f28da7..c01bd32616fb0 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -656,7 +656,8 @@ impl<'a> Builder<'a> { check::Rls, check::RustAnalyzer, check::Rustfmt, - check::Bootstrap + check::Bootstrap, + check::BootstrapShim, ), Kind::Test => describe!( crate::toolstate::ToolStateCheck, @@ -761,6 +762,7 @@ impl<'a> Builder<'a> { dist::LlvmTools, dist::RustDev, dist::Bootstrap, + dist::BootstrapShim, dist::Extended, // It seems that PlainSourceTarball somehow changes how some of the tools // perceive their dependencies (see #93033) which would invalidate fingerprints diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 4b8a58e87b64e..a8a0a29788574 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -473,7 +473,9 @@ tool_check_step!(Rls, "src/tools/rls", SourceType::InTree); tool_check_step!(Rustfmt, "src/tools/rustfmt", SourceType::InTree); tool_check_step!(MiroptTestTools, "src/tools/miropt-test-tools", SourceType::InTree); +// FIXME: currently these are marked as ToolRustc, but they should be ToolBootstrap instead to avoid having to build the compiler first tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree, false); +tool_check_step!(BootstrapShim, "src/bootstrap/bin/bootstrap-shim", SourceType::InTree, false); /// Cargo's output path for the standard library in a given stage, compiled /// by a particular compiler for the specified target. diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 7bd5f33b58c23..73e534818e62d 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -10,10 +10,9 @@ use std::cell::{Cell, RefCell}; use std::cmp; use std::collections::{HashMap, HashSet}; use std::env; -use std::fmt; use std::fs; +use std::ops::{Deref, DerefMut}; use std::path::{Path, PathBuf}; -use std::process::Command; use std::str::FromStr; use crate::builder::TaskPath; @@ -22,10 +21,14 @@ use crate::cc_detect::{ndk_compiler, Language}; use crate::channel::{self, GitInfo}; pub use crate::flags::Subcommand; use crate::flags::{Color, Flags}; +use crate::min_config::get_toml; use crate::util::{exe, output, t}; +use crate::MinimalConfig; use once_cell::sync::OnceCell; use serde::{Deserialize, Deserializer}; +pub use crate::min_config::{DryRun, Stage0Metadata, TargetSelection}; + macro_rules! check_ci_llvm { ($name:expr) => { assert!( @@ -36,17 +39,6 @@ macro_rules! check_ci_llvm { }; } -#[derive(Clone, Default)] -pub enum DryRun { - /// This isn't a dry run. - #[default] - Disabled, - /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. - SelfCheck, - /// This is a dry run enabled by the `--dry-run` flag. - UserSelected, -} - /// Global configuration for the entire build and/or bootstrap. /// /// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters. @@ -61,7 +53,6 @@ pub struct Config { pub ccache: Option, /// Call Build::ninja() instead of this. pub ninja_in_file: bool, - pub verbose: usize, pub submodules: Option, pub compiler_docs: bool, pub library_docs_private_items: bool, @@ -82,20 +73,13 @@ pub struct Config { pub json_output: bool, pub test_compare_mode: bool, pub color: Color, - pub patch_binaries_for_nix: bool, - pub stage0_metadata: Stage0Metadata, - pub on_fail: Option, pub stage: u32, pub keep_stage: Vec, pub keep_stage_std: Vec, - pub src: PathBuf, - /// defaults to `config.toml` - pub config: Option, pub jobs: Option, pub cmd: Subcommand, pub incremental: bool, - pub dry_run: DryRun, /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should. #[cfg(not(test))] download_rustc_commit: Option, @@ -174,7 +158,6 @@ pub struct Config { pub llvm_bolt_profile_generate: bool, pub llvm_bolt_profile_use: Option, - pub build: TargetSelection, pub hosts: Vec, pub targets: Vec, pub local_rebuild: bool, @@ -216,7 +199,6 @@ pub struct Config { pub reuse: Option, pub cargo_native_static: bool, pub configure_args: Vec, - pub out: PathBuf, pub rust_info: channel::GitInfo, // These are either the stage0 downloaded binaries or the locally installed ones. @@ -227,33 +209,24 @@ pub struct Config { initial_rustfmt: RefCell, #[cfg(test)] pub initial_rustfmt: RefCell, -} -#[derive(Clone, Default, Deserialize)] -pub struct Stage0Metadata { - pub compiler: CompilerMetadata, - pub config: Stage0Config, - pub checksums_sha256: HashMap, - pub rustfmt: Option, -} -#[derive(Clone, Default, Deserialize)] -pub struct CompilerMetadata { - pub date: String, - pub version: String, + #[cfg(test)] + pub minimal_config: MinimalConfig, + #[cfg(not(test))] + minimal_config: MinimalConfig, } -#[derive(Clone, Default, Deserialize)] -pub struct Stage0Config { - pub dist_server: String, - pub artifacts_server: String, - pub artifacts_with_llvm_assertions_server: String, - pub git_merge_commit_email: String, - pub nightly_branch: String, +impl Deref for Config { + type Target = MinimalConfig; + fn deref(&self) -> &Self::Target { + &self.minimal_config + } } -#[derive(Clone, Default, Deserialize)] -pub struct RustfmtMetadata { - pub date: String, - pub version: String, + +impl DerefMut for Config { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.minimal_config + } } #[derive(Clone, Debug)] @@ -358,73 +331,6 @@ impl std::str::FromStr for RustcLto { } } -#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct TargetSelection { - pub triple: Interned, - file: Option>, -} - -impl TargetSelection { - pub fn from_user(selection: &str) -> Self { - let path = Path::new(selection); - - let (triple, file) = if path.exists() { - let triple = path - .file_stem() - .expect("Target specification file has no file stem") - .to_str() - .expect("Target specification file stem is not UTF-8"); - - (triple, Some(selection)) - } else { - (selection, None) - }; - - let triple = INTERNER.intern_str(triple); - let file = file.map(|f| INTERNER.intern_str(f)); - - Self { triple, file } - } - - pub fn rustc_target_arg(&self) -> &str { - self.file.as_ref().unwrap_or(&self.triple) - } - - pub fn contains(&self, needle: &str) -> bool { - self.triple.contains(needle) - } - - pub fn starts_with(&self, needle: &str) -> bool { - self.triple.starts_with(needle) - } - - pub fn ends_with(&self, needle: &str) -> bool { - self.triple.ends_with(needle) - } -} - -impl fmt::Display for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.triple)?; - if let Some(file) = self.file { - write!(f, "({})", file)?; - } - Ok(()) - } -} - -impl fmt::Debug for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self) - } -} - -impl PartialEq<&str> for TargetSelection { - fn eq(&self, other: &&str) -> bool { - self.triple == *other - } -} - /// Per-target configuration stored in the global configuration structure. #[derive(Clone, Default)] pub struct Target { @@ -828,31 +734,9 @@ impl Config { } pub fn parse(args: &[String]) -> Config { - #[cfg(test)] - let get_toml = |_: &_| TomlConfig::default(); - #[cfg(not(test))] - let get_toml = |file: &Path| { - let contents = - t!(fs::read_to_string(file), format!("config file {} not found", file.display())); - // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of - // TomlConfig and sub types to be monomorphized 5x by toml. - match toml::from_str(&contents) - .and_then(|table: toml::Value| TomlConfig::deserialize(table)) - { - Ok(table) => table, - Err(err) => { - eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err); - crate::detail_exit(2); - } - } - }; - - Self::parse_inner(args, get_toml) - } - - fn parse_inner<'a>(args: &[String], get_toml: impl 'a + Fn(&Path) -> TomlConfig) -> Config { let flags = Flags::parse(&args); let mut config = Config::default_opts(); + config.minimal_config = MinimalConfig::parse(flags.config.clone()); // Set flags. config.exclude = flags.exclude.into_iter().map(|path| TaskPath::parse(path)).collect(); @@ -884,44 +768,6 @@ impl Config { // Infer the rest of the configuration. - // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, - // running on a completely machine from where it was compiled. - let mut cmd = Command::new("git"); - // NOTE: we cannot support running from outside the repository because the only path we have available - // is set at compile time, which can be wrong if bootstrap was downloaded from source. - // We still support running outside the repository if we find we aren't in a git directory. - cmd.arg("rev-parse").arg("--show-toplevel"); - // Discard stderr because we expect this to fail when building from a tarball. - let output = cmd - .stderr(std::process::Stdio::null()) - .output() - .ok() - .and_then(|output| if output.status.success() { Some(output) } else { None }); - if let Some(output) = output { - let git_root = String::from_utf8(output.stdout).unwrap(); - // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes. - let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap(); - let s = git_root.to_str().unwrap(); - - // Bootstrap is quite bad at handling /? in front of paths - let src = match s.strip_prefix("\\\\?\\") { - Some(p) => PathBuf::from(p), - None => PathBuf::from(git_root), - }; - // If this doesn't have at least `stage0.json`, we guessed wrong. This can happen when, - // for example, the build directory is inside of another unrelated git directory. - // In that case keep the original `CARGO_MANIFEST_DIR` handling. - // - // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside - // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1. - if src.join("src").join("stage0.json").exists() { - config.src = src; - } - } else { - // We're building from a tarball, not git sources. - // We don't support pre-downloaded bootstrap in this case. - } - if cfg!(test) { // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. config.out = Path::new( @@ -1284,17 +1130,19 @@ impl Config { } if config.llvm_from_ci { - let triple = &config.build.triple; + let build_target_selection = config.build; let ci_llvm_bin = config.ci_llvm_root().join("bin"); let mut build_target = config .target_config .entry(config.build) - .or_insert_with(|| Target::from_triple(&triple)); + .or_insert_with(|| Target::from_triple(&build_target_selection.triple)); check_ci_llvm!(build_target.llvm_config); check_ci_llvm!(build_target.llvm_filecheck); - build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); - build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); + build_target.llvm_config = + Some(ci_llvm_bin.join(exe("llvm-config", build_target_selection))); + build_target.llvm_filecheck = + Some(ci_llvm_bin.join(exe("FileCheck", build_target_selection))); } if let Some(t) = toml.dist { @@ -1413,22 +1261,6 @@ impl Config { config } - pub(crate) fn dry_run(&self) -> bool { - match self.dry_run { - DryRun::Disabled => false, - DryRun::SelfCheck | DryRun::UserSelected => true, - } - } - - /// A git invocation which runs inside the source directory. - /// - /// Use this rather than `Command::new("git")` in order to support out-of-tree builds. - pub(crate) fn git(&self) -> Command { - let mut git = Command::new("git"); - git.current_dir(&self.src); - git - } - /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. /// Return the version it would have used for the given commit. pub(crate) fn artifact_version_part(&self, commit: &str) -> String { @@ -1575,12 +1407,6 @@ impl Config { } } - pub fn verbose(&self, msg: &str) { - if self.verbose > 0 { - println!("{}", msg); - } - } - pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool { self.target_config.get(&target).map(|t| t.sanitizers).flatten().unwrap_or(self.sanitizers) } @@ -1633,52 +1459,7 @@ impl Config { } }; - // Handle running from a directory other than the top level - let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); - let top_level = top_level.trim_end(); - let compiler = format!("{top_level}/compiler/"); - let library = format!("{top_level}/library/"); - - // Look for a version to compare to based on the current commit. - // Only commits merged by bors will have CI artifacts. - let merge_base = output( - self.git() - .arg("rev-list") - .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) - .args(&["-n1", "--first-parent", "HEAD"]), - ); - let commit = merge_base.trim_end(); - if commit.is_empty() { - println!("error: could not find commit hash for downloading rustc"); - println!("help: maybe your repository history is too shallow?"); - println!("help: consider disabling `download-rustc`"); - println!("help: or fetch enough history to include one upstream commit"); - crate::detail_exit(1); - } - - // Warn if there were changes to the compiler or standard library since the ancestor commit. - let has_changes = !t!(self - .git() - .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library]) - .status()) - .success(); - if has_changes { - if if_unchanged { - if self.verbose > 0 { - println!( - "warning: saw changes to compiler/ or library/ since {commit}; \ - ignoring `download-rustc`" - ); - } - return None; - } - println!( - "warning: `download-rustc` is enabled, but there are changes to \ - compiler/ or library/" - ); - } - - Some(commit.to_string()) + self.last_modified_commit(&["compiler", "library"], "download-rustc", if_unchanged) } } diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 02e35d2436e2f..60fb5c99a3d20 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2155,6 +2155,41 @@ impl Step for Bootstrap { } } +/// Tarball intended for being able to run `rustup component add bootstrap-shim`. +/// Not stable, but user-facing. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct BootstrapShim { + pub target: TargetSelection, +} + +impl Step for BootstrapShim { + type Output = Option; + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + // Create this even with only `dist bootstrap` to avoid having to update all CI builders. + run.alias("bootstrap-shim") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BootstrapShim { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let target = self.target; + let mut tarball = Tarball::new(builder, "bootstrap-shim", &target.triple); + let bootstrap_outdir = &builder.bootstrap_out; + tarball.add_file( + bootstrap_outdir.join(exe("bootstrap-shim", target)), + tarball.image_dir().join("bin"), + 0o755, + ); + tarball.is_preview(true); + Some(tarball.generate()) + } +} + /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the /// release process to avoid cloning the monorepo and building stuff. /// diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs index 2b135675f32c6..76ab7d83773d0 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/download.rs @@ -11,17 +11,17 @@ use once_cell::sync::OnceCell; use xz2::bufread::XzDecoder; use crate::{ - config::RustfmtMetadata, + min_config::RustfmtMetadata, native::detect_llvm_sha, t, - util::{check_run, exe, program_out_of_date, try_run}, - Config, + util::{check_run, exe, output, program_out_of_date, try_run}, + Config, MinimalConfig, }; static SHOULD_FIX_BINS_AND_DYLIBS: OnceCell = OnceCell::new(); /// Generic helpers that are useful anywhere in bootstrap. -impl Config { +impl MinimalConfig { pub fn is_verbose(&self) -> bool { self.verbose > 0 } @@ -314,6 +314,29 @@ impl Config { } return verified; } + + /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. + /// Return the version it would have used for the given commit. + /// + /// NOTE: this currently doesn't support tarballs, use `Config::artifact_version_part` if you need that support. + pub(crate) fn git_artifact_version_part(&self, commit: &str) -> String { + let (channel, version) = { + let mut channel = self.git(); + channel.arg("show").arg(format!("{}:src/ci/channel", commit)); + let channel = output(&mut channel); + let mut version = self.git(); + version.arg("show").arg(format!("{}:src/version", commit)); + let version = output(&mut version); + (channel.trim().to_owned(), version.trim().to_owned()) + }; + + match channel.as_str() { + "stable" => version, + "beta" => channel, + "nightly" => channel, + other => unreachable!("{:?} is not recognized as a valid channel", other), + } + } } enum DownloadSource { @@ -447,7 +470,102 @@ impl Config { /// Download a single component of a CI-built toolchain (not necessarily a published nightly). // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident fn download_ci_component(&self, filename: String, prefix: &str, commit: &str) { - Self::download_component(self, DownloadSource::CI, filename, prefix, commit, "ci-rustc") + self.download_component(DownloadSource::CI, filename, prefix, commit, "ci-rustc") + } + + pub(crate) fn maybe_download_ci_llvm(&self) { + if !self.llvm_from_ci { + return; + } + let llvm_root = self.ci_llvm_root(); + let llvm_stamp = llvm_root.join(".llvm-stamp"); + let llvm_sha = detect_llvm_sha(&self, self.rust_info.is_managed_git_subrepository()); + let key = format!("{}{}", llvm_sha, self.llvm_assertions); + if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { + self.download_ci_llvm(&llvm_sha); + + if self.should_fix_bins_and_dylibs() { + for entry in t!(fs::read_dir(llvm_root.join("bin"))) { + self.fix_bin_or_dylib(&t!(entry).path()); + } + + let llvm_lib = llvm_root.join("lib"); + for entry in t!(fs::read_dir(&llvm_lib)) { + let lib = t!(entry).path(); + if lib.extension().map_or(false, |ext| ext == "so") { + self.fix_bin_or_dylib(&lib); + } + } + } + + // Update the timestamp of llvm-config to force rustc_llvm to be + // rebuilt. This is a hacky workaround for a deficiency in Cargo where + // the rerun-if-changed directive doesn't handle changes very well. + // https://github.com/rust-lang/cargo/issues/10791 + // Cargo only compares the timestamp of the file relative to the last + // time `rustc_llvm` build script ran. However, the timestamps of the + // files in the tarball are in the past, so it doesn't trigger a + // rebuild. + let now = filetime::FileTime::from_system_time(std::time::SystemTime::now()); + let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build)); + t!(filetime::set_file_times(&llvm_config, now, now)); + + t!(fs::write(llvm_stamp, key)); + } + } + + fn download_ci_llvm(&self, llvm_sha: &str) { + let llvm_assertions = self.llvm_assertions; + + let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions); + let cache_dst = self.out.join("cache"); + let rustc_cache = cache_dst.join(cache_prefix); + if !rustc_cache.exists() { + t!(fs::create_dir_all(&rustc_cache)); + } + let base = if llvm_assertions { + &self.stage0_metadata.config.artifacts_with_llvm_assertions_server + } else { + &self.stage0_metadata.config.artifacts_server + }; + let version = self.artifact_version_part(llvm_sha); + let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple); + let tarball = rustc_cache.join(&filename); + if !tarball.exists() { + let help_on_error = "error: failed to download llvm from ci + + help: old builds get deleted after a certain time + help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml: + + [llvm] + download-ci-llvm = false + "; + self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error); + } + let llvm_root = self.ci_llvm_root(); + self.unpack(&tarball, &llvm_root, "rust-dev"); + } +} + +impl MinimalConfig { + pub fn download_bootstrap(&self, commit: &str) -> PathBuf { + self.verbose(&format!("downloading bootstrap from CI (commit {commit})")); + let host = self.build.triple; + let bin_root = self.out.join(host).join("bootstrap"); + let stamp = bin_root.join(".bootstrap-stamp"); + let bootstrap_bin = bin_root.join("bin").join("bootstrap"); + + if !bootstrap_bin.exists() || program_out_of_date(&stamp, commit) { + let version = self.git_artifact_version_part(commit); + let filename = format!("bootstrap-{version}-{host}.tar.xz"); + self.download_component(DownloadSource::CI, filename, "bootstrap", commit, ""); + if self.should_fix_bins_and_dylibs() { + self.fix_bin_or_dylib(&bootstrap_bin); + } + t!(fs::write(stamp, commit)); + } + + bootstrap_bin } fn download_component( @@ -520,78 +638,4 @@ impl Config { self.unpack(&tarball, &bin_root, prefix); } - - pub(crate) fn maybe_download_ci_llvm(&self) { - if !self.llvm_from_ci { - return; - } - let llvm_root = self.ci_llvm_root(); - let llvm_stamp = llvm_root.join(".llvm-stamp"); - let llvm_sha = detect_llvm_sha(&self, self.rust_info.is_managed_git_subrepository()); - let key = format!("{}{}", llvm_sha, self.llvm_assertions); - if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { - self.download_ci_llvm(&llvm_sha); - if self.should_fix_bins_and_dylibs() { - for entry in t!(fs::read_dir(llvm_root.join("bin"))) { - self.fix_bin_or_dylib(&t!(entry).path()); - } - } - - // Update the timestamp of llvm-config to force rustc_llvm to be - // rebuilt. This is a hacky workaround for a deficiency in Cargo where - // the rerun-if-changed directive doesn't handle changes very well. - // https://github.com/rust-lang/cargo/issues/10791 - // Cargo only compares the timestamp of the file relative to the last - // time `rustc_llvm` build script ran. However, the timestamps of the - // files in the tarball are in the past, so it doesn't trigger a - // rebuild. - let now = filetime::FileTime::from_system_time(std::time::SystemTime::now()); - let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build)); - t!(filetime::set_file_times(&llvm_config, now, now)); - - if self.should_fix_bins_and_dylibs() { - let llvm_lib = llvm_root.join("lib"); - for entry in t!(fs::read_dir(&llvm_lib)) { - let lib = t!(entry).path(); - if lib.extension().map_or(false, |ext| ext == "so") { - self.fix_bin_or_dylib(&lib); - } - } - } - - t!(fs::write(llvm_stamp, key)); - } - } - - fn download_ci_llvm(&self, llvm_sha: &str) { - let llvm_assertions = self.llvm_assertions; - - let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions); - let cache_dst = self.out.join("cache"); - let rustc_cache = cache_dst.join(cache_prefix); - if !rustc_cache.exists() { - t!(fs::create_dir_all(&rustc_cache)); - } - let base = if llvm_assertions { - &self.stage0_metadata.config.artifacts_with_llvm_assertions_server - } else { - &self.stage0_metadata.config.artifacts_server - }; - let version = self.artifact_version_part(llvm_sha); - let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple); - let tarball = rustc_cache.join(&filename); - if !tarball.exists() { - let help_on_error = "error: failed to download llvm from ci - - help: old builds get deleted after a certain time - help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml: - - [llvm] - download-ci-llvm = false - "; - self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error); - } - let llvm_root = self.ci_llvm_root(); - self.unpack(&tarball, &llvm_root, "rust-dev"); - } } diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 267aa3278d8ff..604ccdd5b70e1 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -141,6 +141,7 @@ mod flags; mod format; mod install; mod metadata; +mod min_config; mod native; mod run; mod sanity; @@ -175,6 +176,7 @@ pub use crate::builder::PathSet; use crate::cache::{Interned, INTERNER}; pub use crate::config::Config; pub use crate::flags::Subcommand; +pub use crate::min_config::MinimalConfig; const LLVM_TOOLS: &[&str] = &[ "llvm-cov", // used to generate coverage report diff --git a/src/bootstrap/min_config.rs b/src/bootstrap/min_config.rs new file mode 100644 index 0000000000000..d3be1e7691a3d --- /dev/null +++ b/src/bootstrap/min_config.rs @@ -0,0 +1,382 @@ +use core::fmt; +use std::{ + collections::HashMap, + env, fs, + path::{Path, PathBuf}, + process::Command, +}; + +use serde::Deserialize; + +use crate::{ + cache::{Interned, INTERNER}, + t, + util::output, +}; + +/// The bare minimum config, suitable for `bootstrap-shim`, but sharing code with the main `bootstrap` binary. +#[derive(Default, Clone)] +pub struct MinimalConfig { + // Needed so we know where to store the unpacked bootstrap binary. + pub build: TargetSelection, + // Needed so we know where to load `src/stage0.json` + pub src: PathBuf, + // Needed so we know where to store the cache. + pub out: PathBuf, + pub patch_binaries_for_nix: bool, + // Needed to know which commit to download. + pub stage0_metadata: Stage0Metadata, + + // This isn't currently used, but will eventually let people configure whether to download or build bootstrap. + pub config: Option, + // Not currently used in the shim. + pub verbose: usize, + // Not currently used in the shim. + pub dry_run: DryRun, +} + +#[derive(Default, Deserialize, Clone)] +pub struct Stage0Metadata { + pub compiler: CompilerMetadata, + pub config: Stage0Config, + pub checksums_sha256: HashMap, + pub rustfmt: Option, +} +#[derive(Clone, Default, Deserialize)] +pub struct CompilerMetadata { + pub date: String, + pub version: String, +} +#[derive(Default, Deserialize, Clone)] +pub struct Stage0Config { + pub dist_server: String, + pub artifacts_server: String, + pub artifacts_with_llvm_assertions_server: String, + pub git_merge_commit_email: String, + pub nightly_branch: String, +} + +#[derive(Default, Deserialize, Clone)] +pub struct RustfmtMetadata { + pub date: String, + pub version: String, +} + +impl MinimalConfig { + fn default_opts() -> Self { + let dry_run = DryRun::default(); + let config = None; + let verbose = 0; + let patch_binaries_for_nix = false; + let stage0_metadata = Stage0Metadata::default(); + + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // Undo `src/bootstrap` + let src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); + let out = PathBuf::from("build"); + + // set by build.rs + let build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); + + MinimalConfig { + build, + src, + out, + config, + dry_run, + verbose, + patch_binaries_for_nix, + stage0_metadata, + } + } + + pub fn parse(config_flag: Option) -> MinimalConfig { + let mut config = Self::default_opts(); + + if let Some(src) = src() { + config.src = src; + } + + if cfg!(test) { + // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. + config.out = Path::new( + &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), + ) + .parent() + .unwrap() + .to_path_buf(); + } + + let toml = if let Some(toml_path) = Self::config_path(config.src.clone(), config_flag) { + config.config = Some(toml_path.clone()); + get_toml(&toml_path) + } else { + config.config = None; + TomlConfig::default() + }; + if let Some(build) = toml.build.unwrap_or_default().build { + config.build = TargetSelection::from_user(&build); + } + + // NOTE: Bootstrap spawns various commands with different working directories. + // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. + if !config.out.is_absolute() { + // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. + config.out = crate::util::absolute(&config.out); + } + + if config.dry_run() { + let dir = config.out.join("tmp-dry-run"); + t!(fs::create_dir_all(&dir)); + config.out = dir; + } + + let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); + config.stage0_metadata = t!(serde_json::from_slice::(&stage0_json)); + + config + } + + /// Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. + /// + /// Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, + /// but not if `config.toml` hasn't been created. + fn config_path(src: PathBuf, config_flag: Option) -> Option { + let toml_path = + config_flag.or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); + let using_default_path = toml_path.is_none(); + let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); + if using_default_path && !toml_path.exists() { + toml_path = src.join(toml_path); + } + + if !using_default_path || toml_path.exists() { Some(toml_path) } else { None } + } +} + +impl MinimalConfig { + pub fn verbose(&self, msg: &str) { + if self.verbose > 0 { + println!("{}", msg); + } + } + + pub(crate) fn dry_run(&self) -> bool { + match self.dry_run { + DryRun::Disabled => false, + DryRun::SelfCheck | DryRun::UserSelected => true, + } + } + + /// A git invocation which runs inside the source directory. + /// + /// Use this rather than `Command::new("git")` in order to support out-of-tree builds. + pub(crate) fn git(&self) -> Command { + let mut git = Command::new("git"); + git.current_dir(&self.src); + git + } + + /// Returns the last commit in which any of `modified_paths` were changed, + /// or `None` if there are untracked changes in the working directory and `if_unchanged` is true. + pub fn last_modified_commit( + &self, + modified_paths: &[&str], + option_name: &str, + if_unchanged: bool, + ) -> Option { + // Handle running from a directory other than the top level + let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); + let top_level = top_level.trim_end(); + + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + let merge_base = output( + self.git() + .arg("rev-list") + .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) + .args(&["-n1", "--first-parent", "HEAD"]), + ); + let commit = merge_base.trim_end(); + if commit.is_empty() { + println!("error: could not find commit hash for downloading components from CI"); + println!("help: maybe your repository history is too shallow?"); + println!("help: consider disabling `{option_name}`"); + println!("help: or fetch enough history to include one upstream commit"); + crate::detail_exit(1); + } + + // Warn if there were changes to the compiler or standard library since the ancestor commit. + let mut git = self.git(); + git.args(&["diff-index", "--quiet", &commit, "--"]); + + for path in modified_paths { + git.arg(format!("{top_level}/{path}")); + } + + let has_changes = !t!(git.status()).success(); + if has_changes { + if if_unchanged { + if self.verbose > 0 { + println!( + "warning: saw changes to one of {modified_paths:?} since {commit}; \ + ignoring `{option_name}`" + ); + } + return None; + } + println!( + "warning: `{option_name}` is enabled, but there are changes to one of {modified_paths:?}" + ); + } + + Some(commit.to_string()) + } +} + +#[cfg(test)] +pub(crate) fn get_toml + Default>(_file: &Path) -> T { + T::default() +} +#[cfg(not(test))] +pub(crate) fn get_toml + Default>(file: &Path) -> T { + let contents = + t!(fs::read_to_string(file), format!("config file {} not found", file.display())); + // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of + // TomlConfig and sub types to be monomorphized 5x by toml. + match toml::from_str(&contents).and_then(|table: toml::Value| T::deserialize(table)) { + Ok(table) => table, + Err(err) => { + eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err); + crate::detail_exit(2); + } + } +} + +fn src() -> Option { + // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, + // running on a completely machine from where it was compiled. + let mut cmd = Command::new("git"); + // NOTE: we cannot support running from outside the repository because the only path we have available + // is set at compile time, which can be wrong if bootstrap was downloaded from source. + // We still support running outside the repository if we find we aren't in a git directory. + cmd.arg("rev-parse").arg("--show-toplevel"); + // Discard stderr because we expect this to fail when building from a tarball. + let output = cmd + .stderr(std::process::Stdio::null()) + .output() + .ok() + .and_then(|output| if output.status.success() { Some(output) } else { None }); + if let Some(output) = output { + let git_root = String::from_utf8(output.stdout).unwrap(); + // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes. + let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap(); + let s = git_root.to_str().unwrap(); + + // Bootstrap is quite bad at handling /? in front of paths + let src = match s.strip_prefix("\\\\?\\") { + Some(p) => PathBuf::from(p), + None => PathBuf::from(git_root), + }; + // If this doesn't have at least `stage0.json`, we guessed wrong. This can happen when, + // for example, the build directory is inside of another unrelated git directory. + // In that case keep the original `CARGO_MANIFEST_DIR` handling. + // + // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside + // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1. + if src.join("src").join("stage0.json").exists() { Some(src) } else { None } + } else { + // We're building from a tarball, not git sources. + // We don't support pre-downloaded bootstrap in this case. + None + } +} + +#[derive(Clone, Default)] +pub enum DryRun { + /// This isn't a dry run. + #[default] + Disabled, + /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. + SelfCheck, + /// This is a dry run enabled by the `--dry-run` flag. + UserSelected, +} + +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TargetSelection { + pub triple: Interned, + file: Option>, +} + +impl TargetSelection { + pub fn from_user(selection: &str) -> Self { + let path = Path::new(selection); + + let (triple, file) = if path.exists() { + let triple = path + .file_stem() + .expect("Target specification file has no file stem") + .to_str() + .expect("Target specification file stem is not UTF-8"); + + (triple, Some(selection)) + } else { + (selection, None) + }; + + let triple = INTERNER.intern_str(triple); + let file = file.map(|f| INTERNER.intern_str(f)); + + Self { triple, file } + } + + pub fn rustc_target_arg(&self) -> &str { + self.file.as_ref().unwrap_or(&self.triple) + } + + pub fn contains(&self, needle: &str) -> bool { + self.triple.contains(needle) + } + + pub fn starts_with(&self, needle: &str) -> bool { + self.triple.starts_with(needle) + } + + pub fn ends_with(&self, needle: &str) -> bool { + self.triple.ends_with(needle) + } +} + +impl fmt::Display for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.triple)?; + if let Some(file) = self.file { + write!(f, "({})", file)?; + } + Ok(()) + } +} + +impl fmt::Debug for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self) + } +} + +impl PartialEq<&str> for TargetSelection { + fn eq(&self, other: &&str) -> bool { + self.triple == *other + } +} + +#[derive(Deserialize, Default)] +struct TomlConfig { + build: Option, +} + +/// TOML representation of various global build decisions. +#[derive(Deserialize, Default)] +struct Build { + build: Option, +} diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 5feba4e0605ec..7551406a8a5e4 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -84,7 +84,7 @@ ENV RUST_CONFIGURE_ARGS \ ENV SCRIPT python3 ../src/ci/stage-build.py python3 ../x.py dist \ --host $HOSTS --target $HOSTS \ --include-default-paths \ - build-manifest bootstrap + build-manifest bootstrap bootstrap-shim ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang # This is the only builder which will create source tarballs diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 8ff6e3ba4e801..f527d90541eb8 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -484,7 +484,7 @@ jobs: - name: dist-x86_64-apple env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin + SCRIPT: ./x.py dist bootstrap bootstrap-shim --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -497,7 +497,7 @@ jobs: - name: dist-apple-various env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim + SCRIPT: ./x.py dist bootstrap bootstrap-shim --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -509,7 +509,7 @@ jobs: - name: dist-x86_64-apple-alt env: - SCRIPT: ./x.py dist bootstrap --include-default-paths + SCRIPT: ./x.py dist bootstrap bootstrap-shim --include-default-paths RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -540,7 +540,7 @@ jobs: # This target only needs to support 11.0 and up as nothing else supports the hardware - name: dist-aarch64-apple env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --stage 2 + SCRIPT: ./x.py dist bootstrap bootstrap-shim --include-default-paths --stage 2 RUST_CONFIGURE_ARGS: >- --build=x86_64-apple-darwin --host=aarch64-apple-darwin @@ -680,7 +680,7 @@ jobs: --enable-full-tools --enable-profiler --set rust.lto=thin - SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap --include-default-paths + SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-xl @@ -692,7 +692,7 @@ jobs: --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-xl @@ -703,7 +703,7 @@ jobs: --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 # Hack around this SDK version, because it doesn't work with clang. # See https://github.com/rust-lang/rust/issues/88796 @@ -719,14 +719,14 @@ jobs: # We are intentionally allowing an old toolchain on this builder (and that's # incompatible with LLVM downloads today). NO_DOWNLOAD_CI_LLVM: 1 - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-xl - name: dist-x86_64-mingw env: - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-gnu --enable-full-tools @@ -741,7 +741,7 @@ jobs: - name: dist-x86_64-msvc-alt env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths <<: *job-windows-xl try: diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 21dad9eb74aa9..ec0dbc3c3badc 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -183,7 +183,8 @@ static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin" static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"]; -static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = &[PkgType::Miri, PkgType::JsonDocs]; +static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = + &[PkgType::Miri, PkgType::JsonDocs, PkgType::BootstrapShim]; macro_rules! t { ($e:expr) => { @@ -414,7 +415,8 @@ impl Builder { | PkgType::Rustfmt | PkgType::LlvmTools | PkgType::RustAnalysis - | PkgType::JsonDocs => { + | PkgType::JsonDocs + | PkgType::BootstrapShim => { extensions.push(host_component(pkg)); } PkgType::RustcDev | PkgType::RustcDocs => { diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index dde9745afb785..ee6ed8d27ed03 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -56,6 +56,7 @@ pkg_type! { LlvmTools = "llvm-tools"; preview = true, Miri = "miri"; preview = true, JsonDocs = "rust-docs-json"; preview = true, + BootstrapShim = "bootstrap-shim"; preview = true, } impl PkgType { @@ -91,6 +92,7 @@ impl PkgType { PkgType::ReproducibleArtifacts => true, PkgType::RustMingw => true, PkgType::RustAnalysis => true, + PkgType::BootstrapShim => true, } } @@ -114,6 +116,7 @@ impl PkgType { RustAnalyzer => HOSTS, Clippy => HOSTS, Miri => HOSTS, + BootstrapShim => HOSTS, Rustfmt => HOSTS, RustAnalysis => TARGETS, LlvmTools => TARGETS, diff --git a/src/tools/x/src/main.rs b/src/tools/x/src/main.rs index 01f7187851e38..638cb12a85549 100644 --- a/src/tools/x/src/main.rs +++ b/src/tools/x/src/main.rs @@ -51,7 +51,7 @@ fn exec_or_status(command: &mut Command) -> io::Result { command.status() } -fn main() { +pub fn main() { match env::args().skip(1).next().as_deref() { Some("--wrapper-version") => { let version = env!("CARGO_PKG_VERSION"); From a445f80f49a61003341ae2043ae388cc5e342be7 Mon Sep 17 00:00:00 2001 From: ozkanonur Date: Mon, 30 Jan 2023 16:51:14 +0300 Subject: [PATCH 3/4] Unify `MinimalConfig::parse` and `Config::parse` Signed-off-by: ozkanonur --- .github/workflows/ci.yml | 20 +++---- src/bootstrap/config.rs | 41 +++----------- src/bootstrap/min_config.rs | 107 ++++++++++++++++++++++-------------- 3 files changed, 83 insertions(+), 85 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac3a51df40135..28ffa62c17994 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -316,7 +316,7 @@ jobs: os: ubuntu-20.04-xl - name: dist-x86_64-apple env: - SCRIPT: "./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin" + SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin" RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -328,7 +328,7 @@ jobs: os: macos-latest - name: dist-apple-various env: - SCRIPT: "./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim" + SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim" RUST_CONFIGURE_ARGS: "--enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -339,7 +339,7 @@ jobs: os: macos-latest - name: dist-x86_64-apple-alt env: - SCRIPT: "./x.py dist bootstrap --include-default-paths" + SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths" RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -372,7 +372,7 @@ jobs: os: macos-latest - name: dist-aarch64-apple env: - SCRIPT: "./x.py dist bootstrap --include-default-paths --stage 2" + SCRIPT: "./x.py dist bootstrap bootstrap-shim --include-default-paths --stage 2" RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --host=aarch64-apple-darwin --target=aarch64-apple-darwin --enable-full-tools --enable-sanitizers --enable-profiler --disable-docs --set rust.jemalloc --set llvm.ninja=false" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 SELECT_XCODE: /Applications/Xcode_13.4.1.app @@ -447,19 +447,19 @@ jobs: - name: dist-x86_64-msvc env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler --set rust.lto=thin" - SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap --include-default-paths + SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 os: windows-latest-xl - name: dist-i686-msvc env: RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 os: windows-latest-xl - name: dist-aarch64-msvc env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 WINDOWS_SDK_20348_HACK: 1 os: windows-latest-xl @@ -467,13 +467,13 @@ jobs: env: RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler" NO_DOWNLOAD_CI_LLVM: 1 - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 os: windows-latest-xl - name: dist-x86_64-mingw env: - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler" NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 @@ -482,7 +482,7 @@ jobs: - name: dist-x86_64-msvc-alt env: RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler" - SCRIPT: python x.py dist bootstrap --include-default-paths + SCRIPT: python x.py dist bootstrap bootstrap-shim --include-default-paths os: windows-latest-xl timeout-minutes: 600 runs-on: "${{ matrix.os }}" diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 73e534818e62d..eab0e446accdd 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -21,7 +21,9 @@ use crate::cc_detect::{ndk_compiler, Language}; use crate::channel::{self, GitInfo}; pub use crate::flags::Subcommand; use crate::flags::{Color, Flags}; -use crate::min_config::get_toml; +use crate::min_config::{ + deserialize_stage0_metadata, get_toml, set_and_return_toml_config, set_config_output_dir, +}; use crate::util::{exe, output, t}; use crate::MinimalConfig; use once_cell::sync::OnceCell; @@ -768,40 +770,11 @@ impl Config { // Infer the rest of the configuration. - if cfg!(test) { - // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. - config.out = Path::new( - &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), - ) - .parent() - .unwrap() - .to_path_buf(); - } - - let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); + set_config_output_dir(&mut config.out); + config.stage0_metadata = deserialize_stage0_metadata(&config.src); - config.stage0_metadata = t!(serde_json::from_slice::(&stage0_json)); - - // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. - let toml_path = flags - .config - .clone() - .or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); - let using_default_path = toml_path.is_none(); - let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); - if using_default_path && !toml_path.exists() { - toml_path = config.src.join(toml_path); - } - - // Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, - // but not if `config.toml` hasn't been created. - let mut toml = if !using_default_path || toml_path.exists() { - config.config = Some(toml_path.clone()); - get_toml(&toml_path) - } else { - config.config = None; - TomlConfig::default() - }; + let mut toml: TomlConfig = + set_and_return_toml_config(config.src.clone(), flags.config, &mut config.config); if let Some(include) = &toml.profile { let mut include_path = config.src.clone(); diff --git a/src/bootstrap/min_config.rs b/src/bootstrap/min_config.rs index d3be1e7691a3d..908ca29791b08 100644 --- a/src/bootstrap/min_config.rs +++ b/src/bootstrap/min_config.rs @@ -97,61 +97,31 @@ impl MinimalConfig { config.src = src; } - if cfg!(test) { - // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. - config.out = Path::new( - &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), - ) - .parent() - .unwrap() - .to_path_buf(); - } + set_config_output_dir(&mut config.out); + + let toml: TomlConfig = + set_and_return_toml_config(config.src.clone(), config_flag, &mut config.config); - let toml = if let Some(toml_path) = Self::config_path(config.src.clone(), config_flag) { - config.config = Some(toml_path.clone()); - get_toml(&toml_path) - } else { - config.config = None; - TomlConfig::default() - }; if let Some(build) = toml.build.unwrap_or_default().build { config.build = TargetSelection::from_user(&build); } - // NOTE: Bootstrap spawns various commands with different working directories. - // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. - if !config.out.is_absolute() { - // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. - config.out = crate::util::absolute(&config.out); - } - if config.dry_run() { let dir = config.out.join("tmp-dry-run"); t!(fs::create_dir_all(&dir)); config.out = dir; } + // NOTE: Bootstrap spawns various commands with different working directories. + // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. + else if !config.out.is_absolute() { + // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. + config.out = crate::util::absolute(&config.out); + } - let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); - config.stage0_metadata = t!(serde_json::from_slice::(&stage0_json)); + config.stage0_metadata = deserialize_stage0_metadata(&config.src); config } - - /// Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. - /// - /// Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, - /// but not if `config.toml` hasn't been created. - fn config_path(src: PathBuf, config_flag: Option) -> Option { - let toml_path = - config_flag.or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); - let using_default_path = toml_path.is_none(); - let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); - if using_default_path && !toml_path.exists() { - toml_path = src.join(toml_path); - } - - if !using_default_path || toml_path.exists() { Some(toml_path) } else { None } - } } impl MinimalConfig { @@ -235,10 +205,12 @@ impl MinimalConfig { } #[cfg(test)] +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` pub(crate) fn get_toml + Default>(_file: &Path) -> T { T::default() } #[cfg(not(test))] +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` pub(crate) fn get_toml + Default>(file: &Path) -> T { let contents = t!(fs::read_to_string(file), format!("config file {} not found", file.display())); @@ -253,6 +225,59 @@ pub(crate) fn get_toml + Default>(file: &Path) -> T { } } +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` +/// +/// Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. +#[allow(unused_variables)] +pub(crate) fn set_config_output_dir(output_path: &mut PathBuf) { + #[cfg(test)] + { + *output_path = Path::new( + &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), + ) + .parent() + .unwrap() + .to_path_buf(); + } +} + +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` +pub(crate) fn set_and_return_toml_config + Default>( + src: PathBuf, + config_flag: Option, + cfg_path: &mut Option, +) -> T { + /// Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. + /// + /// Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, + /// but not if `config.toml` hasn't been created. + fn config_path(src: &PathBuf, config_flag: Option) -> Option { + let toml_path = + config_flag.or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); + let using_default_path = toml_path.is_none(); + let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); + if using_default_path && !toml_path.exists() { + toml_path = src.join(toml_path); + } + + if !using_default_path || toml_path.exists() { Some(toml_path) } else { None } + } + + if let Some(toml_path) = config_path(&src, config_flag) { + *cfg_path = Some(toml_path.clone()); + get_toml(&toml_path) + } else { + *cfg_path = None; + T::default() + } +} + +/// Shared helper function to be used in `MinimalConfig::parse` and `bootstrap::config::Config::parse` +pub(crate) fn deserialize_stage0_metadata(stage0_metadata_path: &PathBuf) -> Stage0Metadata { + let stage0_json = t!(std::fs::read(stage0_metadata_path.join("src").join("stage0.json"))); + t!(serde_json::from_slice::(&stage0_json)) +} + fn src() -> Option { // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, // running on a completely machine from where it was compiled. From fedc51a2ebb97029aba965269e2c72d82cd50b64 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Wed, 1 Feb 2023 22:14:08 -0600 Subject: [PATCH 4/4] Also re-download bootstrap if `build_helper` changes --- src/bootstrap/bin/bootstrap-shim.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/bin/bootstrap-shim.rs b/src/bootstrap/bin/bootstrap-shim.rs index d0d044d2373a8..91990bd795e14 100644 --- a/src/bootstrap/bin/bootstrap-shim.rs +++ b/src/bootstrap/bin/bootstrap-shim.rs @@ -25,5 +25,9 @@ fn main() { } fn last_modified_bootstrap_commit(config: &MinimalConfig) -> Option { - config.last_modified_commit(&["src/bootstrap"], "download-bootstrap", true) + config.last_modified_commit( + &["src/bootstrap", "src/tools/build_helper"], + "download-bootstrap", + true, + ) }