Rollup merge of #104512 - jyn514:download-ci-llvm-default, r=Mark-Sim…

…ulacrum

Set `download-ci-llvm = "if-available"` by default when `channel = dev`

See rust-lang/compiler-team#566. The motivation for changing the default is to avoid downloading and building LLVM when someone runs `x build` before running `x setup`. The motivation for only doing it on `channel = "dev"` is to avoid breaking distros or users installing from source. It works because `dev` is also the default channel.

The diff looks larger than it is; most of it is moving the `llvm` branch below the `rust` so `config.channel` is set.

r? `@Mark-Simulacrum` cc `@oli-obk` `@bjorn3` `@cuviper`

config.toml.example

@@ -35,17 +35,16 @@ changelog-seen = 2
 # Unless you're developing for a target where Rust CI doesn't build a compiler
 # toolchain or changing LLVM locally, you probably want to set this to true.
 #
-# This is false by default so that distributions don't unexpectedly download
-# LLVM from the internet.
-#
 # All tier 1 targets are currently supported; set this to `"if-available"` if
 # you are not sure whether you're on a tier 1 target.
 #
 # We also currently only support this when building LLVM for the build triple.
 #
 # Note that many of the LLVM options are not currently supported for
 # downloading. Currently only the "assertions" option can be toggled.
-#download-ci-llvm = false
+#
+# Defaults to "if-available" when `channel = "dev"` and "false" otherwise.
+#download-ci-llvm = "if-available"
 
 # Indicates whether LLVM rebuild should be skipped when running bootstrap. If
 # this is `false` then the compiler's LLVM will be rebuilt whenever the built

src/bootstrap/config.rs

@@ -3,6 +3,9 @@
 //! This module implements parsing `config.toml` configuration files to tweak
 //! how the build runs.
 
+#[cfg(test)]
+mod tests;
+
 use std::cell::{Cell, RefCell};
 use std::cmp;
 use std::collections::{HashMap, HashSet};
@@ -696,7 +699,7 @@ define_config! {
     }
 }
 
-#[derive(Deserialize)]
+#[derive(Debug, Deserialize)]
 #[serde(untagged)]
 enum StringOrBool {
     String(String),
@@ -822,6 +825,29 @@ impl Config {
     }
 
     pub fn parse(args: &[String]) -> Config {
+        #[cfg(test)]
+        let get_toml = |_: &_| TomlConfig::default();
+        #[cfg(not(test))]
+        let get_toml = |file: &Path| {
+            let contents =
+                t!(fs::read_to_string(file), format!("config file {} not found", file.display()));
+            // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of
+            // TomlConfig and sub types to be monomorphized 5x by toml.
+            match toml::from_str(&contents)
+                .and_then(|table: toml::Value| TomlConfig::deserialize(table))
+            {
+                Ok(table) => table,
+                Err(err) => {
+                    eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err);
+                    crate::detail_exit(2);
+                }
+            }
+        };
+
+        Self::parse_inner(args, get_toml)
+    }
+
+    fn parse_inner<'a>(args: &[String], get_toml: impl 'a + Fn(&Path) -> TomlConfig) -> Config {
         let flags = Flags::parse(&args);
         let mut config = Config::default_opts();
 
@@ -907,25 +933,6 @@ impl Config {
 
         config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json));
 
-        #[cfg(test)]
-        let get_toml = |_| TomlConfig::default();
-        #[cfg(not(test))]
-        let get_toml = |file: &Path| {
-            let contents =
-                t!(fs::read_to_string(file), format!("config file {} not found", file.display()));
-            // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of
-            // TomlConfig and sub types to be monomorphized 5x by toml.
-            match toml::from_str(&contents)
-                .and_then(|table: toml::Value| TomlConfig::deserialize(table))
-            {
-                Ok(table) => table,
-                Err(err) => {
-                    eprintln!("failed to parse TOML configuration '{}': {}", file.display(), err);
-                    crate::detail_exit(2);
-                }
-            }
-        };
-
         // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory.
         let toml_path = flags
             .config
@@ -1063,6 +1070,79 @@ impl Config {
         let mut optimize = None;
         let mut ignore_git = None;
 
+        if let Some(rust) = toml.rust {
+            debug = rust.debug;
+            debug_assertions = rust.debug_assertions;
+            debug_assertions_std = rust.debug_assertions_std;
+            overflow_checks = rust.overflow_checks;
+            overflow_checks_std = rust.overflow_checks_std;
+            debug_logging = rust.debug_logging;
+            debuginfo_level = rust.debuginfo_level;
+            debuginfo_level_rustc = rust.debuginfo_level_rustc;
+            debuginfo_level_std = rust.debuginfo_level_std;
+            debuginfo_level_tools = rust.debuginfo_level_tools;
+            debuginfo_level_tests = rust.debuginfo_level_tests;
+            config.rust_split_debuginfo = rust
+                .split_debuginfo
+                .as_deref()
+                .map(SplitDebuginfo::from_str)
+                .map(|v| v.expect("invalid value for rust.split_debuginfo"))
+                .unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple));
+            optimize = rust.optimize;
+            ignore_git = rust.ignore_git;
+            config.rust_new_symbol_mangling = rust.new_symbol_mangling;
+            set(&mut config.rust_optimize_tests, rust.optimize_tests);
+            set(&mut config.codegen_tests, rust.codegen_tests);
+            set(&mut config.rust_rpath, rust.rpath);
+            set(&mut config.jemalloc, rust.jemalloc);
+            set(&mut config.test_compare_mode, rust.test_compare_mode);
+            set(&mut config.backtrace, rust.backtrace);
+            set(&mut config.channel, rust.channel);
+            config.description = rust.description;
+            set(&mut config.rust_dist_src, rust.dist_src);
+            set(&mut config.verbose_tests, rust.verbose_tests);
+            // in the case "false" is set explicitly, do not overwrite the command line args
+            if let Some(true) = rust.incremental {
+                config.incremental = true;
+            }
+            set(&mut config.use_lld, rust.use_lld);
+            set(&mut config.lld_enabled, rust.lld);
+            set(&mut config.llvm_tools_enabled, rust.llvm_tools);
+            config.rustc_parallel = rust.parallel_compiler.unwrap_or(false);
+            config.rustc_default_linker = rust.default_linker;
+            config.musl_root = rust.musl_root.map(PathBuf::from);
+            config.save_toolstates = rust.save_toolstates.map(PathBuf::from);
+            set(&mut config.deny_warnings, flags.deny_warnings.or(rust.deny_warnings));
+            set(&mut config.backtrace_on_ice, rust.backtrace_on_ice);
+            set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir);
+            config.rust_thin_lto_import_instr_limit = rust.thin_lto_import_instr_limit;
+            set(&mut config.rust_remap_debuginfo, rust.remap_debuginfo);
+            set(&mut config.control_flow_guard, rust.control_flow_guard);
+            config.llvm_libunwind_default = rust
+                .llvm_libunwind
+                .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind"));
+
+            if let Some(ref backends) = rust.codegen_backends {
+                config.rust_codegen_backends =
+                    backends.iter().map(|s| INTERNER.intern_str(s)).collect();
+            }
+
+            config.rust_codegen_units = rust.codegen_units.map(threads_from_config);
+            config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
+            config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
+            config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
+            config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc);
+
+            config.rust_lto = rust
+                .lto
+                .as_deref()
+                .map(|value| RustcLto::from_str(value).unwrap())
+                .unwrap_or_default();
+        } else {
+            config.rust_profile_use = flags.rust_profile_use;
+            config.rust_profile_generate = flags.rust_profile_generate;
+        }
+
         if let Some(llvm) = toml.llvm {
             match llvm.ccache {
                 Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()),
@@ -1099,13 +1179,17 @@ impl Config {
             config.llvm_polly = llvm.polly.unwrap_or(false);
             config.llvm_clang = llvm.clang.unwrap_or(false);
             config.llvm_build_config = llvm.build_config.clone().unwrap_or(Default::default());
+
+            let asserts = llvm_assertions.unwrap_or(false);
             config.llvm_from_ci = match llvm.download_ci_llvm {
                 Some(StringOrBool::String(s)) => {
                     assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s);
-                    crate::native::is_ci_llvm_available(&config, llvm_assertions.unwrap_or(false))
+                    crate::native::is_ci_llvm_available(&config, asserts)
                 }
                 Some(StringOrBool::Bool(b)) => b,
-                None => false,
+                None => {
+                    config.channel == "dev" && crate::native::is_ci_llvm_available(&config, asserts)
+                }
             };
 
             if config.llvm_from_ci {
@@ -1145,79 +1229,9 @@ impl Config {
                 // the link step) with each stage.
                 config.llvm_link_shared.set(Some(true));
             }
-        }
-
-        if let Some(rust) = toml.rust {
-            debug = rust.debug;
-            debug_assertions = rust.debug_assertions;
-            debug_assertions_std = rust.debug_assertions_std;
-            overflow_checks = rust.overflow_checks;
-            overflow_checks_std = rust.overflow_checks_std;
-            debug_logging = rust.debug_logging;
-            debuginfo_level = rust.debuginfo_level;
-            debuginfo_level_rustc = rust.debuginfo_level_rustc;
-            debuginfo_level_std = rust.debuginfo_level_std;
-            debuginfo_level_tools = rust.debuginfo_level_tools;
-            debuginfo_level_tests = rust.debuginfo_level_tests;
-            config.rust_split_debuginfo = rust
-                .split_debuginfo
-                .as_deref()
-                .map(SplitDebuginfo::from_str)
-                .map(|v| v.expect("invalid value for rust.split_debuginfo"))
-                .unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple));
-            optimize = rust.optimize;
-            ignore_git = rust.ignore_git;
-            config.rust_new_symbol_mangling = rust.new_symbol_mangling;
-            set(&mut config.rust_optimize_tests, rust.optimize_tests);
-            set(&mut config.codegen_tests, rust.codegen_tests);
-            set(&mut config.rust_rpath, rust.rpath);
-            set(&mut config.jemalloc, rust.jemalloc);
-            set(&mut config.test_compare_mode, rust.test_compare_mode);
-            set(&mut config.backtrace, rust.backtrace);
-            set(&mut config.channel, rust.channel);
-            config.description = rust.description;
-            set(&mut config.rust_dist_src, rust.dist_src);
-            set(&mut config.verbose_tests, rust.verbose_tests);
-            // in the case "false" is set explicitly, do not overwrite the command line args
-            if let Some(true) = rust.incremental {
-                config.incremental = true;
-            }
-            set(&mut config.use_lld, rust.use_lld);
-            set(&mut config.lld_enabled, rust.lld);
-            set(&mut config.llvm_tools_enabled, rust.llvm_tools);
-            config.rustc_parallel = rust.parallel_compiler.unwrap_or(false);
-            config.rustc_default_linker = rust.default_linker;
-            config.musl_root = rust.musl_root.map(PathBuf::from);
-            config.save_toolstates = rust.save_toolstates.map(PathBuf::from);
-            set(&mut config.deny_warnings, flags.deny_warnings.or(rust.deny_warnings));
-            set(&mut config.backtrace_on_ice, rust.backtrace_on_ice);
-            set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir);
-            config.rust_thin_lto_import_instr_limit = rust.thin_lto_import_instr_limit;
-            set(&mut config.rust_remap_debuginfo, rust.remap_debuginfo);
-            set(&mut config.control_flow_guard, rust.control_flow_guard);
-            config.llvm_libunwind_default = rust
-                .llvm_libunwind
-                .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind"));
-
-            if let Some(ref backends) = rust.codegen_backends {
-                config.rust_codegen_backends =
-                    backends.iter().map(|s| INTERNER.intern_str(s)).collect();
-            }
-
-            config.rust_codegen_units = rust.codegen_units.map(threads_from_config);
-            config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
-            config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
-            config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
-            config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc);
-
-            config.rust_lto = rust
-                .lto
-                .as_deref()
-                .map(|value| RustcLto::from_str(value).unwrap())
-                .unwrap_or_default();
         } else {
-            config.rust_profile_use = flags.rust_profile_use;
-            config.rust_profile_generate = flags.rust_profile_generate;
+            config.llvm_from_ci =
+                config.channel == "dev" && crate::native::is_ci_llvm_available(&config, false);
         }
 
         if let Some(t) = toml.target {

src/bootstrap/config/tests.rs

@@ -0,0 +1,24 @@
+use super::{Config, TomlConfig};
+use std::path::Path;
+
+fn toml(config: &str) -> impl '_ + Fn(&Path) -> TomlConfig {
+    |&_| toml::from_str(config).unwrap()
+}
+
+fn parse(config: &str) -> Config {
+    Config::parse_inner(&["check".to_owned(), "--config=/does/not/exist".to_owned()], toml(config))
+}
+
+#[test]
+fn download_ci_llvm() {
+    let parse_llvm = |s| parse(s).llvm_from_ci;
+    let if_available = parse_llvm("llvm.download-ci-llvm = \"if-available\"");
+
+    assert!(parse_llvm("llvm.download-ci-llvm = true"));
+    assert!(!parse_llvm("llvm.download-ci-llvm = false"));
+    assert_eq!(parse_llvm(""), if_available);
+    assert_eq!(parse_llvm("rust.channel = \"dev\""), if_available);
+    assert!(!parse_llvm("rust.channel = \"stable\""));
+}
+
+// FIXME: add test for detecting `src` and `out`

src/bootstrap/defaults/config.user.toml

@@ -7,3 +7,7 @@ test-stage = 2
 doc-stage = 2
 # When compiling from source, you usually want all tools.
 extended = true
+
+[llvm]
+# Most users installing from source want to build all parts of the project from source, not just rustc itself.
+download-ci-llvm = false

src/bootstrap/lib.rs

@@ -1639,10 +1639,10 @@ fn chmod(_path: &Path, _perms: u32) {}
 /// If the test is running and code is an error code, it will cause a panic.
 fn detail_exit(code: i32) -> ! {
     // if in test and code is an error code, panic with status code provided
-    if cfg!(test) && code != 0 {
+    if cfg!(test) {
         panic!("status code: {}", code);
     } else {
-        //otherwise,exit with provided status code
+        // otherwise,exit with provided status code
         std::process::exit(code);
     }
 }