From e287044149edf039ffd584557b4bf05507a9f213 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Tue, 14 May 2024 18:47:15 +0000 Subject: [PATCH 1/6] refactor: add rustc-perf submodule to src/tools Currently, it's very challenging to perform a sandboxed `opt-dist` bootstrap because the tool requires `rustc-perf` to be present, but there is no proper management/tracking of it. Instead, a specific commit is hardcoded where it is needed, and a non-checksummed zip is fetched ad-hoc. This happens in two places: `src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile`: ```dockerfile ENV PERF_COMMIT 4f313add609f43e928e98132358e8426ed3969ae RUN curl -LS -o perf.zip https://ci-mirrors.rust-lang.org/rustc/rustc-perf-$PERF_COMMIT.zip && \ unzip perf.zip && \ mv rustc-perf-$PERF_COMMIT rustc-perf && \ rm perf.zip ``` `src/tools/opt-dist/src/main.rs` ```rust // FIXME: add some mechanism for synchronization of this commit SHA with // Linux (which builds rustc-perf in a Dockerfile) // rustc-perf version from 2023-10-22 const PERF_COMMIT: &str = "4f313add609f43e928e98132358e8426ed3969ae"; let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip"); let client = reqwest::blocking::Client::builder() .timeout(Duration::from_secs(60 * 2)) .connect_timeout(Duration::from_secs(60 * 2)) .build()?; let response = retry_action( || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()), "Download rustc-perf archive", 5, )?; ``` This causes a few issues: 1. Maintainers need to be careful to bump PERF_COMMIT in both places every time 2. In order to run `opt-dist` in a sandbox, you need to provide your own `rustc-perf` (https://github.com/rust-lang/rust/pull/125125), but to figure out which commit to provide you need to grep the Dockerfile 3. Even if you manage to provide the correct `rustc-perf`, its dependencies are not included in the `vendor/` dir created during `dist`, so it will fail to build from the published source tarballs 4. It is hard to provide any level of automation around updating the `rustc-perf` in use, leading to staleness Fundamentally, this means `rustc-src` tarballs no longer contain everything you need to bootstrap Rust, and packagers hoping to leverage `opt-dist` need to go out of their way to keep track of this "hidden" dependency on `rustc-perf`. This change adds rustc-perf as a git submodule, pinned to the current `PERF_COMMIT` 4f313add609f43e928e98132358e8426ed3969ae. Subsequent commits ensure the submodule is initialized when necessary, and make use of it in `opt-dist`. --- .gitmodules | 4 ++++ rustfmt.toml | 1 + src/tools/rustc-perf | 1 + src/tools/tidy/config/black.toml | 1 + src/tools/tidy/config/ruff.toml | 2 ++ src/tools/tidy/src/walk.rs | 1 + 6 files changed, 10 insertions(+) create mode 160000 src/tools/rustc-perf diff --git a/.gitmodules b/.gitmodules index 75faaba71517b..9ad207a0d5226 100644 --- a/.gitmodules +++ b/.gitmodules @@ -43,3 +43,7 @@ path = library/backtrace url = https://github.com/rust-lang/backtrace-rs.git shallow = true +[submodule "src/tools/rustc-perf"] + path = src/tools/rustc-perf + url = https://github.com/rust-lang/rustc-perf.git + shallow = true diff --git a/rustfmt.toml b/rustfmt.toml index 850d01ea7cb81..ef56059feb1c7 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -40,6 +40,7 @@ ignore = [ "src/tools/clippy", "src/tools/miri", "src/tools/rust-analyzer", + "src/tools/rustc-perf", "src/tools/rustfmt", # these are ignored by a standard cargo fmt run diff --git a/src/tools/rustc-perf b/src/tools/rustc-perf new file mode 160000 index 0000000000000..4f313add609f4 --- /dev/null +++ b/src/tools/rustc-perf @@ -0,0 +1 @@ +Subproject commit 4f313add609f43e928e98132358e8426ed3969ae diff --git a/src/tools/tidy/config/black.toml b/src/tools/tidy/config/black.toml index 51a722979f559..e73847a93ba49 100644 --- a/src/tools/tidy/config/black.toml +++ b/src/tools/tidy/config/black.toml @@ -11,5 +11,6 @@ extend-exclude = """(\ src/doc/edition-guide/|\ src/llvm-project/|\ src/doc/embedded-book/|\ + src/tools/rustc-perf/|\ library/backtrace/ )""" diff --git a/src/tools/tidy/config/ruff.toml b/src/tools/tidy/config/ruff.toml index cf08c62648b3e..cf89ffd9ac73f 100644 --- a/src/tools/tidy/config/ruff.toml +++ b/src/tools/tidy/config/ruff.toml @@ -26,6 +26,7 @@ extend-exclude = [ "src/llvm-project/", "src/doc/embedded-book/", "library/backtrace/", + "src/tools/rustc-perf/", # Hack: CI runs from a subdirectory under the main checkout "../src/doc/nomicon/", "../src/tools/cargo/", @@ -38,4 +39,5 @@ extend-exclude = [ "../src/llvm-project/", "../src/doc/embedded-book/", "../library/backtrace/", + "../src/tools/rustc-perf/", ] diff --git a/src/tools/tidy/src/walk.rs b/src/tools/tidy/src/walk.rs index 851c21f1c0f05..f68b7675c769f 100644 --- a/src/tools/tidy/src/walk.rs +++ b/src/tools/tidy/src/walk.rs @@ -18,6 +18,7 @@ pub fn filter_dirs(path: &Path) -> bool { "src/tools/clippy", "src/tools/miri", "src/tools/rust-analyzer", + "src/tools/rustc-perf", "src/tools/rustfmt", "src/doc/book", "src/doc/edition-guide", From 4c075c622cd929c1561fcbfc369813aceefb86f1 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Tue, 14 May 2024 21:18:31 +0000 Subject: [PATCH 2/6] refactor(bootstrap): update rustc-perf submodule when building opt-dist This avoids having normal builds pay the cost of initializing that submodule, while still ensuring it's available whenever `opt-dist` is built. Note that, at this point, `opt-dist` will not yet use the submodule, that will be handled in a subsequent commit. --- src/bootstrap/src/core/build_steps/tool.rs | 41 ++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 21344a4224e40..2db3f8f79364e 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -1,6 +1,6 @@ use std::env; use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::Command; use crate::core::build_steps::compile; @@ -313,10 +313,47 @@ bootstrap_tool!( SuggestTests, "src/tools/suggest-tests", "suggest-tests"; GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys"; RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test"; - OptimizedDist, "src/tools/opt-dist", "opt-dist"; CoverageDump, "src/tools/coverage-dump", "coverage-dump"; ); +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct OptimizedDist { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for OptimizedDist { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/opt-dist") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(OptimizedDist { + compiler: run.builder.compiler(0, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + // We need to ensure the rustc-perf submodule is initialized when building opt-dist since + // the tool requires it to be in place to run. + builder.update_submodule(Path::new("src/tools/rustc-perf")); + + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "opt-dist", + mode: Mode::ToolBootstrap, + path: "src/tools/opt-dist", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + }) + } +} + #[derive(Debug, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] pub struct ErrorIndex { pub compiler: Compiler, From 95c05c3e376376bb5210b62207ab3d9d01468996 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Wed, 15 May 2024 18:48:26 +0000 Subject: [PATCH 3/6] refactor(opt-dist): use rustc-perf from rustc checkout This replaces the hardcoded rustc-perf commit and ad-hoc downloading and unpacking of its zipped source with defaulting to use the new rustc-perf submodule. While it would be nice to make `opt-dist` able to initialize the submodule automatically when pointing to a Rust checkout _other_ than the one opt-dist was built in, that would require a bigger refactor that moved `update_submodule`, from bootstrap, into build_helper. Regardless, I imagine it must be quite rare to use `opt-dist` with a checkout that is neither from a rust-src tarball (which will contain the submodule), nor the checkout opt-dist itself was built (bootstrap will update the submodule when opt-dist is built). --- Cargo.lock | 390 --------------------------------- src/tools/opt-dist/Cargo.toml | 2 - src/tools/opt-dist/src/main.rs | 55 ++--- src/tools/tidy/src/deps.rs | 2 - 4 files changed, 14 insertions(+), 435 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5afd71a4bed67..3a4f028e695f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -825,16 +825,6 @@ dependencies = [ "rand_xorshift", ] -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -1240,15 +1230,6 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" -[[package]] -name = "encoding_rs" -version = "0.8.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" -dependencies = [ - "cfg-if", -] - [[package]] name = "env_filter" version = "0.1.0" @@ -1399,21 +1380,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1637,25 +1603,6 @@ dependencies = [ "serde", ] -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "handlebars" version = "5.1.2" @@ -1766,40 +1713,6 @@ dependencies = [ "syn 2.0.64", ] -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "humansize" version = "2.1.3" @@ -1815,43 +1728,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" -[[package]] -name = "hyper" -version = "0.14.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "iana-time-zone" version = "0.1.60" @@ -2095,12 +1971,6 @@ dependencies = [ "unic-langid", ] -[[package]] -name = "ipnet" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" - [[package]] name = "is_terminal_polyfill" version = "1.70.0" @@ -2529,17 +2399,6 @@ dependencies = [ "rustc-std-workspace-core", ] -[[package]] -name = "mio" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.48.0", -] - [[package]] name = "miow" version = "0.6.0" @@ -2577,24 +2436,6 @@ dependencies = [ name = "miropt-test-tools" version = "0.1.0" -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -2758,32 +2599,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "openssl" -version = "0.10.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" -dependencies = [ - "bitflags 2.5.0", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.64", -] - [[package]] name = "openssl-probe" version = "0.1.5" @@ -2817,7 +2632,6 @@ dependencies = [ "humansize", "humantime", "log", - "reqwest", "serde", "serde_json", "sysinfo", @@ -2825,7 +2639,6 @@ dependencies = [ "tar", "tempfile", "xz2", - "zip", ] [[package]] @@ -3410,46 +3223,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "reqwest" -version = "0.11.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" -dependencies = [ - "base64", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-tls", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "system-configuration", - "tokio", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg", -] - [[package]] name = "rls" version = "2.0.0" @@ -5064,15 +4837,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64", -] - [[package]] name = "rustversion" version = "1.0.17" @@ -5137,29 +4901,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "security-framework" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" -dependencies = [ - "bitflags 2.5.0", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "self_cell" version = "0.10.3" @@ -5225,18 +4966,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - [[package]] name = "sha1" version = "0.10.6" @@ -5516,12 +5245,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - [[package]] name = "synstructure" version = "0.13.1" @@ -5556,27 +5279,6 @@ dependencies = [ "test", ] -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "tabled" version = "0.15.0" @@ -5812,34 +5514,7 @@ checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", - "libc", - "mio", "pin-project-lite", - "socket2", - "windows-sys 0.48.0", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", ] [[package]] @@ -5891,12 +5566,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d" -[[package]] -name = "tower-service" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" - [[package]] name = "tracing" version = "0.1.37" @@ -5982,12 +5651,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - [[package]] name = "twox-hash" version = "1.6.3" @@ -6314,15 +5977,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -6359,18 +6013,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.92" @@ -6419,16 +6061,6 @@ dependencies = [ "semver", ] -[[package]] -name = "web-sys" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "winapi" version = "0.3.9" @@ -6647,16 +6279,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "writeable" version = "0.5.4" @@ -6778,15 +6400,3 @@ dependencies = [ "quote", "syn 2.0.64", ] - -[[package]] -name = "zip" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" -dependencies = [ - "byteorder", - "crc32fast", - "crossbeam-utils", - "flate2", -] diff --git a/src/tools/opt-dist/Cargo.toml b/src/tools/opt-dist/Cargo.toml index 1ff410e723afe..88e8640d56abb 100644 --- a/src/tools/opt-dist/Cargo.toml +++ b/src/tools/opt-dist/Cargo.toml @@ -13,8 +13,6 @@ humansize = "2" sysinfo = { version = "0.30", default-features = false } fs_extra = "1" camino = "1" -reqwest = { version = "0.11", features = ["blocking"] } -zip = { version = "0.6", default-features = false, features = ["deflate"] } tar = "0.4" xz = { version = "0.1", package = "xz2" } serde = { version = "1", features = ["derive"] } diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index bd0a3815855d5..a709076f24580 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -3,10 +3,7 @@ use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; use clap::Parser; use log::LevelFilter; -use std::io::Cursor; -use std::time::Duration; use utils::io; -use zip::ZipArchive; use crate::environment::{Environment, EnvironmentBuilder}; use crate::exec::{cmd, Bootstrap}; @@ -17,9 +14,9 @@ use crate::training::{ rustc_benchmarks, }; use crate::utils::artifact_size::print_binary_sizes; -use crate::utils::io::{copy_directory, move_directory, reset_directory}; +use crate::utils::io::{copy_directory, reset_directory}; use crate::utils::{ - clear_llvm_files, format_env_variables, print_free_disk_space, retry_action, with_log_group, + clear_llvm_files, format_env_variables, print_free_disk_space, with_log_group, write_timer_to_summary, }; @@ -69,7 +66,12 @@ enum EnvironmentCmd { #[arg(long, default_value = "opt-artifacts")] artifact_dir: Utf8PathBuf, - /// Checkout directory of `rustc-perf`, it will be fetched automatically if unspecified. + /// Checkout directory of `rustc-perf`. + /// + /// If unspecified, defaults to the rustc-perf submodule in the rustc checkout dir + /// (`src/tools/rustc-perf`), which should have been initialized when building this tool. + // FIXME: Move update_submodule into build_helper, that way we can also ensure the submodule + // is updated when _running_ opt-dist, rather than building. #[arg(long)] rustc_perf_checkout_dir: Option, @@ -146,8 +148,6 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> .host_llvm_dir(Utf8PathBuf::from("/rustroot")) .artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")) .build_dir(checkout_dir.join("obj")) - // /tmp/rustc-perf comes from the x64 dist Dockerfile - .prebuilt_rustc_perf(Some(Utf8PathBuf::from("/tmp/rustc-perf"))) .shared_llvm(true) .use_bolt(true) .skipped_tests(vec![ @@ -191,9 +191,12 @@ fn execute_pipeline( ) -> anyhow::Result<()> { reset_directory(&env.artifact_dir())?; - with_log_group("Building rustc-perf", || match env.prebuilt_rustc_perf() { - Some(dir) => copy_rustc_perf(env, &dir), - None => download_rustc_perf(env), + with_log_group("Building rustc-perf", || { + let rustc_perf_checkout_dir = match env.prebuilt_rustc_perf() { + Some(dir) => dir, + None => env.checkout_path().join("src").join("tools").join("rustc-perf"), + }; + copy_rustc_perf(env, &rustc_perf_checkout_dir) })?; // Stage 1: Build PGO instrumented rustc @@ -409,36 +412,6 @@ fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> { build_rustc_perf(env) } -// Download and build rustc-perf into the given environment. -fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> { - reset_directory(&env.rustc_perf_dir())?; - - // FIXME: add some mechanism for synchronization of this commit SHA with - // Linux (which builds rustc-perf in a Dockerfile) - // rustc-perf version from 2023-10-22 - const PERF_COMMIT: &str = "4f313add609f43e928e98132358e8426ed3969ae"; - - let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip"); - let client = reqwest::blocking::Client::builder() - .timeout(Duration::from_secs(60 * 2)) - .connect_timeout(Duration::from_secs(60 * 2)) - .build()?; - let response = retry_action( - || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()), - "Download rustc-perf archive", - 5, - )?; - - let mut archive = ZipArchive::new(Cursor::new(response))?; - archive.extract(env.rustc_perf_dir())?; - move_directory( - &env.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")), - &env.rustc_perf_dir(), - )?; - - build_rustc_perf(env) -} - fn build_rustc_perf(env: &Environment) -> anyhow::Result<()> { cmd(&[env.cargo_stage_0().as_str(), "build", "-p", "collector"]) .workdir(&env.rustc_perf_dir()) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 58f4455ae2f4b..efcd2a181ffa2 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -81,12 +81,10 @@ const EXCEPTIONS: ExceptionList = &[ ("ar_archive_writer", "Apache-2.0 WITH LLVM-exception"), // rustc ("colored", "MPL-2.0"), // rustfmt ("dissimilar", "Apache-2.0"), // rustdoc, rustc_lexer (few tests) via expect-test, (dev deps) - ("encoding_rs", "(Apache-2.0 OR MIT) AND BSD-3-Clause"), // opt-dist ("fluent-langneg", "Apache-2.0"), // rustc (fluent translations) ("fortanix-sgx-abi", "MPL-2.0"), // libstd but only for `sgx` target. FIXME: this dependency violates the documentation comment above. ("instant", "BSD-3-Clause"), // rustc_driver/tracing-subscriber/parking_lot ("mdbook", "MPL-2.0"), // mdbook - ("openssl", "Apache-2.0"), // opt-dist ("option-ext", "MPL-2.0"), // cargo-miri (via `directories`) ("rustc_apfloat", "Apache-2.0 WITH LLVM-exception"), // rustc (license is the same as LLVM uses) ("ryu", "Apache-2.0 OR BSL-1.0"), // BSL is not acceptble, but we use it under Apache-2.0 // cargo/... (because of serde) From d9a7868ceb960dba9c33c7801bfa1a9996ffe775 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Wed, 15 May 2024 20:19:49 +0000 Subject: [PATCH 4/6] refactor(ci/docker/dist-x86_64-linux): remove rustc-perf downloading It is now available as a submodule in src/tools/rustc-perf, and is initialized when building opt-dist --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index fe84c23a11c14..4e1aae9822135 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -58,14 +58,6 @@ COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/ RUN ./build-clang.sh ENV CC=clang CXX=clang++ -# rustc-perf version from 2023-10-22 -# Should also be changed in the opt-dist tool for other environments. -ENV PERF_COMMIT 4f313add609f43e928e98132358e8426ed3969ae -RUN curl -LS -o perf.zip https://ci-mirrors.rust-lang.org/rustc/rustc-perf-$PERF_COMMIT.zip && \ - unzip perf.zip && \ - mv rustc-perf-$PERF_COMMIT rustc-perf && \ - rm perf.zip - COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh From 4550fb52d172101115b5c31a41f7d9b171d6c10c Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Wed, 15 May 2024 20:56:50 +0000 Subject: [PATCH 5/6] chore(bootstrap/core/build_steps/dist): add fixme for duplicated vendor logic --- src/bootstrap/src/core/build_steps/dist.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 1f006e1453f4b..2feb59d3aad2f 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1010,6 +1010,9 @@ impl Step for PlainSourceTarball { if builder.rust_info().is_managed_git_subrepository() || builder.rust_info().is_from_tarball() { + // FIXME: This code looks _very_ similar to what we have in `src/core/build_steps/vendor.rs` + // perhaps it should be removed in favor of making `dist` perform the `vendor` step? + // Ensure we have all submodules from src and other directories checked out. for submodule in builder.get_all_submodules() { builder.update_submodule(Path::new(submodule)); From e253718ce445c8f52a55160aaeef42510aa0110b Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Costa Date: Wed, 15 May 2024 20:58:19 +0000 Subject: [PATCH 6/6] refactor(bootstrap/core/build_steps/dist): vendor dependencies needed for opt-dist --- src/bootstrap/src/core/build_steps/dist.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 2feb59d3aad2f..91039d0c8dcc1 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1032,6 +1032,10 @@ impl Step for PlainSourceTarball { .arg(builder.src.join("./compiler/rustc_codegen_gcc/Cargo.toml")) .arg("--sync") .arg(builder.src.join("./src/bootstrap/Cargo.toml")) + .arg("--sync") + .arg(builder.src.join("./src/tools/opt-dist/Cargo.toml")) + .arg("--sync") + .arg(builder.src.join("./src/tools/rustc-perf/Cargo.toml")) // Will read the libstd Cargo.toml // which uses the unstable `public-dependency` feature. .env("RUSTC_BOOTSTRAP", "1")