From a375799f7a4793adc92b76df59dc0be0bb2a90b8 Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Sat, 27 Aug 2016 21:33:38 -0500 Subject: [PATCH 1/3] rustbuild: smarter `git submodule`-ing With this commit, if one bootstraps rust against system llvm then the src/llvm submodule is not updated/checked-out. This saves considerable network bandwith when starting from a fresh clone of rust-lang/rust as the llvm submodule is never cloned. cc #30107 --- src/bootstrap/lib.rs | 83 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 11 deletions(-) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 9eacb5e3924fa..b1a609aac5c2b 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -32,7 +32,7 @@ use std::cell::RefCell; use std::collections::HashMap; use std::env; use std::fs::{self, File}; -use std::path::{PathBuf, Path}; +use std::path::{Component, PathBuf, Path}; use std::process::Command; use build_helper::{run_silent, output}; @@ -475,12 +475,32 @@ impl Build { /// This will detect if any submodules are out of date an run the necessary /// commands to sync them all with upstream. fn update_submodules(&self) { + struct Submodule<'a> { + path: &'a Path, + state: State, + } + + enum State { + // The submodule may have staged/unstaged changes + MaybeDirty, + // Or could be initialized but never updated + NotInitialized, + // The submodule, itself, has extra commits but those changes haven't been commited to + // the (outer) git repository + OutOfSync, + } + if !self.config.submodules { return } if fs::metadata(self.src.join(".git")).is_err() { return } + let git = || { + let mut cmd = Command::new("git"); + cmd.current_dir(&self.src); + return cmd + }; let git_submodule = || { let mut cmd = Command::new("git"); cmd.current_dir(&self.src).arg("submodule"); @@ -492,19 +512,60 @@ impl Build { // of detecting whether we need to run all the submodule commands // below. let out = output(git_submodule().arg("status")); - if !out.lines().any(|l| l.starts_with("+") || l.starts_with("-")) { - return + let mut submodules = vec![]; + for line in out.lines() { + // NOTE `git submodule status` output looks like this: + // + // -5066b7dcab7e700844b0e2ba71b8af9dc627a59b src/liblibc + // +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/rust-llvm-2016-07-18-1-gb37ef24) + // e058ca661692a8d01f8cf9d35939dfe3105ce968 src/jemalloc (3.6.0-533-ge058ca6) + // + // The first character can be '-', '+' or ' ' and denotes the `State` of the submodule + // Right next to this character is the SHA-1 of the submodule HEAD + // And after that comes the path to the submodule + let path = Path::new(line[1..].split(' ').skip(1).next().unwrap()); + let state = if line.starts_with('-') { + State::NotInitialized + } else if line.starts_with('*') { + State::OutOfSync + } else if line.starts_with(' ') { + State::MaybeDirty + } else { + panic!("unexpected git submodule state: {:?}", line.chars().next()); + }; + + submodules.push(Submodule { path: path, state: state }) } self.run(git_submodule().arg("sync")); - self.run(git_submodule().arg("init")); - self.run(git_submodule().arg("update")); - self.run(git_submodule().arg("update").arg("--recursive")); - self.run(git_submodule().arg("status").arg("--recursive")); - self.run(git_submodule().arg("foreach").arg("--recursive") - .arg("git").arg("clean").arg("-fdx")); - self.run(git_submodule().arg("foreach").arg("--recursive") - .arg("git").arg("checkout").arg(".")); + + for submodule in submodules { + // If using llvm-root then don't touch the llvm submodule. + if submodule.path.components().any(|c| c == Component::Normal("llvm".as_ref())) && + self.config.target_config.get(&self.config.build).and_then(|c| c.llvm_config.as_ref()).is_some() + { + continue + } + + match submodule.state { + State::MaybeDirty => { + // drop staged changes + self.run(git().arg("-C").arg(submodule.path).args(&["reset", "--hard"])); + // drops unstaged changes + self.run(git().arg("-C").arg(submodule.path).args(&["clean", "-fdx"])); + }, + State::NotInitialized => { + self.run(git_submodule().arg("init").arg(submodule.path)); + self.run(git_submodule().arg("update").arg(submodule.path)); + }, + State::OutOfSync => { + // drops submodule commits that weren't reported to the (outer) git repository + self.run(git_submodule().arg("update").arg(submodule.path)); + self.run(git().arg("-C").arg(submodule.path).args(&["reset", "--hard"])); + self.run(git().arg("-C").arg(submodule.path).args(&["clean", "-fdx"])); + }, + } + } } /// Clear out `dir` if `input` is newer. From 5683bf9e2020506003d6528482e0673399f61c7d Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Sat, 27 Aug 2016 22:53:19 -0500 Subject: [PATCH 2/3] don't update the src/jemalloc submodule is jemalloc has been disabled i.e. via the --disable-jemalloc configure flag --- src/bootstrap/lib.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index b1a609aac5c2b..8d0a6a81572f8 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -542,7 +542,14 @@ impl Build { for submodule in submodules { // If using llvm-root then don't touch the llvm submodule. if submodule.path.components().any(|c| c == Component::Normal("llvm".as_ref())) && - self.config.target_config.get(&self.config.build).and_then(|c| c.llvm_config.as_ref()).is_some() + self.config.target_config.get(&self.config.build) + .and_then(|c| c.llvm_config.as_ref()).is_some() + { + continue + } + + if submodule.path.components().any(|c| c == Component::Normal("jemalloc".as_ref())) && + !self.config.use_jemalloc { continue } From 4b5007a1a25c09746307c9e8cabdc6292f969582 Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Sun, 28 Aug 2016 22:13:35 -0500 Subject: [PATCH 3/3] fix tidy error --- src/bootstrap/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 8d0a6a81572f8..ef3fed6fd9433 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -517,7 +517,7 @@ impl Build { // NOTE `git submodule status` output looks like this: // // -5066b7dcab7e700844b0e2ba71b8af9dc627a59b src/liblibc - // +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/rust-llvm-2016-07-18-1-gb37ef24) + // +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/..) // e058ca661692a8d01f8cf9d35939dfe3105ce968 src/jemalloc (3.6.0-533-ge058ca6) // // The first character can be '-', '+' or ' ' and denotes the `State` of the submodule