Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] rustbuild: Build jemalloc and libbacktrace only once #38583

Closed
wants to merge 1 commit into from
Closed

[WIP] rustbuild: Build jemalloc and libbacktrace only once #38583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions src/bootstrap/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -478,7 +478,8 @@ impl Build {
//
// These variables are primarily all read by
// src/bootstrap/bin/{rustc.rs,rustdoc.rs}
cargo.env("RUSTC", self.out.join("bootstrap/debug/rustc"))
cargo.env("RUSTBUILD_NATIVE_DIR", self.native_dir(compiler.host))
.env("RUSTC", self.out.join("bootstrap/debug/rustc"))
.env("RUSTC_REAL", self.compiler_path(compiler))
.env("RUSTC_STAGE", stage.to_string())
.env("RUSTC_DEBUGINFO", self.config.rust_debuginfo.to_string())
Expand Down Expand Up @@ -736,10 +737,15 @@ impl Build {
}
}

/// Directory for libraries built from C/C++ code and shared between stages.
fn native_dir(&self, target: &str) -> PathBuf {
self.out.join(target).join("native")
}

/// Root output directory for rust_test_helpers library compiled for
/// `target`
fn test_helpers_out(&self, target: &str) -> PathBuf {
self.out.join(target).join("rust-test-helpers")
self.native_dir(target).join("rust-test-helpers")
}

/// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic
Expand Down
141 changes: 73 additions & 68 deletions src/liballoc_jemalloc/build.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ extern crate build_helper;
extern crate gcc;

use std::env;
use std::fs;
use std::path::PathBuf;
use std::process::Command;
use build_helper::run;
Expand All @@ -24,8 +25,9 @@ fn main() {

let target = env::var("TARGET").expect("TARGET was not set");
let host = env::var("HOST").expect("HOST was not set");
let build_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
let src_dir = env::current_dir().unwrap();
let build_dir = PathBuf::from(env::var_os("RUSTBUILD_NATIVE_DIR").unwrap()).join("jemalloc");
let _ = fs::create_dir_all(&build_dir);

// FIXME: This is a hack to support building targets that don't
// support jemalloc alongside hosts that do. The jemalloc build is
Expand Down Expand Up @@ -81,77 +83,80 @@ fn main() {
}
}

let mut cmd = Command::new("sh");
cmd.arg(src_dir.join("../jemalloc/configure")
.to_str()
.unwrap()
.replace("C:\\", "/c/")
.replace("\\", "/"))
.current_dir(&build_dir)
.env("CC", compiler.path())
.env("EXTRA_CFLAGS", cflags.clone())
// jemalloc generates Makefile deps using GCC's "-MM" flag. This means
// that GCC will run the preprocessor, and only the preprocessor, over
// jemalloc's source files. If we don't specify CPPFLAGS, then at least
// on ARM that step fails with a "Missing implementation for 32-bit
// atomic operations" error. This is because no "-march" flag will be
// passed to GCC, and then GCC won't define the
// "__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4" macro that jemalloc needs to
// select an atomic operation implementation.
.env("CPPFLAGS", cflags.clone())
.env("AR", &ar)
.env("RANLIB", format!("{} s", ar.display()));
if !build_dir.join("Makefile").metadata().is_ok() {
let mut cmd = Command::new("sh");
cmd.arg(src_dir.join("../jemalloc/configure")
.to_str()
.unwrap()
.replace("C:\\", "/c/")
.replace("\\", "/"))
.current_dir(&build_dir)
.env("CC", compiler.path())
.env("EXTRA_CFLAGS", cflags.clone())
// jemalloc generates Makefile deps using GCC's "-MM" flag. This means
// that GCC will run the preprocessor, and only the preprocessor, over
// jemalloc's source files. If we don't specify CPPFLAGS, then at least
// on ARM that step fails with a "Missing implementation for 32-bit
// atomic operations" error. This is because no "-march" flag will be
// passed to GCC, and then GCC won't define the
// "__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4" macro that jemalloc needs to
// select an atomic operation implementation.
.env("CPPFLAGS", cflags.clone())
.env("AR", &ar)
.env("RANLIB", format!("{} s", ar.display()));

if target.contains("windows") {
// A bit of history here, this used to be --enable-lazy-lock added in
// #14006 which was filed with jemalloc in jemalloc/jemalloc#83 which
// was also reported to MinGW:
//
// http://sourceforge.net/p/mingw-w64/bugs/395/
//
// When updating jemalloc to 4.0, however, it was found that binaries
// would exit with the status code STATUS_RESOURCE_NOT_OWNED indicating
// that a thread was unlocking a mutex it never locked. Disabling this
// "lazy lock" option seems to fix the issue, but it was enabled by
// default for MinGW targets in 13473c7 for jemalloc.
//
// As a result of all that, force disabling lazy lock on Windows, and
// after reading some code it at least *appears* that the initialization
// of mutexes is otherwise ok in jemalloc, so shouldn't cause problems
// hopefully...
//
// tl;dr: make windows behave like other platforms by disabling lazy
// locking, but requires passing an option due to a historical
// default with jemalloc.
cmd.arg("--disable-lazy-lock");
} else if target.contains("ios") {
cmd.arg("--disable-tls");
} else if target.contains("android") {
// We force android to have prefixed symbols because apparently
// replacement of the libc allocator doesn't quite work. When this was
// tested (unprefixed symbols), it was found that the `realpath`
// function in libc would allocate with libc malloc (not jemalloc
// malloc), and then the standard library would free with jemalloc free,
// causing a segfault.
//
// If the test suite passes, however, without symbol prefixes then we
// should be good to go!
cmd.arg("--with-jemalloc-prefix=je_");
cmd.arg("--disable-tls");
} else if target.contains("dragonfly") {
cmd.arg("--with-jemalloc-prefix=je_");
}

if target.contains("windows") {
// A bit of history here, this used to be --enable-lazy-lock added in
// #14006 which was filed with jemalloc in jemalloc/jemalloc#83 which
// was also reported to MinGW:
//
// http://sourceforge.net/p/mingw-w64/bugs/395/
//
// When updating jemalloc to 4.0, however, it was found that binaries
// would exit with the status code STATUS_RESOURCE_NOT_OWNED indicating
// that a thread was unlocking a mutex it never locked. Disabling this
// "lazy lock" option seems to fix the issue, but it was enabled by
// default for MinGW targets in 13473c7 for jemalloc.
//
// As a result of all that, force disabling lazy lock on Windows, and
// after reading some code it at least *appears* that the initialization
// of mutexes is otherwise ok in jemalloc, so shouldn't cause problems
// hopefully...
//
// tl;dr: make windows behave like other platforms by disabling lazy
// locking, but requires passing an option due to a historical
// default with jemalloc.
cmd.arg("--disable-lazy-lock");
} else if target.contains("ios") {
cmd.arg("--disable-tls");
} else if target.contains("android") {
// We force android to have prefixed symbols because apparently
// replacement of the libc allocator doesn't quite work. When this was
// tested (unprefixed symbols), it was found that the `realpath`
// function in libc would allocate with libc malloc (not jemalloc
// malloc), and then the standard library would free with jemalloc free,
// causing a segfault.
//
// If the test suite passes, however, without symbol prefixes then we
// should be good to go!
cmd.arg("--with-jemalloc-prefix=je_");
cmd.arg("--disable-tls");
} else if target.contains("dragonfly") {
cmd.arg("--with-jemalloc-prefix=je_");
}
if cfg!(feature = "debug-jemalloc") {
cmd.arg("--enable-debug");
}

if cfg!(feature = "debug-jemalloc") {
cmd.arg("--enable-debug");
}
// Turn off broken quarantine (see jemalloc/jemalloc#161)
cmd.arg("--disable-fill");
cmd.arg(format!("--host={}", build_helper::gnu_target(&target)));
cmd.arg(format!("--build={}", build_helper::gnu_target(&host)));

// Turn off broken quarantine (see jemalloc/jemalloc#161)
cmd.arg("--disable-fill");
cmd.arg(format!("--host={}", build_helper::gnu_target(&target)));
cmd.arg(format!("--build={}", build_helper::gnu_target(&host)));
run(&mut cmd);
}

run(&mut cmd);
let mut make = Command::new(build_helper::make(&host));
make.current_dir(&build_dir)
.arg("build_lib_static");
Expand Down
37 changes: 21 additions & 16 deletions src/libstd/build.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ extern crate gcc;
extern crate build_helper;

use std::env;
use std::fs;
use std::path::PathBuf;
use std::process::Command;

Expand Down Expand Up @@ -67,7 +68,9 @@ fn main() {

fn build_libbacktrace(host: &str, target: &str) {
let src_dir = env::current_dir().unwrap().join("../libbacktrace");
let build_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
let build_dir =
PathBuf::from(env::var_os("RUSTBUILD_NATIVE_DIR").unwrap()).join("libbacktrace");
let _ = fs::create_dir_all(&build_dir);

println!("cargo:rustc-link-lib=static=backtrace");
println!("cargo:rustc-link-search=native={}/.libs", build_dir.display());
Expand All @@ -89,21 +92,23 @@ fn build_libbacktrace(host: &str, target: &str) {
let ar = build_helper::cc2ar(compiler.path(), target).unwrap();
let cflags = compiler.args().iter().map(|s| s.to_str().unwrap())
.collect::<Vec<_>>().join(" ");
run(Command::new("sh")
.current_dir(&build_dir)
.arg(src_dir.join("configure").to_str().unwrap()
.replace("C:\\", "/c/")
.replace("\\", "/"))
.arg("--with-pic")
.arg("--disable-multilib")
.arg("--disable-shared")
.arg("--disable-host-shared")
.arg(format!("--host={}", build_helper::gnu_target(target)))
.arg(format!("--build={}", build_helper::gnu_target(host)))
.env("CC", compiler.path())
.env("AR", &ar)
.env("RANLIB", format!("{} s", ar.display()))
.env("CFLAGS", cflags));
if !build_dir.join("Makefile").metadata().is_ok() {
run(Command::new("sh")
.current_dir(&build_dir)
.arg(src_dir.join("configure").to_str().unwrap()
.replace("C:\\", "/c/")
.replace("\\", "/"))
.arg("--with-pic")
.arg("--disable-multilib")
.arg("--disable-shared")
.arg("--disable-host-shared")
.arg(format!("--host={}", build_helper::gnu_target(target)))
.arg(format!("--build={}", build_helper::gnu_target(host)))
.env("CC", compiler.path())
.env("AR", &ar)
.env("RANLIB", format!("{} s", ar.display()))
.env("CFLAGS", cflags));
}
run(Command::new(build_helper::make(host))
.current_dir(&build_dir)
.arg(format!("INCDIR={}", src_dir.display()))
Expand Down