diff --git a/configure b/configure index 4f1e8f656ae22..213c105efb1c2 100755 --- a/configure +++ b/configure @@ -637,6 +637,7 @@ opt codegen-tests 1 "run the src/test/codegen tests" opt option-checking 1 "complain about unrecognized options in this configure script" opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)" opt vendor 0 "enable usage of vendored Rust crates" +opt rustc-alloc-frame 0 "enable the frame allocator in rustc" # Optimization and debugging options. These may be overridden by the release channel, etc. opt_nosave optimize 1 "build optimized rust code" @@ -1064,6 +1065,12 @@ else CFG_USING_LIBCPP="0" fi +# Disable jemalloc if using alloc frame. +if [ -n "$CFG_ENABLE_RUSTC_ALLOC_FRAME" ] +then + CFG_DISABLE_JEMALLOC=1 +fi + # Same with jemalloc. save the setting here. if [ -n "$CFG_DISABLE_JEMALLOC" ] then diff --git a/src/Cargo.lock b/src/Cargo.lock index 9cd77e71b82dd..ccaff65ba3333 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -13,6 +13,14 @@ dependencies = [ "core 0.0.0", ] +[[package]] +name = "alloc_frame" +version = "0.0.0" +dependencies = [ + "core 0.0.0", + "libc 0.0.0", +] + [[package]] name = "alloc_jemalloc" version = "0.0.0" @@ -259,6 +267,7 @@ dependencies = [ name = "rustc-main" version = "0.0.0" dependencies = [ + "alloc_frame 0.0.0", "rustc_back 0.0.0", "rustc_driver 0.0.0", "rustdoc 0.0.0", @@ -581,6 +590,7 @@ name = "std" version = "0.0.0" dependencies = [ "alloc 0.0.0", + "alloc_frame 0.0.0", "alloc_jemalloc 0.0.0", "alloc_system 0.0.0", "build_helper 0.1.0", diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 6b86e537b7d22..f7fae9f3bfcd6 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -80,6 +80,7 @@ pub struct Config { pub backtrace: bool, // support for RUST_BACKTRACE // misc + pub rustc_alloc_frame: bool, pub channel: String, pub quiet_tests: bool, // Fallback musl-root for all targets @@ -176,6 +177,7 @@ struct Rust { debuginfo_lines: Option, debug_jemalloc: Option, use_jemalloc: Option, + rustc_alloc_frame: Option, backtrace: Option, default_linker: Option, default_ar: Option, @@ -203,6 +205,7 @@ impl Config { let mut config = Config::default(); config.llvm_optimize = true; config.use_jemalloc = true; + config.rustc_alloc_frame = false; config.backtrace = true; config.rust_optimize = true; config.rust_optimize_tests = true; @@ -300,6 +303,7 @@ impl Config { set(&mut config.use_jemalloc, rust.use_jemalloc); set(&mut config.backtrace, rust.backtrace); set(&mut config.channel, rust.channel.clone()); + set(&mut config.rustc_alloc_frame, rust.rustc_alloc_frame); config.rustc_default_linker = rust.default_linker.clone(); config.rustc_default_ar = rust.default_ar.clone(); config.musl_root = rust.musl_root.clone().map(PathBuf::from); @@ -385,6 +389,7 @@ impl Config { ("DEBUGINFO_LINES", self.rust_debuginfo_lines), ("JEMALLOC", self.use_jemalloc), ("DEBUG_JEMALLOC", self.debug_jemalloc), + ("RUSTC_ALLOC_FRAME", self.rustc_alloc_frame), ("RPATH", self.rust_rpath), ("OPTIMIZE_TESTS", self.rust_optimize_tests), ("DEBUGINFO_TESTS", self.rust_debuginfo_tests), diff --git a/src/bootstrap/config.toml.example b/src/bootstrap/config.toml.example index 5fc095137c793..717083ec75d59 100644 --- a/src/bootstrap/config.toml.example +++ b/src/bootstrap/config.toml.example @@ -137,6 +137,12 @@ # Whether or not jemalloc is built with its debug option set #debug-jemalloc = false +# Whether or not the frame allocator is built and enabled in std +#use-alloc-frame = false + +# Whether or not the frame allocator is built and enabled in rustc +#rustc-use-alloc-frame = false + # Whether or not `panic!`s generate backtraces (RUST_BACKTRACE) #backtrace = true diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 665e0c67b7f6c..2bfd38218aa18 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -621,6 +621,9 @@ impl Build { if self.config.use_jemalloc { features.push_str(" jemalloc"); } + if self.config.rustc_alloc_frame { + features.push_str(" rustc_alloc_frame"); + } return features } diff --git a/src/liballoc_frame/Cargo.toml b/src/liballoc_frame/Cargo.toml new file mode 100644 index 0000000000000..b5c34cc2dd025 --- /dev/null +++ b/src/liballoc_frame/Cargo.toml @@ -0,0 +1,13 @@ +[package] +authors = ["The Rust Project Developers"] +name = "alloc_frame" +version = "0.0.0" + +[lib] +name = "alloc_frame" +path = "lib.rs" +test = false + +[dependencies] +core = { path = "../libcore" } +libc = { path = "../rustc/libc_shim" } diff --git a/src/liballoc_frame/lib.rs b/src/liballoc_frame/lib.rs new file mode 100644 index 0000000000000..8a7a8e77e5e77 --- /dev/null +++ b/src/liballoc_frame/lib.rs @@ -0,0 +1,199 @@ +// Copyright 2015 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![crate_name = "alloc_frame"] +#![crate_type = "rlib"] +#![no_std] +#![allocator] +#![cfg_attr(not(stage0), deny(warnings))] +#![unstable(feature = "alloc_frame", + reason = "this library is unlikely to be stabilized in its current \ + form or name", + issue = "0")] +#![feature(allocator)] +#![feature(const_fn)] +#![feature(staged_api)] +#![feature(libc)] + +extern crate libc; + +use core::ptr; +use core::sync::atomic::{AtomicBool, Ordering}; + +// The minimum alignment guaranteed by the architecture. This value is used to +// add fast paths for low alignment values. In practice, the alignment is a +// constant at the call site and the branch will be optimized out. +#[cfg(all(any(target_arch = "x86", + target_arch = "arm", + target_arch = "mips", + target_arch = "powerpc", + target_arch = "powerpc64", + target_arch = "asmjs", + target_arch = "wasm32")))] +const MIN_ALIGN: usize = 8; +#[cfg(all(any(target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "mips64", + target_arch = "s390x")))] +const MIN_ALIGN: usize = 16; + +// The size of the chunk which is actually allocated +const CHUNK_SIZE: usize = 4096 * 16; +const CHUNK_ALIGN: usize = 4096; + +static mut HEAP: *mut u8 = ptr::null_mut(); +static mut HEAP_LEFT: usize = 0; +static HEAP_MUTEX: AtomicBool = AtomicBool::new(false); + +#[no_mangle] +pub extern "C" fn __rust_allocate(size: usize, align: usize) -> *mut u8 { + let new_align = if align < MIN_ALIGN { MIN_ALIGN } else { align }; + let new_size = (size + new_align - 1) & !(new_align - 1); + + unsafe { + if new_size > CHUNK_SIZE { + return imp::allocate(size, align); + } + + while HEAP_MUTEX.compare_and_swap(false, true, Ordering::SeqCst) {} + + if new_size < HEAP_LEFT { + let p = HEAP; + HEAP = p.offset(new_size as isize); + HEAP_LEFT -= new_size; + HEAP_MUTEX.store(false, Ordering::SeqCst); + return p; + } else { + let p = imp::allocate(CHUNK_SIZE, CHUNK_ALIGN); + HEAP = p.offset(new_size as isize); + HEAP_LEFT = CHUNK_SIZE - new_size; + HEAP_MUTEX.store(false, Ordering::SeqCst); + return p; + } + } +} + +#[no_mangle] +pub extern "C" fn __rust_deallocate(_ptr: *mut u8, _old_size: usize, _align: usize) { +} + +#[no_mangle] +pub extern "C" fn __rust_reallocate(ptr: *mut u8, + old_size: usize, + size: usize, + align: usize) + -> *mut u8 { + let new_ptr = __rust_allocate(size, align); + unsafe { libc::memcpy(new_ptr as *mut _, ptr as *mut _, old_size); } + new_ptr +} + +#[no_mangle] +pub extern "C" fn __rust_reallocate_inplace(_ptr: *mut u8, + old_size: usize, + _size: usize, + _align: usize) + -> usize { + old_size +} + +#[no_mangle] +pub extern "C" fn __rust_usable_size(size: usize, _align: usize) -> usize { + size +} + +#[cfg(any(unix, target_os = "redox"))] +mod imp { + use libc; + use core::ptr; + use MIN_ALIGN; + + pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 { + if align <= MIN_ALIGN { + libc::malloc(size as libc::size_t) as *mut u8 + } else { + aligned_malloc(size, align) + } + } + + #[cfg(any(target_os = "android", target_os = "redox"))] + unsafe fn aligned_malloc(size: usize, align: usize) -> *mut u8 { + // On android we currently target API level 9 which unfortunately + // doesn't have the `posix_memalign` API used below. Instead we use + // `memalign`, but this unfortunately has the property on some systems + // where the memory returned cannot be deallocated by `free`! + // + // Upon closer inspection, however, this appears to work just fine with + // Android, so for this platform we should be fine to call `memalign` + // (which is present in API level 9). Some helpful references could + // possibly be chromium using memalign [1], attempts at documenting that + // memalign + free is ok [2] [3], or the current source of chromium + // which still uses memalign on android [4]. + // + // [1]: https://codereview.chromium.org/10796020/ + // [2]: https://code.google.com/p/android/issues/detail?id=35391 + // [3]: https://bugs.chromium.org/p/chromium/issues/detail?id=138579 + // [4]: https://chromium.googlesource.com/chromium/src/base/+/master/ + // /memory/aligned_memory.cc + libc::memalign(align as libc::size_t, size as libc::size_t) as *mut u8 + } + + #[cfg(not(any(target_os = "android", target_os = "redox")))] + unsafe fn aligned_malloc(size: usize, align: usize) -> *mut u8 { + let mut out = ptr::null_mut(); + let ret = libc::posix_memalign(&mut out, align as libc::size_t, size as libc::size_t); + if ret != 0 { + ptr::null_mut() + } else { + out as *mut u8 + } + } +} + +#[cfg(windows)] +#[allow(bad_style)] +mod imp { + use MIN_ALIGN; + + type LPVOID = *mut u8; + type HANDLE = LPVOID; + type SIZE_T = usize; + type DWORD = u32; + + extern "system" { + fn GetProcessHeap() -> HANDLE; + fn HeapAlloc(hHeap: HANDLE, dwFlags: DWORD, dwBytes: SIZE_T) -> LPVOID; + } + + #[repr(C)] + struct Header(*mut u8); + + unsafe fn get_header<'a>(ptr: *mut u8) -> &'a mut Header { + &mut *(ptr as *mut Header).offset(-1) + } + + unsafe fn align_ptr(ptr: *mut u8, align: usize) -> *mut u8 { + let aligned = ptr.offset((align - (ptr as usize & (align - 1))) as isize); + *get_header(aligned) = Header(ptr); + aligned + } + + pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 { + if align <= MIN_ALIGN { + HeapAlloc(GetProcessHeap(), 0, size as SIZE_T) as *mut u8 + } else { + let ptr = HeapAlloc(GetProcessHeap(), 0, (size + align) as SIZE_T) as *mut u8; + if ptr.is_null() { + return ptr; + } + align_ptr(ptr, align) + } + } +} diff --git a/src/librustc_driver/Cargo.toml b/src/librustc_driver/Cargo.toml index 99d3e155e8936..8c2e7f4c90446 100644 --- a/src/librustc_driver/Cargo.toml +++ b/src/librustc_driver/Cargo.toml @@ -36,3 +36,6 @@ serialize = { path = "../libserialize" } syntax = { path = "../libsyntax" } syntax_ext = { path = "../libsyntax_ext" } syntax_pos = { path = "../libsyntax_pos" } + +[features] +rustc_alloc_frame = ["rustc_metadata/rustc_alloc_frame"] diff --git a/src/librustc_metadata/Cargo.toml b/src/librustc_metadata/Cargo.toml index 6f7f03ca216b9..16a8760e722c8 100644 --- a/src/librustc_metadata/Cargo.toml +++ b/src/librustc_metadata/Cargo.toml @@ -22,3 +22,6 @@ serialize = { path = "../libserialize" } syntax = { path = "../libsyntax" } syntax_ext = { path = "../libsyntax_ext" } syntax_pos = { path = "../libsyntax_pos" } + +[features] +rustc_alloc_frame = [] diff --git a/src/librustc_metadata/creader.rs b/src/librustc_metadata/creader.rs index a9af4118c5957..a47f6c7335632 100644 --- a/src/librustc_metadata/creader.rs +++ b/src/librustc_metadata/creader.rs @@ -820,7 +820,10 @@ impl<'a> CrateLoader<'a> { // * Binaries use jemalloc // * Staticlibs and Rust dylibs use system malloc // * Rust dylibs used as dependencies to rust use jemalloc - let name = if need_lib_alloc && !self.sess.opts.cg.prefer_dynamic { + let name = if cfg!(feature = "rustc_alloc_frame") && cfg!(stage0) { + // HACK to make stage1 with alloc_frame + Symbol::intern(&"alloc_frame") + } else if need_lib_alloc && !self.sess.opts.cg.prefer_dynamic { Symbol::intern(&self.sess.target.target.options.lib_allocation_crate) } else { Symbol::intern(&self.sess.target.target.options.exe_allocation_crate) diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml index d66d2001f2304..66dcc37481f02 100644 --- a/src/librustdoc/Cargo.toml +++ b/src/librustdoc/Cargo.toml @@ -30,3 +30,6 @@ log = { path = "../liblog" } [build-dependencies] build_helper = { path = "../build_helper" } gcc = "0.3.27" + +[features] +rustc_alloc_frame = ["rustc_driver/rustc_alloc_frame", "rustc_metadata/rustc_alloc_frame"] diff --git a/src/libstd/Cargo.toml b/src/libstd/Cargo.toml index fcf84cb716917..34b1fe457dc1e 100644 --- a/src/libstd/Cargo.toml +++ b/src/libstd/Cargo.toml @@ -13,6 +13,7 @@ crate-type = ["dylib", "rlib"] alloc = { path = "../liballoc" } alloc_jemalloc = { path = "../liballoc_jemalloc", optional = true } alloc_system = { path = "../liballoc_system" } +alloc_frame = { path = "../liballoc_frame" } panic_unwind = { path = "../libpanic_unwind", optional = true } panic_abort = { path = "../libpanic_abort" } collections = { path = "../libcollections" } diff --git a/src/rustc/Cargo.toml b/src/rustc/Cargo.toml index dce1a0a8ec858..9f867f246d2b1 100644 --- a/src/rustc/Cargo.toml +++ b/src/rustc/Cargo.toml @@ -17,6 +17,8 @@ path = "rustdoc.rs" rustc_back = { path = "../librustc_back" } rustc_driver = { path = "../librustc_driver" } rustdoc = { path = "../librustdoc" } +alloc_frame = { path = "../liballoc_frame", optional = true } [features] jemalloc = ["rustc_back/jemalloc"] +rustc_alloc_frame = ["alloc_frame", "rustc_driver/rustc_alloc_frame"] diff --git a/src/rustc/rustc.rs b/src/rustc/rustc.rs index bfd01146d2c46..c65a2e8abbf12 100644 --- a/src/rustc/rustc.rs +++ b/src/rustc/rustc.rs @@ -9,7 +9,13 @@ // except according to those terms. #![feature(rustc_private)] +#![feature(staged_api)] +#![cfg_attr(all(feature = "rustc_alloc_frame", stage1), feature(alloc_frame))] extern crate rustc_driver; +// Use the frame allocator to speed up runtime +#[cfg(all(feature = "rustc_alloc_frame", stage1))] +extern crate alloc_frame; + fn main() { rustc_driver::main() } diff --git a/src/rustc/rustdoc.rs b/src/rustc/rustdoc.rs index 6fecd3a27a8a4..1fe87275222cb 100644 --- a/src/rustc/rustdoc.rs +++ b/src/rustc/rustdoc.rs @@ -9,7 +9,13 @@ // except according to those terms. #![feature(rustdoc)] +#![feature(staged_api)] +#![cfg_attr(all(feature = "rustc_alloc_frame", stage1), feature(alloc_frame))] extern crate rustdoc; +// Use the frame allocator to speed up runtime +#[cfg(all(feature = "rustc_alloc_frame", stage1))] +extern crate alloc_frame; + fn main() { rustdoc::main() } diff --git a/src/tools/tidy/src/pal.rs b/src/tools/tidy/src/pal.rs index a5e4e5a4c2672..9314d49bedf87 100644 --- a/src/tools/tidy/src/pal.rs +++ b/src/tools/tidy/src/pal.rs @@ -28,6 +28,7 @@ //! - core may not have platform-specific code //! - liballoc_system may have platform-specific code //! - liballoc_jemalloc may have platform-specific code +//! - liballoc_frame may have platform-specific code //! - libpanic_abort may have platform-specific code //! - libpanic_unwind may have platform-specific code //! - libunwind may have platform-specific code @@ -53,6 +54,7 @@ const EXCEPTION_PATHS: &'static [&'static str] = &[ // std crates "src/liballoc_jemalloc", "src/liballoc_system", + "src/liballoc_frame", "src/liblibc", "src/libpanic_abort", "src/libpanic_unwind",