[WIP] Bump allocator for rustc

configure

@@ -637,6 +637,7 @@ opt codegen-tests 1 "run the src/test/codegen tests"
 opt option-checking 1 "complain about unrecognized options in this configure script"
 opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)"
 opt vendor 0 "enable usage of vendored Rust crates"
+opt rustc-alloc-frame 0 "enable the frame allocator in rustc"
 
 # Optimization and debugging options. These may be overridden by the release channel, etc.
 opt_nosave optimize 1 "build optimized rust code"
@@ -1064,6 +1065,12 @@ else
     CFG_USING_LIBCPP="0"
 fi
 
+# Disable jemalloc if using alloc frame.
+if [ -n "$CFG_ENABLE_RUSTC_ALLOC_FRAME" ]
+then
+    CFG_DISABLE_JEMALLOC=1
+fi
+
 # Same with jemalloc.  save the setting here.
 if [ -n "$CFG_DISABLE_JEMALLOC" ]
 then

src/bootstrap/config.rs

@@ -80,6 +80,7 @@ pub struct Config {
     pub backtrace: bool, // support for RUST_BACKTRACE
 
     // misc
+    pub rustc_alloc_frame: bool,
     pub channel: String,
     pub quiet_tests: bool,
     // Fallback musl-root for all targets
@@ -176,6 +177,7 @@ struct Rust {
     debuginfo_lines: Option<bool>,
     debug_jemalloc: Option<bool>,
     use_jemalloc: Option<bool>,
+    rustc_alloc_frame: Option<bool>,
     backtrace: Option<bool>,
     default_linker: Option<String>,
     default_ar: Option<String>,
@@ -203,6 +205,7 @@ impl Config {
         let mut config = Config::default();
         config.llvm_optimize = true;
         config.use_jemalloc = true;
+        config.rustc_alloc_frame = false;
         config.backtrace = true;
         config.rust_optimize = true;
         config.rust_optimize_tests = true;
@@ -300,6 +303,7 @@ impl Config {
             set(&mut config.use_jemalloc, rust.use_jemalloc);
             set(&mut config.backtrace, rust.backtrace);
             set(&mut config.channel, rust.channel.clone());
+            set(&mut config.rustc_alloc_frame, rust.rustc_alloc_frame);
             config.rustc_default_linker = rust.default_linker.clone();
             config.rustc_default_ar = rust.default_ar.clone();
             config.musl_root = rust.musl_root.clone().map(PathBuf::from);
@@ -385,6 +389,7 @@ impl Config {
                 ("DEBUGINFO_LINES", self.rust_debuginfo_lines),
                 ("JEMALLOC", self.use_jemalloc),
                 ("DEBUG_JEMALLOC", self.debug_jemalloc),
+                ("RUSTC_ALLOC_FRAME", self.rustc_alloc_frame),
                 ("RPATH", self.rust_rpath),
                 ("OPTIMIZE_TESTS", self.rust_optimize_tests),
                 ("DEBUGINFO_TESTS", self.rust_debuginfo_tests),

src/bootstrap/config.toml.example

@@ -137,6 +137,12 @@
 # Whether or not jemalloc is built with its debug option set
 #debug-jemalloc = false
 
+# Whether or not the frame allocator is built and enabled in std
+#use-alloc-frame = false
+
+# Whether or not the frame allocator is built and enabled in rustc
+#rustc-use-alloc-frame = false
+
 # Whether or not `panic!`s generate backtraces (RUST_BACKTRACE)
 #backtrace = true
 

src/bootstrap/lib.rs

@@ -621,6 +621,9 @@ impl Build {
         if self.config.use_jemalloc {
             features.push_str(" jemalloc");
         }
+        if self.config.rustc_alloc_frame {
+            features.push_str(" rustc_alloc_frame");
+        }
         return features
     }
 

src/liballoc_frame/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "alloc_frame"
+version = "0.0.0"
+
+[lib]
+name = "alloc_frame"
+path = "lib.rs"
+test = false
+
+[dependencies]
+core = { path = "../libcore" }
+libc = { path = "../rustc/libc_shim" }

src/liballoc_frame/lib.rs

@@ -0,0 +1,199 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![crate_name = "alloc_frame"]
+#![crate_type = "rlib"]
+#![no_std]
+#![allocator]
+#![cfg_attr(not(stage0), deny(warnings))]
+#![unstable(feature = "alloc_frame",
+            reason = "this library is unlikely to be stabilized in its current \
+                      form or name",
+            issue = "0")]
+#![feature(allocator)]
+#![feature(const_fn)]
+#![feature(staged_api)]
+#![feature(libc)]
+
+extern crate libc;
+
+use core::ptr;
+use core::sync::atomic::{AtomicBool, Ordering};
+
+// The minimum alignment guaranteed by the architecture. This value is used to
+// add fast paths for low alignment values. In practice, the alignment is a
+// constant at the call site and the branch will be optimized out.
+#[cfg(all(any(target_arch = "x86",
+              target_arch = "arm",
+              target_arch = "mips",
+              target_arch = "powerpc",
+              target_arch = "powerpc64",
+              target_arch = "asmjs",
+              target_arch = "wasm32")))]
+const MIN_ALIGN: usize = 8;
+#[cfg(all(any(target_arch = "x86_64",
+              target_arch = "aarch64",
+              target_arch = "mips64",
+              target_arch = "s390x")))]
+const MIN_ALIGN: usize = 16;
+
+// The size of the chunk which is actually allocated
+const CHUNK_SIZE: usize = 4096 * 16;
+const CHUNK_ALIGN: usize = 4096;
+
+static mut HEAP: *mut u8 = ptr::null_mut();
+static mut HEAP_LEFT: usize = 0;
+static HEAP_MUTEX: AtomicBool = AtomicBool::new(false);
+
+#[no_mangle]
+pub extern "C" fn __rust_allocate(size: usize, align: usize) -> *mut u8 {
+    let new_align = if align < MIN_ALIGN { MIN_ALIGN } else { align };
+    let new_size = (size + new_align - 1) & !(new_align - 1);
+
+    unsafe {
+        if new_size > CHUNK_SIZE {
+            return imp::allocate(size, align);
+        }
+
+        while HEAP_MUTEX.compare_and_swap(false, true, Ordering::SeqCst) {}
+
+        if new_size < HEAP_LEFT {
+            let p = HEAP;
+            HEAP = p.offset(new_size as isize);
+            HEAP_LEFT -= new_size;
+            HEAP_MUTEX.store(false, Ordering::SeqCst);
+            return p;
+        } else {
+            let p = imp::allocate(CHUNK_SIZE, CHUNK_ALIGN);
+            HEAP = p.offset(new_size as isize);
+            HEAP_LEFT = CHUNK_SIZE - new_size;
+            HEAP_MUTEX.store(false, Ordering::SeqCst);
+            return p;
+        }
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn __rust_deallocate(_ptr: *mut u8, _old_size: usize, _align: usize) {
+}
+
+#[no_mangle]
+pub extern "C" fn __rust_reallocate(ptr: *mut u8,
+                                    old_size: usize,
+                                    size: usize,
+                                    align: usize)
+                                    -> *mut u8 {
+    let new_ptr = __rust_allocate(size, align);
+    unsafe { libc::memcpy(new_ptr as *mut _, ptr as *mut _, old_size); } 
+    new_ptr
+}
+
+#[no_mangle]
+pub extern "C" fn __rust_reallocate_inplace(_ptr: *mut u8,
+                                            old_size: usize,
+                                            _size: usize,
+                                            _align: usize)
+                                            -> usize {
+    old_size
+}
+
+#[no_mangle]
+pub extern "C" fn __rust_usable_size(size: usize, _align: usize) -> usize {
+    size
+}
+
+#[cfg(any(unix, target_os = "redox"))]
+mod imp {
+    use libc;
+    use core::ptr;
+    use MIN_ALIGN;
+
+    pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 {
+        if align <= MIN_ALIGN {
+            libc::malloc(size as libc::size_t) as *mut u8
+        } else {
+            aligned_malloc(size, align)
+        }
+    }
+
+    #[cfg(any(target_os = "android", target_os = "redox"))]
+    unsafe fn aligned_malloc(size: usize, align: usize) -> *mut u8 {
+        // On android we currently target API level 9 which unfortunately
+        // doesn't have the `posix_memalign` API used below. Instead we use
+        // `memalign`, but this unfortunately has the property on some systems
+        // where the memory returned cannot be deallocated by `free`!
+        //
+        // Upon closer inspection, however, this appears to work just fine with
+        // Android, so for this platform we should be fine to call `memalign`
+        // (which is present in API level 9). Some helpful references could
+        // possibly be chromium using memalign [1], attempts at documenting that
+        // memalign + free is ok [2] [3], or the current source of chromium
+        // which still uses memalign on android [4].
+        //
+        // [1]: https://codereview.chromium.org/10796020/
+        // [2]: https://code.google.com/p/android/issues/detail?id=35391
+        // [3]: https://bugs.chromium.org/p/chromium/issues/detail?id=138579
+        // [4]: https://chromium.googlesource.com/chromium/src/base/+/master/
+        //                                       /memory/aligned_memory.cc
+        libc::memalign(align as libc::size_t, size as libc::size_t) as *mut u8
+    }
+
+    #[cfg(not(any(target_os = "android", target_os = "redox")))]
+    unsafe fn aligned_malloc(size: usize, align: usize) -> *mut u8 {
+        let mut out = ptr::null_mut();
+        let ret = libc::posix_memalign(&mut out, align as libc::size_t, size as libc::size_t);
+        if ret != 0 {
+            ptr::null_mut()
+        } else {
+            out as *mut u8
+        }
+    }
+}
+
+#[cfg(windows)]
+#[allow(bad_style)]
+mod imp {
+    use MIN_ALIGN;
+
+    type LPVOID = *mut u8;
+    type HANDLE = LPVOID;
+    type SIZE_T = usize;
+    type DWORD = u32;
+
+    extern "system" {
+        fn GetProcessHeap() -> HANDLE;
+        fn HeapAlloc(hHeap: HANDLE, dwFlags: DWORD, dwBytes: SIZE_T) -> LPVOID;
+    }
+
+    #[repr(C)]
+    struct Header(*mut u8);
+
+    unsafe fn get_header<'a>(ptr: *mut u8) -> &'a mut Header {
+        &mut *(ptr as *mut Header).offset(-1)
+    }
+
+    unsafe fn align_ptr(ptr: *mut u8, align: usize) -> *mut u8 {
+        let aligned = ptr.offset((align - (ptr as usize & (align - 1))) as isize);
+        *get_header(aligned) = Header(ptr);
+        aligned
+    }
+
+    pub unsafe fn allocate(size: usize, align: usize) -> *mut u8 {
+        if align <= MIN_ALIGN {
+            HeapAlloc(GetProcessHeap(), 0, size as SIZE_T) as *mut u8
+        } else {
+            let ptr = HeapAlloc(GetProcessHeap(), 0, (size + align) as SIZE_T) as *mut u8;
+            if ptr.is_null() {
+                return ptr;
+            }
+            align_ptr(ptr, align)
+        }
+    }
+}

src/librustc_driver/Cargo.toml

@@ -36,3 +36,6 @@ serialize = { path = "../libserialize" }
 syntax = { path = "../libsyntax" }
 syntax_ext = { path = "../libsyntax_ext" }
 syntax_pos = { path = "../libsyntax_pos" }
+
+[features]
+rustc_alloc_frame = ["rustc_metadata/rustc_alloc_frame"]

src/librustc_metadata/Cargo.toml

@@ -22,3 +22,6 @@ serialize = { path = "../libserialize" }
 syntax = { path = "../libsyntax" }
 syntax_ext = { path = "../libsyntax_ext" }
 syntax_pos = { path = "../libsyntax_pos" }
+
+[features]
+rustc_alloc_frame = []

src/librustc_metadata/creader.rs

@@ -820,7 +820,10 @@ impl<'a> CrateLoader<'a> {
         // * Binaries use jemalloc
         // * Staticlibs and Rust dylibs use system malloc
         // * Rust dylibs used as dependencies to rust use jemalloc
-        let name = if need_lib_alloc && !self.sess.opts.cg.prefer_dynamic {
+        let name = if cfg!(feature = "rustc_alloc_frame") && cfg!(stage0) {
+            // HACK to make stage1 with alloc_frame
+            Symbol::intern(&"alloc_frame")
+        } else if need_lib_alloc && !self.sess.opts.cg.prefer_dynamic {
             Symbol::intern(&self.sess.target.target.options.lib_allocation_crate)
         } else {
             Symbol::intern(&self.sess.target.target.options.exe_allocation_crate)

src/librustdoc/Cargo.toml

@@ -30,3 +30,6 @@ log = { path = "../liblog" }
 [build-dependencies]
 build_helper = { path = "../build_helper" }
 gcc = "0.3.27"
+
+[features]
+rustc_alloc_frame = ["rustc_driver/rustc_alloc_frame", "rustc_metadata/rustc_alloc_frame"]

src/libstd/Cargo.toml

@@ -13,6 +13,7 @@ crate-type = ["dylib", "rlib"]
 alloc = { path = "../liballoc" }
 alloc_jemalloc = { path = "../liballoc_jemalloc", optional = true }
 alloc_system = { path = "../liballoc_system" }
+alloc_frame = { path = "../liballoc_frame" }
 panic_unwind = { path = "../libpanic_unwind", optional = true }
 panic_abort = { path = "../libpanic_abort" }
 collections = { path = "../libcollections" }

src/rustc/Cargo.toml

@@ -17,6 +17,8 @@ path = "rustdoc.rs"
 rustc_back = { path = "../librustc_back" }
 rustc_driver = { path = "../librustc_driver" }
 rustdoc = { path = "../librustdoc" }
+alloc_frame = { path = "../liballoc_frame", optional = true }
 
 [features]
 jemalloc = ["rustc_back/jemalloc"]
+rustc_alloc_frame = ["alloc_frame", "rustc_driver/rustc_alloc_frame"]

src/rustc/rustc.rs

@@ -9,7 +9,13 @@
 // except according to those terms.
 
 #![feature(rustc_private)]
+#![feature(staged_api)]
+#![cfg_attr(all(feature = "rustc_alloc_frame", stage1), feature(alloc_frame))]
 
 extern crate rustc_driver;
 
+// Use the frame allocator to speed up runtime
+#[cfg(all(feature = "rustc_alloc_frame", stage1))]
+extern crate alloc_frame;
+
 fn main() { rustc_driver::main() }