diff --git a/src/r3_core/src/utils/alloc/rlsf/flex.rs b/src/r3_core/src/utils/alloc/rlsf/flex.rs new file mode 100644 index 00000000000..5e552b73563 --- /dev/null +++ b/src/r3_core/src/utils/alloc/rlsf/flex.rs @@ -0,0 +1,641 @@ +//! An allocator with flexible backing stores +use core::{alloc::Layout, debug_assert, ptr::NonNull, unimplemented}; + +use super::{ + int::BinInteger, + utils::{ + nonnull_slice_end, nonnull_slice_from_raw_parts, nonnull_slice_len, nonnull_slice_start, + }, + Init, Tlsf, GRANULARITY, +}; + +/// The trait for dynamic storage allocators that can back [`FlexTlsf`]. +pub unsafe trait FlexSource { + /// Allocate a memory block of the requested minimum size. + /// + /// Returns the address range of the allocated memory block. + /// + /// # Safety + /// + /// `min_size` must be a multiple of [`GRANULARITY`]. `min_size` must not + /// be zero. + #[inline] + unsafe fn alloc(&mut self, min_size: usize) -> Option> { + let _ = min_size; + None + } + + /// Attempt to grow the specified allocation without moving it. Returns + /// the final allocation size (which must be greater than or equal to + /// `min_new_len`) on success. + /// + /// # Safety + /// + /// `ptr` must be an existing allocation made by this + /// allocator. `min_new_len` must be greater than or equal to `ptr.len()`. + #[inline] + unsafe fn realloc_inplace_grow( + &mut self, + ptr: NonNull<[u8]>, + min_new_len: usize, + ) -> Option { + let _ = (ptr, min_new_len); + None + } + + /// Deallocate a previously allocated memory block. + /// + /// # Safety + /// + /// `ptr` must denote an existing allocation made by this allocator. + #[inline] + unsafe fn dealloc(&mut self, ptr: NonNull<[u8]>) { + let _ = ptr; + unimplemented!("`supports_dealloc` returned `true`, but `dealloc` is not implemented"); + } + + /// Check if this allocator implements [`Self::dealloc`]. + /// + /// If this method returns `false`, [`FlexTlsf`] will not call `dealloc` to + /// release memory blocks. It also applies some optimizations. + /// + /// The returned value must be constant for a particular instance of `Self`. + #[inline] + fn supports_dealloc(&self) -> bool { + false + } + + /// Check if this allocator implements [`Self::realloc_inplace_grow`]. + /// + /// If this method returns `false`, [`FlexTlsf`] will not call + /// `realloc_inplace_grow` to attempt to grow memory blocks. It also applies + /// some optimizations. + /// + /// The returned value must be constant for a particular instance of `Self`. + #[inline] + fn supports_realloc_inplace_grow(&self) -> bool { + false + } + + /// Returns `true` if this allocator is implemented by managing one + /// contiguous region, which is grown every time `alloc` or + /// `realloc_inplace_grow` is called. + /// + /// For example, in WebAssembly, there is usually only one continuous + /// memory region available for data processing, and the only way to acquire + /// more memory is to grow this region by executing `memory.grow` + /// instructions. An implementation of `FlexSource` based on this system + /// would use this instruction to implement both `alloc` and + /// `realloc_inplace_grow` methods. Therefore, it's pointless for + /// [`FlexTlsf`] to call `alloc` when `realloc_inplace_grow` fails. This + /// method can be used to remove such redundant calls to `alloc`. + /// + /// The returned value must be constant for a particular instance of `Self`. + #[inline] + fn is_contiguous_growable(&self) -> bool { + false + } + + /// Get the minimum alignment of allocations made by this allocator. + /// [`FlexTlsf`] may be less efficient if this method returns a value + /// less than [`GRANULARITY`]. + /// + /// The returned value must be constant for a particular instance of `Self`. + #[inline] + fn min_align(&self) -> usize { + 1 + } +} + +trait FlexSourceExt: FlexSource { + #[inline] + fn use_growable_pool(&self) -> bool { + // `growable_pool` is used for deallocation and pool growth. + // Let's not think about the wasted space caused when this method + // returns `false`. + self.supports_dealloc() || self.supports_realloc_inplace_grow() + } +} + +impl FlexSourceExt for T {} + +/// Wraps [`core::alloc::GlobalAlloc`] to implement the [`FlexSource`] trait. +/// +/// Since this type does not implement [`FlexSource::realloc_inplace_grow`], +/// it is likely to end up with terribly fragmented memory pools. +#[derive(Default, Debug, Copy, Clone)] +pub struct GlobalAllocAsFlexSource(pub T); + +impl GlobalAllocAsFlexSource { + const ALIGN: usize = if ALIGN.is_power_of_two() { + if ALIGN < GRANULARITY { + GRANULARITY + } else { + ALIGN + } + } else { + const_panic!("`ALIGN` is not power of two") + }; +} + +impl Init for GlobalAllocAsFlexSource { + const INIT: Self = Self(Init::INIT); +} + +unsafe impl FlexSource + for GlobalAllocAsFlexSource +{ + #[inline] + unsafe fn alloc(&mut self, min_size: usize) -> Option> { + let layout = Layout::from_size_align(min_size, Self::ALIGN) + .ok()? + .pad_to_align(); + // Safety: The caller upholds that `min_size` is not zero + let start = self.0.alloc(layout); + let start = NonNull::new(start)?; + Some(nonnull_slice_from_raw_parts(start, layout.size())) + } + + #[inline] + unsafe fn dealloc(&mut self, ptr: NonNull<[u8]>) { + // Safety: This layout was previously used for allocation, during which + // the layout was checked for validity + let layout = Layout::from_size_align_unchecked(nonnull_slice_len(ptr), Self::ALIGN); + + // Safety: `start` denotes an existing allocation with layout `layout` + self.0.dealloc(ptr.as_ptr() as _, layout); + } + + fn supports_dealloc(&self) -> bool { + true + } + + #[inline] + fn min_align(&self) -> usize { + Self::ALIGN + } +} + +/// A wrapper of [`Tlsf`] that automatically acquires fresh memory pools from +/// [`FlexSource`]. +#[derive(Debug)] +pub struct FlexTlsf +{ + /// The lastly created memory pool. + growable_pool: Option, + source: Source, + tlsf: Tlsf<'static, FLBitmap, SLBitmap, FLLEN, SLLEN>, +} + +#[derive(Debug, Copy, Clone)] +struct Pool { + /// The starting address of the memory allocation. + alloc_start: NonNull, + /// The length of the memory allocation. + alloc_len: usize, + /// The length of the memory pool created within the allocation. + /// This might be slightly less than `alloc_len`. + pool_len: usize, +} + +// Safety: `Pool` is totally thread-safe +unsafe impl Send for Pool {} +unsafe impl Sync for Pool {} + +/// Pool footer stored at the end of each pool. It's only used when +/// supports_dealloc() == true`. +/// +/// The footer is stored in the sentinel block's unused space or any padding +/// present at the end of each pool. This is why `PoolFtr` can't be larger than +/// two `usize`s. +#[repr(C)] +#[derive(Copy, Clone)] +struct PoolFtr { + /// The previous allocation. Forms a singly-linked list. + prev_alloc: Option>, +} + +const _: () = if core::mem::size_of::() != GRANULARITY / 2 { + const_panic!("bad `PoolFtr` size"); +}; + +impl PoolFtr { + /// Get a pointer to `PoolFtr` for a given allocation. + #[inline] + fn get_for_alloc(alloc: NonNull<[u8]>, alloc_align: usize) -> *mut Self { + let alloc_end = nonnull_slice_end(alloc); + let mut ptr = alloc_end.wrapping_sub(core::mem::size_of::()); + // If `alloc_end` is not well-aligned, we need to adjust the location + // of `PoolFtr` + if alloc_align < core::mem::align_of::() { + ptr = (ptr as usize & !(core::mem::align_of::() - 1)) as _; + } + ptr as _ + } +} + +/// Initialization with a [`FlexSource`] provided by [`Default::default`] +impl< + Source: FlexSource + Default, + FLBitmap: BinInteger, + SLBitmap: BinInteger, + const FLLEN: usize, + const SLLEN: usize, + > Default for FlexTlsf +{ + #[inline] + fn default() -> Self { + Self { + source: Source::default(), + tlsf: Tlsf::INIT, + growable_pool: None, + } + } +} + +/// Initialization with a [`FlexSource`] provided by [`Init::INIT`] +impl< + Source: FlexSource + Init, + FLBitmap: BinInteger, + SLBitmap: BinInteger, + const FLLEN: usize, + const SLLEN: usize, + > Init for FlexTlsf +{ + // FIXME: Add `const fn new()` when `const fn`s with type bounds are stabilized + /// An empty pool. + const INIT: Self = Self { + source: Source::INIT, + tlsf: Tlsf::INIT, + growable_pool: None, + }; +} + +impl< + Source: FlexSource, + FLBitmap: BinInteger, + SLBitmap: BinInteger, + const FLLEN: usize, + const SLLEN: usize, + > FlexTlsf +{ + /// Construct a new `FlexTlsf` object. + #[inline] + pub fn new(source: Source) -> Self { + Self { + source, + tlsf: Tlsf::INIT, + growable_pool: None, + } + } + + /// Borrow the contained `Source`. + #[inline] + pub fn source_ref(&self) -> &Source { + &self.source + } + + /// Mutably borrow the contained `Source`. + /// + /// # Safety + /// + /// The caller must not replace the `Source` with another one or modify + /// any existing allocations in the `Source`. + #[inline] + pub unsafe fn source_mut_unchecked(&mut self) -> &mut Source { + &mut self.source + } + + /// Attempt to allocate a block of memory. + /// + /// Returns the starting address of the allocated memory block on success; + /// `None` otherwise. + /// + /// # Time Complexity + /// + /// This method will complete in constant time (assuming `Source`'s methods + /// do so as well). + #[cfg_attr(target_arch = "wasm32", inline(never))] + pub fn allocate(&mut self, layout: Layout) -> Option> { + if let Some(x) = self.tlsf.allocate(layout) { + return Some(x); + } + + self.increase_pool_to_contain_allocation(layout)?; + + self.tlsf.allocate(layout).or_else(|| { + // Not a hard error, but it's still unexpected because + // `increase_pool_to_contain_allocation` was supposed to make this + // allocation possible + debug_assert!( + false, + "the allocation failed despite the effort by \ + `increase_pool_to_contain_allocation`" + ); + None + }) + } + + /// Increase the amount of memory pool to guarantee the success of the + /// given allocation. Returns `Some(())` on success. + #[inline] + fn increase_pool_to_contain_allocation(&mut self, layout: Layout) -> Option<()> { + let use_growable_pool = self.source.use_growable_pool(); + + // How many extra bytes we need to get from the source for the + // allocation to success? + let extra_bytes_well_aligned = + Tlsf::<'static, FLBitmap, SLBitmap, FLLEN, SLLEN>::pool_size_to_contain_allocation( + layout, + )?; + + // The sentinel block + the block to store the allocation + debug_assert!(extra_bytes_well_aligned >= GRANULARITY * 2); + + if let Some(growable_pool) = self.growable_pool.filter(|_| use_growable_pool) { + // Try to extend an existing memory pool first. + let new_pool_len_desired = growable_pool + .pool_len + .checked_add(extra_bytes_well_aligned)?; + + // The following assertion should not trip because... + // - `extra_bytes_well_aligned` returns a value that is at least + // as large as `GRANULARITY * 2`. + // - `growable_pool.alloc_len - growable_pool.pool_len` must be + // less than `GRANULARITY * 2` because of + // `insert_free_block_ptr`'s implementation. + debug_assert!(new_pool_len_desired >= growable_pool.alloc_len); + + // Safety: `new_pool_end_desired >= growable_pool.alloc_len`, and + // `(growable_pool.alloc_start, growable_pool.alloc_len)` + // represents a previous allocation. + if let Some(new_alloc_len) = unsafe { + self.source.realloc_inplace_grow( + nonnull_slice_from_raw_parts( + growable_pool.alloc_start, + growable_pool.alloc_len, + ), + new_pool_len_desired, + ) + } { + if self.source.supports_dealloc() { + // Move `PoolFtr`. Note that `PoolFtr::alloc_start` is + // still uninitialized because this allocation is still in + // `self.growable_pool`, so we only have to move + // `PoolFtr::prev_alloc_end`. + let old_pool_ftr = PoolFtr::get_for_alloc( + nonnull_slice_from_raw_parts( + growable_pool.alloc_start, + growable_pool.alloc_len, + ), + self.source.min_align(), + ); + let new_pool_ftr = PoolFtr::get_for_alloc( + nonnull_slice_from_raw_parts(growable_pool.alloc_start, new_alloc_len), + self.source.min_align(), + ); + // Safety: Both `*new_pool_ftr` and `*old_pool_ftr` + // represent pool footers we control + unsafe { *new_pool_ftr = *old_pool_ftr }; + } + + let num_appended_len = unsafe { + // Safety: `self.source` allocated some memory after + // `alloc_start + pool_len`, so it shouldn't be + // null + let append_start = NonNull::new_unchecked( + growable_pool + .alloc_start + .as_ptr() + .wrapping_add(growable_pool.pool_len), + ); + // Safety: `append_start` follows an existing memory pool, + // and the contained bytes are owned by us + self.tlsf + .append_free_block_ptr(nonnull_slice_from_raw_parts( + append_start, + new_alloc_len - growable_pool.pool_len, + )) + }; + + // This assumption is based on `extra_bytes_well_aligned`'s + // implementation. The `debug_assert!` above depends on this. + debug_assert!( + (growable_pool.pool_len + num_appended_len) - new_alloc_len < GRANULARITY * 2 + ); + + self.growable_pool = Some(Pool { + alloc_start: growable_pool.alloc_start, + alloc_len: new_alloc_len, + pool_len: growable_pool.pool_len + num_appended_len, + }); + + return Some(()); + } // if let Some(new_alloc_len) = ... realloc_inplace_grow + + if self.source.is_contiguous_growable() { + // `is_contiguous_growable` + // indicates that `alloc` will also be fruitless because + // `realloc_inplace_grow` failed. + return None; + } + } // if let Some(growable_pool) = self.growable_pool + + // Create a brand new allocation. `source.min_align` indicates the + // minimum alignment that the created allocation will satisfy. + // `extra_bytes_well_aligned` is the pool size that can contain the + // allocation *if* the pool was well-aligned. If `source.min_align` is + // not well-aligned enough, we need to allocate extra bytes. + let extra_bytes = if self.source.min_align() < GRANULARITY { + // + // wasted wasted + // ╭┴╮ ╭──┴──╮ + // ┌─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┐ + // Allocation: │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ + // └─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┘ + // ┌───────┬───────┬───────┬───────┐ + // Pool created on it: │ │ │ │ │ + // └───────┴───────┴───────┴───────┘ + // ╰───┬───╯ + // GRANULARITY + // + extra_bytes_well_aligned.checked_add(GRANULARITY)? + } else { + extra_bytes_well_aligned + }; + + // Safety: `extra_bytes` is non-zero and aligned to `GRANULARITY` bytes + let alloc = unsafe { self.source.alloc(extra_bytes)? }; + + let is_well_aligned = self.source.min_align() >= super::GRANULARITY; + + // Safety: The passed memory block is what we acquired from + // `self.source`, so we have the ownership + let pool_len = unsafe { + if is_well_aligned { + self.tlsf.insert_free_block_ptr_aligned(alloc) + } else { + self.tlsf.insert_free_block_ptr(alloc) + } + } + .unwrap_or_else(|| unsafe { + debug_assert!(false, "`pool_size_to_contain_allocation` is an impostor"); + // Safety: It's unreachable + core::hint::unreachable_unchecked() + }) + .get(); + + if self.source.supports_dealloc() { + // Link the new memory pool's `PoolFtr::prev_alloc_end` to the + // previous pool (`self.growable_pool`). + let pool_ftr = PoolFtr::get_for_alloc(alloc, self.source.min_align()); + let prev_alloc = self + .growable_pool + .map(|p| nonnull_slice_from_raw_parts(p.alloc_start, p.alloc_len)); + // Safety: `(*pool_ftr).prev_alloc` is within a pool footer + // we control + unsafe { (*pool_ftr).prev_alloc = prev_alloc }; + } + + if use_growable_pool { + self.growable_pool = Some(Pool { + alloc_start: nonnull_slice_start(alloc), + alloc_len: nonnull_slice_len(alloc), + pool_len, + }); + } + + Some(()) + } + + /// Deallocate a previously allocated memory block. + /// + /// # Time Complexity + /// + /// This method will complete in constant time (assuming `Source`'s methods + /// do so as well). + /// + /// # Safety + /// + /// - `ptr` must denote a memory block previously allocated via `self`. + /// - The memory block must have been allocated with the same alignment + /// ([`Layout::align`]) as `align`. + /// + #[cfg_attr(target_arch = "wasm32", inline(never))] + pub unsafe fn deallocate(&mut self, ptr: NonNull, align: usize) { + // Safety: Upheld by the caller + self.tlsf.deallocate(ptr, align) + } + + /// Deallocate a previously allocated memory block with an unknown alignment. + /// + /// Unlike `deallocate`, this function does not require knowing the + /// allocation's alignment but might be less efficient. + /// + /// # Time Complexity + /// + /// This method will complete in constant time (assuming `Source`'s methods + /// do so as well). + /// + /// # Safety + /// + /// - `ptr` must denote a memory block previously allocated via `self`. + /// + pub(crate) unsafe fn deallocate_unknown_align(&mut self, ptr: NonNull) { + // Safety: Upheld by the caller + self.tlsf.deallocate_unknown_align(ptr) + } + + /// Shrink or grow a previously allocated memory block. + /// + /// Returns the new starting address of the memory block on success; + /// `None` otherwise. + /// + /// # Time Complexity + /// + /// Unlike other methods, this method will complete in linear time + /// (`O(old_size)`), assuming `Source`'s methods do so as well. + /// + /// # Safety + /// + /// - `ptr` must denote a memory block previously allocated via `self`. + /// - The memory block must have been allocated with the same alignment + /// ([`Layout::align`]) as `new_layout`. + /// + pub unsafe fn reallocate( + &mut self, + ptr: NonNull, + new_layout: Layout, + ) -> Option> { + // Do this early so that the compiler can de-duplicate the evaluation of + // `size_of_allocation`, which is done here as well as in + // `Tlsf::reallocate`. + let old_size = Tlsf::<'static, FLBitmap, SLBitmap, FLLEN, SLLEN>::size_of_allocation( + ptr, + new_layout.align(), + ); + + // Safety: Upheld by the caller + if let Some(x) = self.tlsf.reallocate(ptr, new_layout) { + return Some(x); + } + + // Allocate a whole new memory block. The following code section looks + // the same as the one in `Tlsf::reallocate`, but `self.allocation` + // here refers to `FlexTlsf::allocate`, which inserts new meory pools + // as necessary. + let new_ptr = self.allocate(new_layout)?; + + // Move the existing data into the new location + debug_assert!(new_layout.size() >= old_size); + core::ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_ptr(), old_size); + + // Deallocate the old memory block. + self.deallocate(ptr, new_layout.align()); + + Some(new_ptr) + } + + /// Get the payload size of the allocation with an unknown alignment. The + /// returned size might be larger than the size specified at the allocation + /// time. + /// + /// # Safety + /// + /// - `ptr` must denote a memory block previously allocated via `Self`. + /// + #[inline] + pub(crate) unsafe fn size_of_allocation_unknown_align(ptr: NonNull) -> usize { + // Safety: Upheld by the caller + Tlsf::<'static, FLBitmap, SLBitmap, FLLEN, SLLEN>::size_of_allocation_unknown_align(ptr) + } +} + +impl Drop + for FlexTlsf +{ + fn drop(&mut self) { + if self.source.supports_dealloc() { + debug_assert!(self.source.use_growable_pool()); + + // Deallocate all memory pools + let align = self.source.min_align(); + let mut cur_alloc_or_none = self + .growable_pool + .map(|p| nonnull_slice_from_raw_parts(p.alloc_start, p.alloc_len)); + + while let Some(cur_alloc) = cur_alloc_or_none { + // Safety: We control the referenced pool footer + let cur_ftr = unsafe { *PoolFtr::get_for_alloc(cur_alloc, align) }; + + // Safety: It's an allocation we allocated from `self.source` + unsafe { self.source.dealloc(cur_alloc) }; + + cur_alloc_or_none = cur_ftr.prev_alloc; + } + } + } +} + +#[cfg(test)] +mod tests; diff --git a/src/r3_core/src/utils/alloc/rlsf/flex/tests.rs b/src/r3_core/src/utils/alloc/rlsf/flex/tests.rs new file mode 100644 index 00000000000..5c137da27ff --- /dev/null +++ b/src/r3_core/src/utils/alloc/rlsf/flex/tests.rs @@ -0,0 +1,415 @@ +use quickcheck_macros::quickcheck; +use std::{fmt, prelude::v1::*}; + +use super::*; +use crate::{ + tests::ShadowAllocator, + utils::{nonnull_slice_end, nonnull_slice_len}, +}; + +trait TestFlexSource: FlexSource { + type Options: quickcheck::Arbitrary; + + fn new(options: Self::Options) -> Self; +} + +impl TestFlexSource + for GlobalAllocAsFlexSource +{ + type Options = (); + + fn new((): ()) -> Self { + Self(T::default()) + } +} + +#[derive(Debug)] +struct TrackingFlexSource { + sa: ShadowAllocator, + inner: T, +} + +impl TestFlexSource for TrackingFlexSource { + type Options = T::Options; + + fn new(options: T::Options) -> Self { + Self { + sa: ShadowAllocator::default(), + inner: T::new(options), + } + } +} + +impl Drop for TrackingFlexSource { + fn drop(&mut self) { + if std::thread::panicking() { + return; + } + + if self.inner.supports_dealloc() { + // All existing pools should have been removed by `FlexTlsf::drop` + self.sa.assert_no_pools(); + } + } +} + +unsafe impl FlexSource for TrackingFlexSource { + unsafe fn alloc(&mut self, min_size: usize) -> Option> { + log::trace!("FlexSource::alloc({:?})", min_size); + let range = self.inner.alloc(min_size)?; + log::trace!(" FlexSource::alloc(...) = {:?}", range); + self.sa.insert_free_block(range.as_ptr()); + Some(range) + } + + unsafe fn realloc_inplace_grow( + &mut self, + ptr: NonNull<[u8]>, + min_new_len: usize, + ) -> Option { + log::trace!("FlexSource::realloc_inplace_grow{:?}", (ptr, min_new_len)); + let new_len = self.inner.realloc_inplace_grow(ptr, min_new_len)?; + log::trace!(" FlexSource::realloc_inplace_grow(...) = {:?}", new_len); + self.sa.append_free_block(std::ptr::slice_from_raw_parts( + nonnull_slice_end(ptr), + new_len - nonnull_slice_len(ptr), + )); + Some(new_len) + } + + #[inline] + fn min_align(&self) -> usize { + self.inner.min_align() + } + + #[inline] + unsafe fn dealloc(&mut self, ptr: NonNull<[u8]>) { + // TODO: check that `ptr` represents an exact allocation, not just + // a part of it + self.inner.dealloc(ptr); + log::trace!("FlexSource::dealloc({:?})", ptr); + self.sa.remove_pool(ptr.as_ptr()); + } + + #[inline] + fn is_contiguous_growable(&self) -> bool { + self.inner.is_contiguous_growable() + } + + #[inline] + fn supports_dealloc(&self) -> bool { + self.inner.supports_dealloc() + } + + #[inline] + fn supports_realloc_inplace_grow(&self) -> bool { + self.inner.supports_realloc_inplace_grow() + } +} + +/// Continuous-growing flex source +struct CgFlexSource { + pool: Vec, + allocated: usize, +} + +impl fmt::Debug for CgFlexSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CgFlexSource") + .field("allocated", &self.allocated) + .finish() + } +} + +impl TestFlexSource for CgFlexSource { + type Options = u8; + + fn new(offset: u8) -> Self { + Self { + pool: std::vec![0u8; 1024 * 32], + allocated: offset as usize, + } + } +} + +unsafe impl FlexSource for CgFlexSource { + unsafe fn alloc(&mut self, min_size: usize) -> Option> { + let allocated = self.allocated; + let new_allocated = allocated + .checked_add(min_size) + .filter(|&x| x <= self.pool.len())?; + + self.allocated = new_allocated; + Some(NonNull::from(&mut self.pool[allocated..new_allocated])) + } + + unsafe fn realloc_inplace_grow( + &mut self, + ptr: NonNull<[u8]>, + min_new_len: usize, + ) -> Option { + self.alloc(min_new_len - nonnull_slice_len(ptr)) + .map(|s| nonnull_slice_len(s) + nonnull_slice_len(ptr)) + } + + fn is_contiguous_growable(&self) -> bool { + true + } + + fn supports_realloc_inplace_grow(&self) -> bool { + true + } + + fn min_align(&self) -> usize { + 1 + } +} + +fn fill_data(p: NonNull<[u8]>) { + use std::mem::MaybeUninit; + let slice = unsafe { &mut *(p.as_ptr() as *mut [MaybeUninit]) }; + for (i, p) in slice.iter_mut().enumerate() { + *p = MaybeUninit::new((i as u8).reverse_bits()); + } +} + +fn verify_data(p: NonNull<[u8]>) { + let slice = unsafe { p.as_ref() }; + for (i, p) in slice.iter().enumerate() { + assert_eq!(*p, (i as u8).reverse_bits()); + } +} + +macro_rules! gen_test { + ($mod:ident, $source:ty, $($tt:tt)*) => { + mod $mod { + use super::*; + type TheTlsf = FlexTlsf, $($tt)*>; + + #[quickcheck] + fn minimal(source_options: <$source as TestFlexSource>::Options) { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut tlsf = TheTlsf::new(TrackingFlexSource::new(source_options)); + + log::trace!("tlsf = {:?}", tlsf); + + let ptr = tlsf.allocate(Layout::from_size_align(1, 1).unwrap()); + log::trace!("ptr = {:?}", ptr); + if let Some(ptr) = ptr { + unsafe { tlsf.deallocate(ptr, 1) }; + } + } + + #[quickcheck] + fn aadaadaraaadr(source_options: <$source as TestFlexSource>::Options) { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut tlsf = TheTlsf::new(TrackingFlexSource::new(source_options)); + + log::trace!("tlsf = {:?}", tlsf); + + let ptr1 = tlsf.allocate(Layout::from_size_align(20, 16).unwrap()); + log::trace!("ptr1 = {:?}", ptr1); + let ptr2 = tlsf.allocate(Layout::from_size_align(21, 1).unwrap()); + log::trace!("ptr2 = {:?}", ptr2); + if let Some(ptr1) = ptr1 { + unsafe { tlsf.deallocate(ptr1, 16) }; + } + let ptr3 = tlsf.allocate(Layout::from_size_align(0, 32).unwrap()); + log::trace!("ptr3 = {:?}", ptr3); + let ptr4 = tlsf.allocate(Layout::from_size_align(10, 8).unwrap()); + log::trace!("ptr4 = {:?}", ptr4); + if let Some(ptr2) = ptr2 { + unsafe { tlsf.deallocate(ptr2, 1) }; + log::trace!("deallocate(ptr2)"); + } + let ptr5 = tlsf.allocate(Layout::from_size_align(12, 8).unwrap()); + log::trace!("ptr5 = {:?}", ptr5); + let ptr3 = ptr3.and_then(|ptr3| unsafe { + tlsf.reallocate(ptr3, Layout::from_size_align(0, 32).unwrap()) + }); + log::trace!("ptr3 = {:?}", ptr3); + let ptr6 = tlsf.allocate(Layout::from_size_align(24, 2).unwrap()); + log::trace!("ptr6 = {:?}", ptr6); + let ptr7 = tlsf.allocate(Layout::from_size_align(11, 16).unwrap()); + log::trace!("ptr7 = {:?}", ptr7); + let ptr8 = tlsf.allocate(Layout::from_size_align(1, 32).unwrap()); + log::trace!("ptr8 = {:?}", ptr8); + if let Some(ptr5) = ptr5 { + unsafe { tlsf.deallocate(ptr5, 8) }; + log::trace!("deallocate(ptr5)"); + } + let ptr3 = ptr3.and_then(|ptr3| unsafe { + tlsf.reallocate(ptr3, Layout::from_size_align(4, 32).unwrap()) + }); + log::trace!("ptr3 = {:?}", ptr3); + } + + #[quickcheck] + fn random(source_options: <$source as TestFlexSource>::Options, max_alloc_size: usize, bytecode: Vec) { + random_inner(source_options, max_alloc_size, bytecode); + } + + fn random_inner(source_options: <$source as TestFlexSource>::Options, max_alloc_size: usize, bytecode: Vec) -> Option<()> { + let max_alloc_size = max_alloc_size % 0x10000; + + let mut tlsf = TheTlsf::new(TrackingFlexSource::new(source_options)); + macro_rules! sa { + () => { + unsafe { tlsf.source_mut_unchecked() }.sa + }; + } + + log::trace!("tlsf = {:?}", tlsf); + + #[derive(Debug)] + struct Alloc { + ptr: NonNull, + layout: Layout, + } + let mut allocs = Vec::new(); + + let mut it = bytecode.iter().cloned(); + loop { + match it.next()? % 8 { + 0..=2 => { + let len = u32::from_le_bytes([ + it.next()?, + it.next()?, + it.next()?, + 0, + ]); + let len = ((len as u64 * max_alloc_size as u64) >> 24) as usize; + let align = 1 << (it.next()? % 6); + let layout = Layout::from_size_align(len, align).unwrap(); + log::trace!("alloc {:?}", layout); + + let ptr = tlsf.allocate(layout); + log::trace!(" → {:?}", ptr); + + if let Some(ptr) = ptr { + allocs.push(Alloc { ptr, layout }); + sa!().allocate(layout, ptr); + + // Fill it with dummy data + fill_data(crate::utils::nonnull_slice_from_raw_parts(ptr, len)); + } + } + 3..=5 => { + let alloc_i = it.next()?; + if allocs.len() > 0 { + let alloc = allocs.swap_remove(alloc_i as usize % allocs.len()); + log::trace!("dealloc {:?}", alloc); + + // Make sure the stored dummy data is not corrupted + verify_data(crate::utils::nonnull_slice_from_raw_parts(alloc.ptr, alloc.layout.size())); + + unsafe { tlsf.deallocate(alloc.ptr, alloc.layout.align()) }; + sa!().deallocate(alloc.layout, alloc.ptr); + } + } + 6..=7 => { + let alloc_i = it.next()?; + if allocs.len() > 0 { + let len = u32::from_le_bytes([ + it.next()?, + it.next()?, + it.next()?, + 0, + ]); + let len = ((len as u64 * max_alloc_size as u64) >> 24) as usize; + + let alloc_i = alloc_i as usize % allocs.len(); + let alloc = &mut allocs[alloc_i]; + log::trace!("realloc {:?} to {:?}", alloc, len); + + let new_layout = Layout::from_size_align(len, alloc.layout.align()).unwrap(); + + if let Some(ptr) = unsafe { tlsf.reallocate(alloc.ptr, new_layout) } { + log::trace!(" {:?} → {:?}", alloc.ptr, ptr); + + // Check and refill the dummy data + verify_data(crate::utils::nonnull_slice_from_raw_parts(ptr, len.min(alloc.layout.size()))); + fill_data(crate::utils::nonnull_slice_from_raw_parts(ptr, len)); + + sa!().deallocate(alloc.layout, alloc.ptr); + alloc.ptr = ptr; + alloc.layout = new_layout; + sa!().allocate(alloc.layout, alloc.ptr); + } else { + log::trace!(" {:?} → fail", alloc.ptr); + + } + } + } + _ => unreachable!(), + } + } + } + } + }; +} + +type SysSource = GlobalAllocAsFlexSource; +gen_test!(tlsf_sys_u8_u8_1_1, SysSource, u8, u8, 1, 1); +gen_test!(tlsf_sys_u8_u8_1_2, SysSource, u8, u8, 1, 2); +gen_test!(tlsf_sys_u8_u8_1_4, SysSource, u8, u8, 1, 4); +gen_test!(tlsf_sys_u8_u8_1_8, SysSource, u8, u8, 1, 8); +gen_test!(tlsf_sys_u8_u8_3_4, SysSource, u8, u8, 3, 4); +gen_test!(tlsf_sys_u8_u8_5_4, SysSource, u8, u8, 5, 4); +gen_test!(tlsf_sys_u8_u8_8_1, SysSource, u8, u8, 8, 1); +gen_test!(tlsf_sys_u8_u8_8_8, SysSource, u8, u8, 8, 8); +gen_test!(tlsf_sys_u16_u8_3_4, SysSource, u16, u8, 3, 4); +gen_test!(tlsf_sys_u16_u8_11_4, SysSource, u16, u8, 11, 4); +gen_test!(tlsf_sys_u16_u8_16_4, SysSource, u16, u8, 16, 4); +gen_test!(tlsf_sys_u16_u16_3_16, SysSource, u16, u16, 3, 16); +gen_test!(tlsf_sys_u16_u16_11_16, SysSource, u16, u16, 11, 16); +gen_test!(tlsf_sys_u16_u16_16_16, SysSource, u16, u16, 16, 16); +gen_test!(tlsf_sys_u16_u32_3_16, SysSource, u16, u32, 3, 16); +gen_test!(tlsf_sys_u16_u32_11_16, SysSource, u16, u32, 11, 16); +gen_test!(tlsf_sys_u16_u32_16_16, SysSource, u16, u32, 16, 16); +gen_test!(tlsf_sys_u16_u32_3_32, SysSource, u16, u32, 3, 32); +gen_test!(tlsf_sys_u16_u32_11_32, SysSource, u16, u32, 11, 32); +gen_test!(tlsf_sys_u16_u32_16_32, SysSource, u16, u32, 16, 32); +gen_test!(tlsf_sys_u32_u32_20_32, SysSource, u32, u32, 20, 32); +gen_test!(tlsf_sys_u32_u32_27_32, SysSource, u32, u32, 27, 32); +gen_test!(tlsf_sys_u32_u32_28_32, SysSource, u32, u32, 28, 32); +gen_test!(tlsf_sys_u32_u32_29_32, SysSource, u32, u32, 29, 32); +gen_test!(tlsf_sys_u32_u32_32_32, SysSource, u32, u32, 32, 32); +gen_test!(tlsf_sys_u64_u8_58_8, SysSource, u64, u64, 58, 8); +gen_test!(tlsf_sys_u64_u8_59_8, SysSource, u64, u64, 59, 8); +gen_test!(tlsf_sys_u64_u8_60_8, SysSource, u64, u64, 60, 8); +gen_test!(tlsf_sys_u64_u8_61_8, SysSource, u64, u64, 61, 8); +gen_test!(tlsf_sys_u64_u8_64_8, SysSource, u64, u64, 64, 8); + +gen_test!(tlsf_cg_u8_u8_1_1, CgFlexSource, u8, u8, 1, 1); +gen_test!(tlsf_cg_u8_u8_1_2, CgFlexSource, u8, u8, 1, 2); +gen_test!(tlsf_cg_u8_u8_1_4, CgFlexSource, u8, u8, 1, 4); +gen_test!(tlsf_cg_u8_u8_1_8, CgFlexSource, u8, u8, 1, 8); +gen_test!(tlsf_cg_u8_u8_3_4, CgFlexSource, u8, u8, 3, 4); +gen_test!(tlsf_cg_u8_u8_5_4, CgFlexSource, u8, u8, 5, 4); +gen_test!(tlsf_cg_u8_u8_8_1, CgFlexSource, u8, u8, 8, 1); +gen_test!(tlsf_cg_u8_u8_8_8, CgFlexSource, u8, u8, 8, 8); +gen_test!(tlsf_cg_u16_u8_3_4, CgFlexSource, u16, u8, 3, 4); +gen_test!(tlsf_cg_u16_u8_11_4, CgFlexSource, u16, u8, 11, 4); +gen_test!(tlsf_cg_u16_u8_16_4, CgFlexSource, u16, u8, 16, 4); +gen_test!(tlsf_cg_u16_u16_3_16, CgFlexSource, u16, u16, 3, 16); +gen_test!(tlsf_cg_u16_u16_11_16, CgFlexSource, u16, u16, 11, 16); +gen_test!(tlsf_cg_u16_u16_16_16, CgFlexSource, u16, u16, 16, 16); +gen_test!(tlsf_cg_u16_u32_3_16, CgFlexSource, u16, u32, 3, 16); +gen_test!(tlsf_cg_u16_u32_11_16, CgFlexSource, u16, u32, 11, 16); +gen_test!(tlsf_cg_u16_u32_16_16, CgFlexSource, u16, u32, 16, 16); +gen_test!(tlsf_cg_u16_u32_3_32, CgFlexSource, u16, u32, 3, 32); +gen_test!(tlsf_cg_u16_u32_11_32, CgFlexSource, u16, u32, 11, 32); +gen_test!(tlsf_cg_u16_u32_16_32, CgFlexSource, u16, u32, 16, 32); +gen_test!(tlsf_cg_u32_u32_20_32, CgFlexSource, u32, u32, 20, 32); +gen_test!(tlsf_cg_u32_u32_27_32, CgFlexSource, u32, u32, 27, 32); +gen_test!(tlsf_cg_u32_u32_28_32, CgFlexSource, u32, u32, 28, 32); +gen_test!(tlsf_cg_u32_u32_29_32, CgFlexSource, u32, u32, 29, 32); +gen_test!(tlsf_cg_u32_u32_32_32, CgFlexSource, u32, u32, 32, 32); +gen_test!(tlsf_cg_u64_u8_58_8, CgFlexSource, u64, u64, 58, 8); +gen_test!(tlsf_cg_u64_u8_59_8, CgFlexSource, u64, u64, 59, 8); +gen_test!(tlsf_cg_u64_u8_60_8, CgFlexSource, u64, u64, 60, 8); +gen_test!(tlsf_cg_u64_u8_61_8, CgFlexSource, u64, u64, 61, 8); +gen_test!(tlsf_cg_u64_u8_64_8, CgFlexSource, u64, u64, 64, 8); diff --git a/src/r3_core/src/utils/alloc/rlsf/int.rs b/src/r3_core/src/utils/alloc/rlsf/int.rs new file mode 100644 index 00000000000..6dd61844f55 --- /dev/null +++ b/src/r3_core/src/utils/alloc/rlsf/int.rs @@ -0,0 +1,174 @@ +//! Provides [`BinInteger`], a trait for types like `u8` and `u32`. +#![allow(unstable_name_collisions)] // `$ty::BITS` +use core::{fmt, marker, ops}; + +/// Integral types with efficient binary operations. +pub trait BinInteger: + Clone + + Copy + + PartialEq + + Eq + + Ord + + PartialOrd + + marker::Unpin + + ops::Add + + ops::Sub + + ops::Mul + + ops::Div + + ops::Rem + + Sized + + ops::AddAssign + + ops::SubAssign + + ops::MulAssign + + ops::DivAssign + + ops::RemAssign + + fmt::Debug + + Send + + Sync + + 'static + + private::Sealed +{ + const ZERO: Self; + const BITS: u32; + + fn ones(range: ops::Range) -> Self; + + fn ones_truncated(range: ops::Range) -> Self; + + /// Return the number of trailing zeros in its binary representation. + fn trailing_zeros(&self) -> u32; + + /// Return the number of leading zeros in its binary representation. + fn leading_zeros(&self) -> u32; + + /// Return the number of ones in its binary representation. + fn count_ones(&self) -> u32; + + /// Return the position of the least significant set bit since the position + /// `start`. + /// + /// Retruns `Self::BITS` if none was found. + fn bit_scan_forward(&self, start: u32) -> u32; + + /// Slice a part of its binary representation as `u32`. + fn extract_u32(&self, range: ops::Range) -> u32; + + /// Retrieve whether the specified bit is set or not. + fn get_bit(&self, i: u32) -> bool; + + /// Set a single bit. + fn set_bit(&mut self, i: u32); + + /// Clear a single bit. + fn clear_bit(&mut self, i: u32); + + /// Perform `ceil` treating the value as a fixed point number with `fp` + /// fractional part digits. + fn checked_ceil_fix(self, fp: u32) -> Option; +} + +macro_rules! impl_binary_integer { + ($type:ty) => { + impl private::Sealed for $type {} + + impl BinInteger for $type { + const ZERO: Self = 0; + const BITS: u32 = core::mem::size_of::<$type>() as u32 * 8; + + #[inline] + fn ones(range: ops::Range) -> Self { + assert!(range.end <= Self::BITS); + Self::ones_truncated(range) + } + #[inline] + fn ones_truncated(range: ops::Range) -> Self { + assert!(range.start <= range.end); + if range.end >= Self::BITS { + (0 as Self).wrapping_sub(1 << range.start) + } else { + ((1 as Self) << range.end).wrapping_sub(1 << range.start) + } + } + #[inline] + fn trailing_zeros(&self) -> u32 { + (*self).trailing_zeros() + } + #[inline] + fn leading_zeros(&self) -> u32 { + (*self).leading_zeros() + } + #[inline] + fn count_ones(&self) -> u32 { + (*self).count_ones() + } + #[inline] + fn bit_scan_forward(&self, start: u32) -> u32 { + if start >= Self::BITS { + Self::BITS + } else { + (*self & !Self::ones(0..start)).trailing_zeros() + } + } + #[inline] + fn extract_u32(&self, range: ops::Range) -> u32 { + let start = range.start; + ((self & Self::ones_truncated(range)) >> start) as u32 + } + #[inline] + fn get_bit(&self, i: u32) -> bool { + if i < Self::BITS { + self & ((1 as Self) << i) != 0 + } else { + false + } + } + #[inline] + fn set_bit(&mut self, i: u32) { + if i < Self::BITS { + *self |= (1 as Self) << i; + } + } + #[inline] + fn clear_bit(&mut self, i: u32) { + if i < Self::BITS { + *self &= !((1 as Self) << i); + } + } + #[inline] + fn checked_ceil_fix(self, fp: u32) -> Option { + if fp >= Self::BITS { + if self == 0 { + Some(0) + } else { + None + } + } else { + let mask = Self::ones(0..fp); + self.checked_add(mask).map(|x| x & !mask) + } + } + } + }; +} + +impl_binary_integer!(i8); +impl_binary_integer!(i16); +impl_binary_integer!(i32); +impl_binary_integer!(i64); +impl_binary_integer!(i128); +impl_binary_integer!(isize); + +impl_binary_integer!(u8); +impl_binary_integer!(u16); +impl_binary_integer!(u32); +impl_binary_integer!(u64); +impl_binary_integer!(u128); +impl_binary_integer!(usize); + +/// Implements [the sealed trait pattern], which protects [`BinInteger`] against +/// downstream implementations. +/// +/// [the sealed trait pattern]: https://rust-lang.github.io/api-guidelines/future-proofing.html +mod private { + pub trait Sealed {} +} diff --git a/src/r3_core/src/utils/alloc/rlsf/tests.rs b/src/r3_core/src/utils/alloc/rlsf/tests.rs new file mode 100644 index 00000000000..197256ac772 --- /dev/null +++ b/src/r3_core/src/utils/alloc/rlsf/tests.rs @@ -0,0 +1,196 @@ +use std::{alloc::Layout, collections::BTreeMap, ops::Range, prelude::v1::*, ptr::NonNull}; + +#[derive(Debug)] +pub struct ShadowAllocator { + regions: BTreeMap, +} + +#[derive(Debug, Eq, PartialEq, Copy, Clone)] +pub enum SaRegion { + Free, + Used, + Invalid, +} + +impl Default for ShadowAllocator { + fn default() -> Self { + Self::new() + } +} + +impl ShadowAllocator { + pub fn new() -> Self { + Self { + regions: Some((0, SaRegion::Invalid)).into_iter().collect(), + } + } + + pub fn new_filled_with_free() -> Self { + Self { + regions: Some((0, SaRegion::Free)).into_iter().collect(), + } + } + + pub fn convert_range( + &mut self, + range: Range, + old_region: SaRegion, + new_region: SaRegion, + ) { + if range.len() == 0 { + return; + } + + assert_ne!(old_region, new_region); + log::trace!( + "sa: converting {:?} from {:?} to {:?}", + range, + old_region, + new_region + ); + + let (&addr, ®ion) = self.regions.range(0..range.end).rev().next().unwrap(); + if addr > range.start { + panic!("there's a discontinuity in range {:?}", range); + } else if region != old_region { + panic!( + "range {:?} is {:?} (expected {:?})", + range, region, old_region + ); + } + + // Insert an element at `range.start` + if addr == range.start { + *self.regions.get_mut(&addr).unwrap() = new_region; + } else { + self.regions.insert(range.start, new_region); + } + + // Each element must represent a discontinuity. If it doesnt't represent + // a discontinuity, it must be removed. + if let Some((_, ®ion)) = self.regions.range(0..range.start).rev().next() { + if region == new_region { + self.regions.remove(&range.start); + } + } + + if let Some(&end_region) = self.regions.get(&range.end) { + // Each element must represent a discontinuity. If it doesnt't + // represent a discontinuity, it must be removed. + if end_region == new_region { + self.regions.remove(&range.end); + } + } else { + // Insert an element at `range.end` + self.regions.insert(range.end, old_region); + } + } + + pub fn assert_no_pools(&mut self) { + assert!( + self.regions.iter().eq(Some((&0, &SaRegion::Invalid))), + "{:?}", + self.regions, + ); + } + + pub fn insert_free_block(&mut self, range: *const [T]) { + let start = range as *const T as usize; + let len = unsafe { &*range }.len(); + self.convert_range(start..start + len, SaRegion::Invalid, SaRegion::Free); + } + + pub fn append_free_block(&mut self, range: *const [T]) { + let start = range as *const T as usize; + let mut it = self.regions.range(0..=start).rev(); + + assert_eq!( + it.next(), + Some((&start, &SaRegion::Invalid)), + "no boundary at `start`" + ); + + assert_ne!( + it.next().expect("no previous allocation to append to").1, + &SaRegion::Invalid, + "no previous allocation to append to" + ); + + self.insert_free_block(range); + } + + pub fn remove_pool(&mut self, range: *const [T]) { + let start = range as *const T as usize; + let end = unsafe { &*range }.len() + start; + if start >= end { + return; + } + log::trace!("sa: invalidating {:?}", start..end); + + // There mustn't be any `Invalid` regions in the range + for (&addr, ®ion) in self.regions.range(0..end).rev() { + if region == SaRegion::Invalid { + panic!("invalid region at {}", addr); + } + if addr <= start { + break; + } + } + + // Create discontinuity at `end` if needed + { + let (&addr, ®ion) = self.regions.range(0..=end).rev().next().unwrap(); + if addr < end && region != SaRegion::Invalid { + self.regions.insert(end, region); + } else if addr == end && region == SaRegion::Invalid { + self.regions.remove(&end); + } + } + + // Create discontinuity at `start` if needed + if let Some((_, ®ion)) = self.regions.range(0..start).rev().next() { + if region != SaRegion::Invalid { + self.regions.insert(start, SaRegion::Invalid); + } else { + self.regions.remove(&start); + } + } else { + assert_eq!(start, 0); + self.regions.insert(start, SaRegion::Invalid); + } + + // Remove anything remaining between `start` and `end` + let keys: Vec<_> = self + .regions + .range(start + 1..end) + .map(|(&addr, _)| addr) + .collect(); + for key in keys.iter() { + self.regions.remove(key); + } + } + + pub fn allocate(&mut self, layout: Layout, start: NonNull) { + let start = start.as_ptr() as usize; + let len = layout.size(); + assert!( + start % layout.align() == 0, + "0x{:x} is not properly aligned (0x{:x} bytes alignment required)", + start, + layout.align() + ); + self.convert_range(start..start + len, SaRegion::Free, SaRegion::Used); + } + + pub fn deallocate(&mut self, layout: Layout, start: NonNull) { + let start = start.as_ptr() as usize; + let len = layout.size(); + assert!( + start % layout.align() == 0, + "0x{:x} is not properly aligned (0x{:x} bytes alignment required)", + start, + layout.align() + ); + self.convert_range(start..start + len, SaRegion::Used, SaRegion::Free); + } +} diff --git a/src/r3_core/src/utils/alloc/rlsf/tlsf.rs b/src/r3_core/src/utils/alloc/rlsf/tlsf.rs new file mode 100644 index 00000000000..8890bc21ed7 --- /dev/null +++ b/src/r3_core/src/utils/alloc/rlsf/tlsf.rs @@ -0,0 +1,1452 @@ +//! The TLSF allocator core +use core::{ + alloc::Layout, + debug_assert, debug_assert_eq, + hint::unreachable_unchecked, + marker::PhantomData, + mem::{self, MaybeUninit}, + num::NonZeroUsize, + ptr::{addr_of, NonNull}, +}; + +use crate::{ + int::BinInteger, + utils::{nonnull_slice_from_raw_parts, nonnull_slice_len, nonnull_slice_start}, +}; + +#[cfg_attr(doc, svgbobdoc::transform)] +/// The TLSF header (top-level) data structure. +/// +/// # Data Structure Overview +/// +///
+/// ```svgbob +/// First level +/// FLLEN = 8 +/// ,-----+-----+-----+-----+-----+-----+-----+-----, +/// fl_bitmap: FLBitmap = | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | +/// +-----+-----+-----+-----+-----+-----+-----+-----+ +/// min size | 2¹¹ | 2¹⁰ | 2⁹ | 2⁸ | 2⁷ | 2⁶ | 2⁵ | 2⁴ | +/// '-----+-----+-----+--+--+-----+-----+-----+-----' +/// | +/// ╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶|╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶ +/// Second Level | +/// v SLLEN = 8 +/// ,-----+-----+-----+-----+-----+-----+-----+-----, +/// "sl_bitmap[4]: SLBitmap"= | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | +/// +-----+-----+-----+-----+-----+-----+-----+-----+ +/// min size 2⁸(1+n/8) | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | +/// +-----+-----+-----+-----+-----+-----+-----+-----+ +/// first_free | | | O | | | | | | +/// '-----+-----+--|--+-----+-----+-----+-----+-----' +/// | +/// | size = 416..448 +/// | +/// ╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶|╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶╶ +/// Free blocks | +/// | +/// ,-----------------------------------' +/// | ,---+---+-------, ,---+---+-------, ,---+---+-------, +/// '-+>O | O-+-------+----+>O | O-+-------+----+>O | | | +/// +---+---' | +---+---' | +---+---' | +/// | | | | | | +/// | | | | | | +/// '---------------' '---------------' '---------------' +/// 416 bytes 432 bytes 416 bytes +/// ``` +///
+/// +/// # Properties +/// +/// The allocation granularity ([`GRANULARITY`]) is `size_of::() * 4` +/// bytes, which is the minimum size of a free block. +/// +/// The maximum block size is `(GRANULARITY << FLLEN) - GRANULARITY`. +/// +#[derive(Debug)] +pub struct Tlsf<'pool, FLBitmap, SLBitmap, const FLLEN: usize, const SLLEN: usize> { + fl_bitmap: FLBitmap, + /// `sl_bitmap[fl].get_bit(sl)` is set iff `first_free[fl][sl].is_some()` + sl_bitmap: [SLBitmap; FLLEN], + first_free: [[Option>; SLLEN]; FLLEN], + _phantom: PhantomData<&'pool mut ()>, +} + +// Safety: All memory block headers directly or indirectly referenced by a +// particular instance of `Tlsf` are logically owned by that `Tlsf` and +// have no interior mutability, so these are safe. +unsafe impl Send + for Tlsf<'_, FLBitmap, SLBitmap, FLLEN, SLLEN> +{ +} + +unsafe impl Sync + for Tlsf<'_, FLBitmap, SLBitmap, FLLEN, SLLEN> +{ +} + +/// The allocation granularity. +/// +/// It is `size_of::() * 4` bytes, which is the minimum size of a TLSF +/// free block. +pub const GRANULARITY: usize = core::mem::size_of::() * 4; + +const GRANULARITY_LOG2: u32 = GRANULARITY.trailing_zeros(); + +// FIXME: Use `usize::BITS` when it's stable +pub(crate) const USIZE_BITS: u32 = core::mem::size_of::() as u32 * 8; + +/// The header of a memory block. +// The header is actually aligned at `size_of::() * 4`-byte boundaries +// but the alignment is set to a half value here not to introduce a padding at +// the end of this struct. +#[repr(C)] +#[cfg_attr(target_pointer_width = "16", repr(align(4)))] +#[cfg_attr(target_pointer_width = "32", repr(align(8)))] +#[cfg_attr(target_pointer_width = "64", repr(align(16)))] +#[derive(Debug)] +struct BlockHdr { + /// The size of the whole memory block, including the header. + /// + /// - `bit[0]` ([`SIZE_USED`]) indicates whether the block is a used memory + /// block or not. + /// + /// - `bit[1]` ([`SIZE_LAST_IN_POOL`]) indicates whether the block is the + /// last one of the pool or not. + /// + /// - `bit[GRANULARITY_LOG2..]` ([`SIZE_SIZE_MASK`]) represents the size. + /// + size: usize, + prev_phys_block: Option>, +} + +/// The bit of [`BlockHdr::size`] indicating whether the block is a used memory +/// block or not. +const SIZE_USED: usize = 1; +/// The bit of [`BlockHdr::size`] indicating whether the block is a sentinel +/// (the last block in a memory pool) or not. If this bit is set, [`SIZE_USED`] +/// must be set, too (`SIZE_SENTINEL ⟹ SIZE_USED`). +const SIZE_SENTINEL: usize = 2; +/// The bits of [`BlockHdr::size`] indicating the block's size. +const SIZE_SIZE_MASK: usize = !((1 << GRANULARITY_LOG2) - 1); + +impl BlockHdr { + /// Get the next block. + /// + /// # Safety + /// + /// `self` must have a next block (it must not be the sentinel block in a + /// pool). + #[inline] + unsafe fn next_phys_block(&self) -> NonNull { + debug_assert!( + (self.size & SIZE_SENTINEL) == 0, + "`self` must not be a sentinel" + ); + + // Safety: Since `self.size & SIZE_LAST_IN_POOL` is not lying, the + // next block should exist at a non-null location. + NonNull::new_unchecked((self as *const _ as *mut u8).add(self.size & SIZE_SIZE_MASK)).cast() + } +} + +/// The header of a free memory block. +#[repr(C)] +#[cfg_attr(target_pointer_width = "16", repr(align(8)))] +#[cfg_attr(target_pointer_width = "32", repr(align(16)))] +#[cfg_attr(target_pointer_width = "64", repr(align(32)))] +#[derive(Debug)] +struct FreeBlockHdr { + common: BlockHdr, + next_free: Option>, + prev_free: Option>, +} + +/// The header of a used memory block. It's `GRANULARITY / 2` bytes long. +/// +/// The payload immediately follows this header. However, if the alignment +/// requirement is greater than or equal to [`GRANULARITY`], an up to +/// `align - GRANULARITY / 2` bytes long padding will be inserted between them, +/// and the last part of the padding ([`UsedBlockPad`]) will encode where the +/// header is located. +#[repr(C)] +#[derive(Debug)] +struct UsedBlockHdr { + common: BlockHdr, +} + +/// In a used memory block with an alignment requirement larger than or equal to +/// `GRANULARITY`, the payload is preceded by this structure. +#[derive(Debug)] +#[repr(C)] +struct UsedBlockPad { + block_hdr: NonNull, +} + +impl UsedBlockPad { + #[inline] + fn get_for_allocation(ptr: NonNull) -> *mut Self { + ptr.cast::().as_ptr().wrapping_sub(1) + } +} + +impl Default + for Tlsf<'_, FLBitmap, SLBitmap, FLLEN, SLLEN> +{ + fn default() -> Self { + Self::INIT + } +} + +impl<'pool, FLBitmap: BinInteger, SLBitmap: BinInteger, const FLLEN: usize, const SLLEN: usize> + Tlsf<'pool, FLBitmap, SLBitmap, FLLEN, SLLEN> +{ + // FIXME: Add `const fn new()` when `const fn`s with type bounds are stabilized + /// An empty pool. + pub const INIT: Self = Self { + fl_bitmap: FLBitmap::ZERO, + sl_bitmap: [SLBitmap::ZERO; FLLEN], + first_free: [[None; SLLEN]; FLLEN], + _phantom: { + let () = Self::VALID; + PhantomData + }, + }; + + // For testing + #[allow(dead_code)] + const FLLEN: usize = FLLEN; + #[allow(dead_code)] + const SLLEN: usize = SLLEN; + + /// Evaluates successfully if the parameters are valid. + const VALID: () = { + if FLLEN == 0 { + const_panic!("`FLLEN` must not be zero"); + } + if SLLEN == 0 { + const_panic!("`SLLEN` must not be zero"); + } + if (FLBitmap::BITS as u128) < FLLEN as u128 { + const_panic!("`FLBitmap` should contain at least `FLLEN` bits"); + } + if (SLBitmap::BITS as u128) < SLLEN as u128 { + const_panic!("`SLBitmap` should contain at least `SLLEN` bits"); + } + }; + + /// The maximum size of each memory pool region. This is constrained by + /// the maximum block size of the segregated list to contain the initial + /// free memory block. + const MAX_POOL_SIZE: Option = { + let shift = GRANULARITY_LOG2 + FLLEN as u32; + if shift < USIZE_BITS { + Some(1 << shift) + } else { + None + } + }; + + /// `SLLEN.log2()` + const SLI: u32 = if SLLEN.is_power_of_two() { + SLLEN.trailing_zeros() + } else { + const_panic!("`SLLEN` is not power of two") + }; + + /// Find the free block list to store a free block of the specified size. + #[inline] + fn map_floor(size: usize) -> Option<(usize, usize)> { + debug_assert!(size >= GRANULARITY); + debug_assert!(size % GRANULARITY == 0); + let fl = USIZE_BITS - GRANULARITY_LOG2 - 1 - size.leading_zeros(); + + // The shift amount can be negative, and rotation lets us handle both + // cases without branching. Underflowed digits can be simply masked out + // in `map_floor`. + let sl = size.rotate_right((fl + GRANULARITY_LOG2).wrapping_sub(Self::SLI)); + + // The most significant one of `size` should be now at `sl[SLI]` + debug_assert!(((sl >> Self::SLI) & 1) == 1); + + // `fl` must be in a valid range + if fl as usize >= FLLEN { + return None; + } + + Some((fl as usize, sl & (SLLEN - 1))) + } + + /// Find the first free block list whose every item is at least as large + /// as the specified size. + #[inline] + fn map_ceil(size: usize) -> Option<(usize, usize)> { + debug_assert!(size >= GRANULARITY); + debug_assert!(size % GRANULARITY == 0); + let mut fl = USIZE_BITS - GRANULARITY_LOG2 - 1 - size.leading_zeros(); + + // The shift amount can be negative, and rotation lets us handle both + // cases without branching. + let mut sl = size.rotate_right((fl + GRANULARITY_LOG2).wrapping_sub(Self::SLI)); + + // The most significant one of `size` should be now at `sl[SLI]` + debug_assert!(((sl >> Self::SLI) & 1) == 1); + + // Underflowed digits appear in `sl[SLI + 1..USIZE-BITS]`. They should + // be rounded up + sl = (sl & (SLLEN - 1)) + (sl >= (1 << (Self::SLI + 1))) as usize; + + // if sl[SLI] { fl += 1; sl = 0; } + fl += (sl >> Self::SLI) as u32; + + // `fl` must be in a valid range + if fl as usize >= FLLEN { + return None; + } + + Some((fl as usize, sl & (SLLEN - 1))) + } + + const MAX_MAP_CEIL_AND_UNMAP_INPUT: usize = { + // The maximum value for which `map_ceil(x)` returns `(USIZE_BITS - + // GRANULARITY_LOG2 - 1, _)`, assuming `FLLEN == ∞` + let max1 = !(usize::MAX >> (Self::SLI + 1)); + + // Now take into account the fact that `FLLEN` is not actually infinity + if FLLEN as u32 - 1 < USIZE_BITS - GRANULARITY_LOG2 - 1 { + max1 >> ((USIZE_BITS - GRANULARITY_LOG2 - 1) - (FLLEN as u32 - 1)) + } else { + max1 + } + }; + + /// Find the first free block list whose every item is at least as large + /// as the specified size and get the list's minimum size. Returns `None` + /// if there isn't such a list, or the list's minimum size is not + /// representable in `usize`. + #[inline] + fn map_ceil_and_unmap(size: usize) -> Option { + debug_assert!(size >= GRANULARITY); + debug_assert!(size % GRANULARITY == 0); + + if size > Self::MAX_MAP_CEIL_AND_UNMAP_INPUT { + return None; + } + + let fl = USIZE_BITS - GRANULARITY_LOG2 - 1 - size.leading_zeros(); + + let list_min_size = if GRANULARITY_LOG2 < Self::SLI && fl < Self::SLI - GRANULARITY_LOG2 { + size + } else { + let shift = fl + GRANULARITY_LOG2 - Self::SLI; + + // round up + (size + ((1 << shift) - 1)) & !((1 << shift) - 1) + }; + + Some(list_min_size) + } + + /// Insert the specified free block to the corresponding free block list. + /// + /// Updates `FreeBlockHdr::{prev_free, next_free}`. + /// + /// # Safety + /// + /// - `*block.as_ptr()` must be owned by `self`. (It does not have to be + /// initialized, however.) + /// - `size` must have a corresponding free list, which does not currently + /// contain `block`. + /// + #[cfg_attr(target_arch = "wasm32", inline(never))] + unsafe fn link_free_block(&mut self, mut block: NonNull, size: usize) { + let (fl, sl) = Self::map_floor(size).unwrap_or_else(|| { + debug_assert!(false, "could not map size {}", size); + // Safety: It's unreachable + unreachable_unchecked() + }); + let first_free = &mut self.first_free[fl][sl]; + let next_free = mem::replace(first_free, Some(block)); + block.as_mut().next_free = next_free; + block.as_mut().prev_free = None; + if let Some(mut next_free) = next_free { + next_free.as_mut().prev_free = Some(block); + } + + self.fl_bitmap.set_bit(fl as u32); + self.sl_bitmap[fl].set_bit(sl as u32); + } + + /// Remove the specified free block from the corresponding free block list. + /// + /// # Safety + /// + /// - `size` must represent the specified free block's size. + /// - The free block must be currently included in a free block list. + /// + #[cfg_attr(target_arch = "wasm32", inline(never))] + unsafe fn unlink_free_block(&mut self, mut block: NonNull, size: usize) { + let next_free = block.as_mut().next_free; + let prev_free = block.as_mut().prev_free; + + if let Some(mut next_free) = next_free { + next_free.as_mut().prev_free = prev_free; + } + + if let Some(mut prev_free) = prev_free { + prev_free.as_mut().next_free = next_free; + } else { + let (fl, sl) = Self::map_floor(size).unwrap_or_else(|| { + debug_assert!(false, "could not map size {}", size); + // Safety: It's unreachable + unreachable_unchecked() + }); + let first_free = &mut self.first_free[fl][sl]; + + debug_assert_eq!(*first_free, Some(block)); + *first_free = next_free; + + if next_free.is_none() { + // The free list is now empty - update the bitmap + self.sl_bitmap[fl].clear_bit(sl as u32); + if self.sl_bitmap[fl] == SLBitmap::ZERO { + self.fl_bitmap.clear_bit(fl as u32); + } + } + } + } + + /// Create a new memory pool at the location specified by a slice pointer. + /// + /// Returns the actual number of bytes (counted from the beginning of + /// `block`) used to create the memory pool. This value is necessary to + /// calculate the start address to pass to [`Self::append_free_block_ptr`]. + /// + /// This method does nothing and returns `None` if the given memory block is + /// too small. + /// + /// # Time Complexity + /// + /// This method will complete in linear time (`O(block.len())`) because + /// it might need to divide the memory block to meet the maximum block size + /// requirement (`(GRANULARITY << FLLEN) - GRANULARITY`). + /// + /// # Examples + /// + /// ``` + /// use rlsf::Tlsf; + /// use std::{mem::MaybeUninit, ptr::NonNull}; + /// static mut POOL: MaybeUninit<[u8; 1024]> = MaybeUninit::uninit(); + /// let mut tlsf: Tlsf = Tlsf::INIT; + /// unsafe { + /// tlsf.insert_free_block_ptr(NonNull::new(POOL.as_mut_ptr()).unwrap()); + /// } + /// ``` + /// + /// # Safety + /// + /// The memory block will be considered owned by `self`. The memory block + /// must outlive `self`. + /// + /// # Panics + /// + /// This method never panics. + pub unsafe fn insert_free_block_ptr(&mut self, block: NonNull<[u8]>) -> Option { + let len = nonnull_slice_len(block); + + // Round up the starting address + let unaligned_start = block.as_ptr() as *mut u8 as usize; + let start = unaligned_start.wrapping_add(GRANULARITY - 1) & !(GRANULARITY - 1); + + let len = if let Some(x) = len + .checked_sub(start.wrapping_sub(unaligned_start)) + .filter(|&x| x >= GRANULARITY * 2) + { + // Round down + x & !(GRANULARITY - 1) + } else { + // The block is too small + return None; + }; + + // Safety: The slice being created here + let pool_len = self.insert_free_block_ptr_aligned(NonNull::new_unchecked( + core::ptr::slice_from_raw_parts_mut(start as *mut u8, len), + ))?; + + // Safety: The sum should not wrap around because it represents the size + // of a memory pool on memory + Some(NonZeroUsize::new_unchecked( + pool_len.get() + start.wrapping_sub(unaligned_start), + )) + } + + /// [`insert_free_block_ptr`] with a well-aligned slice passed by `block`. + pub(crate) unsafe fn insert_free_block_ptr_aligned( + &mut self, + block: NonNull<[u8]>, + ) -> Option { + let start = block.as_ptr() as *mut u8 as usize; + let mut size = nonnull_slice_len(block); + + let mut cursor = start; + + while size >= GRANULARITY * 2 { + let chunk_size = if let Some(max_pool_size) = Self::MAX_POOL_SIZE { + size.min(max_pool_size) + } else { + size + }; + + debug_assert_eq!(chunk_size % GRANULARITY, 0); + + // The new free block + // Safety: `cursor` is not zero. + let mut block = NonNull::new_unchecked(cursor as *mut FreeBlockHdr); + + // Initialize the new free block + block.as_mut().common = BlockHdr { + size: chunk_size - GRANULARITY, + prev_phys_block: None, + }; + + // Cap the end with a sentinel block (a permanently-used block) + let mut sentinel_block = block + .as_ref() + .common + .next_phys_block() + .cast::(); + + sentinel_block.as_mut().common = BlockHdr { + size: GRANULARITY | SIZE_USED | SIZE_SENTINEL, + prev_phys_block: Some(block.cast()), + }; + + // Link the free block to the corresponding free list + self.link_free_block(block, chunk_size - GRANULARITY); + + // `cursor` can reach `usize::MAX + 1`, but in such a case, this + // iteration must be the last one + debug_assert!(cursor.checked_add(chunk_size).is_some() || size == chunk_size); + size -= chunk_size; + cursor = cursor.wrapping_add(chunk_size); + } + + NonZeroUsize::new(cursor.wrapping_sub(start)) + } + + /// Extend an existing memory pool by incorporating the specified memory + /// block. + /// + /// Returns the number of incorporated bytes, counted from the beginning of + /// `block`. + /// + /// In the current implementation, this method can coalesce memory pools + /// only if the maximum pool size is outside the range of `usize`, i.e., + /// `log2(GRANULARITY) + FLLEN >= usize::BITS`. This is because it does not + /// track each pool's size and cannot check whether the resulting pool will + /// have a valid size. + /// + /// # Time Complexity + /// + /// This method will complete in linear time (`O(block.len())`) because + /// it might need to divide the memory block to meet the maximum block size + /// requirement (`(GRANULARITY << FLLEN) - GRANULARITY`). + /// + /// # Examples + /// + /// ``` + /// use rlsf::Tlsf; + /// use std::{mem::MaybeUninit, ptr::NonNull}; + /// + /// static mut POOL: MaybeUninit<[u8; 1024]> = MaybeUninit::uninit(); + /// let mut cursor = unsafe { POOL.as_mut_ptr() } as *mut u8; + /// let mut remaining_len = 1024; + /// + /// let mut tlsf: Tlsf = Tlsf::INIT; + /// let pool0_len = unsafe { + /// tlsf.insert_free_block_ptr(nonnull_slice_from_raw_parts( + /// NonNull::new(cursor).unwrap(), remaining_len / 2)) + /// }.unwrap().get(); + /// cursor = cursor.wrapping_add(pool0_len); + /// remaining_len -= pool0_len; + /// + /// let pool1_len = unsafe { + /// tlsf.append_free_block_ptr(nonnull_slice_from_raw_parts( + /// NonNull::new(cursor).unwrap(), remaining_len / 2)) + /// }; + /// cursor = cursor.wrapping_add(pool1_len); + /// remaining_len -= pool1_len; + /// + /// let pool2_len = unsafe { + /// tlsf.append_free_block_ptr(nonnull_slice_from_raw_parts( + /// NonNull::new(cursor).unwrap(), remaining_len)) + /// }; + /// cursor = cursor.wrapping_add(pool2_len); + /// remaining_len -= pool2_len; + /// + /// // polyfill for + /// fn nonnull_slice_from_raw_parts(ptr: NonNull, len: usize) -> NonNull<[T]> { + /// NonNull::new(std::ptr::slice_from_raw_parts_mut(ptr.as_ptr(), len)).unwrap() + /// } + /// ``` + /// + /// # Safety + /// + /// The memory block will be considered owned by `self`. The memory block + /// must outlive `self`. + /// + /// `block`'s starting address must match an existing memory pool's + /// ending address. See the above example for how to obtain one. + /// + /// # Panics + /// + /// This method never panics. + pub unsafe fn append_free_block_ptr(&mut self, block: NonNull<[u8]>) -> usize { + // Round down the length + let start = nonnull_slice_start(block); + let len = nonnull_slice_len(block) & !(GRANULARITY - 1); + + if Self::MAX_POOL_SIZE.is_some() { + // If `MAX_POOL_SIZE` is `Some(_)`, it's dangerous to coalesce + // memory pools of unknown sizes, so fall back to calling + // `insert_free_block_ptr_aligned`. + let block = nonnull_slice_from_raw_parts(start, len); + return self + .insert_free_block_ptr_aligned(block) + .map(NonZeroUsize::get) + .unwrap_or(0); + } else if len == 0 { + // `block` is so short that the `insert_free_block_ptr` will not + // even create a sentinel block. We'll corrupt the structure if we + // proceed. + return 0; + } + + let original_start = start.as_ptr(); + let mut start = original_start; + let end = (start as usize).wrapping_add(len); + + // The sentinel block from the preceding memory pool will be + // assimilated into `[start..end]`. + start = start.wrapping_sub(super::GRANULARITY); + let sentinel_block = start as *mut UsedBlockHdr; + debug_assert_eq!( + (*sentinel_block).common.size, + GRANULARITY | SIZE_USED | SIZE_SENTINEL + ); + + // The adjacent free block (if there's one) from the preceding memory + // pool will be assimilated into `[start..end]`. + let penultimate_block = (*sentinel_block).common.prev_phys_block.unwrap_or_else(|| { + debug_assert!(false, "sentinel block has no `prev_phys_block`"); + // Safety: It's unreachable + unreachable_unchecked() + }); + let last_nonassimilated_block; + if (penultimate_block.as_ref().size & SIZE_USED) == 0 { + let free_block = penultimate_block.cast::(); + let free_block_size = free_block.as_ref().common.size; + debug_assert_eq!( + free_block_size, + free_block.as_ref().common.size & SIZE_SIZE_MASK + ); + self.unlink_free_block(free_block, free_block_size); + + // Assimilation success + start = free_block.as_ptr() as *mut u8; + last_nonassimilated_block = free_block.as_ref().common.prev_phys_block; + } else { + // Assimilation failed + last_nonassimilated_block = Some(penultimate_block); + } + + // Safety: `start` points to a location inside an existion memory pool, + // so it's non-null + let block = nonnull_slice_from_raw_parts( + NonNull::new_unchecked(start), + end.wrapping_sub(start as usize), + ); + + // Create a memory pool + let pool_len = self + .insert_free_block_ptr_aligned(block) + .unwrap_or_else(|| { + debug_assert!(false, "`pool_size_to_contain_allocation` is an impostor"); + // Safety: It's unreachable + unreachable_unchecked() + }) + .get(); + + // Link the created pool's first block to the preceding memory pool's + // last non-assimilated block to form one continuous memory pool + let mut first_block = nonnull_slice_start(block).cast::(); + first_block.as_mut().common.prev_phys_block = last_nonassimilated_block; + + // Exclude the assimilated part from the returned value + pool_len - (original_start as usize).wrapping_sub(start as usize) + } + + /// Create a new memory pool at the location specified by a slice. + /// + /// This method does nothing if the given memory block is too small. + /// + /// (The return type is yet to be determined.) + /// + /// # Time Complexity + /// + /// See [`Self::insert_free_block_ptr`]. + /// + /// # Examples + /// + /// ``` + /// use rlsf::Tlsf; + /// use std::mem::MaybeUninit; + /// let mut pool = [MaybeUninit::uninit(); 1024]; + /// let mut tlsf: Tlsf = Tlsf::INIT; + /// tlsf.insert_free_block(&mut pool); + /// ``` + /// + /// The insertred memory block must outlive `self`: + /// + /// ```rust,compile_fail + /// use rlsf::Tlsf; + /// use std::mem::MaybeUninit; + /// let mut tlsf: Tlsf = Tlsf::INIT; + /// let mut pool = [MaybeUninit::uninit(); 1024]; + /// tlsf.insert_free_block(&mut pool); + /// drop(pool); // dropping the memory block first is not allowed + /// drop(tlsf); + /// ``` + /// + /// # Panics + /// + /// This method never panics. + #[inline] + pub fn insert_free_block(&mut self, block: &'pool mut [MaybeUninit]) -> impl Send + Sync { + // Safety: `block` is a mutable reference, which guarantees the absence + // of aliasing references. Being `'pool` means it will outlive `self`. + unsafe { self.insert_free_block_ptr(NonNull::new(block as *mut [_] as _).unwrap()) }; + } + + /// Calculate the minimum size of a `GRANULARITY`-byte aligned memory pool + /// (a well-aligned free memory block to be passed to + /// [`Self::insert_free_block`]) that is guaranteed to be able to contain + /// the specified allocation. + /// + /// Returns `None` if no amount of additional memory space can make the + /// allocation containable. + #[inline] + pub(crate) fn pool_size_to_contain_allocation(layout: Layout) -> Option { + // The extra bytes consumed by the header and padding. See + // `Tlsf::allocate` for details. + let max_overhead = + layout.align().saturating_sub(GRANULARITY / 2) + mem::size_of::(); + + // Which segregated list we would look if we were allocating this? + // And what's the minimum size of a free block required for inclusion + // in this list? + let search_size = layout.size().checked_add(max_overhead)?; + let search_size = search_size.checked_add(GRANULARITY - 1)? & !(GRANULARITY - 1); + let list_min_size = Self::map_ceil_and_unmap(search_size)?; + + // Add the sentinel block size + list_min_size.checked_add(GRANULARITY) + } + + /// Attempt to allocate a block of memory. + /// + /// Returns the starting address of the allocated memory block on success; + /// `None` otherwise. + /// + /// # Time Complexity + /// + /// This method will complete in constant time. + pub fn allocate(&mut self, layout: Layout) -> Option> { + unsafe { + // The extra bytes consumed by the header and padding. + // + // After choosing a free block, we need to adjust the payload's location + // to meet the alignment requirement. Every block is aligned to + // `GRANULARITY` bytes. `size_of::` is `GRANULARITY / 2` + // bytes, so the address immediately following `UsedBlockHdr` is only + // aligned to `GRANULARITY / 2` bytes. Consequently, we need to insert + // a padding containing at most `max(align - GRANULARITY / 2, 0)` bytes. + let max_overhead = + layout.align().saturating_sub(GRANULARITY / 2) + mem::size_of::(); + + // Search for a suitable free block + let search_size = layout.size().checked_add(max_overhead)?; + let search_size = search_size.checked_add(GRANULARITY - 1)? & !(GRANULARITY - 1); + let (fl, sl) = self.search_suitable_free_block_list_for_allocation(search_size)?; + + // Get a free block: `block` + let first_free = self.first_free.get_unchecked_mut(fl).get_unchecked_mut(sl); + let block = first_free.unwrap_or_else(|| { + debug_assert!(false, "bitmap outdated"); + // Safety: It's unreachable + unreachable_unchecked() + }); + let mut next_phys_block = block.as_ref().common.next_phys_block(); + let size_and_flags = block.as_ref().common.size; + let size = size_and_flags /* size_and_flags & SIZE_SIZE_MASK */; + debug_assert_eq!(size, size_and_flags & SIZE_SIZE_MASK); + + debug_assert!(size >= search_size); + + // Unlink the free block. We are not using `unlink_free_block` because + // we already know `(fl, sl)` and that `block.prev_free` is `None`. + *first_free = block.as_ref().next_free; + if let Some(mut next_free) = *first_free { + next_free.as_mut().prev_free = None; + } else { + // The free list is now empty - update the bitmap + let sl_bitmap = self.sl_bitmap.get_unchecked_mut(fl); + sl_bitmap.clear_bit(sl as u32); + if *sl_bitmap == SLBitmap::ZERO { + self.fl_bitmap.clear_bit(fl as u32); + } + } + + // Decide the starting address of the payload + let unaligned_ptr = block.as_ptr() as *mut u8 as usize + mem::size_of::(); + let ptr = NonNull::new_unchecked( + (unaligned_ptr.wrapping_add(layout.align() - 1) & !(layout.align() - 1)) as *mut u8, + ); + + if layout.align() < GRANULARITY { + debug_assert_eq!(unaligned_ptr, ptr.as_ptr() as usize); + } else { + debug_assert_ne!(unaligned_ptr, ptr.as_ptr() as usize); + } + + // Calculate the actual overhead and the final block size of the + // used block being created here + let overhead = ptr.as_ptr() as usize - block.as_ptr() as usize; + debug_assert!(overhead <= max_overhead); + + let new_size = overhead + layout.size(); + let new_size = (new_size + GRANULARITY - 1) & !(GRANULARITY - 1); + debug_assert!(new_size <= search_size); + + if new_size == size { + // The allocation completely fills this free block. + // Updating `next_phys_block.prev_phys_block` is unnecessary in this + // case because it's still supposed to point to `block`. + } else { + // The allocation partially fills this free block. Create a new + // free block header at `block + new_size..block + size` + // of length (`new_free_block_size`). + let mut new_free_block: NonNull = + NonNull::new_unchecked(block.cast::().as_ptr().add(new_size)).cast(); + let new_free_block_size = size - new_size; + + // Update `next_phys_block.prev_phys_block` to point to this new + // free block + // Invariant: No two adjacent free blocks + debug_assert!((next_phys_block.as_ref().size & SIZE_USED) != 0); + next_phys_block.as_mut().prev_phys_block = Some(new_free_block.cast()); + + // Create the new free block header + new_free_block.as_mut().common = BlockHdr { + size: new_free_block_size, + prev_phys_block: Some(block.cast()), + }; + self.link_free_block(new_free_block, new_free_block_size); + } + + // Turn `block` into a used memory block and initialize the used block + // header. `prev_phys_block` is already set. + let mut block = block.cast::(); + block.as_mut().common.size = new_size | SIZE_USED; + + // Place a `UsedBlockPad` (used by `used_block_hdr_for_allocation`) + if layout.align() >= GRANULARITY { + (*UsedBlockPad::get_for_allocation(ptr)).block_hdr = block; + } + + Some(ptr) + } + } + + /// Search for a non-empty free block list for allocation. + #[inline] + fn search_suitable_free_block_list_for_allocation( + &self, + min_size: usize, + ) -> Option<(usize, usize)> { + let (mut fl, mut sl) = Self::map_ceil(min_size)?; + + // Search in range `(fl, sl..SLLEN)` + sl = self.sl_bitmap[fl].bit_scan_forward(sl as u32) as usize; + if sl < SLLEN { + debug_assert!(self.sl_bitmap[fl].get_bit(sl as u32)); + + return Some((fl, sl)); + } + + // Search in range `(fl + 1.., ..)` + fl = self.fl_bitmap.bit_scan_forward(fl as u32 + 1) as usize; + if fl < FLLEN { + debug_assert!(self.fl_bitmap.get_bit(fl as u32)); + + sl = self.sl_bitmap[fl].trailing_zeros() as usize; + if sl >= SLLEN { + debug_assert!(false, "bitmap contradiction"); + unsafe { unreachable_unchecked() }; + } + + debug_assert!(self.sl_bitmap[fl].get_bit(sl as u32)); + Some((fl, sl)) + } else { + None + } + } + + /// Find the `UsedBlockHdr` for an allocation (any `NonNull` returned by + /// our allocation functions). + /// + /// # Safety + /// + /// - `ptr` must point to an allocated memory block returned by + /// `Self::{allocate, reallocate}`. + /// + /// - The memory block must have been allocated with the same alignment + /// ([`Layout::align`]) as `align`. + /// + #[inline] + unsafe fn used_block_hdr_for_allocation( + ptr: NonNull, + align: usize, + ) -> NonNull { + if align >= GRANULARITY { + // Read the header pointer + (*UsedBlockPad::get_for_allocation(ptr)).block_hdr + } else { + NonNull::new_unchecked(ptr.as_ptr().sub(GRANULARITY / 2)).cast() + } + } + + /// Find the `UsedBlockHdr` for an allocation (any `NonNull` returned by + /// our allocation functions) with an unknown alignment. + /// + /// Unlike `used_block_hdr_for_allocation`, this function does not require + /// knowing the allocation's alignment but might be less efficient. + /// + /// # Safety + /// + /// - `ptr` must point to an allocated memory block returned by + /// `Self::{allocate, reallocate}`. + /// + #[inline] + unsafe fn used_block_hdr_for_allocation_unknown_align( + ptr: NonNull, + ) -> NonNull { + // Case 1: `align >= GRANULARITY` + let c1_block_hdr_ptr: *const NonNull = + addr_of!((*UsedBlockPad::get_for_allocation(ptr)).block_hdr); + // Case 2: `align < GRANULARITY` + let c2_block_hdr = ptr.cast::().as_ptr().wrapping_sub(1); + let c2_prev_phys_block_ptr: *const Option> = + addr_of!((*c2_block_hdr).common.prev_phys_block); + + // They are both present at the same location, so we can be assured that + // their contents are initialized and we can read them safely without + // knowing which case applies first. + debug_assert_eq!( + c1_block_hdr_ptr as *const usize, + c2_prev_phys_block_ptr as *const usize + ); + + // Read it as `Option>`. + if let Some(block_ptr) = *c2_prev_phys_block_ptr { + // Where does the block represented by `block_ptr` end? + // (Note: `block_ptr.size` might include `SIZE_USED`.) + let block_end = block_ptr.as_ptr() as usize + block_ptr.as_ref().size; + + if ptr.as_ptr() as usize > block_end { + // The block represented by `block_ptr` does not include `ptr`. + // It's Case 2. + NonNull::new_unchecked(c2_block_hdr) + } else { + // `ptr` is inside the block - it's Case 1. + // (Note: `ptr == block_end` should count as being inside + // because the payload might be zero-sized.) + *c1_block_hdr_ptr + } + } else { + // It's non-nullable in Case 1, so we can rule out Case 1. + NonNull::new_unchecked(c2_block_hdr) + } + } + + /// Deallocate a previously allocated memory block. + /// + /// # Time Complexity + /// + /// This method will complete in constant time. + /// + /// # Safety + /// + /// - `ptr` must denote a memory block previously allocated via `self`. + /// - The memory block must have been allocated with the same alignment + /// ([`Layout::align`]) as `align`. + /// + pub unsafe fn deallocate(&mut self, ptr: NonNull, align: usize) { + // Safety: `ptr` is a previously allocated memory block with the same + // alignment as `align`. This is upheld by the caller. + let block = Self::used_block_hdr_for_allocation(ptr, align).cast::(); + self.deallocate_block(block); + } + + /// Deallocate a previously allocated memory block with an unknown alignment. + /// + /// Unlike `deallocate`, this function does not require knowing the + /// allocation's alignment but might be less efficient. + /// + /// # Time Complexity + /// + /// This method will complete in constant time. + /// + /// # Safety + /// + /// - `ptr` must denote a memory block previously allocated via `self`. + /// + pub(crate) unsafe fn deallocate_unknown_align(&mut self, ptr: NonNull) { + // Safety: `ptr` is a previously allocated memory block. This is upheld + // by the caller. + let block = Self::used_block_hdr_for_allocation_unknown_align(ptr).cast::(); + self.deallocate_block(block); + } + + /// Deallocate a previously allocated memory block. Takes a pointer to + /// `BlockHdr` instead of a payload pointer. + #[inline] + unsafe fn deallocate_block(&mut self, mut block: NonNull) { + let mut size = block.as_ref().size & !SIZE_USED; + debug_assert!((block.as_ref().size & SIZE_USED) != 0); + + // This variable tracks whose `prev_phys_block` we should update. + let mut new_next_phys_block; + + // Merge the created hole with the next block if the next block is a + // free block + // Safety: `block.common` should be fully up-to-date and valid + let next_phys_block = block.as_ref().next_phys_block(); + let next_phys_block_size_and_flags = next_phys_block.as_ref().size; + if (next_phys_block_size_and_flags & SIZE_USED) == 0 { + let next_phys_block_size = next_phys_block_size_and_flags; + debug_assert_eq!( + next_phys_block_size_and_flags & SIZE_SIZE_MASK, + next_phys_block_size + ); + + // It's coalescable. Add its size to `size`. This will transfer + // any `SIZE_LAST_IN_POOL` flag `next_phys_block` may have at + // the same time. + size += next_phys_block_size; + + // Safety: `next_phys_block` is a free block and therefore is not a + // sentinel block + new_next_phys_block = next_phys_block.as_ref().next_phys_block(); + + // Unlink `next_phys_block`. + self.unlink_free_block(next_phys_block.cast(), next_phys_block_size); + } else { + new_next_phys_block = next_phys_block; + } + + // Merge with the previous block if it's a free block. + if let Some(prev_phys_block) = block.as_ref().prev_phys_block { + let prev_phys_block_size_and_flags = prev_phys_block.as_ref().size; + + if (prev_phys_block_size_and_flags & SIZE_USED) == 0 { + let prev_phys_block_size = prev_phys_block_size_and_flags; + debug_assert_eq!( + prev_phys_block_size_and_flags & SIZE_SIZE_MASK, + prev_phys_block_size + ); + + // It's coalescable. Add its size to `size`. + size += prev_phys_block_size; + + // Unlink `prev_phys_block`. + self.unlink_free_block(prev_phys_block.cast(), prev_phys_block_size); + + // Move `block` to where `prev_phys_block` is located. By doing + // this, `block` will implicitly inherit `prev_phys_block. + // as_ref().prev_phys_block`. + block = prev_phys_block; + } + } + + // Write the new free block's size and flags. + debug_assert!((size & SIZE_USED) == 0); + block.as_mut().size = size; + + // Link this free block to the corresponding free list + let block = block.cast::(); + self.link_free_block(block, size); + + // Link `new_next_phys_block.prev_phys_block` to `block` + debug_assert_eq!(new_next_phys_block, block.as_ref().common.next_phys_block()); + new_next_phys_block.as_mut().prev_phys_block = Some(block.cast()); + } + + /// Get the payload size of the allocation. The returned size might be + /// larger than the size specified at the allocation time. + /// + /// # Safety + /// + /// - `ptr` must denote a memory block previously allocated via `Self`. + /// - The memory block must have been allocated with the same alignment + /// ([`Layout::align`]) as `align`. + /// + #[inline] + pub(crate) unsafe fn size_of_allocation(ptr: NonNull, align: usize) -> usize { + // Safety: `ptr` is a previously allocated memory block with the same + // alignment as `align`. This is upheld by the caller. + let block = Self::used_block_hdr_for_allocation(ptr, align); + + let size = block.as_ref().common.size - SIZE_USED; + debug_assert_eq!(size, block.as_ref().common.size & SIZE_SIZE_MASK); + + let block_end = block.as_ptr() as usize + size; + let payload_start = ptr.as_ptr() as usize; + block_end - payload_start + } + + /// Get the payload size of the allocation with an unknown alignment. The + /// returned size might be larger than the size specified at the allocation + /// time. + /// + /// # Safety + /// + /// - `ptr` must denote a memory block previously allocated via `Self`. + /// + #[inline] + pub(crate) unsafe fn size_of_allocation_unknown_align(ptr: NonNull) -> usize { + // Safety: `ptr` is a previously allocated memory block. + // This is upheld by the caller. + let block = Self::used_block_hdr_for_allocation_unknown_align(ptr); + + let size = block.as_ref().common.size - SIZE_USED; + debug_assert_eq!(size, block.as_ref().common.size & SIZE_SIZE_MASK); + + let block_end = block.as_ptr() as usize + size; + let payload_start = ptr.as_ptr() as usize; + block_end - payload_start + } + + // TODO: `reallocate_no_move` (constant-time reallocation) + + /// Shrink or grow a previously allocated memory block. + /// + /// Returns the new starting address of the memory block on success; + /// `None` otherwise. + /// + /// # Time Complexity + /// + /// Unlike other methods, this method will complete in linear time + /// (`O(old_size)`). + /// + /// # Safety + /// + /// - `ptr` must denote a memory block previously allocated via `self`. + /// - The memory block must have been allocated with the same alignment + /// ([`Layout::align`]) as `new_layout`. + /// + pub unsafe fn reallocate( + &mut self, + ptr: NonNull, + new_layout: Layout, + ) -> Option> { + // Safety: `ptr` is a previously allocated memory block with the same + // alignment as `align`. This is upheld by the caller. + let block = Self::used_block_hdr_for_allocation(ptr, new_layout.align()); + + // Do this early so that the compiler can de-duplicate common + // subexpressions such as `block.as_ref().common.size - SIZE_USED` + let old_size = Self::size_of_allocation(ptr, new_layout.align()); + + // First try to shrink or grow the block in-place (i.e., without + // allocating a whole new memory block). + if let Some(x) = self.reallocate_inplace(ptr, block, new_layout) { + return Some(x); + } + + // Allocate a whole new memory block + let new_ptr = self.allocate(new_layout)?; + + // Move the existing data into the new location + debug_assert!(new_layout.size() >= old_size); + core::ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_ptr(), old_size); + + // Deallocate the old memory block. + self.deallocate(ptr, new_layout.align()); + + Some(new_ptr) + } + + /// A subroutine of [`Self::reallocate`] that tries to reallocate a memory + /// block in-place. + #[inline] + unsafe fn reallocate_inplace( + &mut self, + ptr: NonNull, + mut block: NonNull, + new_layout: Layout, + ) -> Option> { + // The extra bytes consumed by the header and any padding + let overhead = ptr.as_ptr() as usize - block.as_ptr() as usize; + + // Calculate the new block size. Fail if this causes an overflow. + // Failing at this point does not necessarily mean the whole process of + // reallocation has failed; a new place with a smaller overhead could be + // found later (whether there's actually such a situation or not is yet + // to be proven). + let new_size = overhead.checked_add(new_layout.size())?; + let new_size = new_size.checked_add(GRANULARITY - 1)? & !(GRANULARITY - 1); + + let old_size = block.as_ref().common.size - SIZE_USED; + debug_assert_eq!(old_size, block.as_ref().common.size & SIZE_SIZE_MASK); + + // Shrinking + // ------------------------------------------------------------------ + + if new_size <= old_size { + if new_size == old_size { + // No size change + } else { + // Shrink the block, creating a new free block at the end + let shrink_by = old_size - new_size; + + // We will create a new free block at this address + let mut new_free_block: NonNull = + NonNull::new_unchecked(block.cast::().as_ptr().add(new_size)).cast(); + let mut new_free_block_size = shrink_by; + + // If the next block is a free block... + let mut next_phys_block = block.as_ref().common.next_phys_block(); + let next_phys_block_size_and_flags = next_phys_block.as_ref().size; + if (next_phys_block_size_and_flags & SIZE_USED) == 0 { + let next_phys_block_size = next_phys_block_size_and_flags; + debug_assert_eq!( + next_phys_block_size, + next_phys_block_size_and_flags & SIZE_SIZE_MASK + ); + + // Then we can merge this existing free block (`next_phys_block`) + // into the new one (`new_free_block`). + self.unlink_free_block(next_phys_block.cast(), next_phys_block_size); + new_free_block_size += next_phys_block_size; + + let mut next_next_phys_block = next_phys_block.as_ref().next_phys_block(); + next_next_phys_block.as_mut().prev_phys_block = Some(new_free_block.cast()); + } else { + // We can't merge a used block (`next_phys_block`) and + // a free block (`new_free_block`). + next_phys_block.as_mut().prev_phys_block = Some(new_free_block.cast()); + } + + new_free_block.as_mut().common = BlockHdr { + size: new_free_block_size, + prev_phys_block: Some(block.cast()), + }; + self.link_free_block(new_free_block, new_free_block_size); + + block.as_mut().common.size = new_size | SIZE_USED; + } + + return Some(ptr); + } + + // In-place non-moving reallocation + // ------------------------------------------------------------------ + + debug_assert!(new_size > old_size); + + let grow_by = new_size - old_size; + let next_phys_block = block.as_ref().common.next_phys_block(); + + // If we removed this block, there would be a continous free space of + // `moving_clearance` bytes, which is followed by `moving_clearance_end` + let mut moving_clearance = old_size; + let mut moving_clearance_end = next_phys_block; + + // Grow into the next free block. Fail if there isn't such a block. + #[allow(clippy::never_loop)] + 'nonmoving: loop { + let next_phys_block_size_and_flags = next_phys_block.as_ref().size; + + // Fail it isn't a free block. + if (next_phys_block_size_and_flags & SIZE_USED) != 0 { + break 'nonmoving; + } + + let mut next_phys_block_size = next_phys_block_size_and_flags; + debug_assert_eq!( + next_phys_block_size, + next_phys_block_size_and_flags & SIZE_SIZE_MASK + ); + + // Now we know it's really a free block. + let mut next_phys_block = next_phys_block.cast::(); + let mut next_next_phys_block = next_phys_block.as_ref().common.next_phys_block(); + + moving_clearance += next_phys_block_size; + moving_clearance_end = next_next_phys_block; + + if grow_by > next_phys_block_size { + // Can't fit + break 'nonmoving; + } + + self.unlink_free_block(next_phys_block, next_phys_block_size); + + if grow_by < next_phys_block_size { + // Can fit and there's some slack. Create a free block to fill + // the slack. + next_phys_block_size -= grow_by; + + next_phys_block = + NonNull::new_unchecked(block.cast::().as_ptr().add(new_size)).cast(); + next_phys_block.as_mut().common = BlockHdr { + size: next_phys_block_size, + prev_phys_block: Some(block.cast()), + }; + self.link_free_block(next_phys_block, next_phys_block_size); + + // Update `next_next_phys_block.prev_phys_block` accordingly + next_next_phys_block.as_mut().prev_phys_block = Some(next_phys_block.cast()); + } else { + // Can fit exactly. + debug_assert_eq!(grow_by, next_phys_block_size); + + // Update `next_next_phys_block.prev_phys_block` accordingly + next_next_phys_block.as_mut().prev_phys_block = Some(block.cast()); + } + + block.as_mut().common.size = new_size | SIZE_USED; + + return Some(ptr); + } + + // In-place moving reallocation + // ------------------------------------------------------------------ + + // The non-moving reallocation was failure. Now try the moving approach. + // I.e., grow into the previous free block as well. + // Get the previous block. If there isn't such a block, the moving + // approach will not improve the situation anyway, so return `None`. + let prev_phys_block = block.as_ref().common.prev_phys_block?; + let prev_phys_block_size_and_flags = prev_phys_block.as_ref().size; + + // Fail it isn't a free block. + if (prev_phys_block_size_and_flags & SIZE_USED) != 0 { + return None; + } + + let prev_phys_block_size = prev_phys_block_size_and_flags; + debug_assert_eq!( + prev_phys_block_size, + prev_phys_block_size_and_flags & SIZE_SIZE_MASK + ); + + // Now we know it's really a free block. + moving_clearance += prev_phys_block_size; + + // Decide the starting address of the payload + let unaligned_ptr = + prev_phys_block.as_ptr() as *mut u8 as usize + mem::size_of::(); + let new_ptr = NonNull::new_unchecked( + ((unaligned_ptr + new_layout.align() - 1) & !(new_layout.align() - 1)) as *mut u8, + ); + + // Calculate the new block size + let new_overhead = new_ptr.as_ptr() as usize - prev_phys_block.as_ptr() as usize; + let new_size = new_overhead.checked_add(new_layout.size())?; + let new_size = new_size.checked_add(GRANULARITY - 1)? & !(GRANULARITY - 1); + if new_size > moving_clearance { + // Can't fit + return None; + } + + // Unlink the existing free blocks included in `moving_clearance` + self.unlink_free_block(prev_phys_block.cast(), prev_phys_block_size); + let next_phys_block_size_and_flags = next_phys_block.as_ref().size; + if (next_phys_block_size_and_flags & SIZE_USED) == 0 { + let next_phys_block_size = next_phys_block_size_and_flags; + debug_assert_eq!( + next_phys_block_size, + next_phys_block_size_and_flags & SIZE_SIZE_MASK + ); + self.unlink_free_block(next_phys_block.cast(), next_phys_block_size); + } + + // Move the existing data into the new memory block. + core::ptr::copy( + ptr.as_ptr(), + new_ptr.as_ptr(), + new_layout.size().min(old_size - overhead), + ); + + // We'll replace `prev_phys_block` with a new used block. + let mut new_block = prev_phys_block.cast::(); + + if new_size == moving_clearance { + // The allocation completely fills this free block. + // Update `prev_phys_block` accordingly + moving_clearance_end.as_mut().prev_phys_block = Some(new_block.cast()); + } else { + // The allocation partially fills this free block. Create a new + // free block header at `new_block + new_size..new_block + // + moving_clearance`. + let mut new_free_block: NonNull = + NonNull::new_unchecked(new_block.cast::().as_ptr().add(new_size)).cast(); + let mut new_free_block_size = moving_clearance - new_size; + + // If the following block (`moving_clearance_end`) is a free block... + let moving_clearance_end_size_and_flags = moving_clearance_end.as_ref().size; + if (moving_clearance_end_size_and_flags & SIZE_USED) == 0 { + let moving_clearance_end_size = moving_clearance_end_size_and_flags; + debug_assert_eq!( + moving_clearance_end_size, + moving_clearance_end_size_and_flags & SIZE_SIZE_MASK + ); + + // Then we should merge this existing free block (`moving_clearance_end`) + // into the new one (`new_free_block`). + self.unlink_free_block(moving_clearance_end.cast(), moving_clearance_end_size); + new_free_block_size += moving_clearance_end_size_and_flags; + + let mut next_next_phys_block = moving_clearance_end.as_ref().next_phys_block(); + next_next_phys_block.as_mut().prev_phys_block = Some(new_free_block.cast()); + } else { + // We can't merge a used block (`moving_clearance_end`) and + // a free block (`new_free_block`). + moving_clearance_end.as_mut().prev_phys_block = Some(new_free_block.cast()); + } + + new_free_block.as_mut().common = BlockHdr { + size: new_free_block_size, + prev_phys_block: Some(new_block.cast()), + }; + self.link_free_block(new_free_block, new_free_block_size); + } + + // Turn `new_block` into a used memory block and initialize the used block + // header. `prev_phys_block` is already set. + new_block.as_mut().common.size = new_size | SIZE_USED; + + // Place a header pointer (used by `used_block_hdr_for_allocation`) + if new_layout.align() >= GRANULARITY { + (*UsedBlockPad::get_for_allocation(new_ptr)).block_hdr = new_block; + } + + Some(new_ptr) + } +} + +#[cfg(test)] +mod tests; diff --git a/src/r3_core/src/utils/alloc/rlsf/tlsf/tests.rs b/src/r3_core/src/utils/alloc/rlsf/tlsf/tests.rs new file mode 100644 index 00000000000..4813581cc92 --- /dev/null +++ b/src/r3_core/src/utils/alloc/rlsf/tlsf/tests.rs @@ -0,0 +1,470 @@ +use quickcheck_macros::quickcheck; +use std::{mem::MaybeUninit, prelude::v1::*}; + +use super::*; +use crate::{tests::ShadowAllocator, utils::nonnull_slice_from_raw_parts}; + +#[repr(align(64))] +struct Align(T); + +macro_rules! gen_test { + ($mod:ident, $($tt:tt)*) => { + mod $mod { + use super::*; + type TheTlsf<'a> = Tlsf<'a, $($tt)*>; + + #[test] + fn minimal() { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut tlsf: TheTlsf = Tlsf::INIT; + + let mut pool = [MaybeUninit::uninit(); 65536]; + tlsf.insert_free_block(&mut pool); + + log::trace!("tlsf = {:?}", tlsf); + + let ptr = tlsf.allocate(Layout::from_size_align(1, 1).unwrap()); + log::trace!("ptr = {:?}", ptr); + if let Some(ptr) = ptr { + unsafe { tlsf.deallocate(ptr, 1) }; + } + } + + #[test] + fn adaa() { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut tlsf: TheTlsf = Tlsf::INIT; + + let mut pool = [MaybeUninit::uninit(); 65536]; + tlsf.insert_free_block(&mut pool); + + log::trace!("tlsf = {:?}", tlsf); + + let ptr = tlsf.allocate(Layout::from_size_align(0, 1).unwrap()); + log::trace!("ptr = {:?}", ptr); + if let Some(ptr) = ptr { + unsafe { tlsf.deallocate(ptr, 1) }; + } + + let ptr = tlsf.allocate(Layout::from_size_align(0, 1).unwrap()); + log::trace!("ptr = {:?}", ptr); + + let ptr = tlsf.allocate(Layout::from_size_align(0, 1).unwrap()); + log::trace!("ptr = {:?}", ptr); + } + + #[test] + fn aadd() { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut tlsf: TheTlsf = Tlsf::INIT; + + let mut pool = Align([MaybeUninit::uninit(); 96]); + tlsf.insert_free_block(&mut pool.0); + + log::trace!("tlsf = {:?}", tlsf); + + let ptr1 = tlsf.allocate(Layout::from_size_align(0, 1).unwrap()); + log::trace!("ptr1 = {:?}", ptr1); + + let ptr2 = tlsf.allocate(Layout::from_size_align(0, 1).unwrap()); + log::trace!("ptr2 = {:?}", ptr2); + + if let (Some(ptr1), Some(ptr2)) = (ptr1, ptr2) { + unsafe { tlsf.deallocate(ptr1, 1) }; + unsafe { tlsf.deallocate(ptr2, 1) }; + } + } + + #[test] + fn ara() { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut tlsf: TheTlsf = Tlsf::INIT; + + let mut pool = Align([MaybeUninit::uninit(); 96]); + tlsf.insert_free_block(&mut pool.0); + + log::trace!("tlsf = {:?}", tlsf); + + let ptr = tlsf.allocate(Layout::from_size_align(17, 1).unwrap()); + log::trace!("ptr = {:?}", ptr); + + if let Some(ptr) = ptr { + unsafe { tlsf.reallocate(ptr, Layout::from_size_align(0, 1).unwrap()) }; + log::trace!("ptr = {:?}", ptr); + } + + let ptr = tlsf.allocate(Layout::from_size_align(0, 1).unwrap()); + log::trace!("ptr = {:?}", ptr); + } + + #[test] + fn append_free_block_ptr() { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut tlsf: TheTlsf = Tlsf::INIT; + + let mut pool = Align([MaybeUninit::uninit(); 512]); + let mut cursor = pool.0[0].as_mut_ptr() as *mut u8; + let mut remaining_len = 512; + + let pool0_len = unsafe { + tlsf.insert_free_block_ptr(nonnull_slice_from_raw_parts( + NonNull::new(cursor).unwrap(), remaining_len / 2)) + }.unwrap().get(); + cursor = cursor.wrapping_add(pool0_len); + remaining_len -= pool0_len; + + log::trace!("tlsf = {:?}", tlsf); + + // The memory pool is too small at this point + assert!(tlsf.allocate(Layout::from_size_align(384, 1).unwrap()).is_none()); + + let _pool1_len = unsafe { + tlsf.append_free_block_ptr(nonnull_slice_from_raw_parts( + NonNull::new(cursor).unwrap(), remaining_len)) + }; + + log::trace!("tlsf = {:?}", tlsf); + + let ptr = tlsf.allocate(Layout::from_size_align(384, 1).unwrap()); + log::trace!("ptr = {:?}", ptr); + + if TheTlsf::MAX_POOL_SIZE.is_none() { + // `append_free_block_ptr` coalesces consecutive + // memory pools, so this allocation should succeed + ptr.unwrap(); + } + } + + #[test] + fn insert_free_block_ptr_near_end_fail() { + let mut tlsf: TheTlsf = Tlsf::INIT; + unsafe { + // FIXME: Use `NonNull::<[T]>::slice_from_raw_parts` when it's stable + tlsf.insert_free_block_ptr( + NonNull::new(core::ptr::slice_from_raw_parts_mut( + (usize::MAX - GRANULARITY + 1) as _, + 0, + )) + .unwrap(), + ); + } + + // TODO: Allocation should fail + } + + #[test] + fn insert_free_block_ptr_near_end() { + let _tlsf: TheTlsf = Tlsf::INIT; + // TODO: Find a way to test this case + // + // unsafe { + // tlsf.insert_free_block_ptr(core::ptr::slice_from_raw_parts_mut( + // usize::MAX - GRANULARITY as _, + // GRANULARITY, + // )); + // } + } + + #[quickcheck] + fn random(pool_start: usize, pool_size: usize, bytecode: Vec) { + random_inner(pool_start, pool_size, bytecode); + } + + fn random_inner(pool_start: usize, pool_size: usize, bytecode: Vec) -> Option<()> { + let mut sa = ShadowAllocator::new(); + let mut tlsf: TheTlsf = Tlsf::INIT; + + let mut pool = Align([MaybeUninit::::uninit(); 65536]); + let pool_ptr; + // The end index of the memory pool inserted to `tlsf` + let mut pool_len; + // The end index of `pool` + let pool_limit; + unsafe { + // Insert some part of `pool` to `tlsf` + let pool_start = pool_start % 64; + let pool_size = pool_size % (pool.0.len() - 63); + pool_ptr = pool.0.as_mut_ptr().wrapping_add(pool_start) as *mut u8; + pool_limit = pool.0.len() - pool_start; + + let initial_pool = NonNull::new(std::ptr::slice_from_raw_parts_mut( + pool_ptr, + pool_size + )).unwrap(); + log::trace!("initial_pool = {:p}: [u8; {}]", pool_ptr, pool_size); + + pool_len = if let Some(pool_len) = tlsf.insert_free_block_ptr(initial_pool) { + let pool_len = pool_len.get(); + log::trace!("initial_pool (actual) = {:p}: {}", pool_ptr, pool_len); + sa.insert_free_block(std::ptr::slice_from_raw_parts( + pool_ptr, + pool_len + )); + Some(pool_len) + } else { + None + }; + } + + log::trace!("tlsf = {:?}", tlsf); + + #[derive(Debug)] + struct Alloc { + ptr: NonNull, + layout: Layout, + } + let mut allocs = Vec::new(); + + let mut it = bytecode.iter().cloned(); + loop { + match it.next()? % 8 { + 0..=2 => { + let len = u32::from_le_bytes([ + it.next()?, + it.next()?, + it.next()?, + 0, + ]); + let len = ((len as u64 * pool_size as u64) >> 24) as usize; + let align = 1 << (it.next()? % 6); + let layout = Layout::from_size_align(len, align).unwrap(); + log::trace!("alloc {:?}", layout); + + let ptr = tlsf.allocate(layout); + log::trace!(" → {:?}", ptr); + + if let Some(ptr) = ptr { + allocs.push(Alloc { ptr, layout }); + sa.allocate(layout, ptr); + } + } + 3..=5 => { + let alloc_i = it.next()?; + if allocs.len() > 0 { + let provide_align = (alloc_i as usize / allocs.len()) % 2 == 0; + let alloc = allocs.swap_remove(alloc_i as usize % allocs.len()); + log::trace!("dealloc {:?}", alloc); + + if provide_align { + unsafe { tlsf.deallocate(alloc.ptr, alloc.layout.align()) }; + } else { + unsafe { tlsf.deallocate_unknown_align(alloc.ptr) }; + } + sa.deallocate(alloc.layout, alloc.ptr); + } + } + 6 => { + let alloc_i = it.next()?; + if allocs.len() > 0 { + let len = u32::from_le_bytes([ + it.next()?, + it.next()?, + it.next()?, + 0, + ]); + let len = ((len as u64 * pool_size as u64) >> 24) as usize; + + let alloc_i = alloc_i as usize % allocs.len(); + let alloc = &mut allocs[alloc_i]; + log::trace!("realloc {:?} to {:?}", alloc, len); + + let new_layout = Layout::from_size_align(len, alloc.layout.align()).unwrap(); + + if let Some(ptr) = unsafe { tlsf.reallocate(alloc.ptr, new_layout) } { + log::trace!(" {:?} → {:?}", alloc.ptr, ptr); + sa.deallocate(alloc.layout, alloc.ptr); + alloc.ptr = ptr; + alloc.layout = new_layout; + sa.allocate(alloc.layout, alloc.ptr); + } else { + log::trace!(" {:?} → fail", alloc.ptr); + + } + } + } + 7 => { + let old_pool_len = if let Some(pool_len) = pool_len { + pool_len + } else { + continue; + }; + + // Incorporate some of `pool_len..pool_limit` + let available = pool_limit - old_pool_len; + if available == 0 { + continue; + } + + let num_appended_bytes = + u16::from_le_bytes([it.next()?, it.next()?]) as usize % (available + 1); + + let appended = nonnull_slice_from_raw_parts( + NonNull::new(pool_ptr.wrapping_add(old_pool_len)).unwrap(), + num_appended_bytes, + ); + + log::trace!("appending [{}..][..{}] to pool", old_pool_len, num_appended_bytes); + + let new_actual_appended_bytes = unsafe { tlsf.append_free_block_ptr(appended) }; + log::trace!(" actual appended range = [{}..][..{}]", old_pool_len, new_actual_appended_bytes); + sa.insert_free_block(std::ptr::slice_from_raw_parts( + pool_ptr.wrapping_add(old_pool_len), + new_actual_appended_bytes, + )); + pool_len = Some(old_pool_len + new_actual_appended_bytes); + } + _ => unreachable!(), + } + } + } + + #[test] + fn max_pool_size() { + if let Some(mps) = TheTlsf::MAX_POOL_SIZE { + // `MAX_POOL_SIZE - super::GRANULARITY` should + // be the maximum allowed block size. + assert!(TheTlsf::map_floor(mps - super::GRANULARITY).is_some()); + assert_eq!(TheTlsf::map_floor(mps), None); + } + } + + #[quickcheck] + fn map_ceil_and_unmap(size: usize, shift: u32) -> quickcheck::TestResult { + let size = size.rotate_left(shift % super::USIZE_BITS) + .wrapping_mul(super::GRANULARITY); + if size == 0 { + return quickcheck::TestResult::discard(); + } + let list_min_size = TheTlsf::map_ceil_and_unmap(size); + log::debug!("map_ceil_and_unmap({}) = {:?}", size, list_min_size); + if let Some(list_min_size) = list_min_size { + assert!(list_min_size >= size); + + // `list_min_size` must be the lower bound of some list + let (fl, sl) = TheTlsf::map_floor(list_min_size).unwrap(); + log::debug!("map_floor({}) = {:?}", list_min_size, (fl, sl)); + + // Since `list_min_size` is the lower bound of some list, + // `map_floor(list_min_size)` and `map_ceil(list_min_size)` + // should both return this list + assert_eq!(TheTlsf::map_floor(list_min_size), TheTlsf::map_ceil(list_min_size)); + + // `map_ceil_and_unmap(size)` must be the lower bound of the + // list returned by `map_ceil(size)` + assert_eq!(TheTlsf::map_floor(list_min_size), TheTlsf::map_ceil(size)); + } else { + // Find an explanation for `map_ceil_and_unmap` returning + // `None` + if let Some((fl, _sl)) = TheTlsf::map_ceil(size) { + // The lower bound of `(fl, sl)` is not representable + // in `usize` - this should be why + assert!(fl as u32 + super::GRANULARITY_LOG2 >= super::USIZE_BITS); + } else { + // `map_ceil_and_unmap` is `map_ceil` + infallible + // reverse mapping, and the suboperation `map_ceil` + // failed + } + } + + quickcheck::TestResult::passed() + } + + #[quickcheck] + fn map_ceil_and_unmap_huge(shift: u32) -> quickcheck::TestResult { + let size = usize::MAX << + (shift % (super::USIZE_BITS - super::GRANULARITY_LOG2) + + super::GRANULARITY_LOG2); + + if size == 0 || TheTlsf::map_ceil(size).is_some() { + return quickcheck::TestResult::discard(); + } + + // If `map_ceil` returns `None`, `map_ceil_and_unmap` must + // return `None`, too. + assert_eq!(TheTlsf::map_ceil_and_unmap(size), None); + quickcheck::TestResult::passed() + } + + #[quickcheck] + fn pool_size_to_contain_allocation(size: usize, align: u32)-> quickcheck::TestResult { + let align = (super::GRANULARITY / 2) << (align % 5); + let size = size.wrapping_mul(align); + if size > 500_000 { + // Let's limit pool size + return quickcheck::TestResult::discard(); + } + + let layout = Layout::from_size_align(size, align).unwrap(); + log::debug!("layout = {:?}", layout); + + let pool_size = if let Some(x) = TheTlsf::pool_size_to_contain_allocation(layout) { + x + } else { + return quickcheck::TestResult::discard(); + }; + log::debug!("pool_size_to_contain_allocation = {:?}", pool_size); + + assert_eq!(pool_size % super::GRANULARITY, 0); + + // Create a well-aligned pool + type Bk = Align<[u8; 64]>; + assert_eq!(std::mem::size_of::(), 64); + assert_eq!(std::mem::align_of::(), 64); + let mut pool: Vec> = Vec::new(); + pool.reserve((pool_size + 63) / 64); + let pool = unsafe { + std::slice::from_raw_parts_mut( + pool.as_mut_ptr() as *mut MaybeUninit, + pool_size, + ) + }; + + let mut tlsf: TheTlsf = Tlsf::INIT; + tlsf.insert_free_block(pool); + + // The allocation should success because + // `pool_size_to_contain_allocation` said so + tlsf.allocate(layout) + .expect("allocation unexpectedly failed"); + + quickcheck::TestResult::passed() + } + } + }; +} + +gen_test!(tlsf_u8_u8_1_1, u8, u8, 1, 1); +gen_test!(tlsf_u8_u8_1_2, u8, u8, 1, 2); +gen_test!(tlsf_u8_u8_1_4, u8, u8, 1, 4); +gen_test!(tlsf_u8_u8_1_8, u8, u8, 1, 8); +gen_test!(tlsf_u8_u8_3_4, u8, u8, 3, 4); +gen_test!(tlsf_u8_u8_5_4, u8, u8, 5, 4); +gen_test!(tlsf_u8_u8_8_1, u8, u8, 8, 1); +gen_test!(tlsf_u8_u8_8_8, u8, u8, 8, 8); +gen_test!(tlsf_u16_u8_3_4, u16, u8, 3, 4); +gen_test!(tlsf_u16_u8_11_4, u16, u8, 11, 4); +gen_test!(tlsf_u16_u8_16_4, u16, u8, 16, 4); +gen_test!(tlsf_u16_u16_3_16, u16, u16, 3, 16); +gen_test!(tlsf_u16_u16_11_16, u16, u16, 11, 16); +gen_test!(tlsf_u16_u16_16_16, u16, u16, 16, 16); +gen_test!(tlsf_u16_u32_3_16, u16, u32, 3, 16); +gen_test!(tlsf_u16_u32_11_16, u16, u32, 11, 16); +gen_test!(tlsf_u16_u32_16_16, u16, u32, 16, 16); +gen_test!(tlsf_u16_u32_3_32, u16, u32, 3, 32); +gen_test!(tlsf_u16_u32_11_32, u16, u32, 11, 32); +gen_test!(tlsf_u16_u32_16_32, u16, u32, 16, 32); +gen_test!(tlsf_u32_u32_20_32, u32, u32, 20, 32); +gen_test!(tlsf_u32_u32_27_32, u32, u32, 27, 32); +gen_test!(tlsf_u32_u32_28_32, u32, u32, 28, 32); +gen_test!(tlsf_u32_u32_29_32, u32, u32, 29, 32); +gen_test!(tlsf_u32_u32_32_32, u32, u32, 32, 32); +gen_test!(tlsf_u64_u8_58_8, u64, u64, 58, 8); +gen_test!(tlsf_u64_u8_59_8, u64, u64, 59, 8); +gen_test!(tlsf_u64_u8_60_8, u64, u64, 60, 8); +gen_test!(tlsf_u64_u8_61_8, u64, u64, 61, 8); +gen_test!(tlsf_u64_u8_64_8, u64, u64, 64, 8); diff --git a/src/r3_core/src/utils/alloc/rlsf/utils.rs b/src/r3_core/src/utils/alloc/rlsf/utils.rs new file mode 100644 index 00000000000..275815fc3fb --- /dev/null +++ b/src/r3_core/src/utils/alloc/rlsf/utils.rs @@ -0,0 +1,28 @@ +use core::{mem::MaybeUninit, ptr::NonNull}; + +/// Polyfill for +#[inline] +pub fn nonnull_slice_from_raw_parts(ptr: NonNull, len: usize) -> NonNull<[T]> { + unsafe { NonNull::new_unchecked(core::ptr::slice_from_raw_parts_mut(ptr.as_ptr(), len)) } +} + +/// Polyfill for +#[inline] +pub fn nonnull_slice_len(ptr: NonNull<[T]>) -> usize { + // Safety: We are just reading the slice length embedded in the fat + // pointer and not dereferencing the pointer. We also convert it + // to `*mut [MaybeUninit]` just in case because the slice + // might be uninitialized. + unsafe { (*(ptr.as_ptr() as *const [MaybeUninit])).len() } +} + +// Polyfill for +#[inline] +pub fn nonnull_slice_start(ptr: NonNull<[T]>) -> NonNull { + unsafe { NonNull::new_unchecked(ptr.as_ptr() as *mut T) } +} + +#[inline] +pub fn nonnull_slice_end(ptr: NonNull<[T]>) -> *mut T { + (ptr.as_ptr() as *mut T).wrapping_add(nonnull_slice_len(ptr)) +}