diff --git a/Cargo.toml b/Cargo.toml index 45f528520c..3c2fc8f179 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ sysinfo = "0.30.9" [dev-dependencies] paste = "1.0.8" rand = "0.8.5" +rand_chacha = "0.3.1" criterion = "0.4" [build-dependencies] diff --git a/benches/regular_bench/bulk_meta/bscan.rs b/benches/regular_bench/bulk_meta/bscan.rs new file mode 100644 index 0000000000..b8d57187ce --- /dev/null +++ b/benches/regular_bench/bulk_meta/bscan.rs @@ -0,0 +1,87 @@ +//! Benchmarks for scanning side metadata for non-zero bits. + +use criterion::Criterion; +use mmtk::util::{ + constants::LOG_BITS_IN_WORD, test_private::scan_non_zero_bits_in_metadata_bytes, Address, +}; +use rand::{seq::IteratorRandom, SeedableRng}; +use rand_chacha::ChaCha8Rng; + +fn allocate_aligned(size: usize) -> Address { + let ptr = unsafe { + std::alloc::alloc_zeroed(std::alloc::Layout::from_size_align(size, size).unwrap()) + }; + Address::from_mut_ptr(ptr) +} + +const BLOCK_BYTES: usize = 32768usize; // Match an Immix block size. + +// Asssume one-bit-per-word metadata (matching VO bits). +const BLOCK_META_BYTES: usize = BLOCK_BYTES >> LOG_BITS_IN_WORD; + +/// Set this many distinct bits in the bitmap. +const NUM_OBJECTS: usize = 200; + +/// Get a deterministic seeded Rng. +fn get_rng() -> ChaCha8Rng { + // Create an Rng from a seed and an explicit Rng type. + // Not secure at all, but completely deterministic and reproducible. + // The following seed is read from /dev/random + const SEED64: u64 = 0x4050cb1b5ab26c70; + ChaCha8Rng::seed_from_u64(SEED64) +} + +/// A bitmap, with known location of each bit for assertion. +struct PreparedBitmap { + start: Address, + end: Address, + set_bits: Vec<(Address, u8)>, +} + +/// Make a bitmap of the desired size and set bits. +fn make_standard_bitmap() -> PreparedBitmap { + let start = allocate_aligned(BLOCK_META_BYTES); + let end = start + BLOCK_META_BYTES; + let mut rng = get_rng(); + + let mut set_bits = (0..(BLOCK_BYTES >> LOG_BITS_IN_WORD)) + .choose_multiple(&mut rng, NUM_OBJECTS) + .iter() + .map(|total_bit_offset| { + let word_offset = total_bit_offset >> LOG_BITS_IN_WORD; + let bit_offset = total_bit_offset & ((1 << LOG_BITS_IN_WORD) - 1); + (start + (word_offset << LOG_BITS_IN_WORD), bit_offset as u8) + }) + .collect::>(); + + set_bits.sort(); + + for (addr, bit) in set_bits.iter() { + let word = unsafe { addr.load::() }; + let new_word = word | (1 << bit); + unsafe { addr.store::(new_word) }; + } + + PreparedBitmap { + start, + end, + set_bits, + } +} + +pub fn bench(c: &mut Criterion) { + c.bench_function("bscan_block", |b| { + let bitmap = make_standard_bitmap(); + let mut holder: Vec<(Address, u8)> = Vec::with_capacity(NUM_OBJECTS); + + b.iter(|| { + holder.clear(); + scan_non_zero_bits_in_metadata_bytes(bitmap.start, bitmap.end, &mut |addr, shift| { + holder.push((addr, shift)); + }); + }); + + assert_eq!(holder.len(), NUM_OBJECTS); + assert_eq!(holder, bitmap.set_bits); + }); +} diff --git a/benches/regular_bench/bulk_meta/mod.rs b/benches/regular_bench/bulk_meta/mod.rs index 488258cd96..2abc48b849 100644 --- a/benches/regular_bench/bulk_meta/mod.rs +++ b/benches/regular_bench/bulk_meta/mod.rs @@ -1,7 +1,9 @@ +pub mod bscan; pub mod bzero_bset; pub use criterion::Criterion; pub fn bench(c: &mut Criterion) { + bscan::bench(c); bzero_bset::bench(c); } diff --git a/src/mmtk.rs b/src/mmtk.rs index ed69f31e70..f5f7622130 100644 --- a/src/mmtk.rs +++ b/src/mmtk.rs @@ -6,6 +6,8 @@ use crate::plan::Plan; use crate::policy::sft_map::{create_sft_map, SFTMap}; use crate::scheduler::GCWorkScheduler; +#[cfg(feature = "vo_bit")] +use crate::util::address::ObjectReference; #[cfg(feature = "analysis")] use crate::util::analysis::AnalysisManager; use crate::util::finalizable_processor::FinalizableProcessor; @@ -467,4 +469,54 @@ impl MMTK { pub fn get_options(&self) -> &Options { &self.options } + + /// Enumerate objects in all spaces in this MMTK instance. + /// + /// The call-back function `f` is called for every object that has the valid object bit (VO + /// bit), i.e. objects that are allocated in the heap of this MMTK instance, but has not been + /// reclaimed, yet. + /// + /// # Notes about object initialization and finalization + /// + /// When this function visits an object, it only guarantees that its VO bit must have been set. + /// It is not guaranteed if the object has been "fully initialized" in the sense of the + /// programming language the VM is implementing. For example, the object header and the type + /// information may not have been written. + /// + /// It will also visit objects that have been "finalized" in the sense of the programming + /// langauge the VM is implementing, as long as the object has not been reclaimed by the GC, + /// yet. Be careful. If the object header is destroyed, it may not be safe to access such + /// objects in the high-level language. + /// + /// # Interaction with allocation and GC + /// + /// This function does not mutate the heap. It is safe if multiple threads execute this + /// function concurrently during mutator time. + /// + /// It has *undefined behavior* if allocation or GC happens while this function is being + /// executed. The VM binding must ensure no threads are allocating and GC does not start while + /// executing this function. One way to do this is stopping all mutators before calling this + /// function. + /// + /// Some high-level languages may provide an API that allows the user to allocate objects and + /// trigger GC while enumerating objects. One example is [`ObjectSpace::each_object`][os_eo] in + /// Ruby. The VM binding may use the callback of this function to save all visited object + /// references and let the user visit those references after this function returns. Make sure + /// those saved references are in the root set or in an object that will live through GCs before + /// the high-level language finishes visiting the saved object references. + /// + /// [os_eo]: https://docs.ruby-lang.org/en/master/ObjectSpace.html#method-c-each_object + #[cfg(feature = "vo_bit")] + pub fn enumerate_objects(&self, f: F) + where + F: FnMut(ObjectReference), + { + use crate::util::object_enum; + + let mut enumerator = object_enum::ClosureObjectEnumerator::<_, VM>::new(f); + let plan = self.get_plan(); + plan.for_each_space(&mut |space| { + space.enumerate_objects(&mut enumerator); + }) + } } diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index bf73255033..84e875191e 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -6,10 +6,11 @@ use crate::policy::sft::SFT; use crate::policy::space::{CommonSpace, Space}; use crate::scheduler::GCWorker; use crate::util::alloc::allocator::AllocatorContext; -use crate::util::copy::*; use crate::util::heap::{MonotonePageResource, PageResource}; use crate::util::metadata::{extract_side_metadata, MetadataSpec}; +use crate::util::object_enum::ObjectEnumerator; use crate::util::object_forwarding; +use crate::util::{copy::*, object_enum}; use crate::util::{Address, ObjectReference}; use crate::vm::*; use libc::{mprotect, PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE}; @@ -133,6 +134,10 @@ impl Space for CopySpace { fn set_copy_for_sft_trace(&mut self, semantics: Option) { self.common.copy = semantics; } + + fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { + object_enum::enumerate_blocks_from_monotonic_page_resource(enumerator, &self.pr); + } } impl crate::policy::gc_work::PolicyTraceObject for CopySpace { diff --git a/src/policy/immix/block.rs b/src/policy/immix/block.rs index 8c772924c8..85ed76ddef 100644 --- a/src/policy/immix/block.rs +++ b/src/policy/immix/block.rs @@ -10,6 +10,7 @@ use crate::util::metadata::side_metadata::{MetadataByteArrayRef, SideMetadataSpe use crate::util::metadata::vo_bit; #[cfg(feature = "object_pinning")] use crate::util::metadata::MetadataSpec; +use crate::util::object_enum::BlockMayHaveObjects; use crate::util::Address; use crate::vm::*; use std::sync::atomic::Ordering; @@ -86,6 +87,12 @@ impl Region for Block { } } +impl BlockMayHaveObjects for Block { + fn may_have_objects(&self) -> bool { + self.get_state() != BlockState::Unallocated + } +} + impl Block { /// Log pages in block pub const LOG_PAGES: usize = Self::LOG_BYTES - LOG_BYTES_IN_PAGE as usize; diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index e5be6bbcb4..0e3d303202 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -17,8 +17,9 @@ use crate::util::metadata::side_metadata::SideMetadataSpec; #[cfg(feature = "vo_bit")] use crate::util::metadata::vo_bit; use crate::util::metadata::{self, MetadataSpec}; +use crate::util::object_enum::ObjectEnumerator; use crate::util::object_forwarding; -use crate::util::{copy::*, epilogue}; +use crate::util::{copy::*, epilogue, object_enum}; use crate::util::{Address, ObjectReference}; use crate::vm::*; use crate::{ @@ -189,6 +190,10 @@ impl Space for ImmixSpace { fn set_copy_for_sft_trace(&mut self, _semantics: Option) { panic!("We do not use SFT to trace objects for Immix. set_copy_context() cannot be used.") } + + fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { + object_enum::enumerate_blocks_from_chunk_map::(enumerator, &self.chunk_map); + } } impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace { diff --git a/src/policy/immortalspace.rs b/src/policy/immortalspace.rs index 6d3e63922d..2ebb14e4d4 100644 --- a/src/policy/immortalspace.rs +++ b/src/policy/immortalspace.rs @@ -6,6 +6,7 @@ use crate::util::address::Address; use crate::util::heap::{MonotonePageResource, PageResource}; use crate::util::metadata::mark_bit::MarkState; +use crate::util::object_enum::{self, ObjectEnumerator}; use crate::util::{metadata, ObjectReference}; use crate::plan::{ObjectQueue, VectorObjectQueue}; @@ -112,6 +113,10 @@ impl Space for ImmortalSpace { fn release_multiple_pages(&mut self, _start: Address) { panic!("immortalspace only releases pages enmasse") } + + fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { + object_enum::enumerate_blocks_from_monotonic_page_resource(enumerator, &self.pr); + } } use crate::scheduler::GCWorker; diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index f2b9ec34d1..9d948201bb 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -8,6 +8,7 @@ use crate::policy::space::{CommonSpace, Space}; use crate::util::constants::BYTES_IN_PAGE; use crate::util::heap::{FreeListPageResource, PageResource}; use crate::util::metadata; +use crate::util::object_enum::ObjectEnumerator; use crate::util::opaque_pointer::*; use crate::util::treadmill::TreadMill; use crate::util::{Address, ObjectReference}; @@ -175,6 +176,10 @@ impl Space for LargeObjectSpace { fn release_multiple_pages(&mut self, start: Address) { self.pr.release_pages(start); } + + fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { + self.treadmill.enumerate_objects(enumerator); + } } use crate::scheduler::GCWorker; diff --git a/src/policy/lockfreeimmortalspace.rs b/src/policy/lockfreeimmortalspace.rs index 80b65e288d..dee3205181 100644 --- a/src/policy/lockfreeimmortalspace.rs +++ b/src/policy/lockfreeimmortalspace.rs @@ -16,6 +16,7 @@ use crate::util::heap::VMRequest; use crate::util::memory::MmapStrategy; use crate::util::metadata::side_metadata::SideMetadataContext; use crate::util::metadata::side_metadata::SideMetadataSanity; +use crate::util::object_enum::ObjectEnumerator; use crate::util::opaque_pointer::*; use crate::util::ObjectReference; use crate::vm::VMBinding; @@ -166,6 +167,10 @@ impl Space for LockFreeImmortalSpace { side_metadata_sanity_checker .verify_metadata_context(std::any::type_name::(), &self.metadata) } + + fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { + enumerator.visit_address_range(self.start, self.start + self.total_bytes); + } } use crate::plan::{ObjectQueue, VectorObjectQueue}; diff --git a/src/policy/markcompactspace.rs b/src/policy/markcompactspace.rs index 9db05e56aa..bc0f5659c2 100644 --- a/src/policy/markcompactspace.rs +++ b/src/policy/markcompactspace.rs @@ -11,6 +11,7 @@ use crate::util::constants::LOG_BYTES_IN_WORD; use crate::util::copy::CopySemantics; use crate::util::heap::{MonotonePageResource, PageResource}; use crate::util::metadata::{extract_side_metadata, vo_bit}; +use crate::util::object_enum::{self, ObjectEnumerator}; use crate::util::{Address, ObjectReference}; use crate::{vm::*, ObjectQueue}; use atomic::Ordering; @@ -131,6 +132,10 @@ impl Space for MarkCompactSpace { fn release_multiple_pages(&mut self, _start: Address) { panic!("markcompactspace only releases pages enmasse") } + + fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { + object_enum::enumerate_blocks_from_monotonic_page_resource(enumerator, &self.pr); + } } impl crate::policy::gc_work::PolicyTraceObject for MarkCompactSpace { diff --git a/src/policy/marksweepspace/malloc_ms/global.rs b/src/policy/marksweepspace/malloc_ms/global.rs index d95aa8437b..dbe386db5c 100644 --- a/src/policy/marksweepspace/malloc_ms/global.rs +++ b/src/policy/marksweepspace/malloc_ms/global.rs @@ -15,6 +15,7 @@ use crate::util::metadata::side_metadata::{ SideMetadataContext, SideMetadataSanity, SideMetadataSpec, }; use crate::util::metadata::MetadataSpec; +use crate::util::object_enum::ObjectEnumerator; use crate::util::opaque_pointer::*; use crate::util::Address; use crate::util::ObjectReference; @@ -229,6 +230,10 @@ impl Space for MallocSpace { side_metadata_sanity_checker .verify_metadata_context(std::any::type_name::(), &self.metadata) } + + fn enumerate_objects(&self, _enumerator: &mut dyn ObjectEnumerator) { + unimplemented!() + } } use crate::scheduler::GCWorker; diff --git a/src/policy/marksweepspace/native_ms/block.rs b/src/policy/marksweepspace/native_ms/block.rs index 6727430e9e..a150d974b5 100644 --- a/src/policy/marksweepspace/native_ms/block.rs +++ b/src/policy/marksweepspace/native_ms/block.rs @@ -7,6 +7,7 @@ use super::MarkSweepSpace; use crate::util::constants::LOG_BYTES_IN_PAGE; use crate::util::heap::chunk_map::*; use crate::util::linear_scan::Region; +use crate::util::object_enum::BlockMayHaveObjects; use crate::vm::ObjectModel; use crate::{ util::{ @@ -48,6 +49,12 @@ impl Region for Block { } } +impl BlockMayHaveObjects for Block { + fn may_have_objects(&self) -> bool { + self.get_state() != BlockState::Unallocated + } +} + impl Block { /// Log pages in block pub const LOG_PAGES: usize = Self::LOG_BYTES - LOG_BYTES_IN_PAGE as usize; diff --git a/src/policy/marksweepspace/native_ms/global.rs b/src/policy/marksweepspace/native_ms/global.rs index b4699db30c..4a86dda6fa 100644 --- a/src/policy/marksweepspace/native_ms/global.rs +++ b/src/policy/marksweepspace/native_ms/global.rs @@ -11,6 +11,7 @@ use crate::{ epilogue, heap::{BlockPageResource, PageResource}, metadata::{self, side_metadata::SideMetadataSpec, MetadataSpec}, + object_enum::{self, ObjectEnumerator}, ObjectReference, }, vm::{ActivePlan, VMBinding}, @@ -247,6 +248,10 @@ impl Space for MarkSweepSpace { fn release_multiple_pages(&mut self, _start: crate::util::Address) { todo!() } + + fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { + object_enum::enumerate_blocks_from_chunk_map::(enumerator, &self.chunk_map); + } } impl crate::policy::gc_work::PolicyTraceObject for MarkSweepSpace { diff --git a/src/policy/space.rs b/src/policy/space.rs index 556e0c6bfe..0329c995b7 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -5,6 +5,7 @@ use crate::util::conversions::*; use crate::util::metadata::side_metadata::{ SideMetadataContext, SideMetadataSanity, SideMetadataSpec, }; +use crate::util::object_enum::ObjectEnumerator; use crate::util::Address; use crate::util::ObjectReference; @@ -348,6 +349,28 @@ pub trait Space: 'static + SFT + Sync + Downcast { side_metadata_sanity_checker .verify_metadata_context(std::any::type_name::(), &self.common().metadata) } + + /// Enumerate objects in the current space. + /// + /// Implementers can use the `enumerator` to report + /// + /// - individual objects within the space using `enumerator.visit_object`, and + /// - ranges of address that may contain objects using `enumerator.visit_address_range`. The + /// caller will then enumerate objects in the range using the VO bits metadata. + /// + /// Each object in the space shall be covered by one of the two methods above. + /// + /// # Implementation considerations + /// + /// **Skipping empty ranges**: When enumerating address ranges, spaces can skip ranges (blocks, + /// chunks, etc.) that are guarenteed not to contain objects. + /// + /// **Dynamic dispatch**: Because `Space` is a trait object type and `enumerator` is a `dyn` + /// reference, invoking methods of `enumerator` involves a dynamic dispatching. But the + /// overhead is OK if we call it a block at a time because scanning the VO bits will dominate + /// the execution time. For LOS, it will be cheaper to enumerate individual objects than + /// scanning VO bits because it is sparse. + fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator); } /// Print the VM map for a space. diff --git a/src/policy/vmspace.rs b/src/policy/vmspace.rs index 5d38a77042..38fc6011da 100644 --- a/src/policy/vmspace.rs +++ b/src/policy/vmspace.rs @@ -11,6 +11,7 @@ use crate::util::heap::PageResource; use crate::util::metadata::mark_bit::MarkState; #[cfg(feature = "set_unlog_bits_vm_space")] use crate::util::metadata::MetadataSpec; +use crate::util::object_enum::ObjectEnumerator; use crate::util::opaque_pointer::*; use crate::util::ObjectReference; use crate::vm::{ObjectModel, VMBinding}; @@ -145,6 +146,13 @@ impl Space for VMSpace { // mmapped by the runtime rather than us). So we we use SFT here. SFT_MAP.get_checked(start).name() == self.name() } + + fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { + let external_pages = self.pr.get_external_pages(); + for ep in external_pages.iter() { + enumerator.visit_address_range(ep.start, ep.end); + } + } } use crate::scheduler::GCWorker; diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index 23764525a7..ad39a3d565 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -1115,6 +1115,109 @@ impl SideMetadataSpec { res.map(|addr| addr.align_down(1 << self.log_bytes_in_region)) } + + /// Search for data addresses that have non zero values in the side metadata. This method is + /// primarily used for heap traversal by scanning the VO bits. + /// + /// This function searches the side metadata for the data address range from `data_start_addr` + /// (inclusive) to `data_end_addr` (exclusive). The data address range must be fully mapped. + /// + /// For each data region that has non-zero side metadata, `visit_data` is called with the lowest + /// address of that region. Note that it may not be the original address used to set the + /// metadata bits. + pub fn scan_non_zero_values( + &self, + data_start_addr: Address, + data_end_addr: Address, + visit_data: &mut impl FnMut(Address), + ) { + if self.uses_contiguous_side_metadata() && self.log_num_of_bits == 0 { + // Contiguous one-bit-per-region side metadata + // TODO: VO bits is one-bit-per-word. But if we want to scan other metadata (such as + // the forwarding bits which has two bits per word), we will need to refactor the + // algorithm of `scan_non_zero_values_fast`. + self.scan_non_zero_values_fast(data_start_addr, data_end_addr, visit_data); + } else { + // TODO: VO bits are always contiguous. But if we want to scan other metadata, such as + // side mark bits, we need to refactor `bulk_update_metadata` to support `FnMut`, too, + // and use it to apply `scan_non_zero_values_fast` on each contiguous side metadata + // range. + warn!( + "We are trying to search for non zero bits in a discontiguous side metadata \ + or the metadata has more than one bit per region. \ + The performance is slow, as MMTk does not optimize for this case." + ); + self.scan_non_zero_values_simple::(data_start_addr, data_end_addr, visit_data); + } + } + + fn scan_non_zero_values_simple( + &self, + data_start_addr: Address, + data_end_addr: Address, + visit_data: &mut impl FnMut(Address), + ) { + let region_bytes = 1usize << self.log_bytes_in_region; + + let mut cursor = data_start_addr; + while cursor < data_end_addr { + debug_assert!(cursor.is_mapped()); + + // If we find non-zero value, just call back. + if !unsafe { self.load::(cursor).is_zero() } { + visit_data(cursor); + } + cursor += region_bytes; + } + } + + fn scan_non_zero_values_fast( + &self, + data_start_addr: Address, + data_end_addr: Address, + visit_data: &mut impl FnMut(Address), + ) { + debug_assert!(self.uses_contiguous_side_metadata()); + debug_assert_eq!(self.log_num_of_bits, 0); + + // Then figure out the start and end metadata address and bits. + let start_meta_addr = address_to_contiguous_meta_address(self, data_start_addr); + let start_meta_shift = meta_byte_lshift(self, data_start_addr); + let end_meta_addr = address_to_contiguous_meta_address(self, data_end_addr); + let end_meta_shift = meta_byte_lshift(self, data_end_addr); + + let mut visitor = |range| { + match range { + BitByteRange::Bytes { start, end } => { + helpers::scan_non_zero_bits_in_metadata_bytes(start, end, &mut |addr, bit| { + visit_data(helpers::contiguous_meta_address_to_address(self, addr, bit)); + }); + } + BitByteRange::BitsInByte { + addr, + bit_start, + bit_end, + } => helpers::scan_non_zero_bits_in_metadata_bits( + addr, + bit_start, + bit_end, + &mut |addr, bit| { + visit_data(helpers::contiguous_meta_address_to_address(self, addr, bit)); + }, + ), + } + false + }; + + ranges::break_bit_range( + start_meta_addr, + start_meta_shift, + end_meta_addr, + end_meta_shift, + false, + &mut visitor, + ); + } } impl fmt::Debug for SideMetadataSpec { diff --git a/src/util/metadata/side_metadata/helpers.rs b/src/util/metadata/side_metadata/helpers.rs index 845ad79666..9eb8823649 100644 --- a/src/util/metadata/side_metadata/helpers.rs +++ b/src/util/metadata/side_metadata/helpers.rs @@ -1,3 +1,4 @@ +use super::ranges::BitOffset; use super::SideMetadataSpec; use crate::util::constants::LOG_BYTES_IN_PAGE; use crate::util::constants::{BITS_IN_WORD, BYTES_IN_PAGE, LOG_BITS_IN_BYTE}; @@ -269,6 +270,59 @@ fn find_last_non_zero_bit_in_u8(byte_value: u8) -> Option { } } +pub fn scan_non_zero_bits_in_metadata_bytes( + meta_start: Address, + meta_end: Address, + visit_bit: &mut impl FnMut(Address, BitOffset), +) { + use crate::util::constants::BYTES_IN_ADDRESS; + + let mut cursor = meta_start; + while cursor < meta_end && !cursor.is_aligned_to(BYTES_IN_ADDRESS) { + let byte = unsafe { cursor.load::() }; + scan_non_zero_bits_in_metadata_word(cursor, byte as usize, visit_bit); + cursor += 1usize; + } + + while cursor + BYTES_IN_ADDRESS < meta_end { + let word = unsafe { cursor.load::() }; + scan_non_zero_bits_in_metadata_word(cursor, word, visit_bit); + cursor += BYTES_IN_ADDRESS; + } + + while cursor < meta_end { + let byte = unsafe { cursor.load::() }; + scan_non_zero_bits_in_metadata_word(cursor, byte as usize, visit_bit); + cursor += 1usize; + } +} + +fn scan_non_zero_bits_in_metadata_word( + meta_addr: Address, + mut word: usize, + visit_bit: &mut impl FnMut(Address, BitOffset), +) { + while word != 0 { + let bit = word.trailing_zeros(); + visit_bit(meta_addr, bit as u8); + word = word & (word - 1); + } +} + +pub fn scan_non_zero_bits_in_metadata_bits( + meta_addr: Address, + bit_start: BitOffset, + bit_end: BitOffset, + visit_bit: &mut impl FnMut(Address, BitOffset), +) { + let byte = unsafe { meta_addr.load::() }; + for bit in bit_start..bit_end { + if byte & (1 << bit) != 0 { + visit_bit(meta_addr, bit); + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/util/metadata/side_metadata/mod.rs b/src/util/metadata/side_metadata/mod.rs index 612c69223f..16111b42c1 100644 --- a/src/util/metadata/side_metadata/mod.rs +++ b/src/util/metadata/side_metadata/mod.rs @@ -2,7 +2,7 @@ // For convenience, this module is public and the bindings may create and use side metadata for their purpose. mod constants; -mod helpers; +pub(crate) mod helpers; #[cfg(target_pointer_width = "32")] mod helpers_32; diff --git a/src/util/metadata/vo_bit/mod.rs b/src/util/metadata/vo_bit/mod.rs index e7b0f2551b..3ae0aee8d7 100644 --- a/src/util/metadata/vo_bit/mod.rs +++ b/src/util/metadata/vo_bit/mod.rs @@ -201,7 +201,7 @@ fn get_in_object_address_for_potential_object(potential_obj: Addr } /// Get the object reference from an aligned address where VO bit is set. -fn get_object_ref_for_vo_addr(vo_addr: Address) -> ObjectReference { +pub(crate) fn get_object_ref_for_vo_addr(vo_addr: Address) -> ObjectReference { let addr = vo_addr.offset(-VM::VMObjectModel::IN_OBJECT_ADDRESS_OFFSET); let aligned = addr.align_up(ObjectReference::ALIGNMENT); unsafe { ObjectReference::from_raw_address_unchecked(aligned) } diff --git a/src/util/mod.rs b/src/util/mod.rs index bac77a0b88..d22c29a2e3 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -53,6 +53,7 @@ pub(crate) mod erase_vm; pub(crate) mod finalizable_processor; /// Logger initialization pub(crate) mod logger; +pub(crate) mod object_enum; /// Forwarding word in object copying. pub(crate) mod object_forwarding; /// Reference processing implementation. diff --git a/src/util/object_enum.rs b/src/util/object_enum.rs new file mode 100644 index 0000000000..61378f7a57 --- /dev/null +++ b/src/util/object_enum.rs @@ -0,0 +1,107 @@ +//! Helper types for object enumeration + +use std::marker::PhantomData; + +use crate::vm::VMBinding; + +use super::{ + heap::{ + chunk_map::{ChunkMap, ChunkState}, + MonotonePageResource, + }, + linear_scan::Region, + metadata::{side_metadata::spec_defs::VO_BIT, vo_bit}, + Address, ObjectReference, +}; + +/// A trait for enumerating objects in spaces, used by [`Space::enumerate_objects`]. +/// +/// [`Space::enumerate_objects`]: crate::policy::space::Space::enumerate_objects +pub trait ObjectEnumerator { + /// Visit a single object. + fn visit_object(&mut self, object: ObjectReference); + /// Visit an address range that may contain objects. + fn visit_address_range(&mut self, start: Address, end: Address); +} + +/// An implementation of `ObjectEnumerator` that wraps a callback. +pub struct ClosureObjectEnumerator +where + F: FnMut(ObjectReference), + VM: VMBinding, +{ + object_callback: F, + phantom_data: PhantomData, +} + +impl ClosureObjectEnumerator +where + F: FnMut(ObjectReference), + VM: VMBinding, +{ + pub fn new(object_callback: F) -> Self { + Self { + object_callback, + phantom_data: PhantomData, + } + } +} + +impl ObjectEnumerator for ClosureObjectEnumerator +where + F: FnMut(ObjectReference), + VM: VMBinding, +{ + fn visit_object(&mut self, object: ObjectReference) { + (self.object_callback)(object); + } + + fn visit_address_range(&mut self, start: Address, end: Address) { + VO_BIT.scan_non_zero_values::(start, end, &mut |address| { + let object = vo_bit::get_object_ref_for_vo_addr::(address); + (self.object_callback)(object); + }) + } +} + +/// Allow querying if a block may have objects. `MarkSweepSpace` and `ImmixSpace` use different +/// `Block` types, and they have different block states. This trait lets both `Block` types provide +/// the same `may_have_objects` method. +pub(crate) trait BlockMayHaveObjects: Region { + /// Return `true` if the block may contain valid objects (objects with the VO bit set). Return + /// `false` otherwise. + /// + /// This function is used during object enumeration to filter out memory regions that do not + /// contain objects. Because object enumeration may happen at mutator time, and another mutators + /// may be allocating objects, this function only needs to reflect the state of the block at the + /// time of calling, and is allowed to conservatively return `true`. + fn may_have_objects(&self) -> bool; +} + +pub(crate) fn enumerate_blocks_from_chunk_map( + enumerator: &mut dyn ObjectEnumerator, + chunk_map: &ChunkMap, +) where + B: BlockMayHaveObjects, +{ + for chunk in chunk_map.all_chunks() { + if chunk_map.get(chunk) == ChunkState::Allocated { + for block in chunk.iter_region::() { + if block.may_have_objects() { + enumerator.visit_address_range(block.start(), block.end()); + } + } + } + } +} + +pub(crate) fn enumerate_blocks_from_monotonic_page_resource( + enumerator: &mut dyn ObjectEnumerator, + pr: &MonotonePageResource, +) where + VM: VMBinding, +{ + for (start, size) in pr.iterate_allocated_regions() { + enumerator.visit_address_range(start, start + size); + } +} diff --git a/src/util/test_private/mod.rs b/src/util/test_private/mod.rs index 783dbeeac8..c3f4dfceae 100644 --- a/src/util/test_private/mod.rs +++ b/src/util/test_private/mod.rs @@ -8,6 +8,7 @@ //! and the compiler usually fails to make the right decision given that those functions are not //! used often, and we don't compile the benchmarks using feedback-directed optimizations. +pub use crate::util::metadata::side_metadata::helpers::scan_non_zero_bits_in_metadata_bytes; use crate::util::metadata::side_metadata::SideMetadataSpec; use super::Address; diff --git a/src/util/treadmill.rs b/src/util/treadmill.rs index e8cfe92d4f..96b5eb9e7d 100644 --- a/src/util/treadmill.rs +++ b/src/util/treadmill.rs @@ -4,6 +4,8 @@ use std::sync::Mutex; use crate::util::ObjectReference; +use super::object_enum::ObjectEnumerator; + pub struct TreadMill { from_space: Mutex>, to_space: Mutex>, @@ -99,6 +101,17 @@ impl TreadMill { trace!("Flipped from_space and to_space"); } } + + pub(crate) fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) { + let mut visit_objects = |set: &Mutex>| { + let set = set.lock().unwrap(); + for object in set.iter() { + enumerator.visit_object(*object); + } + }; + visit_objects(&self.alloc_nursery); + visit_objects(&self.to_space); + } } impl Default for TreadMill { diff --git a/src/vm/tests/mock_tests/mock_test_heap_traversal.rs b/src/vm/tests/mock_tests/mock_test_heap_traversal.rs new file mode 100644 index 0000000000..430c46ff2a --- /dev/null +++ b/src/vm/tests/mock_tests/mock_test_heap_traversal.rs @@ -0,0 +1,87 @@ +// GITHUB-CI: MMTK_PLAN=NoGC,MarkSweep,MarkCompact,SemiSpace,Immix +// GITHUB-CI: FEATURES=vo_bit + +// Note on the plans chosen for CI: +// - Those plans cover the MarkSweepSpace, MarkCompactSpace, CopySpace and ImmixSpace. +// Each plan other than NoGC also include ImmortalSpace and LOS. +// - PageProtect consumes too much memory and the test will fail with the default heap size +// chosen by the MutatorFixture. + +use std::collections::HashSet; + +use constants::BYTES_IN_WORD; + +use super::mock_test_prelude::*; + +use crate::{util::*, AllocationSemantics, MMTK}; + +lazy_static! { + static ref FIXTURE: Fixture = Fixture::new(); +} + +pub fn get_all_objects(mmtk: &'static MMTK) -> HashSet { + let mut result = HashSet::new(); + mmtk.enumerate_objects(|object| { + result.insert(object); + }); + result +} + +#[test] +pub fn test_heap_traversal() { + with_mockvm( + default_setup, + || { + FIXTURE.with_fixture_mut(|fixture| { + let mmtk = fixture.mmtk(); + let traversal0 = get_all_objects(mmtk); + assert!(traversal0.is_empty()); + + let mutator = &mut fixture.mutator; + + let align = BYTES_IN_WORD; + + let mut new_obj = |size: usize, semantics: AllocationSemantics| { + let start = memory_manager::alloc(mutator, size, align, 0, semantics); + let object = MockVM::object_start_to_ref(start); + memory_manager::post_alloc(mutator, object, size, semantics); + object + }; + + let mut known_objects = HashSet::new(); + + let mut new_and_assert = |size: usize, semantics: AllocationSemantics| { + let object = new_obj(size, semantics); // a random size + known_objects.insert(object); + let traversal = get_all_objects(mmtk); + assert_eq!(traversal, known_objects); + }; + + { + use AllocationSemantics::*; + + // Add some single objects. Size doesn't matter. + new_and_assert(40, Default); + new_and_assert(64, Default); + new_and_assert(96, Immortal); + new_and_assert(131000, Los); + + // Allocate enough memory to fill up a 64KB Immix block + for _ in 0..1000 { + new_and_assert(112, Default); + } + // Allocate a few objects in the immortal space. + for _ in 0..10 { + new_and_assert(64, Immortal); + } + // And a few objects in the LOS. + for _ in 0..3 { + // The MutatorFixture only has 1MB of memory. Don't allocate too much. + new_and_assert(65504, Immortal); + } + } + }); + }, + no_cleanup, + ) +} diff --git a/src/vm/tests/mock_tests/mod.rs b/src/vm/tests/mock_tests/mod.rs index 751bc58c08..114d8f1859 100644 --- a/src/vm/tests/mock_tests/mod.rs +++ b/src/vm/tests/mock_tests/mod.rs @@ -35,6 +35,8 @@ mod mock_test_conservatism; #[cfg(target_os = "linux")] mod mock_test_handle_mmap_conflict; mod mock_test_handle_mmap_oom; +#[cfg(feature = "vo_bit")] +mod mock_test_heap_traversal; mod mock_test_init_fork; #[cfg(feature = "is_mmtk_object")] mod mock_test_internal_ptr_before_object_ref;