Block-based algo to compute the table

src/hotspot/share/gc/serial/serialCompressor.cpp

@@ -155,7 +155,7 @@ class SCCompacter {
     HeapWord* block_base = align_down(addr, bytes_per_block());
     size_t block = addr_to_block_idx(addr);
     assert(_bot[block] != nullptr, "must have initialized BOT entry");
-    HeapWord* fwd = _bot[block] + _mark_bitmap.count_marked_words(block_base, addr);
+    HeapWord* fwd = _bot[block] + _mark_bitmap.count_marked_words_in_block(block_base, addr);
     assert(SerialHeap::heap()->is_in_reserved(fwd), "forward addresses must be in heap: addr: " PTR_FORMAT ", fwd: " PTR_FORMAT ", block: " SIZE_FORMAT ", bot[block]: " PTR_FORMAT ", block_base: " PTR_FORMAT, p2i(addr), p2i(fwd), block, p2i(_bot[block]), p2i(block_base));
     return fwd;
   }
@@ -190,6 +190,25 @@ class SCCompacter {
     _spaces[index]._first_dead = first_dead;
   }
 
+  // Find the start of the first object at or after addr (but not after limit).
+  HeapWord* first_object_in_block(HeapWord* addr, HeapWord* start, HeapWord* limit) {
+    if (addr >= limit) return limit;
+    if (!_mark_bitmap.is_marked(addr)) {
+      // Easy: find next marked address.
+      return _mark_bitmap.get_next_marked_addr(addr, limit);
+    } else {
+      // Find beginning of live chunk.
+      HeapWord* current = _mark_bitmap.get_last_unmarked_addr(start, addr) + 1;
+      // Forward-search to first object >= addr.
+      while (current < addr) {
+        size_t obj_size = cast_to_oop(current)->size();
+        current += obj_size;
+      }
+      assert(current >= addr, "found object start must be >= addr");
+      return addr;
+    }
+  }
+
   // Build the block-offset-table for space at index.
   void build_table_for_space(uint idx) {
     ContiguousSpace* space = get_space(idx);
@@ -205,8 +224,13 @@ class SCCompacter {
 
     HeapWord* compact_top = get_compaction_top(_index);
     HeapWord* current = bottom;
-    // Scan all live objects in the space.
-    while (current < top) {
+
+    // Scan all live blocks in the space.
+    HeapWord* first_obj_this_block = _mark_bitmap.get_next_marked_addr(current, top);
+    current = align_down(first_obj_this_block, bytes_per_block());
+    while (first_obj_this_block < top) {
+      assert(is_aligned(current, bytes_per_block()), "iterate at block granularity");
+      /*
       HeapWord* next_marked = _mark_bitmap.get_next_marked_addr(current, top);
       // Handle unmarked chunk - either skip it, or dead-space it, to avoid excessive
       // copying by keeping subsequent objects in place.
@@ -219,18 +243,17 @@ class SCCompacter {
           }
           // Store address of next live chunk into first non-live word to allow fast skip to next live during compaction.
           *(HeapWord**)current = next_marked;
-          current = next_marked;
+          current = align_down(next_marked, words_per_block());
         }
       }
-      HeapWord* chunk_end = _mark_bitmap.get_next_unmarked_addr(next_marked, top);
-      HeapWord* next_dead = chunk_end;
-      HeapWord* next_live = _mark_bitmap.get_next_marked_addr(next_dead, top);
-      while (addr_to_block_idx(next_dead) == addr_to_block_idx(next_live) && next_dead < top) {
-        next_dead = _mark_bitmap.get_next_unmarked_addr(next_live, top);
-        next_live = _mark_bitmap.get_next_marked_addr(next_dead, top);
-      }
-      size_t live_in_block = pointer_delta(next_dead, current);
-
+      */
+
+      // Determine if we need to switch to the next compaction space.
+      // Find first object that starts after this block and count
+      // live words up to that object. This is how many words we
+      // must fit into the current compaction space.
+      HeapWord* first_obj_after_block = first_object_in_block(current + words_per_block(), first_obj_this_block, top);
+      size_t live_in_block = _mark_bitmap.count_marked_words(first_obj_this_block, first_obj_after_block);
       while (live_in_block > pointer_delta(_spaces[_index]._space->end(),
                                            compact_top)) {
         // out-of-memory in this space
@@ -240,41 +263,12 @@ class SCCompacter {
         compact_top = _spaces[_index]._compaction_top;
       }
 
-      // Record addresses of the first live word of all blocks covered by the live span.
-      current = next_marked;
-      live_in_block = pointer_delta(chunk_end, current);
-      size_t head = MIN2(pointer_delta(align_up(current, bytes_per_block()), current), live_in_block);
-      if (head > 0) {
-        HeapWord* block_start = align_down(current, bytes_per_block());
-        // Count number of live words preceding the first object in the block. This must
-        // be subtracted, because the BOT stores the forwarding address of the first live
-        // *word*, not the first live *object* in the block.
-        size_t num_live = _mark_bitmap.count_marked_words(block_start, current);
-        // Note that we only record the address for blocks with live words. That
-        // is ok, because we only ask for forwarding address of object-starts, i.e. live words.
-        HeapWord* block_offset = compact_top - num_live;
-        assert(SerialHeap::heap()->is_in_reserved(block_offset), "block-offset must be in heap: " PTR_FORMAT ", compact_top: " PTR_FORMAT ", num_live: " SIZE_FORMAT ", _bot[]: " PTR_FORMAT, p2i(block_offset), p2i(compact_top), num_live, p2i(_bot[addr_to_block_idx(current)]));
-        _bot[addr_to_block_idx(current)] = block_offset;
-        assert(forwardee(current) == compact_top, "must match");
-        compact_top += head;
-        current += head;
-      }
-      // Middle block.
-      while (pointer_delta(chunk_end, current) > words_per_block()) {
-        _bot[addr_to_block_idx(current)] = compact_top;
-        assert(forwardee(current) == compact_top, "must match");
-        current += words_per_block();
-        compact_top += words_per_block();
-      }
-      // Tail.
-      size_t tail = pointer_delta(chunk_end, current);
-      if (tail > 0) {
-        _bot[addr_to_block_idx(current)] = compact_top;
-        assert(forwardee(current) == compact_top, "must match");
-        compact_top += tail;
-        current += tail;
-      }
-      assert(current == chunk_end, "must arrive at next unmarked");
+      // Record address of the first live word of this block.
+      _bot[addr_to_block_idx(current)] = compact_top - _mark_bitmap.count_marked_words_in_block(current, first_obj_this_block);
+      compact_top += live_in_block;
+      // Continue to scan at next block that has an object header.
+      first_obj_this_block = first_obj_after_block;
+      current = align_down(first_obj_after_block, bytes_per_block());
     }
     if (!record_first_dead_done) {
       record_first_dead(idx, top);

src/hotspot/share/gc/shared/markBitMap.hpp

@@ -85,6 +85,8 @@ class MarkBitMap {
                                         HeapWord* limit) const;
   inline HeapWord* get_next_unmarked_addr(const HeapWord* addr,
                                           HeapWord* limit) const;
+  inline HeapWord* get_last_unmarked_addr(const HeapWord* start,
+                                          HeapWord* limit) const;
 
   void print_on_error(outputStream* st, const char* prefix) const;
 
@@ -105,6 +107,7 @@ class MarkBitMap {
 
   // Count the number of marked words in the range [start, end).
   inline size_t count_marked_words(HeapWord* start, HeapWord* end) const;
+  inline size_t count_marked_words_in_block(HeapWord* start, HeapWord* end) const;
 };
 
 #endif // SHARE_GC_SHARED_MARKBITMAP_HPP

src/hotspot/share/gc/shared/markBitMap.inline.hpp

@@ -53,6 +53,16 @@ inline HeapWord* MarkBitMap::get_next_unmarked_addr(const HeapWord* const addr,
   return offset_to_addr(nextOffset);
 }
 
+inline HeapWord* MarkBitMap::get_last_unmarked_addr(const HeapWord* const start,
+                                                    HeapWord* const limit) const {
+  assert(limit != nullptr, "limit must not be null");
+  // Round addr up to a possible object boundary to be safe.
+  size_t const start_offset = addr_to_offset(align_up(start, HeapWordSize << _shifter));
+  size_t const limit_offset = addr_to_offset(limit);
+  size_t const nextOffset = _bm.find_last_clear_bit(start_offset, limit_offset);
+  return offset_to_addr(nextOffset);
+}
+
 inline void MarkBitMap::mark(HeapWord* addr) {
   check_mark(addr);
   _bm.set_bit(addr_to_offset(addr));
@@ -91,6 +101,10 @@ inline void MarkBitMap::clear(oop obj) {
 }
 
 inline size_t MarkBitMap::count_marked_words(HeapWord* start, HeapWord* end) const {
+  return _bm.count_one_bits(addr_to_offset(start), addr_to_offset(end)) /*<< _shifter */;
+}
+
+inline size_t MarkBitMap::count_marked_words_in_block(HeapWord* start, HeapWord* end) const {
   return _bm.count_one_bits_within_word(addr_to_offset(start), addr_to_offset(end)) /*<< _shifter */;
 }
 

src/hotspot/share/memory/metaspace/commitMask.hpp

@@ -26,7 +26,7 @@
 #ifndef SHARE_MEMORY_METASPACE_COMMITMASK_HPP
 #define SHARE_MEMORY_METASPACE_COMMITMASK_HPP
 
-#include "utilities/bitMap.hpp"
+#include "utilities/bitMap.inline.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/globalDefinitions.hpp"
 

src/hotspot/share/utilities/bitMap.cpp

@@ -177,15 +177,6 @@ void BitMap::pretouch() {
   os::pretouch_memory(word_addr(0), word_addr(size()));
 }
 
-void BitMap::set_range_within_word(idx_t beg, idx_t end) {
-  // With a valid range (beg <= end), this test ensures that end != 0, as
-  // required by inverted_bit_mask_for_range.  Also avoids an unnecessary write.
-  if (beg != end) {
-    bm_word_t mask = inverted_bit_mask_for_range(beg, end);
-    *word_addr(beg) |= ~mask;
-  }
-}
-
 void BitMap::clear_range_within_word(idx_t beg, idx_t end) {
   // With a valid range (beg <= end), this test ensures that end != 0, as
   // required by inverted_bit_mask_for_range.  Also avoids an unnecessary write.
@@ -213,25 +204,6 @@ void BitMap::par_put_range_within_word(idx_t beg, idx_t end, bool value) {
   }
 }
 
-void BitMap::set_range(idx_t beg, idx_t end) {
-  verify_range(beg, end);
-
-  idx_t beg_full_word = to_words_align_up(beg);
-  idx_t end_full_word = to_words_align_down(end);
-
-  if (beg_full_word < end_full_word) {
-    // The range includes at least one full word.
-    set_range_within_word(beg, bit_index(beg_full_word));
-    set_range_of_words(beg_full_word, end_full_word);
-    set_range_within_word(bit_index(end_full_word), end);
-  } else {
-    // The range spans at most 2 partial words.
-    idx_t boundary = MIN2(bit_index(beg_full_word), end);
-    set_range_within_word(beg, boundary);
-    set_range_within_word(boundary, end);
-  }
-}
-
 void BitMap::clear_range(idx_t beg, idx_t end) {
   verify_range(beg, end);
 
@@ -595,46 +567,6 @@ void BitMap::clear_large() {
   clear_large_range_of_words(0, size_in_words());
 }
 
-BitMap::idx_t BitMap::count_one_bits_in_range_of_words(idx_t beg_full_word, idx_t end_full_word) const {
-  idx_t sum = 0;
-  for (idx_t i = beg_full_word; i < end_full_word; i++) {
-    bm_word_t w = map()[i];
-    sum += population_count(w);
-  }
-  return sum;
-}
-
-BitMap::idx_t BitMap::count_one_bits() const {
-  return count_one_bits(0, size());
-}
-
-// Returns the number of bits set within  [beg, end).
-BitMap::idx_t BitMap::count_one_bits(idx_t beg, idx_t end) const {
-  verify_range(beg, end);
-
-  idx_t beg_full_word = to_words_align_up(beg);
-  idx_t end_full_word = to_words_align_down(end);
-
-  idx_t sum = 0;
-
-  if (beg_full_word < end_full_word) {
-    // The range includes at least one full word.
-    sum += count_one_bits_within_word(beg, bit_index(beg_full_word));
-    sum += count_one_bits_in_range_of_words(beg_full_word, end_full_word);
-    sum += count_one_bits_within_word(bit_index(end_full_word), end);
-  } else {
-    // The range spans at most 2 partial words.
-    idx_t boundary = MIN2(bit_index(beg_full_word), end);
-    sum += count_one_bits_within_word(beg, boundary);
-    sum += count_one_bits_within_word(boundary, end);
-  }
-
-  assert(sum <= (end - beg), "must be");
-
-  return sum;
-
-}
-
 void BitMap::print_on_error(outputStream* st, const char* prefix) const {
   st->print_cr("%s[" PTR_FORMAT ", " PTR_FORMAT ")",
       prefix, p2i(map()), p2i((char*)map() + (size() >> LogBitsPerByte)));

src/hotspot/share/utilities/bitMap.hpp

@@ -163,7 +163,7 @@ class BitMap {
 
   // Ranges within a single word.
   bm_word_t inverted_bit_mask_for_range(idx_t beg, idx_t end) const;
-  void  set_range_within_word      (idx_t beg, idx_t end);
+  inline void  set_range_within_word      (idx_t beg, idx_t end);
   void  clear_range_within_word    (idx_t beg, idx_t end);
   void  par_put_range_within_word  (idx_t beg, idx_t end, bool value);
 
@@ -174,10 +174,11 @@ class BitMap {
   void      clear_large_range_of_words (idx_t beg, idx_t end);
 
   static void clear_range_of_words(bm_word_t* map, idx_t beg, idx_t end);
+
 public:
   inline idx_t count_one_bits_within_word(idx_t beg, idx_t end) const;
 protected:
-  idx_t count_one_bits_in_range_of_words(idx_t beg_full_word, idx_t end_full_word) const;
+  inline idx_t count_one_bits_in_range_of_words(idx_t beg_full_word, idx_t end_full_word) const;
 
   // Set the map and size.
   void update(bm_word_t* map, idx_t size) {
@@ -234,7 +235,7 @@ class BitMap {
   bool par_at_put(idx_t bit, bool value);
 
   // Update a range of bits.  Ranges are half-open [beg, end).
-  void set_range   (idx_t beg, idx_t end);
+  inline void set_range   (idx_t beg, idx_t end);
   void clear_range (idx_t beg, idx_t end);
   void set_large_range   (idx_t beg, idx_t end);
   void clear_large_range (idx_t beg, idx_t end);
@@ -357,10 +358,10 @@ class BitMap {
   idx_t find_last_set_bit_aligned_left(idx_t beg, idx_t end) const;
 
   // Returns the number of bits set in the bitmap.
-  idx_t count_one_bits() const;
+  inline idx_t count_one_bits() const;
 
   // Returns the number of bits set within  [beg, end).
-  idx_t count_one_bits(idx_t beg, idx_t end) const;
+  inline idx_t count_one_bits(idx_t beg, idx_t end) const;
 
   // Set operations.
   void set_union(const BitMap& bits);