v1

src/hotspot/share/gc/parallel/psPromotionManager.cpp

@@ -30,6 +30,8 @@
 #include "gc/parallel/psPromotionManager.inline.hpp"
 #include "gc/parallel/psScavenge.inline.hpp"
 #include "gc/shared/gcTrace.hpp"
+#include "gc/shared/partialArrayState.hpp"
+#include "gc/shared/partialArrayTaskStepper.inline.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
 #include "logging/log.hpp"
@@ -47,6 +49,7 @@ PSPromotionManager::PSScannerTasksQueueSet* PSPromotionManager::_stack_array_dep
 PreservedMarksSet*             PSPromotionManager::_preserved_marks_set = NULL;
 PSOldGen*                      PSPromotionManager::_old_gen = NULL;
 MutableSpace*                  PSPromotionManager::_young_space = NULL;
+PartialArrayStateAllocator*    PSPromotionManager::_partial_array_state_allocator = NULL;
 
 void PSPromotionManager::initialize() {
   ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
@@ -61,11 +64,16 @@ void PSPromotionManager::initialize() {
   assert(_manager_array == NULL, "Attempt to initialize twice");
   _manager_array = PaddedArray<PSPromotionManager, mtGC>::create_unfreeable(promotion_manager_num);
 
+  assert(_partial_array_state_allocator == NULL, "Attempt to initialize twice");
+  _partial_array_state_allocator
+    = new PartialArrayStateAllocator(ParallelGCThreads);
+
   _stack_array_depth = new PSScannerTasksQueueSet(ParallelGCThreads);
 
   // Create and register the PSPromotionManager(s) for the worker threads.
   for(uint i=0; i<ParallelGCThreads; i++) {
     stack_array_depth()->register_queue(i, _manager_array[i].claimed_stack_depth());
+    _manager_array[i]._partial_array_state_allocator_index = i;
   }
   // The VMThread gets its own PSPromotionManager, which is not available
   // for work stealing.
@@ -122,6 +130,10 @@ bool PSPromotionManager::post_scavenge(YoungGCTracer& gc_tracer) {
     }
     manager->flush_labs();
   }
+  // All PartialArrayStates have been returned to the allocator, since the
+  // claimed_stack_depths are all empty.  Leave them there for use by future
+  // collections.
+
   if (!promotion_failure_occurred) {
     // If there was no promotion failure, the preserved mark stacks
     // should be empty.
@@ -183,9 +195,10 @@ PSPromotionManager::reset_stats() {
 }
 #endif // TASKQUEUE_STATS
 
-PSPromotionManager::PSPromotionManager() {
-  ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
-
+// Most members are initialized either by initialize() or reset().
+PSPromotionManager::PSPromotionManager()
+  : _partial_array_stepper(ParallelGCThreads, ParGCArrayScanChunk)
+{
   // We set the old lab's start array.
   _old_lab.set_start_array(old_gen()->start_array());
 
@@ -202,9 +215,11 @@ PSPromotionManager::PSPromotionManager() {
                                      (uint) (queue_size / 4));
   }
 
-  _array_chunk_size = ParGCArrayScanChunk;
+  // Initialize to a bad value; fixed by initialize().
+  _partial_array_state_allocator_index = UINT_MAX;
+
   // let's choose 1.5x the chunk size
-  _min_array_size_for_chunking = 3 * _array_chunk_size / 2;
+  _min_array_size_for_chunking = (3 * ParGCArrayScanChunk / 2);
 
   _preserved_marks = NULL;
 
@@ -310,37 +325,57 @@ template <class T> void PSPromotionManager::process_array_chunk_work(
   }
 }
 
-void PSPromotionManager::process_array_chunk(PartialArrayScanTask task) {
-  assert(PSChunkLargeArrays, "invariant");
-
-  oop old = task.to_source_array();
-  assert(old->is_objArray(), "invariant");
-  assert(old->is_forwarded(), "invariant");
-
+void PSPromotionManager::process_array_chunk(PartialArrayState* state) {
   TASKQUEUE_STATS_ONLY(++_array_chunks_processed);
 
-  oop const obj = old->forwardee();
-
-  int start;
-  int const end = arrayOop(old)->length();
-  if (end > (int) _min_array_size_for_chunking) {
-    // we'll chunk more
-    start = end - _array_chunk_size;
-    assert(start > 0, "invariant");
-    arrayOop(old)->set_length(start);
-    push_depth(ScannerTask(PartialArrayScanTask(old)));
-    TASKQUEUE_STATS_ONLY(++_array_chunk_pushes);
+  // Claim a chunk.  Push additional tasks before processing the claimed
+  // chunk to allow other workers to steal while we're processing.
+  PartialArrayTaskStepper::Step step = _partial_array_stepper.next(state);
+  if (step._ncreate > 0) {
+    state->add_references(step._ncreate);
+    for (uint i = 0; i < step._ncreate; ++i) {
+      push_depth(ScannerTask(state));
+    }
+    TASKQUEUE_STATS_ONLY(_array_chunk_pushes += step._ncreate);
+  }
+  int start = checked_cast<int>(step._index);
+  int end = checked_cast<int>(step._index + _partial_array_stepper.chunk_size());
+  assert(start < end, "invariant");
+  if (UseCompressedOops) {
+    process_array_chunk_work<narrowOop>(state->destination(), start, end);
   } else {
-    // this is the final chunk for this array
-    start = 0;
-    int const actual_length = arrayOop(obj)->length();
-    arrayOop(old)->set_length(actual_length);
+    process_array_chunk_work<oop>(state->destination(), start, end);
   }
+  // Release reference to state, now that we're done with it.
+  _partial_array_state_allocator->release(_partial_array_state_allocator_index, state);
+}
 
+void PSPromotionManager::push_objArray(oop old_obj, oop new_obj) {
+  assert(old_obj->is_objArray(), "precondition");
+  assert(old_obj->is_forwarded(), "precondition");
+  assert(old_obj->forwardee() == new_obj, "precondition");
+  assert(new_obj->is_objArray(), "precondition");
+
+  size_t array_length = objArrayOop(new_obj)->length();
+  PartialArrayTaskStepper::Step step = _partial_array_stepper.start(array_length);
+
+  if (step._ncreate > 0) {
+    TASKQUEUE_STATS_ONLY(++_arrays_chunked);
+    PartialArrayState* state =
+      _partial_array_state_allocator->allocate(_partial_array_state_allocator_index,
+                                               old_obj, new_obj,
+                                               step._index,
+                                               array_length,
+                                               step._ncreate);
+    for (uint i = 0; i < step._ncreate; ++i) {
+      push_depth(ScannerTask(state));
+    }
+    TASKQUEUE_STATS_ONLY(_array_chunk_pushes += step._ncreate);
+  }
   if (UseCompressedOops) {
-    process_array_chunk_work<narrowOop>(obj, start, end);
+    process_array_chunk_work<narrowOop>(new_obj, 0, checked_cast<int>(step._index));
   } else {
-    process_array_chunk_work<oop>(obj, start, end);
+    process_array_chunk_work<oop>(new_obj, 0, checked_cast<int>(step._index));
   }
 }
 

src/hotspot/share/gc/parallel/psPromotionManager.hpp

@@ -28,6 +28,7 @@
 #include "gc/parallel/psPromotionLAB.hpp"
 #include "gc/shared/copyFailedInfo.hpp"
 #include "gc/shared/gcTrace.hpp"
+#include "gc/shared/partialArrayTaskStepper.hpp"
 #include "gc/shared/preservedMarks.hpp"
 #include "gc/shared/taskqueue.hpp"
 #include "memory/padded.hpp"
@@ -48,6 +49,8 @@
 class MutableSpace;
 class PSOldGen;
 class ParCompactionManager;
+class PartialArrayState;
+class PartialArrayStateAllocator;
 
 class PSPromotionManager {
   friend class PSScavenge;
@@ -86,7 +89,9 @@ class PSPromotionManager {
   bool                                _totally_drain;
   uint                                _target_stack_size;
 
-  uint                                _array_chunk_size;
+  static PartialArrayStateAllocator*  _partial_array_state_allocator;
+  PartialArrayTaskStepper             _partial_array_stepper;
+  uint                                _partial_array_state_allocator_index;
   uint                                _min_array_size_for_chunking;
 
   PreservedMarks*                     _preserved_marks;
@@ -100,7 +105,8 @@ class PSPromotionManager {
 
   template <class T> void  process_array_chunk_work(oop obj,
                                                     int start, int end);
-  void process_array_chunk(PartialArrayScanTask task);
+  void process_array_chunk(PartialArrayState* state);
+  void push_objArray(oop old_obj, oop new_obj);
 
   void push_depth(ScannerTask task);
 

src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp

@@ -276,9 +276,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
     if (new_obj_size > _min_array_size_for_chunking &&
         new_obj->is_objArray() &&
         PSChunkLargeArrays) {
-      // we'll chunk it
-      push_depth(ScannerTask(PartialArrayScanTask(o)));
-      TASKQUEUE_STATS_ONLY(++_arrays_chunked; ++_array_chunk_pushes);
+      push_objArray(o, new_obj);
     } else {
       // we'll just push its contents
       push_contents(new_obj);
@@ -330,9 +328,9 @@ inline void PSPromotionManager::copy_and_push_safe_barrier(T* p) {
 }
 
 inline void PSPromotionManager::process_popped_location_depth(ScannerTask task) {
-  if (task.is_partial_array_task()) {
+  if (task.is_partial_array_state()) {
     assert(PSChunkLargeArrays, "invariant");
-    process_array_chunk(task.to_partial_array_task());
+    process_array_chunk(task.to_partial_array_state());
   } else {
     if (task.is_narrow_oop_ptr()) {
       assert(UseCompressedOops, "Error");
@@ -349,7 +347,7 @@ inline bool PSPromotionManager::steal_depth(int queue_num, ScannerTask& t) {
 
 #if TASKQUEUE_STATS
 void PSPromotionManager::record_steal(ScannerTask task) {
-  if (task.is_partial_array_task()) {
+  if (task.is_partial_array_state()) {
     ++_array_chunk_steals;
   }
 }

src/hotspot/share/gc/shared/partialArrayState.hpp

@@ -25,6 +25,7 @@
 #ifndef SHARE_GC_SHARED_PARTIALARRAYSTATE_HPP
 #define SHARE_GC_SHARED_PARTIALARRAYSTATE_HPP
 
+#include "memory/allocation.hpp"
 #include "oops/oopsHierarchy.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/macros.hpp"
@@ -109,7 +110,7 @@ class PartialArrayState {
 // the worker_id used in the operation.  This avoids locking and such on those
 // data structures, at the cost of possibly doing more total arena allocation
 // that would be needed with a single shared arena and free-list.
-class PartialArrayStateAllocator {
+class PartialArrayStateAllocator : public CHeapObj<mtGC> {
   class Impl;
   Impl* _impl;