#2209: collection: order rooted constructions with new objgroup locat…

…ion manager
DARMA-tasking · Nov 30, 2023 · caa1a05 · caa1a05
1 parent 04aa0f4
commit caa1a05
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 90 deletions.
diff --git a/src/vt/vrt/collection/collection_builder.impl.h b/src/vt/vrt/collection/collection_builder.impl.h
@@ -66,9 +66,7 @@ std::tuple<EpochType, VirtualProxyType> CollectionManager::makeCollection(
       "collection construction", term::UseDS{false}
     );
     theMsg()->pushEpoch(ep);
-    using MsgType = param::ConstructParamMsg<ColT>;
-    auto m = makeMessage<MsgType>(po);
-    theMsg()->broadcastMsg<MsgType, makeCollectionHandler>(m);
+    theMsg()->send<makeCollectionHandler<ColT>>(vt::Node(0), po, true);
     theMsg()->popEpoch(ep);
     theTerm()->finishedEpoch(ep);
     return std::make_tuple(ep, proxy_bits);
@@ -82,11 +80,37 @@ std::tuple<EpochType, VirtualProxyType> CollectionManager::makeCollection(
   }
 }
 
+/*static*/ inline void CollectionManager::finishedRootedConstruction() {
+  theCollection()->has_pending_construction_ = false;
+  if (theCollection()->pending_rooted_constructions_.size() > 0) {
+    auto action = theCollection()->pending_rooted_constructions_.back();
+    theCollection()->pending_rooted_constructions_.pop_back();
+    action();
+  }
+}
+
 template <typename ColT>
 /*static*/ void CollectionManager::makeCollectionHandler(
-  param::ConstructParamMsg<ColT>* msg
+  param::ConstructParams<ColT> po, bool is_root
 ) {
-  theCollection()->makeCollectionImpl(*msg->po);
+  if (is_root) {
+    if (theCollection()->has_pending_construction_) {
+      auto ep = theMsg()->getEpoch();
+      theTerm()->produce(ep);
+      theCollection()->pending_rooted_constructions_.push_back([=]{
+        theTerm()->pushEpoch(ep);
+        makeCollectionHandler(po, true);
+        theTerm()->consume(ep);
+        theTerm()->popEpoch(ep);
+      });
+    } else {
+      theMsg()->broadcast<makeCollectionHandler<ColT>>(po, false);
+    }
+  } else {
+    theCollection()->makeCollectionImpl(po);
+    auto r = theCollection()->reducer();
+    r->reduce<finishedRootedConstruction>(vt::Node(0));
+  }
 }
 
 namespace detail {

diff --git a/src/vt/vrt/collection/manager.h b/src/vt/vrt/collection/manager.h
@@ -72,7 +72,6 @@
 #include "vt/runtime/component/component_pack.h"
 #include "vt/context/runnable_context/lb_data.fwd.h"
 #include "vt/vrt/collection/param/construct_params.h"
-#include "vt/vrt/collection/param/construct_params_msg.h"
 #include "vt/utils/fntraits/fntraits.h"
 
 #include <memory>
@@ -1697,10 +1696,18 @@ struct CollectionManager
    * \internal \brief Handler for receiving a new collection configuration to
    * construct on this node
    *
-   * \param[in] msg the configuration message
+   * \param[in] po construct parameters
+   * \param[in] is_root whether if it's boucing off the root first
    */
   template <typename ColT>
-  static void makeCollectionHandler(param::ConstructParamMsg<ColT>* msg);
+  static void makeCollectionHandler(
+    param::ConstructParams<ColT> po, bool is_root
+  );
+
+  /**
+   * \brief Finished a rooted construction--do the next if needed
+   */
+  static void finishedRootedConstruction();
 
   /**
    * \internal \brief System function to actually constructing the collection
@@ -1766,6 +1773,8 @@ struct CollectionManager
   VirtualIDType next_rooted_id_ = 0;
   TypelessHolder typeless_holder_;
   std::unordered_map<VirtualProxyType, SequentialIDType> reduce_stamp_;
+  bool has_pending_construction_ = false;
+  std::list<ActionType> pending_rooted_constructions_;
 };
 
 }}} /* end namespace vt::vrt::collection */

diff --git a/src/vt/vrt/collection/param/construct_params_msg.h b/src/vt/vrt/collection/param/construct_params_msg.h