#868: collection: switch to using element id struct

src/vt/vrt/collection/balance/baselb/baselb.cc

@@ -108,9 +108,9 @@ void BaseLB::importProcessorData(
 
     vt_debug_print_verbose(
       lb, node,
-      "\t {}: importProcessorData: this_load={}, obj={}, load={}, "
+      "\t {}: importProcessorData: this_load={}, obj={}, home={}, load={}, "
       "load_milli={}, bin={}\n",
-      this_node, this_load, obj, load, load_milli, bin
+      this_node, this_load, obj.id, obj.home_node, load, load_milli, bin
     );
   }
 
@@ -191,8 +191,8 @@ void BaseLB::applyMigrations(TransferVecType const &transfers) {
 
       vt_debug_print(
         lb, node,
-        "migrateObjectTo, obj_id={}, from={}, to={}, found={}\n",
-        obj_id, from, to, has_object
+        "migrateObjectTo, obj_id={}, home={}, from={}, to={}, found={}\n",
+        obj_id.id, obj_id.home_node, from, to, has_object
       );
 
       local_migration_count_++;

src/vt/vrt/collection/balance/baselb/baselb.h

@@ -64,7 +64,7 @@ namespace vt { namespace vrt { namespace collection { namespace lb {
 static constexpr int32_t const default_bin_size = 10;
 
 struct BaseLB {
-  using ObjIDType        = balance::ElementIDType;
+  using ObjIDType        = balance::ElementIDStruct;
   using ObjBinType       = int32_t;
   using ObjBinListType   = std::list<ObjIDType>;
   using ObjSampleType    = std::map<ObjBinType, ObjBinListType>;

src/vt/vrt/collection/balance/elm_stats.cc

@@ -88,26 +88,26 @@ void ElementStats::recvComm(
 }
 
 void ElementStats::recvObjData(
-  ElementIDType pto, ElementIDType tto,
-  ElementIDType pfrom, ElementIDType tfrom, double bytes, bool bcast
+  ElementIDStruct pto,
+  ElementIDStruct pfrom, double bytes, bool bcast
 ) {
-  LBCommKey key(LBCommKey::CollectionTag{}, pfrom, tfrom, pto, tto, bcast);
+  LBCommKey key(LBCommKey::CollectionTag{}, pfrom, pto, bcast);
   recvComm(key, bytes);
 }
 
 void ElementStats::recvFromNode(
-  ElementIDType pto, ElementIDType tto, NodeType from,
+  ElementIDStruct pto, NodeType from,
   double bytes, bool bcast
 ) {
-  LBCommKey key(LBCommKey::NodeToCollectionTag{}, from, pto, tto, bcast);
+  LBCommKey key(LBCommKey::NodeToCollectionTag{}, from, pto, bcast);
   recvComm(key, bytes);
 }
 
 void ElementStats::recvToNode(
-  NodeType to, ElementIDType pfrom, ElementIDType tfrom,
+  NodeType to, ElementIDStruct pfrom,
   double bytes, bool bcast
 ) {
-  LBCommKey key(LBCommKey::CollectionToNodeTag{}, pfrom, tfrom, to, bcast);
+  LBCommKey key(LBCommKey::CollectionToNodeTag{}, pfrom, to, bcast);
   recvComm(key, bytes);
 }
 

src/vt/vrt/collection/balance/elm_stats.h

@@ -71,15 +71,15 @@ struct ElementStats {
   void addTime(TimeType const& time);
   void recvComm(LBCommKey key, double bytes);
   void recvObjData(
-    ElementIDType to_perm, ElementIDType to_temp,
-    ElementIDType from_perm, ElementIDType from_temp, double bytes, bool bcast
+    ElementIDStruct to_perm,
+    ElementIDStruct from_perm, double bytes, bool bcast
   );
   void recvFromNode(
-    ElementIDType to_perm, ElementIDType to_temp, NodeType from,
+    ElementIDStruct to_perm, NodeType from,
     double bytes, bool bcast
   );
   void recvToNode(
-    NodeType to, ElementIDType from_perm, ElementIDType from_temp,
+    NodeType to, ElementIDStruct from_perm,
     double bytes, bool bcast
   );
   void updatePhase(PhaseType const& inc = 1);

src/vt/vrt/collection/balance/gossiplb/gossiplb.cc

@@ -420,12 +420,13 @@ void GossipLB::decide() {
         vt_debug_print(
           gossiplb, node,
           "GossipLB::decide: under.size()={}, selected_node={}, selected_load={},"
-          "obj_id={:x}, obj_load={}, avg={}, this_new_load_={}, "
+          "obj_id={:x}, home={}, obj_load={}, avg={}, this_new_load_={}, "
           "criterion={}\n",
           under.size(),
           selected_node,
           selected_load,
-          obj_id,
+          obj_id.id,
+          obj_id.home_node,
           obj_load,
           avg,
           this_new_load_,

src/vt/vrt/collection/balance/greedylb/greedylb.cc

@@ -210,8 +210,9 @@ void GreedyLB::runBalancer(
 GreedyLB::ObjIDType GreedyLB::objSetNode(
   NodeType const& node, ObjIDType const& id
 ) {
-  auto const new_id = id & 0xFFFFFFFF0000000;
-  return new_id | node;
+  auto new_id = id;
+  new_id.curr_node = node;
+  return new_id;
 }
 
 void GreedyLB::recvObjsDirect(GreedyLBTypes::ObjIDType* objs) {
@@ -223,7 +224,7 @@ void GreedyLB::recvObjsDirect(GreedyLBTypes::ObjIDType* objs) {
     "recvObjsDirect: num_recs={}\n", num_recs
   );
 
-  for (decltype(+num_recs) i = 0; i < num_recs; i++) {
+  for (decltype(+num_recs.id) i = 0; i < num_recs.id; i++) {
     auto const to_node = objGetNode(recs[i]);
     auto const new_obj_id = objSetNode(this_node,recs[i]);
     vt_debug_print(
@@ -274,7 +275,7 @@ void GreedyLB::transferObjs(std::vector<GreedyProc>&& in_load) {
       auto ptr_out = reinterpret_cast<GreedyLBTypes::ObjIDType*>(ptr);
       auto const& proc = node_transfer[node];
       auto const& rec_size = proc.size();
-      *ptr_out = rec_size;
+      ptr_out->id = rec_size;
       for (size_t i = 0; i < rec_size; i++) {
         *(ptr_out + i + 1) = proc[i];
       }

src/vt/vrt/collection/balance/greedylb/greedylb_types.h

@@ -56,7 +56,7 @@
 namespace vt { namespace vrt { namespace collection { namespace lb {
 
 struct GreedyLBTypes {
-  using ObjIDType = balance::ElementIDType;
+  using ObjIDType = balance::ElementIDStruct;
   using ObjBinType = int32_t;
   using ObjBinListType = std::list<ObjIDType>;
   using ObjSampleType = std::map<ObjBinType, ObjBinListType>;
@@ -76,7 +76,9 @@ struct GreedyRecord {
   ObjType getObj() const { return obj_; }
 
 private:
-  GreedyLBTypes::ObjIDType obj_ = 0;
+  GreedyLBTypes::ObjIDType obj_ = {
+    0, uninitialized_destination, uninitialized_destination
+  };
   LoadType load_ = 0.0f;
 };
 

src/vt/vrt/collection/balance/hierarchicallb/hierlb_types.h

@@ -56,7 +56,7 @@
 namespace vt { namespace vrt { namespace collection { namespace lb {
 
 struct HierLBTypes {
-  using ObjIDType = balance::ElementIDType;
+  using ObjIDType = balance::ElementIDStruct;
   using ObjBinType = int32_t;
   using ObjBinListType = std::list<ObjIDType>;
   using ObjSampleType = std::map<ObjBinType, ObjBinListType>;

src/vt/vrt/collection/balance/lb_comm.h

@@ -62,8 +62,8 @@ enum struct CommCategory : int8_t {
   CollectiveToCollectionBcast = 7
 };
 
-inline NodeType objGetNode(ElementIDType const id) {
-  return id & 0x0000000FFFFFFFF;
+inline NodeType objGetNode(ElementIDStruct const id) {
+  return id.curr_node;
 }
 
 struct LBCommKey {
@@ -79,52 +79,48 @@ struct LBCommKey {
 
   LBCommKey(
     CollectionTag,
-    ElementIDType from, ElementIDType from_temp,
-    ElementIDType to,   ElementIDType to_temp,
+    ElementIDStruct from, ElementIDStruct to,
     bool bcast
-  ) : from_(from), from_temp_(from_temp), to_(to), to_temp_(to_temp),
+  ) : from_(from), to_(to),
       cat_(bcast ? CommCategory::Broadcast : CommCategory::SendRecv)
   { }
   LBCommKey(
     CollectionToNodeTag,
-    ElementIDType from, ElementIDType from_temp, NodeType to,
+    ElementIDStruct from, NodeType to,
     bool bcast
-  ) : from_(from), from_temp_(from_temp), nto_(to),
+  ) : from_(from), nto_(to),
       cat_(bcast ? CommCategory::CollectionToNodeBcast : CommCategory::CollectionToNode)
   { }
   LBCommKey(
     NodeToCollectionTag,
-    NodeType from, ElementIDType to, ElementIDType to_temp,
+    NodeType from, ElementIDStruct to,
     bool bcast
-  ) : to_(to), to_temp_(to_temp), nfrom_(from),
+  ) : to_(to), nfrom_(from),
       cat_(bcast ? CommCategory::NodeToCollectionBcast : CommCategory::NodeToCollection)
   { }
 
-  ElementIDType from_      = no_element_id;
-  ElementIDType from_temp_ = no_element_id;
-  ElementIDType to_        = no_element_id;
-  ElementIDType to_temp_   = no_element_id;
+  ElementIDStruct from_ = { no_element_id, uninitialized_destination, uninitialized_destination };
+  ElementIDStruct to_   = { no_element_id, uninitialized_destination, uninitialized_destination };
+
   ElementIDType edge_id_   = no_element_id;
   NodeType nfrom_          = uninitialized_destination;
   NodeType nto_            = uninitialized_destination;
   CommCategory  cat_       = CommCategory::SendRecv;
 
-  ElementIDType fromObj()      const { return from_; }
-  ElementIDType toObj()        const { return to_; }
-  ElementIDType fromObjTemp()  const { return from_temp_; }
-  ElementIDType toObjTemp()    const { return to_temp_; }
+  ElementIDStruct fromObj()    const { return from_; }
+  ElementIDStruct toObj()      const { return to_; }
   ElementIDType fromNode()     const { return nfrom_; }
   ElementIDType toNode()       const { return nto_; }
   ElementIDType edgeID()       const { return edge_id_; }
 
   bool selfEdge() const { return cat_ == CommCategory::SendRecv and from_ == to_; }
   bool offNode() const {
     if (cat_ == CommCategory::SendRecv) {
-      return objGetNode(from_temp_) != objGetNode(to_temp_);
+      return objGetNode(from_) != objGetNode(to_);
     } else if (cat_ == CommCategory::CollectionToNode) {
-      return objGetNode(from_temp_) != nto_;
+      return objGetNode(from_) != nto_;
     } else if (cat_ == CommCategory::NodeToCollection) {
-      return objGetNode(to_temp_) != nfrom_;
+      return objGetNode(to_) != nfrom_;
     } else {
       return true;
     }
@@ -140,7 +136,7 @@ struct LBCommKey {
 
   template <typename SerializerT>
   void serialize(SerializerT& s) {
-    s | from_ | to_ | from_temp_ | to_temp_ | nfrom_ | nto_ | cat_ | edge_id_;
+    s | from_ | to_ | nfrom_ | nto_ | cat_ | edge_id_;
   }
 };
 
@@ -174,8 +170,9 @@ using CommMapType   = std::unordered_map<CommKeyType,CommVolume>;
 
 namespace std {
 
-using CommCategoryType = vt::vrt::collection::balance::CommCategory;
-using LBCommKeyType    = vt::vrt::collection::balance::LBCommKey;
+using CommCategoryType    = vt::vrt::collection::balance::CommCategory;
+using LBCommKeyType       = vt::vrt::collection::balance::LBCommKey;
+using ElementIDStructType = vt::vrt::collection::balance::ElementIDStruct;
 
 template <>
 struct hash<CommCategoryType> {
@@ -189,7 +186,10 @@ struct hash<CommCategoryType> {
 template <>
 struct hash<LBCommKeyType> {
   size_t operator()(LBCommKeyType const& in) const {
-    return std::hash<uint64_t>()(in.from_ ^ in.to_ ^ in.nfrom_ ^ in.nto_);
+    return std::hash<uint64_t>()(
+      std::hash<ElementIDStructType>()(in.from_) ^ 
+      std::hash<ElementIDStructType>()(in.to_) ^ in.nfrom_ ^ in.nto_
+    );
   }
 };
 

src/vt/vrt/collection/balance/lb_common.cc

@@ -0,0 +1,59 @@
+/*
+//@HEADER
+// *****************************************************************************
+//
+//                                 lb_common.cc
+//                           DARMA Toolkit v. 1.0.0
+//                       DARMA/vt => Virtual Transport
+//
+// Copyright 2019 National Technology & Engineering Solutions of Sandia, LLC
+// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S.
+// Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+//
+// * Neither the name of the copyright holder nor the names of its
+//   contributors may be used to endorse or promote products derived from this
+//   software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact [email protected]
+//
+// *****************************************************************************
+//@HEADER
+*/
+
+#include "vt/config.h"
+#include "vt/vrt/collection/balance/lb_common.h"
+
+#include <ostream>
+
+namespace vt { namespace vrt { namespace collection { namespace balance {
+
+std::ostream& operator<<(
+  std::ostream& os, const ::vt::vrt::collection::balance::ElementIDStruct& id
+) {
+  os << "(" << id.id << "," << id.home_node << "," << id.curr_node << ")";
+  return os;
+}
+
+}}}} /* end namespace vt::vrt::collection::balance */

src/vt/vrt/collection/balance/lb_common.h

@@ -50,6 +50,7 @@
 
 #include <cstdlib>
 #include <unordered_map>
+#include <ostream>
 
 namespace vt { namespace vrt { namespace collection {
 namespace balance {
@@ -78,8 +79,8 @@ std::ostream& operator<<(
 
 static constexpr ElementIDType const no_element_id = 0;
 
-using LoadMapType         = std::unordered_map<ElementIDType,TimeType>;
-using SubphaseLoadMapType = std::unordered_map<ElementIDType, std::vector<TimeType>>;
+using LoadMapType         = std::unordered_map<ElementIDStruct,TimeType>;
+using SubphaseLoadMapType = std::unordered_map<ElementIDStruct, std::vector<TimeType>>;
 } /* end namespace balance */
 
 namespace lb {

src/vt/vrt/collection/balance/model/comm_overhead.cc

@@ -62,7 +62,7 @@ void CommOverhead::setLoads(std::unordered_map<PhaseType, LoadMapType> const* pr
   ComposedModel::setLoads(proc_load, proc_subphase_load, proc_comm);
 }
 
-TimeType CommOverhead::getWork(ElementIDType object, PhaseOffset offset) {
+TimeType CommOverhead::getWork(ElementIDStruct object, PhaseOffset offset) {
   auto work = ComposedModel::getWork(object, offset);
 
   auto phase = getNumCompletedPhases() + offset.phases;
@@ -71,7 +71,7 @@ TimeType CommOverhead::getWork(ElementIDType object, PhaseOffset offset) {
   TimeType overhead = 0.;
   for (auto&& c : comm) {
     // find messages that go off-node and are sent to this object
-    if (c.first.offNode() and c.first.toObjTemp() == object) {
+    if (c.first.offNode() and c.first.toObj() == object) {
       overhead += per_msg_weight_ * c.second.messages;
       overhead += per_byte_weight_ * c.second.bytes;
     }

src/vt/vrt/collection/balance/model/comm_overhead.h

@@ -70,7 +70,7 @@ struct CommOverhead : public ComposedModel {
                 std::unordered_map<PhaseType, SubphaseLoadMapType> const* proc_subphase_load,
                 std::unordered_map<PhaseType, CommMapType> const* proc_comm) override;
 
-  TimeType getWork(ElementIDType object, PhaseOffset when) override;
+  TimeType getWork(ElementIDStruct object, PhaseOffset when) override;
 
 private:
   std::unordered_map<PhaseType, CommMapType> const* proc_comm_; /**< Underlying comm data */

src/vt/vrt/collection/balance/model/composed_model.cc

@@ -56,7 +56,7 @@ void ComposedModel::updateLoads(PhaseType last_completed_phase) {
   base_->updateLoads(last_completed_phase);
 }
 
-TimeType ComposedModel::getWork(ElementIDType object, PhaseOffset when) {
+TimeType ComposedModel::getWork(ElementIDStruct object, PhaseOffset when) {
   return base_->getWork(object, when);
 }
 

src/vt/vrt/collection/balance/model/composed_model.h

@@ -71,7 +71,7 @@ class ComposedModel : public LoadModel
 
   void updateLoads(PhaseType last_completed_phase) override;
 
-  TimeType getWork(ElementIDType object, PhaseOffset when) override;
+  TimeType getWork(ElementIDStruct object, PhaseOffset when) override;
   unsigned int getNumPastPhasesNeeded(unsigned int look_back) override;
 
   ObjectIterator begin() override;