From a96259d784ca878bef7e010b10a0b0786a1b15ac Mon Sep 17 00:00:00 2001 From: Caleb Schilly Date: Mon, 26 Aug 2024 06:39:40 -0700 Subject: [PATCH 1/8] #2338: begin work encoding object features --- .../vrt/collection/balance/lb_data_holder.cc | 9 ++++++- .../vrt/collection/balance/workload_replay.cc | 2 ++ tests/unit/lb/test_offlinelb.cc | 24 ++++++++++++++----- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/vt/vrt/collection/balance/lb_data_holder.cc b/src/vt/vrt/collection/balance/lb_data_holder.cc index 5ed8b917c0..dd992912d0 100644 --- a/src/vt/vrt/collection/balance/lb_data_holder.cc +++ b/src/vt/vrt/collection/balance/lb_data_holder.cc @@ -42,7 +42,10 @@ */ #include "vt/vrt/collection/balance/lb_data_holder.h" +#include "vt/vrt/collection/balance/node_lb_data.h" #include "vt/context/context.h" +#include "vt/elm/elm_id_bits.h" +#include "vt/elm/elm_comm.h" #include @@ -296,6 +299,10 @@ LBDataHolder::LBDataHolder(nlohmann::json const& j) auto node = task["node"]; auto time = task["time"]; auto etype = task["entity"]["type"]; + auto home = task["entity"]["home"]; + bool migratable = task["entity"]["migratable"]; + auto seq_id = theNodeLBData()->getNextElm(); + vtAssertExpr(time.is_number()); vtAssertExpr(node.is_number()); @@ -303,7 +310,7 @@ LBDataHolder::LBDataHolder(nlohmann::json const& j) auto object = task["entity"]["id"]; vtAssertExpr(object.is_number()); - auto elm = ElementIDStruct{object, node}; + auto elm = elm::ElmIDBits::createCollectionImpl(migratable, seq_id, home, node); this->node_data_[id][elm].whole_phase_load = time; if ( diff --git a/src/vt/vrt/collection/balance/workload_replay.cc b/src/vt/vrt/collection/balance/workload_replay.cc index 7d6f8e145b..4497a24caa 100644 --- a/src/vt/vrt/collection/balance/workload_replay.cc +++ b/src/vt/vrt/collection/balance/workload_replay.cc @@ -51,6 +51,7 @@ #include #include +#include namespace vt { namespace vrt { namespace collection { namespace balance { namespace replay { @@ -60,6 +61,7 @@ void replayWorkloads( ) { // read in object loads from json files auto const filename = theConfig()->getLBDataFileIn(); + std::cout << "Reading in file: " << filename << std::endl; auto workloads = readInWorkloads(filename); // use the default stats handler diff --git a/tests/unit/lb/test_offlinelb.cc b/tests/unit/lb/test_offlinelb.cc index 541b9225e7..a74940dba3 100644 --- a/tests/unit/lb/test_offlinelb.cc +++ b/tests/unit/lb/test_offlinelb.cc @@ -45,6 +45,8 @@ #include #include +#include + #include #include "test_parallel_harness.h" @@ -67,12 +69,22 @@ struct SimCol : vt::Collection { auto const next_node = (this_node + 1) % num_nodes; auto const prev_node = this_node - 1 >= 0 ? this_node - 1 : num_nodes - 1; vt_debug_print(terse, lb, "handler: idx={}: elm={}\n", getIndex(), getElmID()); - if (m->iter == 0 or m->iter == 3 or m->iter == 6) { - EXPECT_EQ(getIndex().x() / 2, this_node); - } else if (m->iter == 1 or m-> iter == 2) { - EXPECT_EQ(getIndex().x() / 2, prev_node); - } else if (m->iter == 4 or m-> iter == 5) { - EXPECT_EQ(getIndex().x() / 2, next_node); + int this_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &this_rank); + if (this_rank == 0) { + std::cout << std::endl << "m->iter: " << m->iter << "\n" << + "getIndex().x(): " << getIndex().x() << "\n" << + "getIndex().x()/2: " << getIndex().x() / 2 << "\n" << + "this node: " << this_node << "\n" << + "prev node: " << prev_node << "\n" << + "next node: " << next_node << "\n" << std::endl; + if (m->iter == 0 or m->iter == 3 or m->iter == 6) { + EXPECT_EQ(getIndex().x() / 2, this_node); + } else if (m->iter == 1 or m-> iter == 2) { + EXPECT_EQ(getIndex().x() / 2, prev_node); + } else if (m->iter == 4 or m-> iter == 5) { + EXPECT_EQ(getIndex().x() / 2, next_node); + } } } From a59637700d70a3a75a09bc6f2001fd28f3eb0078 Mon Sep 17 00:00:00 2001 From: Caleb Schilly Date: Mon, 26 Aug 2024 13:41:42 -0400 Subject: [PATCH 2/8] #2338: use object as seq_id; only use createCollectionImpl if collection id exists --- src/vt/vrt/collection/balance/lb_data_holder.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/vt/vrt/collection/balance/lb_data_holder.cc b/src/vt/vrt/collection/balance/lb_data_holder.cc index dd992912d0..c975cee85c 100644 --- a/src/vt/vrt/collection/balance/lb_data_holder.cc +++ b/src/vt/vrt/collection/balance/lb_data_holder.cc @@ -301,7 +301,6 @@ LBDataHolder::LBDataHolder(nlohmann::json const& j) auto etype = task["entity"]["type"]; auto home = task["entity"]["home"]; bool migratable = task["entity"]["migratable"]; - auto seq_id = theNodeLBData()->getNextElm(); vtAssertExpr(time.is_number()); vtAssertExpr(node.is_number()); @@ -310,13 +309,13 @@ LBDataHolder::LBDataHolder(nlohmann::json const& j) auto object = task["entity"]["id"]; vtAssertExpr(object.is_number()); - auto elm = elm::ElmIDBits::createCollectionImpl(migratable, seq_id, home, node); - this->node_data_[id][elm].whole_phase_load = time; + auto elm = ElementIDStruct{object, node}; if ( task["entity"].find("collection_id") != task["entity"].end() and task["entity"].find("index") != task["entity"].end() ) { + elm = elm::ElmIDBits::createCollectionImpl(migratable, object, home, node); auto cid = task["entity"]["collection_id"]; auto idx = task["entity"]["index"]; if (cid.is_number() && idx.is_array()) { @@ -326,6 +325,8 @@ LBDataHolder::LBDataHolder(nlohmann::json const& j) } } + this->node_data_[id][elm].whole_phase_load = time; + if (task.find("subphases") != task.end()) { auto subphases = task["subphases"]; if (subphases.is_array()) { From 7c7e96fb81026bd4d2f32aede7f5c313d533e2df Mon Sep 17 00:00:00 2001 From: Caleb Schilly Date: Tue, 27 Aug 2024 15:12:42 -0400 Subject: [PATCH 3/8] #2338: determine ids of encoded objects via bit stripping --- src/vt/vrt/collection/balance/lb_data_holder.cc | 11 ++++++++++- .../vrt/collection/balance/workload_replay.cc | 2 -- tests/unit/utils/test_bit_packing.nompi.cc | 17 +++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/vt/vrt/collection/balance/lb_data_holder.cc b/src/vt/vrt/collection/balance/lb_data_holder.cc index c975cee85c..69e91f1f0f 100644 --- a/src/vt/vrt/collection/balance/lb_data_holder.cc +++ b/src/vt/vrt/collection/balance/lb_data_holder.cc @@ -315,9 +315,18 @@ LBDataHolder::LBDataHolder(nlohmann::json const& j) task["entity"].find("collection_id") != task["entity"].end() and task["entity"].find("index") != task["entity"].end() ) { - elm = elm::ElmIDBits::createCollectionImpl(migratable, object, home, node); + using Field = uint64_t; + auto strippedObject = BitPackerType::getField< + vt::elm::eElmIDProxyBitsNonObjGroup::ID, + vt::elm::elm_id_num_bits, + Field + >(static_cast(object)); auto cid = task["entity"]["collection_id"]; auto idx = task["entity"]["index"]; + elm = elm::ElmIDBits::createCollectionImpl(migratable, + strippedObject, + home, + node); if (cid.is_number() && idx.is_array()) { std::vector arr = idx; auto proxy = static_cast(cid); diff --git a/src/vt/vrt/collection/balance/workload_replay.cc b/src/vt/vrt/collection/balance/workload_replay.cc index 4497a24caa..7d6f8e145b 100644 --- a/src/vt/vrt/collection/balance/workload_replay.cc +++ b/src/vt/vrt/collection/balance/workload_replay.cc @@ -51,7 +51,6 @@ #include #include -#include namespace vt { namespace vrt { namespace collection { namespace balance { namespace replay { @@ -61,7 +60,6 @@ void replayWorkloads( ) { // read in object loads from json files auto const filename = theConfig()->getLBDataFileIn(); - std::cout << "Reading in file: " << filename << std::endl; auto workloads = readInWorkloads(filename); // use the default stats handler diff --git a/tests/unit/utils/test_bit_packing.nompi.cc b/tests/unit/utils/test_bit_packing.nompi.cc index a2946ac028..e3f845bb49 100644 --- a/tests/unit/utils/test_bit_packing.nompi.cc +++ b/tests/unit/utils/test_bit_packing.nompi.cc @@ -44,6 +44,7 @@ #include #include "vt/utils/bits/bits_packer.h" +#include "vt/elm/elm_id_bits.h" #include "test_harness.h" namespace vt { namespace tests { namespace unit { @@ -125,4 +126,20 @@ TEST_F(TestBitPacking, test_bit_packing_dynamic_check_masking_5) { EXPECT_EQ(x, 0x0000FCCD0000FEEDull); } +TEST_F(TestBitPacking, test_bit_packing_and_create_collection) { + using Field = uint64_t; + bool migratable = true; + int home = 0; + int node = 0; + auto seq_id = 3; + auto init_elm = elm::ElmIDBits::createCollectionImpl(migratable, seq_id, home, node); + auto derived_id = BitPackerType::getField< + vt::elm::eElmIDProxyBitsNonObjGroup::ID, + vt::elm::elm_id_num_bits, + Field + >(init_elm.id); + auto derived_elm = elm::ElmIDBits::createCollectionImpl(migratable, derived_id, home, node); + EXPECT_EQ(init_elm.id, derived_elm.id); +} + }}} /* end namespace vt::tests::unit */ From c766b4149a7078e9eb3e200b5d68c156ba207cf4 Mon Sep 17 00:00:00 2001 From: Caleb Schilly Date: Tue, 27 Aug 2024 15:27:32 -0400 Subject: [PATCH 4/8] #2338: clean up includes --- src/vt/vrt/collection/balance/lb_data_holder.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/vt/vrt/collection/balance/lb_data_holder.cc b/src/vt/vrt/collection/balance/lb_data_holder.cc index 69e91f1f0f..27d4966f01 100644 --- a/src/vt/vrt/collection/balance/lb_data_holder.cc +++ b/src/vt/vrt/collection/balance/lb_data_holder.cc @@ -42,10 +42,8 @@ */ #include "vt/vrt/collection/balance/lb_data_holder.h" -#include "vt/vrt/collection/balance/node_lb_data.h" #include "vt/context/context.h" #include "vt/elm/elm_id_bits.h" -#include "vt/elm/elm_comm.h" #include From ae05fee2e0c58ce95a89ee0f6902ff30fd73f314 Mon Sep 17 00:00:00 2001 From: Caleb Schilly Date: Tue, 27 Aug 2024 15:28:49 -0400 Subject: [PATCH 5/8] #2338: restore original ordering of code --- src/vt/vrt/collection/balance/lb_data_holder.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vt/vrt/collection/balance/lb_data_holder.cc b/src/vt/vrt/collection/balance/lb_data_holder.cc index 27d4966f01..333dc1babd 100644 --- a/src/vt/vrt/collection/balance/lb_data_holder.cc +++ b/src/vt/vrt/collection/balance/lb_data_holder.cc @@ -319,12 +319,12 @@ LBDataHolder::LBDataHolder(nlohmann::json const& j) vt::elm::elm_id_num_bits, Field >(static_cast(object)); - auto cid = task["entity"]["collection_id"]; - auto idx = task["entity"]["index"]; elm = elm::ElmIDBits::createCollectionImpl(migratable, strippedObject, home, node); + auto cid = task["entity"]["collection_id"]; + auto idx = task["entity"]["index"]; if (cid.is_number() && idx.is_array()) { std::vector arr = idx; auto proxy = static_cast(cid); From 8f48c0082d8e76b62d99becc015bafcc37dab2e1 Mon Sep 17 00:00:00 2001 From: Caleb Schilly Date: Tue, 27 Aug 2024 15:30:07 -0400 Subject: [PATCH 6/8] #2338: remove debug prints --- tests/unit/lb/test_offlinelb.cc | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/tests/unit/lb/test_offlinelb.cc b/tests/unit/lb/test_offlinelb.cc index a74940dba3..541b9225e7 100644 --- a/tests/unit/lb/test_offlinelb.cc +++ b/tests/unit/lb/test_offlinelb.cc @@ -45,8 +45,6 @@ #include #include -#include - #include #include "test_parallel_harness.h" @@ -69,22 +67,12 @@ struct SimCol : vt::Collection { auto const next_node = (this_node + 1) % num_nodes; auto const prev_node = this_node - 1 >= 0 ? this_node - 1 : num_nodes - 1; vt_debug_print(terse, lb, "handler: idx={}: elm={}\n", getIndex(), getElmID()); - int this_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &this_rank); - if (this_rank == 0) { - std::cout << std::endl << "m->iter: " << m->iter << "\n" << - "getIndex().x(): " << getIndex().x() << "\n" << - "getIndex().x()/2: " << getIndex().x() / 2 << "\n" << - "this node: " << this_node << "\n" << - "prev node: " << prev_node << "\n" << - "next node: " << next_node << "\n" << std::endl; - if (m->iter == 0 or m->iter == 3 or m->iter == 6) { - EXPECT_EQ(getIndex().x() / 2, this_node); - } else if (m->iter == 1 or m-> iter == 2) { - EXPECT_EQ(getIndex().x() / 2, prev_node); - } else if (m->iter == 4 or m-> iter == 5) { - EXPECT_EQ(getIndex().x() / 2, next_node); - } + if (m->iter == 0 or m->iter == 3 or m->iter == 6) { + EXPECT_EQ(getIndex().x() / 2, this_node); + } else if (m->iter == 1 or m-> iter == 2) { + EXPECT_EQ(getIndex().x() / 2, prev_node); + } else if (m->iter == 4 or m-> iter == 5) { + EXPECT_EQ(getIndex().x() / 2, next_node); } } From 47b07661cabc96e8399cbebd63f793edbd3c0d3c Mon Sep 17 00:00:00 2001 From: Caleb Schilly Date: Tue, 27 Aug 2024 15:47:41 -0400 Subject: [PATCH 7/8] #2338: set home to 1 in unit test --- tests/unit/utils/test_bit_packing.nompi.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/utils/test_bit_packing.nompi.cc b/tests/unit/utils/test_bit_packing.nompi.cc index e3f845bb49..4cd11d7a9d 100644 --- a/tests/unit/utils/test_bit_packing.nompi.cc +++ b/tests/unit/utils/test_bit_packing.nompi.cc @@ -129,7 +129,7 @@ TEST_F(TestBitPacking, test_bit_packing_dynamic_check_masking_5) { TEST_F(TestBitPacking, test_bit_packing_and_create_collection) { using Field = uint64_t; bool migratable = true; - int home = 0; + int home = 1; int node = 0; auto seq_id = 3; auto init_elm = elm::ElmIDBits::createCollectionImpl(migratable, seq_id, home, node); From 2dfae5d50cc98b444b61e5a55fc6d3998f3c32ba Mon Sep 17 00:00:00 2001 From: Caleb Schilly Date: Tue, 27 Aug 2024 17:11:10 -0400 Subject: [PATCH 8/8] #2338: small change to ping ci --- tests/unit/utils/test_bit_packing.nompi.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/utils/test_bit_packing.nompi.cc b/tests/unit/utils/test_bit_packing.nompi.cc index 4cd11d7a9d..09b356e7df 100644 --- a/tests/unit/utils/test_bit_packing.nompi.cc +++ b/tests/unit/utils/test_bit_packing.nompi.cc @@ -127,7 +127,6 @@ TEST_F(TestBitPacking, test_bit_packing_dynamic_check_masking_5) { } TEST_F(TestBitPacking, test_bit_packing_and_create_collection) { - using Field = uint64_t; bool migratable = true; int home = 1; int node = 0; @@ -136,7 +135,7 @@ TEST_F(TestBitPacking, test_bit_packing_and_create_collection) { auto derived_id = BitPackerType::getField< vt::elm::eElmIDProxyBitsNonObjGroup::ID, vt::elm::elm_id_num_bits, - Field + uint64_t >(init_elm.id); auto derived_elm = elm::ElmIDBits::createCollectionImpl(migratable, derived_id, home, node); EXPECT_EQ(init_elm.id, derived_elm.id);