Skip to content

Commit

Permalink
#868: lb: adjust ZoltanLB for new IDs
Browse files Browse the repository at this point in the history
  • Loading branch information
@lifflander @nlslatt
lifflander authored and nlslatt committed Feb 26, 2021
1 parent ce41ee6 commit 39ecdc9
Show file tree
Hide file tree
Showing 4 changed files with 61 additions and 43 deletions.
4 changes: 2 additions & 2 deletions src/vt/vrt/collection/balance/lb_comm.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ struct LBCommKey {
ElementIDStruct from_ = { no_element_id, uninitialized_destination, uninitialized_destination };
ElementIDStruct to_ = { no_element_id, uninitialized_destination, uninitialized_destination };

ElementIDType edge_id_ = no_element_id;
ElementIDStruct edge_id_ = { no_element_id, uninitialized_destination, uninitialized_destination };
NodeType nfrom_ = uninitialized_destination;
NodeType nto_ = uninitialized_destination;
CommCategory cat_ = CommCategory::SendRecv;
Expand All @@ -112,7 +112,7 @@ struct LBCommKey {
ElementIDStruct toObj() const { return to_; }
ElementIDType fromNode() const { return nfrom_; }
ElementIDType toNode() const { return nto_; }
ElementIDType edgeID() const { return edge_id_; }
ElementIDStruct edgeID() const { return edge_id_; }

bool selfEdge() const { return cat_ == CommCategory::SendRecv and from_ == to_; }
bool offNode() const {
Expand Down
1 change: 1 addition & 0 deletions src/vt/vrt/collection/balance/lb_common.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
#include "vt/vrt/collection/balance/lb_common.h"

#include <ostream>
#include <fmt/ostream.h>

namespace vt { namespace vrt { namespace collection { namespace balance {

Expand Down
2 changes: 2 additions & 0 deletions src/vt/vrt/collection/balance/lb_common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@
#include <unordered_map>
#include <ostream>

#include <fmt/ostream.h>

namespace vt { namespace vrt { namespace collection {
namespace balance {

Expand Down
97 changes: 56 additions & 41 deletions src/vt/vrt/collection/balance/zoltanlb/zoltanlb.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,11 @@ void ZoltanLB::runLB() {
num_export, num_import
);

std::set<ObjIDType> load_objs;
for (auto obj : *load_model_) {
load_objs.insert(obj);
}

for (int i = 0; i < num_export; i++) {
int to_node = export_procs[i];

Expand All @@ -161,7 +166,15 @@ void ZoltanLB::runLB() {
to_node, export_global_ids[i]
);

migrateObjectTo(export_global_ids[i], static_cast<NodeType>(to_node));
auto const obj_id = export_global_ids[i];
auto iter = load_objs.find(
ObjIDType{
obj_id, uninitialized_destination, uninitialized_destination
}
);
vtAssert(iter != load_objs.end(), "Object must exist here");

migrateObjectTo(*iter, static_cast<NodeType>(to_node));
}

Zoltan_LB_Free_Part(
Expand All @@ -186,11 +199,11 @@ void ZoltanLB::makeGraphSymmetric() {
elm.first.cat_ == balance::CommCategory::SendRecv and
not elm.first.selfEdge()
) {
auto from = elm.first.fromObjTemp();
auto to = elm.first.toObjTemp();
auto from = elm.first.fromObj();
auto to = elm.first.toObj();

auto from_node = objGetNode(from);
auto to_node = objGetNode(to);
auto from_node = from.curr_node;
auto to_node = to.curr_node;

vtAssert(
from_node == this_node or to_node == this_node,
Expand All @@ -199,7 +212,7 @@ void ZoltanLB::makeGraphSymmetric() {

vt_debug_print(
lb, node,
"makeGraphSymmetric: from={:x}, to={:x}\n",
"makeGraphSymmetric: from={}, to={}\n",
from, to
);

Expand All @@ -225,8 +238,8 @@ void ZoltanLB::recvSharedEdges(CommMsg* msg) {
for (auto&& elm : comm) {
vt_debug_print(
lb, node,
"recv shared edge: from={:x}, to={:x}\n",
elm.first.fromObjTemp(), elm.first.toObjTemp()
"recv shared edge: from={}, to={}\n",
elm.first.fromObj(), elm.first.toObj()
);

load_comm_symm[elm.first] += elm.second;
Expand All @@ -239,12 +252,11 @@ void ZoltanLB::combineEdges() {
ElementCommType load_comm_combined;

for (auto&& e1 : load_comm_symm) {
auto from = std::max(e1.first.fromObjTemp(), e1.first.toObjTemp());
auto to = std::min(e1.first.fromObjTemp(), e1.first.toObjTemp());
auto from = std::max(e1.first.fromObj(), e1.first.toObj());
auto to = std::min(e1.first.fromObj(), e1.first.toObj());

auto key = balance::LBCommKey{
balance::LBCommKey::CollectionTag{},
from, from, to, to, false
balance::LBCommKey::CollectionTag{}, from, to, false
};
load_comm_combined[key] += e1.second;
}
Expand All @@ -265,18 +277,18 @@ void ZoltanLB::countEdges() {
elm.first.cat_ == balance::CommCategory::SendRecv and
not elm.first.selfEdge()
) {
auto from = elm.first.fromObjTemp();
auto to = elm.first.toObjTemp();
auto from = elm.first.fromObj();
auto to = elm.first.toObj();

auto from_node = objGetNode(from);
auto to_node = objGetNode(to);
auto from_node = from.curr_node;
auto to_node = to.curr_node;

if (from_node == to_node and from_node == this_node) {
local_edge++;
} else {
// Break ties on non-local object edges based on obj ID
auto large_obj_id = std::max(from, to);
if (objGetNode(large_obj_id) == this_node) {
if (large_obj_id.curr_node == this_node) {
remote_owned_edge++;
}
}
Expand Down Expand Up @@ -307,42 +319,42 @@ void ZoltanLB::allocateShareEdgeGIDs() {

auto const this_node = theContext()->getNode();
for (auto&& elm : load_comm_symm) {
auto from = elm.first.fromObjTemp();
auto to = elm.first.toObjTemp();
auto from = elm.first.fromObj();
auto to = elm.first.toObj();

auto from_node = objGetNode(from);
auto to_node = objGetNode(to);
auto from_node = from.curr_node;
auto to_node = to.curr_node;

if (from_node == to_node and from_node == this_node) {
auto offset = max_edges_per_node_ * this_node;
auto id = 1 + offset + edge_id_++;
auto key = elm.first;
key.edge_id_ = id;
key.edge_id_.id = id;
load_comm_edge_id[key] = elm.second;

vt_debug_print(
lb, node,
"allocate: local edge_id={:x}, from={:x}, to={:x}\n",
key.edge_id_,
key.fromObjTemp(),
key.toObjTemp()
key.edge_id_.id,
key.fromObj(),
key.toObj()
);

} else {
auto large_obj_id = std::max(from, to);
if (objGetNode(large_obj_id) == this_node) {
if (large_obj_id.curr_node == this_node) {
auto offset = max_edges_per_node_ * this_node;
auto id = 1 + offset + edge_id_++;
auto key = elm.first;
key.edge_id_ = id;
key.edge_id_.id = id;
load_comm_edge_id[key] = elm.second;

vt_debug_print(
lb, node,
"allocate: remote edge_id={:x}, from={:x}, to={:x}\n",
key.edge_id_,
key.fromObjTemp(),
key.toObjTemp()
key.edge_id_.id,
key.fromObj(),
key.toObj()
);
} else {
// The other node will provide the edge
Expand Down Expand Up @@ -401,7 +413,7 @@ std::unique_ptr<ZoltanLB::Graph> ZoltanLB::makeGraph() {
graph->vertex_weight = std::make_unique<int[]>(graph->num_vertices);

static_assert(
sizeof(ZOLTAN_ID_TYPE) == sizeof(ObjIDType),
sizeof(ZOLTAN_ID_TYPE) == sizeof(uint64_t),
"ObjIDType must be exactly the same size as ZOLTAN_ID_TYPE\n"
"Please recompile with \"-D Zoltan_ENABLE_ULLONG_IDS:Bool=ON\""
);
Expand All @@ -417,11 +429,11 @@ std::unique_ptr<ZoltanLB::Graph> ZoltanLB::makeGraph() {
{
int idx = 0;
for (auto&& obj : load_objs) {
graph->vertex_gid[idx++] = obj;
graph->vertex_gid[idx++] = obj.id;

vt_debug_print(
lb, node,
"makeVertexGraph: vertex_id={}: obj={:x}\n",
"makeVertexGraph: vertex_idx={}: obj={}\n",
idx - 1, obj
);
}
Expand All @@ -432,7 +444,10 @@ std::unique_ptr<ZoltanLB::Graph> ZoltanLB::makeGraph() {
{
int idx = 0;
for (auto&& obj : load_objs) {
auto load = load_model_->getWork(obj, {balance::PhaseOffset::NEXT_PHASE, balance::PhaseOffset::WHOLE_PHASE});
auto load = load_model_->getWork(
obj,
{balance::PhaseOffset::NEXT_PHASE, balance::PhaseOffset::WHOLE_PHASE}
);
auto time = static_cast<int>(loadMilli(load));
graph->vertex_weight[idx++] = time;
}
Expand Down Expand Up @@ -475,11 +490,11 @@ std::unique_ptr<ZoltanLB::Graph> ZoltanLB::makeGraph() {
auto comm = iter->second;

vtAssert(
iter->first.edge_id_ != balance::no_element_id,
iter->first.edge_id_.id != balance::no_element_id,
"Must have element ID"
);

graph->edge_gid[edge_idx] = iter->first.edge_id_;
graph->edge_gid[edge_idx] = iter->first.edge_id_.id;
graph->edge_weight[edge_idx] = comm.bytes;

vt_debug_print(
Expand All @@ -494,18 +509,18 @@ std::unique_ptr<ZoltanLB::Graph> ZoltanLB::makeGraph() {
vt_debug_print(
lb, node,
"makeEdgeGraph: \t edge_id={}: edge_idx={}, obj={:x}\n",
iter->first.edge_id_, edge_idx, iter->first.fromObjTemp()
iter->first.edge_id_, edge_idx, iter->first.fromObj().id
);

graph->neighbor_gid[neighbor_idx++] = iter->first.fromObjTemp();
graph->neighbor_gid[neighbor_idx++] = iter->first.fromObj().id;

vt_debug_print(
lb, node,
"makeEdgeGraph: \t edge_id={}: edge_idx={}, obj={:x}\n",
iter->first.edge_id_, edge_idx, iter->first.toObjTemp()
iter->first.edge_id_, edge_idx, iter->first.toObj().id
);

graph->neighbor_gid[neighbor_idx++] = iter->first.toObjTemp();
graph->neighbor_gid[neighbor_idx++] = iter->first.toObj().id;

// This edge begins at neighbor_idx
graph->neighbor_idx[edge_idx + 1] = neighbor_idx;
Expand All @@ -514,7 +529,7 @@ std::unique_ptr<ZoltanLB::Graph> ZoltanLB::makeGraph() {
lb, node,
"edge_id={:x} from={:x}, to={:x}\n",
iter->first.edge_id_,
iter->first.fromObjTemp(), iter->first.toObjTemp()
iter->first.fromObj(), iter->first.toObj()
);

edge_idx++;
Expand Down

0 comments on commit 39ecdc9

Please sign in to comment.