Skip to content

Commit

Permalink
#2342: update schema to support seq_id in tasks field when id is not …
Browse files Browse the repository at this point in the history
…bit-encoded; add unit test
  • Loading branch information
cwschilly authored and cz4rs committed Sep 20, 2024
1 parent 6aa4287 commit 69df764
Show file tree
Hide file tree
Showing 4 changed files with 169 additions and 19 deletions.
7 changes: 4 additions & 3 deletions scripts/JSON_data_files_validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -435,11 +435,12 @@ def validate_comm_links(all_jsons):
for data in all_jsons:
if data["phases"][n].get("communications") is not None:
comms = data["phases"][n]["communications"]
comm_ids.update({int(comm["from"]["id"]) for comm in comms})
comm_ids.update({int(comm["to"]["id"]) for comm in comms})
id_string = "id" if "id" in comms[0]["from"] else "seq_id"
comm_ids.update({int(comm["from"][id_string]) for comm in comms})
comm_ids.update({int(comm["to"][id_string]) for comm in comms})

tasks = data["phases"][n]["tasks"]
task_ids.update({int(task["entity"]["id"]) for task in tasks})
task_ids.update({int(task["entity"][id_string]) for task in tasks})

if not comm_ids.issubset(task_ids):
logging.error(
Expand Down
27 changes: 18 additions & 9 deletions scripts/LBDatafile_schema.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
from schema import And, Optional, Schema

def validate_id_and_seq_id(field):
"""Ensure that either seq_id or id is provided."""
if 'seq_id' not in field and 'id' not in field:
raise ValueError('Either id (bit-encoded) or seq_id must be provided.')
return field

LBDatafile_schema = Schema(
{
Optional('type'): And(str, "LBDatafile", error="'LBDatafile' must be chosen."),
Expand Down Expand Up @@ -30,15 +36,16 @@
'id': int,
'tasks': [
{
'entity': {
'entity': And({
Optional('collection_id'): int,
'home': int,
'id': int,
Optional('id'): int,
Optional('seq_id'): int,
Optional('index'): [int],
'type': str,
'migratable': bool,
Optional('objgroup_id'): int
},
}, validate_id_and_seq_id),
'node': int,
'resource': str,
Optional('subphases'): [
Expand All @@ -55,25 +62,27 @@
Optional('communications'): [
{
'type': str,
'to': {
'to': And({
'type': str,
'id': int,
Optional('id'): int,
Optional('seq_id'): int,
Optional('home'): int,
Optional('collection_id'): int,
Optional('migratable'): bool,
Optional('index'): [int],
Optional('objgroup_id'): int,
},
}, validate_id_and_seq_id),
'messages': int,
'from': {
'from': And({
'type': str,
'id': int,
Optional('id'): int,
Optional('seq_id'): int,
Optional('home'): int,
Optional('collection_id'): int,
Optional('migratable'): bool,
Optional('index'): [int],
Optional('objgroup_id'): int,
},
}, validate_id_and_seq_id),
'bytes': float
}
],
Expand Down
26 changes: 19 additions & 7 deletions src/vt/vrt/collection/balance/lb_data_holder.cc
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,14 @@ LBDataHolder::LBDataHolder(nlohmann::json const& j)
vtAssertExpr(node.is_number());

if (etype == "object") {
auto object = task["entity"]["id"];
nlohmann::json object;
bool bitpacked_id = false;
if (task["entity"].find("id") != task["entity"].end()) {
object = task["entity"]["id"];
bitpacked_id = true;
} else {
object = task["entity"]["seq_id"];
}
vtAssertExpr(object.is_number());

auto elm = ElementIDStruct{object, node};
Expand All @@ -314,13 +321,18 @@ LBDataHolder::LBDataHolder(nlohmann::json const& j)
task["entity"].find("index") != task["entity"].end()
) {
using Field = uint64_t;
auto strippedObject = BitPackerType::getField<
vt::elm::eElmIDProxyBitsNonObjGroup::ID,
vt::elm::elm_id_num_bits,
Field
>(static_cast<Field>(object));
Field object_id;
if (bitpacked_id) {
object_id = BitPackerType::getField<
vt::elm::eElmIDProxyBitsNonObjGroup::ID,
vt::elm::elm_id_num_bits,
Field
>(static_cast<Field>(object));
} else {
object_id = static_cast<Field>(object);
}
elm = elm::ElmIDBits::createCollectionImpl(migratable,
strippedObject,
object_id,
home,
node);
auto cid = task["entity"]["collection_id"];
Expand Down
128 changes: 128 additions & 0 deletions tests/unit/lb/test_lb_data_holder.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
/*
//@HEADER
// *****************************************************************************
//
// test_lb_data_holder.cc
// DARMA/vt => Virtual Transport
//
// Copyright 2019-2021 National Technology & Engineering Solutions of Sandia, LLC
// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S.
// Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * Neither the name of the copyright holder nor the names of its
// contributors may be used to endorse or promote products derived from this
// software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact [email protected]
//
// *****************************************************************************
//@HEADER
*/

#include "vt/elm/elm_id_bits.h"
#include <vt/collective/startup.h>
#include <vt/vrt/collection/balance/lb_data_holder.h>

#include "test_harness.h"

#include <nlohmann/json.hpp>

namespace vt { namespace tests { namespace unit { namespace lb {

using TestLBDataHolder = TestHarness;

nlohmann::json create_basic_json(std::string id_type, int id,
int home, int node, bool migratable) {
nlohmann::json j = {
{"metadata", {
{"rank", 0},
{"type", "LBDatafile"}
}},
{"phases", {
{
{"id", 0},
{"tasks", {
{
{"entity", {
{"collection_id", 7},
{"index", {0}},
{"home", home},
{id_type, id},
{"migratable", migratable},
{"type", "object"}
}},
{"node", node},
{"resource", "cpu"},
{"time", 0.5},
}
}}
}
}}
};

return j;
}

void test_data_holder_elms(int seq_id, int home, int node, bool migratable) {
// Determine encoded ID
auto elm = elm::ElmIDBits::createCollectionImpl(migratable, seq_id, home, node);
auto encoded_id = elm.id;

// Create DataHolder and get resulting object elm
auto simple_json_id = create_basic_json("id", encoded_id, home, node, migratable);
auto dh_id = vt::vrt::collection::balance::LBDataHolder(simple_json_id);
auto elm_id = dh_id.node_data_[0].begin()->first;

// Create new DataHolder using "seq_id" and get elm
auto simple_json_seq = create_basic_json("seq_id", seq_id, home, node, migratable);
auto dh_seq = vt::vrt::collection::balance::LBDataHolder(simple_json_seq);
auto elm_seq = dh_seq.node_data_[0].begin()->first;

// Assert that both elms are equal (have the same id)
EXPECT_EQ(elm_id, elm_seq);
EXPECT_EQ(elm_id, elm);
}

TEST_F(TestLBDataHolder, test_lb_data_holder_no_comms_object_id) {
// Initialize
int argc = 0;
char** argv = nullptr;
MPI_Comm comm = MPI_COMM_WORLD;
vt::initialize(argc, argv, &comm);

// Run a variety of test cases (seq_id, home, node, migratable)
test_data_holder_elms(0,0,0,false);
test_data_holder_elms(0,0,0,true);
test_data_holder_elms(0,0,2,false);
test_data_holder_elms(0,0,1,true);
test_data_holder_elms(1,1,0,false);
test_data_holder_elms(2,1,9,true);
test_data_holder_elms(3,0,1,false);

// Finalize
// vt::finalize();
}

}}}} // end namespace vt::tests::unit::lb

0 comments on commit 69df764

Please sign in to comment.