Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ADIOS2: Fix BTD Particle Resize w/ Empty Ranks #3657

Merged
merged 3 commits into from
Feb 2, 2023
Merged

ADIOS2: Fix BTD Particle Resize w/ Empty Ranks #3657

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bc3cc55
ADIOS2: Fix BTD Resize w/ Empty Ranks
ax3l Jan 31, 2023
82440ca
[Draft] Start Duplicating All Put Logic
ax3l Feb 1, 2023
d1967a5
Better Work-Around
ax3l Feb 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions Source/Diagnostics/WarpXOpenPMD.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -744,6 +744,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc,
m_Series->flush();

// dump individual particles
bool contributed_particles = false; // did the local MPI rank contribute particles?
for (auto currentLevel = 0; currentLevel <= pc->finestLevel(); currentLevel++) {
uint64_t offset = static_cast<uint64_t>( counter.m_ParticleOffsetAtRank[currentLevel] );
// For BTD, the offset include the number of particles already flushed
Expand All @@ -754,8 +755,11 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc,

// Do not call storeChunk() with zero-sized particle tiles:
// https://github.com/openPMD/openPMD-api/issues/1147
// https://github.com/ECP-WarpX/WarpX/pull/1898#discussion_r745008290
if (numParticleOnTile == 0) continue;

contributed_particles = true;

// get position and particle ID from aos
// note: this implementation iterates the AoS 4x...
// if we flush late as we do now, we can also copy out the data in one go
Expand Down Expand Up @@ -833,8 +837,59 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc,
write_int_comp, int_comp_names);

offset += numParticleOnTile64;
} // pti
} // currentLevel

// work-around for BTD particle resize in ADIOS2
//
// This issues an empty ADIOS2 Put to make sure the new global shape
// meta-data is committed for each variable.
//
// Refs.:
// https://github.com/ECP-WarpX/WarpX/issues/3389
// https://github.com/ornladios/ADIOS2/issues/3455
// BP4 (ADIOS 2.8): last MPI rank's `Put` meta-data wins
// BP5 (ADIOS 2.8): everyone has to write an empty block
if (is_resizing_flush && !contributed_particles && isBTD && m_Series->backend() == "ADIOS2") {
for( auto & [record_name, record] : currSpecies ) {
for( auto & [comp_name, comp] : record ) {
if (comp.constant()) continue;

auto dtype = comp.getDatatype();
switch (dtype) {
case openPMD::Datatype::FLOAT :
[[fallthrough]];
case openPMD::Datatype::DOUBLE : {
auto empty_data = std::make_shared<amrex::ParticleReal>();
comp.storeChunk(empty_data, {uint64_t(0)}, {uint64_t(0)});
break;
}
case openPMD::Datatype::UINT : {
auto empty_data = std::make_shared<unsigned int>();
comp.storeChunk(empty_data, {uint64_t(0)}, {uint64_t(0)});
break;
}
case openPMD::Datatype::ULONG : {
auto empty_data = std::make_shared<unsigned long>();
comp.storeChunk(empty_data, {uint64_t(0)}, {uint64_t(0)});
break;
}
case openPMD::Datatype::ULONGLONG : {
auto empty_data = std::make_shared<unsigned long long>();
comp.storeChunk(empty_data, {uint64_t(0)}, {uint64_t(0)});
break;
}
default : {
std::string msg = "WarpX openPMD ADIOS2 work-around has unknown dtype: ";
msg += datatypeToString(dtype);
amrex::Abort(msg);
break;
}
}
}
}
}

m_Series->flush();
}

Expand Down