diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 2cee7db218e..63707576fa0 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -744,6 +744,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, m_Series->flush(); // dump individual particles + bool contributed_particles = false; // did the local MPI rank contribute particles? for (auto currentLevel = 0; currentLevel <= pc->finestLevel(); currentLevel++) { uint64_t offset = static_cast( counter.m_ParticleOffsetAtRank[currentLevel] ); // For BTD, the offset include the number of particles already flushed @@ -754,8 +755,11 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, // Do not call storeChunk() with zero-sized particle tiles: // https://github.com/openPMD/openPMD-api/issues/1147 + // https://github.com/ECP-WarpX/WarpX/pull/1898#discussion_r745008290 if (numParticleOnTile == 0) continue; + contributed_particles = true; + // get position and particle ID from aos // note: this implementation iterates the AoS 4x... // if we flush late as we do now, we can also copy out the data in one go @@ -833,8 +837,59 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, write_int_comp, int_comp_names); offset += numParticleOnTile64; + } // pti + } // currentLevel + + // work-around for BTD particle resize in ADIOS2 + // + // This issues an empty ADIOS2 Put to make sure the new global shape + // meta-data is committed for each variable. + // + // Refs.: + // https://github.com/ECP-WarpX/WarpX/issues/3389 + // https://github.com/ornladios/ADIOS2/issues/3455 + // BP4 (ADIOS 2.8): last MPI rank's `Put` meta-data wins + // BP5 (ADIOS 2.8): everyone has to write an empty block + if (is_resizing_flush && !contributed_particles && isBTD && m_Series->backend() == "ADIOS2") { + for( auto & [record_name, record] : currSpecies ) { + for( auto & [comp_name, comp] : record ) { + if (comp.constant()) continue; + + auto dtype = comp.getDatatype(); + switch (dtype) { + case openPMD::Datatype::FLOAT : + [[fallthrough]]; + case openPMD::Datatype::DOUBLE : { + auto empty_data = std::make_shared(); + comp.storeChunk(empty_data, {uint64_t(0)}, {uint64_t(0)}); + break; + } + case openPMD::Datatype::UINT : { + auto empty_data = std::make_shared(); + comp.storeChunk(empty_data, {uint64_t(0)}, {uint64_t(0)}); + break; + } + case openPMD::Datatype::ULONG : { + auto empty_data = std::make_shared(); + comp.storeChunk(empty_data, {uint64_t(0)}, {uint64_t(0)}); + break; + } + case openPMD::Datatype::ULONGLONG : { + auto empty_data = std::make_shared(); + comp.storeChunk(empty_data, {uint64_t(0)}, {uint64_t(0)}); + break; + } + default : { + std::string msg = "WarpX openPMD ADIOS2 work-around has unknown dtype: "; + msg += datatypeToString(dtype); + amrex::Abort(msg); + break; + } + } + } } } + m_Series->flush(); }