From 8126ff4737cc0a1269ceeab8d8ff42a1afb5bd5f Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Fri, 23 Apr 2021 01:06:37 -0400 Subject: [PATCH 1/8] Added support to write to one file --- Source/Diagnostics/WarpXOpenPMD.cpp | 35 ++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 2ff21de2aab..505b2d55d1d 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -267,9 +267,10 @@ void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush) // close BTD file only when isLastBTDFlush is true if (isBTD and !isLastBTDFlush) callClose = false; if (callClose) { - if (m_Series) - m_Series->iterations[m_CurrentStep].close(); - + if (m_Series) { + auto iterations = m_Series->writeIterations(); + iterations[m_CurrentStep].close(); + } // create a little helper file for ParaView 5.9+ if (amrex::ParallelDescriptor::IOProcessor()) { @@ -295,15 +296,29 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) std::string filepath = m_dirPrefix; GetFileName(filepath); + std::string useSteps = R"( + { + "adios2": { + "engine": { + "type": "bp4", + "usesteps": true + } + } + } + )"; // close a previously open series before creating a new one // see ADIOS1 limitation: https://github.com/openPMD/openPMD-api/pull/686 - m_Series = nullptr; + if (m_OneFilePerTS) + m_Series = nullptr; + else if (m_Series != nullptr) + return; if (amrex::ParallelDescriptor::NProcs() > 1) { #if defined(AMREX_USE_MPI) m_Series = std::make_unique( filepath, access, - amrex::ParallelDescriptor::Communicator() + amrex::ParallelDescriptor::Communicator(), + useSteps ); m_MPISize = amrex::ParallelDescriptor::NProcs(); m_MPIRank = amrex::ParallelDescriptor::MyProc(); @@ -311,7 +326,7 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) amrex::Abort("openPMD-api not built with MPI support!"); #endif } else { - m_Series = std::make_unique(filepath, access); + m_Series = std::make_unique(filepath, access, useSteps); m_MPISize = 1; m_MPIRank = 1; } @@ -430,7 +445,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, WarpXParticleCounter counter(pc); - openPMD::Iteration currIteration = m_Series->iterations[iteration]; + openPMD::WriteIterations iterations = m_Series->writeIterations(); + auto currIteration = iterations[iteration]; + openPMD::ParticleSpecies currSpecies = currIteration.particles[name]; // meta data for ED-PIC extension currSpecies.setAttribute( "particleShape", double( WarpX::noz ) ); @@ -811,7 +828,9 @@ WarpXOpenPMDPlot::WriteOpenPMDFields ( //const std::string& filename, auto const dataset = openPMD::Dataset(datatype, global_size); // meta data - auto series_iteration = m_Series->iterations[iteration]; + openPMD::WriteIterations iterations = m_Series->writeIterations(); + auto series_iteration = iterations[iteration]; + auto meshes = series_iteration.meshes; if( first_write_to_iteration ) { series_iteration.setTime( time ); From 0ae19c1f1a516bec44ca11277a3e8671a00e1789 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Fri, 23 Apr 2021 20:49:58 -0400 Subject: [PATCH 2/8] updated with the latest openPMD version --- Source/Diagnostics/WarpXOpenPMD.cpp | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 505b2d55d1d..d208211f112 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -296,16 +296,6 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) std::string filepath = m_dirPrefix; GetFileName(filepath); - std::string useSteps = R"( - { - "adios2": { - "engine": { - "type": "bp4", - "usesteps": true - } - } - } - )"; // close a previously open series before creating a new one // see ADIOS1 limitation: https://github.com/openPMD/openPMD-api/pull/686 if (m_OneFilePerTS) @@ -317,8 +307,7 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) #if defined(AMREX_USE_MPI) m_Series = std::make_unique( filepath, access, - amrex::ParallelDescriptor::Communicator(), - useSteps + amrex::ParallelDescriptor::Communicator() ); m_MPISize = amrex::ParallelDescriptor::NProcs(); m_MPIRank = amrex::ParallelDescriptor::MyProc(); @@ -326,11 +315,15 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) amrex::Abort("openPMD-api not built with MPI support!"); #endif } else { - m_Series = std::make_unique(filepath, access, useSteps); + m_Series = std::make_unique(filepath, access); m_MPISize = 1; m_MPIRank = 1; } + if ( m_OpenPMDFileType.compare("bp") == 0 ) + m_Series->setIterationEncoding( openPMD::IterationEncoding::variableBased ); + + // input file / simulation setup author if( WarpX::authors.size() > 0u ) m_Series->setAuthor( WarpX::authors ); From 78253dbe945addfb0ae3c32005cefd10345267fc Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 23 Apr 2021 18:33:50 -0700 Subject: [PATCH 3/8] [draft] openPMD-api: dev temporarily update to dev branch --- .azure-pipelines.yml | 2 +- .github/workflows/ubuntu.yml | 4 ++-- Docs/source/install/cmake.rst | 2 +- cmake/dependencies/openPMD.cmake | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 2510da8ffb1..516287dec2b 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -49,7 +49,7 @@ jobs: sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY sudo chmod a+x /usr/local/bin/cmake-easyinstall if [ "${WARPX_CI_OPENPMD:-FALSE}" == "TRUE" ]; then - cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@0.13.2 \ + cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@dev \ -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF python -m pip install --upgrade openpmd-api fi diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 87652adcf92..189d0b5a812 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -56,7 +56,7 @@ jobs: run: | .github/workflows/dependencies/nvcc11.sh export CEI_SUDO="sudo" - cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@0.13.2 -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF + cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@dev -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF - name: build WarpX run: | export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} @@ -103,7 +103,7 @@ jobs: sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" - CXX=$(which icpc) CC=$(which icc) cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@0.13.2 -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF + CXX=$(which icpc) CC=$(which icc) cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@dev -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF - name: build WarpX run: | set +e diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 32aeb3580e6..5b2cfab20d4 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -107,7 +107,7 @@ CMake Option Default & Values Des ``WarpX_amrex_internal`` **ON**/OFF Needs a pre-installed AMReX library if set to ``OFF`` ``WarpX_openpmd_src`` *None* Path to openPMD-api source directory (preferred if set) ``WarpX_openpmd_repo`` ``https://github.com/openPMD/openPMD-api.git`` Repository URI to pull and build openPMD-api from -``WarpX_openpmd_branch`` ``0.13.2`` Repository branch for ``WarpX_openpmd_repo`` +``WarpX_openpmd_branch`` ``dev`` Repository branch for ``WarpX_openpmd_repo`` ``WarpX_openpmd_internal`` **ON**/OFF Needs a pre-installed openPMD-api library if set to ``OFF`` ``WarpX_picsar_src`` *None* Path to PICSAR source directory (preferred if set) ``WarpX_picsar_repo`` ``https://github.com/ECP-WarpX/picsar.git`` Repository URI to pull and build PICSAR from diff --git a/cmake/dependencies/openPMD.cmake b/cmake/dependencies/openPMD.cmake index db1213ce452..bc1e5417bf1 100644 --- a/cmake/dependencies/openPMD.cmake +++ b/cmake/dependencies/openPMD.cmake @@ -81,7 +81,7 @@ if(WarpX_OPENPMD) set(WarpX_openpmd_repo "https://github.com/openPMD/openPMD-api.git" CACHE STRING "Repository URI to pull and build openPMD-api from if(WarpX_openpmd_internal)") - set(WarpX_openpmd_branch "0.13.2" + set(WarpX_openpmd_branch "dev" CACHE STRING "Repository branch for WarpX_openpmd_repo if(WarpX_openpmd_internal)") From 99cfcd636c45e5cf72a8f382d7708651034024d2 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Fri, 23 Apr 2021 21:34:36 -0400 Subject: [PATCH 4/8] backward compatible comparison --- Source/Diagnostics/WarpXOpenPMD.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index d208211f112..e12dac72ada 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -321,8 +321,11 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) } if ( m_OpenPMDFileType.compare("bp") == 0 ) +#if OPENPMDAPI_VERSION_GE(0, 14, 0) m_Series->setIterationEncoding( openPMD::IterationEncoding::variableBased ); - +#else + m_Series->setIterationEncoding( openPMD::IterationEncoding::groupBased ); +#endif // input file / simulation setup author if( WarpX::authors.size() > 0u ) From e14ac05a315ce77cfc3eec2eebd760150baf1c3c Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Tue, 18 May 2021 01:32:31 -0400 Subject: [PATCH 5/8] updated openpmd_tspf to openpmd_encoding --- Docs/source/usage/parameters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 33010505440..6bc03ed54a0 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1572,7 +1572,7 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a ``json`` only works with serial/single-rank jobs. When WarpX is compiled with openPMD support, the first available backend in the order given above is taken. -* ``.openpmd_tspf`` (`bool`, optional, default ``true``) only read if ``.format = openpmd``. +* ``.openpmd_encoding`` (optional, ``v`` (default, variable based), ``f`` (file based) or ``g`` (group based) ) only read if ``.format = openpmd``. Whether to write one file per timestep. * ``.fields_to_plot`` (list of `strings`, optional) From a2f58b434d483391869bff5cb80d2e55ef85e470 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Tue, 18 May 2021 01:33:10 -0400 Subject: [PATCH 6/8] replaced binary option openpmd_tspf by openpmd_encoding, taking v/g/f for variable based, group based or file based encoding --- .../FlushFormats/FlushFormatOpenPMD.cpp | 28 +++++++++++++++++-- Source/Diagnostics/WarpXOpenPMD.H | 6 ++-- Source/Diagnostics/WarpXOpenPMD.cpp | 16 ++++------- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp index eb3846e69c4..0c55d424b2a 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp @@ -12,12 +12,34 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) // Which backend to use (ADIOS, ADIOS2 or HDF5). Default depends on what is available std::string openpmd_backend {"default"}; // one file per timestep (or one file for all steps) - bool openpmd_tspf = true; + std::string openpmd_encoding {"v"}; pp_diag_name.query("openpmd_backend", openpmd_backend); - pp_diag_name.query("openpmd_tspf", openpmd_tspf); + bool encodingDefined = pp_diag_name.query("openpmd_encoding", openpmd_encoding); + + openPMD::IterationEncoding encoding = openPMD::IterationEncoding::groupBased; + if ( 0 == openpmd_encoding.compare("v") ) +#if OPENPMDAPI_VERSION_GE(0, 14, 0) + encoding = openPMD::IterationEncoding::variableBased; +#else + encoding = openPMD::IterationEncoding::groupBased; +#endif + else if ( 0 == openpmd_encoding.compare("f") ) + encoding = openPMD::IterationEncoding::fileBased; + + // + // if no encoding is defined, then check to see if tspf is defined. + // (backward compatibility) + // + if ( !encodingDefined ) + { + bool openpmd_tspf = false; + bool tspfDefined = pp_diag_name.query("openpmd_tspf", openpmd_tspf); + if ( tspfDefined && openpmd_tspf ) + encoding = openPMD::IterationEncoding::fileBased; + } auto & warpx = WarpX::GetInstance(); m_OpenPMDPlotWriter = std::make_unique( - openpmd_tspf, openpmd_backend, warpx.getPMLdirections() + encoding, openpmd_backend, warpx.getPMLdirections() ); } diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 4152dd89692..ca4123f6271 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -93,11 +93,11 @@ public: /** Initialize openPMD I/O routines * - * @param oneFilePerTS write one file per timestep + * @param ie iteration encoding from openPMD: "group, file, variable" * @param filetype file backend, e.g. "bp" or "h5" * @param fieldPMLdirections PML field solver, @see WarpX::getPMLdirections() */ - WarpXOpenPMDPlot (bool oneFilePerTS, std::string filetype, std::vector fieldPMLdirections); + WarpXOpenPMDPlot (openPMD::IterationEncoding ie, std::string filetype, std::vector fieldPMLdirections); ~WarpXOpenPMDPlot (); @@ -227,7 +227,7 @@ private: //int m_NumSoAIntAttributes = PIdx::nattribs; //! WarpX' additional int particle attributes in SoA int m_NumAoSIntAttributes = 0; //! WarpX definition: no additional int attributes in particle AoS - bool m_OneFilePerTS = true; //! write in openPMD fileBased manner for individual time steps + openPMD::IterationEncoding m_Encoding = openPMD::IterationEncoding::groupBased; std::string m_OpenPMDFileType = "bp"; //! MPI-parallel openPMD backend: bp or h5 int m_CurrentStep = -1; diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index e12dac72ada..69844b2c49f 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -195,10 +195,10 @@ namespace detail } #ifdef WARPX_USE_OPENPMD -WarpXOpenPMDPlot::WarpXOpenPMDPlot(bool oneFilePerTS, +WarpXOpenPMDPlot::WarpXOpenPMDPlot(openPMD::IterationEncoding ie, std::string openPMDFileType, std::vector fieldPMLdirections) :m_Series(nullptr), - m_OneFilePerTS(oneFilePerTS), + m_Encoding(ie), m_OpenPMDFileType(std::move(openPMDFileType)), m_fieldPMLdirections(std::move(fieldPMLdirections)) { @@ -232,7 +232,7 @@ WarpXOpenPMDPlot::GetFileName (std::string& filepath) // // OpenPMD supports timestepped names // - if (m_OneFilePerTS) + if (m_Encoding == openPMD::IterationEncoding::fileBased) filename = filename.append("_%06T"); filename.append(".").append(m_OpenPMDFileType); filepath.append(filename); @@ -298,7 +298,8 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) // close a previously open series before creating a new one // see ADIOS1 limitation: https://github.com/openPMD/openPMD-api/pull/686 - if (m_OneFilePerTS) + //if (m_OneFilePerTS) + if (m_Encoding == openPMD::IterationEncoding::fileBased) m_Series = nullptr; else if (m_Series != nullptr) return; @@ -320,12 +321,7 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) m_MPIRank = 1; } - if ( m_OpenPMDFileType.compare("bp") == 0 ) -#if OPENPMDAPI_VERSION_GE(0, 14, 0) - m_Series->setIterationEncoding( openPMD::IterationEncoding::variableBased ); -#else - m_Series->setIterationEncoding( openPMD::IterationEncoding::groupBased ); -#endif + m_Series->setIterationEncoding( m_Encoding ); // input file / simulation setup author if( WarpX::authors.size() > 0u ) From 49ab5204994f19228e11f51ce819235467a3941e Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Tue, 18 May 2021 01:50:28 -0400 Subject: [PATCH 7/8] fixed eol --- Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp index 0c55d424b2a..ef0bd9db2bf 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp @@ -24,18 +24,18 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) encoding = openPMD::IterationEncoding::groupBased; #endif else if ( 0 == openpmd_encoding.compare("f") ) - encoding = openPMD::IterationEncoding::fileBased; + encoding = openPMD::IterationEncoding::fileBased; // // if no encoding is defined, then check to see if tspf is defined. - // (backward compatibility) + // (backward compatibility) // if ( !encodingDefined ) { bool openpmd_tspf = false; - bool tspfDefined = pp_diag_name.query("openpmd_tspf", openpmd_tspf); - if ( tspfDefined && openpmd_tspf ) - encoding = openPMD::IterationEncoding::fileBased; + bool tspfDefined = pp_diag_name.query("openpmd_tspf", openpmd_tspf); + if ( tspfDefined && openpmd_tspf ) + encoding = openPMD::IterationEncoding::fileBased; } auto & warpx = WarpX::GetInstance(); m_OpenPMDPlotWriter = std::make_unique( From a8d91ff6956ed9d955aff79607dcdc263c89a4f5 Mon Sep 17 00:00:00 2001 From: Junmin Gu Date: Tue, 18 May 2021 20:10:59 -0400 Subject: [PATCH 8/8] updated doc --- Docs/source/usage/parameters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 9be758389e9..89582e62f61 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1588,7 +1588,7 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a When WarpX is compiled with openPMD support, the first available backend in the order given above is taken. * ``.openpmd_encoding`` (optional, ``v`` (default, variable based), ``f`` (file based) or ``g`` (group based) ) only read if ``.format = openpmd``. - Whether to write one file per timestep. + openPMD file output encoding (file based will write one file per timestep). * ``.fields_to_plot`` (list of `strings`, optional) Fields written to output.