diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 9feb668d866..630ba492e85 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -52,7 +52,7 @@ jobs: sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY sudo chmod a+x /usr/local/bin/cmake-easyinstall if [ "${WARPX_CI_OPENPMD:-FALSE}" == "TRUE" ]; then - cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@0.13.2 \ + cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@dev \ -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF python -m pip install --upgrade openpmd-api fi diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 87652adcf92..189d0b5a812 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -56,7 +56,7 @@ jobs: run: | .github/workflows/dependencies/nvcc11.sh export CEI_SUDO="sudo" - cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@0.13.2 -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF + cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@dev -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF - name: build WarpX run: | export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} @@ -103,7 +103,7 @@ jobs: sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" - CXX=$(which icpc) CC=$(which icc) cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@0.13.2 -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF + CXX=$(which icpc) CC=$(which icc) cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@dev -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF - name: build WarpX run: | set +e diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 32aeb3580e6..5b2cfab20d4 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -107,7 +107,7 @@ CMake Option Default & Values Des ``WarpX_amrex_internal`` **ON**/OFF Needs a pre-installed AMReX library if set to ``OFF`` ``WarpX_openpmd_src`` *None* Path to openPMD-api source directory (preferred if set) ``WarpX_openpmd_repo`` ``https://github.com/openPMD/openPMD-api.git`` Repository URI to pull and build openPMD-api from -``WarpX_openpmd_branch`` ``0.13.2`` Repository branch for ``WarpX_openpmd_repo`` +``WarpX_openpmd_branch`` ``dev`` Repository branch for ``WarpX_openpmd_repo`` ``WarpX_openpmd_internal`` **ON**/OFF Needs a pre-installed openPMD-api library if set to ``OFF`` ``WarpX_picsar_src`` *None* Path to PICSAR source directory (preferred if set) ``WarpX_picsar_repo`` ``https://github.com/ECP-WarpX/picsar.git`` Repository URI to pull and build PICSAR from diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 3ffb6d6be35..89582e62f61 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1587,8 +1587,8 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a ``json`` only works with serial/single-rank jobs. When WarpX is compiled with openPMD support, the first available backend in the order given above is taken. -* ``.openpmd_tspf`` (`bool`, optional, default ``true``) only read if ``.format = openpmd``. - Whether to write one file per timestep. +* ``.openpmd_encoding`` (optional, ``v`` (default, variable based), ``f`` (file based) or ``g`` (group based) ) only read if ``.format = openpmd``. + openPMD file output encoding (file based will write one file per timestep). * ``.fields_to_plot`` (list of `strings`, optional) Fields written to output. diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp index eb3846e69c4..ef0bd9db2bf 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp @@ -12,12 +12,34 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) // Which backend to use (ADIOS, ADIOS2 or HDF5). Default depends on what is available std::string openpmd_backend {"default"}; // one file per timestep (or one file for all steps) - bool openpmd_tspf = true; + std::string openpmd_encoding {"v"}; pp_diag_name.query("openpmd_backend", openpmd_backend); - pp_diag_name.query("openpmd_tspf", openpmd_tspf); + bool encodingDefined = pp_diag_name.query("openpmd_encoding", openpmd_encoding); + + openPMD::IterationEncoding encoding = openPMD::IterationEncoding::groupBased; + if ( 0 == openpmd_encoding.compare("v") ) +#if OPENPMDAPI_VERSION_GE(0, 14, 0) + encoding = openPMD::IterationEncoding::variableBased; +#else + encoding = openPMD::IterationEncoding::groupBased; +#endif + else if ( 0 == openpmd_encoding.compare("f") ) + encoding = openPMD::IterationEncoding::fileBased; + + // + // if no encoding is defined, then check to see if tspf is defined. + // (backward compatibility) + // + if ( !encodingDefined ) + { + bool openpmd_tspf = false; + bool tspfDefined = pp_diag_name.query("openpmd_tspf", openpmd_tspf); + if ( tspfDefined && openpmd_tspf ) + encoding = openPMD::IterationEncoding::fileBased; + } auto & warpx = WarpX::GetInstance(); m_OpenPMDPlotWriter = std::make_unique( - openpmd_tspf, openpmd_backend, warpx.getPMLdirections() + encoding, openpmd_backend, warpx.getPMLdirections() ); } diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 4152dd89692..ca4123f6271 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -93,11 +93,11 @@ public: /** Initialize openPMD I/O routines * - * @param oneFilePerTS write one file per timestep + * @param ie iteration encoding from openPMD: "group, file, variable" * @param filetype file backend, e.g. "bp" or "h5" * @param fieldPMLdirections PML field solver, @see WarpX::getPMLdirections() */ - WarpXOpenPMDPlot (bool oneFilePerTS, std::string filetype, std::vector fieldPMLdirections); + WarpXOpenPMDPlot (openPMD::IterationEncoding ie, std::string filetype, std::vector fieldPMLdirections); ~WarpXOpenPMDPlot (); @@ -227,7 +227,7 @@ private: //int m_NumSoAIntAttributes = PIdx::nattribs; //! WarpX' additional int particle attributes in SoA int m_NumAoSIntAttributes = 0; //! WarpX definition: no additional int attributes in particle AoS - bool m_OneFilePerTS = true; //! write in openPMD fileBased manner for individual time steps + openPMD::IterationEncoding m_Encoding = openPMD::IterationEncoding::groupBased; std::string m_OpenPMDFileType = "bp"; //! MPI-parallel openPMD backend: bp or h5 int m_CurrentStep = -1; diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 2ff21de2aab..69844b2c49f 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -195,10 +195,10 @@ namespace detail } #ifdef WARPX_USE_OPENPMD -WarpXOpenPMDPlot::WarpXOpenPMDPlot(bool oneFilePerTS, +WarpXOpenPMDPlot::WarpXOpenPMDPlot(openPMD::IterationEncoding ie, std::string openPMDFileType, std::vector fieldPMLdirections) :m_Series(nullptr), - m_OneFilePerTS(oneFilePerTS), + m_Encoding(ie), m_OpenPMDFileType(std::move(openPMDFileType)), m_fieldPMLdirections(std::move(fieldPMLdirections)) { @@ -232,7 +232,7 @@ WarpXOpenPMDPlot::GetFileName (std::string& filepath) // // OpenPMD supports timestepped names // - if (m_OneFilePerTS) + if (m_Encoding == openPMD::IterationEncoding::fileBased) filename = filename.append("_%06T"); filename.append(".").append(m_OpenPMDFileType); filepath.append(filename); @@ -267,9 +267,10 @@ void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush) // close BTD file only when isLastBTDFlush is true if (isBTD and !isLastBTDFlush) callClose = false; if (callClose) { - if (m_Series) - m_Series->iterations[m_CurrentStep].close(); - + if (m_Series) { + auto iterations = m_Series->writeIterations(); + iterations[m_CurrentStep].close(); + } // create a little helper file for ParaView 5.9+ if (amrex::ParallelDescriptor::IOProcessor()) { @@ -297,7 +298,11 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) // close a previously open series before creating a new one // see ADIOS1 limitation: https://github.com/openPMD/openPMD-api/pull/686 - m_Series = nullptr; + //if (m_OneFilePerTS) + if (m_Encoding == openPMD::IterationEncoding::fileBased) + m_Series = nullptr; + else if (m_Series != nullptr) + return; if (amrex::ParallelDescriptor::NProcs() > 1) { #if defined(AMREX_USE_MPI) @@ -316,6 +321,8 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) m_MPIRank = 1; } + m_Series->setIterationEncoding( m_Encoding ); + // input file / simulation setup author if( WarpX::authors.size() > 0u ) m_Series->setAuthor( WarpX::authors ); @@ -430,7 +437,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, WarpXParticleCounter counter(pc); - openPMD::Iteration currIteration = m_Series->iterations[iteration]; + openPMD::WriteIterations iterations = m_Series->writeIterations(); + auto currIteration = iterations[iteration]; + openPMD::ParticleSpecies currSpecies = currIteration.particles[name]; // meta data for ED-PIC extension currSpecies.setAttribute( "particleShape", double( WarpX::noz ) ); @@ -811,7 +820,9 @@ WarpXOpenPMDPlot::WriteOpenPMDFields ( //const std::string& filename, auto const dataset = openPMD::Dataset(datatype, global_size); // meta data - auto series_iteration = m_Series->iterations[iteration]; + openPMD::WriteIterations iterations = m_Series->writeIterations(); + auto series_iteration = iterations[iteration]; + auto meshes = series_iteration.meshes; if( first_write_to_iteration ) { series_iteration.setTime( time );