[WIP] openPMD: variableBased IterationEncoding

.azure-pipelines.yml

@@ -52,7 +52,7 @@ jobs:
       sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY
       sudo chmod a+x /usr/local/bin/cmake-easyinstall
       if [ "${WARPX_CI_OPENPMD:-FALSE}" == "TRUE" ]; then
-        cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@0.13.2 \
+        cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@dev \
           -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF
         python -m pip install --upgrade openpmd-api
       fi

.github/workflows/ubuntu.yml

@@ -56,7 +56,7 @@ jobs:
       run: |
         .github/workflows/dependencies/nvcc11.sh
         export CEI_SUDO="sudo"
-        cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@0.13.2 -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF
+        cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@dev -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF
     - name: build WarpX
       run: |
         export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
@@ -103,7 +103,7 @@ jobs:
         sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY
         sudo chmod a+x /usr/local/bin/cmake-easyinstall
         export CEI_SUDO="sudo"
-        CXX=$(which icpc) CC=$(which icc) cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@0.13.2 -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF
+        CXX=$(which icpc) CC=$(which icc) cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/openPMD-api.git@dev -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF
     - name: build WarpX
       run: |
         set +e

Docs/source/install/cmake.rst

@@ -107,7 +107,7 @@ CMake Option                  Default & Values                               Des
 ``WarpX_amrex_internal``      **ON**/OFF                                     Needs a pre-installed AMReX library if set to ``OFF``
 ``WarpX_openpmd_src``         *None*                                         Path to openPMD-api source directory (preferred if set)
 ``WarpX_openpmd_repo``        ``https://github.com/openPMD/openPMD-api.git`` Repository URI to pull and build openPMD-api from
-``WarpX_openpmd_branch``      ``0.13.2``                                     Repository branch for ``WarpX_openpmd_repo``
+``WarpX_openpmd_branch``      ``dev``                                        Repository branch for ``WarpX_openpmd_repo``
 ``WarpX_openpmd_internal``    **ON**/OFF                                     Needs a pre-installed openPMD-api library if set to ``OFF``
 ``WarpX_picsar_src``          *None*                                         Path to PICSAR source directory (preferred if set)
 ``WarpX_picsar_repo``         ``https://github.com/ECP-WarpX/picsar.git``    Repository URI to pull and build PICSAR from

Docs/source/usage/parameters.rst

@@ -1587,8 +1587,8 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a
     ``json`` only works with serial/single-rank jobs.
     When WarpX is compiled with openPMD support, the first available backend in the order given above is taken.
 
-* ``<diag_name>.openpmd_tspf`` (`bool`, optional, default ``true``) only read if ``<diag_name>.format = openpmd``.
-    Whether to write one file per timestep.
+* ``<diag_name>.openpmd_encoding`` (optional, ``v`` (default, variable based), ``f`` (file based) or ``g`` (group based) ) only read if ``<diag_name>.format = openpmd``.
+    openPMD file output encoding (file based will write one file per timestep).
 
 * ``<diag_name>.fields_to_plot`` (list of `strings`, optional)
     Fields written to output.

Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp

@@ -12,12 +12,34 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name)
     // Which backend to use (ADIOS, ADIOS2 or HDF5). Default depends on what is available
     std::string openpmd_backend {"default"};
     // one file per timestep (or one file for all steps)
-    bool openpmd_tspf = true;
+    std::string  openpmd_encoding {"v"};
     pp_diag_name.query("openpmd_backend", openpmd_backend);
-    pp_diag_name.query("openpmd_tspf", openpmd_tspf);
+    bool encodingDefined = pp_diag_name.query("openpmd_encoding", openpmd_encoding);
+
+    openPMD::IterationEncoding encoding = openPMD::IterationEncoding::groupBased;
+    if ( 0 == openpmd_encoding.compare("v") )
+#if OPENPMDAPI_VERSION_GE(0, 14, 0)
+      encoding = openPMD::IterationEncoding::variableBased;
+#else
+      encoding = openPMD::IterationEncoding::groupBased;
+#endif
+    else if ( 0 == openpmd_encoding.compare("f") )
+      encoding = openPMD::IterationEncoding::fileBased;
+
+    //
+    // if no encoding is defined, then check to see if tspf is defined.
+    // (backward compatibility)
+    //
+    if ( !encodingDefined )
+    {
+      bool openpmd_tspf = false;
+      bool tspfDefined = pp_diag_name.query("openpmd_tspf", openpmd_tspf);
+      if ( tspfDefined && openpmd_tspf )
+           encoding = openPMD::IterationEncoding::fileBased;
+    }
     auto & warpx = WarpX::GetInstance();
     m_OpenPMDPlotWriter = std::make_unique<WarpXOpenPMDPlot>(
-        openpmd_tspf, openpmd_backend, warpx.getPMLdirections()
+        encoding, openpmd_backend, warpx.getPMLdirections()
     );
 }
 

Source/Diagnostics/WarpXOpenPMD.H

@@ -93,11 +93,11 @@ public:
 
   /** Initialize openPMD I/O routines
    *
-   * @param oneFilePerTS write one file per timestep
+   * @param ie  iteration encoding from openPMD: "group, file, variable"
    * @param filetype file backend, e.g. "bp" or "h5"
    * @param fieldPMLdirections PML field solver, @see WarpX::getPMLdirections()
    */
-  WarpXOpenPMDPlot (bool oneFilePerTS, std::string filetype, std::vector<bool> fieldPMLdirections);
+  WarpXOpenPMDPlot (openPMD::IterationEncoding ie, std::string filetype, std::vector<bool> fieldPMLdirections);
 
   ~WarpXOpenPMDPlot ();
 
@@ -227,7 +227,7 @@ private:
   //int m_NumSoAIntAttributes = PIdx::nattribs; //! WarpX' additional int particle attributes in SoA
   int m_NumAoSIntAttributes = 0; //! WarpX definition: no additional int attributes in particle AoS
 
-  bool m_OneFilePerTS = true;  //! write in openPMD fileBased manner for individual time steps
+  openPMD::IterationEncoding m_Encoding = openPMD::IterationEncoding::groupBased;
   std::string m_OpenPMDFileType = "bp"; //! MPI-parallel openPMD backend: bp or h5
   int m_CurrentStep  = -1;
 

Source/Diagnostics/WarpXOpenPMD.cpp

@@ -195,10 +195,10 @@ namespace detail
 }
 
 #ifdef WARPX_USE_OPENPMD
-WarpXOpenPMDPlot::WarpXOpenPMDPlot(bool oneFilePerTS,
+WarpXOpenPMDPlot::WarpXOpenPMDPlot(openPMD::IterationEncoding ie,
     std::string openPMDFileType, std::vector<bool> fieldPMLdirections)
   :m_Series(nullptr),
-   m_OneFilePerTS(oneFilePerTS),
+   m_Encoding(ie),
    m_OpenPMDFileType(std::move(openPMDFileType)),
    m_fieldPMLdirections(std::move(fieldPMLdirections))
 {
@@ -232,7 +232,7 @@ WarpXOpenPMDPlot::GetFileName (std::string& filepath)
   //
   // OpenPMD supports timestepped names
   //
-  if (m_OneFilePerTS)
+  if (m_Encoding == openPMD::IterationEncoding::fileBased)
       filename = filename.append("_%06T");
   filename.append(".").append(m_OpenPMDFileType);
   filepath.append(filename);
@@ -267,9 +267,10 @@ void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush)
     // close BTD file only when isLastBTDFlush is true
     if (isBTD and !isLastBTDFlush) callClose = false;
     if (callClose) {
-        if (m_Series)
-            m_Series->iterations[m_CurrentStep].close();
-
+        if (m_Series) {
+            auto iterations = m_Series->writeIterations();
+            iterations[m_CurrentStep].close();
+        }
         // create a little helper file for ParaView 5.9+
         if (amrex::ParallelDescriptor::IOProcessor())
         {
@@ -297,7 +298,11 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD)
 
     // close a previously open series before creating a new one
     // see ADIOS1 limitation: https://github.com/openPMD/openPMD-api/pull/686
-    m_Series = nullptr;
+    //if (m_OneFilePerTS)
+    if (m_Encoding == openPMD::IterationEncoding::fileBased)
+        m_Series = nullptr;
+    else if (m_Series != nullptr)
+        return;
 
     if (amrex::ParallelDescriptor::NProcs() > 1) {
 #if defined(AMREX_USE_MPI)
@@ -316,6 +321,8 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD)
         m_MPIRank = 1;
     }
 
+    m_Series->setIterationEncoding( m_Encoding );
+
     // input file / simulation setup author
     if( WarpX::authors.size() > 0u )
         m_Series->setAuthor( WarpX::authors );
@@ -430,7 +437,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc,
 
   WarpXParticleCounter counter(pc);
 
-  openPMD::Iteration currIteration = m_Series->iterations[iteration];
+  openPMD::WriteIterations iterations = m_Series->writeIterations();
+  auto currIteration = iterations[iteration];
+
   openPMD::ParticleSpecies currSpecies = currIteration.particles[name];
   // meta data for ED-PIC extension
   currSpecies.setAttribute( "particleShape", double( WarpX::noz ) );
@@ -811,7 +820,9 @@ WarpXOpenPMDPlot::WriteOpenPMDFields ( //const std::string& filename,
   auto const dataset = openPMD::Dataset(datatype, global_size);
 
   // meta data
-  auto series_iteration = m_Series->iterations[iteration];
+  openPMD::WriteIterations iterations = m_Series->writeIterations();
+  auto series_iteration = iterations[iteration];
+
   auto meshes = series_iteration.meshes;
   if( first_write_to_iteration ) {
       series_iteration.setTime( time );