Added new option <diag>.openpmd_encoding

Docs/source/usage/parameters.rst

@@ -1591,8 +1591,8 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a
     ``json`` only works with serial/single-rank jobs.
     When WarpX is compiled with openPMD support, the first available backend in the order given above is taken.
 
-* ``<diag_name>.openpmd_tspf`` (`bool`, optional, default ``true``) only read if ``<diag_name>.format = openpmd``.
-    Whether to write one file per timestep.
+* ``<diag_name>.openpmd_encoding`` (optional, ``v`` (default, variable based), ``f`` (file based) or ``g`` (group based) ) only read if ``<diag_name>.format = openpmd``.
+     openPMD file output encoding (file based will write one file per timestep).
 
 * ``<diag_name>.fields_to_plot`` (list of `strings`, optional)
     Fields written to output.

Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp

@@ -12,13 +12,35 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name)
     // Which backend to use (ADIOS, ADIOS2 or HDF5). Default depends on what is available
     std::string openpmd_backend {"default"};
     // one file per timestep (or one file for all steps)
-    bool openpmd_tspf = true;
+    std::string  openpmd_encoding {"v"};
     pp_diag_name.query("openpmd_backend", openpmd_backend);
-    pp_diag_name.query("openpmd_tspf", openpmd_tspf);
-    auto & warpx = WarpX::GetInstance();
-    m_OpenPMDPlotWriter = std::make_unique<WarpXOpenPMDPlot>(
-        openpmd_tspf, openpmd_backend, warpx.getPMLdirections()
-    );
+    bool encodingDefined = pp_diag_name.query("openpmd_encoding", openpmd_encoding);
+
+    openPMD::IterationEncoding encoding = openPMD::IterationEncoding::groupBased;
+    if ( 0 == openpmd_encoding.compare("v") )
+#if OPENPMDAPI_VERSION_GE(0, 14, 0)
+      encoding = openPMD::IterationEncoding::variableBased;
+#else
+    encoding = openPMD::IterationEncoding::groupBased;
+#endif
+    else if ( 0 == openpmd_encoding.compare("f") )
+      encoding = openPMD::IterationEncoding::fileBased;
+
+  //
+  // if no encoding is defined, then check to see if tspf is defined.
+  // (backward compatibility)
+  //
+  if ( !encodingDefined )
+    {
+      bool openpmd_tspf = false;
+      bool tspfDefined = pp_diag_name.query("openpmd_tspf", openpmd_tspf);
+      if ( tspfDefined && openpmd_tspf )
+    encoding = openPMD::IterationEncoding::fileBased;
+    }
+  auto & warpx = WarpX::GetInstance();
+  m_OpenPMDPlotWriter = std::make_unique<WarpXOpenPMDPlot>(
+                               encoding, openpmd_backend, warpx.getPMLdirections()
+                               );
 }
 
 void

Source/Diagnostics/WarpXOpenPMD.H

@@ -93,11 +93,11 @@ public:
 
   /** Initialize openPMD I/O routines
    *
-   * @param oneFilePerTS write one file per timestep
+   * @param ie  iteration encoding from openPMD: "group, file, variable"
    * @param filetype file backend, e.g. "bp" or "h5"
    * @param fieldPMLdirections PML field solver, @see WarpX::getPMLdirections()
    */
-  WarpXOpenPMDPlot (bool oneFilePerTS, std::string filetype, std::vector<bool> fieldPMLdirections);
+  WarpXOpenPMDPlot (openPMD::IterationEncoding ie, std::string filetype, std::vector<bool> fieldPMLdirections);
 
   ~WarpXOpenPMDPlot ();
 
@@ -243,7 +243,7 @@ private:
   //int m_NumSoAIntAttributes = PIdx::nattribs; //! WarpX' additional int particle attributes in SoA
   int m_NumAoSIntAttributes = 0; //! WarpX definition: no additional int attributes in particle AoS
 
-  bool m_OneFilePerTS = true;  //! write in openPMD fileBased manner for individual time steps
+  openPMD::IterationEncoding m_Encoding = openPMD::IterationEncoding::groupBased;
   std::string m_OpenPMDFileType = "bp"; //! MPI-parallel openPMD backend: bp or h5
   int m_CurrentStep  = -1;
 

Source/Diagnostics/WarpXOpenPMD.cpp

@@ -195,10 +195,10 @@ namespace detail
 }
 
 #ifdef WARPX_USE_OPENPMD
-WarpXOpenPMDPlot::WarpXOpenPMDPlot(bool oneFilePerTS,
+WarpXOpenPMDPlot::WarpXOpenPMDPlot(openPMD::IterationEncoding ie,
     std::string openPMDFileType, std::vector<bool> fieldPMLdirections)
   :m_Series(nullptr),
-   m_OneFilePerTS(oneFilePerTS),
+   m_Encoding(ie),
    m_OpenPMDFileType(std::move(openPMDFileType)),
    m_fieldPMLdirections(std::move(fieldPMLdirections))
 {
@@ -232,7 +232,7 @@ WarpXOpenPMDPlot::GetFileName (std::string& filepath)
   //
   // OpenPMD supports timestepped names
   //
-  if (m_OneFilePerTS)
+  if (m_Encoding == openPMD::IterationEncoding::fileBased)
       filename = filename.append("_%06T");
   filename.append(".").append(m_OpenPMDFileType);
   filepath.append(filename);
@@ -256,19 +256,43 @@ void WarpXOpenPMDPlot::SetStep (int ts, const std::string& dirPrefix,
             amrex::Warning(warnMsg);
         }
     }
+
+    if ( isBTD )
+    {
+      if ( ( openPMD::IterationEncoding::fileBased != m_Encoding ) &&
+           ( openPMD::IterationEncoding::groupBased != m_Encoding ) )
+      {
+        std::string warnMsg = "Unable to support BTD with streaming. Using GroupBased ";
+        amrex::Warning(warnMsg);
+        m_Encoding = openPMD::IterationEncoding::groupBased;
+      }
+    }
     m_CurrentStep = ts;
     Init(openPMD::Access::CREATE, isBTD);
 }
 
 void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush)
 {
+  auto lf_iterationClose = [&] ()
+    {
+      if ( isBTD ) {
+        auto it = m_Series->iterations[m_CurrentStep];
+        it.close();
+      } else {
+        auto iterations = m_Series->writeIterations();
+        auto it = iterations[m_CurrentStep];
+        it.close();
+      }
+    };
+
     // default close is true
     bool callClose = true;
     // close BTD file only when isLastBTDFlush is true
     if (isBTD and !isLastBTDFlush) callClose = false;
     if (callClose) {
-        if (m_Series)
-            m_Series->iterations[m_CurrentStep].close();
+        if (m_Series) {
+      lf_iterationClose();
+        }
 
         // create a little helper file for ParaView 5.9+
         if (amrex::ParallelDescriptor::IOProcessor())
@@ -297,7 +321,10 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD)
 
     // close a previously open series before creating a new one
     // see ADIOS1 limitation: https://github.com/openPMD/openPMD-api/pull/686
-    m_Series = nullptr;
+    if ( m_Encoding == openPMD::IterationEncoding::fileBased )
+        m_Series = nullptr;
+    else if ( m_Series != nullptr )
+        return;
 
     if (amrex::ParallelDescriptor::NProcs() > 1) {
 #if defined(AMREX_USE_MPI)
@@ -316,6 +343,8 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD)
         m_MPIRank = 1;
     }
 
+    m_Series->setIterationEncoding( m_Encoding );
+
     // input file / simulation setup author
     if( WarpX::authors.size() > 0u )
         m_Series->setAuthor( WarpX::authors );
@@ -430,7 +459,22 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc,
 
   WarpXParticleCounter counter(pc);
 
-  openPMD::Iteration currIteration = m_Series->iterations[iteration];
+  auto  lfs_test =[&] () -> openPMD::Iteration&
+  {
+    // do not use steps with these two encodings, so BTD will be able to revisit previous steps
+    if (  (openPMD::IterationEncoding::fileBased == m_Encoding ) ||
+          (openPMD::IterationEncoding::groupBased == m_Encoding )  )
+    {
+        openPMD::Iteration& it = m_Series->iterations[iteration];
+        return it;
+    } else {
+        auto iterations = m_Series->writeIterations();
+        openPMD::Iteration& it = iterations[iteration];
+        return it;
+    }
+  };
+
+  openPMD::Iteration& currIteration = lfs_test();
 
   openPMD::ParticleSpecies currSpecies = currIteration.particles[name];
   // meta data for ED-PIC extension
@@ -933,12 +977,27 @@ WarpXOpenPMDPlot::WriteOpenPMDFieldsAll ( //const std::string& filename,
 
   AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_Series != nullptr, "openPMD series must be initialized");
 
+  auto  lfs_test =[&] () -> openPMD::Iteration&
+  {
+    // do not use steps with these two encodings, so BTD will be able to revisit previous steps
+    if (  (openPMD::IterationEncoding::fileBased == m_Encoding ) ||
+          (openPMD::IterationEncoding::groupBased == m_Encoding )  )
+    {
+        openPMD::Iteration& it = m_Series->iterations[m_CurrentStep];
+        return it;
+    } else {
+        auto iterations = m_Series->writeIterations();
+        openPMD::Iteration& it = iterations[m_CurrentStep];
+        return it;
+    }
+  };
+
   // is this either a regular write (true) or the first write in a
   // backtransformed diagnostic (BTD):
   bool const first_write_to_iteration = ! m_Series->iterations.contains( iteration );
 
   // meta data
-  openPMD::Iteration series_iteration = m_Series->iterations[iteration];
+  openPMD::Iteration& series_iteration = lfs_test();
 
   auto meshes = series_iteration.meshes;
   if (first_write_to_iteration) {