Encapsulate multi-process communication (part 2)

bindings/C/adios2/c/adios2_c_adios.cpp

@@ -10,10 +10,13 @@
 
 #include "adios2_c_adios.h"
 
-#include "adios2/common/ADIOSMPI.h"
 #include "adios2/core/ADIOS.h"
 #include "adios2/helper/adiosFunctions.h"
 
+#ifdef ADIOS2_HAVE_MPI
+#include "adios2/helper/adiosCommMPI.h"
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -35,7 +38,8 @@ adios2_adios *adios2_init_config_glue(const char *config_file, MPI_Comm comm,
         const bool debugBool =
             (debug_mode == adios2_debug_mode_on) ? true : false;
         adios = reinterpret_cast<adios2_adios *>(new adios2::core::ADIOS(
-            config_file, comm, debugBool, host_language));
+            config_file, adios2::helper::CommFromMPI(comm), debugBool,
+            host_language));
     }
     catch (...)
     {

bindings/C/adios2/c/adios2_c_io.cpp

@@ -12,11 +12,14 @@
 
 #include <vector>
 
-#include "adios2/common/ADIOSMPI.h"
 #include "adios2/core/IO.h"
 #include "adios2/helper/adiosFunctions.h" //GetType<T>
 #include "adios2_c_internal.h"
 
+#ifdef ADIOS2_HAVE_MPI
+#include "adios2/helper/adiosCommMPI.h"
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -729,7 +732,8 @@ adios2_engine *adios2_open_new_comm(adios2_io *io, const char *name,
             io, "for adios2_io, in call to adios2_open");
         engine = reinterpret_cast<adios2_engine *>(
             &reinterpret_cast<adios2::core::IO *>(io)->Open(
-                name, adios2_ToOpenMode(mode), comm));
+                name, adios2_ToOpenMode(mode),
+                adios2::helper::CommFromMPI(comm)));
     }
     catch (...)
     {

bindings/CXX11/adios2/cxx11/ADIOS.cpp

@@ -8,16 +8,20 @@
 
 #include "ADIOS.h"
 
-#include "adios2/common/ADIOSMPI.h"
 #include "adios2/core/ADIOS.h"
 #include "adios2/core/IO.h"
 #include "adios2/helper/adiosFunctions.h" //CheckForNullptr
 
+#ifdef ADIOS2_HAVE_MPI
+#include "adios2/helper/adiosCommMPI.h"
+#endif
+
 namespace adios2
 {
 #ifdef ADIOS2_HAVE_MPI
 ADIOS::ADIOS(const std::string &configFile, MPI_Comm comm, const bool debugMode)
-: m_ADIOS(std::make_shared<core::ADIOS>(configFile, comm, debugMode, "C++"))
+: m_ADIOS(std::make_shared<core::ADIOS>(configFile, helper::CommFromMPI(comm),
+                                        debugMode, "C++"))
 {
 }
 

bindings/CXX11/adios2/cxx11/IO.cpp

@@ -11,9 +11,12 @@
 #include "IO.h"
 #include "IO.tcc"
 
-#include "adios2/common/ADIOSMPI.h"
 #include "adios2/core/IO.h"
 
+#ifdef ADIOS2_HAVE_MPI
+#include "adios2/helper/adiosCommMPI.h"
+#endif
+
 namespace adios2
 {
 
@@ -109,7 +112,7 @@ Engine IO::Open(const std::string &name, const Mode mode, MPI_Comm comm)
 {
     helper::CheckForNullptr(m_IO,
                             "for engine " + name + ", in call to IO::Open");
-    return Engine(&m_IO->Open(name, mode, comm));
+    return Engine(&m_IO->Open(name, mode, helper::CommFromMPI(comm)));
 }
 #endif
 

bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.cpp

@@ -11,23 +11,26 @@
 #include "ADIOS2fstream.h"
 #include "ADIOS2fstream.tcc"
 
-#include "adios2/common/ADIOSMPI.h"
+#ifdef ADIOS2_HAVE_MPI
+#include "adios2/helper/adiosCommMPI.h"
+#endif
 
 namespace adios2
 {
 
 #ifdef ADIOS2_HAVE_MPI
 fstream::fstream(const std::string &name, const openmode mode, MPI_Comm comm,
                  const std::string engineType)
-: m_Stream(std::make_shared<core::Stream>(name, ToMode(mode), comm, engineType,
-                                          "C++"))
+: m_Stream(std::make_shared<core::Stream>(
+      name, ToMode(mode), helper::CommFromMPI(comm), engineType, "C++"))
 {
 }
 
 fstream::fstream(const std::string &name, const openmode mode, MPI_Comm comm,
                  const std::string &configFile,
                  const std::string ioInConfigFile)
-: m_Stream(std::make_shared<core::Stream>(name, ToMode(mode), comm, configFile,
+: m_Stream(std::make_shared<core::Stream>(name, ToMode(mode),
+                                          helper::CommFromMPI(comm), configFile,
                                           ioInConfigFile, "C++"))
 {
 }
@@ -54,8 +57,8 @@ void fstream::open(const std::string &name, const openmode mode, MPI_Comm comm,
                    const std::string engineType)
 {
     CheckOpen(name);
-    m_Stream = std::make_shared<core::Stream>(name, ToMode(mode), comm,
-                                              engineType, "C++");
+    m_Stream = std::make_shared<core::Stream>(
+        name, ToMode(mode), helper::CommFromMPI(comm), engineType, "C++");
 }
 
 void fstream::open(const std::string &name, const openmode mode, MPI_Comm comm,
@@ -64,7 +67,8 @@ void fstream::open(const std::string &name, const openmode mode, MPI_Comm comm,
 {
     CheckOpen(name);
     m_Stream = std::make_shared<core::Stream>(
-        name, ToMode(mode), comm, configFile, ioInConfigFile, "C++");
+        name, ToMode(mode), helper::CommFromMPI(comm), configFile,
+        ioInConfigFile, "C++");
 }
 #else
 void fstream::open(const std::string &name, const openmode mode,

bindings/Python/py11ADIOS.cpp

@@ -10,7 +10,9 @@
 
 #include "py11ADIOS.h"
 
-#include "adios2/common/ADIOSMPI.h"
+#ifdef ADIOS2_HAVE_MPI
+#include "adios2/helper/adiosCommMPI.h"
+#endif
 
 namespace adios2
 {
@@ -20,8 +22,8 @@ namespace py11
 #ifdef ADIOS2_HAVE_MPI
 ADIOS::ADIOS(const std::string &configFile, MPI4PY_Comm mpiComm,
              const bool debugMode)
-: m_ADIOS(std::make_shared<adios2::core::ADIOS>(configFile, mpiComm, debugMode,
-                                                "Python"))
+: m_ADIOS(std::make_shared<adios2::core::ADIOS>(
+      configFile, helper::CommFromMPI(mpiComm), debugMode, "Python"))
 {
 }
 

bindings/Python/py11File.cpp

@@ -14,42 +14,52 @@
 #include <algorithm>
 #include <iostream>
 
-#include "adios2/common/ADIOSMPI.h"
 #include "adios2/common/ADIOSMacros.h"
+#include "adios2/helper/adiosCommDummy.h"
 #include "adios2/helper/adiosFunctions.h"
 
+#ifdef ADIOS2_HAVE_MPI
+#include "adios2/helper/adiosCommMPI.h"
+#endif
+
 #include "py11types.h"
 
 namespace adios2
 {
 namespace py11
 {
 
+#ifdef ADIOS2_HAVE_MPI
 File::File(const std::string &name, const std::string mode, MPI_Comm comm,
            const std::string engineType)
 : m_Name(name), m_Mode(mode),
-  m_Stream(std::make_shared<core::Stream>(name, ToMode(mode), comm, engineType,
-                                          "Python"))
+  m_Stream(std::make_shared<core::Stream>(
+      name, ToMode(mode), helper::CommFromMPI(comm), engineType, "Python"))
 {
 }
 
 File::File(const std::string &name, const std::string mode, MPI_Comm comm,
            const std::string &configFile, const std::string ioInConfigFile)
 : m_Name(name), m_Mode(mode),
-  m_Stream(std::make_shared<core::Stream>(name, ToMode(mode), comm, configFile,
+  m_Stream(std::make_shared<core::Stream>(name, ToMode(mode),
+                                          helper::CommFromMPI(comm), configFile,
                                           ioInConfigFile, "Python"))
 {
 }
+#endif
 
 File::File(const std::string &name, const std::string mode,
            const std::string engineType)
-: File(name, mode, MPI_COMM_NULL, engineType)
+: m_Name(name), m_Mode(mode), m_Stream(std::make_shared<core::Stream>(
+                                  name, ToMode(mode), engineType, "Python"))
 {
 }
 
 File::File(const std::string &name, const std::string mode,
            const std::string &configFile, const std::string ioInConfigFile)
-: File(name, mode, MPI_COMM_NULL, configFile, ioInConfigFile)
+: m_Name(name), m_Mode(mode),
+  m_Stream(std::make_shared<core::Stream>(name, ToMode(mode), configFile,
+                                          ioInConfigFile, "Python"))
 {
 }
 

bindings/Python/py11File.h

@@ -16,6 +16,10 @@
 #include "adios2/common/ADIOSTypes.h"
 #include "adios2/core/Stream.h"
 
+#ifdef ADIOS2_HAVE_MPI
+#include "adios2/common/ADIOSMPI.h"
+#endif
+
 namespace adios2
 {
 namespace py11
@@ -27,11 +31,13 @@ class File
     const std::string m_Name;
     const std::string m_Mode;
 
+#ifdef ADIOS2_HAVE_MPI
     File(const std::string &name, const std::string mode, MPI_Comm comm,
          const std::string engineType = "BPFile");
 
     File(const std::string &name, const std::string mode, MPI_Comm comm,
          const std::string &configFile, const std::string ioInConfigFile);
+#endif
 
     File(const std::string &name, const std::string mode,
          const std::string engineType = "BPFile");

bindings/Python/py11IO.cpp

@@ -14,6 +14,7 @@
 #include "adios2/helper/adiosFunctions.h" //GetType<T>
 
 #ifdef ADIOS2_HAVE_MPI
+#include "adios2/helper/adiosCommMPI.h"
 #include <mpi4py/mpi4py.h>
 #endif
 
@@ -252,7 +253,8 @@ Engine IO::Open(const std::string &name, const int mode, MPI4PY_Comm comm)
     helper::CheckForNullptr(m_IO,
                             "for engine " + name + ", in call to IO::Open");
 
-    return Engine(&m_IO->Open(name, static_cast<adios2::Mode>(mode), comm));
+    return Engine(&m_IO->Open(name, static_cast<adios2::Mode>(mode),
+                              helper::CommFromMPI(comm)));
 }
 #endif
 

source/adios2/CMakeLists.txt

@@ -33,6 +33,8 @@ add_library(adios2
 
 #helper
   helper/adiosComm.h  helper/adiosComm.cpp
+  helper/adiosCommDummy.h  helper/adiosCommDummy.cpp
+  helper/adiosCommMPI.h  helper/adiosCommMPI.cpp
   helper/adiosDynamicBinder.h  helper/adiosDynamicBinder.cpp
   helper/adiosMath.cpp
   helper/adiosMemory.cpp

source/adios2/core/ADIOS.cpp

@@ -13,8 +13,8 @@
 #include <algorithm> // std::transform
 #include <ios>       //std::ios_base::failure
 
-#include "adios2/common/ADIOSMPI.h"
 #include "adios2/core/IO.h"
+#include "adios2/helper/adiosCommDummy.h"
 #include "adios2/helper/adiosFunctions.h" //InquireKey, BroadcastFile
 
 // OPERATORS
@@ -57,10 +57,10 @@ namespace adios2
 namespace core
 {
 
-ADIOS::ADIOS(const std::string configFile, MPI_Comm mpiComm,
+ADIOS::ADIOS(const std::string configFile, helper::Comm comm,
              const bool debugMode, const std::string hostLanguage)
 : m_ConfigFile(configFile), m_DebugMode(debugMode),
-  m_HostLanguage(hostLanguage), m_Comm(helper::Comm::Duplicate(mpiComm))
+  m_HostLanguage(hostLanguage), m_Comm(std::move(comm))
 {
     if (!configFile.empty())
     {
@@ -74,18 +74,18 @@ ADIOS::ADIOS(const std::string configFile, MPI_Comm mpiComm,
 
 ADIOS::ADIOS(const std::string configFile, const bool debugMode,
              const std::string hostLanguage)
-: ADIOS(configFile, MPI_COMM_NULL, debugMode, hostLanguage)
+: ADIOS(configFile, helper::CommDummy(), debugMode, hostLanguage)
 {
 }
 
-ADIOS::ADIOS(MPI_Comm mpiComm, const bool debugMode,
+ADIOS::ADIOS(helper::Comm comm, const bool debugMode,
              const std::string hostLanguage)
-: ADIOS("", mpiComm, debugMode, hostLanguage)
+: ADIOS("", std::move(comm), debugMode, hostLanguage)
 {
 }
 
 ADIOS::ADIOS(const bool debugMode, const std::string hostLanguage)
-: ADIOS("", MPI_COMM_NULL, debugMode, hostLanguage)
+: ADIOS("", helper::CommDummy(), debugMode, hostLanguage)
 {
 }
 

source/adios2/core/ADIOS.h

@@ -19,7 +19,6 @@
 /// \endcond
 
 #include "adios2/common/ADIOSConfig.h"
-#include "adios2/common/ADIOSMPI.h"
 #include "adios2/common/ADIOSTypes.h"
 #include "adios2/core/Operator.h"
 #include "adios2/helper/adiosComm.h"
@@ -55,7 +54,7 @@ class ADIOS
      * false: optional feature to turn off checks on user input data,
      * recommended in stable flows
      */
-    ADIOS(const std::string configFile, MPI_Comm mpiComm, const bool debugMode,
+    ADIOS(const std::string configFile, helper::Comm comm, const bool debugMode,
           const std::string hostLanguage);
 
     /**
@@ -77,7 +76,7 @@ class ADIOS
      * false: optional feature to turn off checks on user input data,
      * recommended in stable flows
      */
-    ADIOS(MPI_Comm mpiComm, const bool debugMode,
+    ADIOS(helper::Comm comm, const bool debugMode,
           const std::string hostLanguage);
 
     /**

source/adios2/core/IO.cpp

@@ -14,7 +14,6 @@
 #include <sstream>
 #include <utility> // std::pair
 
-#include "adios2/common/ADIOSMPI.h"
 #include "adios2/common/ADIOSMacros.h"
 
 #include "adios2/engine/bp3/BP3Reader.h"
@@ -472,7 +471,7 @@ size_t IO::AddOperation(Operator &op, const Params &parameters) noexcept
     return m_Operations.size() - 1;
 }
 
-Engine &IO::Open(const std::string &name, const Mode mode, MPI_Comm mpiComm)
+Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm)
 {
     TAU_SCOPED_TIMER("IO::Open");
     auto itEngineFound = m_Engines.find(name);
@@ -504,7 +503,6 @@ Engine &IO::Open(const std::string &name, const Mode mode, MPI_Comm mpiComm)
         }
     }
 
-    auto comm = helper::Comm::Duplicate(mpiComm);
     std::shared_ptr<Engine> engine;
     const bool isDefaultEngine = m_EngineType.empty() ? true : false;
     std::string engineTypeLC = m_EngineType;
@@ -765,7 +763,7 @@ Engine &IO::Open(const std::string &name, const Mode mode, MPI_Comm mpiComm)
 
 Engine &IO::Open(const std::string &name, const Mode mode)
 {
-    return Open(name, mode, m_ADIOS.GetComm().AsMPI());
+    return Open(name, mode, m_ADIOS.GetComm().Duplicate());
 }
 
 Engine &IO::GetEngine(const std::string &name)

source/adios2/core/IO.h

@@ -360,7 +360,7 @@ class IO
      * @exception std::invalid_argument if Engine with unique name is already
      * created with another Open, in debug mode only
      */
-    Engine &Open(const std::string &name, const Mode mode, MPI_Comm mpiComm);
+    Engine &Open(const std::string &name, const Mode mode, helper::Comm comm);
 
     /**
      * Overloaded version that reuses the MPI_Comm object passed