From 7e9cbb9c7e118a19ec5c8cebcc8d7983fc7d469e Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 4 Oct 2019 13:08:46 -0400 Subject: [PATCH 01/34] mpidummy: Add MPI_OP_NULL constant Use value `0` for consistency with `MPI_DATATYPE_NULL` and increment the index of other "op" constants. --- source/adios2/helper/mpidummy.h | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/source/adios2/helper/mpidummy.h b/source/adios2/helper/mpidummy.h index 1454f087c6..360b041912 100644 --- a/source/adios2/helper/mpidummy.h +++ b/source/adios2/helper/mpidummy.h @@ -82,20 +82,21 @@ typedef int MPI_Op; #define MPI_ANY_SOURCE 0 #define MPI_ANY_TAG 0 -#define MPI_MAX 0 -#define MPI_MIN 1 -#define MPI_SUM 2 -#define MPI_PROD 3 -#define MPI_LAND 4 -#define MPI_BAND 5 -#define MPI_LOR 6 -#define MPI_BOR 7 -#define MPI_LXOR 8 -#define MPI_BXOR 9 -#define MPI_MAXLOC 10 -#define MPI_MINLOC 11 -#define MPI_REPLACE 12 -#define MPI_NO_OP 13 +#define MPI_OP_NULL 0 +#define MPI_MAX 1 +#define MPI_MIN 2 +#define MPI_SUM 3 +#define MPI_PROD 4 +#define MPI_LAND 5 +#define MPI_BAND 6 +#define MPI_LOR 7 +#define MPI_BOR 8 +#define MPI_LXOR 9 +#define MPI_BXOR 10 +#define MPI_MAXLOC 11 +#define MPI_MINLOC 12 +#define MPI_REPLACE 13 +#define MPI_NO_OP 14 #define MPI_MAX_PROCESSOR_NAME 32 From c0bfedb65ceb9327b1e02207fb5fb273f2d633d3 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 19 Sep 2019 14:00:14 -0400 Subject: [PATCH 02/34] Add missing includes of ADIOSMPI.h in places that use MPI directly --- source/adios2/core/Stream.h | 1 + source/adios2/engine/insitumpi/InSituMPIReader.h | 1 + source/adios2/toolkit/format/dataman/DataManSerializer.cpp | 2 ++ 3 files changed, 4 insertions(+) diff --git a/source/adios2/core/Stream.h b/source/adios2/core/Stream.h index c5ecdde69b..585061e7e7 100644 --- a/source/adios2/core/Stream.h +++ b/source/adios2/core/Stream.h @@ -13,6 +13,7 @@ #include //std::shared_ptr +#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSMacros.h" #include "adios2/common/ADIOSTypes.h" #include "adios2/core/ADIOS.h" diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.h b/source/adios2/engine/insitumpi/InSituMPIReader.h index 3790c1b247..525e15f9de 100644 --- a/source/adios2/engine/insitumpi/InSituMPIReader.h +++ b/source/adios2/engine/insitumpi/InSituMPIReader.h @@ -15,6 +15,7 @@ #define ADIOS2_ENGINE_INSITUMPI_INSITUMPIREADER_H_ #include "adios2/common/ADIOSConfig.h" +#include "adios2/common/ADIOSMPI.h" #include "adios2/core/ADIOS.h" #include "adios2/core/Engine.h" #include "adios2/helper/adiosComm.h" diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp index e1f3ffa715..f09245a17f 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp @@ -11,6 +11,8 @@ #include "DataManSerializer.h" #include "DataManSerializer.tcc" +#include "adios2/common/ADIOSMPI.h" + #include #include From 3234c58ba43f7ee50795449b49e09daa9eefa858 Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 13 Sep 2019 15:00:19 -0400 Subject: [PATCH 03/34] Remove unused ADIOSMPI.h includes --- source/adios2/engine/bp3/BP3Writer.cpp | 1 - source/adios2/engine/bp4/BP4Writer.cpp | 1 - source/adios2/engine/sst/SstParamParser.cpp | 1 - source/adios2/engine/sst/SstReader.h | 2 -- source/adios2/engine/sst/SstReader.tcc | 1 - source/adios2/engine/sst/SstWriter.cpp | 1 - source/adios2/engine/sst/SstWriter.h | 2 -- source/adios2/engine/sst/SstWriter.tcc | 1 - source/adios2/helper/adiosSystem.cpp | 1 - source/adios2/helper/adiosXML.cpp | 1 - source/adios2/toolkit/aggregator/mpi/MPIAggregator.h | 1 - source/adios2/toolkit/aggregator/mpi/MPIChain.cpp | 2 -- source/adios2/toolkit/query/Worker.h | 1 - source/adios2/toolkit/transport/Transport.cpp | 1 - source/adios2/toolkit/transport/Transport.h | 1 - source/utils/bpls/bpls.h | 1 - 16 files changed, 19 deletions(-) diff --git a/source/adios2/engine/bp3/BP3Writer.cpp b/source/adios2/engine/bp3/BP3Writer.cpp index 773ab85049..fc25db1002 100644 --- a/source/adios2/engine/bp3/BP3Writer.cpp +++ b/source/adios2/engine/bp3/BP3Writer.cpp @@ -11,7 +11,6 @@ #include "BP3Writer.h" #include "BP3Writer.tcc" -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSMacros.h" #include "adios2/core/IO.h" #include "adios2/helper/adiosFunctions.h" //CheckIndexRange diff --git a/source/adios2/engine/bp4/BP4Writer.cpp b/source/adios2/engine/bp4/BP4Writer.cpp index 3b2e0bb646..c24d347d1d 100644 --- a/source/adios2/engine/bp4/BP4Writer.cpp +++ b/source/adios2/engine/bp4/BP4Writer.cpp @@ -11,7 +11,6 @@ #include "BP4Writer.h" #include "BP4Writer.tcc" -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSMacros.h" #include "adios2/core/IO.h" #include "adios2/helper/adiosFunctions.h" //CheckIndexRange diff --git a/source/adios2/engine/sst/SstParamParser.cpp b/source/adios2/engine/sst/SstParamParser.cpp index 0517fdc218..ccee2ad386 100644 --- a/source/adios2/engine/sst/SstParamParser.cpp +++ b/source/adios2/engine/sst/SstParamParser.cpp @@ -1,5 +1,4 @@ -#include #include #include "SstParamParser.h" diff --git a/source/adios2/engine/sst/SstReader.h b/source/adios2/engine/sst/SstReader.h index 7cf9170dd3..bc631de4b0 100644 --- a/source/adios2/engine/sst/SstReader.h +++ b/source/adios2/engine/sst/SstReader.h @@ -11,8 +11,6 @@ #ifndef ADIOS2_ENGINE_SST_SSTREADER_H_ #define ADIOS2_ENGINE_SST_SSTREADER_H_ -#include - #include "adios2/toolkit/sst/sst.h" #include "adios2/core/Engine.h" diff --git a/source/adios2/engine/sst/SstReader.tcc b/source/adios2/engine/sst/SstReader.tcc index f2f97f0902..f5c3ab34b6 100644 --- a/source/adios2/engine/sst/SstReader.tcc +++ b/source/adios2/engine/sst/SstReader.tcc @@ -13,7 +13,6 @@ #include "SstReader.h" -#include "adios2/common/ADIOSMPI.h" #include "adios2/helper/adiosFunctions.h" //GetType #include "adios2/toolkit/profiling/taustubs/tautimer.hpp" diff --git a/source/adios2/engine/sst/SstWriter.cpp b/source/adios2/engine/sst/SstWriter.cpp index 8a310459ef..0a4e5ec2a3 100644 --- a/source/adios2/engine/sst/SstWriter.cpp +++ b/source/adios2/engine/sst/SstWriter.cpp @@ -8,7 +8,6 @@ * Author: Greg Eisenhauer */ -#include #include #include "SstParamParser.h" diff --git a/source/adios2/engine/sst/SstWriter.h b/source/adios2/engine/sst/SstWriter.h index 511f114f5c..3c99ca71e9 100644 --- a/source/adios2/engine/sst/SstWriter.h +++ b/source/adios2/engine/sst/SstWriter.h @@ -11,8 +11,6 @@ #ifndef ADIOS2_ENGINE_SST_SST_WRITER_H_ #define ADIOS2_ENGINE_SST_SST_WRITER_H_ -#include - #include "adios2/common/ADIOSConfig.h" #include "adios2/core/Engine.h" #include "adios2/helper/adiosComm.h" diff --git a/source/adios2/engine/sst/SstWriter.tcc b/source/adios2/engine/sst/SstWriter.tcc index 34a9cccc9f..3f0cdfcbe0 100644 --- a/source/adios2/engine/sst/SstWriter.tcc +++ b/source/adios2/engine/sst/SstWriter.tcc @@ -13,7 +13,6 @@ #include "SstWriter.h" -#include "adios2/common/ADIOSMPI.h" #include "adios2/helper/adiosFunctions.h" //GetType #include "adios2/toolkit/profiling/taustubs/tautimer.hpp" diff --git a/source/adios2/helper/adiosSystem.cpp b/source/adios2/helper/adiosSystem.cpp index 0a96c13e96..06ef1e423d 100644 --- a/source/adios2/helper/adiosSystem.cpp +++ b/source/adios2/helper/adiosSystem.cpp @@ -16,7 +16,6 @@ #include -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSTypes.h" #include "adios2/helper/adiosComm.h" #include "adios2/helper/adiosString.h" diff --git a/source/adios2/helper/adiosXML.cpp b/source/adios2/helper/adiosXML.cpp index d41bd32916..2f05a5a7d9 100644 --- a/source/adios2/helper/adiosXML.cpp +++ b/source/adios2/helper/adiosXML.cpp @@ -17,7 +17,6 @@ #include //std::invalid_argument /// \endcond -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSTypes.h" #include "adios2/core/IO.h" #include "adios2/helper/adiosString.h" diff --git a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h index 7ab8ae876f..d15d9751b5 100644 --- a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h +++ b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h @@ -13,7 +13,6 @@ #include //std::unique_ptr -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSTypes.h" #include "adios2/helper/adiosComm.h" #include "adios2/toolkit/format/buffer/Buffer.h" diff --git a/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp b/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp index 9cf24b17b8..4db96961bb 100644 --- a/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp +++ b/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp @@ -9,8 +9,6 @@ */ #include "MPIChain.h" -#include "adios2/common/ADIOSMPI.h" - #include "adios2/toolkit/format/buffer/heap/BufferSTL.h" namespace adios2 diff --git a/source/adios2/toolkit/query/Worker.h b/source/adios2/toolkit/query/Worker.h index 8e9fd6f149..bb4cb0c923 100644 --- a/source/adios2/toolkit/query/Worker.h +++ b/source/adios2/toolkit/query/Worker.h @@ -8,7 +8,6 @@ #include #include -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSTypes.h" #include diff --git a/source/adios2/toolkit/transport/Transport.cpp b/source/adios2/toolkit/transport/Transport.cpp index 9f0e764113..829e2a4f4a 100644 --- a/source/adios2/toolkit/transport/Transport.cpp +++ b/source/adios2/toolkit/transport/Transport.cpp @@ -10,7 +10,6 @@ #include "Transport.h" -#include "adios2/common/ADIOSMPI.h" #include "adios2/helper/adiosFunctions.h" //CreateDirectory namespace adios2 diff --git a/source/adios2/toolkit/transport/Transport.h b/source/adios2/toolkit/transport/Transport.h index b715f54a7f..e25149f73f 100644 --- a/source/adios2/toolkit/transport/Transport.h +++ b/source/adios2/toolkit/transport/Transport.h @@ -17,7 +17,6 @@ /// \endcond #include "adios2/common/ADIOSConfig.h" -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSTypes.h" #include "adios2/helper/adiosComm.h" #include "adios2/toolkit/profiling/iochrono/IOChrono.h" diff --git a/source/utils/bpls/bpls.h b/source/utils/bpls/bpls.h index 0265acde59..9bd1d4265f 100644 --- a/source/utils/bpls/bpls.h +++ b/source/utils/bpls/bpls.h @@ -6,7 +6,6 @@ */ #include "adios2/common/ADIOSConfig.h" -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSMacros.h" #include "adios2/core/ADIOS.h" #include "adios2/core/Engine.h" From 9160f88e93c1bc4760c60f36138390eda4d5f61f Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 4 Sep 2019 15:42:55 -0400 Subject: [PATCH 04/34] helper: Encapsulate MPI_Gatherv as Comm::Gatherv --- source/adios2/helper/adiosComm.cpp | 27 +++++++++++++ source/adios2/helper/adiosComm.h | 11 ++++++ source/adios2/helper/adiosComm.inl | 9 +++++ source/adios2/helper/adiosComm.tcc | 62 +++++++----------------------- 4 files changed, 61 insertions(+), 48 deletions(-) diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp index 2033bd3796..bfe0849fc2 100644 --- a/source/adios2/helper/adiosComm.cpp +++ b/source/adios2/helper/adiosComm.cpp @@ -8,7 +8,9 @@ #include "adiosComm.h" #include "adiosComm.tcc" +#include #include //std::ios_base::failure +#include #include #include "adios2/common/ADIOSMPI.h" @@ -174,6 +176,31 @@ void Comm::GatherImpl(const void *sendbuf, size_t sendcount, hint); } +void Comm::GathervImpl(const void *sendbuf, size_t sendcount, + MPI_Datatype sendtype, void *recvbuf, + const size_t *recvcounts, const size_t *displs, + MPI_Datatype recvtype, int root, + const std::string &hint) const +{ + std::vector countsInt; + std::vector displsInt; + if (root == this->Rank()) + { + auto cast = [](size_t sz) -> int { return int(sz); }; + const int size = this->Size(); + countsInt.reserve(size); + std::transform(recvcounts, recvcounts + size, + std::back_inserter(countsInt), cast); + displsInt.reserve(size); + std::transform(displs, displs + size, std::back_inserter(displsInt), + cast); + } + CheckMPIReturn(SMPI_Gatherv(sendbuf, static_cast(sendcount), sendtype, + recvbuf, countsInt.data(), displsInt.data(), + recvtype, root, m_MPIComm), + hint); +} + void Comm::ReduceImpl(const void *sendbuf, void *recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op, int root, const std::string &hint) const diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h index 45094a48c1..724a4e1f73 100644 --- a/source/adios2/helper/adiosComm.h +++ b/source/adios2/helper/adiosComm.h @@ -193,6 +193,11 @@ class Comm size_t recvcount, int root, const std::string &hint = std::string()) const; + template + void Gatherv(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, + const size_t *recvcounts, const size_t *displs, int root, + const std::string &hint = std::string()) const; + template void Reduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op, int root, const std::string &hint = std::string()) const; @@ -252,6 +257,12 @@ class Comm MPI_Datatype recvtype, int root, const std::string &hint) const; + void GathervImpl(const void *sendbuf, size_t sendcount, + MPI_Datatype sendtype, void *recvbuf, + const size_t *recvcounts, const size_t *displs, + MPI_Datatype recvtype, int root, + const std::string &hint) const; + void ReduceImpl(const void *sendbuf, void *recvbuf, size_t count, MPI_Datatype datatype, MPI_Op op, int root, const std::string &hint) const; diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index f18b58cfba..58604d19d5 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -171,6 +171,15 @@ void Comm::Gather(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, Datatype(), root, hint); } +template +void Comm::Gatherv(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, + const size_t *recvcounts, const size_t *displs, int root, + const std::string &hint) const +{ + return GathervImpl(sendbuf, sendcount, Datatype(), recvbuf, + recvcounts, displs, Datatype(), root, hint); +} + template void Comm::Reduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op, int root, const std::string &hint) const diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc index 084386369a..dd995963b9 100644 --- a/source/adios2/helper/adiosComm.tcc +++ b/source/adios2/helper/adiosComm.tcc @@ -12,8 +12,6 @@ #include "adiosComm.h" #include "adios2/common/ADIOSMPI.h" -#include "adios2/common/ADIOSTypes.h" -#include "adios2/helper/adiosType.h" #include //std::runtime_error @@ -25,16 +23,15 @@ namespace helper namespace { -std::vector GetGathervDisplacements(const size_t *counts, - const size_t countsSize) +std::vector GetGathervDisplacements(const size_t *counts, + const size_t countsSize) { - std::vector displacements(countsSize); + std::vector displacements(countsSize); displacements[0] = 0; for (size_t i = 1; i < countsSize; ++i) { - displacements[i] = - displacements[i - 1] + static_cast(counts[i - 1]); + displacements[i] = displacements[i - 1] + counts[i - 1]; } return displacements; } @@ -80,29 +77,13 @@ void Comm::GathervArrays(const char *source, size_t sourceCount, const size_t *counts, size_t countsSize, char *destination, int rankDestination) const { - int result = 0; - int rank; - SMPI_Comm_rank(m_MPIComm, &rank); - - std::vector countsInt, displacementsInt; - - if (rank == rankDestination) - { - countsInt = NewVectorTypeFromArray(counts, countsSize); - displacementsInt = GetGathervDisplacements(counts, countsSize); - } - - int sourceCountInt = static_cast(sourceCount); - result = - SMPI_Gatherv(const_cast(source), sourceCountInt, MPI_CHAR, - destination, countsInt.data(), displacementsInt.data(), - MPI_CHAR, rankDestination, m_MPIComm); - - if (result != MPI_SUCCESS) + std::vector displs; + if (rankDestination == this->Rank()) { - throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI " - "Gatherv type MPI_CHAR function\n"); + displs = GetGathervDisplacements(counts, countsSize); } + this->Gatherv(source, sourceCount, destination, counts, displs.data(), + rankDestination); } template <> @@ -110,28 +91,13 @@ void Comm::GathervArrays(const size_t *source, size_t sourceCount, const size_t *counts, size_t countsSize, size_t *destination, int rankDestination) const { - int result = 0; - int rank; - SMPI_Comm_rank(m_MPIComm, &rank); - - std::vector countsInt = - NewVectorTypeFromArray(counts, countsSize); - - std::vector displacementsInt = - GetGathervDisplacements(counts, countsSize); - - int sourceCountInt = static_cast(sourceCount); - - result = SMPI_Gatherv(const_cast(source), sourceCountInt, - ADIOS2_MPI_SIZE_T, destination, countsInt.data(), - displacementsInt.data(), ADIOS2_MPI_SIZE_T, - rankDestination, m_MPIComm); - - if (result != MPI_SUCCESS) + std::vector displs; + if (rankDestination == this->Rank()) { - throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI " - "Gather type size_t function\n"); + displs = GetGathervDisplacements(counts, countsSize); } + this->Gatherv(source, sourceCount, destination, counts, displs.data(), + rankDestination); } // AllGatherArrays full specializations forward-declared in 'adiosComm.inl'. From 4bd86ba3c23e97427311233a4d3ec3100b5270e3 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 4 Sep 2019 15:42:22 -0400 Subject: [PATCH 05/34] helper: Implement Comm::GatherArrays in terms of our encapsulation --- source/adios2/helper/adiosComm.tcc | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc index dd995963b9..024a6da332 100644 --- a/source/adios2/helper/adiosComm.tcc +++ b/source/adios2/helper/adiosComm.tcc @@ -43,32 +43,16 @@ template <> void Comm::GatherArrays(const char *source, size_t sourceCount, char *destination, int rankDestination) const { - int countsInt = static_cast(sourceCount); - int result = SMPI_Gather(const_cast(source), countsInt, MPI_CHAR, - destination, countsInt, MPI_CHAR, rankDestination, - m_MPIComm); - - if (result != MPI_SUCCESS) - { - throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI " - "Gather type MPI_CHAR function\n"); - } + this->Gather(source, sourceCount, destination, sourceCount, + rankDestination); } template <> void Comm::GatherArrays(const size_t *source, size_t sourceCount, size_t *destination, int rankDestination) const { - int countsInt = static_cast(sourceCount); - int result = SMPI_Gather(const_cast(source), countsInt, - ADIOS2_MPI_SIZE_T, destination, countsInt, - ADIOS2_MPI_SIZE_T, rankDestination, m_MPIComm); - - if (result != MPI_SUCCESS) - { - throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI " - "Gather type size_t function\n"); - } + this->Gather(source, sourceCount, destination, sourceCount, + rankDestination); } // GathervArrays full specializations forward-declared in 'adiosComm.inl'. From fff4435df892eb1ab5719f2ae1474014c3a4f683 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 4 Sep 2019 16:45:33 -0400 Subject: [PATCH 06/34] helper: Implement Comm::GatherValues in terms of our encapsulation --- source/adios2/helper/adiosComm.inl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index 58604d19d5..718892503d 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -31,9 +31,8 @@ void Comm::GatherArrays(const size_t *source, size_t sourceCount, template std::vector Comm::GatherValues(T source, int rankDestination) const { - int rank, size; - SMPI_Comm_rank(m_MPIComm, &rank); - SMPI_Comm_size(m_MPIComm, &size); + int rank = this->Rank(); + int size = this->Size(); std::vector output; From 8b14d294c8c45565ce4b5944e45c9134854d1c2a Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 4 Sep 2019 16:46:06 -0400 Subject: [PATCH 07/34] helper: Implement Comm::GathervVectors in terms of our encapsulation --- source/adios2/helper/adiosComm.inl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index 718892503d..56c4a81af0 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -68,8 +68,7 @@ void Comm::GathervVectors(const std::vector &in, std::vector &out, size_t gatheredSize = 0; - int rank; - SMPI_Comm_rank(m_MPIComm, &rank); + int rank = this->Rank(); if (rank == rankDestination) // pre-allocate vector { From 14edaa47c6d5ac0e5cff08197021d20b4d961a0f Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 4 Sep 2019 16:26:07 -0400 Subject: [PATCH 08/34] helper: Implement Comm::AllGatherArrays in terms of our encapsulation --- source/adios2/helper/adiosComm.tcc | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc index 024a6da332..2792f7737c 100644 --- a/source/adios2/helper/adiosComm.tcc +++ b/source/adios2/helper/adiosComm.tcc @@ -13,8 +13,6 @@ #include "adios2/common/ADIOSMPI.h" -#include //std::runtime_error - namespace adios2 { namespace helper @@ -89,16 +87,7 @@ template <> void Comm::AllGatherArrays(const size_t *source, const size_t sourceCount, size_t *destination) const { - int countsInt = static_cast(sourceCount); - int result = MPI_Allgather(const_cast(source), countsInt, - ADIOS2_MPI_SIZE_T, destination, countsInt, - ADIOS2_MPI_SIZE_T, m_MPIComm); - - if (result != MPI_SUCCESS) - { - throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI " - "Allgather type size_t function\n"); - } + this->Allgather(source, sourceCount, destination, sourceCount); } // ReduceValues full specializations forward-declared in 'adiosComm.inl'. From 81e1cbaf5f4f7d82050a87eb0efc41c43c04a1b5 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 4 Sep 2019 16:46:25 -0400 Subject: [PATCH 09/34] helper: Implement Comm::AllGatherValues in terms of our encapsulation --- source/adios2/helper/adiosComm.inl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index 56c4a81af0..623d8d8a50 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -97,8 +97,7 @@ void Comm::GathervVectors(const std::vector &in, std::vector &out, template std::vector Comm::AllGatherValues(const T source) const { - int size; - SMPI_Comm_size(m_MPIComm, &size); + int size = this->Size(); std::vector output(size); T sourceCopy = source; // so we can have an address for rvalues From c7f0d71e4c9a4abd99d6100b1dacb30958100f13 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 4 Sep 2019 16:27:59 -0400 Subject: [PATCH 10/34] helper: Implement Comm::ReduceValues in terms of our encapsulation --- source/adios2/helper/adiosComm.tcc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc index 2792f7737c..d3f450f662 100644 --- a/source/adios2/helper/adiosComm.tcc +++ b/source/adios2/helper/adiosComm.tcc @@ -97,8 +97,7 @@ unsigned int Comm::ReduceValues(const unsigned int source, MPI_Op operation, { unsigned int sourceLocal = source; unsigned int reduceValue = 0; - SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED, operation, - rankDestination, m_MPIComm); + this->Reduce(&sourceLocal, &reduceValue, 1, operation, rankDestination); return reduceValue; } @@ -109,8 +108,7 @@ unsigned long int Comm::ReduceValues(const unsigned long int source, { unsigned long int sourceLocal = source; unsigned long int reduceValue = 0; - SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG, operation, - rankDestination, m_MPIComm); + this->Reduce(&sourceLocal, &reduceValue, 1, operation, rankDestination); return reduceValue; } @@ -121,8 +119,7 @@ unsigned long long int Comm::ReduceValues(const unsigned long long int source, { unsigned long long int sourceLocal = source; unsigned long long int reduceValue = 0; - SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG_LONG, - operation, rankDestination, m_MPIComm); + this->Reduce(&sourceLocal, &reduceValue, 1, operation, rankDestination); return reduceValue; } From 8ab36c262b244016674a476689cb571e190624e6 Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 4 Sep 2019 16:32:05 -0400 Subject: [PATCH 11/34] helper: Implement Comm::BroadcastValue in terms of our encapsulation --- source/adios2/helper/adiosComm.tcc | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc index d3f450f662..6c4728430a 100644 --- a/source/adios2/helper/adiosComm.tcc +++ b/source/adios2/helper/adiosComm.tcc @@ -127,16 +127,13 @@ unsigned long long int Comm::ReduceValues(const unsigned long long int source, template <> size_t Comm::BroadcastValue(const size_t &input, const int rankSource) const { - int rank; - SMPI_Comm_rank(m_MPIComm, &rank); size_t output = 0; - - if (rank == rankSource) + if (rankSource == this->Rank()) { output = input; } - SMPI_Bcast(&output, 1, ADIOS2_MPI_SIZE_T, rankSource, m_MPIComm); + this->Bcast(&output, 1, rankSource); return output; } @@ -145,13 +142,11 @@ template <> std::string Comm::BroadcastValue(const std::string &input, const int rankSource) const { - int rank; - SMPI_Comm_rank(m_MPIComm, &rank); const size_t inputSize = input.size(); const size_t length = this->BroadcastValue(inputSize, rankSource); std::string output; - if (rank == rankSource) + if (rankSource == this->Rank()) { output = input; } @@ -160,8 +155,7 @@ std::string Comm::BroadcastValue(const std::string &input, output.resize(length); } - SMPI_Bcast(const_cast(output.data()), static_cast(length), - MPI_CHAR, rankSource, m_MPIComm); + this->Bcast(const_cast(output.data()), length, rankSource); return output; } From c88c1a86e3d6d8aecc8bcd9cc9599a63f1331d9b Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 4 Sep 2019 16:41:17 -0400 Subject: [PATCH 12/34] helper: Implement Comm::BroadcastVector in terms of our encapsulation --- source/adios2/helper/adiosComm.cpp | 19 +++++++++++--- source/adios2/helper/adiosComm.h | 3 ++- source/adios2/helper/adiosComm.inl | 2 +- source/adios2/helper/adiosComm.tcc | 42 +++++------------------------- 4 files changed, 24 insertions(+), 42 deletions(-) diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp index bfe0849fc2..82f60c7742 100644 --- a/source/adios2/helper/adiosComm.cpp +++ b/source/adios2/helper/adiosComm.cpp @@ -158,11 +158,22 @@ void Comm::AllreduceImpl(const void *sendbuf, void *recvbuf, size_t count, } void Comm::BcastImpl(void *buffer, size_t count, MPI_Datatype datatype, - int root, const std::string &hint) const + size_t datatypeSize, int root, + const std::string &hint) const { - CheckMPIReturn( - SMPI_Bcast(buffer, static_cast(count), datatype, root, m_MPIComm), - hint); + size_t inputSize = count; + const int MAXBCASTSIZE = 1073741824; + size_t blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize); + unsigned char *blockBuf = static_cast(buffer); + while (inputSize > 0) + { + CheckMPIReturn(SMPI_Bcast(blockBuf, static_cast(blockSize), + datatype, root, m_MPIComm), + hint); + blockBuf += blockSize * datatypeSize; + inputSize -= blockSize; + blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize); + } } void Comm::GatherImpl(const void *sendbuf, size_t sendcount, diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h index 724a4e1f73..bff73bbd27 100644 --- a/source/adios2/helper/adiosComm.h +++ b/source/adios2/helper/adiosComm.h @@ -249,7 +249,8 @@ class Comm MPI_Datatype datatype, MPI_Op op, const std::string &hint) const; - void BcastImpl(void *buffer, size_t count, MPI_Datatype datatype, int root, + void BcastImpl(void *buffer, size_t count, MPI_Datatype datatype, + size_t datatypeSize, int root, const std::string &hint) const; void GatherImpl(const void *sendbuf, size_t sendcount, diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index 623d8d8a50..b3f5a940bf 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -157,7 +157,7 @@ template void Comm::Bcast(T *buffer, const size_t count, int root, const std::string &hint) const { - return BcastImpl(buffer, count, Datatype(), root, hint); + return BcastImpl(buffer, count, Datatype(), sizeof(T), root, hint); } template diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc index 6c4728430a..cc272ad760 100644 --- a/source/adios2/helper/adiosComm.tcc +++ b/source/adios2/helper/adiosComm.tcc @@ -165,70 +165,40 @@ template <> void Comm::BroadcastVector(std::vector &vector, const int rankSource) const { - int size; - SMPI_Comm_size(m_MPIComm, &size); - - if (size == 1) + if (this->Size() == 1) { return; } // First Broadcast the size, then the contents size_t inputSize = this->BroadcastValue(vector.size(), rankSource); - int rank; - SMPI_Comm_rank(m_MPIComm, &rank); - if (rank != rankSource) + if (rankSource != this->Rank()) { vector.resize(inputSize); } - const int MAXBCASTSIZE = 1073741824; - size_t blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize); - char *buffer = vector.data(); - while (inputSize > 0) - { - SMPI_Bcast(buffer, static_cast(blockSize), MPI_CHAR, rankSource, - m_MPIComm); - buffer += blockSize; - inputSize -= blockSize; - blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize); - } + this->Bcast(vector.data(), inputSize, rankSource); } template <> void Comm::BroadcastVector(std::vector &vector, const int rankSource) const { - int size; - SMPI_Comm_size(m_MPIComm, &size); - - if (size == 1) + if (this->Size() == 1) { return; } // First Broadcast the size, then the contents size_t inputSize = this->BroadcastValue(vector.size(), rankSource); - int rank; - SMPI_Comm_rank(m_MPIComm, &rank); - if (rank != rankSource) + if (rankSource != this->Rank()) { vector.resize(inputSize); } - const int MAXBCASTSIZE = 1073741824 / sizeof(size_t); - size_t blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize); - size_t *buffer = vector.data(); - while (inputSize > 0) - { - SMPI_Bcast(buffer, static_cast(blockSize), ADIOS2_MPI_SIZE_T, - rankSource, m_MPIComm); - buffer += blockSize; - inputSize -= blockSize; - blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize); - } + this->Bcast(vector.data(), inputSize, rankSource); } // Datatype full specializations forward-declared in 'adiosComm.inl'. From 4afe40dc9d5ecc19cb6024fd85caf54c7dac890e Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 12 Sep 2019 13:50:00 -0400 Subject: [PATCH 13/34] helper: Implement Comm::BroadcastFile in terms of our encapsulation --- source/adios2/helper/adiosComm.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp index 82f60c7742..5496d07de2 100644 --- a/source/adios2/helper/adiosComm.cpp +++ b/source/adios2/helper/adiosComm.cpp @@ -123,8 +123,7 @@ std::string Comm::BroadcastFile(const std::string &fileName, const std::string hint, const int rankSource) const { - int rank; - MPI_Comm_rank(m_MPIComm, &rank); + int rank = this->Rank(); std::string fileContents; // Read the file on rank 0 and broadcast it to everybody else From 1c62294cd222f43849e8a514cd356395b174babb Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 12 Sep 2019 13:54:06 -0400 Subject: [PATCH 14/34] helper: Move Comm::BroadcastFile to after MPI-specific methods --- source/adios2/helper/adiosComm.cpp | 36 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp index 5496d07de2..fa22f38d67 100644 --- a/source/adios2/helper/adiosComm.cpp +++ b/source/adios2/helper/adiosComm.cpp @@ -119,24 +119,6 @@ void Comm::Barrier(const std::string &hint) const CheckMPIReturn(SMPI_Barrier(m_MPIComm), hint); } -std::string Comm::BroadcastFile(const std::string &fileName, - const std::string hint, - const int rankSource) const -{ - int rank = this->Rank(); - std::string fileContents; - - // Read the file on rank 0 and broadcast it to everybody else - if (rank == rankSource) - { - // load file contents - fileContents = FileToString(fileName, hint); - } - fileContents = this->BroadcastValue(fileContents, rankSource); - - return fileContents; -} - void Comm::AllgatherImpl(const void *sendbuf, size_t sendcount, MPI_Datatype sendtype, void *recvbuf, size_t recvcount, MPI_Datatype recvtype, const std::string &hint) const @@ -367,6 +349,24 @@ Comm::Req Comm::IrecvImpl(void *buffer, size_t count, MPI_Datatype datatype, return req; } +std::string Comm::BroadcastFile(const std::string &fileName, + const std::string hint, + const int rankSource) const +{ + int rank = this->Rank(); + std::string fileContents; + + // Read the file on rank 0 and broadcast it to everybody else + if (rank == rankSource) + { + // load file contents + fileContents = FileToString(fileName, hint); + } + fileContents = this->BroadcastValue(fileContents, rankSource); + + return fileContents; +} + Comm::Req::Req() = default; Comm::Req::Req(MPI_Datatype datatype) : m_MPIDatatype(datatype) {} From 337156181737ba22c0acfb0e6ab158f7ceb4d522 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 12 Sep 2019 13:28:04 -0400 Subject: [PATCH 15/34] helper: Factor out MPI-specific Comm interfaces Create a dedicated header to hold functions to convert between the Comm encapsulation and MPI_Comm. In the future the main Comm header will not need to include raw MPI interfaces at all. --- source/adios2/CMakeLists.txt | 1 + source/adios2/core/ADIOS.cpp | 4 +-- source/adios2/core/IO.cpp | 5 +-- .../engine/dataspaces/DataSpacesReader.cpp | 3 +- .../engine/dataspaces/DataSpacesWriter.cpp | 3 +- source/adios2/engine/hdf5/HDF5ReaderP.cpp | 4 +-- source/adios2/engine/hdf5/HDF5WriterP.cpp | 8 ++--- .../engine/insitumpi/InSituMPIReader.cpp | 3 +- .../engine/insitumpi/InSituMPIWriter.cpp | 3 +- source/adios2/engine/sst/SstReader.cpp | 3 +- source/adios2/engine/sst/SstWriter.cpp | 3 +- source/adios2/helper/adiosComm.cpp | 4 +-- source/adios2/helper/adiosComm.h | 15 +++----- source/adios2/helper/adiosCommMPI.cpp | 34 ++++++++++++++++++ source/adios2/helper/adiosCommMPI.h | 36 +++++++++++++++++++ 15 files changed, 101 insertions(+), 28 deletions(-) create mode 100644 source/adios2/helper/adiosCommMPI.cpp create mode 100644 source/adios2/helper/adiosCommMPI.h diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 49844d99bf..95a6b76f93 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -33,6 +33,7 @@ add_library(adios2 #helper helper/adiosComm.h helper/adiosComm.cpp + helper/adiosCommMPI.h helper/adiosCommMPI.cpp helper/adiosDynamicBinder.h helper/adiosDynamicBinder.cpp helper/adiosMath.cpp helper/adiosMemory.cpp diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index 5e271cb597..19d7c330c0 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -13,8 +13,8 @@ #include // std::transform #include //std::ios_base::failure -#include "adios2/common/ADIOSMPI.h" #include "adios2/core/IO.h" +#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" //InquireKey, BroadcastFile // OPERATORS @@ -60,7 +60,7 @@ namespace core ADIOS::ADIOS(const std::string configFile, MPI_Comm mpiComm, const bool debugMode, const std::string hostLanguage) : m_ConfigFile(configFile), m_DebugMode(debugMode), - m_HostLanguage(hostLanguage), m_Comm(helper::Comm::Duplicate(mpiComm)) + m_HostLanguage(hostLanguage), m_Comm(helper::CommFromMPI(mpiComm)) { if (!configFile.empty()) { diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index d32c101612..45732cc939 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -29,6 +29,7 @@ #include "adios2/engine/skeleton/SkeletonWriter.h" #include "adios2/helper/adiosComm.h" +#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" //BuildParametersMap #include "adios2/toolkit/profiling/taustubs/tautimer.hpp" #include // FileIsDirectory() @@ -504,7 +505,7 @@ Engine &IO::Open(const std::string &name, const Mode mode, MPI_Comm mpiComm) } } - auto comm = helper::Comm::Duplicate(mpiComm); + auto comm = helper::CommFromMPI(mpiComm); std::shared_ptr engine; const bool isDefaultEngine = m_EngineType.empty() ? true : false; std::string engineTypeLC = m_EngineType; @@ -765,7 +766,7 @@ Engine &IO::Open(const std::string &name, const Mode mode, MPI_Comm mpiComm) Engine &IO::Open(const std::string &name, const Mode mode) { - return Open(name, mode, m_ADIOS.GetComm().AsMPI()); + return Open(name, mode, CommAsMPI(m_ADIOS.GetComm())); } Engine &IO::GetEngine(const std::string &name) diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.cpp b/source/adios2/engine/dataspaces/DataSpacesReader.cpp index 79c698f00e..9ee4555d41 100644 --- a/source/adios2/engine/dataspaces/DataSpacesReader.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesReader.cpp @@ -12,6 +12,7 @@ #include "DataSpacesReader.h" #include "DataSpacesReader.tcc" +#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" //CSVToVector #include "adios2/toolkit/dataspaces/ds_data.h" #include "dataspaces.h" @@ -52,7 +53,7 @@ DataSpacesReader::DataSpacesReader(IO &io, const std::string &name, { m_ProvideLatest = false; } - MPI_Comm mpiComm = m_Comm.AsMPI(); + MPI_Comm mpiComm = CommAsMPI(m_Comm); ret = adios_read_dataspaces_init(&mpiComm, &m_data); if (ret < 0) { diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp index 11d97972a2..b783c85414 100644 --- a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp +++ b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp @@ -13,6 +13,7 @@ #include "DataSpacesWriter.h" #include "DataSpacesWriter.tcc" +#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" //CSVToVector #include "adios2/toolkit/dataspaces/ds_data.h" #include "dataspaces.h" @@ -40,7 +41,7 @@ DataSpacesWriter::DataSpacesWriter(IO &io, const std::string &name, { m_data.appid = 0; } - MPI_Comm mpiComm = m_Comm.AsMPI(); + MPI_Comm mpiComm = CommAsMPI(m_Comm); ret = adios_dataspaces_init(&mpiComm, &m_data); if (ret < 0) fprintf(stderr, "Unable to connect to DataSpaces. Err: %d\n", ret); diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.cpp b/source/adios2/engine/hdf5/HDF5ReaderP.cpp index 8d5638562b..fe7aaebb7f 100644 --- a/source/adios2/engine/hdf5/HDF5ReaderP.cpp +++ b/source/adios2/engine/hdf5/HDF5ReaderP.cpp @@ -11,7 +11,7 @@ #include "HDF5ReaderP.h" #include "HDF5ReaderP.tcc" -#include "adios2/common/ADIOSMPI.h" +#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" //CSVToVector #include @@ -62,7 +62,7 @@ void HDF5ReaderP::Init() ", in call to Open\n"); } - m_H5File.Init(m_Name, m_Comm.AsMPI(), false); + m_H5File.Init(m_Name, CommAsMPI(m_Comm), false); m_H5File.ParseParameters(m_IO); /* diff --git a/source/adios2/engine/hdf5/HDF5WriterP.cpp b/source/adios2/engine/hdf5/HDF5WriterP.cpp index d3394239b9..818a60a4e9 100644 --- a/source/adios2/engine/hdf5/HDF5WriterP.cpp +++ b/source/adios2/engine/hdf5/HDF5WriterP.cpp @@ -10,7 +10,7 @@ #include "HDF5WriterP.h" -#include "adios2/common/ADIOSMPI.h" +#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" //CSVToVector namespace adios2 @@ -58,7 +58,7 @@ void HDF5WriterP::Init() } #ifdef NEVER - m_H5File.Init(m_Name, m_Comm.AsMPI(), true); + m_H5File.Init(m_Name, CommAsMPI(m_Comm), true); #else // enforce .h5 ending std::string suffix = ".h5"; @@ -71,11 +71,11 @@ void HDF5WriterP::Init() { // is a file with .bp ending std::string updatedName = m_Name.substr(0, wpos) + suffix; - m_H5File.Init(updatedName, m_Comm.AsMPI(), true); + m_H5File.Init(updatedName, CommAsMPI(m_Comm), true); } else { - m_H5File.Init(m_Name, m_Comm.AsMPI(), true); + m_H5File.Init(m_Name, CommAsMPI(m_Comm), true); } m_H5File.ParseParameters(m_IO); #endif diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.cpp b/source/adios2/engine/insitumpi/InSituMPIReader.cpp index f310fd9e8e..776a5909de 100644 --- a/source/adios2/engine/insitumpi/InSituMPIReader.cpp +++ b/source/adios2/engine/insitumpi/InSituMPIReader.cpp @@ -14,6 +14,7 @@ #include "InSituMPIReader.tcc" +#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" // CSVToVector #include "adios2/toolkit/profiling/taustubs/tautimer.hpp" @@ -38,7 +39,7 @@ InSituMPIReader::InSituMPIReader(IO &io, const std::string &name, Init(); m_RankAllPeers = - insitumpi::FindPeers(m_Comm.AsMPI(), m_Name, false, m_CommWorld); + insitumpi::FindPeers(CommAsMPI(m_Comm), m_Name, false, m_CommWorld); MPI_Comm_rank(m_CommWorld, &m_GlobalRank); MPI_Comm_size(m_CommWorld, &m_GlobalNproc); m_ReaderRank = m_Comm.Rank(); diff --git a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp index 7b2f1980ab..543ce95dbb 100644 --- a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp +++ b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp @@ -11,6 +11,7 @@ * Author: Norbert Podhorszki pnorbert@ornl.gov */ +#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosMath.h" #include "adios2/toolkit/profiling/taustubs/tautimer.hpp" @@ -39,7 +40,7 @@ InSituMPIWriter::InSituMPIWriter(IO &io, const std::string &name, m_BP3Serializer.Init(m_IO.m_Parameters, "in call to InSituMPI::Open write"); m_RankAllPeers = - insitumpi::FindPeers(m_Comm.AsMPI(), m_Name, true, m_CommWorld); + insitumpi::FindPeers(CommAsMPI(m_Comm), m_Name, true, m_CommWorld); for (int i = 0; i < m_RankAllPeers.size(); i++) { m_RankToPeerID[m_RankAllPeers[i]] = i; diff --git a/source/adios2/engine/sst/SstReader.cpp b/source/adios2/engine/sst/SstReader.cpp index 0f02ac5540..66b38deb74 100644 --- a/source/adios2/engine/sst/SstReader.cpp +++ b/source/adios2/engine/sst/SstReader.cpp @@ -15,6 +15,7 @@ #include #include +#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" #include "adios2/toolkit/profiling/taustubs/tautimer.hpp" @@ -34,7 +35,7 @@ SstReader::SstReader(IO &io, const std::string &name, const Mode mode, Init(); - m_Input = SstReaderOpen(cstr, &Params, m_Comm.AsMPI()); + m_Input = SstReaderOpen(cstr, &Params, CommAsMPI(m_Comm)); if (!m_Input) { throw std::runtime_error( diff --git a/source/adios2/engine/sst/SstWriter.cpp b/source/adios2/engine/sst/SstWriter.cpp index 0a4e5ec2a3..2ca4925e12 100644 --- a/source/adios2/engine/sst/SstWriter.cpp +++ b/source/adios2/engine/sst/SstWriter.cpp @@ -8,6 +8,7 @@ * Author: Greg Eisenhauer */ +#include "adios2/helper/adiosCommMPI.h" #include #include "SstParamParser.h" @@ -108,7 +109,7 @@ SstWriter::SstWriter(IO &io, const std::string &name, const Mode mode, Init(); - m_Output = SstWriterOpen(name.c_str(), &Params, m_Comm.AsMPI()); + m_Output = SstWriterOpen(name.c_str(), &Params, CommAsMPI(m_Comm)); if (m_MarshalMethod == SstMarshalBP) { diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp index fa22f38d67..5554ffa499 100644 --- a/source/adios2/helper/adiosComm.cpp +++ b/source/adios2/helper/adiosComm.cpp @@ -2,7 +2,7 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * Comm.cpp + * adiosComm.cpp */ #include "adiosComm.h" @@ -54,7 +54,7 @@ Comm &Comm::operator=(Comm &&comm) void Comm::swap(Comm &comm) { std::swap(this->m_MPIComm, comm.m_MPIComm); } -Comm Comm::Duplicate(MPI_Comm mpiComm) +Comm Comm::FromMPI(MPI_Comm mpiComm) { MPI_Comm newComm; SMPI_Comm_dup(mpiComm, &newComm); diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h index bff73bbd27..39bf180676 100644 --- a/source/adios2/helper/adiosComm.h +++ b/source/adios2/helper/adiosComm.h @@ -65,16 +65,6 @@ class Comm */ void swap(Comm &comm); - /** - * @brief Get the underlying raw MPI communicator. - */ - MPI_Comm AsMPI() const { return m_MPIComm; } - - /** - * @brief Create a communicator by duplicating a MPI communicator. - */ - static Comm Duplicate(MPI_Comm mpiComm); - /** * @brief Free the communicator. * @param hint Description of std::runtime_error exception on error. @@ -291,6 +281,11 @@ class Comm /** Return MPI datatype id for type T. */ template static MPI_Datatype Datatype(); + + friend Comm CommFromMPI(MPI_Comm mpiComm); + friend MPI_Comm CommAsMPI(Comm const &comm); + MPI_Comm AsMPI() const { return m_MPIComm; } + static Comm FromMPI(MPI_Comm mpiComm); }; class Comm::Req diff --git a/source/adios2/helper/adiosCommMPI.cpp b/source/adios2/helper/adiosCommMPI.cpp new file mode 100644 index 0000000000..91f7cd2cfd --- /dev/null +++ b/source/adios2/helper/adiosCommMPI.cpp @@ -0,0 +1,34 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * adiosCommMPI.cpp + */ + +#include "adiosCommMPI.h" + +#include + +#include "adiosComm.h" + +#include "adios2/common/ADIOSMPI.h" + +namespace adios2 +{ +namespace helper +{ + +Comm CommFromMPI(MPI_Comm mpiComm) +{ + // Forward to the private API. + return Comm::FromMPI(mpiComm); +} + +MPI_Comm CommAsMPI(Comm const &comm) +{ + // Forward to the private API. + return comm.AsMPI(); +} + +} // end namespace helper +} // end namespace adios2 diff --git a/source/adios2/helper/adiosCommMPI.h b/source/adios2/helper/adiosCommMPI.h new file mode 100644 index 0000000000..adfcb2b8f4 --- /dev/null +++ b/source/adios2/helper/adiosCommMPI.h @@ -0,0 +1,36 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * adiosCommMPI.h : MPI-specific Comm APIs + */ + +#ifndef ADIOS2_HELPER_ADIOSCOMMMPI_H_ +#define ADIOS2_HELPER_ADIOSCOMMMPI_H_ + +#include "adiosComm.h" + +#include "adios2/common/ADIOSMPI.h" + +namespace adios2 +{ +namespace helper +{ + +/** + * @brief Create a communicator by duplicating a MPI communicator. + */ +Comm CommFromMPI(MPI_Comm mpiComm); + +/** + * @brief Get the underlying raw MPI communicator. + * + * Returns MPI_COMM_NULL if the communicator is empty or is not + * backed by MPI. + */ +MPI_Comm CommAsMPI(Comm const &comm); + +} // end namespace helper +} // end namespace adios2 + +#endif // ADIOS2_HELPER_ADIOSCOMMMPI_H_ From 2b39dfac89cbca3ec6e31a4518975116cd158477 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 12 Sep 2019 14:00:32 -0400 Subject: [PATCH 16/34] helper: Move Comm MPI implementation behind an abstraction Move the MPI-backed implementation into a separate file. Use an abstraction layer with runtime dispatch so that the main Comm does not need MPI. Note that we still use a few MPI types and constants in the Comm interface, but those can be refactored later. --- source/adios2/helper/adiosComm.cpp | 416 ++-------------------- source/adios2/helper/adiosComm.h | 164 ++++----- source/adios2/helper/adiosComm.inl | 33 +- source/adios2/helper/adiosCommMPI.cpp | 475 +++++++++++++++++++++++++- 4 files changed, 601 insertions(+), 487 deletions(-) diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp index 5554ffa499..83e70b8f78 100644 --- a/source/adios2/helper/adiosComm.cpp +++ b/source/adios2/helper/adiosComm.cpp @@ -8,12 +8,8 @@ #include "adiosComm.h" #include "adiosComm.tcc" -#include -#include //std::ios_base::failure -#include #include -#include "adios2/common/ADIOSMPI.h" #include "adios2/helper/adiosString.h" namespace adios2 @@ -23,331 +19,26 @@ namespace helper Comm::Comm() = default; -Comm::Comm(MPI_Comm mpiComm) : m_MPIComm(mpiComm) {} +Comm::Comm(std::unique_ptr impl) : m_Impl(std::move(impl)) {} -Comm::~Comm() -{ - // Handle the case where MPI is finalized before the ADIOS destructor is - // called, which happens, e.g., with global / static ADIOS objects - int flag; - MPI_Finalized(&flag); - if (!flag) - { - if (m_MPIComm != MPI_COMM_NULL && m_MPIComm != MPI_COMM_WORLD && - m_MPIComm != MPI_COMM_SELF) - { - SMPI_Comm_free(&m_MPIComm); - } - } -} +Comm::~Comm() = default; -Comm::Comm(Comm &&comm) : m_MPIComm(comm.m_MPIComm) -{ - comm.m_MPIComm = MPI_COMM_NULL; -} +Comm::Comm(Comm &&comm) = default; -Comm &Comm::operator=(Comm &&comm) -{ - Comm(std::move(comm)).swap(*this); - return *this; -} +Comm &Comm::operator=(Comm &&comm) = default; -void Comm::swap(Comm &comm) { std::swap(this->m_MPIComm, comm.m_MPIComm); } - -Comm Comm::FromMPI(MPI_Comm mpiComm) -{ - MPI_Comm newComm; - SMPI_Comm_dup(mpiComm, &newComm); - return Comm(newComm); -} - -void Comm::CheckMPIReturn(const int value, const std::string &hint) -{ - if (value == MPI_SUCCESS) - { - return; - } - - std::string error; - switch (value) - { - case MPI_ERR_COMM: - error = "MPI_ERR_COMM"; - break; - case MPI_ERR_INTERN: - error = "MPI_ERR_INTERN"; - break; - default: - error = "MPI_ERR number: " + std::to_string(value); - } - - throw std::runtime_error("ERROR: ADIOS2 detected " + error + ", " + hint); -} - -void Comm::Free(const std::string &hint) -{ - if (m_MPIComm != MPI_COMM_NULL && m_MPIComm != MPI_COMM_WORLD && - m_MPIComm != MPI_COMM_SELF) - { - CheckMPIReturn(SMPI_Comm_free(&m_MPIComm), hint); - } -} +void Comm::Free(const std::string &hint) { m_Impl->Free(hint); } Comm Comm::Split(int color, int key, const std::string &hint) const { - MPI_Comm newComm; - CheckMPIReturn(MPI_Comm_split(m_MPIComm, color, key, &newComm), hint); - return Comm(newComm); -} - -int Comm::Rank() const -{ - int rank; - CheckMPIReturn(SMPI_Comm_rank(m_MPIComm, &rank), {}); - return rank; -} - -int Comm::Size() const -{ - int size; - CheckMPIReturn(SMPI_Comm_size(m_MPIComm, &size), {}); - return size; -} - -void Comm::Barrier(const std::string &hint) const -{ - CheckMPIReturn(SMPI_Barrier(m_MPIComm), hint); -} - -void Comm::AllgatherImpl(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, size_t recvcount, - MPI_Datatype recvtype, const std::string &hint) const -{ - CheckMPIReturn( - SMPI_Allgather(sendbuf, static_cast(sendcount), sendtype, recvbuf, - static_cast(recvcount), recvtype, m_MPIComm), - hint); -} - -void Comm::AllreduceImpl(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, MPI_Op op, - const std::string &hint) const -{ - CheckMPIReturn(SMPI_Allreduce(sendbuf, recvbuf, static_cast(count), - datatype, op, m_MPIComm), - hint); -} - -void Comm::BcastImpl(void *buffer, size_t count, MPI_Datatype datatype, - size_t datatypeSize, int root, - const std::string &hint) const -{ - size_t inputSize = count; - const int MAXBCASTSIZE = 1073741824; - size_t blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize); - unsigned char *blockBuf = static_cast(buffer); - while (inputSize > 0) - { - CheckMPIReturn(SMPI_Bcast(blockBuf, static_cast(blockSize), - datatype, root, m_MPIComm), - hint); - blockBuf += blockSize * datatypeSize; - inputSize -= blockSize; - blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize); - } -} - -void Comm::GatherImpl(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, size_t recvcount, - MPI_Datatype recvtype, int root, - const std::string &hint) const -{ - CheckMPIReturn(SMPI_Gather(sendbuf, static_cast(sendcount), sendtype, - recvbuf, static_cast(recvcount), recvtype, - root, m_MPIComm), - hint); -} - -void Comm::GathervImpl(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, - const size_t *recvcounts, const size_t *displs, - MPI_Datatype recvtype, int root, - const std::string &hint) const -{ - std::vector countsInt; - std::vector displsInt; - if (root == this->Rank()) - { - auto cast = [](size_t sz) -> int { return int(sz); }; - const int size = this->Size(); - countsInt.reserve(size); - std::transform(recvcounts, recvcounts + size, - std::back_inserter(countsInt), cast); - displsInt.reserve(size); - std::transform(displs, displs + size, std::back_inserter(displsInt), - cast); - } - CheckMPIReturn(SMPI_Gatherv(sendbuf, static_cast(sendcount), sendtype, - recvbuf, countsInt.data(), displsInt.data(), - recvtype, root, m_MPIComm), - hint); -} - -void Comm::ReduceImpl(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, MPI_Op op, int root, - const std::string &hint) const -{ - CheckMPIReturn(SMPI_Reduce(sendbuf, recvbuf, static_cast(count), - datatype, op, root, m_MPIComm), - hint); -} - -void Comm::ReduceInPlaceImpl(void *buf, size_t count, MPI_Datatype datatype, - MPI_Op op, int root, const std::string &hint) const -{ - CheckMPIReturn(SMPI_Reduce(MPI_IN_PLACE, buf, static_cast(count), - datatype, op, root, m_MPIComm), - hint); + return Comm(m_Impl->Split(color, key, hint)); } -void Comm::SendImpl(const void *buf, size_t count, MPI_Datatype datatype, - int dest, int tag, const std::string &hint) const -{ - CheckMPIReturn( - MPI_Send(buf, static_cast(count), datatype, dest, tag, m_MPIComm), - hint); -} +int Comm::Rank() const { return m_Impl->Rank(); } -Comm::Status Comm::RecvImpl(void *buf, size_t count, MPI_Datatype datatype, - int source, int tag, const std::string &hint) const -{ - MPI_Status mpiStatus; - CheckMPIReturn(MPI_Recv(buf, static_cast(count), datatype, source, tag, - m_MPIComm, &mpiStatus), - hint); +int Comm::Size() const { return m_Impl->Size(); } - Status status; -#ifdef ADIOS2_HAVE_MPI - status.Source = mpiStatus.MPI_SOURCE; - status.Tag = mpiStatus.MPI_TAG; - { - int mpiCount = 0; - CheckMPIReturn(MPI_Get_count(&mpiStatus, datatype, &mpiCount), hint); - status.Count = mpiCount; - } -#endif - return status; -} - -void Comm::ScatterImpl(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, size_t recvcount, - MPI_Datatype recvtype, int root, - const std::string &hint) const -{ - CheckMPIReturn(MPI_Scatter(sendbuf, static_cast(sendcount), sendtype, - recvbuf, static_cast(recvcount), recvtype, - root, m_MPIComm), - hint); -} - -Comm::Req Comm::IsendImpl(const void *buffer, size_t count, - MPI_Datatype datatype, int dest, int tag, - const std::string &hint) const -{ - Comm::Req req(datatype); - - if (count > DefaultMaxFileBatchSize) - { - const size_t batches = count / DefaultMaxFileBatchSize; - - size_t position = 0; - for (size_t b = 0; b < batches; ++b) - { - int batchSize = static_cast(DefaultMaxFileBatchSize); - MPI_Request mpiReq; - CheckMPIReturn( - MPI_Isend(static_cast(const_cast(buffer)) + - position, - batchSize, datatype, dest, tag, m_MPIComm, &mpiReq), - "in call to Isend batch " + std::to_string(b) + " " + hint + - "\n"); - req.m_MPIReqs.emplace_back(mpiReq); - - position += DefaultMaxFileBatchSize; - } - const size_t remainder = count % DefaultMaxFileBatchSize; - if (remainder > 0) - { - int batchSize = static_cast(remainder); - MPI_Request mpiReq; - CheckMPIReturn( - MPI_Isend(static_cast(const_cast(buffer)) + - position, - batchSize, datatype, dest, tag, m_MPIComm, &mpiReq), - "in call to Isend remainder batch " + hint + "\n"); - req.m_MPIReqs.emplace_back(mpiReq); - } - } - else - { - int batchSize = static_cast(count); - MPI_Request mpiReq; - CheckMPIReturn( - MPI_Isend(static_cast(const_cast(buffer)), - batchSize, datatype, dest, tag, m_MPIComm, &mpiReq), - " in call to Isend with single batch " + hint + "\n"); - req.m_MPIReqs.emplace_back(mpiReq); - } - return req; -} - -Comm::Req Comm::IrecvImpl(void *buffer, size_t count, MPI_Datatype datatype, - int source, int tag, const std::string &hint) const -{ - Comm::Req req(datatype); - - if (count > DefaultMaxFileBatchSize) - { - const size_t batches = count / DefaultMaxFileBatchSize; - size_t position = 0; - for (size_t b = 0; b < batches; ++b) - { - int batchSize = static_cast(DefaultMaxFileBatchSize); - MPI_Request mpiReq; - CheckMPIReturn(MPI_Irecv(static_cast(buffer) + position, - batchSize, datatype, source, tag, - m_MPIComm, &mpiReq), - "in call to Irecv batch " + std::to_string(b) + " " + - hint + "\n"); - req.m_MPIReqs.emplace_back(mpiReq); - - position += DefaultMaxFileBatchSize; - } - - const size_t remainder = count % DefaultMaxFileBatchSize; - if (remainder > 0) - { - int batchSize = static_cast(remainder); - MPI_Request mpiReq; - CheckMPIReturn(MPI_Irecv(static_cast(buffer) + position, - batchSize, datatype, source, tag, - m_MPIComm, &mpiReq), - "in call to Irecv remainder batch " + hint + "\n"); - req.m_MPIReqs.emplace_back(mpiReq); - } - } - else - { - int batchSize = static_cast(count); - MPI_Request mpiReq; - CheckMPIReturn(MPI_Irecv(buffer, batchSize, datatype, source, tag, - m_MPIComm, &mpiReq), - " in call to Isend with single batch " + hint + "\n"); - req.m_MPIReqs.emplace_back(mpiReq); - } - - return req; -} +void Comm::Barrier(const std::string &hint) const { m_Impl->Barrier(hint); } std::string Comm::BroadcastFile(const std::string &fileName, const std::string hint, @@ -369,89 +60,40 @@ std::string Comm::BroadcastFile(const std::string &fileName, Comm::Req::Req() = default; -Comm::Req::Req(MPI_Datatype datatype) : m_MPIDatatype(datatype) {} +Comm::Req::Req(std::unique_ptr impl) : m_Impl(std::move(impl)) {} -Comm::Req::~Req() {} - -Comm::Req::Req(Req &&req) -: m_MPIDatatype(req.m_MPIDatatype), m_MPIReqs(std::move(req.m_MPIReqs)) -{ -} +Comm::Req::~Req() = default; -Comm::Req &Comm::Req::operator=(Req &&req) -{ - Req(std::move(req)).swap(*this); - return *this; -} +Comm::Req::Req(Req &&req) = default; -void Comm::Req::swap(Req &req) -{ - std::swap(this->m_MPIDatatype, req.m_MPIDatatype); - std::swap(this->m_MPIReqs, req.m_MPIReqs); -} +Comm::Req &Comm::Req::operator=(Req &&req) = default; Comm::Status Comm::Req::Wait(const std::string &hint) { Comm::Status status; - if (m_MPIReqs.empty()) + if (m_Impl) { - return status; + status = m_Impl->Wait(hint); + m_Impl.reset(); } + return status; +} -#ifdef ADIOS2_HAVE_MPI - std::vector mpiRequests = std::move(m_MPIReqs); - std::vector mpiStatuses(mpiRequests.size()); - - if (mpiRequests.size() > 1) - { - int mpiReturn = MPI_Waitall(static_cast(mpiRequests.size()), - mpiRequests.data(), mpiStatuses.data()); - if (mpiReturn == MPI_ERR_IN_STATUS) - { - for (auto &mpiStatus : mpiStatuses) - { - if (mpiStatus.MPI_ERROR != MPI_SUCCESS) - { - mpiReturn = mpiStatus.MPI_ERROR; - break; - } - } - } - CheckMPIReturn(mpiReturn, hint); - } - else - { - CheckMPIReturn(MPI_Wait(mpiRequests.data(), mpiStatuses.data()), hint); - } +CommImpl::~CommImpl() = default; - // Our batched operation should be from only one source and have one tag. - status.Source = mpiStatuses.front().MPI_SOURCE; - status.Tag = mpiStatuses.front().MPI_TAG; +Comm CommImpl::MakeComm(std::unique_ptr impl) +{ + return Comm(std::move(impl)); +} - // Accumulate the total count of our batched operation. - for (auto &mpiStatus : mpiStatuses) - { - int mpiCount = 0; - CheckMPIReturn(MPI_Get_count(&mpiStatus, m_MPIDatatype, &mpiCount), - hint); - status.Count += mpiCount; - } +Comm::Req CommImpl::MakeReq(std::unique_ptr impl) +{ + return Comm::Req(std::move(impl)); +} - // Our batched operation was cancelled if any member was cancelled. - for (auto &mpiStatus : mpiStatuses) - { - int mpiCancelled = 0; - MPI_Test_cancelled(&mpiStatus, &mpiCancelled); - if (mpiCancelled) - { - status.Cancelled = true; - break; - } - } -#endif +CommImpl *CommImpl::Get(Comm const &comm) { return comm.m_Impl.get(); } - return status; -} +CommReqImpl::~CommReqImpl() = default; } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h index 39bf180676..dbd35f1dea 100644 --- a/source/adios2/helper/adiosComm.h +++ b/source/adios2/helper/adiosComm.h @@ -10,6 +10,7 @@ #include "adios2/common/ADIOSMPI.h" +#include #include #include @@ -18,6 +19,9 @@ namespace adios2 namespace helper { +class CommImpl; +class CommReqImpl; + /** @brief Encapsulation for communication in a multi-process environment. */ class Comm { @@ -60,11 +64,6 @@ class Comm */ Comm &operator=(Comm const &) = delete; - /** - * @brief Swap communicator state with another. - */ - void swap(Comm &comm); - /** * @brief Free the communicator. * @param hint Description of std::runtime_error exception on error. @@ -218,74 +217,15 @@ class Comm const std::string &hint = std::string()) const; private: - /** - * @brief Construct by taking ownership of a MPI communicator. - * - * This is a private implementation detail used by static - * methods like Duplicate. - */ - explicit Comm(MPI_Comm mpiComm); - - /** Encapsulated MPI communicator instance. */ - MPI_Comm m_MPIComm = MPI_COMM_NULL; - - static void CheckMPIReturn(const int value, const std::string &hint); - - void AllgatherImpl(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, size_t recvcount, - MPI_Datatype recvtype, const std::string &hint) const; - - void AllreduceImpl(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, MPI_Op op, - const std::string &hint) const; - - void BcastImpl(void *buffer, size_t count, MPI_Datatype datatype, - size_t datatypeSize, int root, - const std::string &hint) const; - - void GatherImpl(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, size_t recvcount, - MPI_Datatype recvtype, int root, - const std::string &hint) const; - - void GathervImpl(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, - const size_t *recvcounts, const size_t *displs, - MPI_Datatype recvtype, int root, - const std::string &hint) const; - - void ReduceImpl(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, MPI_Op op, int root, - const std::string &hint) const; - - void ReduceInPlaceImpl(void *buf, size_t count, MPI_Datatype datatype, - MPI_Op op, int root, const std::string &hint) const; + friend class CommImpl; - void SendImpl(const void *buf, size_t count, MPI_Datatype datatype, - int dest, int tag, const std::string &hint) const; + explicit Comm(std::unique_ptr impl); - Status RecvImpl(void *buf, size_t count, MPI_Datatype datatype, int source, - int tag, const std::string &hint) const; - - void ScatterImpl(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, size_t recvcount, - MPI_Datatype recvtype, int root, - const std::string &hint) const; - - Req IsendImpl(const void *buffer, size_t count, MPI_Datatype datatype, - int dest, int tag, const std::string &hint) const; - - Req IrecvImpl(void *buffer, size_t count, MPI_Datatype datatype, int source, - int tag, const std::string &hint) const; + std::unique_ptr m_Impl; /** Return MPI datatype id for type T. */ template static MPI_Datatype Datatype(); - - friend Comm CommFromMPI(MPI_Comm mpiComm); - friend MPI_Comm CommAsMPI(Comm const &comm); - MPI_Comm AsMPI() const { return m_MPIComm; } - static Comm FromMPI(MPI_Comm mpiComm); }; class Comm::Req @@ -324,11 +264,6 @@ class Comm::Req */ Req &operator=(Req const &) = delete; - /** - * @brief Swap request state with another. - */ - void swap(Req &req); - /** * @brief Wait for the request to finish. * @@ -337,16 +272,11 @@ class Comm::Req Comm::Status Wait(const std::string &hint = std::string()); private: - friend class Comm; - - Req(MPI_Datatype datatype); + friend class CommImpl; - /** Encapsulated MPI datatype of the requested operation. */ - MPI_Datatype m_MPIDatatype = MPI_DATATYPE_NULL; + explicit Req(std::unique_ptr impl); - /** Encapsulated MPI request instances. There may be more than - * one when we batch requests too large for MPI interfaces. */ - std::vector m_MPIReqs; + std::unique_ptr m_Impl; }; class Comm::Status @@ -373,6 +303,80 @@ class Comm::Status bool Cancelled = false; }; +class CommImpl +{ +public: + virtual ~CommImpl() = 0; + virtual void Free(const std::string &hint) = 0; + virtual std::unique_ptr Split(int color, int key, + const std::string &hint) const = 0; + virtual int Rank() const = 0; + virtual int Size() const = 0; + virtual void Barrier(const std::string &hint) const = 0; + virtual void Allgather(const void *sendbuf, size_t sendcount, + MPI_Datatype sendtype, void *recvbuf, + size_t recvcount, MPI_Datatype recvtype, + const std::string &hint) const = 0; + + virtual void Allreduce(const void *sendbuf, void *recvbuf, size_t count, + MPI_Datatype datatype, MPI_Op op, + const std::string &hint) const = 0; + + virtual void Bcast(void *buffer, size_t count, MPI_Datatype datatype, + size_t datatypeSize, int root, + const std::string &hint) const = 0; + + virtual void Gather(const void *sendbuf, size_t sendcount, + MPI_Datatype sendtype, void *recvbuf, size_t recvcount, + MPI_Datatype recvtype, int root, + const std::string &hint) const = 0; + + virtual void Gatherv(const void *sendbuf, size_t sendcount, + MPI_Datatype sendtype, void *recvbuf, + const size_t *recvcounts, const size_t *displs, + MPI_Datatype recvtype, int root, + const std::string &hint) const = 0; + + virtual void Reduce(const void *sendbuf, void *recvbuf, size_t count, + MPI_Datatype datatype, MPI_Op op, int root, + const std::string &hint) const = 0; + + virtual void ReduceInPlace(void *buf, size_t count, MPI_Datatype datatype, + MPI_Op op, int root, + const std::string &hint) const = 0; + + virtual void Send(const void *buf, size_t count, MPI_Datatype datatype, + int dest, int tag, const std::string &hint) const = 0; + + virtual Comm::Status Recv(void *buf, size_t count, MPI_Datatype datatype, + int source, int tag, + const std::string &hint) const = 0; + + virtual void Scatter(const void *sendbuf, size_t sendcount, + MPI_Datatype sendtype, void *recvbuf, size_t recvcount, + MPI_Datatype recvtype, int root, + const std::string &hint) const = 0; + + virtual Comm::Req Isend(const void *buffer, size_t count, + MPI_Datatype datatype, int dest, int tag, + const std::string &hint) const = 0; + + virtual Comm::Req Irecv(void *buffer, size_t count, MPI_Datatype datatype, + int source, int tag, + const std::string &hint) const = 0; + + static Comm MakeComm(std::unique_ptr impl); + static Comm::Req MakeReq(std::unique_ptr impl); + static CommImpl *Get(Comm const &comm); +}; + +class CommReqImpl +{ +public: + virtual ~CommReqImpl() = 0; + virtual Comm::Status Wait(const std::string &hint) = 0; +}; + } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index b3f5a940bf..fed6d765fd 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -142,30 +142,30 @@ template void Comm::Allgather(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, size_t recvcount, const std::string &hint) const { - return AllgatherImpl(sendbuf, sendcount, Datatype(), recvbuf, - recvcount, Datatype(), hint); + return m_Impl->Allgather(sendbuf, sendcount, Datatype(), recvbuf, + recvcount, Datatype(), hint); } template void Comm::Allreduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op, const std::string &hint) const { - return AllreduceImpl(sendbuf, recvbuf, count, Datatype(), op, hint); + return m_Impl->Allreduce(sendbuf, recvbuf, count, Datatype(), op, hint); } template void Comm::Bcast(T *buffer, const size_t count, int root, const std::string &hint) const { - return BcastImpl(buffer, count, Datatype(), sizeof(T), root, hint); + return m_Impl->Bcast(buffer, count, Datatype(), sizeof(T), root, hint); } template void Comm::Gather(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, size_t recvcount, int root, const std::string &hint) const { - return GatherImpl(sendbuf, sendcount, Datatype(), recvbuf, recvcount, - Datatype(), root, hint); + return m_Impl->Gather(sendbuf, sendcount, Datatype(), recvbuf, + recvcount, Datatype(), root, hint); } template @@ -173,58 +173,59 @@ void Comm::Gatherv(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, const size_t *recvcounts, const size_t *displs, int root, const std::string &hint) const { - return GathervImpl(sendbuf, sendcount, Datatype(), recvbuf, - recvcounts, displs, Datatype(), root, hint); + return m_Impl->Gatherv(sendbuf, sendcount, Datatype(), recvbuf, + recvcounts, displs, Datatype(), root, hint); } template void Comm::Reduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op, int root, const std::string &hint) const { - return ReduceImpl(sendbuf, recvbuf, count, Datatype(), op, root, hint); + return m_Impl->Reduce(sendbuf, recvbuf, count, Datatype(), op, root, + hint); } template void Comm::ReduceInPlace(T *buf, size_t count, MPI_Op op, int root, const std::string &hint) const { - return ReduceInPlaceImpl(buf, count, Datatype(), op, root, hint); + return m_Impl->ReduceInPlace(buf, count, Datatype(), op, root, hint); } template void Comm::Send(const T *buf, size_t count, int dest, int tag, const std::string &hint) const { - return SendImpl(buf, count, Datatype(), dest, tag, hint); + return m_Impl->Send(buf, count, Datatype(), dest, tag, hint); } template Comm::Status Comm::Recv(T *buf, size_t count, int source, int tag, const std::string &hint) const { - return RecvImpl(buf, count, Datatype(), source, tag, hint); + return m_Impl->Recv(buf, count, Datatype(), source, tag, hint); } template void Comm::Scatter(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, size_t recvcount, int root, const std::string &hint) const { - return ScatterImpl(sendbuf, sendcount, Datatype(), recvbuf, - recvcount, Datatype(), root, hint); + return m_Impl->Scatter(sendbuf, sendcount, Datatype(), recvbuf, + recvcount, Datatype(), root, hint); } template Comm::Req Comm::Isend(const T *buffer, const size_t count, int dest, int tag, const std::string &hint) const { - return IsendImpl(buffer, count, Datatype(), dest, tag, hint); + return m_Impl->Isend(buffer, count, Datatype(), dest, tag, hint); } template Comm::Req Comm::Irecv(T *buffer, const size_t count, int source, int tag, const std::string &hint) const { - return IrecvImpl(buffer, count, Datatype(), source, tag, hint); + return m_Impl->Irecv(buffer, count, Datatype(), source, tag, hint); } // Datatype full specializations implemented in 'adiosComm.tcc'. diff --git a/source/adios2/helper/adiosCommMPI.cpp b/source/adios2/helper/adiosCommMPI.cpp index 91f7cd2cfd..0d550d104f 100644 --- a/source/adios2/helper/adiosCommMPI.cpp +++ b/source/adios2/helper/adiosCommMPI.cpp @@ -7,27 +7,494 @@ #include "adiosCommMPI.h" +#include +#include //std::ios_base::failure +#include #include #include "adiosComm.h" #include "adios2/common/ADIOSMPI.h" +#include "adios2/common/ADIOSTypes.h" namespace adios2 { namespace helper { +namespace +{ +void CheckMPIReturn(const int value, const std::string &hint) +{ + if (value == MPI_SUCCESS) + { + return; + } + + std::string error; + switch (value) + { + case MPI_ERR_COMM: + error = "MPI_ERR_COMM"; + break; + case MPI_ERR_INTERN: + error = "MPI_ERR_INTERN"; + break; + default: + error = "MPI_ERR number: " + std::to_string(value); + } + + throw std::runtime_error("ERROR: ADIOS2 detected " + error + ", " + hint); +} +} + +class CommReqImplMPI : public CommReqImpl +{ +public: + CommReqImplMPI(MPI_Datatype datatype) : m_MPIDatatype(datatype) {} + ~CommReqImplMPI() override; + + Comm::Status Wait(const std::string &hint) override; + + /** Encapsulated MPI datatype of the requested operation. */ + MPI_Datatype m_MPIDatatype = MPI_DATATYPE_NULL; + + /** Encapsulated MPI request instances. There may be more than + * one when we batch requests too large for MPI interfaces. */ + std::vector m_MPIReqs; +}; + +CommReqImplMPI::~CommReqImplMPI() = default; + +class CommImplMPI : public CommImpl +{ +public: + CommImplMPI(MPI_Comm mpiComm) : m_MPIComm(mpiComm) {} + + MPI_Comm m_MPIComm; + + ~CommImplMPI() override; + + void Free(const std::string &hint) override; + std::unique_ptr Split(int color, int key, + const std::string &hint) const override; + + int Rank() const override; + int Size() const override; + void Barrier(const std::string &hint) const override; + + void Allgather(const void *sendbuf, size_t sendcount, MPI_Datatype sendtype, + void *recvbuf, size_t recvcount, MPI_Datatype recvtype, + const std::string &hint) const override; + + void Allreduce(const void *sendbuf, void *recvbuf, size_t count, + MPI_Datatype datatype, MPI_Op op, + const std::string &hint) const override; + + void Bcast(void *buffer, size_t count, MPI_Datatype datatype, + size_t datatypeSize, int root, + const std::string &hint) const override; + + void Gather(const void *sendbuf, size_t sendcount, MPI_Datatype sendtype, + void *recvbuf, size_t recvcount, MPI_Datatype recvtype, + int root, const std::string &hint) const override; + + void Gatherv(const void *sendbuf, size_t sendcount, MPI_Datatype sendtype, + void *recvbuf, const size_t *recvcounts, const size_t *displs, + MPI_Datatype recvtype, int root, + const std::string &hint) const override; + + void Reduce(const void *sendbuf, void *recvbuf, size_t count, + MPI_Datatype datatype, MPI_Op op, int root, + const std::string &hint) const override; + + void ReduceInPlace(void *buf, size_t count, MPI_Datatype datatype, + MPI_Op op, int root, + const std::string &hint) const override; + + void Send(const void *buf, size_t count, MPI_Datatype datatype, int dest, + int tag, const std::string &hint) const override; + + Comm::Status Recv(void *buf, size_t count, MPI_Datatype datatype, + int source, int tag, + const std::string &hint) const override; + + void Scatter(const void *sendbuf, size_t sendcount, MPI_Datatype sendtype, + void *recvbuf, size_t recvcount, MPI_Datatype recvtype, + int root, const std::string &hint) const override; + + Comm::Req Isend(const void *buffer, size_t count, MPI_Datatype datatype, + int dest, int tag, const std::string &hint) const override; + + Comm::Req Irecv(void *buffer, size_t count, MPI_Datatype datatype, + int source, int tag, + const std::string &hint) const override; +}; + +CommImplMPI::~CommImplMPI() +{ + // Handle the case where MPI is finalized before the ADIOS destructor is + // called, which happens, e.g., with global / static ADIOS objects + int flag; + MPI_Finalized(&flag); + if (!flag) + { + if (m_MPIComm != MPI_COMM_NULL && m_MPIComm != MPI_COMM_WORLD && + m_MPIComm != MPI_COMM_SELF) + { + SMPI_Comm_free(&m_MPIComm); + } + } +} + +void CommImplMPI::Free(const std::string &hint) +{ + if (m_MPIComm != MPI_COMM_NULL && m_MPIComm != MPI_COMM_WORLD && + m_MPIComm != MPI_COMM_SELF) + { + CheckMPIReturn(SMPI_Comm_free(&m_MPIComm), hint); + } +} + +std::unique_ptr CommImplMPI::Split(int color, int key, + const std::string &hint) const +{ + MPI_Comm newComm; + CheckMPIReturn(MPI_Comm_split(m_MPIComm, color, key, &newComm), hint); + return std::unique_ptr(new CommImplMPI(newComm)); +} + +int CommImplMPI::Rank() const +{ + int rank; + CheckMPIReturn(SMPI_Comm_rank(m_MPIComm, &rank), {}); + return rank; +} + +int CommImplMPI::Size() const +{ + int size; + CheckMPIReturn(SMPI_Comm_size(m_MPIComm, &size), {}); + return size; +} + +void CommImplMPI::Barrier(const std::string &hint) const +{ + CheckMPIReturn(SMPI_Barrier(m_MPIComm), hint); +} + +void CommImplMPI::Allgather(const void *sendbuf, size_t sendcount, + MPI_Datatype sendtype, void *recvbuf, + size_t recvcount, MPI_Datatype recvtype, + const std::string &hint) const +{ + CheckMPIReturn( + SMPI_Allgather(sendbuf, static_cast(sendcount), sendtype, recvbuf, + static_cast(recvcount), recvtype, m_MPIComm), + hint); +} + +void CommImplMPI::Allreduce(const void *sendbuf, void *recvbuf, size_t count, + MPI_Datatype datatype, MPI_Op op, + const std::string &hint) const +{ + CheckMPIReturn(SMPI_Allreduce(sendbuf, recvbuf, static_cast(count), + datatype, op, m_MPIComm), + hint); +} + +void CommImplMPI::Bcast(void *buffer, size_t count, MPI_Datatype datatype, + size_t datatypeSize, int root, + const std::string &hint) const +{ + size_t inputSize = count; + const int MAXBCASTSIZE = 1073741824; + size_t blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize); + unsigned char *blockBuf = static_cast(buffer); + while (inputSize > 0) + { + CheckMPIReturn(SMPI_Bcast(blockBuf, static_cast(blockSize), + datatype, root, m_MPIComm), + hint); + blockBuf += blockSize * datatypeSize; + inputSize -= blockSize; + blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize); + } +} + +void CommImplMPI::Gather(const void *sendbuf, size_t sendcount, + MPI_Datatype sendtype, void *recvbuf, size_t recvcount, + MPI_Datatype recvtype, int root, + const std::string &hint) const +{ + CheckMPIReturn(SMPI_Gather(sendbuf, static_cast(sendcount), sendtype, + recvbuf, static_cast(recvcount), recvtype, + root, m_MPIComm), + hint); +} + +void CommImplMPI::Gatherv(const void *sendbuf, size_t sendcount, + MPI_Datatype sendtype, void *recvbuf, + const size_t *recvcounts, const size_t *displs, + MPI_Datatype recvtype, int root, + const std::string &hint) const +{ + std::vector countsInt; + std::vector displsInt; + if (root == this->Rank()) + { + auto cast = [](size_t sz) -> int { return int(sz); }; + const int size = this->Size(); + countsInt.reserve(size); + std::transform(recvcounts, recvcounts + size, + std::back_inserter(countsInt), cast); + displsInt.reserve(size); + std::transform(displs, displs + size, std::back_inserter(displsInt), + cast); + } + CheckMPIReturn(SMPI_Gatherv(sendbuf, static_cast(sendcount), sendtype, + recvbuf, countsInt.data(), displsInt.data(), + recvtype, root, m_MPIComm), + hint); +} + +void CommImplMPI::Reduce(const void *sendbuf, void *recvbuf, size_t count, + MPI_Datatype datatype, MPI_Op op, int root, + const std::string &hint) const +{ + CheckMPIReturn(SMPI_Reduce(sendbuf, recvbuf, static_cast(count), + datatype, op, root, m_MPIComm), + hint); +} + +void CommImplMPI::ReduceInPlace(void *buf, size_t count, MPI_Datatype datatype, + MPI_Op op, int root, + const std::string &hint) const +{ + CheckMPIReturn(SMPI_Reduce(MPI_IN_PLACE, buf, static_cast(count), + datatype, op, root, m_MPIComm), + hint); +} + +void CommImplMPI::Send(const void *buf, size_t count, MPI_Datatype datatype, + int dest, int tag, const std::string &hint) const +{ + CheckMPIReturn( + MPI_Send(buf, static_cast(count), datatype, dest, tag, m_MPIComm), + hint); +} + +Comm::Status CommImplMPI::Recv(void *buf, size_t count, MPI_Datatype datatype, + int source, int tag, + const std::string &hint) const +{ + MPI_Status mpiStatus; + CheckMPIReturn(MPI_Recv(buf, static_cast(count), datatype, source, tag, + m_MPIComm, &mpiStatus), + hint); + + Comm::Status status; +#ifdef ADIOS2_HAVE_MPI + status.Source = mpiStatus.MPI_SOURCE; + status.Tag = mpiStatus.MPI_TAG; + { + int mpiCount = 0; + CheckMPIReturn(MPI_Get_count(&mpiStatus, datatype, &mpiCount), hint); + status.Count = mpiCount; + } +#endif + return status; +} + +void CommImplMPI::Scatter(const void *sendbuf, size_t sendcount, + MPI_Datatype sendtype, void *recvbuf, + size_t recvcount, MPI_Datatype recvtype, int root, + const std::string &hint) const +{ + CheckMPIReturn(MPI_Scatter(sendbuf, static_cast(sendcount), sendtype, + recvbuf, static_cast(recvcount), recvtype, + root, m_MPIComm), + hint); +} + +Comm::Req CommImplMPI::Isend(const void *buffer, size_t count, + MPI_Datatype datatype, int dest, int tag, + const std::string &hint) const +{ + auto req = std::unique_ptr(new CommReqImplMPI(datatype)); + + if (count > DefaultMaxFileBatchSize) + { + const size_t batches = count / DefaultMaxFileBatchSize; + + size_t position = 0; + for (size_t b = 0; b < batches; ++b) + { + int batchSize = static_cast(DefaultMaxFileBatchSize); + MPI_Request mpiReq; + CheckMPIReturn( + MPI_Isend(static_cast(const_cast(buffer)) + + position, + batchSize, datatype, dest, tag, m_MPIComm, &mpiReq), + "in call to Isend batch " + std::to_string(b) + " " + hint + + "\n"); + req->m_MPIReqs.emplace_back(mpiReq); + + position += DefaultMaxFileBatchSize; + } + const size_t remainder = count % DefaultMaxFileBatchSize; + if (remainder > 0) + { + int batchSize = static_cast(remainder); + MPI_Request mpiReq; + CheckMPIReturn( + MPI_Isend(static_cast(const_cast(buffer)) + + position, + batchSize, datatype, dest, tag, m_MPIComm, &mpiReq), + "in call to Isend remainder batch " + hint + "\n"); + req->m_MPIReqs.emplace_back(mpiReq); + } + } + else + { + int batchSize = static_cast(count); + MPI_Request mpiReq; + CheckMPIReturn( + MPI_Isend(static_cast(const_cast(buffer)), + batchSize, datatype, dest, tag, m_MPIComm, &mpiReq), + " in call to Isend with single batch " + hint + "\n"); + req->m_MPIReqs.emplace_back(mpiReq); + } + + return MakeReq(std::move(req)); +} + +Comm::Req CommImplMPI::Irecv(void *buffer, size_t count, MPI_Datatype datatype, + int source, int tag, const std::string &hint) const +{ + auto req = std::unique_ptr(new CommReqImplMPI(datatype)); + + if (count > DefaultMaxFileBatchSize) + { + const size_t batches = count / DefaultMaxFileBatchSize; + size_t position = 0; + for (size_t b = 0; b < batches; ++b) + { + int batchSize = static_cast(DefaultMaxFileBatchSize); + MPI_Request mpiReq; + CheckMPIReturn(MPI_Irecv(static_cast(buffer) + position, + batchSize, datatype, source, tag, + m_MPIComm, &mpiReq), + "in call to Irecv batch " + std::to_string(b) + " " + + hint + "\n"); + req->m_MPIReqs.emplace_back(mpiReq); + + position += DefaultMaxFileBatchSize; + } + + const size_t remainder = count % DefaultMaxFileBatchSize; + if (remainder > 0) + { + int batchSize = static_cast(remainder); + MPI_Request mpiReq; + CheckMPIReturn(MPI_Irecv(static_cast(buffer) + position, + batchSize, datatype, source, tag, + m_MPIComm, &mpiReq), + "in call to Irecv remainder batch " + hint + "\n"); + req->m_MPIReqs.emplace_back(mpiReq); + } + } + else + { + int batchSize = static_cast(count); + MPI_Request mpiReq; + CheckMPIReturn(MPI_Irecv(buffer, batchSize, datatype, source, tag, + m_MPIComm, &mpiReq), + " in call to Isend with single batch " + hint + "\n"); + req->m_MPIReqs.emplace_back(mpiReq); + } + + return MakeReq(std::move(req)); +} + +Comm::Status CommReqImplMPI::Wait(const std::string &hint) +{ + Comm::Status status; + if (m_MPIReqs.empty()) + { + return status; + } + +#ifdef ADIOS2_HAVE_MPI + std::vector mpiRequests = std::move(m_MPIReqs); + std::vector mpiStatuses(mpiRequests.size()); + + if (mpiRequests.size() > 1) + { + int mpiReturn = MPI_Waitall(static_cast(mpiRequests.size()), + mpiRequests.data(), mpiStatuses.data()); + if (mpiReturn == MPI_ERR_IN_STATUS) + { + for (auto &mpiStatus : mpiStatuses) + { + if (mpiStatus.MPI_ERROR != MPI_SUCCESS) + { + mpiReturn = mpiStatus.MPI_ERROR; + break; + } + } + } + CheckMPIReturn(mpiReturn, hint); + } + else + { + CheckMPIReturn(MPI_Wait(mpiRequests.data(), mpiStatuses.data()), hint); + } + + // Our batched operation should be from only one source and have one tag. + status.Source = mpiStatuses.front().MPI_SOURCE; + status.Tag = mpiStatuses.front().MPI_TAG; + + // Accumulate the total count of our batched operation. + for (auto &mpiStatus : mpiStatuses) + { + int mpiCount = 0; + CheckMPIReturn(MPI_Get_count(&mpiStatus, m_MPIDatatype, &mpiCount), + hint); + status.Count += mpiCount; + } + + // Our batched operation was cancelled if any member was cancelled. + for (auto &mpiStatus : mpiStatuses) + { + int mpiCancelled = 0; + MPI_Test_cancelled(&mpiStatus, &mpiCancelled); + if (mpiCancelled) + { + status.Cancelled = true; + break; + } + } +#endif + + return status; +} + Comm CommFromMPI(MPI_Comm mpiComm) { - // Forward to the private API. - return Comm::FromMPI(mpiComm); + MPI_Comm newComm; + SMPI_Comm_dup(mpiComm, &newComm); + auto comm = std::unique_ptr(new CommImplMPI(newComm)); + return CommImpl::MakeComm(std::move(comm)); } MPI_Comm CommAsMPI(Comm const &comm) { - // Forward to the private API. - return comm.AsMPI(); + if (CommImplMPI *mpi = dynamic_cast(CommImpl::Get(comm))) + { + return mpi->m_MPIComm; + } + return MPI_COMM_NULL; } } // end namespace helper From e5db85b00f005df823433c5239a1ac8caca54f4a Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 12 Sep 2019 14:29:42 -0400 Subject: [PATCH 17/34] helper: Drop unused Comm::AllGatherArrays method It was used only inside our own implementation in one place, and its implementation was just a thin wrapper around our `Allgather` interface, so simply use that directly. --- source/adios2/helper/adiosComm.h | 10 ---------- source/adios2/helper/adiosComm.inl | 7 +------ source/adios2/helper/adiosComm.tcc | 8 -------- 3 files changed, 1 insertion(+), 24 deletions(-) diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h index dbd35f1dea..f5a1885fb2 100644 --- a/source/adios2/helper/adiosComm.h +++ b/source/adios2/helper/adiosComm.h @@ -139,16 +139,6 @@ class Comm template std::vector AllGatherValues(const T source) const; - /** - * Perform AllGather for equal size arrays - * @param source - * @param sourceCount - * @param destination - */ - template - void AllGatherArrays(const T *source, const size_t sourceCount, - T *destination) const; - template T ReduceValues(const T source, MPI_Op operation = MPI_SUM, const int rankDestination = 0) const; diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index fed6d765fd..385a0678cb 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -101,15 +101,10 @@ std::vector Comm::AllGatherValues(const T source) const std::vector output(size); T sourceCopy = source; // so we can have an address for rvalues - this->AllGatherArrays(&sourceCopy, 1, output.data()); + this->Allgather(&sourceCopy, 1, output.data(), 1); return output; } -// AllGatherArrays full specializations implemented in 'adiosComm.tcc'. -template <> -void Comm::AllGatherArrays(const size_t *source, const size_t sourceCount, - size_t *destination) const; - // ReduceValues full specializations implemented in 'adiosComm.tcc'. template <> unsigned int Comm::ReduceValues(const unsigned int source, MPI_Op operation, diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc index cc272ad760..b11ea41f86 100644 --- a/source/adios2/helper/adiosComm.tcc +++ b/source/adios2/helper/adiosComm.tcc @@ -82,14 +82,6 @@ void Comm::GathervArrays(const size_t *source, size_t sourceCount, rankDestination); } -// AllGatherArrays full specializations forward-declared in 'adiosComm.inl'. -template <> -void Comm::AllGatherArrays(const size_t *source, const size_t sourceCount, - size_t *destination) const -{ - this->Allgather(source, sourceCount, destination, sourceCount); -} - // ReduceValues full specializations forward-declared in 'adiosComm.inl'. template <> unsigned int Comm::ReduceValues(const unsigned int source, MPI_Op operation, From a1b5a40f1f1434ed67e89c079dd350a443e2bd61 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 12 Sep 2019 14:21:41 -0400 Subject: [PATCH 18/34] helper: Move private GetGathervDisplacements into Comm --- source/adios2/helper/adiosComm.cpp | 13 +++++++++++++ source/adios2/helper/adiosComm.h | 3 +++ source/adios2/helper/adiosComm.tcc | 18 ------------------ 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp index 83e70b8f78..8a5d560da8 100644 --- a/source/adios2/helper/adiosComm.cpp +++ b/source/adios2/helper/adiosComm.cpp @@ -58,6 +58,19 @@ std::string Comm::BroadcastFile(const std::string &fileName, return fileContents; } +std::vector Comm::GetGathervDisplacements(const size_t *counts, + const size_t countsSize) +{ + std::vector displacements(countsSize); + displacements[0] = 0; + + for (size_t i = 1; i < countsSize; ++i) + { + displacements[i] = displacements[i - 1] + counts[i - 1]; + } + return displacements; +} + Comm::Req::Req() = default; Comm::Req::Req(std::unique_ptr impl) : m_Impl(std::move(impl)) {} diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h index f5a1885fb2..6bf4cdec4e 100644 --- a/source/adios2/helper/adiosComm.h +++ b/source/adios2/helper/adiosComm.h @@ -216,6 +216,9 @@ class Comm /** Return MPI datatype id for type T. */ template static MPI_Datatype Datatype(); + + static std::vector GetGathervDisplacements(const size_t *counts, + const size_t countsSize); }; class Comm::Req diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc index b11ea41f86..bcd82ddf21 100644 --- a/source/adios2/helper/adiosComm.tcc +++ b/source/adios2/helper/adiosComm.tcc @@ -18,24 +18,6 @@ namespace adios2 namespace helper { -namespace -{ - -std::vector GetGathervDisplacements(const size_t *counts, - const size_t countsSize) -{ - std::vector displacements(countsSize); - displacements[0] = 0; - - for (size_t i = 1; i < countsSize; ++i) - { - displacements[i] = displacements[i - 1] + counts[i - 1]; - } - return displacements; -} - -} - // GatherArrays full specializations forward-declared in 'adiosComm.inl'. template <> void Comm::GatherArrays(const char *source, size_t sourceCount, From 93045489912220b58eb5fe8abf1d61fc7c193a4c Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 12 Sep 2019 14:24:34 -0400 Subject: [PATCH 19/34] helper: Generalize Comm::GathervArrays implementation With our general `Gatherv` interface we no longer need a per-type specialization of `GathervArrays`. --- source/adios2/helper/adiosComm.h | 2 -- source/adios2/helper/adiosComm.inl | 20 ++++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h index 6bf4cdec4e..82398390b2 100644 --- a/source/adios2/helper/adiosComm.h +++ b/source/adios2/helper/adiosComm.h @@ -112,8 +112,6 @@ class Comm /** * Gather arrays of the same type into a destination (must be pre-allocated) - * if countsSize == 1, calls MPI_Gather, otherwise calls MPI_Gatherv. - * This function must be specialized for each MPI_Type. * @param source input from each rank * @param counts counts for each source * @param countsSize number of counts diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index 385a0678cb..0cf52ab326 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -47,15 +47,19 @@ std::vector Comm::GatherValues(T source, int rankDestination) const return output; } -// GathervArrays full specializations implemented in 'adiosComm.tcc'. -template <> -void Comm::GathervArrays(const char *source, size_t sourceCount, - const size_t *counts, size_t countsSize, - char *destination, int rankDestination) const; -template <> -void Comm::GathervArrays(const size_t *source, size_t sourceCount, +template +void Comm::GathervArrays(const T *source, size_t sourceCount, const size_t *counts, size_t countsSize, - size_t *destination, int rankDestination) const; + T *destination, int rankDestination) const +{ + std::vector displs; + if (rankDestination == this->Rank()) + { + displs = GetGathervDisplacements(counts, countsSize); + } + this->Gatherv(source, sourceCount, destination, counts, displs.data(), + rankDestination); +} template void Comm::GathervVectors(const std::vector &in, std::vector &out, From 6fc159c037a15fc3152cdddf6c34f466d24b5bd6 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 12 Sep 2019 14:26:50 -0400 Subject: [PATCH 20/34] helper: Generalize Comm::GatherArrays implementation With our general `Gather` interface we no longer need a per-type specialization of `GatherArrays`. --- source/adios2/helper/adiosComm.inl | 14 +++++++------- source/adios2/helper/adiosComm.tcc | 17 ----------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index 0cf52ab326..55d0d9fdf5 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -20,13 +20,13 @@ namespace adios2 namespace helper { -// GatherArrays full specializations implemented in 'adiosComm.tcc'. -template <> -void Comm::GatherArrays(const char *source, size_t sourceCount, - char *destination, int rankDestination) const; -template <> -void Comm::GatherArrays(const size_t *source, size_t sourceCount, - size_t *destination, int rankDestination) const; +template +void Comm::GatherArrays(const T *source, size_t sourceCount, T *destination, + int rankDestination) const +{ + this->Gather(source, sourceCount, destination, sourceCount, + rankDestination); +} template std::vector Comm::GatherValues(T source, int rankDestination) const diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc index bcd82ddf21..376bb3fd3c 100644 --- a/source/adios2/helper/adiosComm.tcc +++ b/source/adios2/helper/adiosComm.tcc @@ -18,23 +18,6 @@ namespace adios2 namespace helper { -// GatherArrays full specializations forward-declared in 'adiosComm.inl'. -template <> -void Comm::GatherArrays(const char *source, size_t sourceCount, - char *destination, int rankDestination) const -{ - this->Gather(source, sourceCount, destination, sourceCount, - rankDestination); -} - -template <> -void Comm::GatherArrays(const size_t *source, size_t sourceCount, - size_t *destination, int rankDestination) const -{ - this->Gather(source, sourceCount, destination, sourceCount, - rankDestination); -} - // GathervArrays full specializations forward-declared in 'adiosComm.inl'. template <> void Comm::GathervArrays(const char *source, size_t sourceCount, From d58189d4f9200f417dcde4f75890e37a16d6d365 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 12 Sep 2019 14:32:17 -0400 Subject: [PATCH 21/34] helper: Generalize Comm::ReduceValues implementation With our general `Reduce` interface we no longer need a per-type specialization of `ReduceValues`. --- source/adios2/helper/adiosComm.inl | 21 +++++----- source/adios2/helper/adiosComm.tcc | 62 ------------------------------ 2 files changed, 9 insertions(+), 74 deletions(-) diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index 55d0d9fdf5..640de3ec5f 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -109,18 +109,15 @@ std::vector Comm::AllGatherValues(const T source) const return output; } -// ReduceValues full specializations implemented in 'adiosComm.tcc'. -template <> -unsigned int Comm::ReduceValues(const unsigned int source, MPI_Op operation, - const int rankDestination) const; -template <> -unsigned long int Comm::ReduceValues(const unsigned long int source, - MPI_Op operation, - const int rankDestination) const; -template <> -unsigned long long int Comm::ReduceValues(const unsigned long long int source, - MPI_Op operation, - const int rankDestination) const; +template +T Comm::ReduceValues(const T source, MPI_Op operation, + const int rankDestination) const +{ + T sourceLocal = source; + T reduceValue = 0; + this->Reduce(&sourceLocal, &reduceValue, 1, operation, rankDestination); + return reduceValue; +} // BroadcastValue full specializations implemented in 'adiosComm.tcc'. template <> diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc index 376bb3fd3c..63806e973c 100644 --- a/source/adios2/helper/adiosComm.tcc +++ b/source/adios2/helper/adiosComm.tcc @@ -18,68 +18,6 @@ namespace adios2 namespace helper { -// GathervArrays full specializations forward-declared in 'adiosComm.inl'. -template <> -void Comm::GathervArrays(const char *source, size_t sourceCount, - const size_t *counts, size_t countsSize, - char *destination, int rankDestination) const -{ - std::vector displs; - if (rankDestination == this->Rank()) - { - displs = GetGathervDisplacements(counts, countsSize); - } - this->Gatherv(source, sourceCount, destination, counts, displs.data(), - rankDestination); -} - -template <> -void Comm::GathervArrays(const size_t *source, size_t sourceCount, - const size_t *counts, size_t countsSize, - size_t *destination, int rankDestination) const -{ - std::vector displs; - if (rankDestination == this->Rank()) - { - displs = GetGathervDisplacements(counts, countsSize); - } - this->Gatherv(source, sourceCount, destination, counts, displs.data(), - rankDestination); -} - -// ReduceValues full specializations forward-declared in 'adiosComm.inl'. -template <> -unsigned int Comm::ReduceValues(const unsigned int source, MPI_Op operation, - const int rankDestination) const -{ - unsigned int sourceLocal = source; - unsigned int reduceValue = 0; - this->Reduce(&sourceLocal, &reduceValue, 1, operation, rankDestination); - return reduceValue; -} - -template <> -unsigned long int Comm::ReduceValues(const unsigned long int source, - MPI_Op operation, - const int rankDestination) const -{ - unsigned long int sourceLocal = source; - unsigned long int reduceValue = 0; - this->Reduce(&sourceLocal, &reduceValue, 1, operation, rankDestination); - return reduceValue; -} - -template <> -unsigned long long int Comm::ReduceValues(const unsigned long long int source, - MPI_Op operation, - const int rankDestination) const -{ - unsigned long long int sourceLocal = source; - unsigned long long int reduceValue = 0; - this->Reduce(&sourceLocal, &reduceValue, 1, operation, rankDestination); - return reduceValue; -} - // BroadcastValue full specializations forward-declared in 'adiosComm.inl'. template <> size_t Comm::BroadcastValue(const size_t &input, const int rankSource) const From 0a216239a70c09db46254cc4a03fc0f106f86d04 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 12 Sep 2019 14:41:37 -0400 Subject: [PATCH 22/34] helper: Generalize Comm::BroadcastValue implementation With our general `Bcast` interface we no longer need a per-type specialization of `BroadcastValue`. --- source/adios2/helper/adiosComm.inl | 16 ++++++++++++++-- source/adios2/helper/adiosComm.tcc | 14 -------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index 640de3ec5f..36ddf401fe 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -119,10 +119,22 @@ T Comm::ReduceValues(const T source, MPI_Op operation, return reduceValue; } +template +T Comm::BroadcastValue(const T &input, const int rankSource) const +{ + T output = 0; + if (rankSource == this->Rank()) + { + output = input; + } + + this->Bcast(&output, 1, rankSource); + + return output; +} + // BroadcastValue full specializations implemented in 'adiosComm.tcc'. template <> -size_t Comm::BroadcastValue(const size_t &input, const int rankSource) const; -template <> std::string Comm::BroadcastValue(const std::string &input, const int rankSource) const; diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc index 63806e973c..5ade8ee207 100644 --- a/source/adios2/helper/adiosComm.tcc +++ b/source/adios2/helper/adiosComm.tcc @@ -19,20 +19,6 @@ namespace helper { // BroadcastValue full specializations forward-declared in 'adiosComm.inl'. -template <> -size_t Comm::BroadcastValue(const size_t &input, const int rankSource) const -{ - size_t output = 0; - if (rankSource == this->Rank()) - { - output = input; - } - - this->Bcast(&output, 1, rankSource); - - return output; -} - template <> std::string Comm::BroadcastValue(const std::string &input, const int rankSource) const From ccdc0845beef69b1f1603be409745c931f1e5f33 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 12 Sep 2019 14:43:17 -0400 Subject: [PATCH 23/34] helper: Generalize Comm::BroadcastVector implementation With our general `Bcast` interface we no longer need a per-type specialization of `BroadcastVector`. --- source/adios2/helper/adiosComm.inl | 25 +++++++++++++----- source/adios2/helper/adiosComm.tcc | 41 ------------------------------ 2 files changed, 18 insertions(+), 48 deletions(-) diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index 36ddf401fe..f32a982901 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -138,13 +138,24 @@ template <> std::string Comm::BroadcastValue(const std::string &input, const int rankSource) const; -// BroadcastVector full specializations implemented in 'adiosComm.tcc'. -template <> -void Comm::BroadcastVector(std::vector &vector, - const int rankSource) const; -template <> -void Comm::BroadcastVector(std::vector &vector, - const int rankSource) const; +template +void Comm::BroadcastVector(std::vector &vector, const int rankSource) const +{ + if (this->Size() == 1) + { + return; + } + + // First Broadcast the size, then the contents + size_t inputSize = this->BroadcastValue(vector.size(), rankSource); + + if (rankSource != this->Rank()) + { + vector.resize(inputSize); + } + + this->Bcast(vector.data(), inputSize, rankSource); +} template void Comm::Allgather(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc index 5ade8ee207..15aaeec6c8 100644 --- a/source/adios2/helper/adiosComm.tcc +++ b/source/adios2/helper/adiosComm.tcc @@ -41,47 +41,6 @@ std::string Comm::BroadcastValue(const std::string &input, return output; } -// BroadcastVector full specializations forward-declared in 'adiosComm.inl'. -template <> -void Comm::BroadcastVector(std::vector &vector, - const int rankSource) const -{ - if (this->Size() == 1) - { - return; - } - - // First Broadcast the size, then the contents - size_t inputSize = this->BroadcastValue(vector.size(), rankSource); - - if (rankSource != this->Rank()) - { - vector.resize(inputSize); - } - - this->Bcast(vector.data(), inputSize, rankSource); -} - -template <> -void Comm::BroadcastVector(std::vector &vector, - const int rankSource) const -{ - if (this->Size() == 1) - { - return; - } - - // First Broadcast the size, then the contents - size_t inputSize = this->BroadcastValue(vector.size(), rankSource); - - if (rankSource != this->Rank()) - { - vector.resize(inputSize); - } - - this->Bcast(vector.data(), inputSize, rankSource); -} - // Datatype full specializations forward-declared in 'adiosComm.inl'. template <> MPI_Datatype Comm::Datatype() From c85b97b77157a4f21055dfda1d45447b4e0bf651 Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 13 Sep 2019 14:11:46 -0400 Subject: [PATCH 24/34] helper: Encapsulate MPI_Op as Comm::Op --- .../engine/insitumpi/InSituMPIReader.cpp | 13 ++++--- source/adios2/helper/adiosComm.h | 36 +++++++++++++++---- source/adios2/helper/adiosComm.inl | 13 ++++--- source/adios2/helper/adiosCommMPI.cpp | 27 +++++++++----- .../format/dataman/DataManSerializer.cpp | 2 +- 5 files changed, 62 insertions(+), 29 deletions(-) diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.cpp b/source/adios2/engine/insitumpi/InSituMPIReader.cpp index 776a5909de..93c3798b21 100644 --- a/source/adios2/engine/insitumpi/InSituMPIReader.cpp +++ b/source/adios2/engine/insitumpi/InSituMPIReader.cpp @@ -192,7 +192,7 @@ StepStatus InSituMPIReader::BeginStep(const StepMode mode, } /* Exchange steps */ int maxstep; - m_Comm.Allreduce(&step, &maxstep, 1, MPI_MAX); + m_Comm.Allreduce(&step, &maxstep, 1, helper::Comm::Op::Max); if (m_Verbosity == 5 && !m_ReaderRank) { @@ -423,12 +423,13 @@ void InSituMPIReader::SendReadSchedule( if (m_ReaderRootRank == m_ReaderRank) { m_Comm.ReduceInPlace(nReaderPerWriter.data(), nReaderPerWriter.size(), - MPI_SUM, m_ReaderRootRank); + helper::Comm::Op::Sum, m_ReaderRootRank); } else { m_Comm.Reduce(nReaderPerWriter.data(), nReaderPerWriter.data(), - nReaderPerWriter.size(), MPI_SUM, m_ReaderRootRank); + nReaderPerWriter.size(), helper::Comm::Op::Sum, + m_ReaderRootRank); } // Reader root sends nReaderPerWriter to writer root @@ -593,8 +594,10 @@ void InSituMPIReader::DoClose(const int transportIndex) if (m_Verbosity > 2) { uint64_t inPlaceBytes, inTempBytes; - m_Comm.Reduce(&m_BytesReceivedInPlace, &inPlaceBytes, 1, MPI_SUM, 0); - m_Comm.Reduce(&m_BytesReceivedInTemporary, &inTempBytes, 1, MPI_SUM, 0); + m_Comm.Reduce(&m_BytesReceivedInPlace, &inPlaceBytes, 1, + helper::Comm::Op::Sum, 0); + m_Comm.Reduce(&m_BytesReceivedInTemporary, &inTempBytes, 1, + helper::Comm::Op::Sum, 0); if (m_ReaderRank == 0) { std::cout << "ADIOS InSituMPI Reader for " << m_Name << " received " diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h index 82398390b2..4b5e2b2051 100644 --- a/source/adios2/helper/adiosComm.h +++ b/source/adios2/helper/adiosComm.h @@ -29,6 +29,28 @@ class Comm class Req; class Status; + /** + * @brief Enumeration of element-wise accumulation operations. + */ + enum class Op + { + Null, + Max, + Min, + Sum, + Product, + LogicalAnd, + BitwiseAnd, + LogicalOr, + BitwiseOr, + LogicalXor, + BitwiseXor, + MaxLoc, + MinLoc, + Replace, + None, + }; + /** * @brief Default constructor. Produces an empty communicator. * @@ -138,7 +160,7 @@ class Comm std::vector AllGatherValues(const T source) const; template - T ReduceValues(const T source, MPI_Op operation = MPI_SUM, + T ReduceValues(const T source, Op op = Op::Sum, const int rankDestination = 0) const; template @@ -158,7 +180,7 @@ class Comm const std::string &hint = std::string()) const; template - void Allreduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op, + void Allreduce(const T *sendbuf, T *recvbuf, size_t count, Op op, const std::string &hint = std::string()) const; template @@ -176,11 +198,11 @@ class Comm const std::string &hint = std::string()) const; template - void Reduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op, int root, + void Reduce(const T *sendbuf, T *recvbuf, size_t count, Op op, int root, const std::string &hint = std::string()) const; template - void ReduceInPlace(T *buf, size_t count, MPI_Op op, int root, + void ReduceInPlace(T *buf, size_t count, Op op, int root, const std::string &hint = std::string()) const; template @@ -310,7 +332,7 @@ class CommImpl const std::string &hint) const = 0; virtual void Allreduce(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, MPI_Op op, + MPI_Datatype datatype, Comm::Op op, const std::string &hint) const = 0; virtual void Bcast(void *buffer, size_t count, MPI_Datatype datatype, @@ -329,11 +351,11 @@ class CommImpl const std::string &hint) const = 0; virtual void Reduce(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, MPI_Op op, int root, + MPI_Datatype datatype, Comm::Op op, int root, const std::string &hint) const = 0; virtual void ReduceInPlace(void *buf, size_t count, MPI_Datatype datatype, - MPI_Op op, int root, + Comm::Op op, int root, const std::string &hint) const = 0; virtual void Send(const void *buf, size_t count, MPI_Datatype datatype, diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index f32a982901..a54c072209 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -110,12 +110,11 @@ std::vector Comm::AllGatherValues(const T source) const } template -T Comm::ReduceValues(const T source, MPI_Op operation, - const int rankDestination) const +T Comm::ReduceValues(const T source, Op op, const int rankDestination) const { T sourceLocal = source; T reduceValue = 0; - this->Reduce(&sourceLocal, &reduceValue, 1, operation, rankDestination); + this->Reduce(&sourceLocal, &reduceValue, 1, op, rankDestination); return reduceValue; } @@ -166,7 +165,7 @@ void Comm::Allgather(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, } template -void Comm::Allreduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op, +void Comm::Allreduce(const T *sendbuf, T *recvbuf, size_t count, Op op, const std::string &hint) const { return m_Impl->Allreduce(sendbuf, recvbuf, count, Datatype(), op, hint); @@ -197,15 +196,15 @@ void Comm::Gatherv(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, } template -void Comm::Reduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op, - int root, const std::string &hint) const +void Comm::Reduce(const T *sendbuf, T *recvbuf, size_t count, Op op, int root, + const std::string &hint) const { return m_Impl->Reduce(sendbuf, recvbuf, count, Datatype(), op, root, hint); } template -void Comm::ReduceInPlace(T *buf, size_t count, MPI_Op op, int root, +void Comm::ReduceInPlace(T *buf, size_t count, Op op, int root, const std::string &hint) const { return m_Impl->ReduceInPlace(buf, count, Datatype(), op, root, hint); diff --git a/source/adios2/helper/adiosCommMPI.cpp b/source/adios2/helper/adiosCommMPI.cpp index 0d550d104f..582a8c152d 100644 --- a/source/adios2/helper/adiosCommMPI.cpp +++ b/source/adios2/helper/adiosCommMPI.cpp @@ -24,6 +24,15 @@ namespace helper namespace { + +const MPI_Op OpToMPI[] = { + MPI_OP_NULL, MPI_MAX, MPI_MIN, MPI_SUM, MPI_PROD, + MPI_LAND, MPI_BAND, MPI_LOR, MPI_BOR, MPI_LXOR, + MPI_BXOR, MPI_MAXLOC, MPI_MINLOC, MPI_REPLACE, MPI_NO_OP, +}; + +MPI_Op ToMPI(Comm::Op op) { return OpToMPI[int(op)]; } + void CheckMPIReturn(const int value, const std::string &hint) { if (value == MPI_SUCCESS) @@ -88,7 +97,7 @@ class CommImplMPI : public CommImpl const std::string &hint) const override; void Allreduce(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, MPI_Op op, + MPI_Datatype datatype, Comm::Op op, const std::string &hint) const override; void Bcast(void *buffer, size_t count, MPI_Datatype datatype, @@ -105,11 +114,11 @@ class CommImplMPI : public CommImpl const std::string &hint) const override; void Reduce(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, MPI_Op op, int root, + MPI_Datatype datatype, Comm::Op op, int root, const std::string &hint) const override; void ReduceInPlace(void *buf, size_t count, MPI_Datatype datatype, - MPI_Op op, int root, + Comm::Op op, int root, const std::string &hint) const override; void Send(const void *buf, size_t count, MPI_Datatype datatype, int dest, @@ -195,11 +204,11 @@ void CommImplMPI::Allgather(const void *sendbuf, size_t sendcount, } void CommImplMPI::Allreduce(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, MPI_Op op, + MPI_Datatype datatype, Comm::Op op, const std::string &hint) const { CheckMPIReturn(SMPI_Allreduce(sendbuf, recvbuf, static_cast(count), - datatype, op, m_MPIComm), + datatype, ToMPI(op), m_MPIComm), hint); } @@ -259,20 +268,20 @@ void CommImplMPI::Gatherv(const void *sendbuf, size_t sendcount, } void CommImplMPI::Reduce(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, MPI_Op op, int root, + MPI_Datatype datatype, Comm::Op op, int root, const std::string &hint) const { CheckMPIReturn(SMPI_Reduce(sendbuf, recvbuf, static_cast(count), - datatype, op, root, m_MPIComm), + datatype, ToMPI(op), root, m_MPIComm), hint); } void CommImplMPI::ReduceInPlace(void *buf, size_t count, MPI_Datatype datatype, - MPI_Op op, int root, + Comm::Op op, int root, const std::string &hint) const { CheckMPIReturn(SMPI_Reduce(MPI_IN_PLACE, buf, static_cast(count), - datatype, op, root, m_MPIComm), + datatype, ToMPI(op), root, m_MPIComm), hint); } diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp index f09245a17f..f4890b534a 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp @@ -72,7 +72,7 @@ void DataManSerializer::AggregateMetadata() auto localJsonPack = SerializeJson(m_MetadataJson); unsigned int size = localJsonPack->size(); unsigned int maxSize; - m_Comm.Allreduce(&size, &maxSize, 1, MPI_MAX); + m_Comm.Allreduce(&size, &maxSize, 1, helper::Comm::Op::Max); maxSize += sizeof(uint64_t); localJsonPack->resize(maxSize, '\0'); *(reinterpret_cast(localJsonPack->data() + From 4e76dbc7190a8532b5b8e5d2a2dcc5e8413ab5be Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 13 Sep 2019 14:36:35 -0400 Subject: [PATCH 25/34] helper: Encapsulate MPI_Datatype as CommImpl::Datatype --- source/adios2/helper/adiosComm.h | 65 ++++++---- source/adios2/helper/adiosComm.inl | 83 ++++++------ source/adios2/helper/adiosComm.tcc | 77 ++++++----- source/adios2/helper/adiosCommMPI.cpp | 177 +++++++++++++++----------- 4 files changed, 229 insertions(+), 173 deletions(-) diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h index 4b5e2b2051..7296be0f36 100644 --- a/source/adios2/helper/adiosComm.h +++ b/source/adios2/helper/adiosComm.h @@ -8,8 +8,6 @@ #ifndef ADIOS2_HELPER_ADIOSCOMM_H_ #define ADIOS2_HELPER_ADIOSCOMM_H_ -#include "adios2/common/ADIOSMPI.h" - #include #include #include @@ -233,10 +231,6 @@ class Comm std::unique_ptr m_Impl; - /** Return MPI datatype id for type T. */ - template - static MPI_Datatype Datatype(); - static std::vector GetGathervDisplacements(const size_t *counts, const size_t countsSize); }; @@ -319,6 +313,31 @@ class Comm::Status class CommImpl { public: + enum class Datatype + { + SignedChar, + Char, + Short, + Int, + Long, + UnsignedChar, + UnsignedShort, + UnsignedInt, + UnsignedLong, + UnsignedLongLong, + LongLong, + Double, + LongDouble, + Int_Int, + Float_Int, + Double_Int, + LongDouble_Int, + Short_Int, + }; + + template + static Datatype GetDatatype(); + virtual ~CommImpl() = 0; virtual void Free(const std::string &hint) = 0; virtual std::unique_ptr Split(int color, int key, @@ -327,54 +346,54 @@ class CommImpl virtual int Size() const = 0; virtual void Barrier(const std::string &hint) const = 0; virtual void Allgather(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, - size_t recvcount, MPI_Datatype recvtype, + Datatype sendtype, void *recvbuf, size_t recvcount, + Datatype recvtype, const std::string &hint) const = 0; virtual void Allreduce(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, Comm::Op op, + Datatype datatype, Comm::Op op, const std::string &hint) const = 0; - virtual void Bcast(void *buffer, size_t count, MPI_Datatype datatype, + virtual void Bcast(void *buffer, size_t count, Datatype datatype, size_t datatypeSize, int root, const std::string &hint) const = 0; virtual void Gather(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, size_t recvcount, - MPI_Datatype recvtype, int root, + Datatype sendtype, void *recvbuf, size_t recvcount, + Datatype recvtype, int root, const std::string &hint) const = 0; virtual void Gatherv(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, + Datatype sendtype, void *recvbuf, const size_t *recvcounts, const size_t *displs, - MPI_Datatype recvtype, int root, + Datatype recvtype, int root, const std::string &hint) const = 0; virtual void Reduce(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, Comm::Op op, int root, + Datatype datatype, Comm::Op op, int root, const std::string &hint) const = 0; - virtual void ReduceInPlace(void *buf, size_t count, MPI_Datatype datatype, + virtual void ReduceInPlace(void *buf, size_t count, Datatype datatype, Comm::Op op, int root, const std::string &hint) const = 0; - virtual void Send(const void *buf, size_t count, MPI_Datatype datatype, + virtual void Send(const void *buf, size_t count, Datatype datatype, int dest, int tag, const std::string &hint) const = 0; - virtual Comm::Status Recv(void *buf, size_t count, MPI_Datatype datatype, + virtual Comm::Status Recv(void *buf, size_t count, Datatype datatype, int source, int tag, const std::string &hint) const = 0; virtual void Scatter(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, size_t recvcount, - MPI_Datatype recvtype, int root, + Datatype sendtype, void *recvbuf, size_t recvcount, + Datatype recvtype, int root, const std::string &hint) const = 0; - virtual Comm::Req Isend(const void *buffer, size_t count, - MPI_Datatype datatype, int dest, int tag, + virtual Comm::Req Isend(const void *buffer, size_t count, Datatype datatype, + int dest, int tag, const std::string &hint) const = 0; - virtual Comm::Req Irecv(void *buffer, size_t count, MPI_Datatype datatype, + virtual Comm::Req Irecv(void *buffer, size_t count, Datatype datatype, int source, int tag, const std::string &hint) const = 0; diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index a54c072209..9dd14a9182 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -160,30 +160,34 @@ template void Comm::Allgather(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, size_t recvcount, const std::string &hint) const { - return m_Impl->Allgather(sendbuf, sendcount, Datatype(), recvbuf, - recvcount, Datatype(), hint); + return m_Impl->Allgather(sendbuf, sendcount, CommImpl::GetDatatype(), + recvbuf, recvcount, CommImpl::GetDatatype(), + hint); } template void Comm::Allreduce(const T *sendbuf, T *recvbuf, size_t count, Op op, const std::string &hint) const { - return m_Impl->Allreduce(sendbuf, recvbuf, count, Datatype(), op, hint); + return m_Impl->Allreduce(sendbuf, recvbuf, count, + CommImpl::GetDatatype(), op, hint); } template void Comm::Bcast(T *buffer, const size_t count, int root, const std::string &hint) const { - return m_Impl->Bcast(buffer, count, Datatype(), sizeof(T), root, hint); + return m_Impl->Bcast(buffer, count, CommImpl::GetDatatype(), sizeof(T), + root, hint); } template void Comm::Gather(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, size_t recvcount, int root, const std::string &hint) const { - return m_Impl->Gather(sendbuf, sendcount, Datatype(), recvbuf, - recvcount, Datatype(), root, hint); + return m_Impl->Gather(sendbuf, sendcount, CommImpl::GetDatatype(), + recvbuf, recvcount, CommImpl::GetDatatype(), + root, hint); } template @@ -191,98 +195,105 @@ void Comm::Gatherv(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, const size_t *recvcounts, const size_t *displs, int root, const std::string &hint) const { - return m_Impl->Gatherv(sendbuf, sendcount, Datatype(), recvbuf, - recvcounts, displs, Datatype(), root, hint); + return m_Impl->Gatherv(sendbuf, sendcount, CommImpl::GetDatatype(), + recvbuf, recvcounts, displs, + CommImpl::GetDatatype(), root, hint); } template void Comm::Reduce(const T *sendbuf, T *recvbuf, size_t count, Op op, int root, const std::string &hint) const { - return m_Impl->Reduce(sendbuf, recvbuf, count, Datatype(), op, root, - hint); + return m_Impl->Reduce(sendbuf, recvbuf, count, CommImpl::GetDatatype(), + op, root, hint); } template void Comm::ReduceInPlace(T *buf, size_t count, Op op, int root, const std::string &hint) const { - return m_Impl->ReduceInPlace(buf, count, Datatype(), op, root, hint); + return m_Impl->ReduceInPlace(buf, count, CommImpl::GetDatatype(), op, + root, hint); } template void Comm::Send(const T *buf, size_t count, int dest, int tag, const std::string &hint) const { - return m_Impl->Send(buf, count, Datatype(), dest, tag, hint); + return m_Impl->Send(buf, count, CommImpl::GetDatatype(), dest, tag, + hint); } template Comm::Status Comm::Recv(T *buf, size_t count, int source, int tag, const std::string &hint) const { - return m_Impl->Recv(buf, count, Datatype(), source, tag, hint); + return m_Impl->Recv(buf, count, CommImpl::GetDatatype(), source, tag, + hint); } template void Comm::Scatter(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf, size_t recvcount, int root, const std::string &hint) const { - return m_Impl->Scatter(sendbuf, sendcount, Datatype(), recvbuf, - recvcount, Datatype(), root, hint); + return m_Impl->Scatter(sendbuf, sendcount, CommImpl::GetDatatype(), + recvbuf, recvcount, CommImpl::GetDatatype(), + root, hint); } template Comm::Req Comm::Isend(const T *buffer, const size_t count, int dest, int tag, const std::string &hint) const { - return m_Impl->Isend(buffer, count, Datatype(), dest, tag, hint); + return m_Impl->Isend(buffer, count, CommImpl::GetDatatype(), dest, tag, + hint); } template Comm::Req Comm::Irecv(T *buffer, const size_t count, int source, int tag, const std::string &hint) const { - return m_Impl->Irecv(buffer, count, Datatype(), source, tag, hint); + return m_Impl->Irecv(buffer, count, CommImpl::GetDatatype(), source, tag, + hint); } -// Datatype full specializations implemented in 'adiosComm.tcc'. +// CommImpl::GetDatatype full specializations implemented in 'adiosComm.tcc'. template <> -MPI_Datatype Comm::Datatype(); +CommImpl::Datatype CommImpl::GetDatatype(); template <> -MPI_Datatype Comm::Datatype(); +CommImpl::Datatype CommImpl::GetDatatype(); template <> -MPI_Datatype Comm::Datatype(); +CommImpl::Datatype CommImpl::GetDatatype(); template <> -MPI_Datatype Comm::Datatype(); +CommImpl::Datatype CommImpl::GetDatatype(); template <> -MPI_Datatype Comm::Datatype(); +CommImpl::Datatype CommImpl::GetDatatype(); template <> -MPI_Datatype Comm::Datatype(); +CommImpl::Datatype CommImpl::GetDatatype(); template <> -MPI_Datatype Comm::Datatype(); +CommImpl::Datatype CommImpl::GetDatatype(); template <> -MPI_Datatype Comm::Datatype(); +CommImpl::Datatype CommImpl::GetDatatype(); template <> -MPI_Datatype Comm::Datatype(); +CommImpl::Datatype CommImpl::GetDatatype(); template <> -MPI_Datatype Comm::Datatype(); +CommImpl::Datatype CommImpl::GetDatatype(); template <> -MPI_Datatype Comm::Datatype(); +CommImpl::Datatype CommImpl::GetDatatype(); template <> -MPI_Datatype Comm::Datatype(); +CommImpl::Datatype CommImpl::GetDatatype(); template <> -MPI_Datatype Comm::Datatype(); +CommImpl::Datatype CommImpl::GetDatatype(); template <> -MPI_Datatype Comm::Datatype>(); +CommImpl::Datatype CommImpl::GetDatatype>(); template <> -MPI_Datatype Comm::Datatype>(); +CommImpl::Datatype CommImpl::GetDatatype>(); template <> -MPI_Datatype Comm::Datatype>(); +CommImpl::Datatype CommImpl::GetDatatype>(); template <> -MPI_Datatype Comm::Datatype>(); +CommImpl::Datatype CommImpl::GetDatatype>(); template <> -MPI_Datatype Comm::Datatype>(); +CommImpl::Datatype CommImpl::GetDatatype>(); } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc index 15aaeec6c8..ca2d769163 100644 --- a/source/adios2/helper/adiosComm.tcc +++ b/source/adios2/helper/adiosComm.tcc @@ -11,8 +11,6 @@ #include "adiosComm.h" -#include "adios2/common/ADIOSMPI.h" - namespace adios2 { namespace helper @@ -41,113 +39,114 @@ std::string Comm::BroadcastValue(const std::string &input, return output; } -// Datatype full specializations forward-declared in 'adiosComm.inl'. +// Comm::Impl::GetDatatype full specializations forward-declared in +// 'adiosComm.inl'. template <> -MPI_Datatype Comm::Datatype() +CommImpl::Datatype CommImpl::GetDatatype() { - return MPI_SIGNED_CHAR; + return CommImpl::Datatype::SignedChar; } template <> -MPI_Datatype Comm::Datatype() +CommImpl::Datatype CommImpl::GetDatatype() { - return MPI_CHAR; + return CommImpl::Datatype::Char; } template <> -MPI_Datatype Comm::Datatype() +CommImpl::Datatype CommImpl::GetDatatype() { - return MPI_SHORT; + return CommImpl::Datatype::Short; } template <> -MPI_Datatype Comm::Datatype() +CommImpl::Datatype CommImpl::GetDatatype() { - return MPI_INT; + return CommImpl::Datatype::Int; } template <> -MPI_Datatype Comm::Datatype() +CommImpl::Datatype CommImpl::GetDatatype() { - return MPI_LONG; + return CommImpl::Datatype::Long; } template <> -MPI_Datatype Comm::Datatype() +CommImpl::Datatype CommImpl::GetDatatype() { - return MPI_UNSIGNED_CHAR; + return CommImpl::Datatype::UnsignedChar; } template <> -MPI_Datatype Comm::Datatype() +CommImpl::Datatype CommImpl::GetDatatype() { - return MPI_UNSIGNED_SHORT; + return CommImpl::Datatype::UnsignedShort; } template <> -MPI_Datatype Comm::Datatype() +CommImpl::Datatype CommImpl::GetDatatype() { - return MPI_UNSIGNED; + return CommImpl::Datatype::UnsignedInt; } template <> -MPI_Datatype Comm::Datatype() +CommImpl::Datatype CommImpl::GetDatatype() { - return MPI_UNSIGNED_LONG; + return CommImpl::Datatype::UnsignedLong; } template <> -MPI_Datatype Comm::Datatype() +CommImpl::Datatype CommImpl::GetDatatype() { - return MPI_UNSIGNED_LONG_LONG; + return CommImpl::Datatype::UnsignedLongLong; } template <> -MPI_Datatype Comm::Datatype() +CommImpl::Datatype CommImpl::GetDatatype() { - return MPI_LONG_LONG_INT; + return CommImpl::Datatype::LongLong; } template <> -MPI_Datatype Comm::Datatype() +CommImpl::Datatype CommImpl::GetDatatype() { - return MPI_DOUBLE; + return CommImpl::Datatype::Double; } template <> -MPI_Datatype Comm::Datatype() +CommImpl::Datatype CommImpl::GetDatatype() { - return MPI_LONG_DOUBLE; + return CommImpl::Datatype::LongDouble; } template <> -MPI_Datatype Comm::Datatype>() +CommImpl::Datatype CommImpl::GetDatatype>() { - return MPI_2INT; + return CommImpl::Datatype::Int_Int; } template <> -MPI_Datatype Comm::Datatype>() +CommImpl::Datatype CommImpl::GetDatatype>() { - return MPI_FLOAT_INT; + return CommImpl::Datatype::Float_Int; } template <> -MPI_Datatype Comm::Datatype>() +CommImpl::Datatype CommImpl::GetDatatype>() { - return MPI_DOUBLE_INT; + return CommImpl::Datatype::Double_Int; } template <> -MPI_Datatype Comm::Datatype>() +CommImpl::Datatype CommImpl::GetDatatype>() { - return MPI_LONG_DOUBLE_INT; + return CommImpl::Datatype::LongDouble_Int; } template <> -MPI_Datatype Comm::Datatype>() +CommImpl::Datatype CommImpl::GetDatatype>() { - return MPI_SHORT_INT; + return CommImpl::Datatype::Short_Int; } } // end namespace helper diff --git a/source/adios2/helper/adiosCommMPI.cpp b/source/adios2/helper/adiosCommMPI.cpp index 582a8c152d..2bde011456 100644 --- a/source/adios2/helper/adiosCommMPI.cpp +++ b/source/adios2/helper/adiosCommMPI.cpp @@ -33,6 +33,29 @@ const MPI_Op OpToMPI[] = { MPI_Op ToMPI(Comm::Op op) { return OpToMPI[int(op)]; } +const MPI_Datatype DatatypeToMPI[] = { + MPI_SIGNED_CHAR, + MPI_CHAR, + MPI_SHORT, + MPI_INT, + MPI_LONG, + MPI_UNSIGNED_CHAR, + MPI_UNSIGNED_SHORT, + MPI_UNSIGNED, + MPI_UNSIGNED_LONG, + MPI_UNSIGNED_LONG_LONG, + MPI_LONG_LONG_INT, + MPI_DOUBLE, + MPI_LONG_DOUBLE, + MPI_2INT, + MPI_FLOAT_INT, + MPI_DOUBLE_INT, + MPI_LONG_DOUBLE_INT, + MPI_SHORT_INT, +}; + +MPI_Datatype ToMPI(CommImpl::Datatype dt) { return DatatypeToMPI[int(dt)]; } + void CheckMPIReturn(const int value, const std::string &hint) { if (value == MPI_SUCCESS) @@ -92,52 +115,49 @@ class CommImplMPI : public CommImpl int Size() const override; void Barrier(const std::string &hint) const override; - void Allgather(const void *sendbuf, size_t sendcount, MPI_Datatype sendtype, - void *recvbuf, size_t recvcount, MPI_Datatype recvtype, + void Allgather(const void *sendbuf, size_t sendcount, Datatype sendtype, + void *recvbuf, size_t recvcount, Datatype recvtype, const std::string &hint) const override; void Allreduce(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, Comm::Op op, + Datatype datatype, Comm::Op op, const std::string &hint) const override; - void Bcast(void *buffer, size_t count, MPI_Datatype datatype, + void Bcast(void *buffer, size_t count, Datatype datatype, size_t datatypeSize, int root, const std::string &hint) const override; - void Gather(const void *sendbuf, size_t sendcount, MPI_Datatype sendtype, - void *recvbuf, size_t recvcount, MPI_Datatype recvtype, - int root, const std::string &hint) const override; + void Gather(const void *sendbuf, size_t sendcount, Datatype sendtype, + void *recvbuf, size_t recvcount, Datatype recvtype, int root, + const std::string &hint) const override; - void Gatherv(const void *sendbuf, size_t sendcount, MPI_Datatype sendtype, + void Gatherv(const void *sendbuf, size_t sendcount, Datatype sendtype, void *recvbuf, const size_t *recvcounts, const size_t *displs, - MPI_Datatype recvtype, int root, + Datatype recvtype, int root, const std::string &hint) const override; void Reduce(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, Comm::Op op, int root, + Datatype datatype, Comm::Op op, int root, const std::string &hint) const override; - void ReduceInPlace(void *buf, size_t count, MPI_Datatype datatype, - Comm::Op op, int root, - const std::string &hint) const override; + void ReduceInPlace(void *buf, size_t count, Datatype datatype, Comm::Op op, + int root, const std::string &hint) const override; - void Send(const void *buf, size_t count, MPI_Datatype datatype, int dest, + void Send(const void *buf, size_t count, Datatype datatype, int dest, int tag, const std::string &hint) const override; - Comm::Status Recv(void *buf, size_t count, MPI_Datatype datatype, - int source, int tag, - const std::string &hint) const override; + Comm::Status Recv(void *buf, size_t count, Datatype datatype, int source, + int tag, const std::string &hint) const override; - void Scatter(const void *sendbuf, size_t sendcount, MPI_Datatype sendtype, - void *recvbuf, size_t recvcount, MPI_Datatype recvtype, - int root, const std::string &hint) const override; + void Scatter(const void *sendbuf, size_t sendcount, Datatype sendtype, + void *recvbuf, size_t recvcount, Datatype recvtype, int root, + const std::string &hint) const override; - Comm::Req Isend(const void *buffer, size_t count, MPI_Datatype datatype, + Comm::Req Isend(const void *buffer, size_t count, Datatype datatype, int dest, int tag, const std::string &hint) const override; - Comm::Req Irecv(void *buffer, size_t count, MPI_Datatype datatype, - int source, int tag, - const std::string &hint) const override; + Comm::Req Irecv(void *buffer, size_t count, Datatype datatype, int source, + int tag, const std::string &hint) const override; }; CommImplMPI::~CommImplMPI() @@ -193,26 +213,26 @@ void CommImplMPI::Barrier(const std::string &hint) const } void CommImplMPI::Allgather(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, - size_t recvcount, MPI_Datatype recvtype, - const std::string &hint) const + Datatype sendtype, void *recvbuf, size_t recvcount, + Datatype recvtype, const std::string &hint) const { - CheckMPIReturn( - SMPI_Allgather(sendbuf, static_cast(sendcount), sendtype, recvbuf, - static_cast(recvcount), recvtype, m_MPIComm), - hint); + CheckMPIReturn(SMPI_Allgather(sendbuf, static_cast(sendcount), + ToMPI(sendtype), recvbuf, + static_cast(recvcount), ToMPI(recvtype), + m_MPIComm), + hint); } void CommImplMPI::Allreduce(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, Comm::Op op, + Datatype datatype, Comm::Op op, const std::string &hint) const { CheckMPIReturn(SMPI_Allreduce(sendbuf, recvbuf, static_cast(count), - datatype, ToMPI(op), m_MPIComm), + ToMPI(datatype), ToMPI(op), m_MPIComm), hint); } -void CommImplMPI::Bcast(void *buffer, size_t count, MPI_Datatype datatype, +void CommImplMPI::Bcast(void *buffer, size_t count, Datatype datatype, size_t datatypeSize, int root, const std::string &hint) const { @@ -223,7 +243,7 @@ void CommImplMPI::Bcast(void *buffer, size_t count, MPI_Datatype datatype, while (inputSize > 0) { CheckMPIReturn(SMPI_Bcast(blockBuf, static_cast(blockSize), - datatype, root, m_MPIComm), + ToMPI(datatype), root, m_MPIComm), hint); blockBuf += blockSize * datatypeSize; inputSize -= blockSize; @@ -232,20 +252,21 @@ void CommImplMPI::Bcast(void *buffer, size_t count, MPI_Datatype datatype, } void CommImplMPI::Gather(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, size_t recvcount, - MPI_Datatype recvtype, int root, + Datatype sendtype, void *recvbuf, size_t recvcount, + Datatype recvtype, int root, const std::string &hint) const { - CheckMPIReturn(SMPI_Gather(sendbuf, static_cast(sendcount), sendtype, - recvbuf, static_cast(recvcount), recvtype, + CheckMPIReturn(SMPI_Gather(sendbuf, static_cast(sendcount), + ToMPI(sendtype), recvbuf, + static_cast(recvcount), ToMPI(recvtype), root, m_MPIComm), hint); } void CommImplMPI::Gatherv(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, + Datatype sendtype, void *recvbuf, const size_t *recvcounts, const size_t *displs, - MPI_Datatype recvtype, int root, + Datatype recvtype, int root, const std::string &hint) const { std::vector countsInt; @@ -261,45 +282,46 @@ void CommImplMPI::Gatherv(const void *sendbuf, size_t sendcount, std::transform(displs, displs + size, std::back_inserter(displsInt), cast); } - CheckMPIReturn(SMPI_Gatherv(sendbuf, static_cast(sendcount), sendtype, - recvbuf, countsInt.data(), displsInt.data(), - recvtype, root, m_MPIComm), + CheckMPIReturn(SMPI_Gatherv(sendbuf, static_cast(sendcount), + ToMPI(sendtype), recvbuf, countsInt.data(), + displsInt.data(), ToMPI(recvtype), root, + m_MPIComm), hint); } void CommImplMPI::Reduce(const void *sendbuf, void *recvbuf, size_t count, - MPI_Datatype datatype, Comm::Op op, int root, + Datatype datatype, Comm::Op op, int root, const std::string &hint) const { CheckMPIReturn(SMPI_Reduce(sendbuf, recvbuf, static_cast(count), - datatype, ToMPI(op), root, m_MPIComm), + ToMPI(datatype), ToMPI(op), root, m_MPIComm), hint); } -void CommImplMPI::ReduceInPlace(void *buf, size_t count, MPI_Datatype datatype, +void CommImplMPI::ReduceInPlace(void *buf, size_t count, Datatype datatype, Comm::Op op, int root, const std::string &hint) const { CheckMPIReturn(SMPI_Reduce(MPI_IN_PLACE, buf, static_cast(count), - datatype, ToMPI(op), root, m_MPIComm), + ToMPI(datatype), ToMPI(op), root, m_MPIComm), hint); } -void CommImplMPI::Send(const void *buf, size_t count, MPI_Datatype datatype, +void CommImplMPI::Send(const void *buf, size_t count, Datatype datatype, int dest, int tag, const std::string &hint) const { - CheckMPIReturn( - MPI_Send(buf, static_cast(count), datatype, dest, tag, m_MPIComm), - hint); + CheckMPIReturn(MPI_Send(buf, static_cast(count), ToMPI(datatype), dest, + tag, m_MPIComm), + hint); } -Comm::Status CommImplMPI::Recv(void *buf, size_t count, MPI_Datatype datatype, +Comm::Status CommImplMPI::Recv(void *buf, size_t count, Datatype datatype, int source, int tag, const std::string &hint) const { MPI_Status mpiStatus; - CheckMPIReturn(MPI_Recv(buf, static_cast(count), datatype, source, tag, - m_MPIComm, &mpiStatus), + CheckMPIReturn(MPI_Recv(buf, static_cast(count), ToMPI(datatype), + source, tag, m_MPIComm, &mpiStatus), hint); Comm::Status status; @@ -308,7 +330,8 @@ Comm::Status CommImplMPI::Recv(void *buf, size_t count, MPI_Datatype datatype, status.Tag = mpiStatus.MPI_TAG; { int mpiCount = 0; - CheckMPIReturn(MPI_Get_count(&mpiStatus, datatype, &mpiCount), hint); + CheckMPIReturn(MPI_Get_count(&mpiStatus, ToMPI(datatype), &mpiCount), + hint); status.Count = mpiCount; } #endif @@ -316,21 +339,23 @@ Comm::Status CommImplMPI::Recv(void *buf, size_t count, MPI_Datatype datatype, } void CommImplMPI::Scatter(const void *sendbuf, size_t sendcount, - MPI_Datatype sendtype, void *recvbuf, - size_t recvcount, MPI_Datatype recvtype, int root, + Datatype sendtype, void *recvbuf, size_t recvcount, + Datatype recvtype, int root, const std::string &hint) const { - CheckMPIReturn(MPI_Scatter(sendbuf, static_cast(sendcount), sendtype, - recvbuf, static_cast(recvcount), recvtype, + CheckMPIReturn(MPI_Scatter(sendbuf, static_cast(sendcount), + ToMPI(sendtype), recvbuf, + static_cast(recvcount), ToMPI(recvtype), root, m_MPIComm), hint); } Comm::Req CommImplMPI::Isend(const void *buffer, size_t count, - MPI_Datatype datatype, int dest, int tag, + Datatype datatype, int dest, int tag, const std::string &hint) const { - auto req = std::unique_ptr(new CommReqImplMPI(datatype)); + auto req = + std::unique_ptr(new CommReqImplMPI(ToMPI(datatype))); if (count > DefaultMaxFileBatchSize) { @@ -342,9 +367,9 @@ Comm::Req CommImplMPI::Isend(const void *buffer, size_t count, int batchSize = static_cast(DefaultMaxFileBatchSize); MPI_Request mpiReq; CheckMPIReturn( - MPI_Isend(static_cast(const_cast(buffer)) + - position, - batchSize, datatype, dest, tag, m_MPIComm, &mpiReq), + MPI_Isend( + static_cast(const_cast(buffer)) + position, + batchSize, ToMPI(datatype), dest, tag, m_MPIComm, &mpiReq), "in call to Isend batch " + std::to_string(b) + " " + hint + "\n"); req->m_MPIReqs.emplace_back(mpiReq); @@ -357,9 +382,9 @@ Comm::Req CommImplMPI::Isend(const void *buffer, size_t count, int batchSize = static_cast(remainder); MPI_Request mpiReq; CheckMPIReturn( - MPI_Isend(static_cast(const_cast(buffer)) + - position, - batchSize, datatype, dest, tag, m_MPIComm, &mpiReq), + MPI_Isend( + static_cast(const_cast(buffer)) + position, + batchSize, ToMPI(datatype), dest, tag, m_MPIComm, &mpiReq), "in call to Isend remainder batch " + hint + "\n"); req->m_MPIReqs.emplace_back(mpiReq); } @@ -370,7 +395,8 @@ Comm::Req CommImplMPI::Isend(const void *buffer, size_t count, MPI_Request mpiReq; CheckMPIReturn( MPI_Isend(static_cast(const_cast(buffer)), - batchSize, datatype, dest, tag, m_MPIComm, &mpiReq), + batchSize, ToMPI(datatype), dest, tag, m_MPIComm, + &mpiReq), " in call to Isend with single batch " + hint + "\n"); req->m_MPIReqs.emplace_back(mpiReq); } @@ -378,10 +404,11 @@ Comm::Req CommImplMPI::Isend(const void *buffer, size_t count, return MakeReq(std::move(req)); } -Comm::Req CommImplMPI::Irecv(void *buffer, size_t count, MPI_Datatype datatype, +Comm::Req CommImplMPI::Irecv(void *buffer, size_t count, Datatype datatype, int source, int tag, const std::string &hint) const { - auto req = std::unique_ptr(new CommReqImplMPI(datatype)); + auto req = + std::unique_ptr(new CommReqImplMPI(ToMPI(datatype))); if (count > DefaultMaxFileBatchSize) { @@ -392,7 +419,7 @@ Comm::Req CommImplMPI::Irecv(void *buffer, size_t count, MPI_Datatype datatype, int batchSize = static_cast(DefaultMaxFileBatchSize); MPI_Request mpiReq; CheckMPIReturn(MPI_Irecv(static_cast(buffer) + position, - batchSize, datatype, source, tag, + batchSize, ToMPI(datatype), source, tag, m_MPIComm, &mpiReq), "in call to Irecv batch " + std::to_string(b) + " " + hint + "\n"); @@ -407,7 +434,7 @@ Comm::Req CommImplMPI::Irecv(void *buffer, size_t count, MPI_Datatype datatype, int batchSize = static_cast(remainder); MPI_Request mpiReq; CheckMPIReturn(MPI_Irecv(static_cast(buffer) + position, - batchSize, datatype, source, tag, + batchSize, ToMPI(datatype), source, tag, m_MPIComm, &mpiReq), "in call to Irecv remainder batch " + hint + "\n"); req->m_MPIReqs.emplace_back(mpiReq); @@ -417,8 +444,8 @@ Comm::Req CommImplMPI::Irecv(void *buffer, size_t count, MPI_Datatype datatype, { int batchSize = static_cast(count); MPI_Request mpiReq; - CheckMPIReturn(MPI_Irecv(buffer, batchSize, datatype, source, tag, - m_MPIComm, &mpiReq), + CheckMPIReturn(MPI_Irecv(buffer, batchSize, ToMPI(datatype), source, + tag, m_MPIComm, &mpiReq), " in call to Isend with single batch " + hint + "\n"); req->m_MPIReqs.emplace_back(mpiReq); } From e5adee0f6881de893fc3d8670e7b70ac428fd809 Mon Sep 17 00:00:00 2001 From: Brad King Date: Mon, 16 Sep 2019 15:03:44 -0400 Subject: [PATCH 26/34] helper: Add internal data type size table to Comm Avoid explicitly passing `sizeof(T)` to `Bcast`. --- source/adios2/helper/adiosComm.cpp | 28 +++++++++++++++++++++++++++ source/adios2/helper/adiosComm.h | 5 +++-- source/adios2/helper/adiosComm.inl | 3 +-- source/adios2/helper/adiosCommMPI.cpp | 8 +++----- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp index 8a5d560da8..bc7f43b65b 100644 --- a/source/adios2/helper/adiosComm.cpp +++ b/source/adios2/helper/adiosComm.cpp @@ -17,6 +17,32 @@ namespace adios2 namespace helper { +namespace +{ +const size_t DatatypeToSize[] = { + sizeof(signed char), + sizeof(char), + sizeof(short), + sizeof(int), + sizeof(long), + sizeof(unsigned char), + sizeof(unsigned short), + sizeof(unsigned int), + sizeof(unsigned long), + sizeof(unsigned long long), + sizeof(long long), + sizeof(double), + sizeof(long double), + sizeof(std::pair), + sizeof(std::pair), + sizeof(std::pair), + sizeof(std::pair), + sizeof(std::pair), +}; + +size_t ToSize(CommImpl::Datatype dt) { return DatatypeToSize[int(dt)]; } +} + Comm::Comm() = default; Comm::Comm(std::unique_ptr impl) : m_Impl(std::move(impl)) {} @@ -94,6 +120,8 @@ Comm::Status Comm::Req::Wait(const std::string &hint) CommImpl::~CommImpl() = default; +size_t CommImpl::SizeOf(Datatype datatype) { return ToSize(datatype); } + Comm CommImpl::MakeComm(std::unique_ptr impl) { return Comm(std::move(impl)); diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h index 7296be0f36..b4471e84a0 100644 --- a/source/adios2/helper/adiosComm.h +++ b/source/adios2/helper/adiosComm.h @@ -354,8 +354,7 @@ class CommImpl Datatype datatype, Comm::Op op, const std::string &hint) const = 0; - virtual void Bcast(void *buffer, size_t count, Datatype datatype, - size_t datatypeSize, int root, + virtual void Bcast(void *buffer, size_t count, Datatype datatype, int root, const std::string &hint) const = 0; virtual void Gather(const void *sendbuf, size_t sendcount, @@ -397,6 +396,8 @@ class CommImpl int source, int tag, const std::string &hint) const = 0; + static size_t SizeOf(Datatype datatype); + static Comm MakeComm(std::unique_ptr impl); static Comm::Req MakeReq(std::unique_ptr impl); static CommImpl *Get(Comm const &comm); diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl index 9dd14a9182..03eb3522de 100644 --- a/source/adios2/helper/adiosComm.inl +++ b/source/adios2/helper/adiosComm.inl @@ -177,8 +177,7 @@ template void Comm::Bcast(T *buffer, const size_t count, int root, const std::string &hint) const { - return m_Impl->Bcast(buffer, count, CommImpl::GetDatatype(), sizeof(T), - root, hint); + return m_Impl->Bcast(buffer, count, CommImpl::GetDatatype(), root, hint); } template diff --git a/source/adios2/helper/adiosCommMPI.cpp b/source/adios2/helper/adiosCommMPI.cpp index 2bde011456..1c62fcc05d 100644 --- a/source/adios2/helper/adiosCommMPI.cpp +++ b/source/adios2/helper/adiosCommMPI.cpp @@ -123,8 +123,7 @@ class CommImplMPI : public CommImpl Datatype datatype, Comm::Op op, const std::string &hint) const override; - void Bcast(void *buffer, size_t count, Datatype datatype, - size_t datatypeSize, int root, + void Bcast(void *buffer, size_t count, Datatype datatype, int root, const std::string &hint) const override; void Gather(const void *sendbuf, size_t sendcount, Datatype sendtype, @@ -232,8 +231,7 @@ void CommImplMPI::Allreduce(const void *sendbuf, void *recvbuf, size_t count, hint); } -void CommImplMPI::Bcast(void *buffer, size_t count, Datatype datatype, - size_t datatypeSize, int root, +void CommImplMPI::Bcast(void *buffer, size_t count, Datatype datatype, int root, const std::string &hint) const { size_t inputSize = count; @@ -245,7 +243,7 @@ void CommImplMPI::Bcast(void *buffer, size_t count, Datatype datatype, CheckMPIReturn(SMPI_Bcast(blockBuf, static_cast(blockSize), ToMPI(datatype), root, m_MPIComm), hint); - blockBuf += blockSize * datatypeSize; + blockBuf += blockSize * CommImpl::SizeOf(datatype); inputSize -= blockSize; blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize); } From f3df8de86b140435272eaa347bae3432b6d635d1 Mon Sep 17 00:00:00 2001 From: Brad King Date: Mon, 16 Sep 2019 16:41:45 -0400 Subject: [PATCH 27/34] helper: Encapsulate MPI_Comm_dup as Comm::Duplicate --- source/adios2/helper/adiosComm.cpp | 5 +++++ source/adios2/helper/adiosComm.h | 10 ++++++++++ source/adios2/helper/adiosCommMPI.cpp | 8 ++++++++ 3 files changed, 23 insertions(+) diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp index bc7f43b65b..e30085ff90 100644 --- a/source/adios2/helper/adiosComm.cpp +++ b/source/adios2/helper/adiosComm.cpp @@ -55,6 +55,11 @@ Comm &Comm::operator=(Comm &&comm) = default; void Comm::Free(const std::string &hint) { m_Impl->Free(hint); } +Comm Comm::Duplicate(const std::string &hint) const +{ + return Comm(m_Impl->Duplicate(hint)); +} + Comm Comm::Split(int color, int key, const std::string &hint) const { return Comm(m_Impl->Split(color, key, hint)); diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h index b4471e84a0..edafbeb63b 100644 --- a/source/adios2/helper/adiosComm.h +++ b/source/adios2/helper/adiosComm.h @@ -92,6 +92,14 @@ class Comm */ void Free(const std::string &hint = std::string()); + /** + * @brief Duplicate the communicator. + * @param hint Description of std::runtime_error exception on error. + * + * Creates a new communicator covering the same processes as the original. + */ + Comm Duplicate(const std::string &hint = std::string()) const; + /** * @brief Split the communicator. * @param color Control of subset assignment (nonnegative integer). @@ -340,6 +348,8 @@ class CommImpl virtual ~CommImpl() = 0; virtual void Free(const std::string &hint) = 0; + virtual std::unique_ptr + Duplicate(const std::string &hint) const = 0; virtual std::unique_ptr Split(int color, int key, const std::string &hint) const = 0; virtual int Rank() const = 0; diff --git a/source/adios2/helper/adiosCommMPI.cpp b/source/adios2/helper/adiosCommMPI.cpp index 1c62fcc05d..e912e13c91 100644 --- a/source/adios2/helper/adiosCommMPI.cpp +++ b/source/adios2/helper/adiosCommMPI.cpp @@ -108,6 +108,7 @@ class CommImplMPI : public CommImpl ~CommImplMPI() override; void Free(const std::string &hint) override; + std::unique_ptr Duplicate(const std::string &hint) const override; std::unique_ptr Split(int color, int key, const std::string &hint) const override; @@ -184,6 +185,13 @@ void CommImplMPI::Free(const std::string &hint) } } +std::unique_ptr CommImplMPI::Duplicate(const std::string &hint) const +{ + MPI_Comm newComm; + CheckMPIReturn(MPI_Comm_dup(m_MPIComm, &newComm), hint); + return std::unique_ptr(new CommImplMPI(newComm)); +} + std::unique_ptr CommImplMPI::Split(int color, int key, const std::string &hint) const { From 06ab278ad4fa726f7bdf7da3be4af236b5df4920 Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 20 Sep 2019 13:23:55 -0400 Subject: [PATCH 28/34] helper: Encapsulate MPI_COMM_WORLD as Comm::World --- source/adios2/helper/adiosComm.cpp | 5 +++++ source/adios2/helper/adiosComm.h | 7 +++++++ source/adios2/helper/adiosCommMPI.cpp | 6 ++++++ source/adios2/toolkit/format/dataman/DataManSerializer.cpp | 5 +---- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp index e30085ff90..fd44297e39 100644 --- a/source/adios2/helper/adiosComm.cpp +++ b/source/adios2/helper/adiosComm.cpp @@ -65,6 +65,11 @@ Comm Comm::Split(int color, int key, const std::string &hint) const return Comm(m_Impl->Split(color, key, hint)); } +Comm Comm::World(const std::string &hint) const +{ + return Comm(m_Impl->World(hint)); +} + int Comm::Rank() const { return m_Impl->Rank(); } int Comm::Size() const { return m_Impl->Size(); } diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h index edafbeb63b..22d13a63c2 100644 --- a/source/adios2/helper/adiosComm.h +++ b/source/adios2/helper/adiosComm.h @@ -112,6 +112,12 @@ class Comm Comm Split(int color, int key, const std::string &hint = std::string()) const; + /** + * @brief Create a communicator covering all processes. + * @param hint Description of std::runtime_error exception on error. + */ + Comm World(const std::string &hint = std::string()) const; + int Rank() const; int Size() const; @@ -352,6 +358,7 @@ class CommImpl Duplicate(const std::string &hint) const = 0; virtual std::unique_ptr Split(int color, int key, const std::string &hint) const = 0; + virtual std::unique_ptr World(const std::string &hint) const = 0; virtual int Rank() const = 0; virtual int Size() const = 0; virtual void Barrier(const std::string &hint) const = 0; diff --git a/source/adios2/helper/adiosCommMPI.cpp b/source/adios2/helper/adiosCommMPI.cpp index e912e13c91..6c10cca505 100644 --- a/source/adios2/helper/adiosCommMPI.cpp +++ b/source/adios2/helper/adiosCommMPI.cpp @@ -111,6 +111,7 @@ class CommImplMPI : public CommImpl std::unique_ptr Duplicate(const std::string &hint) const override; std::unique_ptr Split(int color, int key, const std::string &hint) const override; + std::unique_ptr World(const std::string &hint) const override; int Rank() const override; int Size() const override; @@ -200,6 +201,11 @@ std::unique_ptr CommImplMPI::Split(int color, int key, return std::unique_ptr(new CommImplMPI(newComm)); } +std::unique_ptr CommImplMPI::World(const std::string &) const +{ + return std::unique_ptr(new CommImplMPI(MPI_COMM_WORLD)); +} + int CommImplMPI::Rank() const { int rank; diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp index f4890b534a..565224db0c 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp @@ -11,8 +11,6 @@ #include "DataManSerializer.h" #include "DataManSerializer.tcc" -#include "adios2/common/ADIOSMPI.h" - #include #include @@ -1195,8 +1193,7 @@ void DataManSerializer::Log(const int level, const std::string &message, const bool mpi, const bool endline) { TAU_SCOPED_TIMER_FUNC(); - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); + const int rank = m_Comm.World().Rank(); if (m_Verbosity >= level) { From f5939f93bc8ec1577dc4f658864166aa422127cb Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 4 Oct 2019 13:56:33 -0400 Subject: [PATCH 29/34] engine: Port HDF5 engines to Comm encapsulation --- source/adios2/engine/hdf5/HDF5ReaderP.cpp | 3 +- source/adios2/engine/hdf5/HDF5WriterP.cpp | 7 ++-- source/adios2/engine/mixer/HDFMixer.cpp | 7 +--- source/adios2/engine/mixer/HDFMixerWriter.cpp | 27 ++++++------- source/adios2/engine/mixer/HDFMixerWriter.h | 39 ++++--------------- .../toolkit/interop/hdf5/HDF5Common.cpp | 25 +++++++++--- .../adios2/toolkit/interop/hdf5/HDF5Common.h | 3 +- 7 files changed, 47 insertions(+), 64 deletions(-) diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.cpp b/source/adios2/engine/hdf5/HDF5ReaderP.cpp index fe7aaebb7f..1d98a42191 100644 --- a/source/adios2/engine/hdf5/HDF5ReaderP.cpp +++ b/source/adios2/engine/hdf5/HDF5ReaderP.cpp @@ -11,7 +11,6 @@ #include "HDF5ReaderP.h" #include "HDF5ReaderP.tcc" -#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" //CSVToVector #include @@ -62,7 +61,7 @@ void HDF5ReaderP::Init() ", in call to Open\n"); } - m_H5File.Init(m_Name, CommAsMPI(m_Comm), false); + m_H5File.Init(m_Name, m_Comm, false); m_H5File.ParseParameters(m_IO); /* diff --git a/source/adios2/engine/hdf5/HDF5WriterP.cpp b/source/adios2/engine/hdf5/HDF5WriterP.cpp index 818a60a4e9..32a4a19c48 100644 --- a/source/adios2/engine/hdf5/HDF5WriterP.cpp +++ b/source/adios2/engine/hdf5/HDF5WriterP.cpp @@ -10,7 +10,6 @@ #include "HDF5WriterP.h" -#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" //CSVToVector namespace adios2 @@ -58,7 +57,7 @@ void HDF5WriterP::Init() } #ifdef NEVER - m_H5File.Init(m_Name, CommAsMPI(m_Comm), true); + m_H5File.Init(m_Name, m_Comm, true); #else // enforce .h5 ending std::string suffix = ".h5"; @@ -71,11 +70,11 @@ void HDF5WriterP::Init() { // is a file with .bp ending std::string updatedName = m_Name.substr(0, wpos) + suffix; - m_H5File.Init(updatedName, CommAsMPI(m_Comm), true); + m_H5File.Init(updatedName, m_Comm, true); } else { - m_H5File.Init(m_Name, CommAsMPI(m_Comm), true); + m_H5File.Init(m_Name, m_Comm, true); } m_H5File.ParseParameters(m_IO); #endif diff --git a/source/adios2/engine/mixer/HDFMixer.cpp b/source/adios2/engine/mixer/HDFMixer.cpp index 1b2cebbbe5..36bde32ed6 100644 --- a/source/adios2/engine/mixer/HDFMixer.cpp +++ b/source/adios2/engine/mixer/HDFMixer.cpp @@ -11,7 +11,6 @@ #include "HDFMixer.h" #include "HDFMixer.tcc" -#include "adios2/common/ADIOSMPI.h" #include "adios2/core/IO.h" #include "adios2/helper/adiosFunctions.h" //CheckIndexRange #include "adios2/toolkit/transport/file/FileFStream.h" @@ -27,7 +26,7 @@ HDFMixer::HDFMixer(IO &io, const std::string &name, const Mode openMode, helper::Comm comm) : Engine("HDFMixer", io, name, openMode, std::move(comm)), m_HDFVDSWriter(m_Comm, m_DebugMode), - m_HDFSerialWriter(MPI_COMM_SELF, m_DebugMode), + m_HDFSerialWriter(helper::Comm(), m_DebugMode), m_TransportsManager(m_Comm, m_DebugMode) { m_EndMessage = " in call to IO Open HDFMixer " + m_Name + "\n"; @@ -102,9 +101,7 @@ void HDFMixer::InitTransports() */ #else - int rank; - MPI_Comm_rank(m_Comm, &rank); - m_HDFSerialWriter.Init(m_Name, rank); + m_HDFSerialWriter.Init(m_Name, m_Comm.Rank()); m_HDFVDSWriter.Init(m_Name); /* auto transportsNames = m_TransportsManager.GetFilesBaseNames( diff --git a/source/adios2/engine/mixer/HDFMixerWriter.cpp b/source/adios2/engine/mixer/HDFMixerWriter.cpp index 697ecf32a1..9d8ad4fe0a 100644 --- a/source/adios2/engine/mixer/HDFMixerWriter.cpp +++ b/source/adios2/engine/mixer/HDFMixerWriter.cpp @@ -11,7 +11,6 @@ #include #include "HDFMixerWriter.h" -#include "adios2/common/ADIOSMPI.h" #include "adios2/helper/adiosFunctions.h" // @@ -24,11 +23,11 @@ namespace core namespace engine { -HDFVDSWriter::HDFVDSWriter(MPI_Comm mpiComm, bool debugMode) -: m_MPISubfileComm(mpiComm), m_VDSFile(debugMode), m_Rank(-1) +HDFVDSWriter::HDFVDSWriter(helper::Comm const &comm, bool debugMode) +: m_SubfileComm(comm), m_VDSFile(debugMode), m_Rank(-1) { - MPI_Comm_size(m_MPISubfileComm, &m_NumSubFiles); - MPI_Comm_rank(m_MPISubfileComm, &m_Rank); + m_NumSubFiles = m_SubfileComm.Size(); + m_Rank = m_SubfileComm.Rank(); } void HDFVDSWriter::Init(const std::string &name) @@ -42,7 +41,7 @@ void HDFVDSWriter::Init(const std::string &name) // VDS can only operate on one process. So let rank = 0 handle it // std::string h5Name = adios2::helper::AddExtension(name, ".h5"); - m_VDSFile.Init(h5Name, MPI_COMM_SELF, true); + m_VDSFile.Init(h5Name, helper::Comm(), true); // m_FileName = h5Name; m_FileName = name; } @@ -129,11 +128,8 @@ void HDFVDSWriter::AddVar(const VariableBase &var, hid_t h5Type) GetVarInfo(var, dimsf, nDims, start, count, one); // - MPI_Gather(start.data(), nDims, ADIOS2_MPI_SIZE_T, all_starts, nDims, - ADIOS2_MPI_SIZE_T, 0, m_MPISubfileComm); - - MPI_Gather(count.data(), nDims, ADIOS2_MPI_SIZE_T, all_counts, nDims, - ADIOS2_MPI_SIZE_T, 0, m_MPISubfileComm); + m_SubfileComm.Gather(start.data(), nDims, all_starts[0], nDims, 0); + m_SubfileComm.Gather(count.data(), nDims, all_counts[0], nDims, 0); herr_t status; if (m_Rank == 0) @@ -196,7 +192,7 @@ void HDFVDSWriter::AddVar(const VariableBase &var, hid_t h5Type) } // m_VDSFile.Close(); - MPI_Barrier(m_MPISubfileComm); + m_SubfileComm.Barrier(); } void HDFVDSWriter::Advance(const float timeoutSeconds) @@ -222,8 +218,9 @@ void HDFVDSWriter::Close(const int transportIndex) // // class HDFSerialWriter // -HDFSerialWriter::HDFSerialWriter(MPI_Comm mpiComm, const bool debugMode = false) -: m_MPILocalComm(mpiComm), m_DebugMode(debugMode), m_H5File(debugMode) +HDFSerialWriter::HDFSerialWriter(helper::Comm const &comm, + const bool debugMode = false) +: m_LocalComm(comm), m_DebugMode(debugMode), m_H5File(debugMode) { } @@ -291,7 +288,7 @@ void HDFSerialWriter::Init(const std::string &name, int rank) StaticCreateName(baseName, rootTag, h5Name, name, rank); // std::cout<<"rank="<) { return MPI_2INT; } -inline MPI_Datatype mpi_typeof(std::pair) { return MPI_FLOAT_INT; } -inline MPI_Datatype mpi_typeof(std::pair) -{ - return MPI_DOUBLE_INT; -} -inline MPI_Datatype mpi_typeof(std::pair) -{ - return MPI_LONG_DOUBLE_INT; -} -inline MPI_Datatype mpi_typeof(std::pair) { return MPI_SHORT_INT; } - -#define ADIOS_MPI_SIZE_T (mpi_typeof(size_t())) -*/ namespace adios2 { namespace core @@ -49,7 +24,7 @@ namespace engine class HDFVDSWriter { public: - HDFVDSWriter(MPI_Comm mpiComm, bool debugMode); + HDFVDSWriter(helper::Comm const &comm, bool debugMode); void Init(const std::string &name); void AddVar(const VariableBase &var, hid_t h5Type); void @@ -66,13 +41,14 @@ class HDFVDSWriter int m_NumSubFiles; std::string m_FileName; - MPI_Comm m_MPISubfileComm; // only rank 0 in this comm can build VDS; + helper::Comm const + &m_SubfileComm; // only rank 0 in this comm can build VDS; }; class HDFSerialWriter { public: - HDFSerialWriter(MPI_Comm mpiComm, bool debugMode); + HDFSerialWriter(helper::Comm const &comm, bool debugMode); void Advance(const float timeoutSeconds = std::numeric_limits::max()); void Close(const int transportIndex = -1); @@ -84,12 +60,13 @@ class HDFSerialWriter /** contains data buffer and position */ // capsule::STLVector m_HeapBuffer; - // int m_MPIRank; + // int m_Rank; interop::HDF5Common m_H5File; std::string m_FileName; private: - MPI_Comm m_MPILocalComm; // all ranks in this comm write to the same file + helper::Comm const + &m_LocalComm; // all ranks in this comm write to the same file const bool m_DebugMode = false; int m_Rank; }; diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp index e46bde7ebf..691a236f78 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp @@ -17,10 +17,13 @@ #include #include -#include "adios2/common/ADIOSMPI.h" #include "adios2/helper/adiosFunctions.h" // IsRowMajor #include // strlen +#ifdef ADIOS2_HAVE_MPI +#include "adios2/helper/adiosCommMPI.h" +#endif + namespace adios2 { namespace interop @@ -112,17 +115,27 @@ void HDF5Common::ParseParameters(core::IO &io) } } -void HDF5Common::Init(const std::string &name, MPI_Comm comm, bool toWrite) +void HDF5Common::Init(const std::string &name, helper::Comm const &comm, + bool toWrite) { m_WriteMode = toWrite; m_PropertyListId = H5Pcreate(H5P_FILE_ACCESS); #ifdef ADIOS2_HAVE_MPI - SMPI_Comm_rank(comm, &m_CommRank); - SMPI_Comm_size(comm, &m_CommSize); - if (m_CommSize != 1) + MPI_Comm mpiComm = helper::CommAsMPI(comm); + if (mpiComm != MPI_COMM_NULL) + { + MPI_Comm_rank(mpiComm, &m_CommRank); + MPI_Comm_size(mpiComm, &m_CommSize); + if (m_CommSize != 1) + { + H5Pset_fapl_mpio(m_PropertyListId, mpiComm, MPI_INFO_NULL); + } + } + else { - H5Pset_fapl_mpio(m_PropertyListId, comm, MPI_INFO_NULL); + m_CommRank = 0; + m_CommSize = 1; } #endif diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.h b/source/adios2/toolkit/interop/hdf5/HDF5Common.h index 9a7e1ceb82..d899b27111 100644 --- a/source/adios2/toolkit/interop/hdf5/HDF5Common.h +++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.h @@ -19,6 +19,7 @@ #include "adios2/common/ADIOSTypes.h" #include "adios2/core/IO.h" // for CreateVar #include "adios2/core/Variable.h" +#include "adios2/helper/adiosComm.h" #include // for Intel Compiler @@ -120,7 +121,7 @@ class HDF5Common static const std::string PARAMETER_CHUNK_VARS; void ParseParameters(core::IO &io); - void Init(const std::string &name, MPI_Comm comm, bool toWrite); + void Init(const std::string &name, helper::Comm const &comm, bool toWrite); template void Write(core::Variable &variable, const T *values); From c3991a2562b6c5c6fbbc9d831818c261a121765d Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 19 Sep 2019 14:21:59 -0400 Subject: [PATCH 30/34] utils: Clarify adios_reorganize Reorganize class member names --- source/utils/adios_reorganize/Reorganize.cpp | 38 ++++++++++---------- source/utils/adios_reorganize/Reorganize.h | 6 ++-- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/source/utils/adios_reorganize/Reorganize.cpp b/source/utils/adios_reorganize/Reorganize.cpp index 424e8be339..5bd2ad5195 100644 --- a/source/utils/adios_reorganize/Reorganize.cpp +++ b/source/utils/adios_reorganize/Reorganize.cpp @@ -44,9 +44,9 @@ namespace utils Reorganize::Reorganize(int argc, char *argv[]) : Utils("adios_reorganize", argc, argv) { - MPI_Comm_split(MPI_COMM_WORLD, m_MPISplitColor, rank, &comm); - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &numproc); + MPI_Comm_split(MPI_COMM_WORLD, m_MPISplitColor, 0, &m_Comm); + MPI_Comm_rank(m_Comm, &m_Rank); + MPI_Comm_size(m_Comm, &m_Size); if (argc < 5) { @@ -92,14 +92,14 @@ Reorganize::Reorganize(int argc, char *argv[]) prod *= decomp_values[i]; } - if (prod > numproc) + if (prod > m_Size) { print0("ERROR: Product of decomposition numbers %d > number of " "processes %d\n", - prod, numproc); + prod, m_Size); std::string errmsg("ERROR: The product of decomposition numbers " + std::to_string(prod) + " > number of processes " + - std::to_string(numproc) + "\n"); + std::to_string(m_Size) + "\n"); PrintUsage(); throw std::invalid_argument(errmsg); } @@ -119,7 +119,7 @@ void Reorganize::Run() print0("Write method parameters = ", wmethodparam_str); #ifdef ADIOS2_HAVE_MPI - core::ADIOS adios(comm, true, "C++"); + core::ADIOS adios(m_Comm, true, "C++"); #else core::ADIOS adios(true, "C++"); #endif @@ -145,7 +145,7 @@ void Reorganize::Run() rStream.BeginStep(adios2::StepMode::Read, 10.0); if (status == adios2::StepStatus::NotReady) { - if (!rank) + if (!m_Rank) { std::cout << " No new steps arrived in a while " << std::endl; } @@ -161,7 +161,7 @@ void Reorganize::Run() if (rStream.CurrentStep() != static_cast(curr_step + 1)) { // we missed some steps - std::cout << "rank " << rank << " WARNING: steps " << curr_step + std::cout << "rank " << m_Rank << " WARNING: steps " << curr_step << ".." << rStream.CurrentStep() - 1 << "were missed when advancing." << std::endl; } @@ -195,7 +195,7 @@ void Reorganize::Run() template void Reorganize::osprint0(std::ostream &out, Arg &&arg, Args &&... args) { - if (!rank) + if (!m_Rank) { out << std::forward(arg); using expander = int[]; @@ -207,7 +207,7 @@ void Reorganize::osprint0(std::ostream &out, Arg &&arg, Args &&... args) template void Reorganize::print0(Arg &&arg, Args &&... args) { - if (!rank) + if (!m_Rank) { std::cout << std::forward(arg); using expander = int[]; @@ -447,13 +447,13 @@ int Reorganize::ProcessMetadata(core::Engine &rStream, core::IO &io, if (variable == nullptr) { - std::cerr << "rank " << rank << ": ERROR: Variable " << name + std::cerr << "rank " << m_Rank << ": ERROR: Variable " << name << " inquiry failed" << std::endl; return 1; } // print variable type and dimensions - if (!rank) + if (!m_Rank) { std::cout << " " << type << " " << name; if (variable->GetShape().size() > 0) @@ -473,7 +473,7 @@ int Reorganize::ProcessMetadata(core::Engine &rStream, core::IO &io, // determine subset we will write size_t sum_count = - Decompose(numproc, rank, varinfo[varidx], decomp_values); + Decompose(m_Size, m_Rank, varinfo[varidx], decomp_values); varinfo[varidx].writesize = sum_count * variable->m_ElementSize; if (varinfo[varidx].writesize != 0) @@ -490,7 +490,7 @@ int Reorganize::ProcessMetadata(core::Engine &rStream, core::IO &io, write_total + variables.size() * 200 + attributes.size() * 32 + 1024; if (bufsize > max_write_buffer_size) { - std::cerr << "ERROR: rank " << rank + std::cerr << "ERROR: rank " << m_Rank << ": write buffer size needs to hold about " << bufsize << "bytes but max is set to " << max_write_buffer_size << std::endl; @@ -499,7 +499,7 @@ int Reorganize::ProcessMetadata(core::Engine &rStream, core::IO &io, if (bufsize > max_read_buffer_size) { - std::cerr << "ERROR: rank " << rank + std::cerr << "ERROR: rank " << m_Rank << ": read buffer size needs to hold at least " << bufsize << "bytes but max is set to " << max_read_buffer_size << std::endl; @@ -518,7 +518,7 @@ int Reorganize::ReadWrite(core::Engine &rStream, core::Engine &wStream, if (nvars != varinfo.size()) { std::cerr - << "ERROR rank " << rank + << "ERROR rank " << m_Rank << ": Invalid program state, number " "of variables (" << nvars @@ -536,7 +536,7 @@ int Reorganize::ReadWrite(core::Engine &rStream, core::Engine &wStream, if (varinfo[varidx].writesize != 0) { // read variable subset - std::cout << "rank " << rank << ": Read variable " << name + std::cout << "rank " << m_Rank << ": Read variable " << name << std::endl; const std::string &type = variables.at(name).first; if (type == "compound") @@ -577,7 +577,7 @@ int Reorganize::ReadWrite(core::Engine &rStream, core::Engine &wStream, if (varinfo[varidx].writesize != 0) { // Write variable subset - std::cout << "rank " << rank << ": Write variable " << name + std::cout << "rank " << m_Rank << ": Write variable " << name << std::endl; const std::string &type = variables.at(name).first; if (type == "compound") diff --git a/source/utils/adios_reorganize/Reorganize.h b/source/utils/adios_reorganize/Reorganize.h index 34274c2928..2fbc44ab97 100644 --- a/source/utils/adios_reorganize/Reorganize.h +++ b/source/utils/adios_reorganize/Reorganize.h @@ -82,9 +82,9 @@ class Reorganize : public Utils static const int timeout_sec = 300; // Global variables - int rank = 0; - int numproc = 1; - MPI_Comm comm; + int m_Rank = 0; + int m_Size = 1; + MPI_Comm m_Comm; // Read/write method parameters Params rmethodparams; From 554a756f633c1c573d9aab53a7374584edcd00cb Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 19 Sep 2019 14:19:33 -0400 Subject: [PATCH 31/34] utils: Port adios_reorganize to Comm encapsulation --- source/utils/adios_reorganize/Reorganize.cpp | 17 +++++++++-------- source/utils/adios_reorganize/Reorganize.h | 5 +++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/source/utils/adios_reorganize/Reorganize.cpp b/source/utils/adios_reorganize/Reorganize.cpp index 5bd2ad5195..072abbf25a 100644 --- a/source/utils/adios_reorganize/Reorganize.cpp +++ b/source/utils/adios_reorganize/Reorganize.cpp @@ -29,6 +29,8 @@ #include "adios2/core/ADIOS.h" #include "adios2/core/Engine.h" #include "adios2/core/IO.h" +#include "adios2/helper/adiosComm.h" +#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" #include "adios2/helper/adiosString.h" @@ -44,9 +46,12 @@ namespace utils Reorganize::Reorganize(int argc, char *argv[]) : Utils("adios_reorganize", argc, argv) { - MPI_Comm_split(MPI_COMM_WORLD, m_MPISplitColor, 0, &m_Comm); - MPI_Comm_rank(m_Comm, &m_Rank); - MPI_Comm_size(m_Comm, &m_Size); + { + auto commWorld = helper::CommFromMPI(MPI_COMM_WORLD); + m_Comm = commWorld.Split(m_CommSplitColor, 0); + } + m_Rank = m_Comm.Rank(); + m_Size = m_Comm.Size(); if (argc < 5) { @@ -118,11 +123,7 @@ void Reorganize::Run() print0("Write method = ", wmethodname); print0("Write method parameters = ", wmethodparam_str); -#ifdef ADIOS2_HAVE_MPI - core::ADIOS adios(m_Comm, true, "C++"); -#else - core::ADIOS adios(true, "C++"); -#endif + core::ADIOS adios(CommAsMPI(m_Comm), true, "C++"); core::IO &io = adios.DeclareIO("group"); print0("Waiting to open stream ", infilename, "..."); diff --git a/source/utils/adios_reorganize/Reorganize.h b/source/utils/adios_reorganize/Reorganize.h index 2fbc44ab97..904c2139e6 100644 --- a/source/utils/adios_reorganize/Reorganize.h +++ b/source/utils/adios_reorganize/Reorganize.h @@ -13,6 +13,7 @@ #include "adios2.h" #include "adios2/core/IO.h" // DataMap +#include "adios2/helper/adiosComm.h" #include "utils/Utils.h" namespace adios2 @@ -40,7 +41,7 @@ class Reorganize : public Utils void Run() final; private: - static const int m_MPISplitColor = 23731; // color in MPI_Split_comm() call + static const int m_CommSplitColor = 23731; // color in Comm::Split() call static const std::string m_HelpMessage; static const Params m_Options; @@ -84,7 +85,7 @@ class Reorganize : public Utils // Global variables int m_Rank = 0; int m_Size = 1; - MPI_Comm m_Comm; + helper::Comm m_Comm; // Read/write method parameters Params rmethodparams; From 41219da7b9af22824ee44f0b0cceb329fe91aeae Mon Sep 17 00:00:00 2001 From: Brad King Date: Mon, 7 Oct 2019 15:58:28 -0400 Subject: [PATCH 32/34] helper: Add dummy Comm implementation Create a Comm implementation for serial programs. Use implementations inspired by `source/adios2/helper/mpidummy.cpp`. --- source/adios2/CMakeLists.txt | 1 + source/adios2/helper/adiosCommDummy.cpp | 272 ++++++++++++++++++++++++ source/adios2/helper/adiosCommDummy.h | 26 +++ 3 files changed, 299 insertions(+) create mode 100644 source/adios2/helper/adiosCommDummy.cpp create mode 100644 source/adios2/helper/adiosCommDummy.h diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 95a6b76f93..44b9800e2b 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -33,6 +33,7 @@ add_library(adios2 #helper helper/adiosComm.h helper/adiosComm.cpp + helper/adiosCommDummy.h helper/adiosCommDummy.cpp helper/adiosCommMPI.h helper/adiosCommMPI.cpp helper/adiosDynamicBinder.h helper/adiosDynamicBinder.cpp helper/adiosMath.cpp diff --git a/source/adios2/helper/adiosCommDummy.cpp b/source/adios2/helper/adiosCommDummy.cpp new file mode 100644 index 0000000000..5637940f0b --- /dev/null +++ b/source/adios2/helper/adiosCommDummy.cpp @@ -0,0 +1,272 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * adiosCommDummy.cpp + */ + +#include "adiosCommDummy.h" + +#include +#include + +#include "adiosComm.h" + +namespace adios2 +{ +namespace helper +{ + +namespace +{ +void CommDummyError(const std::string &msg) +{ + std::cerr << "CommDummy: a function returned error code '" << msg + << "'. Aborting!" << std::endl; + std::abort(); +} +} + +class CommReqImplDummy : public CommReqImpl +{ +public: + CommReqImplDummy() {} + ~CommReqImplDummy() override; + + Comm::Status Wait(const std::string &hint) override; +}; + +CommReqImplDummy::~CommReqImplDummy() = default; + +class CommImplDummy : public CommImpl +{ +public: + CommImplDummy() = default; + ~CommImplDummy() override; + + void Free(const std::string &hint) override; + std::unique_ptr Duplicate(const std::string &hint) const override; + std::unique_ptr Split(int color, int key, + const std::string &hint) const override; + std::unique_ptr World(const std::string &hint) const override; + + int Rank() const override; + int Size() const override; + void Barrier(const std::string &hint) const override; + + void Allgather(const void *sendbuf, size_t sendcount, Datatype sendtype, + void *recvbuf, size_t recvcount, Datatype recvtype, + const std::string &hint) const override; + + void Allreduce(const void *sendbuf, void *recvbuf, size_t count, + Datatype datatype, Comm::Op op, + const std::string &hint) const override; + + void Bcast(void *buffer, size_t count, Datatype datatype, int root, + const std::string &hint) const override; + + void Gather(const void *sendbuf, size_t sendcount, Datatype sendtype, + void *recvbuf, size_t recvcount, Datatype recvtype, int root, + const std::string &hint) const override; + + void Gatherv(const void *sendbuf, size_t sendcount, Datatype sendtype, + void *recvbuf, const size_t *recvcounts, const size_t *displs, + Datatype recvtype, int root, + const std::string &hint) const override; + + void Reduce(const void *sendbuf, void *recvbuf, size_t count, + Datatype datatype, Comm::Op op, int root, + const std::string &hint) const override; + + void ReduceInPlace(void *buf, size_t count, Datatype datatype, Comm::Op op, + int root, const std::string &hint) const override; + + void Send(const void *buf, size_t count, Datatype datatype, int dest, + int tag, const std::string &hint) const override; + + Comm::Status Recv(void *buf, size_t count, Datatype datatype, int source, + int tag, const std::string &hint) const override; + + void Scatter(const void *sendbuf, size_t sendcount, Datatype sendtype, + void *recvbuf, size_t recvcount, Datatype recvtype, int root, + const std::string &hint) const override; + + Comm::Req Isend(const void *buffer, size_t count, Datatype datatype, + int dest, int tag, const std::string &hint) const override; + + Comm::Req Irecv(void *buffer, size_t count, Datatype datatype, int source, + int tag, const std::string &hint) const override; +}; + +CommImplDummy::~CommImplDummy() = default; + +void CommImplDummy::Free(const std::string &) {} + +std::unique_ptr CommImplDummy::Duplicate(const std::string &) const +{ + return std::unique_ptr(new CommImplDummy()); +} + +std::unique_ptr CommImplDummy::Split(int, int, + const std::string &) const +{ + return std::unique_ptr(new CommImplDummy()); +} + +std::unique_ptr CommImplDummy::World(const std::string &) const +{ + return std::unique_ptr(new CommImplDummy()); +} + +int CommImplDummy::Rank() const { return 0; } + +int CommImplDummy::Size() const { return 1; } + +void CommImplDummy::Barrier(const std::string &) const {} + +void CommImplDummy::Allgather(const void *sendbuf, size_t sendcount, + Datatype sendtype, void *recvbuf, + size_t recvcount, Datatype recvtype, + const std::string &hint) const +{ + CommImplDummy::Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, + recvtype, 0, hint); +} + +void CommImplDummy::Allreduce(const void *sendbuf, void *recvbuf, size_t count, + Datatype datatype, Comm::Op op, + const std::string &hint) const +{ + CommImplDummy::Reduce(sendbuf, recvbuf, count, datatype, op, 0, hint); +} + +void CommImplDummy::Bcast(void *, size_t, Datatype, int, + const std::string &) const +{ +} + +void CommImplDummy::Gather(const void *sendbuf, size_t sendcount, + Datatype sendtype, void *recvbuf, size_t recvcount, + Datatype recvtype, int root, + const std::string &) const +{ + if (sendcount > 0 && !sendbuf) + { + return CommDummyError("sendbuf is null"); + } + if (recvcount > 0 && !recvbuf) + { + return CommDummyError("recvbuf is null"); + } + if (root != 0) + { + return CommDummyError("root is not 0"); + } + + const size_t nsent = sendcount * CommImpl::SizeOf(sendtype); + const size_t nrecv = recvcount * CommImpl::SizeOf(recvtype); + + if (nrecv != nsent) + { + return CommDummyError("send and recv sizes differ"); + } + + std::memcpy(recvbuf, sendbuf, nsent); +} + +void CommImplDummy::Gatherv(const void *sendbuf, size_t sendcount, + Datatype sendtype, void *recvbuf, + const size_t *recvcounts, const size_t *displs, + Datatype recvtype, int root, + const std::string &hint) const +{ + const size_t recvcount = recvcounts[0]; + if (recvcount != sendcount) + { + return CommDummyError("send and recv counts differ"); + } + CommImplDummy::Gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, + recvtype, root, hint); +} + +void CommImplDummy::Reduce(const void *sendbuf, void *recvbuf, size_t count, + Datatype datatype, Comm::Op, int, + const std::string &) const +{ + std::memcpy(recvbuf, sendbuf, count * CommImpl::SizeOf(datatype)); +} + +void CommImplDummy::ReduceInPlace(void *, size_t, Datatype, Comm::Op, int, + const std::string &) const +{ +} + +void CommImplDummy::Send(const void *, size_t, Datatype, int, int, + const std::string &) const +{ +} + +Comm::Status CommImplDummy::Recv(void *, size_t, Datatype, int, int, + const std::string &) const +{ + Comm::Status status; + return status; +} + +void CommImplDummy::Scatter(const void *sendbuf, size_t sendcount, + Datatype sendtype, void *recvbuf, size_t recvcount, + Datatype recvtype, int root, + const std::string &) const +{ + if (sendcount > 0 && !sendbuf) + { + return CommDummyError("sendbuf is null"); + } + if (recvcount > 0 && !recvbuf) + { + return CommDummyError("recvbuf is null"); + } + if (root != 0) + { + return CommDummyError("root is not 0"); + } + + const size_t nsent = sendcount * CommImpl::SizeOf(sendtype); + const size_t nrecv = recvcount * CommImpl::SizeOf(recvtype); + + if (nrecv != nsent) + { + return CommDummyError("send and recv sizes differ"); + } + + std::memcpy(recvbuf, sendbuf, nsent); +} + +Comm::Req CommImplDummy::Isend(const void *, size_t, Datatype, int, int, + const std::string &) const +{ + auto req = std::unique_ptr(new CommReqImplDummy()); + return MakeReq(std::move(req)); +} + +Comm::Req CommImplDummy::Irecv(void *, size_t, Datatype, int, int, + const std::string &) const +{ + auto req = std::unique_ptr(new CommReqImplDummy()); + return MakeReq(std::move(req)); +} + +Comm::Status CommReqImplDummy::Wait(const std::string &hint) +{ + Comm::Status status; + return status; +} + +Comm CommDummy() +{ + auto comm = std::unique_ptr(new CommImplDummy()); + return CommImpl::MakeComm(std::move(comm)); +} + +} // end namespace helper +} // end namespace adios2 diff --git a/source/adios2/helper/adiosCommDummy.h b/source/adios2/helper/adiosCommDummy.h new file mode 100644 index 0000000000..08aa013566 --- /dev/null +++ b/source/adios2/helper/adiosCommDummy.h @@ -0,0 +1,26 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * adiosCommDummy.h : A dummy Comm that does not actually communicate. + */ + +#ifndef ADIOS2_HELPER_ADIOSCOMMDUMMY_H_ +#define ADIOS2_HELPER_ADIOSCOMMDUMMY_H_ + +#include "adiosComm.h" + +namespace adios2 +{ +namespace helper +{ + +/** + * @brief Create a dummy communicator. + */ +Comm CommDummy(); + +} // end namespace helper +} // end namespace adios2 + +#endif // ADIOS2_HELPER_ADIOSCOMMDUMMY_H_ From 89891238bf72753e4b51f25940755fa2d0eb5026 Mon Sep 17 00:00:00 2001 From: Brad King Date: Mon, 7 Oct 2019 16:00:19 -0400 Subject: [PATCH 33/34] core: Replace MPI with Comm in ADIOS, IO, and Stream interfaces --- bindings/C/adios2/c/adios2_c_adios.cpp | 8 +++++-- bindings/C/adios2/c/adios2_c_io.cpp | 8 +++++-- bindings/CXX11/adios2/cxx11/ADIOS.cpp | 8 +++++-- bindings/CXX11/adios2/cxx11/IO.cpp | 7 ++++-- .../adios2/cxx11/fstream/ADIOS2fstream.cpp | 18 +++++++++------ bindings/Python/py11ADIOS.cpp | 8 ++++--- bindings/Python/py11File.cpp | 22 ++++++++++++++----- bindings/Python/py11File.h | 6 +++++ bindings/Python/py11IO.cpp | 4 +++- source/adios2/core/ADIOS.cpp | 14 ++++++------ source/adios2/core/ADIOS.h | 5 ++--- source/adios2/core/IO.cpp | 7 ++---- source/adios2/core/IO.h | 2 +- source/adios2/core/IO.tcc | 1 - source/adios2/core/Stream.cpp | 18 ++++++++------- source/adios2/core/Stream.h | 6 ++--- source/utils/adios_reorganize/Reorganize.cpp | 14 +++++++++--- 17 files changed, 100 insertions(+), 56 deletions(-) diff --git a/bindings/C/adios2/c/adios2_c_adios.cpp b/bindings/C/adios2/c/adios2_c_adios.cpp index af5678d241..a1d003d46e 100644 --- a/bindings/C/adios2/c/adios2_c_adios.cpp +++ b/bindings/C/adios2/c/adios2_c_adios.cpp @@ -10,10 +10,13 @@ #include "adios2_c_adios.h" -#include "adios2/common/ADIOSMPI.h" #include "adios2/core/ADIOS.h" #include "adios2/helper/adiosFunctions.h" +#ifdef ADIOS2_HAVE_MPI +#include "adios2/helper/adiosCommMPI.h" +#endif + #ifdef __cplusplus extern "C" { #endif @@ -35,7 +38,8 @@ adios2_adios *adios2_init_config_glue(const char *config_file, MPI_Comm comm, const bool debugBool = (debug_mode == adios2_debug_mode_on) ? true : false; adios = reinterpret_cast(new adios2::core::ADIOS( - config_file, comm, debugBool, host_language)); + config_file, adios2::helper::CommFromMPI(comm), debugBool, + host_language)); } catch (...) { diff --git a/bindings/C/adios2/c/adios2_c_io.cpp b/bindings/C/adios2/c/adios2_c_io.cpp index 62c5c67ef0..0aff38b6df 100644 --- a/bindings/C/adios2/c/adios2_c_io.cpp +++ b/bindings/C/adios2/c/adios2_c_io.cpp @@ -12,11 +12,14 @@ #include -#include "adios2/common/ADIOSMPI.h" #include "adios2/core/IO.h" #include "adios2/helper/adiosFunctions.h" //GetType #include "adios2_c_internal.h" +#ifdef ADIOS2_HAVE_MPI +#include "adios2/helper/adiosCommMPI.h" +#endif + #ifdef __cplusplus extern "C" { #endif @@ -729,7 +732,8 @@ adios2_engine *adios2_open_new_comm(adios2_io *io, const char *name, io, "for adios2_io, in call to adios2_open"); engine = reinterpret_cast( &reinterpret_cast(io)->Open( - name, adios2_ToOpenMode(mode), comm)); + name, adios2_ToOpenMode(mode), + adios2::helper::CommFromMPI(comm))); } catch (...) { diff --git a/bindings/CXX11/adios2/cxx11/ADIOS.cpp b/bindings/CXX11/adios2/cxx11/ADIOS.cpp index d36c72309b..e308a010a5 100644 --- a/bindings/CXX11/adios2/cxx11/ADIOS.cpp +++ b/bindings/CXX11/adios2/cxx11/ADIOS.cpp @@ -8,16 +8,20 @@ #include "ADIOS.h" -#include "adios2/common/ADIOSMPI.h" #include "adios2/core/ADIOS.h" #include "adios2/core/IO.h" #include "adios2/helper/adiosFunctions.h" //CheckForNullptr +#ifdef ADIOS2_HAVE_MPI +#include "adios2/helper/adiosCommMPI.h" +#endif + namespace adios2 { #ifdef ADIOS2_HAVE_MPI ADIOS::ADIOS(const std::string &configFile, MPI_Comm comm, const bool debugMode) -: m_ADIOS(std::make_shared(configFile, comm, debugMode, "C++")) +: m_ADIOS(std::make_shared(configFile, helper::CommFromMPI(comm), + debugMode, "C++")) { } diff --git a/bindings/CXX11/adios2/cxx11/IO.cpp b/bindings/CXX11/adios2/cxx11/IO.cpp index ec40df3217..23147379da 100644 --- a/bindings/CXX11/adios2/cxx11/IO.cpp +++ b/bindings/CXX11/adios2/cxx11/IO.cpp @@ -11,9 +11,12 @@ #include "IO.h" #include "IO.tcc" -#include "adios2/common/ADIOSMPI.h" #include "adios2/core/IO.h" +#ifdef ADIOS2_HAVE_MPI +#include "adios2/helper/adiosCommMPI.h" +#endif + namespace adios2 { @@ -109,7 +112,7 @@ Engine IO::Open(const std::string &name, const Mode mode, MPI_Comm comm) { helper::CheckForNullptr(m_IO, "for engine " + name + ", in call to IO::Open"); - return Engine(&m_IO->Open(name, mode, comm)); + return Engine(&m_IO->Open(name, mode, helper::CommFromMPI(comm))); } #endif diff --git a/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.cpp b/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.cpp index 3229fbfeec..0f13e047f8 100644 --- a/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.cpp +++ b/bindings/CXX11/adios2/cxx11/fstream/ADIOS2fstream.cpp @@ -11,7 +11,9 @@ #include "ADIOS2fstream.h" #include "ADIOS2fstream.tcc" -#include "adios2/common/ADIOSMPI.h" +#ifdef ADIOS2_HAVE_MPI +#include "adios2/helper/adiosCommMPI.h" +#endif namespace adios2 { @@ -19,15 +21,16 @@ namespace adios2 #ifdef ADIOS2_HAVE_MPI fstream::fstream(const std::string &name, const openmode mode, MPI_Comm comm, const std::string engineType) -: m_Stream(std::make_shared(name, ToMode(mode), comm, engineType, - "C++")) +: m_Stream(std::make_shared( + name, ToMode(mode), helper::CommFromMPI(comm), engineType, "C++")) { } fstream::fstream(const std::string &name, const openmode mode, MPI_Comm comm, const std::string &configFile, const std::string ioInConfigFile) -: m_Stream(std::make_shared(name, ToMode(mode), comm, configFile, +: m_Stream(std::make_shared(name, ToMode(mode), + helper::CommFromMPI(comm), configFile, ioInConfigFile, "C++")) { } @@ -54,8 +57,8 @@ void fstream::open(const std::string &name, const openmode mode, MPI_Comm comm, const std::string engineType) { CheckOpen(name); - m_Stream = std::make_shared(name, ToMode(mode), comm, - engineType, "C++"); + m_Stream = std::make_shared( + name, ToMode(mode), helper::CommFromMPI(comm), engineType, "C++"); } void fstream::open(const std::string &name, const openmode mode, MPI_Comm comm, @@ -64,7 +67,8 @@ void fstream::open(const std::string &name, const openmode mode, MPI_Comm comm, { CheckOpen(name); m_Stream = std::make_shared( - name, ToMode(mode), comm, configFile, ioInConfigFile, "C++"); + name, ToMode(mode), helper::CommFromMPI(comm), configFile, + ioInConfigFile, "C++"); } #else void fstream::open(const std::string &name, const openmode mode, diff --git a/bindings/Python/py11ADIOS.cpp b/bindings/Python/py11ADIOS.cpp index 2a289f70e3..b1bba07063 100644 --- a/bindings/Python/py11ADIOS.cpp +++ b/bindings/Python/py11ADIOS.cpp @@ -10,7 +10,9 @@ #include "py11ADIOS.h" -#include "adios2/common/ADIOSMPI.h" +#ifdef ADIOS2_HAVE_MPI +#include "adios2/helper/adiosCommMPI.h" +#endif namespace adios2 { @@ -20,8 +22,8 @@ namespace py11 #ifdef ADIOS2_HAVE_MPI ADIOS::ADIOS(const std::string &configFile, MPI4PY_Comm mpiComm, const bool debugMode) -: m_ADIOS(std::make_shared(configFile, mpiComm, debugMode, - "Python")) +: m_ADIOS(std::make_shared( + configFile, helper::CommFromMPI(mpiComm), debugMode, "Python")) { } diff --git a/bindings/Python/py11File.cpp b/bindings/Python/py11File.cpp index 92c729696b..a442f79b76 100644 --- a/bindings/Python/py11File.cpp +++ b/bindings/Python/py11File.cpp @@ -14,10 +14,14 @@ #include #include -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSMacros.h" +#include "adios2/helper/adiosCommDummy.h" #include "adios2/helper/adiosFunctions.h" +#ifdef ADIOS2_HAVE_MPI +#include "adios2/helper/adiosCommMPI.h" +#endif + #include "py11types.h" namespace adios2 @@ -25,31 +29,37 @@ namespace adios2 namespace py11 { +#ifdef ADIOS2_HAVE_MPI File::File(const std::string &name, const std::string mode, MPI_Comm comm, const std::string engineType) : m_Name(name), m_Mode(mode), - m_Stream(std::make_shared(name, ToMode(mode), comm, engineType, - "Python")) + m_Stream(std::make_shared( + name, ToMode(mode), helper::CommFromMPI(comm), engineType, "Python")) { } File::File(const std::string &name, const std::string mode, MPI_Comm comm, const std::string &configFile, const std::string ioInConfigFile) : m_Name(name), m_Mode(mode), - m_Stream(std::make_shared(name, ToMode(mode), comm, configFile, + m_Stream(std::make_shared(name, ToMode(mode), + helper::CommFromMPI(comm), configFile, ioInConfigFile, "Python")) { } +#endif File::File(const std::string &name, const std::string mode, const std::string engineType) -: File(name, mode, MPI_COMM_NULL, engineType) +: m_Name(name), m_Mode(mode), m_Stream(std::make_shared( + name, ToMode(mode), engineType, "Python")) { } File::File(const std::string &name, const std::string mode, const std::string &configFile, const std::string ioInConfigFile) -: File(name, mode, MPI_COMM_NULL, configFile, ioInConfigFile) +: m_Name(name), m_Mode(mode), + m_Stream(std::make_shared(name, ToMode(mode), configFile, + ioInConfigFile, "Python")) { } diff --git a/bindings/Python/py11File.h b/bindings/Python/py11File.h index 47ed83a20c..6a6a990789 100644 --- a/bindings/Python/py11File.h +++ b/bindings/Python/py11File.h @@ -16,6 +16,10 @@ #include "adios2/common/ADIOSTypes.h" #include "adios2/core/Stream.h" +#ifdef ADIOS2_HAVE_MPI +#include "adios2/common/ADIOSMPI.h" +#endif + namespace adios2 { namespace py11 @@ -27,11 +31,13 @@ class File const std::string m_Name; const std::string m_Mode; +#ifdef ADIOS2_HAVE_MPI File(const std::string &name, const std::string mode, MPI_Comm comm, const std::string engineType = "BPFile"); File(const std::string &name, const std::string mode, MPI_Comm comm, const std::string &configFile, const std::string ioInConfigFile); +#endif File(const std::string &name, const std::string mode, const std::string engineType = "BPFile"); diff --git a/bindings/Python/py11IO.cpp b/bindings/Python/py11IO.cpp index 8ba1615b7b..51245fef88 100644 --- a/bindings/Python/py11IO.cpp +++ b/bindings/Python/py11IO.cpp @@ -14,6 +14,7 @@ #include "adios2/helper/adiosFunctions.h" //GetType #ifdef ADIOS2_HAVE_MPI +#include "adios2/helper/adiosCommMPI.h" #include #endif @@ -252,7 +253,8 @@ Engine IO::Open(const std::string &name, const int mode, MPI4PY_Comm comm) helper::CheckForNullptr(m_IO, "for engine " + name + ", in call to IO::Open"); - return Engine(&m_IO->Open(name, static_cast(mode), comm)); + return Engine(&m_IO->Open(name, static_cast(mode), + helper::CommFromMPI(comm))); } #endif diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index 19d7c330c0..4e132b1d24 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -14,7 +14,7 @@ #include //std::ios_base::failure #include "adios2/core/IO.h" -#include "adios2/helper/adiosCommMPI.h" +#include "adios2/helper/adiosCommDummy.h" #include "adios2/helper/adiosFunctions.h" //InquireKey, BroadcastFile // OPERATORS @@ -57,10 +57,10 @@ namespace adios2 namespace core { -ADIOS::ADIOS(const std::string configFile, MPI_Comm mpiComm, +ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const bool debugMode, const std::string hostLanguage) : m_ConfigFile(configFile), m_DebugMode(debugMode), - m_HostLanguage(hostLanguage), m_Comm(helper::CommFromMPI(mpiComm)) + m_HostLanguage(hostLanguage), m_Comm(std::move(comm)) { if (!configFile.empty()) { @@ -74,18 +74,18 @@ ADIOS::ADIOS(const std::string configFile, MPI_Comm mpiComm, ADIOS::ADIOS(const std::string configFile, const bool debugMode, const std::string hostLanguage) -: ADIOS(configFile, MPI_COMM_NULL, debugMode, hostLanguage) +: ADIOS(configFile, helper::CommDummy(), debugMode, hostLanguage) { } -ADIOS::ADIOS(MPI_Comm mpiComm, const bool debugMode, +ADIOS::ADIOS(helper::Comm comm, const bool debugMode, const std::string hostLanguage) -: ADIOS("", mpiComm, debugMode, hostLanguage) +: ADIOS("", std::move(comm), debugMode, hostLanguage) { } ADIOS::ADIOS(const bool debugMode, const std::string hostLanguage) -: ADIOS("", MPI_COMM_NULL, debugMode, hostLanguage) +: ADIOS("", helper::CommDummy(), debugMode, hostLanguage) { } diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h index 00c65262c9..bcce1dfdae 100644 --- a/source/adios2/core/ADIOS.h +++ b/source/adios2/core/ADIOS.h @@ -19,7 +19,6 @@ /// \endcond #include "adios2/common/ADIOSConfig.h" -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSTypes.h" #include "adios2/core/Operator.h" #include "adios2/helper/adiosComm.h" @@ -55,7 +54,7 @@ class ADIOS * false: optional feature to turn off checks on user input data, * recommended in stable flows */ - ADIOS(const std::string configFile, MPI_Comm mpiComm, const bool debugMode, + ADIOS(const std::string configFile, helper::Comm comm, const bool debugMode, const std::string hostLanguage); /** @@ -77,7 +76,7 @@ class ADIOS * false: optional feature to turn off checks on user input data, * recommended in stable flows */ - ADIOS(MPI_Comm mpiComm, const bool debugMode, + ADIOS(helper::Comm comm, const bool debugMode, const std::string hostLanguage); /** diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 45732cc939..2384b85a0c 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -14,7 +14,6 @@ #include #include // std::pair -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSMacros.h" #include "adios2/engine/bp3/BP3Reader.h" @@ -29,7 +28,6 @@ #include "adios2/engine/skeleton/SkeletonWriter.h" #include "adios2/helper/adiosComm.h" -#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" //BuildParametersMap #include "adios2/toolkit/profiling/taustubs/tautimer.hpp" #include // FileIsDirectory() @@ -473,7 +471,7 @@ size_t IO::AddOperation(Operator &op, const Params ¶meters) noexcept return m_Operations.size() - 1; } -Engine &IO::Open(const std::string &name, const Mode mode, MPI_Comm mpiComm) +Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) { TAU_SCOPED_TIMER("IO::Open"); auto itEngineFound = m_Engines.find(name); @@ -505,7 +503,6 @@ Engine &IO::Open(const std::string &name, const Mode mode, MPI_Comm mpiComm) } } - auto comm = helper::CommFromMPI(mpiComm); std::shared_ptr engine; const bool isDefaultEngine = m_EngineType.empty() ? true : false; std::string engineTypeLC = m_EngineType; @@ -766,7 +763,7 @@ Engine &IO::Open(const std::string &name, const Mode mode, MPI_Comm mpiComm) Engine &IO::Open(const std::string &name, const Mode mode) { - return Open(name, mode, CommAsMPI(m_ADIOS.GetComm())); + return Open(name, mode, m_ADIOS.GetComm().Duplicate()); } Engine &IO::GetEngine(const std::string &name) diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index 75363befbd..b59fccd6ae 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -360,7 +360,7 @@ class IO * @exception std::invalid_argument if Engine with unique name is already * created with another Open, in debug mode only */ - Engine &Open(const std::string &name, const Mode mode, MPI_Comm mpiComm); + Engine &Open(const std::string &name, const Mode mode, helper::Comm comm); /** * Overloaded version that reuses the MPI_Comm object passed diff --git a/source/adios2/core/IO.tcc b/source/adios2/core/IO.tcc index 2832fa3377..c19f4356ad 100644 --- a/source/adios2/core/IO.tcc +++ b/source/adios2/core/IO.tcc @@ -18,7 +18,6 @@ #include //std::invalid_argument /// \endcond -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSMacros.h" #include "adios2/helper/adiosFunctions.h" //helper::GetType #include "adios2/toolkit/profiling/taustubs/tautimer.hpp" diff --git a/source/adios2/core/Stream.cpp b/source/adios2/core/Stream.cpp index 1ee0bb29c2..27cb944bbe 100644 --- a/source/adios2/core/Stream.cpp +++ b/source/adios2/core/Stream.cpp @@ -11,16 +11,17 @@ #include "Stream.h" #include "Stream.tcc" -#include "adios2/common/ADIOSMPI.h" +#include "adios2/helper/adiosCommDummy.h" namespace adios2 { namespace core { -Stream::Stream(const std::string &name, const Mode mode, MPI_Comm comm, +Stream::Stream(const std::string &name, const Mode mode, helper::Comm comm, const std::string engineType, const std::string hostLanguage) -: m_Name(name), m_ADIOS(std::make_shared(comm, DebugON, hostLanguage)), +: m_Name(name), + m_ADIOS(std::make_shared(std::move(comm), DebugON, hostLanguage)), m_IO(&m_ADIOS->DeclareIO(name)), m_Mode(mode), m_EngineType(engineType) { if (mode == adios2::Mode::Read) @@ -31,15 +32,15 @@ Stream::Stream(const std::string &name, const Mode mode, MPI_Comm comm, Stream::Stream(const std::string &name, const Mode mode, const std::string engineType, const std::string hostLanguage) -: Stream(name, mode, MPI_COMM_NULL, engineType, hostLanguage) +: Stream(name, mode, helper::CommDummy(), engineType, hostLanguage) { } -Stream::Stream(const std::string &name, const Mode mode, MPI_Comm comm, +Stream::Stream(const std::string &name, const Mode mode, helper::Comm comm, const std::string configFile, const std::string ioInConfigFile, const std::string hostLanguage) -: m_Name(name), - m_ADIOS(std::make_shared(configFile, comm, DebugON, hostLanguage)), +: m_Name(name), m_ADIOS(std::make_shared(configFile, std::move(comm), + DebugON, hostLanguage)), m_IO(&m_ADIOS->DeclareIO(ioInConfigFile)), m_Mode(mode) { if (mode == adios2::Mode::Read) @@ -51,7 +52,8 @@ Stream::Stream(const std::string &name, const Mode mode, MPI_Comm comm, Stream::Stream(const std::string &name, const Mode mode, const std::string configFile, const std::string ioInConfigFile, const std::string hostLanguage) -: Stream(name, mode, MPI_COMM_NULL, configFile, ioInConfigFile, hostLanguage) +: Stream(name, mode, helper::CommDummy(), configFile, ioInConfigFile, + hostLanguage) { } diff --git a/source/adios2/core/Stream.h b/source/adios2/core/Stream.h index 585061e7e7..c5ec533459 100644 --- a/source/adios2/core/Stream.h +++ b/source/adios2/core/Stream.h @@ -13,12 +13,12 @@ #include //std::shared_ptr -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSMacros.h" #include "adios2/common/ADIOSTypes.h" #include "adios2/core/ADIOS.h" #include "adios2/core/Engine.h" #include "adios2/core/IO.h" +#include "adios2/helper/adiosComm.h" namespace adios2 { @@ -47,13 +47,13 @@ class Stream */ Engine *m_Engine = nullptr; - Stream(const std::string &name, const Mode mode, MPI_Comm comm, + Stream(const std::string &name, const Mode mode, helper::Comm comm, const std::string engineType, const std::string hostLanguage); Stream(const std::string &name, const Mode mode, const std::string engineType, const std::string hostLanguage); - Stream(const std::string &name, const Mode mode, MPI_Comm comm, + Stream(const std::string &name, const Mode mode, helper::Comm comm, const std::string configFile, const std::string ioInConfigFile, const std::string hostLanguage); diff --git a/source/utils/adios_reorganize/Reorganize.cpp b/source/utils/adios_reorganize/Reorganize.cpp index 072abbf25a..ab6ffbf5c2 100644 --- a/source/utils/adios_reorganize/Reorganize.cpp +++ b/source/utils/adios_reorganize/Reorganize.cpp @@ -24,16 +24,20 @@ #include #include -#include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSMacros.h" #include "adios2/core/ADIOS.h" #include "adios2/core/Engine.h" #include "adios2/core/IO.h" #include "adios2/helper/adiosComm.h" -#include "adios2/helper/adiosCommMPI.h" #include "adios2/helper/adiosFunctions.h" #include "adios2/helper/adiosString.h" +#ifdef ADIOS2_HAVE_MPI +#include "adios2/helper/adiosCommMPI.h" +#else +#include "adios2/helper/adiosCommDummy.h" +#endif + // C headers #include #include @@ -46,10 +50,14 @@ namespace utils Reorganize::Reorganize(int argc, char *argv[]) : Utils("adios_reorganize", argc, argv) { +#ifdef ADIOS2_HAVE_MPI { auto commWorld = helper::CommFromMPI(MPI_COMM_WORLD); m_Comm = commWorld.Split(m_CommSplitColor, 0); } +#else + m_Comm = helper::CommDummy(); +#endif m_Rank = m_Comm.Rank(); m_Size = m_Comm.Size(); @@ -123,7 +131,7 @@ void Reorganize::Run() print0("Write method = ", wmethodname); print0("Write method parameters = ", wmethodparam_str); - core::ADIOS adios(CommAsMPI(m_Comm), true, "C++"); + core::ADIOS adios(m_Comm.Duplicate(), true, "C++"); core::IO &io = adios.DeclareIO("group"); print0("Waiting to open stream ", infilename, "..."); From c47c2b965dd6533acd1ee3edcf1e135669463794 Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 20 Sep 2019 13:57:04 -0400 Subject: [PATCH 34/34] helper: Port Comm MPI backend away from SMPI_ wrappers If `CommFromMPI()` is given `MPI_COMM_NULL`, fall back to `CommDummy()` instead. --- source/adios2/helper/adiosCommMPI.cpp | 57 +++++++++++++++------------ 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/source/adios2/helper/adiosCommMPI.cpp b/source/adios2/helper/adiosCommMPI.cpp index 6c10cca505..639a5d23fe 100644 --- a/source/adios2/helper/adiosCommMPI.cpp +++ b/source/adios2/helper/adiosCommMPI.cpp @@ -13,6 +13,7 @@ #include #include "adiosComm.h" +#include "adiosCommDummy.h" #include "adios2/common/ADIOSMPI.h" #include "adios2/common/ADIOSTypes.h" @@ -172,7 +173,7 @@ CommImplMPI::~CommImplMPI() if (m_MPIComm != MPI_COMM_NULL && m_MPIComm != MPI_COMM_WORLD && m_MPIComm != MPI_COMM_SELF) { - SMPI_Comm_free(&m_MPIComm); + MPI_Comm_free(&m_MPIComm); } } } @@ -182,7 +183,7 @@ void CommImplMPI::Free(const std::string &hint) if (m_MPIComm != MPI_COMM_NULL && m_MPIComm != MPI_COMM_WORLD && m_MPIComm != MPI_COMM_SELF) { - CheckMPIReturn(SMPI_Comm_free(&m_MPIComm), hint); + CheckMPIReturn(MPI_Comm_free(&m_MPIComm), hint); } } @@ -209,30 +210,30 @@ std::unique_ptr CommImplMPI::World(const std::string &) const int CommImplMPI::Rank() const { int rank; - CheckMPIReturn(SMPI_Comm_rank(m_MPIComm, &rank), {}); + CheckMPIReturn(MPI_Comm_rank(m_MPIComm, &rank), {}); return rank; } int CommImplMPI::Size() const { int size; - CheckMPIReturn(SMPI_Comm_size(m_MPIComm, &size), {}); + CheckMPIReturn(MPI_Comm_size(m_MPIComm, &size), {}); return size; } void CommImplMPI::Barrier(const std::string &hint) const { - CheckMPIReturn(SMPI_Barrier(m_MPIComm), hint); + CheckMPIReturn(MPI_Barrier(m_MPIComm), hint); } void CommImplMPI::Allgather(const void *sendbuf, size_t sendcount, Datatype sendtype, void *recvbuf, size_t recvcount, Datatype recvtype, const std::string &hint) const { - CheckMPIReturn(SMPI_Allgather(sendbuf, static_cast(sendcount), - ToMPI(sendtype), recvbuf, - static_cast(recvcount), ToMPI(recvtype), - m_MPIComm), + CheckMPIReturn(MPI_Allgather(sendbuf, static_cast(sendcount), + ToMPI(sendtype), recvbuf, + static_cast(recvcount), ToMPI(recvtype), + m_MPIComm), hint); } @@ -240,8 +241,8 @@ void CommImplMPI::Allreduce(const void *sendbuf, void *recvbuf, size_t count, Datatype datatype, Comm::Op op, const std::string &hint) const { - CheckMPIReturn(SMPI_Allreduce(sendbuf, recvbuf, static_cast(count), - ToMPI(datatype), ToMPI(op), m_MPIComm), + CheckMPIReturn(MPI_Allreduce(sendbuf, recvbuf, static_cast(count), + ToMPI(datatype), ToMPI(op), m_MPIComm), hint); } @@ -254,8 +255,8 @@ void CommImplMPI::Bcast(void *buffer, size_t count, Datatype datatype, int root, unsigned char *blockBuf = static_cast(buffer); while (inputSize > 0) { - CheckMPIReturn(SMPI_Bcast(blockBuf, static_cast(blockSize), - ToMPI(datatype), root, m_MPIComm), + CheckMPIReturn(MPI_Bcast(blockBuf, static_cast(blockSize), + ToMPI(datatype), root, m_MPIComm), hint); blockBuf += blockSize * CommImpl::SizeOf(datatype); inputSize -= blockSize; @@ -268,10 +269,10 @@ void CommImplMPI::Gather(const void *sendbuf, size_t sendcount, Datatype recvtype, int root, const std::string &hint) const { - CheckMPIReturn(SMPI_Gather(sendbuf, static_cast(sendcount), - ToMPI(sendtype), recvbuf, - static_cast(recvcount), ToMPI(recvtype), - root, m_MPIComm), + CheckMPIReturn(MPI_Gather(sendbuf, static_cast(sendcount), + ToMPI(sendtype), recvbuf, + static_cast(recvcount), ToMPI(recvtype), + root, m_MPIComm), hint); } @@ -294,10 +295,10 @@ void CommImplMPI::Gatherv(const void *sendbuf, size_t sendcount, std::transform(displs, displs + size, std::back_inserter(displsInt), cast); } - CheckMPIReturn(SMPI_Gatherv(sendbuf, static_cast(sendcount), - ToMPI(sendtype), recvbuf, countsInt.data(), - displsInt.data(), ToMPI(recvtype), root, - m_MPIComm), + CheckMPIReturn(MPI_Gatherv(sendbuf, static_cast(sendcount), + ToMPI(sendtype), recvbuf, countsInt.data(), + displsInt.data(), ToMPI(recvtype), root, + m_MPIComm), hint); } @@ -305,8 +306,8 @@ void CommImplMPI::Reduce(const void *sendbuf, void *recvbuf, size_t count, Datatype datatype, Comm::Op op, int root, const std::string &hint) const { - CheckMPIReturn(SMPI_Reduce(sendbuf, recvbuf, static_cast(count), - ToMPI(datatype), ToMPI(op), root, m_MPIComm), + CheckMPIReturn(MPI_Reduce(sendbuf, recvbuf, static_cast(count), + ToMPI(datatype), ToMPI(op), root, m_MPIComm), hint); } @@ -314,8 +315,8 @@ void CommImplMPI::ReduceInPlace(void *buf, size_t count, Datatype datatype, Comm::Op op, int root, const std::string &hint) const { - CheckMPIReturn(SMPI_Reduce(MPI_IN_PLACE, buf, static_cast(count), - ToMPI(datatype), ToMPI(op), root, m_MPIComm), + CheckMPIReturn(MPI_Reduce(MPI_IN_PLACE, buf, static_cast(count), + ToMPI(datatype), ToMPI(op), root, m_MPIComm), hint); } @@ -530,8 +531,12 @@ Comm::Status CommReqImplMPI::Wait(const std::string &hint) Comm CommFromMPI(MPI_Comm mpiComm) { + if (mpiComm == MPI_COMM_NULL) + { + return CommDummy(); + } MPI_Comm newComm; - SMPI_Comm_dup(mpiComm, &newComm); + MPI_Comm_dup(mpiComm, &newComm); auto comm = std::unique_ptr(new CommImplMPI(newComm)); return CommImpl::MakeComm(std::move(comm)); }