From 497c68cebb56ba8c79a4557643c12bd4dc8ecee4 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Jul 2020 01:32:28 -0400 Subject: [PATCH 1/6] replaced MPI based handshake with file based, removed multi-app support from SSC --- source/adios2/engine/ssc/SscReader.cpp | 44 +- source/adios2/engine/ssc/SscReader.h | 2 - source/adios2/engine/ssc/SscWriter.cpp | 44 +- source/adios2/engine/ssc/SscWriter.h | 2 - source/adios2/helper/adiosMpiHandshake.cpp | 408 +++++----------- source/adios2/helper/adiosMpiHandshake.h | 127 ++--- testing/adios2/engine/ssc/CMakeLists.txt | 3 - testing/adios2/engine/ssc/TestSscMultiApp.cpp | 443 ------------------ 8 files changed, 160 insertions(+), 913 deletions(-) delete mode 100644 testing/adios2/engine/ssc/TestSscMultiApp.cpp diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index 6ec60472e6..73c3e7e455 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -251,47 +251,19 @@ void SscReader::SyncMpiPattern() { TAU_SCOPED_TIMER_FUNC(); - if (m_Verbosity >= 5) - { - std::cout << "SscReader::SyncMpiPattern, World Rank " << m_StreamRank - << ", Reader Rank " << m_ReaderRank << std::endl; - } - - m_MpiHandshake.Handshake(m_Name, 'r', m_OpenTimeoutSecs, m_MaxStreamsPerApp, - m_MaxFilenameLength, m_RendezvousAppCount, - CommAsMPI(m_Comm)); - - std::vector allStreamRanks; - std::vector allWriterRanks; - - for (const auto &app : m_MpiHandshake.GetWriterMap(m_Name)) - { - for (int rank : app.second) - { - allWriterRanks.push_back(rank); - allStreamRanks.push_back(rank); - } - } - - for (const auto &app : m_MpiHandshake.GetReaderMap(m_Name)) - { - for (int rank : app.second) - { - allStreamRanks.push_back(rank); - } - } + auto appRankMaps = + helper::Handshake(m_Name, 'r', m_OpenTimeoutSecs, CommAsMPI(m_Comm)); MPI_Group worldGroup; + MPI_Group streamGroup; + MPI_Comm_group(MPI_COMM_WORLD, &worldGroup); - MPI_Group_incl(worldGroup, allWriterRanks.size(), allWriterRanks.data(), + MPI_Group_incl(worldGroup, appRankMaps[1].size(), appRankMaps[1].data(), &m_MpiAllWritersGroup); + MPI_Group_incl(worldGroup, appRankMaps[0].size(), appRankMaps[0].data(), + &streamGroup); - MPI_Comm_group(MPI_COMM_WORLD, &worldGroup); - std::sort(allStreamRanks.begin(), allStreamRanks.end()); - MPI_Group allWorkersGroup; - MPI_Group_incl(worldGroup, allStreamRanks.size(), allStreamRanks.data(), - &allWorkersGroup); - MPI_Comm_create_group(MPI_COMM_WORLD, allWorkersGroup, 0, &m_StreamComm); + MPI_Comm_create_group(MPI_COMM_WORLD, streamGroup, 0, &m_StreamComm); } void SscReader::SyncWritePattern() diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h index 4f702a6c53..2b9bf5b5ae 100644 --- a/source/adios2/engine/ssc/SscReader.h +++ b/source/adios2/engine/ssc/SscReader.h @@ -60,8 +60,6 @@ class SscReader : public Engine int m_ReaderRank; int m_ReaderSize; - helper::MpiHandshake m_MpiHandshake; - void SyncMpiPattern(); void SyncWritePattern(); void SyncReadPattern(); diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp index d16f2cd1bb..e97fa9b181 100644 --- a/source/adios2/engine/ssc/SscWriter.cpp +++ b/source/adios2/engine/ssc/SscWriter.cpp @@ -217,47 +217,19 @@ void SscWriter::SyncMpiPattern() { TAU_SCOPED_TIMER_FUNC(); - if (m_Verbosity >= 5) - { - std::cout << "SscWriter::SyncMpiPattern, World Rank " << m_StreamRank - << ", Writer Rank " << m_WriterRank << std::endl; - } - - m_MpiHandshake.Handshake(m_Name, 'w', m_OpenTimeoutSecs, m_MaxStreamsPerApp, - m_MaxFilenameLength, m_RendezvousAppCount, - CommAsMPI(m_Comm)); + auto appRankMaps = + helper::Handshake(m_Name, 'w', m_OpenTimeoutSecs, CommAsMPI(m_Comm)); - std::vector allStreamRanks; - std::vector allReaderRanks; - - for (const auto &app : m_MpiHandshake.GetWriterMap(m_Name)) - { - for (int rank : app.second) - { - allStreamRanks.push_back(rank); - } - } - - for (const auto &app : m_MpiHandshake.GetReaderMap(m_Name)) - { - for (int rank : app.second) - { - allStreamRanks.push_back(rank); - allReaderRanks.push_back(rank); - } - } MPI_Group worldGroup; - MPI_Group allReadersGroup; + MPI_Group streamGroup; + MPI_Comm_group(MPI_COMM_WORLD, &worldGroup); - MPI_Group_incl(worldGroup, allReaderRanks.size(), allReaderRanks.data(), + MPI_Group_incl(worldGroup, appRankMaps[2].size(), appRankMaps[2].data(), &m_MpiAllReadersGroup); + MPI_Group_incl(worldGroup, appRankMaps[0].size(), appRankMaps[0].data(), + &streamGroup); - MPI_Comm_group(MPI_COMM_WORLD, &worldGroup); - std::sort(allStreamRanks.begin(), allStreamRanks.end()); - MPI_Group allWorkersGroup; - MPI_Group_incl(worldGroup, allStreamRanks.size(), allStreamRanks.data(), - &allWorkersGroup); - MPI_Comm_create_group(MPI_COMM_WORLD, allWorkersGroup, 0, &m_StreamComm); + MPI_Comm_create_group(MPI_COMM_WORLD, streamGroup, 0, &m_StreamComm); } void SscWriter::SyncWritePattern() diff --git a/source/adios2/engine/ssc/SscWriter.h b/source/adios2/engine/ssc/SscWriter.h index 74bf4e6487..abe2676a07 100644 --- a/source/adios2/engine/ssc/SscWriter.h +++ b/source/adios2/engine/ssc/SscWriter.h @@ -62,8 +62,6 @@ class SscWriter : public Engine int m_WriterRank; int m_WriterSize; - helper::MpiHandshake m_MpiHandshake; - void SyncMpiPattern(); void SyncWritePattern(); void SyncReadPattern(); diff --git a/source/adios2/helper/adiosMpiHandshake.cpp b/source/adios2/helper/adiosMpiHandshake.cpp index 8fa3301f05..435abf884f 100644 --- a/source/adios2/helper/adiosMpiHandshake.cpp +++ b/source/adios2/helper/adiosMpiHandshake.cpp @@ -9,345 +9,171 @@ */ #include "adiosMpiHandshake.h" -#include #include -#include -#include +#include +#include #include +#include +#include namespace adios2 { namespace helper { -std::vector MpiHandshake::m_Buffer; -std::vector> MpiHandshake::m_SendRequests; -std::vector> MpiHandshake::m_RecvRequests; -size_t MpiHandshake::m_MaxStreamsPerApp; -size_t MpiHandshake::m_MaxFilenameLength; -size_t MpiHandshake::m_ItemSize; -std::map MpiHandshake::m_RendezvousAppCounts; -size_t MpiHandshake::m_StreamID = 0; -int MpiHandshake::m_WorldSize; -int MpiHandshake::m_WorldRank; -int MpiHandshake::m_LocalSize; -int MpiHandshake::m_LocalRank; -int MpiHandshake::m_LocalMasterRank; -std::map>> MpiHandshake::m_WritersMap; -std::map>> MpiHandshake::m_ReadersMap; -std::map MpiHandshake::m_AppsSize; - -size_t MpiHandshake::PlaceInBuffer(size_t stream, int rank) +const std::vector> Handshake(const std::string &filename, + const char mode, + const int timeoutSeconds, + MPI_Comm localComm) { - return rank * m_MaxStreamsPerApp * m_ItemSize + stream * m_ItemSize; -} + std::vector> ret(3); -void MpiHandshake::Test() -{ - int success = 0; - MPI_Status status; + int localRank; + int localSize; + int worldRank; + int worldSize; + + MPI_Comm_rank(localComm, &localRank); + MPI_Comm_size(localComm, &localSize); + + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + std::vector allLocalRanks(localSize); + + MPI_Gather(&worldRank, 1, MPI_INT, allLocalRanks.data(), 1, MPI_INT, 0, + localComm); - for (int rank = 0; rank < m_WorldSize; ++rank) + if (localRank == 0) { - for (size_t stream = 0; stream < m_MaxStreamsPerApp; ++stream) + std::ofstream fs; + fs.open(filename + "." + mode); + for (auto rank : allLocalRanks) { - MPI_Test(&m_RecvRequests[rank][stream], &success, &status); - if (success) + fs << rank << std::endl; + } + fs.close(); + + if (mode == 'r') + { + + for (auto i : allLocalRanks) { - size_t offset = PlaceInBuffer(stream, rank); - char mode = m_Buffer[offset]; - offset += sizeof(char); - int appMasterRank; - std::memcpy(&appMasterRank, m_Buffer.data() + offset, - sizeof(appMasterRank)); - offset += sizeof(int); - int appSize; - std::memcpy(&appSize, m_Buffer.data() + offset, - sizeof(appSize)); - offset += sizeof(int); - std::string filename = m_Buffer.data() + offset; - m_AppsSize[appMasterRank] = appSize; - if (mode == 'w') + ret[0].push_back(i); + ret[2].push_back(i); + } + + std::ofstream fsc; + fsc.open(filename + ".r.c"); + fsc << "completed"; + fsc.close(); + + while (true) + { + std::ifstream fs; + try { - auto &ranks = m_WritersMap[filename][appMasterRank]; - ranks.insert(rank); + fs.open(filename + ".w.c"); + std::string line; + std::getline(fs, line); + if (line != "completed") + { + continue; + } + fs.close(); + remove((filename + ".w.c\0").c_str()); + break; } - else if (mode == 'r') + catch (...) { - auto &ranks = m_ReadersMap[filename][appMasterRank]; - ranks.insert(rank); + continue; } } - } - } -} -bool MpiHandshake::Check(const std::string &filename, const bool verbose) -{ - Test(); - - // check if RendezvousAppCount reached - - if (m_WritersMap[filename].size() + m_ReadersMap[filename].size() != - m_RendezvousAppCounts[filename]) - { - if (verbose) - { - std::cout << "MpiHandshake Rank " << m_WorldRank << " Stream " - << filename << ": " << m_WritersMap[filename].size() - << " writers and " << m_ReadersMap[filename].size() - << " readers found out of " - << m_RendezvousAppCounts[filename] - << " total rendezvous apps" << std::endl; + std::ifstream fs; + fs.open(filename + ".w"); + for (std::string line; std::getline(fs, line);) + { + ret[0].push_back(std::stoi(line)); + ret[1].push_back(std::stoi(line)); + } + fs.close(); + remove((filename + ".w\0").c_str()); } - return false; - } + else if (mode == 'w') + { + for (auto i : allLocalRanks) + { + ret[0].push_back(i); + ret[1].push_back(i); + } - // check if all ranks' info is received + std::ofstream fsc; + fsc.open(filename + ".w.c"); + fsc << "completed"; + fsc.close(); - for (const auto &app : m_WritersMap[filename]) - { - if (app.second.size() != m_AppsSize[app.first]) - { - if (verbose) + while (true) { - std::cout << "MpiHandshake Rank " << m_WorldRank << " Stream " - << filename << ": " - << " App master rank " << app.first << ", Expected " - << m_AppsSize[app.first] << " ranks " - << ", Received " << app.second.size() << " ranks: "; - for (const auto r : app.second) + std::ifstream fs; + try + { + fs.open(filename + ".r.c"); + std::string line; + std::getline(fs, line); + if (line != "completed") + { + continue; + } + fs.close(); + remove((filename + ".r.c\0").c_str()); + break; + } + catch (...) { - std::cout << r << ", "; + continue; } - std::cout << std::endl; } - return false; - } - } - for (const auto &app : m_ReadersMap[filename]) - { - if (app.second.size() != m_AppsSize[app.first]) - { - if (verbose) + std::ifstream fs; + fs.open(filename + ".r"); + for (std::string line; std::getline(fs, line);) { - std::cout << "MpiHandshake Rank " << m_WorldRank << " Stream " - << filename << ": " - << " App master rank " << app.first << ", Expected " - << m_AppsSize[app.first] << " ranks " - << ", Received " << app.second.size() << " ranks: "; - for (const auto r : app.second) - { - std::cout << r << ", "; - } - std::cout << std::endl; + ret[0].push_back(std::stoi(line)); + ret[2].push_back(std::stoi(line)); } - return false; + fs.close(); + remove((filename + ".r\0").c_str()); } } - return true; -} - -void MpiHandshake::Handshake(const std::string &filename, const char mode, - const int timeoutSeconds, - const size_t maxStreamsPerApp, - const size_t maxFilenameLength, - const size_t rendezvousAppCountForStream, - MPI_Comm localComm) -{ - - // initialize variables - - if (filename.size() > maxFilenameLength) - { - throw(std::runtime_error("Filename too long")); - } - - MPI_Comm_size(MPI_COMM_WORLD, &m_WorldSize); - MPI_Comm_rank(MPI_COMM_WORLD, &m_WorldRank); - MPI_Comm_size(localComm, &m_LocalSize); - MPI_Comm_rank(localComm, &m_LocalRank); - m_MaxStreamsPerApp = maxStreamsPerApp; - m_MaxFilenameLength = maxFilenameLength; - m_RendezvousAppCounts[filename] = rendezvousAppCountForStream; - - m_SendRequests.resize(m_WorldSize); - m_RecvRequests.resize(m_WorldSize); - for (int rank = 0; rank < m_WorldSize; ++rank) - { - m_SendRequests[rank].resize(maxStreamsPerApp); - m_RecvRequests[rank].resize(maxStreamsPerApp); - } - - m_ItemSize = maxFilenameLength + sizeof(char) + sizeof(int) * 2; - m_Buffer.resize(m_WorldSize * maxStreamsPerApp * m_ItemSize, '\0'); - - // broadcast local master rank's world rank to use as app ID + int dims[3]; - if (m_LocalRank == 0) + if (localRank == 0) { - m_LocalMasterRank = m_WorldRank; - } - MPI_Bcast(&m_LocalMasterRank, 1, MPI_INT, 0, localComm); - - // start receiving - - for (int rank = 0; rank < m_WorldSize; ++rank) - { - for (size_t stream = 0; stream < maxStreamsPerApp; ++stream) + for (int i = 0; i < 3; ++i) { - MPI_Irecv(m_Buffer.data() + PlaceInBuffer(stream, rank), - static_cast(m_ItemSize), MPI_CHAR, rank, rank, - MPI_COMM_WORLD, &m_RecvRequests[rank][stream]); + dims[i] = ret[i].size(); + std::sort(ret[i].begin(), ret[i].end()); } } - // start sending - - size_t offset = 0; - std::vector buffer(m_ItemSize); - std::memcpy(buffer.data(), &mode, sizeof(char)); - offset += sizeof(char); - std::memcpy(buffer.data() + offset, &m_LocalMasterRank, sizeof(int)); - offset += sizeof(int); - std::memcpy(buffer.data() + offset, &m_LocalSize, sizeof(int)); - offset += sizeof(int); - std::memcpy(buffer.data() + offset, filename.data(), filename.size()); - - for (int rank = 0; rank < m_WorldSize; ++rank) - { - MPI_Isend(buffer.data(), static_cast(m_ItemSize), MPI_CHAR, rank, - m_WorldRank, MPI_COMM_WORLD, - &m_SendRequests[rank][m_StreamID]); - } - - // wait and check if required RendezvousAppCount reached + MPI_Bcast(dims, 3, MPI_INT, 0, localComm); - auto startTime = std::chrono::system_clock::now(); - while (!Check(filename, false)) + if (localRank != 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(5)); - auto nowTime = std::chrono::system_clock::now(); - auto duration = std::chrono::duration_cast( - nowTime - startTime); - if (duration.count() > timeoutSeconds) + for (int i = 0; i < 3; ++i) { - Check(filename, true); - throw(std::runtime_error("Mpi handshake timeout on Rank " + - std::to_string(m_WorldRank) + - " for Stream " + filename)); + ret[i].resize(dims[i]); } } - // clean up MPI requests - - for (auto &rs : m_RecvRequests) + for (int i = 0; i < 3; ++i) { - for (auto &r : rs) - { - MPI_Status status; - int success; - MPI_Test(&r, &success, &status); - if (!success) - { - MPI_Cancel(&r); - } - } + MPI_Bcast(ret[i].data(), ret[i].size(), MPI_INT, 0, localComm); } - m_RecvRequests.clear(); - - ++m_StreamID; -} - -const std::map> & -MpiHandshake::GetWriterMap(const std::string &filename) -{ - return m_WritersMap[filename]; -} -const std::map> & -MpiHandshake::GetReaderMap(const std::string &filename) -{ - return m_ReadersMap[filename]; -} - -void MpiHandshake::PrintMaps(const int printRank, const std::string &filename) -{ - if (m_WorldRank == printRank) - { - std::cout << "Printing MPI handshake map for Stream " << filename - << " from Rank " << printRank << std::endl; - std::cout << " Writers: " << std::endl; - - for (const auto &app : m_WritersMap[filename]) - { - std::cout << " App Master Rank " << app.first << std::endl; - std::cout << " "; - for (const auto &rank : app.second) - { - std::cout << rank << ", "; - } - std::cout << std::endl; - } - std::cout << " Readers: " << std::endl; - for (const auto &app : m_ReadersMap[filename]) - { - std::cout << " App Master Rank " << app.first << std::endl; - std::cout << " "; - for (const auto &rank : app.second) - { - std::cout << rank << ", "; - } - std::cout << std::endl; - } - } -} -void MpiHandshake::PrintMaps() -{ - for (int printRank = 0; printRank < m_WorldSize; ++printRank) - { - MPI_Barrier(MPI_COMM_WORLD); - if (m_WorldRank == printRank) - { - std::cout << "For rank " << printRank - << "============================================" - << std::endl; - std::cout << "Writers: " << std::endl; - for (const auto &stream : m_WritersMap) - { - std::cout << " Stream " << stream.first << std::endl; - for (const auto &app : stream.second) - { - std::cout << " App Master Rank " << app.first - << std::endl; - std::cout << " "; - for (const auto &rank : app.second) - { - std::cout << rank << ", "; - } - std::cout << std::endl; - } - } - std::cout << "Readers: " << std::endl; - for (const auto &stream : m_ReadersMap) - { - std::cout << " Stream " << stream.first << std::endl; - for (const auto &app : stream.second) - { - std::cout << " App Master Rank " << app.first - << std::endl; - std::cout << " "; - for (const auto &rank : app.second) - { - std::cout << rank << ", "; - } - std::cout << std::endl; - } - } - } - } + return ret; } } // end namespace helper diff --git a/source/adios2/helper/adiosMpiHandshake.h b/source/adios2/helper/adiosMpiHandshake.h index 1a35db086e..e8cc462a39 100644 --- a/source/adios2/helper/adiosMpiHandshake.h +++ b/source/adios2/helper/adiosMpiHandshake.h @@ -16,114 +16,41 @@ #error "Do not include adiosMpiHandshake.h without ADIOS2_HAVE_MPI." #endif -#include #include -#include #include -#include namespace adios2 { namespace helper { -class MpiHandshake -{ -public: - /** - * Start the handshake operations and wait until the rendezvous conditions - * are reached, or timeout. - * - * @param filename: name of the staging stream, must be within the length of - * maxFilenameLength - * - * @param mode: 'r' or 'w', read or write - * - * @param timeoutSeconds: timeout for the handshake, will throw exception - * when reaching this timeout - * - * @param maxStreamsPerApp: the maximum number of streams that all apps - * sharing this MPI_COMM_WORLD can possibly open. It is required that this - * number is consistent across all ranks. This is used for pre-allocating - * the vectors holding MPI requests and must be specified correctly, - * otherwise strange errors could occur. This class does not provide any - * mechanism to check whether this number being passed is actually correct - * or not accross all ranks, because implementing this logic for an - * arbitrary communication pattern is overly expensive, if not impossible. - * - * @param maxFilenameLength: the maximum possible length of filename that - * all apps sharing this MPI_COMM_WORLD could possibly define. It is - * required that this number is consistent across all ranks. This is used - * for pre-allocating the buffer for aggregating the global MPI information. - * An exception will be thrown if any filename on any rank is found to be - * longer than this. - * - * @param rendezvousAppCountForStream: the number of apps, including both - * writers and readers, that will work on this stream. The function will - * block until it receives the MPI handshake information from all these - * apps, or until timeoutSeconds is passed. - * - * @param localComm: local MPI communicator for the app - */ - static void Handshake(const std::string &filename, const char mode, - const int timeoutSeconds, - const size_t maxStreamsPerApp, - const size_t maxFilenameLength, - const size_t rendezvousAppCountForStream, - MPI_Comm localComm); - - /** - * Get the writer map of all apps participating the stream. - * - * @param filename: name of the staging stream - * - * @return map of all writer apps participating the stream. Key is the world - * rank of the master rank of the local communicator of a participating - * writer app. Value is a set of all world ranks of this writer app - */ - static const std::map> & - GetWriterMap(const std::string &filename); - - /** - * Get the reader map of all apps participating the stream. - * - * @param filename: name of the staging stream - * - * @return map of all reader apps participating the stream. Key is the world - * rank of the master rank of the local communicator of a participating - * reader app. Value is a set of all world ranks of this reader app - */ - static const std::map> & - GetReaderMap(const std::string &filename); - -private: - static void Test(); - static bool Check(const std::string &filename, const bool verbose); - static size_t PlaceInBuffer(const size_t stream, const int rank); - static void PrintMaps(); - static void PrintMaps(const int printRank, const std::string &filename); - - static std::vector m_Buffer; - static std::vector> m_SendRequests; - static std::vector> m_RecvRequests; - static size_t m_MaxStreamsPerApp; - static size_t m_MaxFilenameLength; - static size_t m_ItemSize; - static std::map m_RendezvousAppCounts; - static size_t m_StreamID; - static int m_WorldSize; - static int m_WorldRank; - static int m_LocalSize; - static int m_LocalRank; - static int m_LocalMasterRank; - - // > - static std::map>> m_WritersMap; - static std::map>> m_ReadersMap; - - // - static std::map m_AppsSize; -}; +/** + * Start the handshake operations and wait until the rendezvous conditions + * are reached, or timeout. + * + * @param filename: name of the staging stream, must be within the length of + * maxFilenameLength + * + * @param mode: 'r' or 'w', read or write + * + * @param timeoutSeconds: timeout for the handshake, will throw exception + * when reaching this timeout + * + * @param rendezvousAppCountForStream: the number of apps, including both + * writers and readers, that will work on this stream. The function will + * block until it receives the MPI handshake information from all these + * apps, or until timeoutSeconds is passed. + * + * @param localComm: local MPI communicator for the app + * + * @return 3 vectors of ranks. [0] is the vector of all writer and reader ranks + * for stream *filename*. [1] is the vector of all writer ranks for stream + * *filename*. [2] is the vector of all reader ranks for stream *filename*. + */ +const std::vector> Handshake(const std::string &filename, + const char mode, + const int timeoutSeconds, + MPI_Comm localComm); } // end namespace helper } // end namespace adios2 diff --git a/testing/adios2/engine/ssc/CMakeLists.txt b/testing/adios2/engine/ssc/CMakeLists.txt index 1ff74c3330..9f7b008819 100644 --- a/testing/adios2/engine/ssc/CMakeLists.txt +++ b/testing/adios2/engine/ssc/CMakeLists.txt @@ -45,9 +45,6 @@ if(ADIOS2_HAVE_MPI) gtest_add_tests_helper(MoreWritersThanReaders MPI_ONLY Ssc Engine.SSC. "") SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscMoreWritersThanReaders.MPI "" TRUE) - gtest_add_tests_helper(MultiApp MPI_ONLY Ssc Engine.SSC. "") - SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscMultiApp.MPI "" TRUE) - gtest_add_tests_helper(Xgc2Way MPI_ONLY Ssc Engine.SSC. "") SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscXgc2Way.MPI "" TRUE) diff --git a/testing/adios2/engine/ssc/TestSscMultiApp.cpp b/testing/adios2/engine/ssc/TestSscMultiApp.cpp deleted file mode 100644 index 60886ba6d1..0000000000 --- a/testing/adios2/engine/ssc/TestSscMultiApp.cpp +++ /dev/null @@ -1,443 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - */ - -#include "TestSscCommon.h" -#include -#include -#include -#include -#include - -using namespace adios2; -int mpiRank = 0; -int mpiSize = 1; -int mpiGroup; -MPI_Comm mpiComm; - -class SscEngineTest : public ::testing::Test -{ -public: - SscEngineTest() = default; -}; - -void Writer1(const Dims &shape, const Dims &start, const Dims &count, - const size_t steps, const adios2::Params &engineParams, - const std::string &name) -{ - size_t datasize = std::accumulate(count.begin(), count.end(), 1, - std::multiplies()); - adios2::ADIOS adios(mpiComm); - adios2::IO dataManIO = adios.DeclareIO("WAN"); - dataManIO.SetEngine("ssc"); - dataManIO.SetParameters(engineParams); - std::vector myChars(datasize); - std::vector myUChars(datasize); - std::vector myShorts(datasize); - std::vector myUShorts(datasize); - std::vector myInts(datasize); - std::vector myUInts(datasize); - std::vector myFloats(datasize); - std::vector myDoubles(datasize); - std::vector> myComplexes(datasize); - std::vector> myDComplexes(datasize); - auto bpChars = - dataManIO.DefineVariable("bpChars", shape, start, count); - auto bpUChars = dataManIO.DefineVariable("bpUChars", shape, - start, count); - auto bpShorts = - dataManIO.DefineVariable("bpShorts", shape, start, count); - auto bpUShorts = dataManIO.DefineVariable( - "bpUShorts", shape, start, count); - auto bpInts = dataManIO.DefineVariable("bpInts", shape, start, count); - auto bpUInts = - dataManIO.DefineVariable("bpUInts", shape, start, count); - auto bpFloats = - dataManIO.DefineVariable("bpFloats", shape, start, count); - auto bpDoubles = - dataManIO.DefineVariable("bpDoubles", shape, start, count); - auto bpComplexes = dataManIO.DefineVariable>( - "bpComplexes", shape, start, count); - auto bpDComplexes = dataManIO.DefineVariable>( - "bpDComplexes", shape, start, count); - dataManIO.DefineAttribute("AttInt", 110); - adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write); - for (int i = 0; i < steps; ++i) - { - dataManWriter.BeginStep(); - GenData(myChars, i, start, count, shape); - GenData(myUChars, i, start, count, shape); - GenData(myShorts, i, start, count, shape); - GenData(myUShorts, i, start, count, shape); - GenData(myInts, i, start, count, shape); - GenData(myUInts, i, start, count, shape); - GenData(myFloats, i, start, count, shape); - GenData(myDoubles, i, start, count, shape); - GenData(myComplexes, i, start, count, shape); - GenData(myDComplexes, i, start, count, shape); - dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync); - dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync); - dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync); - dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync); - dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync); - dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync); - dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync); - dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync); - dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync); - dataManWriter.Put(bpDComplexes, myDComplexes.data(), - adios2::Mode::Sync); - dataManWriter.EndStep(); - } - dataManWriter.Close(); -} - -void Writer2(const Dims &shape, const Dims &start, const Dims &count, - const size_t steps, const adios2::Params &engineParams, - const std::string &name) -{ - size_t datasize = std::accumulate(count.begin(), count.end(), 1, - std::multiplies()); - adios2::ADIOS adios(mpiComm); - adios2::IO dataManIO = adios.DeclareIO("WAN"); - dataManIO.SetEngine("ssc"); - dataManIO.SetParameters(engineParams); - std::vector myChars(datasize); - std::vector myUChars(datasize); - std::vector myShorts(datasize); - std::vector myUShorts(datasize); - std::vector myInts(datasize); - std::vector myUInts(datasize); - std::vector myFloats(datasize); - std::vector myDoubles(datasize); - std::vector> myComplexes(datasize); - std::vector> myDComplexes(datasize); - auto bpChars = - dataManIO.DefineVariable("bpChars2", shape, start, count); - auto bpUChars = dataManIO.DefineVariable("bpUChars2", shape, - start, count); - auto bpShorts = - dataManIO.DefineVariable("bpShorts2", shape, start, count); - auto bpUShorts = dataManIO.DefineVariable( - "bpUShorts2", shape, start, count); - auto bpInts = dataManIO.DefineVariable("bpInts2", shape, start, count); - auto bpUInts = - dataManIO.DefineVariable("bpUInts2", shape, start, count); - auto bpFloats = - dataManIO.DefineVariable("bpFloats2", shape, start, count); - auto bpDoubles = - dataManIO.DefineVariable("bpDoubles2", shape, start, count); - auto bpComplexes = dataManIO.DefineVariable>( - "bpComplexes2", shape, start, count); - auto bpDComplexes = dataManIO.DefineVariable>( - "bpDComplexes2", shape, start, count); - dataManIO.DefineAttribute("AttInt2", 111); - adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write); - for (int i = 0; i < steps; ++i) - { - dataManWriter.BeginStep(); - GenData(myChars, i, start, count, shape); - GenData(myUChars, i, start, count, shape); - GenData(myShorts, i, start, count, shape); - GenData(myUShorts, i, start, count, shape); - GenData(myInts, i, start, count, shape); - GenData(myUInts, i, start, count, shape); - GenData(myFloats, i, start, count, shape); - GenData(myDoubles, i, start, count, shape); - GenData(myComplexes, i, start, count, shape); - GenData(myDComplexes, i, start, count, shape); - dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync); - dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync); - dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync); - dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync); - dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync); - dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync); - dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync); - dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync); - dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync); - dataManWriter.Put(bpDComplexes, myDComplexes.data(), - adios2::Mode::Sync); - dataManWriter.EndStep(); - } - dataManWriter.Close(); -} - -void Reader1(const Dims &shape, const Dims &start, const Dims &count, - const size_t steps, const adios2::Params &engineParams, - const std::string &name) -{ - adios2::ADIOS adios(mpiComm); - adios2::IO dataManIO = adios.DeclareIO("Test"); - dataManIO.SetEngine("ssc"); - dataManIO.SetParameters(engineParams); - adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read); - - size_t datasize = std::accumulate(shape.begin(), shape.end(), 1, - std::multiplies()); - std::vector myChars(datasize); - std::vector myUChars(datasize); - std::vector myShorts(datasize); - std::vector myUShorts(datasize); - std::vector myInts(datasize); - std::vector myUInts(datasize); - std::vector myFloats(datasize); - std::vector myDoubles(datasize); - std::vector> myComplexes(datasize); - std::vector> myDComplexes(datasize); - - while (true) - { - adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5); - if (status == adios2::StepStatus::OK) - { - const auto &vars = dataManIO.AvailableVariables(); - ASSERT_EQ(vars.size(), 20); - size_t currentStep = dataManReader.CurrentStep(); - adios2::Variable bpChars = - dataManIO.InquireVariable("bpChars"); - adios2::Variable bpUChars = - dataManIO.InquireVariable("bpUChars"); - adios2::Variable bpShorts = - dataManIO.InquireVariable("bpShorts"); - adios2::Variable bpUShorts = - dataManIO.InquireVariable("bpUShorts"); - adios2::Variable bpInts = - dataManIO.InquireVariable("bpInts"); - adios2::Variable bpUInts = - dataManIO.InquireVariable("bpUInts2"); - adios2::Variable bpFloats = - dataManIO.InquireVariable("bpFloats2"); - adios2::Variable bpDoubles = - dataManIO.InquireVariable("bpDoubles2"); - adios2::Variable> bpComplexes = - dataManIO.InquireVariable>("bpComplexes2"); - adios2::Variable> bpDComplexes = - dataManIO.InquireVariable>( - "bpDComplexes2"); - - dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync); - dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync); - dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync); - dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync); - dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync); - dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync); - dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync); - dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync); - dataManReader.Get(bpComplexes, myComplexes.data(), - adios2::Mode::Sync); - dataManReader.Get(bpDComplexes, myDComplexes.data(), - adios2::Mode::Sync); - VerifyData(myChars.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myUChars.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myShorts.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myUShorts.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myInts.data(), currentStep, Dims(shape.size(), 0), shape, - shape, mpiRank); - VerifyData(myUInts.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myFloats.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myDoubles.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myComplexes.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myDComplexes.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - dataManReader.EndStep(); - } - else if (status == adios2::StepStatus::EndOfStream) - { - std::cout << "[Rank " + std::to_string(mpiRank) + - "] SscTest reader end of stream!" - << std::endl; - break; - } - } - dataManReader.Close(); -} - -void Reader2(const Dims &shape, const Dims &start, const Dims &count, - const size_t steps, const adios2::Params &engineParams, - const std::string &name) -{ - adios2::ADIOS adios(mpiComm); - adios2::IO dataManIO = adios.DeclareIO("Test"); - dataManIO.SetEngine("ssc"); - dataManIO.SetParameters(engineParams); - adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read); - - size_t datasize = std::accumulate(shape.begin(), shape.end(), 1, - std::multiplies()); - std::vector myChars(datasize); - std::vector myUChars(datasize); - std::vector myShorts(datasize); - std::vector myUShorts(datasize); - std::vector myInts(datasize); - std::vector myUInts(datasize); - std::vector myFloats(datasize); - std::vector myDoubles(datasize); - std::vector> myComplexes(datasize); - std::vector> myDComplexes(datasize); - - while (true) - { - adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5); - if (status == adios2::StepStatus::OK) - { - const auto &vars = dataManIO.AvailableVariables(); - ASSERT_EQ(vars.size(), 20); - size_t currentStep = dataManReader.CurrentStep(); - adios2::Variable bpChars = - dataManIO.InquireVariable("bpChars2"); - adios2::Variable bpUChars = - dataManIO.InquireVariable("bpUChars2"); - adios2::Variable bpShorts = - dataManIO.InquireVariable("bpShorts2"); - adios2::Variable bpUShorts = - dataManIO.InquireVariable("bpUShorts2"); - adios2::Variable bpInts = - dataManIO.InquireVariable("bpInts2"); - adios2::Variable bpUInts = - dataManIO.InquireVariable("bpUInts"); - adios2::Variable bpFloats = - dataManIO.InquireVariable("bpFloats"); - adios2::Variable bpDoubles = - dataManIO.InquireVariable("bpDoubles"); - adios2::Variable> bpComplexes = - dataManIO.InquireVariable>("bpComplexes"); - adios2::Variable> bpDComplexes = - dataManIO.InquireVariable>("bpDComplexes"); - - dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync); - dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync); - dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync); - dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync); - dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync); - dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync); - dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync); - dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync); - dataManReader.Get(bpComplexes, myComplexes.data(), - adios2::Mode::Sync); - dataManReader.Get(bpDComplexes, myDComplexes.data(), - adios2::Mode::Sync); - VerifyData(myChars.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myUChars.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myShorts.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myUShorts.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myInts.data(), currentStep, Dims(shape.size(), 0), shape, - shape, mpiRank); - VerifyData(myUInts.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myFloats.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myDoubles.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myComplexes.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - VerifyData(myDComplexes.data(), currentStep, Dims(shape.size(), 0), - shape, shape, mpiRank); - dataManReader.EndStep(); - } - else if (status == adios2::StepStatus::EndOfStream) - { - std::cout << "[Rank " + std::to_string(mpiRank) + - "] SscTest reader end of stream!" - << std::endl; - break; - } - } - dataManReader.Close(); -} - -TEST_F(SscEngineTest, TestSscMultiApp) -{ - std::string filename = "TestSscMultiApp"; - adios2::Params engineParams = {{"RendezvousAppCount", "4"}}; - - int worldRank, worldSize; - Dims start, count, shape; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - if (worldSize < 8) - { - return; - } - if (worldRank == 0 or worldRank == 1) - { - mpiGroup = 0; - } - else if (worldRank == 2 or worldRank == 3) - { - mpiGroup = 1; - } - else if (worldRank == 4 or worldRank == 5) - { - mpiGroup = 2; - } - else if (worldRank == 6 or worldRank == 7) - { - mpiGroup = 3; - } - - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); - - size_t steps = 20; - - if (mpiGroup == 0) - { - shape = {2, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - Writer1(shape, start, count, steps, engineParams, filename); - } - - if (mpiGroup == 1) - { - shape = {2, 10}; - start = {0, 0}; - count = shape; - Reader1(shape, start, shape, steps, engineParams, filename); - } - - if (mpiGroup == 2) - { - shape = {2, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - Writer2(shape, start, count, steps, engineParams, filename); - } - - if (mpiGroup == 3) - { - shape = {2, 10}; - start = {0, 0}; - count = shape; - Reader2(shape, start, shape, steps, engineParams, filename); - } - - MPI_Barrier(MPI_COMM_WORLD); -} - -int main(int argc, char **argv) -{ - MPI_Init(&argc, &argv); - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - ::testing::InitGoogleTest(&argc, argv); - int result = RUN_ALL_TESTS(); - - MPI_Finalize(); - return result; -} From cfe3160572d66392d310786aabedc4ee239eaa9b Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Jul 2020 02:10:40 -0400 Subject: [PATCH 2/6] added algorithm.h --- source/adios2/helper/adiosMpiHandshake.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/source/adios2/helper/adiosMpiHandshake.cpp b/source/adios2/helper/adiosMpiHandshake.cpp index 435abf884f..830517f1ba 100644 --- a/source/adios2/helper/adiosMpiHandshake.cpp +++ b/source/adios2/helper/adiosMpiHandshake.cpp @@ -9,6 +9,7 @@ */ #include "adiosMpiHandshake.h" +#include #include #include #include From 5208d05e6bf23750f741a8d0311ad57241f7a8a5 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Jul 2020 02:52:37 -0400 Subject: [PATCH 3/6] added vector.h --- source/adios2/helper/adiosMpiHandshake.cpp | 1 - source/adios2/helper/adiosMpiHandshake.h | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/helper/adiosMpiHandshake.cpp b/source/adios2/helper/adiosMpiHandshake.cpp index 830517f1ba..e0a669c139 100644 --- a/source/adios2/helper/adiosMpiHandshake.cpp +++ b/source/adios2/helper/adiosMpiHandshake.cpp @@ -15,7 +15,6 @@ #include #include #include -#include namespace adios2 { diff --git a/source/adios2/helper/adiosMpiHandshake.h b/source/adios2/helper/adiosMpiHandshake.h index e8cc462a39..0f38a4bf5e 100644 --- a/source/adios2/helper/adiosMpiHandshake.h +++ b/source/adios2/helper/adiosMpiHandshake.h @@ -18,6 +18,7 @@ #include #include +#include namespace adios2 { From eb8470b33f9539de60278df9f031ae91cd7ab085 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Jul 2020 15:49:38 -0400 Subject: [PATCH 4/6] replaced int with size_t to fix warnings --- source/adios2/helper/adiosMpiHandshake.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/adios2/helper/adiosMpiHandshake.cpp b/source/adios2/helper/adiosMpiHandshake.cpp index e0a669c139..f905b77ccd 100644 --- a/source/adios2/helper/adiosMpiHandshake.cpp +++ b/source/adios2/helper/adiosMpiHandshake.cpp @@ -147,7 +147,7 @@ const std::vector> Handshake(const std::string &filename, } } - int dims[3]; + size_t dims[3]; if (localRank == 0) { @@ -158,7 +158,7 @@ const std::vector> Handshake(const std::string &filename, } } - MPI_Bcast(dims, 3, MPI_INT, 0, localComm); + MPI_Bcast(dims, 3, MPI_UNSIGNED_LONG_LONG, 0, localComm); if (localRank != 0) { From 980c0e05011eb15eaff9b7b41111d68fec4ee363 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Jul 2020 16:59:54 -0400 Subject: [PATCH 5/6] another try to pass the windows CI --- source/adios2/helper/adiosMpiHandshake.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/adios2/helper/adiosMpiHandshake.cpp b/source/adios2/helper/adiosMpiHandshake.cpp index f905b77ccd..0260235b1d 100644 --- a/source/adios2/helper/adiosMpiHandshake.cpp +++ b/source/adios2/helper/adiosMpiHandshake.cpp @@ -147,18 +147,18 @@ const std::vector> Handshake(const std::string &filename, } } - size_t dims[3]; + int dims[3]; if (localRank == 0) { for (int i = 0; i < 3; ++i) { - dims[i] = ret[i].size(); + dims[i] = static_cast(ret[i].size()); std::sort(ret[i].begin(), ret[i].end()); } } - MPI_Bcast(dims, 3, MPI_UNSIGNED_LONG_LONG, 0, localComm); + MPI_Bcast(dims, 3, MPI_INT, 0, localComm); if (localRank != 0) { From f574d7739eee78fd6999b49529c5f8f759276d42 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Sun, 5 Jul 2020 17:23:28 -0400 Subject: [PATCH 6/6] the third try to pass the windows CI --- source/adios2/helper/adiosMpiHandshake.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/adios2/helper/adiosMpiHandshake.cpp b/source/adios2/helper/adiosMpiHandshake.cpp index 0260235b1d..390f49da78 100644 --- a/source/adios2/helper/adiosMpiHandshake.cpp +++ b/source/adios2/helper/adiosMpiHandshake.cpp @@ -170,7 +170,8 @@ const std::vector> Handshake(const std::string &filename, for (int i = 0; i < 3; ++i) { - MPI_Bcast(ret[i].data(), ret[i].size(), MPI_INT, 0, localComm); + MPI_Bcast(ret[i].data(), static_cast(ret[i].size()), MPI_INT, 0, + localComm); } return ret;