Merge pull request #2358 from JasonRuonanWang/mpihandshake

replaced MPI based handshake with file based, removed multi-app support from SSC
ornladios · Jul 5, 2020 · 4e6d572 · 4e6d572
2 parents afd3924 + f574d77
commit 4e6d572
Show file tree

Hide file tree

Showing 8 changed files with 160 additions and 911 deletions.
diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp
@@ -251,47 +251,19 @@ void SscReader::SyncMpiPattern()
 {
     TAU_SCOPED_TIMER_FUNC();
 
-    if (m_Verbosity >= 5)
-    {
-        std::cout << "SscReader::SyncMpiPattern, World Rank " << m_StreamRank
-                  << ", Reader Rank " << m_ReaderRank << std::endl;
-    }
-
-    m_MpiHandshake.Handshake(m_Name, 'r', m_OpenTimeoutSecs, m_MaxStreamsPerApp,
-                             m_MaxFilenameLength, m_RendezvousAppCount,
-                             CommAsMPI(m_Comm));
-
-    std::vector<int> allStreamRanks;
-    std::vector<int> allWriterRanks;
-
-    for (const auto &app : m_MpiHandshake.GetWriterMap(m_Name))
-    {
-        for (int rank : app.second)
-        {
-            allWriterRanks.push_back(rank);
-            allStreamRanks.push_back(rank);
-        }
-    }
-
-    for (const auto &app : m_MpiHandshake.GetReaderMap(m_Name))
-    {
-        for (int rank : app.second)
-        {
-            allStreamRanks.push_back(rank);
-        }
-    }
+    auto appRankMaps =
+        helper::Handshake(m_Name, 'r', m_OpenTimeoutSecs, CommAsMPI(m_Comm));
 
     MPI_Group worldGroup;
+    MPI_Group streamGroup;
+
     MPI_Comm_group(MPI_COMM_WORLD, &worldGroup);
-    MPI_Group_incl(worldGroup, allWriterRanks.size(), allWriterRanks.data(),
+    MPI_Group_incl(worldGroup, appRankMaps[1].size(), appRankMaps[1].data(),
                    &m_MpiAllWritersGroup);
+    MPI_Group_incl(worldGroup, appRankMaps[0].size(), appRankMaps[0].data(),
+                   &streamGroup);
 
-    MPI_Comm_group(MPI_COMM_WORLD, &worldGroup);
-    std::sort(allStreamRanks.begin(), allStreamRanks.end());
-    MPI_Group allWorkersGroup;
-    MPI_Group_incl(worldGroup, allStreamRanks.size(), allStreamRanks.data(),
-                   &allWorkersGroup);
-    MPI_Comm_create_group(MPI_COMM_WORLD, allWorkersGroup, 0, &m_StreamComm);
+    MPI_Comm_create_group(MPI_COMM_WORLD, streamGroup, 0, &m_StreamComm);
 }
 
 void SscReader::SyncWritePattern()

diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h
@@ -60,8 +60,6 @@ class SscReader : public Engine
     int m_ReaderRank;
     int m_ReaderSize;
 
-    helper::MpiHandshake m_MpiHandshake;
-
     void SyncMpiPattern();
     void SyncWritePattern();
     void SyncReadPattern();

diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp
@@ -217,47 +217,19 @@ void SscWriter::SyncMpiPattern()
 {
     TAU_SCOPED_TIMER_FUNC();
 
-    if (m_Verbosity >= 5)
-    {
-        std::cout << "SscWriter::SyncMpiPattern, World Rank " << m_StreamRank
-                  << ", Writer Rank " << m_WriterRank << std::endl;
-    }
-
-    m_MpiHandshake.Handshake(m_Name, 'w', m_OpenTimeoutSecs, m_MaxStreamsPerApp,
-                             m_MaxFilenameLength, m_RendezvousAppCount,
-                             CommAsMPI(m_Comm));
+    auto appRankMaps =
+        helper::Handshake(m_Name, 'w', m_OpenTimeoutSecs, CommAsMPI(m_Comm));
 
-    std::vector<int> allStreamRanks;
-    std::vector<int> allReaderRanks;
-
-    for (const auto &app : m_MpiHandshake.GetWriterMap(m_Name))
-    {
-        for (int rank : app.second)
-        {
-            allStreamRanks.push_back(rank);
-        }
-    }
-
-    for (const auto &app : m_MpiHandshake.GetReaderMap(m_Name))
-    {
-        for (int rank : app.second)
-        {
-            allStreamRanks.push_back(rank);
-            allReaderRanks.push_back(rank);
-        }
-    }
     MPI_Group worldGroup;
-    MPI_Group allReadersGroup;
+    MPI_Group streamGroup;
+
     MPI_Comm_group(MPI_COMM_WORLD, &worldGroup);
-    MPI_Group_incl(worldGroup, allReaderRanks.size(), allReaderRanks.data(),
+    MPI_Group_incl(worldGroup, appRankMaps[2].size(), appRankMaps[2].data(),
                    &m_MpiAllReadersGroup);
+    MPI_Group_incl(worldGroup, appRankMaps[0].size(), appRankMaps[0].data(),
+                   &streamGroup);
 
-    MPI_Comm_group(MPI_COMM_WORLD, &worldGroup);
-    std::sort(allStreamRanks.begin(), allStreamRanks.end());
-    MPI_Group allWorkersGroup;
-    MPI_Group_incl(worldGroup, allStreamRanks.size(), allStreamRanks.data(),
-                   &allWorkersGroup);
-    MPI_Comm_create_group(MPI_COMM_WORLD, allWorkersGroup, 0, &m_StreamComm);
+    MPI_Comm_create_group(MPI_COMM_WORLD, streamGroup, 0, &m_StreamComm);
 }
 
 void SscWriter::SyncWritePattern()

diff --git a/source/adios2/engine/ssc/SscWriter.h b/source/adios2/engine/ssc/SscWriter.h
@@ -62,8 +62,6 @@ class SscWriter : public Engine
     int m_WriterRank;
     int m_WriterSize;
 
-    helper::MpiHandshake m_MpiHandshake;
-
     void SyncMpiPattern();
     void SyncWritePattern();
     void SyncReadPattern();