Merge pull request #154 from NexGenAnalytics/muelu-remove-hardcoded-m…

…pi-comm-world

MueLu: Don't use hardcoded MPI_COMM_WORLD

packages/muelu/adapters/amgx/MueLu_AMGXOperator_decl.hpp

@@ -61,6 +61,7 @@
 #include "MueLu_TimeMonitor.hpp"
 #include "MueLu_TpetraOperator.hpp"
 #include "MueLu_VerboseObject.hpp"
+#include "MueLu_GlobalComm.hpp"
 
 #include <cuda_runtime.h>
 #include <amgx_c.h>
@@ -150,7 +151,7 @@ namespace MueLu {
     typedef Tpetra::Map<LO,GO,NO>            Map;
     typedef Tpetra::MultiVector<SC,LO,GO,NO> MultiVector;
 
-   
+
     void printMaps(Teuchos::RCP<const Teuchos::Comm<int> >& comm, const std::vector<std::vector<int> >& vec, const std::vector<int>& perm,
                    const int* nbrs, const Map& map, const std::string& label) {
       for (int p = 0; p < comm->getSize(); p++) {
@@ -179,15 +180,15 @@ namespace MueLu {
       RCP<const Teuchos::Comm<int> > comm = inA->getRowMap()->getComm();
       int numProcs = comm->getSize();
       int myRank   = comm->getRank();
-     
+
 
       RCP<Teuchos::Time> amgxTimer = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: initialize");
       amgxTimer->start();
       // Initialize
       //AMGX_SAFE_CALL(AMGX_initialize());
       //AMGX_SAFE_CALL(AMGX_initialize_plugins());
-    
-                
+
+
       /*system*/
       //AMGX_SAFE_CALL(AMGX_register_print_callback(&print_callback));
       AMGX_SAFE_CALL(AMGX_install_signal_handler());
@@ -241,7 +242,7 @@ namespace MueLu {
 #ifdef NEW_COMM
         AMGX_resources_create(&Resources_, Config_, &mpiComm, 1/* number of GPU devices utilized by this rank */, device);
 #else
-        AMGX_resources_create(&Resources_, Config_, MPI_COMM_WORLD, 1/* number of GPU devices utilized by this rank */, device);
+        AMGX_resources_create(&Resources_, Config_, MueLu::get_global_comm(), 1/* number of GPU devices utilized by this rank */, device);
 #endif
       }
 
@@ -464,7 +465,7 @@ namespace MueLu {
       AMGX_SAFE_CALL(AMGX_config_destroy(Config_));
     }
 
-   
+
     //! Returns the Tpetra::Map object associated with the domain of this operator.
     Teuchos::RCP<const Map> getDomainMap() const;
 

packages/muelu/src/MueCentral/MueLu_VerboseObject.cpp

@@ -55,6 +55,7 @@
 #include "MueLu_ConfigDefs.hpp"
 #include "MueLu_VerbosityLevel.hpp"
 #include "MueLu_Exceptions.hpp"
+#include "MueLu_GlobalComm.hpp"
 
 namespace MueLu {
 
@@ -74,8 +75,8 @@ namespace MueLu {
 #ifdef HAVE_MPI
     int mpiStarted = 0; MPI_Initialized(&mpiStarted);
     if (mpiStarted)     {
-      MPI_Comm_rank(MPI_COMM_WORLD, &procRank_);
-      MPI_Comm_size(MPI_COMM_WORLD, &numProcs_);
+      MPI_Comm_rank(MueLu::get_global_comm(), &procRank_);
+      MPI_Comm_size(MueLu::get_global_comm(), &numProcs_);
     }
 #endif
     }
@@ -140,7 +141,7 @@ namespace MueLu {
     int mpiStarted = 0; MPI_Initialized(&mpiStarted);
     if (mpiStarted)     {
       int procRank;
-      MPI_Comm_rank(MPI_COMM_WORLD, &procRank);
+      MPI_Comm_rank(MueLu::get_global_comm(), &procRank);
       fn = filename + "." + std::to_string(procRank);
     } else
 #endif

packages/muelu/src/Utils/MueLu_GlobalComm.hpp

@@ -0,0 +1,75 @@
+// @HEADER
+//
+// ***********************************************************************
+//
+//        MueLu: A package for multigrid based preconditioning
+//                  Copyright 2012 Sandia Corporation
+//
+// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
+// the U.S. Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the Corporation nor the names of the
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact
+//                    Jonathan Hu       ([email protected])
+//                    Andrey Prokopenko ([email protected])
+//                    Ray Tuminaro      ([email protected])
+//
+// ***********************************************************************
+//
+// @HEADER
+
+#ifndef MUELU_GLOBAL_COMM_HPP
+#define MUELU_GLOBAL_COMM_HPP
+
+#include "MueLu_config.hpp"
+
+#ifdef HAVE_MPI
+
+#include <mpi.h>
+#include <mutex>
+
+namespace MueLu {
+
+static std::mutex mpi_mutex;
+static MPI_Comm Global_MueLu_Comm = MPI_COMM_WORLD;
+
+inline void initialize_global_comm(MPI_Comm comm) {
+  std::lock_guard<std::mutex> guard(mpi_mutex);
+  Global_MueLu_Comm = comm;
+}
+
+inline MPI_Comm get_global_comm() {
+  std::lock_guard<std::mutex> guard(mpi_mutex);
+  return Global_MueLu_Comm;
+}
+
+} // namespace MueLu
+
+#endif // HAVE_MPI
+#endif // MUELU_GLOBAL_COMM_HPP

packages/muelu/src/Utils/MueLu_Memory.cpp

@@ -46,6 +46,7 @@
 #include <sstream>
 #include <fstream>
 #include "MueLu_Memory.hpp"
+#include "MueLu_GlobalComm.hpp"
 
 #include <iostream> // TODO: remove
 #include <unistd.h>
@@ -122,9 +123,9 @@ namespace MueLu {
       double localTime = timer.totalElapsedTime();
 #ifdef HAVE_MPI
       int ntimers=1, root=0;
-      MPI_Reduce(&localTime,&maxTime,ntimers,MPI_DOUBLE,MPI_MAX,root,MPI_COMM_WORLD);
-      MPI_Reduce(&localTime,&minTime,ntimers,MPI_DOUBLE,MPI_MIN,root,MPI_COMM_WORLD);
-      MPI_Reduce(&localTime,&avgTime,ntimers,MPI_DOUBLE,MPI_SUM,root,MPI_COMM_WORLD);
+      MPI_Reduce(&localTime,&maxTime,ntimers,MPI_DOUBLE,MPI_MAX,root,MueLu::get_global_comm());
+      MPI_Reduce(&localTime,&minTime,ntimers,MPI_DOUBLE,MPI_MIN,root,MueLu::get_global_comm());
+      MPI_Reduce(&localTime,&avgTime,ntimers,MPI_DOUBLE,MPI_SUM,root,MueLu::get_global_comm());
 #else
       maxTime = localTime;
       minTime = localTime;