#93: Remove use of MPI_COMM_WORLD in Ifpack package

packages/ifpack/src/CMakeLists.txt

@@ -109,6 +109,7 @@ SET(HEADERS ${HEADERS}
   Ifpack_SILU.h
   Ifpack_IHSS.h
   Ifpack_SORa.h
+  Ifpack_DefaultComm.h
   )
 
 SET(SOURCES ${SOURCES}

packages/ifpack/src/Ifpack_DefaultComm.h

@@ -0,0 +1,76 @@
+/*@HEADER
+// ***********************************************************************
+//
+//       Ifpack: Object-Oriented Algebraic Preconditioner Package
+//                 Copyright (2002) Sandia Corporation
+//
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the Corporation nor the names of the
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact Michael A. Heroux ([email protected])
+//
+// ***********************************************************************
+//@HEADER
+*/
+
+#ifndef IFPACK_DEFAULTCOMM_H
+#define IFPACK_DEFAULTCOMM_H
+
+#include <mpi.h>
+#include <pthread.h>
+
+#ifdef __cplusplus
+/* if C++, define the rest of this header file as extern C */
+extern "C" {
+#endif
+
+static pthread_mutex_t global_comm_lock;
+static MPI_Comm Global_MPI_Comm = MPI_COMM_WORLD;
+
+/* Function to set the default communicator */
+static void initialize_global_comm(MPI_Comm comm) {
+  pthread_mutex_lock(&global_comm_lock);
+  Global_MPI_Comm = comm;
+  pthread_mutex_unlock(&global_comm_lock);
+}
+
+/* Function to get the default communicator */
+static MPI_Comm get_global_comm() {
+  pthread_mutex_lock(&global_comm_lock);
+  MPI_Comm comm = Global_MPI_Comm;
+  pthread_mutex_unlock(&global_comm_lock);
+  return comm;
+}
+
+#ifdef __cplusplus
+} /* closing bracket for extern "C" */
+#endif
+
+#endif /* IFPACK_DEFAULTCOMM_H */

packages/ifpack/src/euclid/guards_dh.h

@@ -44,15 +44,15 @@
 #define GUARDS_DH
 
 #include "euclid_common.h"
-
+#include "Ifpack_DefaultComm.h"
 
 /*
    This file defines the INITIALIZE_DH and FINALIZE_DH macros
 */
 
 #define  INITIALIZE_DH(argc, argv, help) \
             MPI_Init(&argc,&argv);  \
-            comm_dh = MPI_COMM_WORLD;    \
+            comm_dh = get_global_comm();    \
             MPI_Errhandler_set(comm_dh, MPI_ERRORS_RETURN); \
             EuclidInitialize(argc, argv, help); \
             dh_StartFunc(__FUNC__, __FILE__, __LINE__, 1); \

packages/ifpack/src/euclid/mat_dh_private.c

@@ -46,6 +46,7 @@
 #include "Mat_dh.h"
 #include "Mem_dh.h"
 #include "Vec_dh.h"
+#include "Ifpack_DefaultComm.h"
 
 #define IS_UPPER_TRI 97
 #define IS_LOWER_TRI 98
@@ -1166,7 +1167,7 @@ partition_and_distribute_metis_private (Mat_dh A, Mat_dh * Bout)
   /* broadcast number of rows to all processors */
   if (myid_dh == 0)
     m = A->m;
-  MPI_Bcast (&m, 1, MPI_INT, 0, MPI_COMM_WORLD);
+  MPI_Bcast (&m, 1, MPI_INT, 0, get_global_comm());
 
   /* broadcast number of nonzeros in each row to all processors */
   rowLengths = (int *) MALLOC_DH (m * sizeof (int));
@@ -1359,7 +1360,7 @@ partition_and_distribute_private (Mat_dh A, Mat_dh * Bout)
   /* broadcast number of rows to all processors */
   if (myid_dh == 0)
     m = A->m;
-  MPI_Bcast (&m, 1, MPI_INT, 0, MPI_COMM_WORLD);
+  MPI_Bcast (&m, 1, MPI_INT, 0, get_global_comm());
 
   /* broadcast number of nonzeros in each row to all processors */
   rowLengths = (int *) MALLOC_DH (m * sizeof (int));