Merge pull request #12192 from NexGenAnalytics/nga-fy23-pr-candidate-14

Belos: Remove use of hardcoded MPI_COMM_WORLD
trilinos · Sep 7, 2023 · b16d43e · b16d43e
2 parents 3d3d3fd + bd5eab8
commit b16d43e
Show file tree

Hide file tree

Showing 4 changed files with 74 additions and 3 deletions.
diff --git a/packages/belos/epetra/src/BelosEpetraUtils.cpp b/packages/belos/epetra/src/BelosEpetraUtils.cpp
@@ -48,6 +48,7 @@
 #include "Epetra_Import.h"
 #ifdef EPETRA_MPI
 #include "Epetra_MpiComm.h"
+#include "BelosGlobalComm.hpp"
 #else
 #include "Epetra_SerialComm.h"
 #endif
@@ -98,7 +99,7 @@ int createEpetraProblem( std::string             &filename,
 
   RCP<Epetra_Comm> epetraComm;
 #ifdef EPETRA_MPI	
-  epetraComm = rcp(new Epetra_MpiComm( MPI_COMM_WORLD ) );	
+  epetraComm = rcp(new Epetra_MpiComm( Belos::get_global_comm() ) );
 #else	
   epetraComm = rcp(new Epetra_SerialComm());
 #endif
@@ -273,7 +274,7 @@ namespace Test {
 
       if (comm_.is_null()) {
 #ifdef EPETRA_MPI
-        comm_ = rcp (new Epetra_MpiComm (MPI_COMM_WORLD));
+        comm_ = rcp (new Epetra_MpiComm (Belos::get_global_comm()));
 #else
         comm_ = rcp (new Epetra_SerialComm);
 #endif // EPETRA_MPI

diff --git a/packages/belos/src/BelosGlobalComm.hpp b/packages/belos/src/BelosGlobalComm.hpp
@@ -0,0 +1,68 @@
+//@HEADER
+// ************************************************************************
+//
+//                 Belos: Block Linear Solvers Package
+//                  Copyright 2004 Sandia Corporation
+//
+// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
+// the U.S. Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the Corporation nor the names of the
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact Michael A. Heroux ([email protected])
+//
+// ************************************************************************
+//@HEADER
+
+#ifndef BELOS_GLOBAL_COMM_HPP
+#define BELOS_GLOBAL_COMM_HPP
+
+#ifdef HAVE_MPI
+
+#include <mpi.h>
+#include <mutex>
+
+namespace Belos {
+
+static std::mutex mpi_mutex;
+static MPI_Comm Global_MPI_Comm = MPI_COMM_WORLD; // CHECK: ALLOW MPI_COMM_WORLD
+
+inline void initialize_global_comm(MPI_Comm comm) {
+    std::lock_guard<std::mutex> guard(mpi_mutex);
+    Global_MPI_Comm = comm;
+}
+
+inline MPI_Comm get_global_comm() {
+    std::lock_guard<std::mutex> guard(mpi_mutex);
+    return Global_MPI_Comm;
+}
+
+}
+
+#endif // HAVE_MPI
+#endif // BELOS_GLOBAL_COMM_HPP
diff --git a/packages/belos/src/BelosOutputManager.hpp b/packages/belos/src/BelosOutputManager.hpp
@@ -53,6 +53,7 @@
 
 #ifdef HAVE_MPI
 #include <mpi.h>
+#include "BelosGlobalComm.hpp"
 #endif
 
 /*!	\class Belos::OutputManager
@@ -160,7 +161,7 @@ namespace Belos {
     // Initialize MPI
     int mpiStarted = 0;
     MPI_Initialized(&mpiStarted);
-    if (mpiStarted) MPI_Comm_rank(MPI_COMM_WORLD, &MyPID);
+    if (mpiStarted) MPI_Comm_rank(Belos::get_global_comm(), &MyPID);
     else MyPID=0;
 #else 
     MyPID = 0;

diff --git a/packages/belos/src/CMakeLists.txt b/packages/belos/src/CMakeLists.txt
@@ -108,6 +108,7 @@ APPEND_SET(HEADERS
   Belos_Details_registerLinearSolverFactory.hpp
   Belos_Details_registerSolverFactory.hpp
   BelosUtils.hpp
+  BelosGlobalComm.hpp
   )
 
 # UTIL