diff --git a/packages/belos/epetra/src/BelosEpetraUtils.cpp b/packages/belos/epetra/src/BelosEpetraUtils.cpp index 8a2a89ccc0c3..b346def7425d 100644 --- a/packages/belos/epetra/src/BelosEpetraUtils.cpp +++ b/packages/belos/epetra/src/BelosEpetraUtils.cpp @@ -48,6 +48,7 @@ #include "Epetra_Import.h" #ifdef EPETRA_MPI #include "Epetra_MpiComm.h" +#include "BelosGlobalComm.hpp" #else #include "Epetra_SerialComm.h" #endif @@ -98,7 +99,7 @@ int createEpetraProblem( std::string &filename, RCP epetraComm; #ifdef EPETRA_MPI - epetraComm = rcp(new Epetra_MpiComm( MPI_COMM_WORLD ) ); + epetraComm = rcp(new Epetra_MpiComm( Belos::get_global_comm() ) ); #else epetraComm = rcp(new Epetra_SerialComm()); #endif @@ -273,7 +274,7 @@ namespace Test { if (comm_.is_null()) { #ifdef EPETRA_MPI - comm_ = rcp (new Epetra_MpiComm (MPI_COMM_WORLD)); + comm_ = rcp (new Epetra_MpiComm (Belos::get_global_comm())); #else comm_ = rcp (new Epetra_SerialComm); #endif // EPETRA_MPI diff --git a/packages/belos/src/BelosGlobalComm.hpp b/packages/belos/src/BelosGlobalComm.hpp new file mode 100644 index 000000000000..bfedc384d014 --- /dev/null +++ b/packages/belos/src/BelosGlobalComm.hpp @@ -0,0 +1,68 @@ +//@HEADER +// ************************************************************************ +// +// Belos: Block Linear Solvers Package +// Copyright 2004 Sandia Corporation +// +// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Michael A. Heroux (maherou@sandia.gov) +// +// ************************************************************************ +//@HEADER + +#ifndef BELOS_GLOBAL_COMM_HPP +#define BELOS_GLOBAL_COMM_HPP + +#ifdef HAVE_MPI + +#include +#include + +namespace Belos { + +static std::mutex mpi_mutex; +static MPI_Comm Global_MPI_Comm = MPI_COMM_WORLD; // CHECK: ALLOW MPI_COMM_WORLD + +inline void initialize_global_comm(MPI_Comm comm) { + std::lock_guard guard(mpi_mutex); + Global_MPI_Comm = comm; +} + +inline MPI_Comm get_global_comm() { + std::lock_guard guard(mpi_mutex); + return Global_MPI_Comm; +} + +} + +#endif // HAVE_MPI +#endif // BELOS_GLOBAL_COMM_HPP diff --git a/packages/belos/src/BelosOutputManager.hpp b/packages/belos/src/BelosOutputManager.hpp index 12fd2aebfc44..2b8e5a893445 100644 --- a/packages/belos/src/BelosOutputManager.hpp +++ b/packages/belos/src/BelosOutputManager.hpp @@ -53,6 +53,7 @@ #ifdef HAVE_MPI #include +#include "BelosGlobalComm.hpp" #endif /*! \class Belos::OutputManager @@ -160,7 +161,7 @@ namespace Belos { // Initialize MPI int mpiStarted = 0; MPI_Initialized(&mpiStarted); - if (mpiStarted) MPI_Comm_rank(MPI_COMM_WORLD, &MyPID); + if (mpiStarted) MPI_Comm_rank(Belos::get_global_comm(), &MyPID); else MyPID=0; #else MyPID = 0; diff --git a/packages/belos/src/CMakeLists.txt b/packages/belos/src/CMakeLists.txt index 36df2b008ce6..6033962d7ec3 100644 --- a/packages/belos/src/CMakeLists.txt +++ b/packages/belos/src/CMakeLists.txt @@ -108,6 +108,7 @@ APPEND_SET(HEADERS Belos_Details_registerLinearSolverFactory.hpp Belos_Details_registerSolverFactory.hpp BelosUtils.hpp + BelosGlobalComm.hpp ) # UTIL