From ed65c005b109f6554fc57d0da1546fd632ff33d2 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Thu, 22 Aug 2019 12:56:56 -0400
Subject: [PATCH 01/42] helper: add more MPI_Datatype values to mpidummy

---
 source/adios2/helper/mpidummy.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/adios2/helper/mpidummy.h b/source/adios2/helper/mpidummy.h
index edf89455b6..236b9e216d 100644
--- a/source/adios2/helper/mpidummy.h
+++ b/source/adios2/helper/mpidummy.h
@@ -57,6 +57,7 @@ typedef int MPI_Op;
 #define MPI_COMM_WORLD 1
 #define MPI_COMM_SELF 2
 
+#define MPI_DATATYPE_NULL 0
 #define MPI_INT 1
 #define MPI_CHAR 2
 #define MPI_DOUBLE 3
@@ -74,6 +75,7 @@ typedef int MPI_Op;
 #define MPI_DOUBLE_INT 15
 #define MPI_LONG_DOUBLE_INT 16
 #define MPI_SHORT_INT 17
+#define MPI_SIGNED_CHAR 18
 
 #define MPI_ANY_SOURCE 0
 #define MPI_ANY_TAG 0

From b1e41aed69136e17ff6cfe3d4a3e5b23739e6dc0 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Thu, 22 Aug 2019 12:56:34 -0400
Subject: [PATCH 02/42] helper: add MPI_IN_PLACE to mpidummy

---
 source/adios2/helper/mpidummy.cpp | 4 ++++
 source/adios2/helper/mpidummy.h   | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/source/adios2/helper/mpidummy.cpp b/source/adios2/helper/mpidummy.cpp
index 618a0ce721..349729dc50 100644
--- a/source/adios2/helper/mpidummy.cpp
+++ b/source/adios2/helper/mpidummy.cpp
@@ -316,6 +316,10 @@ int MPI_Get_processor_name(char *name, int *resultlen)
 int MPI_Reduce(const void *sendbuf, void *recvbuf, int count,
                MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm)
 {
+    if (sendbuf == MPI_IN_PLACE)
+    {
+        RETURN_CHECK(MPI_SUCCESS);
+    }
     int ier, size_of_type;
     ier = MPIDUMMY::MPI_Type_size(datatype, &size_of_type);
     if (ier != MPI_SUCCESS)
diff --git a/source/adios2/helper/mpidummy.h b/source/adios2/helper/mpidummy.h
index 236b9e216d..1454f087c6 100644
--- a/source/adios2/helper/mpidummy.h
+++ b/source/adios2/helper/mpidummy.h
@@ -53,6 +53,8 @@ typedef int MPI_Op;
 #define MPI_BYTE 1 /* I need the size of the type here */
 #define MPI_INFO_NULL 0
 
+#define MPI_IN_PLACE ((void *)1)
+
 #define MPI_COMM_NULL 0
 #define MPI_COMM_WORLD 1
 #define MPI_COMM_SELF 2

From 541f380fea1378b319a3851ad0eca522e83b5941 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 23 Jul 2019 15:00:23 -0400
Subject: [PATCH 03/42] helper: Take CheckMPIReturn hint string by reference

---
 source/adios2/helper/adiosMPIFunctions.cpp | 2 +-
 source/adios2/helper/adiosMPIFunctions.h   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/adios2/helper/adiosMPIFunctions.cpp b/source/adios2/helper/adiosMPIFunctions.cpp
index 37ff056436..17f344c1a4 100644
--- a/source/adios2/helper/adiosMPIFunctions.cpp
+++ b/source/adios2/helper/adiosMPIFunctions.cpp
@@ -34,7 +34,7 @@ std::vector<int> GetGathervDisplacements(const size_t *counts,
     return displacements;
 }
 
-void CheckMPIReturn(const int value, const std::string hint)
+void CheckMPIReturn(const int value, const std::string &hint)
 {
     if (value == MPI_SUCCESS)
     {
diff --git a/source/adios2/helper/adiosMPIFunctions.h b/source/adios2/helper/adiosMPIFunctions.h
index 258cb5f12f..e7da203067 100644
--- a/source/adios2/helper/adiosMPIFunctions.h
+++ b/source/adios2/helper/adiosMPIFunctions.h
@@ -111,7 +111,7 @@ void GathervVectors(const std::vector<T> &in, std::vector<T> &out,
 std::vector<int> GetGathervDisplacements(const size_t *counts,
                                          const size_t countsSize);
 
-void CheckMPIReturn(const int value, const std::string hint);
+void CheckMPIReturn(const int value, const std::string &hint);
 
 std::string BroadcastFile(const std::string &fileName, MPI_Comm mpiComm,
                           const std::string hint = "",

From 78311d8cca124737541fe17ef21c19e3878fabad Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Wed, 5 Jun 2019 11:56:45 -0400
Subject: [PATCH 04/42] helper: Add placeholder for Comm encapsulation of
 multi-process communication

---
 source/adios2/CMakeLists.txt       |  1 +
 source/adios2/helper/adiosComm.cpp | 19 +++++++++++++++++++
 source/adios2/helper/adiosComm.h   | 28 ++++++++++++++++++++++++++++
 source/adios2/helper/adiosComm.inl | 24 ++++++++++++++++++++++++
 source/adios2/helper/adiosComm.tcc | 24 ++++++++++++++++++++++++
 5 files changed, 96 insertions(+)
 create mode 100644 source/adios2/helper/adiosComm.cpp
 create mode 100644 source/adios2/helper/adiosComm.h
 create mode 100644 source/adios2/helper/adiosComm.inl
 create mode 100644 source/adios2/helper/adiosComm.tcc

diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt
index 2492f76a65..e92343945d 100644
--- a/source/adios2/CMakeLists.txt
+++ b/source/adios2/CMakeLists.txt
@@ -32,6 +32,7 @@ add_library(adios2
   operator/callback/Signature2.cpp
 
 #helper
+  helper/adiosComm.h  helper/adiosComm.cpp
   helper/adiosDynamicBinder.h  helper/adiosDynamicBinder.cpp
   helper/adiosMath.cpp
   helper/adiosMemory.cpp
diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
new file mode 100644
index 0000000000..f70cd1b2c5
--- /dev/null
+++ b/source/adios2/helper/adiosComm.cpp
@@ -0,0 +1,19 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * Comm.cpp
+ */
+
+#include "adiosComm.h"
+#include "adiosComm.tcc"
+
+namespace adios2
+{
+namespace helper
+{
+
+Comm::~Comm() = default;
+
+} // end namespace helper
+} // end namespace adios2
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
new file mode 100644
index 0000000000..06f707d3fc
--- /dev/null
+++ b/source/adios2/helper/adiosComm.h
@@ -0,0 +1,28 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * adiosComm.h : Communicate in a multi-process environment.
+ */
+
+#ifndef ADIOS2_HELPER_ADIOSCOMM_H_
+#define ADIOS2_HELPER_ADIOSCOMM_H_
+
+namespace adios2
+{
+namespace helper
+{
+
+/** @brief Encapsulation for communication in a multi-process environment.  */
+class Comm
+{
+public:
+    ~Comm();
+};
+
+} // end namespace helper
+} // end namespace adios2
+
+#include "adiosComm.inl"
+
+#endif // ADIOS2_HELPER_ADIOSCOMM_H_
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
new file mode 100644
index 0000000000..b8ea0af00b
--- /dev/null
+++ b/source/adios2/helper/adiosComm.inl
@@ -0,0 +1,24 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * adiosComm.inl
+ */
+
+#ifndef ADIOS2_HELPER_ADIOSCOMM_INL_
+#define ADIOS2_HELPER_ADIOSCOMM_INL_
+#ifndef ADIOS2_HELPER_ADIOSCOMM_H_
+#error "Inline file should only be included from it's header, never on it's own"
+#endif
+
+namespace adios2
+{
+namespace helper
+{
+
+// Add template definitions here.
+
+} // end namespace helper
+} // end namespace adios2
+
+#endif /* ADIOS2_HELPER_ADIOSCOMM_INL_ */
diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc
new file mode 100644
index 0000000000..1b109bf4ca
--- /dev/null
+++ b/source/adios2/helper/adiosComm.tcc
@@ -0,0 +1,24 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * adiosComm.tcc : specialization of template functions defined in
+ * adiosComm.h
+ */
+
+#ifndef ADIOS2_HELPER_ADIOSCOMM_TCC_
+#define ADIOS2_HELPER_ADIOSCOMM_TCC_
+
+#include "adiosComm.h"
+
+namespace adios2
+{
+namespace helper
+{
+
+// Add template specializations here.
+
+} // end namespace helper
+} // end namespace adios2
+
+#endif /* ADIOS2_HELPER_ADIOSCOMM_TCC_ */

From 33280a2cd3d2e75f1f05204a40ae7f9d0f9c1516 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Thu, 6 Jun 2019 09:53:49 -0400
Subject: [PATCH 05/42] core: Refactor IO to get communicator from ADIOS
 factory

Avoid storing a redundant copy in an IO class member.
---
 source/adios2/core/ADIOS.cpp      |  2 +-
 source/adios2/core/IO.cpp         | 12 +++++-------
 source/adios2/core/IO.h           | 10 ++--------
 source/adios2/helper/adiosXML.cpp |  7 +++----
 4 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp
index e608958108..cf2e3fcd50 100644
--- a/source/adios2/core/ADIOS.cpp
+++ b/source/adios2/core/ADIOS.cpp
@@ -128,7 +128,7 @@ IO &ADIOS::DeclareIO(const std::string name)
     }
 
     auto ioPair = m_IOs.emplace(
-        name, IO(*this, name, m_MPIComm, false, m_HostLanguage, m_DebugMode));
+        name, IO(*this, name, false, m_HostLanguage, m_DebugMode));
     IO &io = ioPair.first->second;
     io.SetDeclared();
     return io;
diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp
index c693a252ee..9f741327c1 100644
--- a/source/adios2/core/IO.cpp
+++ b/source/adios2/core/IO.cpp
@@ -73,12 +73,10 @@ namespace adios2
 namespace core
 {
 
-IO::IO(ADIOS &adios, const std::string name, MPI_Comm mpiComm,
-       const bool inConfigFile, const std::string hostLanguage,
-       const bool debugMode)
-: m_ADIOS(adios), m_Name(name), m_MPIComm(mpiComm),
-  m_InConfigFile(inConfigFile), m_HostLanguage(hostLanguage),
-  m_DebugMode(debugMode)
+IO::IO(ADIOS &adios, const std::string name, const bool inConfigFile,
+       const std::string hostLanguage, const bool debugMode)
+: m_ADIOS(adios), m_Name(name), m_InConfigFile(inConfigFile),
+  m_HostLanguage(hostLanguage), m_DebugMode(debugMode)
 {
 }
 
@@ -698,7 +696,7 @@ Engine &IO::Open(const std::string &name, const Mode mode,
 
 Engine &IO::Open(const std::string &name, const Mode mode)
 {
-    return Open(name, mode, m_MPIComm);
+    return Open(name, mode, m_ADIOS.m_MPIComm);
 }
 
 Engine &IO::GetEngine(const std::string &name)
diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h
index c4dea3f426..d5808ac651 100644
--- a/source/adios2/core/IO.h
+++ b/source/adios2/core/IO.h
@@ -53,10 +53,6 @@ class IO
     /** unique identifier */
     const std::string m_Name;
 
-    /** from ADIOS class passed to Engine created with Open
-     *  if no new communicator is passed */
-    MPI_Comm m_MPIComm;
-
     /** true: extra exceptions checks */
     const bool m_DebugMode = false;
 
@@ -99,14 +95,12 @@ class IO
      * Not to be used direclty in applications.
      * @param adios reference to ADIOS object that owns current IO
      * @param name unique identifier for this IO object
-     * @param mpiComm MPI communicator from ADIOS factory class
      * @param inConfigFile IO defined in config file (XML)
      * @param hostLanguage current language using the adios2 library
      * @param debugMode true: extra exception checks (recommended)
      */
-    IO(ADIOS &adios, const std::string name, MPI_Comm mpiComm,
-       const bool inConfigFile, const std::string hostLanguage,
-       const bool debugMode);
+    IO(ADIOS &adios, const std::string name, const bool inConfigFile,
+       const std::string hostLanguage, const bool debugMode);
 
     ~IO() = default;
 
diff --git a/source/adios2/helper/adiosXML.cpp b/source/adios2/helper/adiosXML.cpp
index 1567a742b3..f580ed8e21 100644
--- a/source/adios2/helper/adiosXML.cpp
+++ b/source/adios2/helper/adiosXML.cpp
@@ -287,10 +287,9 @@ void ParseConfigXML(
             helper::XMLAttribute("name", io, adios.m_DebugMode, hint);
 
         // Build the IO object
-        auto itCurrentIO =
-            ios.emplace(ioName.value(),
-                        core::IO(adios, ioName.value(), adios.m_MPIComm, true,
-                                 adios.m_HostLanguage, adios.m_DebugMode));
+        auto itCurrentIO = ios.emplace(
+            ioName.value(), core::IO(adios, ioName.value(), true,
+                                     adios.m_HostLanguage, adios.m_DebugMode));
         core::IO &currentIO = itCurrentIO.first->second;
 
         // must be unique per io

From a9a46ce96f2c5c1ab630eefec28e42d8a4f7c65b Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Wed, 5 Jun 2019 14:24:50 -0400
Subject: [PATCH 06/42] core: Refactor ADIOS factory to make communicator
 member private

Provide a `GetComm()` method to get it publicly.
---
 source/adios2/core/ADIOS.h        | 7 +++++--
 source/adios2/core/IO.cpp         | 2 +-
 source/adios2/helper/adiosXML.cpp | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h
index d1a4a9e9e8..bef4846438 100644
--- a/source/adios2/core/ADIOS.h
+++ b/source/adios2/core/ADIOS.h
@@ -38,8 +38,8 @@ class ADIOS
     /** if true will do more checks, exceptions, warnings, expect slower code */
     const bool m_DebugMode = true;
 
-    /** Passed from parallel constructor, MPI_Comm is a pointer itself. */
-    MPI_Comm m_MPIComm;
+    /** Get the communicator passed to constructor for parallel case.  */
+    MPI_Comm GetComm() const { return m_MPIComm; }
 
     /** Changed by language bindings in constructor */
     const std::string m_HostLanguage = "C++";
@@ -184,6 +184,9 @@ class ADIOS
     void RemoveAllIOs() noexcept;
 
 private:
+    /** Passed from parallel constructor, MPI_Comm is a pointer itself. */
+    MPI_Comm m_MPIComm;
+
     /** XML File to be read containing configuration information */
     const std::string m_ConfigFile;
 
diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp
index 9f741327c1..f49821639b 100644
--- a/source/adios2/core/IO.cpp
+++ b/source/adios2/core/IO.cpp
@@ -696,7 +696,7 @@ Engine &IO::Open(const std::string &name, const Mode mode,
 
 Engine &IO::Open(const std::string &name, const Mode mode)
 {
-    return Open(name, mode, m_ADIOS.m_MPIComm);
+    return Open(name, mode, m_ADIOS.GetComm());
 }
 
 Engine &IO::GetEngine(const std::string &name)
diff --git a/source/adios2/helper/adiosXML.cpp b/source/adios2/helper/adiosXML.cpp
index f580ed8e21..931af2f2e3 100644
--- a/source/adios2/helper/adiosXML.cpp
+++ b/source/adios2/helper/adiosXML.cpp
@@ -151,7 +151,7 @@ void ParseConfigXML(
 
     auto lf_FileContents = [&](const std::string &configXML) -> std::string {
         const std::string fileContents(helper::BroadcastFile(
-            configXML, adios.m_MPIComm,
+            configXML, adios.GetComm(),
             "when parsing configXML file, in call to the ADIOS constructor"));
 
         if (adios.m_DebugMode)

From f7fec28f4e0accb7e7d3e0ef2cf6f74b9a99a684 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Fri, 21 Jun 2019 14:11:21 -0400
Subject: [PATCH 07/42] core: Refactor ADIOS factory to use Comm encapsulation

Implement the Comm encapsulation by moving MPI code out of the factory.
---
 source/adios2/core/ADIOS.cpp       | 17 ++-------
 source/adios2/core/ADIOS.h         |  7 ++--
 source/adios2/helper/adiosComm.cpp | 44 ++++++++++++++++++++-
 source/adios2/helper/adiosComm.h   | 61 ++++++++++++++++++++++++++++++
 4 files changed, 111 insertions(+), 18 deletions(-)

diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp
index cf2e3fcd50..5e271cb597 100644
--- a/source/adios2/core/ADIOS.cpp
+++ b/source/adios2/core/ADIOS.cpp
@@ -59,10 +59,9 @@ namespace core
 
 ADIOS::ADIOS(const std::string configFile, MPI_Comm mpiComm,
              const bool debugMode, const std::string hostLanguage)
-: m_ConfigFile(configFile), m_DebugMode(debugMode), m_HostLanguage(hostLanguage)
+: m_ConfigFile(configFile), m_DebugMode(debugMode),
+  m_HostLanguage(hostLanguage), m_Comm(helper::Comm::Duplicate(mpiComm))
 {
-    SMPI_Comm_dup(mpiComm, &m_MPIComm);
-
     if (!configFile.empty())
     {
         if (configFile.substr(configFile.size() - 3) == "xml")
@@ -90,17 +89,7 @@ ADIOS::ADIOS(const bool debugMode, const std::string hostLanguage)
 {
 }
 
-ADIOS::~ADIOS()
-{
-    // Handle the case where MPI is finalized before the ADIOS destructor is
-    // called, which happens, e.g., with global / static ADIOS objects
-    int flag;
-    MPI_Finalized(&flag);
-    if (!flag)
-    {
-        SMPI_Comm_free(&m_MPIComm);
-    }
-}
+ADIOS::~ADIOS() = default;
 
 IO &ADIOS::DeclareIO(const std::string name)
 {
diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h
index bef4846438..c334563882 100644
--- a/source/adios2/core/ADIOS.h
+++ b/source/adios2/core/ADIOS.h
@@ -22,6 +22,7 @@
 #include "adios2/common/ADIOSMPI.h"
 #include "adios2/common/ADIOSTypes.h"
 #include "adios2/core/Operator.h"
+#include "adios2/helper/adiosComm.h"
 
 namespace adios2
 {
@@ -39,7 +40,7 @@ class ADIOS
     const bool m_DebugMode = true;
 
     /** Get the communicator passed to constructor for parallel case.  */
-    MPI_Comm GetComm() const { return m_MPIComm; }
+    MPI_Comm GetComm() const { return m_Comm; }
 
     /** Changed by language bindings in constructor */
     const std::string m_HostLanguage = "C++";
@@ -184,8 +185,8 @@ class ADIOS
     void RemoveAllIOs() noexcept;
 
 private:
-    /** Passed from parallel constructor, MPI_Comm is a pointer itself. */
-    MPI_Comm m_MPIComm;
+    /** Communicator given to parallel constructor. */
+    helper::Comm m_Comm;
 
     /** XML File to be read containing configuration information */
     const std::string m_ConfigFile;
diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index f70cd1b2c5..b6ca087f25 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -8,12 +8,54 @@
 #include "adiosComm.h"
 #include "adiosComm.tcc"
 
+#include <ios> //std::ios_base::failure
+
+#include "adios2/common/ADIOSMPI.h"
+
 namespace adios2
 {
 namespace helper
 {
 
-Comm::~Comm() = default;
+Comm::Comm() = default;
+
+Comm::Comm(MPI_Comm mpiComm) : m_MPIComm(mpiComm) {}
+
+Comm::~Comm()
+{
+    // Handle the case where MPI is finalized before the ADIOS destructor is
+    // called, which happens, e.g., with global / static ADIOS objects
+    int flag;
+    MPI_Finalized(&flag);
+    if (!flag)
+    {
+        if (m_MPIComm != MPI_COMM_NULL && m_MPIComm != MPI_COMM_WORLD &&
+            m_MPIComm != MPI_COMM_SELF)
+        {
+            SMPI_Comm_free(&m_MPIComm);
+        }
+    }
+}
+
+Comm::Comm(Comm &&comm) : m_MPIComm(comm.m_MPIComm)
+{
+    comm.m_MPIComm = MPI_COMM_NULL;
+}
+
+Comm &Comm::operator=(Comm &&comm)
+{
+    Comm(std::move(comm)).swap(*this);
+    return *this;
+}
+
+void Comm::swap(Comm &comm) { std::swap(this->m_MPIComm, comm.m_MPIComm); }
+
+Comm Comm::Duplicate(MPI_Comm mpiComm)
+{
+    MPI_Comm newComm;
+    SMPI_Comm_dup(mpiComm, &newComm);
+    return Comm(newComm);
+}
 
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 06f707d3fc..fc9cbd99e1 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -8,6 +8,8 @@
 #ifndef ADIOS2_HELPER_ADIOSCOMM_H_
 #define ADIOS2_HELPER_ADIOSCOMM_H_
 
+#include "adios2/common/ADIOSMPI.h"
+
 namespace adios2
 {
 namespace helper
@@ -17,7 +19,66 @@ namespace helper
 class Comm
 {
 public:
+    /**
+     * @brief Default constructor.  Produces an empty communicator.
+     *
+     * An empty communicator may not be used for communcation.
+     */
+    Comm();
+
+    /**
+     * @brief Move constructor.  Moves communicator state from that given.
+     *
+     * The moved-from communicator is left empty and may not be used for
+     * communication.
+     */
+    Comm(Comm &&);
+
+    /**
+     * @brief Deleted copy constructor.  A communicator may not be copied.
+     */
+    Comm(Comm const &) = delete;
+
     ~Comm();
+
+    /**
+     * @brief Move assignment.  Moves communicator state from that given.
+     *
+     * The moved-from communicator is left empty and may not be used for
+     * communication.
+     */
+    Comm &operator=(Comm &&);
+
+    /**
+     * @brief Deleted copy assignment.  A communicator may not be copied.
+     */
+    Comm &operator=(Comm const &) = delete;
+
+    /**
+     * @brief Swap communicator state with another.
+     */
+    void swap(Comm &comm);
+
+    // FIXME: Remove conversion after clients transition to encapsulation.
+    /** Convert to a concrete MPI communicator.  */
+    operator MPI_Comm() const { return m_MPIComm; }
+
+    /**
+     * @brief Create a communicator by duplicating a MPI communicator.
+     */
+    static Comm Duplicate(MPI_Comm mpiComm);
+
+private:
+    /**
+     * @brief Construct by taking ownership of a MPI communicator.
+     *
+     * This is a private implementation detail used by static
+     * methods like Duplicate.
+     */
+    explicit Comm(MPI_Comm mpiComm);
+
+    /** Encapsulated MPI communicator instance.  */
+    MPI_Comm m_MPIComm = MPI_COMM_NULL;
 };
 
 } // end namespace helper

From 9e7cdf68497805f15914b740bade74b5990a2e4b Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 18 Jun 2019 10:52:51 -0400
Subject: [PATCH 08/42] core: Expose Comm encapsulation outside ADIOS factory

For now it has an implicit conversion to MPI_Comm anyway.
---
 source/adios2/core/ADIOS.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h
index c334563882..00c65262c9 100644
--- a/source/adios2/core/ADIOS.h
+++ b/source/adios2/core/ADIOS.h
@@ -40,7 +40,7 @@ class ADIOS
     const bool m_DebugMode = true;
 
     /** Get the communicator passed to constructor for parallel case.  */
-    MPI_Comm GetComm() const { return m_Comm; }
+    helper::Comm const &GetComm() const { return m_Comm; }
 
     /** Changed by language bindings in constructor */
     const std::string m_HostLanguage = "C++";

From 64f04be61e74ba2302a8e9306d56814ac6f4d95f Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 18 Jun 2019 11:02:18 -0400
Subject: [PATCH 09/42] helper: Encapsulate MPI_Comm_free as Comm::Free

---
 source/adios2/helper/adiosComm.cpp | 12 ++++++++++++
 source/adios2/helper/adiosComm.h   | 10 ++++++++++
 2 files changed, 22 insertions(+)

diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index b6ca087f25..e36629a790 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -9,9 +9,12 @@
 #include "adiosComm.tcc"
 
 #include <ios> //std::ios_base::failure
+#include <utility>
 
 #include "adios2/common/ADIOSMPI.h"
 
+#include "adiosMPIFunctions.h"
+
 namespace adios2
 {
 namespace helper
@@ -57,5 +60,14 @@ Comm Comm::Duplicate(MPI_Comm mpiComm)
     return Comm(newComm);
 }
 
+void Comm::Free(const std::string &hint)
+{
+    if (m_MPIComm != MPI_COMM_NULL && m_MPIComm != MPI_COMM_WORLD &&
+        m_MPIComm != MPI_COMM_SELF)
+    {
+        CheckMPIReturn(SMPI_Comm_free(&m_MPIComm), hint);
+    }
+}
+
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index fc9cbd99e1..4a434c87b5 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -10,6 +10,8 @@
 
 #include "adios2/common/ADIOSMPI.h"
 
+#include <string>
+
 namespace adios2
 {
 namespace helper
@@ -68,6 +70,14 @@ class Comm
      */
     static Comm Duplicate(MPI_Comm mpiComm);
 
+    /**
+     * @brief Free the communicator.
+     * @param hint Description of std::runtime_error exception on error.
+     *
+     * The communicator is left empty and may not be used for communication.
+     */
+    void Free(const std::string &hint = std::string());
+
 private:
     /**
      * @brief Construct by taking ownership of a MPI communicator.

From 8b8b797cbfa00c05a45e52e24d536ad95de841a7 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 18 Jun 2019 11:12:34 -0400
Subject: [PATCH 10/42] engine: Rename m_MPIComm to m_Comm

---
 source/adios2/core/Engine.cpp                 |  4 ++--
 source/adios2/core/Engine.h                   |  2 +-
 source/adios2/engine/bp3/BP3Reader.cpp        |  2 +-
 source/adios2/engine/bp3/BP3Writer.cpp        |  4 ++--
 source/adios2/engine/bp4/BP4Reader.cpp        | 16 +++++++---------
 source/adios2/engine/bp4/BP4Writer.cpp        |  6 +++---
 .../adios2/engine/dataman/DataManWriter.cpp   |  4 ++--
 source/adios2/engine/hdf5/HDF5ReaderP.cpp     |  2 +-
 source/adios2/engine/hdf5/HDF5WriterP.cpp     |  6 +++---
 .../engine/insitumpi/InSituMPIReader.cpp      | 19 +++++++++----------
 .../engine/insitumpi/InSituMPIWriter.cpp      |  8 ++++----
 source/adios2/engine/mixer/HDFMixer.cpp       |  2 +-
 source/adios2/engine/ssc/SscReader.cpp        |  6 +++---
 source/adios2/engine/ssc/SscWriter.cpp        |  8 ++++----
 source/adios2/engine/sst/SstReader.cpp        |  2 +-
 source/adios2/engine/sst/SstWriter.cpp        |  4 ++--
 source/adios2/engine/table/TableWriter.cpp    | 10 +++++-----
 17 files changed, 51 insertions(+), 54 deletions(-)

diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp
index 2ba47bab56..a468d82a67 100644
--- a/source/adios2/core/Engine.cpp
+++ b/source/adios2/core/Engine.cpp
@@ -23,7 +23,7 @@ namespace core
 Engine::Engine(const std::string engineType, IO &io, const std::string &name,
                const Mode openMode, MPI_Comm mpiComm)
 : m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode),
-  m_MPIComm(mpiComm), m_DebugMode(io.m_DebugMode)
+  m_Comm(mpiComm), m_DebugMode(io.m_DebugMode)
 {
 }
 
@@ -69,7 +69,7 @@ void Engine::Close(const int transportIndex)
 
     if (transportIndex == -1)
     {
-        helper::CheckMPIReturn(SMPI_Comm_free(&m_MPIComm),
+        helper::CheckMPIReturn(SMPI_Comm_free(&m_Comm),
                                "freeing comm in Engine " + m_Name +
                                    ", in call to Close");
         m_IsClosed = true;
diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h
index 60aaecc565..27f063e1b5 100644
--- a/source/adios2/core/Engine.h
+++ b/source/adios2/core/Engine.h
@@ -447,7 +447,7 @@ class Engine
 protected:
     /** from ADIOS class passed to Engine created with Open
      *  if no new communicator is passed */
-    MPI_Comm m_MPIComm;
+    MPI_Comm m_Comm;
 
     /** true: additional exceptions */
     const bool m_DebugMode = false;
diff --git a/source/adios2/engine/bp3/BP3Reader.cpp b/source/adios2/engine/bp3/BP3Reader.cpp
index bee3c4bd97..8b6471e5fd 100644
--- a/source/adios2/engine/bp3/BP3Reader.cpp
+++ b/source/adios2/engine/bp3/BP3Reader.cpp
@@ -202,7 +202,7 @@ void BP3Reader::InitBuffer()
     }
 
     // broadcast metadata buffer to all ranks from zero
-    helper::BroadcastVector(m_BP3Deserializer.m_Metadata.m_Buffer, m_MPIComm);
+    helper::BroadcastVector(m_BP3Deserializer.m_Metadata.m_Buffer, m_Comm);
 
     // fills IO with available Variables and Attributes
     m_BP3Deserializer.ParseMetadata(m_BP3Deserializer.m_Metadata, *this);
diff --git a/source/adios2/engine/bp3/BP3Writer.cpp b/source/adios2/engine/bp3/BP3Writer.cpp
index 24b50f02ac..106cdf564a 100644
--- a/source/adios2/engine/bp3/BP3Writer.cpp
+++ b/source/adios2/engine/bp3/BP3Writer.cpp
@@ -282,7 +282,7 @@ void BP3Writer::WriteProfilingJSONFile()
 
     if (m_BP3Serializer.m_RankMPI == 0)
     {
-        transport::FileFStream profilingJSONStream(m_MPIComm, m_DebugMode);
+        transport::FileFStream profilingJSONStream(m_Comm, m_DebugMode);
         auto bpBaseNames = m_BP3Serializer.GetBPBaseNames({m_Name});
         profilingJSONStream.Open(bpBaseNames[0] + "/profiling.json",
                                  Mode::Write);
@@ -295,7 +295,7 @@ void BP3Writer::WriteCollectiveMetadataFile(const bool isFinal)
 {
     TAU_SCOPED_TIMER("BP3Writer::WriteCollectiveMetadataFile");
     m_BP3Serializer.AggregateCollectiveMetadata(
-        m_MPIComm, m_BP3Serializer.m_Metadata, true);
+        m_Comm, m_BP3Serializer.m_Metadata, true);
 
     if (m_BP3Serializer.m_RankMPI == 0)
     {
diff --git a/source/adios2/engine/bp4/BP4Reader.cpp b/source/adios2/engine/bp4/BP4Reader.cpp
index e3d2598245..29c95965f4 100644
--- a/source/adios2/engine/bp4/BP4Reader.cpp
+++ b/source/adios2/engine/bp4/BP4Reader.cpp
@@ -217,7 +217,7 @@ void BP4Reader::OpenFiles()
         }
     }
 
-    flag = helper::BroadcastValue(flag, m_MPIComm, 0);
+    flag = helper::BroadcastValue(flag, m_Comm, 0);
     if (flag == 2)
     {
         if (m_BP4Deserializer.m_RankMPI == 0 && !lasterrmsg.empty())
@@ -269,7 +269,7 @@ void BP4Reader::OpenFiles()
         }
     }
 
-    flag = helper::BroadcastValue(flag, m_MPIComm, 0);
+    flag = helper::BroadcastValue(flag, m_Comm, 0);
     if (flag == 1)
     {
         throw std::runtime_error("ERROR: File " + m_Name +
@@ -317,11 +317,10 @@ void BP4Reader::InitBuffer()
                                  fileSize);
     }
     // broadcast buffer to all ranks from zero
-    helper::BroadcastVector(m_BP4Deserializer.m_Metadata.m_Buffer, m_MPIComm);
+    helper::BroadcastVector(m_BP4Deserializer.m_Metadata.m_Buffer, m_Comm);
 
     // broadcast metadata index buffer to all ranks from zero
-    helper::BroadcastVector(m_BP4Deserializer.m_MetadataIndex.m_Buffer,
-                            m_MPIComm);
+    helper::BroadcastVector(m_BP4Deserializer.m_MetadataIndex.m_Buffer, m_Comm);
 
     /* Parse metadata index table */
     m_BP4Deserializer.ParseMetadataIndex(m_BP4Deserializer.m_MetadataIndex);
@@ -386,18 +385,17 @@ size_t BP4Reader::UpdateBuffer()
         }
     }
 
-    helper::BroadcastVector(sizes, m_MPIComm, 0);
+    helper::BroadcastVector(sizes, m_Comm, 0);
     size_t newIdxSize = sizes[0];
 
     if (newIdxSize > 0)
     {
         // broadcast buffer to all ranks from zero
-        helper::BroadcastVector(m_BP4Deserializer.m_Metadata.m_Buffer,
-                                m_MPIComm);
+        helper::BroadcastVector(m_BP4Deserializer.m_Metadata.m_Buffer, m_Comm);
 
         // broadcast metadata index buffer to all ranks from zero
         helper::BroadcastVector(m_BP4Deserializer.m_MetadataIndex.m_Buffer,
-                                m_MPIComm);
+                                m_Comm);
 
         if (m_BP4Deserializer.m_RankMPI != 0)
         {
diff --git a/source/adios2/engine/bp4/BP4Writer.cpp b/source/adios2/engine/bp4/BP4Writer.cpp
index 912ff9294d..7af9a8c3d5 100644
--- a/source/adios2/engine/bp4/BP4Writer.cpp
+++ b/source/adios2/engine/bp4/BP4Writer.cpp
@@ -256,7 +256,7 @@ void BP4Writer::InitBPBuffer()
             m_FileMetadataIndexManager.ReadFile(
                 preMetadataIndex.m_Buffer.data(), preMetadataIndexFileSize);
         }
-        helper::BroadcastVector(preMetadataIndex.m_Buffer, m_MPIComm);
+        helper::BroadcastVector(preMetadataIndex.m_Buffer, m_Comm);
         preMetadataIndexFileSize = preMetadataIndex.m_Buffer.size();
         if (preMetadataIndexFileSize > 0)
         {
@@ -421,7 +421,7 @@ void BP4Writer::WriteProfilingJSONFile()
     if (m_BP4Serializer.m_RankMPI == 0)
     {
         // std::cout << "write profiling file!" << std::endl;
-        transport::FileFStream profilingJSONStream(m_MPIComm, m_DebugMode);
+        transport::FileFStream profilingJSONStream(m_Comm, m_DebugMode);
         auto bpBaseNames = m_BP4Serializer.GetBPBaseNames({m_Name});
         profilingJSONStream.Open(bpBaseNames[0] + "/profiling.json",
                                  Mode::Write);
@@ -478,7 +478,7 @@ void BP4Writer::WriteCollectiveMetadataFile(const bool isFinal)
         return;
     }
     m_BP4Serializer.AggregateCollectiveMetadata(
-        m_MPIComm, m_BP4Serializer.m_Metadata, true);
+        m_Comm, m_BP4Serializer.m_Metadata, true);
 
     if (m_BP4Serializer.m_RankMPI == 0)
     {
diff --git a/source/adios2/engine/dataman/DataManWriter.cpp b/source/adios2/engine/dataman/DataManWriter.cpp
index bb4dd4ec00..4e141fe5de 100644
--- a/source/adios2/engine/dataman/DataManWriter.cpp
+++ b/source/adios2/engine/dataman/DataManWriter.cpp
@@ -44,9 +44,9 @@ DataManWriter::DataManWriter(IO &io, const std::string &name,
         std::vector<char> allCaVec(32 * m_MpiSize, '\0');
 
         MPI_Allgather(m_DataAddress.data(), m_DataAddress.size(), MPI_CHAR,
-                      allDaVec.data(), 32, MPI_CHAR, m_MPIComm);
+                      allDaVec.data(), 32, MPI_CHAR, m_Comm);
         MPI_Allgather(m_ControlAddress.data(), m_ControlAddress.size(),
-                      MPI_CHAR, allCaVec.data(), 32, MPI_CHAR, m_MPIComm);
+                      MPI_CHAR, allCaVec.data(), 32, MPI_CHAR, m_Comm);
 
         std::vector<std::string> daVec;
         std::vector<std::string> caVec;
diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.cpp b/source/adios2/engine/hdf5/HDF5ReaderP.cpp
index 5428552466..3f5b082edc 100644
--- a/source/adios2/engine/hdf5/HDF5ReaderP.cpp
+++ b/source/adios2/engine/hdf5/HDF5ReaderP.cpp
@@ -61,7 +61,7 @@ void HDF5ReaderP::Init()
             ", in call to Open\n");
     }
 
-    m_H5File.Init(m_Name, m_MPIComm, false);
+    m_H5File.Init(m_Name, m_Comm, false);
     m_H5File.ParseParameters(m_IO);
 
     /*
diff --git a/source/adios2/engine/hdf5/HDF5WriterP.cpp b/source/adios2/engine/hdf5/HDF5WriterP.cpp
index 33ad7b27ce..d874fe1643 100644
--- a/source/adios2/engine/hdf5/HDF5WriterP.cpp
+++ b/source/adios2/engine/hdf5/HDF5WriterP.cpp
@@ -57,7 +57,7 @@ void HDF5WriterP::Init()
     }
 
 #ifdef NEVER
-    m_H5File.Init(m_Name, m_MPIComm, true);
+    m_H5File.Init(m_Name, m_Comm, true);
 #else
     // enforce .h5 ending
     std::string suffix = ".h5";
@@ -70,11 +70,11 @@ void HDF5WriterP::Init()
     {
         // is a file with .bp ending
         std::string updatedName = m_Name.substr(0, wpos) + suffix;
-        m_H5File.Init(updatedName, m_MPIComm, true);
+        m_H5File.Init(updatedName, m_Comm, true);
     }
     else
     {
-        m_H5File.Init(m_Name, m_MPIComm, true);
+        m_H5File.Init(m_Name, m_Comm, true);
     }
     m_H5File.ParseParameters(m_IO);
 #endif
diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.cpp b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
index 9b49fe4854..471ac025b6 100644
--- a/source/adios2/engine/insitumpi/InSituMPIReader.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
@@ -76,8 +76,7 @@ InSituMPIReader::InSituMPIReader(IO &io, const std::string &name,
 
     // figure out who is the Reader Root
     std::vector<int> v(m_ReaderNproc);
-    MPI_Allgather(&m_ReaderRootRank, 1, MPI_INT, v.data(), 1, MPI_INT,
-                  m_MPIComm);
+    MPI_Allgather(&m_ReaderRootRank, 1, MPI_INT, v.data(), 1, MPI_INT, m_Comm);
     for (int i = 0; i < m_ReaderNproc; i++)
     {
         if (v[i] != -1)
@@ -191,7 +190,7 @@ StepStatus InSituMPIReader::BeginStep(const StepMode mode,
         }
         /* Exchange steps */
         int maxstep;
-        MPI_Allreduce(&step, &maxstep, 1, MPI_INT, MPI_MAX, m_MPIComm);
+        MPI_Allreduce(&step, &maxstep, 1, MPI_INT, MPI_MAX, m_Comm);
 
         if (m_Verbosity == 5 && !m_ReaderRank)
         {
@@ -246,10 +245,10 @@ StepStatus InSituMPIReader::BeginStep(const StepMode mode,
         }
 
         // broadcast metadata to every reader
-        MPI_Bcast(&mdLen, 1, MPI_UNSIGNED_LONG, m_ReaderRootRank, m_MPIComm);
+        MPI_Bcast(&mdLen, 1, MPI_UNSIGNED_LONG, m_ReaderRootRank, m_Comm);
         m_BP3Deserializer.m_Metadata.m_Buffer.resize(mdLen);
         MPI_Bcast(m_BP3Deserializer.m_Metadata.m_Buffer.data(), mdLen, MPI_CHAR,
-                  m_ReaderRootRank, m_MPIComm);
+                  m_ReaderRootRank, m_Comm);
 
         // Parse metadata into Variables and Attributes maps
         m_IO.RemoveAllVariables();
@@ -278,7 +277,7 @@ StepStatus InSituMPIReader::BeginStep(const StepMode mode,
         }
 
         // broadcast fixed schedule flag to every reader
-        MPI_Bcast(&fixed, 1, MPI_INT, m_ReaderRootRank, m_MPIComm);
+        MPI_Bcast(&fixed, 1, MPI_INT, m_ReaderRootRank, m_Comm);
         m_RemoteDefinitionsLocked = (fixed ? true : false);
         if (m_ReaderRootRank == m_ReaderRank)
         {
@@ -425,7 +424,7 @@ void InSituMPIReader::SendReadSchedule(
         sendBuf = MPI_IN_PLACE;
     }
     MPI_Reduce(sendBuf, nReaderPerWriter.data(), nReaderPerWriter.size(),
-               MPI_INT, MPI_SUM, m_ReaderRootRank, m_MPIComm);
+               MPI_INT, MPI_SUM, m_ReaderRootRank, m_Comm);
 
     // Reader root sends nReaderPerWriter to writer root
     if (m_ReaderRootRank == m_ReaderRank)
@@ -511,7 +510,7 @@ void InSituMPIReader::ProcessReceives()
 
     // Send final acknowledgment to the Writer
     int dummy = 1;
-    MPI_Bcast(&dummy, 1, MPI_INT, m_ReaderRootRank, m_MPIComm);
+    MPI_Bcast(&dummy, 1, MPI_INT, m_ReaderRootRank, m_Comm);
     if (m_ReaderRootRank == m_ReaderRank)
     {
         MPI_Send(&dummy, 1, MPI_INT, m_WriteRootGlobalRank,
@@ -590,9 +589,9 @@ void InSituMPIReader::DoClose(const int transportIndex)
     {
         uint64_t inPlaceBytes, inTempBytes;
         MPI_Reduce(&m_BytesReceivedInPlace, &inPlaceBytes, 1, MPI_LONG_LONG_INT,
-                   MPI_SUM, 0, m_MPIComm);
+                   MPI_SUM, 0, m_Comm);
         MPI_Reduce(&m_BytesReceivedInTemporary, &inTempBytes, 1,
-                   MPI_LONG_LONG_INT, MPI_SUM, 0, m_MPIComm);
+                   MPI_LONG_LONG_INT, MPI_SUM, 0, m_Comm);
         if (m_ReaderRank == 0)
         {
             std::cout << "ADIOS InSituMPI Reader for " << m_Name << " received "
diff --git a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
index 039d77dc48..2474b50de9 100644
--- a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
@@ -155,7 +155,7 @@ void InSituMPIWriter::PerformPuts()
             m_BP3Serializer.SerializeData(m_IO, true); // advance timestep
             m_BP3Serializer.SerializeMetadataInData();
             m_BP3Serializer.AggregateCollectiveMetadata(
-                m_MPIComm, m_BP3Serializer.m_Metadata, true);
+                m_Comm, m_BP3Serializer.m_Metadata, true);
 
             // store length long enough to survive Isend() completion
             // so don't move this into the next if branch
@@ -219,7 +219,7 @@ void InSituMPIWriter::PerformPuts()
                          &status);
             }
             // broadcast fixed schedule flag to every reader
-            MPI_Bcast(&fixed, 1, MPI_INT, 0, m_MPIComm);
+            MPI_Bcast(&fixed, 1, MPI_INT, 0, m_Comm);
             m_RemoteDefinitionsLocked = (fixed ? true : false);
             if (m_BP3Serializer.m_RankMPI == 0)
             {
@@ -327,7 +327,7 @@ void InSituMPIWriter::EndStep()
         MPI_Recv(&dummy, 1, MPI_INT, m_RankDirectPeers[0],
                  insitumpi::MpiTags::ReadCompleted, m_CommWorld, &status);
     }
-    MPI_Bcast(&dummy, 1, MPI_INT, 0, m_MPIComm);
+    MPI_Bcast(&dummy, 1, MPI_INT, 0, m_Comm);
     TAU_STOP("WaitForReaderAck");
 
     if (m_Verbosity == 5)
@@ -442,7 +442,7 @@ void InSituMPIWriter::ReceiveReadSchedule(
 
     // Each writer receives the number of its peer readers
     MPI_Scatter(nReaderPerWriter.data(), 1, MPI_INT, &nPeerReaders, 1, MPI_INT,
-                0, m_MPIComm);
+                0, m_Comm);
 
     std::vector<MPI_Request> requests(nPeerReaders);
     // Reader global rank -> length of serialized read schedule
diff --git a/source/adios2/engine/mixer/HDFMixer.cpp b/source/adios2/engine/mixer/HDFMixer.cpp
index cf3286884b..9e58ee7e30 100644
--- a/source/adios2/engine/mixer/HDFMixer.cpp
+++ b/source/adios2/engine/mixer/HDFMixer.cpp
@@ -103,7 +103,7 @@ void HDFMixer::InitTransports()
 #else
 
     int rank;
-    MPI_Comm_rank(m_MPIComm, &rank);
+    MPI_Comm_rank(m_Comm, &rank);
     m_HDFSerialWriter.Init(m_Name, rank);
     m_HDFVDSWriter.Init(m_Name);
 /*
diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp
index cb097e339d..a04995ec0d 100644
--- a/source/adios2/engine/ssc/SscReader.cpp
+++ b/source/adios2/engine/ssc/SscReader.cpp
@@ -50,7 +50,7 @@ SscReader::SscReader(IO &io, const std::string &name, const Mode mode,
     }
 
     // retrieve all writer addresses
-    helper::HandshakeReader(m_MPIComm, m_AppID, m_FullAddresses, m_Name, "ssc");
+    helper::HandshakeReader(m_Comm, m_AppID, m_FullAddresses, m_Name, "ssc");
 
     // initialize transports
     m_DataTransport.OpenRequester(m_Timeout, m_DataReceiverBufferSize);
@@ -70,7 +70,7 @@ SscReader::SscReader(IO &io, const std::string &name, const Mode mode,
                                                 address);
         }
     }
-    m_DataManSerializer.PutAggregatedMetadata(reply, m_MPIComm);
+    m_DataManSerializer.PutAggregatedMetadata(reply, m_Comm);
     m_DataManSerializer.GetAttributes(m_IO);
 
     if (m_Verbosity >= 5)
@@ -378,7 +378,7 @@ void SscReader::RequestMetadata(const int64_t step)
         reply = m_MetadataTransport.Request(request.data(), request.size(),
                                             address);
     }
-    m_DataManSerializer.PutAggregatedMetadata(reply, m_MPIComm);
+    m_DataManSerializer.PutAggregatedMetadata(reply, m_Comm);
 }
 
 void SscReader::DoClose(const int transportIndex)
diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp
index bd0c9f500c..8ab66f457c 100644
--- a/source/adios2/engine/ssc/SscWriter.cpp
+++ b/source/adios2/engine/ssc/SscWriter.cpp
@@ -109,11 +109,11 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 void SscWriter::Init()
 {
     TAU_SCOPED_TIMER_FUNC();
-    MPI_Comm_rank(m_MPIComm, &m_MpiRank);
-    MPI_Comm_size(m_MPIComm, &m_MpiSize);
+    MPI_Comm_rank(m_Comm, &m_MpiRank);
+    MPI_Comm_size(m_Comm, &m_MpiSize);
     srand(time(NULL));
     InitParameters();
-    helper::HandshakeWriter(m_MPIComm, m_AppID, m_FullAddresses, m_Name, "ssc",
+    helper::HandshakeWriter(m_Comm, m_AppID, m_FullAddresses, m_Name, "ssc",
                             m_Port, m_Channels, m_MaxRanksPerNode,
                             m_MaxAppsPerNode);
     InitTransports();
@@ -231,7 +231,7 @@ void SscWriter::ReplyThread(const std::string &address)
 
 void SscWriter::DoClose(const int transportIndex)
 {
-    MPI_Barrier(m_MPIComm);
+    MPI_Barrier(m_Comm);
     m_Listening = false;
     for (auto &i : m_ReplyThreads)
     {
diff --git a/source/adios2/engine/sst/SstReader.cpp b/source/adios2/engine/sst/SstReader.cpp
index e9947da436..ef2e3bb018 100644
--- a/source/adios2/engine/sst/SstReader.cpp
+++ b/source/adios2/engine/sst/SstReader.cpp
@@ -311,7 +311,7 @@ StepStatus SstReader::BeginStep(StepMode Mode, const float timeout_sec)
         //   whatever transport it is using.  But it is opaque to the Engine
         //   (and to the control plane).)
 
-        m_BP3Deserializer = new format::BP3Deserializer(m_MPIComm, m_DebugMode);
+        m_BP3Deserializer = new format::BP3Deserializer(m_Comm, m_DebugMode);
         m_BP3Deserializer->InitParameters(m_IO.m_Parameters);
 
         m_BP3Deserializer->m_Metadata.Resize(
diff --git a/source/adios2/engine/sst/SstWriter.cpp b/source/adios2/engine/sst/SstWriter.cpp
index 553e6167ab..16d09cfb1f 100644
--- a/source/adios2/engine/sst/SstWriter.cpp
+++ b/source/adios2/engine/sst/SstWriter.cpp
@@ -140,7 +140,7 @@ StepStatus SstWriter::BeginStep(StepMode mode, const float timeout_sec)
     {
         // initialize BP serializer, deleted in
         // SstWriter::EndStep()::lf_FreeBlocks()
-        m_BP3Serializer = new format::BP3Serializer(m_MPIComm, m_DebugMode);
+        m_BP3Serializer = new format::BP3Serializer(m_Comm, m_DebugMode);
         m_BP3Serializer->InitParameters(m_IO.m_Parameters);
         m_BP3Serializer->m_MetadataSet.TimeStep = 1;
         m_BP3Serializer->m_MetadataSet.CurrentStep = m_WriterStep;
@@ -249,7 +249,7 @@ void SstWriter::EndStep()
 
         m_BP3Serializer->CloseStream(m_IO, true);
         m_BP3Serializer->AggregateCollectiveMetadata(
-            m_MPIComm, m_BP3Serializer->m_Metadata, true);
+            m_Comm, m_BP3Serializer->m_Metadata, true);
         BP3DataBlock *newblock = new BP3DataBlock;
         newblock->metadata.DataSize = m_BP3Serializer->m_Metadata.m_Position;
         newblock->metadata.block = m_BP3Serializer->m_Metadata.m_Buffer.data();
diff --git a/source/adios2/engine/table/TableWriter.cpp b/source/adios2/engine/table/TableWriter.cpp
index 69e0a28b3b..5d0cc0b12b 100644
--- a/source/adios2/engine/table/TableWriter.cpp
+++ b/source/adios2/engine/table/TableWriter.cpp
@@ -27,7 +27,7 @@ TableWriter::TableWriter(IO &io, const std::string &name, const Mode mode,
                          MPI_Comm mpiComm)
 : Engine("TableWriter", io, name, mode, mpiComm),
   m_IsRowMajor(helper::IsRowMajor(m_IO.m_HostLanguage)),
-  m_Deserializer(m_MPIComm, m_IsRowMajor),
+  m_Deserializer(m_Comm, m_IsRowMajor),
   m_SubAdios(MPI_COMM_WORLD, adios2::DebugOFF),
   m_SubIO(m_SubAdios.DeclareIO("SubIO"))
 {
@@ -83,7 +83,7 @@ void TableWriter::EndStep()
         }
     }
 
-    MPI_Barrier(m_MPIComm);
+    MPI_Barrier(m_Comm);
 
     m_Listening = false;
     if (m_Verbosity >= 5)
@@ -206,7 +206,7 @@ void TableWriter::InitParameters()
     std::vector<char> cvAll(128 * m_MpiSize);
     std::memcpy(cv.data(), a.c_str(), a.size());
     MPI_Allgather(cv.data(), cv.size(), MPI_CHAR, cvAll.data(), cv.size(),
-                  MPI_CHAR, m_MPIComm);
+                  MPI_CHAR, m_Comm);
     for (int i = 0; i < m_MpiSize; ++i)
     {
         auto j = nlohmann::json::parse(cvAll.data() + i * 128);
@@ -225,8 +225,8 @@ void TableWriter::InitTransports()
 
     for (int i = 0; i < m_Aggregators; ++i)
     {
-        auto s = std::make_shared<format::DataManSerializer>(m_MPIComm,
-                                                             m_IsRowMajor);
+        auto s =
+            std::make_shared<format::DataManSerializer>(m_Comm, m_IsRowMajor);
         s->NewWriterBuffer(m_SerializerBufferSize);
         s->SetDestination(m_AllAddresses[i]);
         m_Serializers.push_back(s);

From 29177b01c281f1943bde377d501912f4adc86eab Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 18 Jun 2019 11:05:05 -0400
Subject: [PATCH 11/42] engine: Construct with Comm encapsulation

---
 source/adios2/core/Engine.cpp                 |  9 +-
 source/adios2/core/Engine.h                   |  9 +-
 source/adios2/core/IO.cpp                     | 85 +++++++++----------
 source/adios2/engine/bp3/BP3Reader.cpp        |  9 +-
 source/adios2/engine/bp3/BP3Reader.h          |  5 +-
 source/adios2/engine/bp3/BP3Writer.cpp        |  8 +-
 source/adios2/engine/bp3/BP3Writer.h          |  5 +-
 source/adios2/engine/bp4/BP4Reader.cpp        | 11 ++-
 source/adios2/engine/bp4/BP4Reader.h          |  5 +-
 source/adios2/engine/bp4/BP4Writer.cpp        | 11 ++-
 source/adios2/engine/bp4/BP4Writer.h          |  5 +-
 .../adios2/engine/dataman/DataManCommon.cpp   | 10 +--
 source/adios2/engine/dataman/DataManCommon.h  |  3 +-
 .../adios2/engine/dataman/DataManReader.cpp   |  4 +-
 source/adios2/engine/dataman/DataManReader.h  |  4 +-
 .../adios2/engine/dataman/DataManWriter.cpp   |  4 +-
 source/adios2/engine/dataman/DataManWriter.h  |  3 +-
 .../engine/dataspaces/DataSpacesReader.cpp    |  5 +-
 .../engine/dataspaces/DataSpacesReader.h      |  3 +-
 .../engine/dataspaces/DataSpacesWriter.cpp    |  5 +-
 .../engine/dataspaces/DataSpacesWriter.h      |  5 +-
 source/adios2/engine/hdf5/HDF5ReaderP.cpp     |  5 +-
 source/adios2/engine/hdf5/HDF5ReaderP.h       |  5 +-
 source/adios2/engine/hdf5/HDF5WriterP.cpp     |  5 +-
 source/adios2/engine/hdf5/HDF5WriterP.h       |  5 +-
 source/adios2/engine/inline/InlineReader.cpp  |  6 +-
 source/adios2/engine/inline/InlineReader.h    |  5 +-
 source/adios2/engine/inline/InlineWriter.cpp  |  6 +-
 source/adios2/engine/inline/InlineWriter.h    |  5 +-
 .../engine/insitumpi/InSituMPIReader.cpp      | 12 +--
 .../adios2/engine/insitumpi/InSituMPIReader.h |  5 +-
 .../engine/insitumpi/InSituMPIWriter.cpp      | 12 +--
 .../adios2/engine/insitumpi/InSituMPIWriter.h |  5 +-
 source/adios2/engine/mixer/HDFMixer.cpp       |  8 +-
 source/adios2/engine/mixer/HDFMixer.h         |  5 +-
 source/adios2/engine/null/NullEngine.cpp      |  4 +-
 source/adios2/engine/null/NullEngine.h        |  3 +-
 .../adios2/engine/nullcore/NullCoreWriter.cpp |  4 +-
 .../adios2/engine/nullcore/NullCoreWriter.h   |  3 +-
 .../adios2/engine/skeleton/SkeletonReader.cpp |  6 +-
 .../adios2/engine/skeleton/SkeletonReader.h   |  5 +-
 .../adios2/engine/skeleton/SkeletonWriter.cpp |  6 +-
 .../adios2/engine/skeleton/SkeletonWriter.h   |  5 +-
 source/adios2/engine/ssc/SscReader.cpp        |  8 +-
 source/adios2/engine/ssc/SscReader.h          |  3 +-
 source/adios2/engine/ssc/SscWriter.cpp        |  6 +-
 source/adios2/engine/ssc/SscWriter.h          |  3 +-
 source/adios2/engine/sst/SstReader.cpp        |  6 +-
 source/adios2/engine/sst/SstReader.h          |  5 +-
 source/adios2/engine/sst/SstWriter.cpp        |  6 +-
 source/adios2/engine/sst/SstWriter.h          |  3 +-
 source/adios2/engine/table/TableWriter.cpp    |  8 +-
 source/adios2/engine/table/TableWriter.h      |  3 +-
 .../format/dataman/DataManSerializer.cpp      | 18 ++--
 .../format/dataman/DataManSerializer.h        |  7 +-
 55 files changed, 220 insertions(+), 189 deletions(-)

diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp
index a468d82a67..bcb969c083 100644
--- a/source/adios2/core/Engine.cpp
+++ b/source/adios2/core/Engine.cpp
@@ -12,6 +12,7 @@
 #include "Engine.tcc"
 
 #include <stdexcept>
+#include <utility>
 
 #include "adios2/core/IO.h"
 
@@ -21,9 +22,9 @@ namespace core
 {
 
 Engine::Engine(const std::string engineType, IO &io, const std::string &name,
-               const Mode openMode, MPI_Comm mpiComm)
+               const Mode openMode, helper::Comm comm)
 : m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode),
-  m_Comm(mpiComm), m_DebugMode(io.m_DebugMode)
+  m_Comm(std::move(comm)), m_DebugMode(io.m_DebugMode)
 {
 }
 
@@ -69,9 +70,7 @@ void Engine::Close(const int transportIndex)
 
     if (transportIndex == -1)
     {
-        helper::CheckMPIReturn(SMPI_Comm_free(&m_Comm),
-                               "freeing comm in Engine " + m_Name +
-                                   ", in call to Close");
+        m_Comm.Free("freeing comm in Engine " + m_Name + ", in call to Close");
         m_IsClosed = true;
     }
 }
diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h
index 27f063e1b5..755e38a0cb 100644
--- a/source/adios2/core/Engine.h
+++ b/source/adios2/core/Engine.h
@@ -29,6 +29,7 @@
 #include "adios2/core/IO.h"
 #include "adios2/core/Variable.h"
 #include "adios2/core/VariableCompound.h"
+#include "adios2/helper/adiosComm.h"
 
 namespace adios2
 {
@@ -62,10 +63,10 @@ class Engine
      * @param io object that generates this Engine
      * @param name unique engine name within IO class object
      * @param mode  open mode from ADIOSTypes.h Mode
-     * @param mpiComm new communicator passed at Open or from ADIOS class
+     * @param comm  communicator passed at Open or from ADIOS class
      */
     Engine(const std::string engineType, IO &io, const std::string &name,
-           const Mode mode, MPI_Comm mpiComm);
+           const Mode mode, helper::Comm comm);
 
     virtual ~Engine();
 
@@ -446,8 +447,8 @@ class Engine
 
 protected:
     /** from ADIOS class passed to Engine created with Open
-     *  if no new communicator is passed */
-    MPI_Comm m_Comm;
+     *  if no communicator is passed */
+    helper::Comm m_Comm;
 
     /** true: additional exceptions */
     const bool m_DebugMode = false;
diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp
index f49821639b..4817b7e9ce 100644
--- a/source/adios2/core/IO.cpp
+++ b/source/adios2/core/IO.cpp
@@ -27,6 +27,7 @@
 #include "adios2/engine/skeleton/SkeletonReader.h"
 #include "adios2/engine/skeleton/SkeletonWriter.h"
 
+#include "adios2/helper/adiosComm.h"
 #include "adios2/helper/adiosFunctions.h" //BuildParametersMap
 #include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
 #include <adios2sys/SystemTools.hxx> // FileIsDirectory()
@@ -424,8 +425,7 @@ size_t IO::AddOperation(Operator &op, const Params &parameters) noexcept
     return m_Operations.size() - 1;
 }
 
-Engine &IO::Open(const std::string &name, const Mode mode,
-                 MPI_Comm mpiComm_orig)
+Engine &IO::Open(const std::string &name, const Mode mode, MPI_Comm mpiComm)
 {
     TAU_SCOPED_TIMER("IO::Open");
     auto itEngineFound = m_Engines.find(name);
@@ -457,8 +457,7 @@ Engine &IO::Open(const std::string &name, const Mode mode,
         }
     }
 
-    MPI_Comm mpiComm;
-    SMPI_Comm_dup(mpiComm_orig, &mpiComm);
+    auto comm = helper::Comm::Duplicate(mpiComm);
     std::shared_ptr<Engine> engine;
     const bool isDefaultEngine = m_EngineType.empty() ? true : false;
     std::string engineTypeLC = m_EngineType;
@@ -493,26 +492,26 @@ Engine &IO::Open(const std::string &name, const Mode mode,
     {
         if (mode == Mode::Read)
         {
-            engine =
-                std::make_shared<engine::BP3Reader>(*this, name, mode, mpiComm);
+            engine = std::make_shared<engine::BP3Reader>(*this, name, mode,
+                                                         std::move(comm));
         }
         else
         {
-            engine =
-                std::make_shared<engine::BP3Writer>(*this, name, mode, mpiComm);
+            engine = std::make_shared<engine::BP3Writer>(*this, name, mode,
+                                                         std::move(comm));
         }
     }
     else if (engineTypeLC == "bp4")
     {
         if (mode == Mode::Read)
         {
-            engine =
-                std::make_shared<engine::BP4Reader>(*this, name, mode, mpiComm);
+            engine = std::make_shared<engine::BP4Reader>(*this, name, mode,
+                                                         std::move(comm));
         }
         else
         {
-            engine =
-                std::make_shared<engine::BP4Writer>(*this, name, mode, mpiComm);
+            engine = std::make_shared<engine::BP4Writer>(*this, name, mode,
+                                                         std::move(comm));
         }
     }
     else if (engineTypeLC == "hdfmixer")
@@ -521,10 +520,10 @@ Engine &IO::Open(const std::string &name, const Mode mode,
 #if H5_VERSION_GE(1, 11, 0)
         if (mode == Mode::Read)
             engine = std::make_shared<engine::HDF5ReaderP>(*this, name, mode,
-                                                           mpiComm);
+                                                           std::move(comm));
         else
-            engine =
-                std::make_shared<engine::HDFMixer>(*this, name, mode, mpiComm);
+            engine = std::make_shared<engine::HDFMixer>(*this, name, mode,
+                                                        std::move(comm));
 #else
         throw std::invalid_argument(
             "ERROR: update HDF5 >= 1.11 to support VDS.");
@@ -539,10 +538,10 @@ Engine &IO::Open(const std::string &name, const Mode mode,
 #ifdef ADIOS2_HAVE_DATAMAN
         if (mode == Mode::Read)
             engine = std::make_shared<engine::DataManReader>(*this, name, mode,
-                                                             mpiComm);
+                                                             std::move(comm));
         else
             engine = std::make_shared<engine::DataManWriter>(*this, name, mode,
-                                                             mpiComm);
+                                                             std::move(comm));
 #else
         throw std::invalid_argument(
             "ERROR: this version didn't compile with "
@@ -553,11 +552,11 @@ Engine &IO::Open(const std::string &name, const Mode mode,
     {
 #ifdef ADIOS2_HAVE_SSC
         if (mode == Mode::Read)
-            engine =
-                std::make_shared<engine::SscReader>(*this, name, mode, mpiComm);
+            engine = std::make_shared<engine::SscReader>(*this, name, mode,
+                                                         std::move(comm));
         else
-            engine =
-                std::make_shared<engine::SscWriter>(*this, name, mode, mpiComm);
+            engine = std::make_shared<engine::SscWriter>(*this, name, mode,
+                                                         std::move(comm));
 #else
         throw std::invalid_argument("ERROR: this version didn't compile with "
                                     "SSC library, can't use SSC engine\n");
@@ -568,7 +567,7 @@ Engine &IO::Open(const std::string &name, const Mode mode,
 #ifdef ADIOS2_HAVE_TABLE
         if (mode == Mode::Write)
             engine = std::make_shared<engine::TableWriter>(*this, name, mode,
-                                                           mpiComm);
+                                                           std::move(comm));
         else
             throw std::invalid_argument(
                 "ERROR: Table engine only supports Write. It uses other "
@@ -583,11 +582,11 @@ Engine &IO::Open(const std::string &name, const Mode mode,
     {
 #ifdef ADIOS2_HAVE_SST
         if (mode == Mode::Read)
-            engine =
-                std::make_shared<engine::SstReader>(*this, name, mode, mpiComm);
+            engine = std::make_shared<engine::SstReader>(*this, name, mode,
+                                                         std::move(comm));
         else
-            engine =
-                std::make_shared<engine::SstWriter>(*this, name, mode, mpiComm);
+            engine = std::make_shared<engine::SstWriter>(*this, name, mode,
+                                                         std::move(comm));
 #else
         throw std::invalid_argument("ERROR: this version didn't compile with "
                                     "Sst library, can't use Sst engine\n");
@@ -597,11 +596,11 @@ Engine &IO::Open(const std::string &name, const Mode mode,
     {
 #ifdef ADIOS2_HAVE_DATASPACES
         if (mode == Mode::Read)
-            engine = std::make_shared<engine::DataSpacesReader>(*this, name,
-                                                                mode, mpiComm);
+            engine = std::make_shared<engine::DataSpacesReader>(
+                *this, name, mode, std::move(comm));
         else
-            engine = std::make_shared<engine::DataSpacesWriter>(*this, name,
-                                                                mode, mpiComm);
+            engine = std::make_shared<engine::DataSpacesWriter>(
+                *this, name, mode, std::move(comm));
 #else
         throw std::invalid_argument(
             "ERROR: this version didn't compile with "
@@ -613,10 +612,10 @@ Engine &IO::Open(const std::string &name, const Mode mode,
 #ifdef ADIOS2_HAVE_HDF5
         if (mode == Mode::Read)
             engine = std::make_shared<engine::HDF5ReaderP>(*this, name, mode,
-                                                           mpiComm);
+                                                           std::move(comm));
         else
             engine = std::make_shared<engine::HDF5WriterP>(*this, name, mode,
-                                                           mpiComm);
+                                                           std::move(comm));
 #else
         throw std::invalid_argument("ERROR: this version didn't compile with "
                                     "HDF5 library, can't use HDF5 engine\n");
@@ -626,11 +625,11 @@ Engine &IO::Open(const std::string &name, const Mode mode,
     {
 #ifdef ADIOS2_HAVE_MPI
         if (mode == Mode::Read)
-            engine = std::make_shared<engine::InSituMPIReader>(*this, name,
-                                                               mode, mpiComm);
+            engine = std::make_shared<engine::InSituMPIReader>(
+                *this, name, mode, std::move(comm));
         else
-            engine = std::make_shared<engine::InSituMPIWriter>(*this, name,
-                                                               mode, mpiComm);
+            engine = std::make_shared<engine::InSituMPIWriter>(
+                *this, name, mode, std::move(comm));
 #else
         throw std::invalid_argument("ERROR: this version didn't compile with "
                                     "MPI, can't use InSituMPI engine\n");
@@ -640,24 +639,24 @@ Engine &IO::Open(const std::string &name, const Mode mode,
     {
         if (mode == Mode::Read)
             engine = std::make_shared<engine::SkeletonReader>(*this, name, mode,
-                                                              mpiComm);
+                                                              std::move(comm));
         else
             engine = std::make_shared<engine::SkeletonWriter>(*this, name, mode,
-                                                              mpiComm);
+                                                              std::move(comm));
     }
     else if (engineTypeLC == "inline")
     {
         if (mode == Mode::Read)
             engine = std::make_shared<engine::InlineReader>(*this, name, mode,
-                                                            mpiComm);
+                                                            std::move(comm));
         else
             engine = std::make_shared<engine::InlineWriter>(*this, name, mode,
-                                                            mpiComm);
+                                                            std::move(comm));
     }
     else if (engineTypeLC == "null")
     {
-        engine =
-            std::make_shared<engine::NullEngine>(*this, name, mode, mpiComm);
+        engine = std::make_shared<engine::NullEngine>(*this, name, mode,
+                                                      std::move(comm));
     }
     else if (engineTypeLC == "nullcore")
     {
@@ -666,7 +665,7 @@ Engine &IO::Open(const std::string &name, const Mode mode,
                 "ERROR: nullcore engine does not support read mode");
         else
             engine = std::make_shared<engine::NullCoreWriter>(*this, name, mode,
-                                                              mpiComm);
+                                                              std::move(comm));
     }
     else
     {
diff --git a/source/adios2/engine/bp3/BP3Reader.cpp b/source/adios2/engine/bp3/BP3Reader.cpp
index 8b6471e5fd..3a0b3c5095 100644
--- a/source/adios2/engine/bp3/BP3Reader.cpp
+++ b/source/adios2/engine/bp3/BP3Reader.cpp
@@ -11,6 +11,7 @@
 #include "BP3Reader.h"
 #include "BP3Reader.tcc"
 
+#include "adios2/helper/adiosComm.h"
 #include "adios2/helper/adiosFunctions.h" // MPI BroadcastVector
 #include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
 
@@ -22,10 +23,10 @@ namespace engine
 {
 
 BP3Reader::BP3Reader(IO &io, const std::string &name, const Mode mode,
-                     MPI_Comm mpiComm)
-: Engine("BP3", io, name, mode, mpiComm),
-  m_BP3Deserializer(mpiComm, m_DebugMode), m_FileManager(mpiComm, m_DebugMode),
-  m_SubFileManager(mpiComm, m_DebugMode)
+                     helper::Comm comm)
+: Engine("BP3", io, name, mode, std::move(comm)),
+  m_BP3Deserializer(m_Comm, m_DebugMode), m_FileManager(m_Comm, m_DebugMode),
+  m_SubFileManager(m_Comm, m_DebugMode)
 {
     TAU_SCOPED_TIMER("BP3Reader::Open");
     Init();
diff --git a/source/adios2/engine/bp3/BP3Reader.h b/source/adios2/engine/bp3/BP3Reader.h
index 7360d25505..dec58e3fce 100644
--- a/source/adios2/engine/bp3/BP3Reader.h
+++ b/source/adios2/engine/bp3/BP3Reader.h
@@ -13,6 +13,7 @@
 
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/format/bp3/BP3.h" //format::BP3Deserializer
 #include "adios2/toolkit/transportman/TransportMan.h"
 
@@ -32,10 +33,10 @@ class BP3Reader : public Engine
      * @param io
      * @param name
      * @param openMode only read
-     * @param mpiComm
+     * @param comm
      */
     BP3Reader(IO &io, const std::string &name, const Mode mode,
-              MPI_Comm mpiComm);
+              helper::Comm comm);
 
     ~BP3Reader() = default;
 
diff --git a/source/adios2/engine/bp3/BP3Writer.cpp b/source/adios2/engine/bp3/BP3Writer.cpp
index 106cdf564a..c4a0925352 100644
--- a/source/adios2/engine/bp3/BP3Writer.cpp
+++ b/source/adios2/engine/bp3/BP3Writer.cpp
@@ -26,10 +26,10 @@ namespace engine
 {
 
 BP3Writer::BP3Writer(IO &io, const std::string &name, const Mode mode,
-                     MPI_Comm mpiComm)
-: Engine("BP3", io, name, mode, mpiComm), m_BP3Serializer(mpiComm, m_DebugMode),
-  m_FileDataManager(mpiComm, m_DebugMode),
-  m_FileMetadataManager(mpiComm, m_DebugMode)
+                     helper::Comm comm)
+: Engine("BP3", io, name, mode, std::move(comm)),
+  m_BP3Serializer(m_Comm, m_DebugMode), m_FileDataManager(m_Comm, m_DebugMode),
+  m_FileMetadataManager(m_Comm, m_DebugMode)
 {
     TAU_SCOPED_TIMER("BP3Writer::Open");
     m_IO.m_ReadStreaming = false;
diff --git a/source/adios2/engine/bp3/BP3Writer.h b/source/adios2/engine/bp3/BP3Writer.h
index 27138b14e4..7215518ba7 100644
--- a/source/adios2/engine/bp3/BP3Writer.h
+++ b/source/adios2/engine/bp3/BP3Writer.h
@@ -13,6 +13,7 @@
 
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/format/bp3/BP3.h"
 #include "adios2/toolkit/transportman/TransportMan.h" //transport::TransportsMan
 
@@ -31,10 +32,10 @@ class BP3Writer : public core::Engine
      * Constructor for file Writer in BP format
      * @param name unique name given to the engine
      * @param openMode w (supported), r, a from OpenMode in ADIOSTypes.h
-     * @param mpiComm MPI communicator
+     * @param comm multi-process communicator
      */
     BP3Writer(IO &io, const std::string &name, const Mode mode,
-              MPI_Comm mpiComm);
+              helper::Comm comm);
 
     ~BP3Writer() = default;
 
diff --git a/source/adios2/engine/bp4/BP4Reader.cpp b/source/adios2/engine/bp4/BP4Reader.cpp
index 29c95965f4..7e8f79b10f 100644
--- a/source/adios2/engine/bp4/BP4Reader.cpp
+++ b/source/adios2/engine/bp4/BP4Reader.cpp
@@ -25,12 +25,11 @@ namespace engine
 {
 
 BP4Reader::BP4Reader(IO &io, const std::string &name, const Mode mode,
-                     MPI_Comm mpiComm)
-: Engine("BP4Reader", io, name, mode, mpiComm),
-  m_BP4Deserializer(mpiComm, m_DebugMode),
-  m_MDFileManager(mpiComm, m_DebugMode),
-  m_DataFileManager(mpiComm, m_DebugMode),
-  m_MDIndexFileManager(mpiComm, m_DebugMode)
+                     helper::Comm comm)
+: Engine("BP4Reader", io, name, mode, std::move(comm)),
+  m_BP4Deserializer(m_Comm, m_DebugMode), m_MDFileManager(m_Comm, m_DebugMode),
+  m_DataFileManager(m_Comm, m_DebugMode),
+  m_MDIndexFileManager(m_Comm, m_DebugMode)
 {
     TAU_SCOPED_TIMER("BP4Reader::Open");
     Init();
diff --git a/source/adios2/engine/bp4/BP4Reader.h b/source/adios2/engine/bp4/BP4Reader.h
index 49e1545450..2dc2d2c7b5 100644
--- a/source/adios2/engine/bp4/BP4Reader.h
+++ b/source/adios2/engine/bp4/BP4Reader.h
@@ -13,6 +13,7 @@
 
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/format/bp4/BP4.h" //format::BP4Deserializer
 #include "adios2/toolkit/transportman/TransportMan.h"
 
@@ -32,10 +33,10 @@ class BP4Reader : public Engine
      * @param io
      * @param name
      * @param openMode only read
-     * @param mpiComm
+     * @param comm
      */
     BP4Reader(IO &io, const std::string &name, const Mode mode,
-              MPI_Comm mpiComm);
+              helper::Comm comm);
 
     virtual ~BP4Reader() = default;
 
diff --git a/source/adios2/engine/bp4/BP4Writer.cpp b/source/adios2/engine/bp4/BP4Writer.cpp
index 7af9a8c3d5..bb5f71a03b 100644
--- a/source/adios2/engine/bp4/BP4Writer.cpp
+++ b/source/adios2/engine/bp4/BP4Writer.cpp
@@ -29,12 +29,11 @@ namespace engine
 {
 
 BP4Writer::BP4Writer(IO &io, const std::string &name, const Mode mode,
-                     MPI_Comm mpiComm)
-: Engine("BP4Writer", io, name, mode, mpiComm),
-  m_BP4Serializer(mpiComm, m_DebugMode),
-  m_FileDataManager(mpiComm, m_DebugMode),
-  m_FileMetadataManager(mpiComm, m_DebugMode),
-  m_FileMetadataIndexManager(mpiComm, m_DebugMode)
+                     helper::Comm comm)
+: Engine("BP4Writer", io, name, mode, std::move(comm)),
+  m_BP4Serializer(m_Comm, m_DebugMode), m_FileDataManager(m_Comm, m_DebugMode),
+  m_FileMetadataManager(m_Comm, m_DebugMode),
+  m_FileMetadataIndexManager(m_Comm, m_DebugMode)
 {
     TAU_SCOPED_TIMER("BP4Writer::Open");
     m_IO.m_ReadStreaming = false;
diff --git a/source/adios2/engine/bp4/BP4Writer.h b/source/adios2/engine/bp4/BP4Writer.h
index 075888c31a..0e161d3014 100644
--- a/source/adios2/engine/bp4/BP4Writer.h
+++ b/source/adios2/engine/bp4/BP4Writer.h
@@ -13,6 +13,7 @@
 
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/format/bp4/BP4.h"
 #include "adios2/toolkit/transportman/TransportMan.h" //transport::TransportsMan
 
@@ -31,10 +32,10 @@ class BP4Writer : public core::Engine
      * Constructor for file Writer in BP4 format
      * @param name unique name given to the engine
      * @param openMode w (supported), r, a from OpenMode in ADIOSTypes.h
-     * @param mpiComm MPI communicator
+     * @param comm multi-process communicator
      */
     BP4Writer(IO &io, const std::string &name, const Mode mode,
-              MPI_Comm mpiComm);
+              helper::Comm comm);
 
     ~BP4Writer();
 
diff --git a/source/adios2/engine/dataman/DataManCommon.cpp b/source/adios2/engine/dataman/DataManCommon.cpp
index c54b2eb485..9b399bec79 100644
--- a/source/adios2/engine/dataman/DataManCommon.cpp
+++ b/source/adios2/engine/dataman/DataManCommon.cpp
@@ -19,13 +19,13 @@ namespace engine
 
 DataManCommon::DataManCommon(const std::string engineType, IO &io,
                              const std::string &name, const Mode mode,
-                             MPI_Comm mpiComm)
-: Engine(engineType, io, name, mode, mpiComm),
+                             helper::Comm comm)
+: Engine(engineType, io, name, mode, std::move(comm)),
   m_IsRowMajor(helper::IsRowMajor(io.m_HostLanguage)),
-  m_DataManSerializer(mpiComm, m_IsRowMajor)
+  m_DataManSerializer(m_Comm, m_IsRowMajor)
 {
-    MPI_Comm_rank(mpiComm, &m_MpiRank);
-    MPI_Comm_size(mpiComm, &m_MpiSize);
+    MPI_Comm_rank(m_Comm, &m_MpiRank);
+    MPI_Comm_size(m_Comm, &m_MpiSize);
     GetParameter(m_IO.m_Parameters, "IPAddress", m_IPAddress);
     GetParameter(m_IO.m_Parameters, "Port", m_Port);
     GetParameter(m_IO.m_Parameters, "StagingMode", m_StagingMode);
diff --git a/source/adios2/engine/dataman/DataManCommon.h b/source/adios2/engine/dataman/DataManCommon.h
index 78b09ccbbe..c638c1b7c1 100644
--- a/source/adios2/engine/dataman/DataManCommon.h
+++ b/source/adios2/engine/dataman/DataManCommon.h
@@ -14,6 +14,7 @@
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/common/ADIOSMacros.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/helper/adiosSystem.h"
 #include "adios2/toolkit/format/dataman/DataManSerializer.h"
 #include "adios2/toolkit/format/dataman/DataManSerializer.tcc"
@@ -32,7 +33,7 @@ class DataManCommon : public Engine
 
 public:
     DataManCommon(const std::string engineType, IO &io, const std::string &name,
-                  const Mode mode, MPI_Comm mpiComm);
+                  const Mode mode, helper::Comm comm);
     virtual ~DataManCommon();
 
 protected:
diff --git a/source/adios2/engine/dataman/DataManReader.cpp b/source/adios2/engine/dataman/DataManReader.cpp
index 7a24d5507e..fffedc628c 100644
--- a/source/adios2/engine/dataman/DataManReader.cpp
+++ b/source/adios2/engine/dataman/DataManReader.cpp
@@ -23,8 +23,8 @@ namespace engine
 {
 
 DataManReader::DataManReader(IO &io, const std::string &name, const Mode mode,
-                             MPI_Comm mpiComm)
-: DataManCommon("DataManReader", io, name, mode, mpiComm)
+                             helper::Comm comm)
+: DataManCommon("DataManReader", io, name, mode, std::move(comm))
 {
     GetParameter(m_IO.m_Parameters, "AlwaysProvideLatestTimestep",
                  m_ProvideLatest);
diff --git a/source/adios2/engine/dataman/DataManReader.h b/source/adios2/engine/dataman/DataManReader.h
index 32cc2f1ea5..b42705eade 100644
--- a/source/adios2/engine/dataman/DataManReader.h
+++ b/source/adios2/engine/dataman/DataManReader.h
@@ -14,6 +14,8 @@
 
 #include "DataManCommon.h"
 
+#include "adios2/helper/adiosComm.h"
+
 namespace adios2
 {
 namespace core
@@ -26,7 +28,7 @@ class DataManReader : public DataManCommon
 
 public:
     DataManReader(IO &io, const std::string &name, const Mode mode,
-                  MPI_Comm mpiComm);
+                  helper::Comm comm);
     virtual ~DataManReader();
     StepStatus BeginStep(StepMode stepMode, const float timeoutSeconds) final;
     size_t CurrentStep() const final;
diff --git a/source/adios2/engine/dataman/DataManWriter.cpp b/source/adios2/engine/dataman/DataManWriter.cpp
index 4e141fe5de..b125cff789 100644
--- a/source/adios2/engine/dataman/DataManWriter.cpp
+++ b/source/adios2/engine/dataman/DataManWriter.cpp
@@ -24,8 +24,8 @@ namespace engine
 {
 
 DataManWriter::DataManWriter(IO &io, const std::string &name,
-                             const Mode openMode, MPI_Comm mpiComm)
-: DataManCommon("DataManWriter", io, name, openMode, mpiComm)
+                             const Mode openMode, helper::Comm comm)
+: DataManCommon("DataManWriter", io, name, openMode, std::move(comm))
 {
     if (m_StagingMode == "wide")
     {
diff --git a/source/adios2/engine/dataman/DataManWriter.h b/source/adios2/engine/dataman/DataManWriter.h
index fa05213a16..2b1c9b42e5 100644
--- a/source/adios2/engine/dataman/DataManWriter.h
+++ b/source/adios2/engine/dataman/DataManWriter.h
@@ -13,6 +13,7 @@
 #define ADIOS2_ENGINE_DATAMAN_DATAMAN_WRITER_H_
 
 #include "DataManCommon.h"
+#include "adios2/helper/adiosComm.h"
 
 namespace adios2
 {
@@ -26,7 +27,7 @@ class DataManWriter : public DataManCommon
 
 public:
     DataManWriter(IO &io, const std::string &name, const Mode mode,
-                  MPI_Comm mpiComm);
+                  helper::Comm comm);
     virtual ~DataManWriter();
 
     StepStatus BeginStep(StepMode mode,
diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.cpp b/source/adios2/engine/dataspaces/DataSpacesReader.cpp
index 389f322c5f..e1eddfc8a4 100644
--- a/source/adios2/engine/dataspaces/DataSpacesReader.cpp
+++ b/source/adios2/engine/dataspaces/DataSpacesReader.cpp
@@ -24,8 +24,8 @@ namespace engine
 {
 
 DataSpacesReader::DataSpacesReader(IO &io, const std::string &name,
-                                   const Mode mode, MPI_Comm mpiComm)
-: Engine("DataSpacesReader", io, name, mode, mpiComm)
+                                   const Mode mode, helper::Comm comm)
+: Engine("DataSpacesReader", io, name, mode, std::move(comm))
 {
 
     f_Name = name;
@@ -52,6 +52,7 @@ DataSpacesReader::DataSpacesReader(IO &io, const std::string &name,
     {
         m_ProvideLatest = false;
     }
+    MPI_Comm mpiComm = m_Comm;
     ret = adios_read_dataspaces_init(&mpiComm, &m_data);
     if (ret < 0)
     {
diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.h b/source/adios2/engine/dataspaces/DataSpacesReader.h
index ce948fe2d0..1c79c9de73 100644
--- a/source/adios2/engine/dataspaces/DataSpacesReader.h
+++ b/source/adios2/engine/dataspaces/DataSpacesReader.h
@@ -15,6 +15,7 @@
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/Engine.h"
 #include "adios2/core/IO.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/dataspaces/ds.h"
 #include "mpi.h"
 
@@ -30,7 +31,7 @@ class DataSpacesReader : public Engine
 
 public:
     DataSpacesReader(IO &adios, const std::string &name, const Mode openMode,
-                     MPI_Comm mpiComm);
+                     helper::Comm comm);
 
     ~DataSpacesReader();
     StepStatus BeginStep();
diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp
index 010f2434fb..43b96d6637 100644
--- a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp
+++ b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp
@@ -25,8 +25,8 @@ namespace engine
 {
 
 DataSpacesWriter::DataSpacesWriter(IO &io, const std::string &name,
-                                   const Mode mode, MPI_Comm mpiComm)
-: Engine("DataSpacesWriter", io, name, mode, mpiComm)
+                                   const Mode mode, helper::Comm comm)
+: Engine("DataSpacesWriter", io, name, mode, std::move(comm))
 {
 
     f_Name = name;
@@ -40,6 +40,7 @@ DataSpacesWriter::DataSpacesWriter(IO &io, const std::string &name,
     {
         m_data.appid = 0;
     }
+    MPI_Comm mpiComm = m_Comm;
     ret = adios_dataspaces_init(&mpiComm, &m_data);
     if (ret < 0)
         fprintf(stderr, "Unable to connect to DataSpaces. Err: %d\n", ret);
diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.h b/source/adios2/engine/dataspaces/DataSpacesWriter.h
index 35df4d9ecb..7e038236ed 100644
--- a/source/adios2/engine/dataspaces/DataSpacesWriter.h
+++ b/source/adios2/engine/dataspaces/DataSpacesWriter.h
@@ -15,6 +15,7 @@
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/Engine.h"
 #include "adios2/core/IO.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/dataspaces/ds.h"
 #include "mpi.h"
 
@@ -33,11 +34,11 @@ class DataSpacesWriter : public Engine
      * Constructor for DataSpaces writer engine,
      * @param name unique name given to the engine
      * @param accessMode
-     * @param mpiComm
+     * @param comm
      * @param method
      */
     DataSpacesWriter(IO &io, const std::string &name, const Mode mode,
-                     MPI_Comm mpiComm);
+                     helper::Comm comm);
 
     ~DataSpacesWriter();
 
diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.cpp b/source/adios2/engine/hdf5/HDF5ReaderP.cpp
index 3f5b082edc..6a9fbfe962 100644
--- a/source/adios2/engine/hdf5/HDF5ReaderP.cpp
+++ b/source/adios2/engine/hdf5/HDF5ReaderP.cpp
@@ -24,8 +24,9 @@ namespace engine
 {
 
 HDF5ReaderP::HDF5ReaderP(IO &io, const std::string &name, const Mode openMode,
-                         MPI_Comm mpiComm)
-: Engine("HDF5Reader", io, name, openMode, mpiComm), m_H5File(io.m_DebugMode)
+                         helper::Comm comm)
+: Engine("HDF5Reader", io, name, openMode, std::move(comm)),
+  m_H5File(io.m_DebugMode)
 {
     m_EndMessage = ", in call to IO HDF5Reader Open " + m_Name + "\n";
     Init();
diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.h b/source/adios2/engine/hdf5/HDF5ReaderP.h
index 96e4620a75..9208fe6c99 100644
--- a/source/adios2/engine/hdf5/HDF5ReaderP.h
+++ b/source/adios2/engine/hdf5/HDF5ReaderP.h
@@ -13,6 +13,7 @@
 
 #include "adios2/core/Engine.h"
 #include "adios2/core/IO.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/interop/hdf5/HDF5Common.h"
 
 #include <map>
@@ -33,11 +34,11 @@ class HDF5ReaderP : public Engine
      * Constructor for single HDF5 reader engine, reads from HDF5 format
      * @param name unique name given to the engine
      * @param accessMode
-     * @param mpiComm
+     * @param comm
      * @param method
      */
     HDF5ReaderP(IO &adios, const std::string &name, const Mode openMode,
-                MPI_Comm mpiComm);
+                helper::Comm comm);
 
     ~HDF5ReaderP();
 
diff --git a/source/adios2/engine/hdf5/HDF5WriterP.cpp b/source/adios2/engine/hdf5/HDF5WriterP.cpp
index d874fe1643..c9bd9d804a 100644
--- a/source/adios2/engine/hdf5/HDF5WriterP.cpp
+++ b/source/adios2/engine/hdf5/HDF5WriterP.cpp
@@ -21,8 +21,9 @@ namespace engine
 {
 
 HDF5WriterP::HDF5WriterP(IO &io, const std::string &name, const Mode mode,
-                         MPI_Comm mpiComm)
-: Engine("HDF5Writer", io, name, mode, mpiComm), m_H5File(io.m_DebugMode)
+                         helper::Comm comm)
+: Engine("HDF5Writer", io, name, mode, std::move(comm)),
+  m_H5File(io.m_DebugMode)
 {
     m_IO.m_ReadStreaming = false;
     m_EndMessage = ", in call to IO HDF5Writer Open " + m_Name + "\n";
diff --git a/source/adios2/engine/hdf5/HDF5WriterP.h b/source/adios2/engine/hdf5/HDF5WriterP.h
index 797d3de5f4..98087c45f3 100644
--- a/source/adios2/engine/hdf5/HDF5WriterP.h
+++ b/source/adios2/engine/hdf5/HDF5WriterP.h
@@ -16,6 +16,7 @@
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/Engine.h"
 #include "adios2/core/IO.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/interop/hdf5/HDF5Common.h"
 
 namespace adios2
@@ -33,11 +34,11 @@ class HDF5WriterP : public Engine
      * Constructor for HDF5 writer engine, writes in hdf5 format
      * @param name unique name given to the engine
      * @param accessMode
-     * @param mpiComm
+     * @param comm
      * @param method
      */
     HDF5WriterP(IO &io, const std::string &name, const Mode mode,
-                MPI_Comm mpiComm);
+                helper::Comm comm);
 
     ~HDF5WriterP();
 
diff --git a/source/adios2/engine/inline/InlineReader.cpp b/source/adios2/engine/inline/InlineReader.cpp
index b22d1eb294..72d8f5e827 100644
--- a/source/adios2/engine/inline/InlineReader.cpp
+++ b/source/adios2/engine/inline/InlineReader.cpp
@@ -23,11 +23,11 @@ namespace engine
 {
 
 InlineReader::InlineReader(IO &io, const std::string &name, const Mode mode,
-                           MPI_Comm mpiComm)
-: Engine("InlineReader", io, name, mode, mpiComm)
+                           helper::Comm comm)
+: Engine("InlineReader", io, name, mode, std::move(comm))
 {
     m_EndMessage = " in call to IO Open InlineReader " + m_Name + "\n";
-    MPI_Comm_rank(mpiComm, &m_ReaderRank);
+    MPI_Comm_rank(m_Comm, &m_ReaderRank);
     Init();
     Engine &writer = io.GetEngine(m_WriterID);
     if (m_Verbosity == 5)
diff --git a/source/adios2/engine/inline/InlineReader.h b/source/adios2/engine/inline/InlineReader.h
index 6a3feda3b9..c9fedfdcd1 100644
--- a/source/adios2/engine/inline/InlineReader.h
+++ b/source/adios2/engine/inline/InlineReader.h
@@ -15,6 +15,7 @@
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/ADIOS.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/helper/adiosFunctions.h"
 
 namespace adios2
@@ -33,13 +34,13 @@ class InlineReader : public Engine
      * heap capsule
      * @param name unique name given to the engine
      * @param accessMode
-     * @param mpiComm
+     * @param comm
      * @param method
      * @param debugMode
      * @param hostLanguage
      */
     InlineReader(IO &adios, const std::string &name, const Mode mode,
-                 MPI_Comm mpiComm);
+                 helper::Comm comm);
 
     ~InlineReader();
     StepStatus BeginStep(StepMode mode = StepMode::Read,
diff --git a/source/adios2/engine/inline/InlineWriter.cpp b/source/adios2/engine/inline/InlineWriter.cpp
index 364a0c3d24..ebcb081fd3 100644
--- a/source/adios2/engine/inline/InlineWriter.cpp
+++ b/source/adios2/engine/inline/InlineWriter.cpp
@@ -23,11 +23,11 @@ namespace engine
 {
 
 InlineWriter::InlineWriter(IO &io, const std::string &name, const Mode mode,
-                           MPI_Comm mpiComm)
-: Engine("InlineWriter", io, name, mode, mpiComm)
+                           helper::Comm comm)
+: Engine("InlineWriter", io, name, mode, std::move(comm))
 {
     m_EndMessage = " in call to InlineWriter " + m_Name + " Open\n";
-    MPI_Comm_rank(mpiComm, &m_WriterRank);
+    MPI_Comm_rank(m_Comm, &m_WriterRank);
     Init();
     if (m_Verbosity == 5)
     {
diff --git a/source/adios2/engine/inline/InlineWriter.h b/source/adios2/engine/inline/InlineWriter.h
index 865f6c180c..a5adb70c33 100644
--- a/source/adios2/engine/inline/InlineWriter.h
+++ b/source/adios2/engine/inline/InlineWriter.h
@@ -14,6 +14,7 @@
 
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 
 namespace adios2
 {
@@ -30,12 +31,12 @@ class InlineWriter : public Engine
      * Constructor for Writer
      * @param name unique name given to the engine
      * @param accessMode
-     * @param mpiComm
+     * @param comm
      * @param method
      * @param debugMode
      */
     InlineWriter(IO &adios, const std::string &name, const Mode mode,
-                 MPI_Comm mpiComm);
+                 helper::Comm comm);
 
     ~InlineWriter() = default;
 
diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.cpp b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
index 471ac025b6..fb450abe7c 100644
--- a/source/adios2/engine/insitumpi/InSituMPIReader.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
@@ -29,19 +29,19 @@ namespace engine
 {
 
 InSituMPIReader::InSituMPIReader(IO &io, const std::string &name,
-                                 const Mode mode, MPI_Comm mpiComm)
-: Engine("InSituMPIReader", io, name, mode, mpiComm),
-  m_BP3Deserializer(mpiComm, m_DebugMode)
+                                 const Mode mode, helper::Comm comm)
+: Engine("InSituMPIReader", io, name, mode, std::move(comm)),
+  m_BP3Deserializer(m_Comm, m_DebugMode)
 {
     TAU_SCOPED_TIMER("InSituMPIReader::Open");
     m_EndMessage = " in call to IO Open InSituMPIReader " + m_Name + "\n";
     Init();
 
-    m_RankAllPeers = insitumpi::FindPeers(mpiComm, m_Name, false, m_CommWorld);
+    m_RankAllPeers = insitumpi::FindPeers(m_Comm, m_Name, false, m_CommWorld);
     MPI_Comm_rank(m_CommWorld, &m_GlobalRank);
     MPI_Comm_size(m_CommWorld, &m_GlobalNproc);
-    MPI_Comm_rank(mpiComm, &m_ReaderRank);
-    MPI_Comm_size(mpiComm, &m_ReaderNproc);
+    MPI_Comm_rank(m_Comm, &m_ReaderRank);
+    MPI_Comm_size(m_Comm, &m_ReaderNproc);
     m_RankDirectPeers =
         insitumpi::AssignPeers(m_ReaderRank, m_ReaderNproc, m_RankAllPeers);
     if (m_Verbosity == 5)
diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.h b/source/adios2/engine/insitumpi/InSituMPIReader.h
index e45b145e4f..17631d1d9f 100644
--- a/source/adios2/engine/insitumpi/InSituMPIReader.h
+++ b/source/adios2/engine/insitumpi/InSituMPIReader.h
@@ -17,6 +17,7 @@
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/ADIOS.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/helper/adiosFunctions.h"
 #include "adios2/toolkit/format/bp3/BP3.h"
 
@@ -36,13 +37,13 @@ class InSituMPIReader : public Engine
      * heap capsule
      * @param name unique name given to the engine
      * @param accessMode
-     * @param mpiComm
+     * @param comm
      * @param method
      * @param debugMode
      * @param hostLanguage
      */
     InSituMPIReader(IO &adios, const std::string &name, const Mode mode,
-                    MPI_Comm mpiComm);
+                    helper::Comm comm);
 
     ~InSituMPIReader();
     StepStatus BeginStep(StepMode mode = StepMode::Read,
diff --git a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
index 2474b50de9..4d6967340b 100644
--- a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
@@ -29,24 +29,24 @@ namespace engine
 {
 
 InSituMPIWriter::InSituMPIWriter(IO &io, const std::string &name,
-                                 const Mode mode, MPI_Comm mpiComm)
-: Engine("InSituMPIWriter", io, name, mode, mpiComm),
-  m_BP3Serializer(mpiComm, m_DebugMode)
+                                 const Mode mode, helper::Comm comm)
+: Engine("InSituMPIWriter", io, name, mode, std::move(comm)),
+  m_BP3Serializer(m_Comm, m_DebugMode)
 {
     TAU_SCOPED_TIMER("InSituMPIWriter::Open");
     m_EndMessage = " in call to InSituMPIWriter " + m_Name + " Open\n";
     Init();
     m_BP3Serializer.InitParameters(m_IO.m_Parameters);
 
-    m_RankAllPeers = insitumpi::FindPeers(mpiComm, m_Name, true, m_CommWorld);
+    m_RankAllPeers = insitumpi::FindPeers(m_Comm, m_Name, true, m_CommWorld);
     for (int i = 0; i < m_RankAllPeers.size(); i++)
     {
         m_RankToPeerID[m_RankAllPeers[i]] = i;
     }
     MPI_Comm_rank(m_CommWorld, &m_GlobalRank);
     MPI_Comm_size(m_CommWorld, &m_GlobalNproc);
-    MPI_Comm_rank(mpiComm, &m_WriterRank);
-    MPI_Comm_size(mpiComm, &m_WriterNproc);
+    MPI_Comm_rank(m_Comm, &m_WriterRank);
+    MPI_Comm_size(m_Comm, &m_WriterNproc);
     m_RankDirectPeers =
         insitumpi::AssignPeers(m_WriterRank, m_WriterNproc, m_RankAllPeers);
     int primaryContact = insitumpi::ConnectDirectPeers(
diff --git a/source/adios2/engine/insitumpi/InSituMPIWriter.h b/source/adios2/engine/insitumpi/InSituMPIWriter.h
index ae8c4b712a..2fbd623901 100644
--- a/source/adios2/engine/insitumpi/InSituMPIWriter.h
+++ b/source/adios2/engine/insitumpi/InSituMPIWriter.h
@@ -17,6 +17,7 @@
 #include "InSituMPISchedules.h"
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/format/bp3/BP3.h"
 
 namespace adios2
@@ -34,12 +35,12 @@ class InSituMPIWriter : public Engine
      * Constructor for Writer writes in ADIOS 1.x BP format
      * @param name unique name given to the engine
      * @param accessMode
-     * @param mpiComm
+     * @param comm
      * @param method
      * @param debugMode
      */
     InSituMPIWriter(IO &adios, const std::string &name, const Mode openMode,
-                    MPI_Comm mpiComm);
+                    helper::Comm comm);
 
     ~InSituMPIWriter();
 
diff --git a/source/adios2/engine/mixer/HDFMixer.cpp b/source/adios2/engine/mixer/HDFMixer.cpp
index 9e58ee7e30..1b2cebbbe5 100644
--- a/source/adios2/engine/mixer/HDFMixer.cpp
+++ b/source/adios2/engine/mixer/HDFMixer.cpp
@@ -24,11 +24,11 @@ namespace engine
 {
 
 HDFMixer::HDFMixer(IO &io, const std::string &name, const Mode openMode,
-                   MPI_Comm mpiComm)
-: Engine("HDFMixer", io, name, openMode, mpiComm),
-  m_HDFVDSWriter(mpiComm, m_DebugMode),
+                   helper::Comm comm)
+: Engine("HDFMixer", io, name, openMode, std::move(comm)),
+  m_HDFVDSWriter(m_Comm, m_DebugMode),
   m_HDFSerialWriter(MPI_COMM_SELF, m_DebugMode),
-  m_TransportsManager(mpiComm, m_DebugMode)
+  m_TransportsManager(m_Comm, m_DebugMode)
 {
     m_EndMessage = " in call to IO Open HDFMixer " + m_Name + "\n";
     Init();
diff --git a/source/adios2/engine/mixer/HDFMixer.h b/source/adios2/engine/mixer/HDFMixer.h
index 3a830a9c15..75ba6fc154 100644
--- a/source/adios2/engine/mixer/HDFMixer.h
+++ b/source/adios2/engine/mixer/HDFMixer.h
@@ -26,6 +26,7 @@
 #include "adios2/common/ADIOSMacros.h"
 #include "adios2/common/ADIOSTypes.h"
 #include "adios2/core/Variable.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/helper/adiosFunctions.h"
 //#include "adios2/toolkit/capsule/heap/STLVector.h"
 #include "adios2/toolkit/transportman/TransportMan.h" //transport::TransportsMan
@@ -47,10 +48,10 @@ class HDFMixer : public Engine
      * Constructor for file Writer in H5 format
      * @param name unique name given to the engine
      * @param openMode w (supported), r, a from OpenMode in ADIOSTypes.h
-     * @param mpiComm MPI communicator
+     * @param comm multi-process communicator
      */
     HDFMixer(IO &io, const std::string &name, const Mode openMode,
-             MPI_Comm mpiComm);
+             helper::Comm comm);
 
     ~HDFMixer();
 
diff --git a/source/adios2/engine/null/NullEngine.cpp b/source/adios2/engine/null/NullEngine.cpp
index 67b6095833..80be7dc656 100644
--- a/source/adios2/engine/null/NullEngine.cpp
+++ b/source/adios2/engine/null/NullEngine.cpp
@@ -18,8 +18,8 @@ namespace engine
 {
 
 NullEngine::NullEngine(IO &io, const std::string &name, const Mode mode,
-                       MPI_Comm mpiComm)
-: Engine("NULL", io, name, mode, mpiComm)
+                       helper::Comm comm)
+: Engine("NULL", io, name, mode, std::move(comm))
 {
 }
 
diff --git a/source/adios2/engine/null/NullEngine.h b/source/adios2/engine/null/NullEngine.h
index 16e63de527..6cd54c7812 100644
--- a/source/adios2/engine/null/NullEngine.h
+++ b/source/adios2/engine/null/NullEngine.h
@@ -13,6 +13,7 @@
 
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 
 namespace adios2
 {
@@ -25,7 +26,7 @@ class NullEngine : public Engine
 {
 public:
     NullEngine(IO &adios, const std::string &name, const Mode mode,
-               MPI_Comm mpiComm);
+               helper::Comm comm);
 
     ~NullEngine() = default;
 
diff --git a/source/adios2/engine/nullcore/NullCoreWriter.cpp b/source/adios2/engine/nullcore/NullCoreWriter.cpp
index dd484aed5e..970b7864c0 100644
--- a/source/adios2/engine/nullcore/NullCoreWriter.cpp
+++ b/source/adios2/engine/nullcore/NullCoreWriter.cpp
@@ -16,8 +16,8 @@ struct NullCoreWriter::NullCoreWriterImpl
 };
 
 NullCoreWriter::NullCoreWriter(IO &io, const std::string &name, const Mode mode,
-                               MPI_Comm mpiComm)
-: Engine("NullCoreWriter", io, name, mode, mpiComm),
+                               helper::Comm comm)
+: Engine("NullCoreWriter", io, name, mode, std::move(comm)),
   Impl(new NullCoreWriter::NullCoreWriterImpl)
 {
 }
diff --git a/source/adios2/engine/nullcore/NullCoreWriter.h b/source/adios2/engine/nullcore/NullCoreWriter.h
index e1aa37c4d9..3fff8b7969 100644
--- a/source/adios2/engine/nullcore/NullCoreWriter.h
+++ b/source/adios2/engine/nullcore/NullCoreWriter.h
@@ -15,6 +15,7 @@
 
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 
 namespace adios2
 {
@@ -28,7 +29,7 @@ class NullCoreWriter : public core::Engine
 
 public:
     NullCoreWriter(IO &io, const std::string &name, const Mode mode,
-                   MPI_Comm mpiComm);
+                   helper::Comm comm);
 
     virtual ~NullCoreWriter();
 
diff --git a/source/adios2/engine/skeleton/SkeletonReader.cpp b/source/adios2/engine/skeleton/SkeletonReader.cpp
index 8fc36f28a5..687c5d8a00 100644
--- a/source/adios2/engine/skeleton/SkeletonReader.cpp
+++ b/source/adios2/engine/skeleton/SkeletonReader.cpp
@@ -23,11 +23,11 @@ namespace engine
 {
 
 SkeletonReader::SkeletonReader(IO &io, const std::string &name, const Mode mode,
-                               MPI_Comm mpiComm)
-: Engine("SkeletonReader", io, name, mode, mpiComm)
+                               helper::Comm comm)
+: Engine("SkeletonReader", io, name, mode, std::move(comm))
 {
     m_EndMessage = " in call to IO Open SkeletonReader " + m_Name + "\n";
-    MPI_Comm_rank(mpiComm, &m_ReaderRank);
+    MPI_Comm_rank(m_Comm, &m_ReaderRank);
     Init();
     if (m_Verbosity == 5)
     {
diff --git a/source/adios2/engine/skeleton/SkeletonReader.h b/source/adios2/engine/skeleton/SkeletonReader.h
index 0c6ff9c463..63e55ff773 100644
--- a/source/adios2/engine/skeleton/SkeletonReader.h
+++ b/source/adios2/engine/skeleton/SkeletonReader.h
@@ -15,6 +15,7 @@
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/ADIOS.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/helper/adiosFunctions.h"
 
 namespace adios2
@@ -33,13 +34,13 @@ class SkeletonReader : public Engine
      * heap capsule
      * @param name unique name given to the engine
      * @param accessMode
-     * @param mpiComm
+     * @param comm
      * @param method
      * @param debugMode
      * @param hostLanguage
      */
     SkeletonReader(IO &adios, const std::string &name, const Mode mode,
-                   MPI_Comm mpiComm);
+                   helper::Comm comm);
 
     ~SkeletonReader();
     StepStatus BeginStep(StepMode mode = StepMode::Read,
diff --git a/source/adios2/engine/skeleton/SkeletonWriter.cpp b/source/adios2/engine/skeleton/SkeletonWriter.cpp
index 1db1c64dab..0d9d9427aa 100644
--- a/source/adios2/engine/skeleton/SkeletonWriter.cpp
+++ b/source/adios2/engine/skeleton/SkeletonWriter.cpp
@@ -24,11 +24,11 @@ namespace engine
 {
 
 SkeletonWriter::SkeletonWriter(IO &io, const std::string &name, const Mode mode,
-                               MPI_Comm mpiComm)
-: Engine("SkeletonWriter", io, name, mode, mpiComm)
+                               helper::Comm comm)
+: Engine("SkeletonWriter", io, name, mode, std::move(comm))
 {
     m_EndMessage = " in call to SkeletonWriter " + m_Name + " Open\n";
-    MPI_Comm_rank(mpiComm, &m_WriterRank);
+    MPI_Comm_rank(m_Comm, &m_WriterRank);
     Init();
     if (m_Verbosity == 5)
     {
diff --git a/source/adios2/engine/skeleton/SkeletonWriter.h b/source/adios2/engine/skeleton/SkeletonWriter.h
index 55cd281df0..0bcab56812 100644
--- a/source/adios2/engine/skeleton/SkeletonWriter.h
+++ b/source/adios2/engine/skeleton/SkeletonWriter.h
@@ -14,6 +14,7 @@
 
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 
 namespace adios2
 {
@@ -30,12 +31,12 @@ class SkeletonWriter : public Engine
      * Constructor for Writer
      * @param name unique name given to the engine
      * @param accessMode
-     * @param mpiComm
+     * @param comm
      * @param method
      * @param debugMode
      */
     SkeletonWriter(IO &adios, const std::string &name, const Mode mode,
-                   MPI_Comm mpiComm);
+                   helper::Comm comm);
 
     ~SkeletonWriter() = default;
 
diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp
index a04995ec0d..bdacb225fc 100644
--- a/source/adios2/engine/ssc/SscReader.cpp
+++ b/source/adios2/engine/ssc/SscReader.cpp
@@ -26,14 +26,14 @@ namespace engine
 {
 
 SscReader::SscReader(IO &io, const std::string &name, const Mode mode,
-                     MPI_Comm mpiComm)
-: Engine("SscReader", io, name, mode, mpiComm),
-  m_DataManSerializer(mpiComm, helper::IsRowMajor(io.m_HostLanguage)),
+                     helper::Comm comm)
+: Engine("SscReader", io, name, mode, std::move(comm)),
+  m_DataManSerializer(m_Comm, helper::IsRowMajor(io.m_HostLanguage)),
   m_RepliedMetadata(std::make_shared<std::vector<char>>())
 {
     TAU_SCOPED_TIMER_FUNC();
 
-    MPI_Comm_rank(mpiComm, &m_MpiRank);
+    MPI_Comm_rank(m_Comm, &m_MpiRank);
     srand(time(NULL));
 
     // initialize parameters from IO
diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h
index 70786a7525..95acc79848 100644
--- a/source/adios2/engine/ssc/SscReader.h
+++ b/source/adios2/engine/ssc/SscReader.h
@@ -16,6 +16,7 @@
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/ADIOS.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/helper/adiosFunctions.h"
 #include "adios2/toolkit/format/dataman/DataManSerializer.h"
 #include "adios2/toolkit/format/dataman/DataManSerializer.tcc"
@@ -33,7 +34,7 @@ class SscReader : public Engine
 {
 public:
     SscReader(IO &adios, const std::string &name, const Mode mode,
-              MPI_Comm mpiComm);
+              helper::Comm comm);
 
     ~SscReader();
     StepStatus BeginStep(
diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp
index 8ab66f457c..7fdf3e398a 100644
--- a/source/adios2/engine/ssc/SscWriter.cpp
+++ b/source/adios2/engine/ssc/SscWriter.cpp
@@ -26,9 +26,9 @@ namespace engine
 {
 
 SscWriter::SscWriter(IO &io, const std::string &name, const Mode mode,
-                     MPI_Comm mpiComm)
-: Engine("SscWriter", io, name, mode, mpiComm),
-  m_DataManSerializer(mpiComm, helper::IsRowMajor(io.m_HostLanguage))
+                     helper::Comm comm)
+: Engine("SscWriter", io, name, mode, std::move(comm)),
+  m_DataManSerializer(m_Comm, helper::IsRowMajor(io.m_HostLanguage))
 {
     TAU_SCOPED_TIMER_FUNC();
     Init();
diff --git a/source/adios2/engine/ssc/SscWriter.h b/source/adios2/engine/ssc/SscWriter.h
index 0b7abde9f6..b22340f2ff 100644
--- a/source/adios2/engine/ssc/SscWriter.h
+++ b/source/adios2/engine/ssc/SscWriter.h
@@ -14,6 +14,7 @@
 #include <queue>
 
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/format/dataman/DataManSerializer.h"
 #include "adios2/toolkit/format/dataman/DataManSerializer.tcc"
 #include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
@@ -31,7 +32,7 @@ class SscWriter : public Engine
 
 public:
     SscWriter(IO &adios, const std::string &name, const Mode mode,
-              MPI_Comm mpiComm);
+              helper::Comm comm);
 
     ~SscWriter() = default;
 
diff --git a/source/adios2/engine/sst/SstReader.cpp b/source/adios2/engine/sst/SstReader.cpp
index ef2e3bb018..3158a6484c 100644
--- a/source/adios2/engine/sst/SstReader.cpp
+++ b/source/adios2/engine/sst/SstReader.cpp
@@ -27,15 +27,15 @@ namespace engine
 {
 
 SstReader::SstReader(IO &io, const std::string &name, const Mode mode,
-                     MPI_Comm mpiComm)
-: Engine("SstReader", io, name, mode, mpiComm)
+                     helper::Comm comm)
+: Engine("SstReader", io, name, mode, std::move(comm))
 {
     char *cstr = new char[name.length() + 1];
     std::strcpy(cstr, name.c_str());
 
     Init();
 
-    m_Input = SstReaderOpen(cstr, &Params, mpiComm);
+    m_Input = SstReaderOpen(cstr, &Params, m_Comm);
     if (!m_Input)
     {
         throw std::runtime_error(
diff --git a/source/adios2/engine/sst/SstReader.h b/source/adios2/engine/sst/SstReader.h
index fa43a2cced..97932b7438 100644
--- a/source/adios2/engine/sst/SstReader.h
+++ b/source/adios2/engine/sst/SstReader.h
@@ -17,6 +17,7 @@
 
 #include "adios2/core/Engine.h"
 #include "adios2/core/IO.h"
+#include "adios2/helper/adiosComm.h"
 
 namespace adios2
 {
@@ -34,13 +35,13 @@ class SstReader : public Engine
      * @param adios
      * @param name
      * @param accessMode
-     * @param mpiComm
+     * @param comm
      * @param method
      * @param debugMode
      * @param nthreads
      */
     SstReader(IO &io, const std::string &name, const Mode mode,
-              MPI_Comm mpiComm);
+              helper::Comm comm);
 
     virtual ~SstReader();
 
diff --git a/source/adios2/engine/sst/SstWriter.cpp b/source/adios2/engine/sst/SstWriter.cpp
index 16d09cfb1f..456171b9cd 100644
--- a/source/adios2/engine/sst/SstWriter.cpp
+++ b/source/adios2/engine/sst/SstWriter.cpp
@@ -24,8 +24,8 @@ namespace engine
 {
 
 SstWriter::SstWriter(IO &io, const std::string &name, const Mode mode,
-                     MPI_Comm mpiComm)
-: Engine("SstWriter", io, name, mode, mpiComm)
+                     helper::Comm comm)
+: Engine("SstWriter", io, name, mode, std::move(comm))
 {
     auto AssembleMetadata = [](void *writer, int CohortSize,
                                struct _SstData * /*PerRankMetadata*/,
@@ -109,7 +109,7 @@ SstWriter::SstWriter(IO &io, const std::string &name, const Mode mode,
 
     Init();
 
-    m_Output = SstWriterOpen(name.c_str(), &Params, mpiComm);
+    m_Output = SstWriterOpen(name.c_str(), &Params, m_Comm);
 
     if (m_MarshalMethod == SstMarshalBP)
     {
diff --git a/source/adios2/engine/sst/SstWriter.h b/source/adios2/engine/sst/SstWriter.h
index 59785c5c3e..6795eeb3d0 100644
--- a/source/adios2/engine/sst/SstWriter.h
+++ b/source/adios2/engine/sst/SstWriter.h
@@ -18,6 +18,7 @@
 
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/core/Engine.h"
+#include "adios2/helper/adiosComm.h"
 
 namespace adios2
 {
@@ -31,7 +32,7 @@ class SstWriter : public Engine
 
 public:
     SstWriter(IO &io, const std::string &name, const Mode mode,
-              MPI_Comm mpiComm);
+              helper::Comm comm);
 
     virtual ~SstWriter();
 
diff --git a/source/adios2/engine/table/TableWriter.cpp b/source/adios2/engine/table/TableWriter.cpp
index 5d0cc0b12b..c99717d0b4 100644
--- a/source/adios2/engine/table/TableWriter.cpp
+++ b/source/adios2/engine/table/TableWriter.cpp
@@ -24,15 +24,15 @@ namespace engine
 {
 
 TableWriter::TableWriter(IO &io, const std::string &name, const Mode mode,
-                         MPI_Comm mpiComm)
-: Engine("TableWriter", io, name, mode, mpiComm),
+                         helper::Comm comm)
+: Engine("TableWriter", io, name, mode, std::move(comm)),
   m_IsRowMajor(helper::IsRowMajor(m_IO.m_HostLanguage)),
   m_Deserializer(m_Comm, m_IsRowMajor),
   m_SubAdios(MPI_COMM_WORLD, adios2::DebugOFF),
   m_SubIO(m_SubAdios.DeclareIO("SubIO"))
 {
-    MPI_Comm_rank(mpiComm, &m_MpiRank);
-    MPI_Comm_size(mpiComm, &m_MpiSize);
+    MPI_Comm_rank(m_Comm, &m_MpiRank);
+    MPI_Comm_size(m_Comm, &m_MpiSize);
     Init();
 }
 
diff --git a/source/adios2/engine/table/TableWriter.h b/source/adios2/engine/table/TableWriter.h
index a16f0960fc..16de48e222 100644
--- a/source/adios2/engine/table/TableWriter.h
+++ b/source/adios2/engine/table/TableWriter.h
@@ -14,6 +14,7 @@
 #include "adios2/core/Engine.h"
 #include "adios2/engine/bp3/BP3Writer.h"
 #include "adios2/engine/bp4/BP4Writer.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/format/dataman/DataManSerializer.h"
 #include "adios2/toolkit/format/dataman/DataManSerializer.tcc"
 #include "adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h"
@@ -32,7 +33,7 @@ class TableWriter : public Engine
 
 public:
     TableWriter(IO &adios, const std::string &name, const Mode mode,
-                MPI_Comm mpiComm);
+                helper::Comm comm);
 
     virtual ~TableWriter();
 
diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
index f9fa744c5e..2923eb7b22 100644
--- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
+++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
@@ -21,13 +21,14 @@ namespace adios2
 namespace format
 {
 
-DataManSerializer::DataManSerializer(MPI_Comm mpiComm, const bool isRowMajor)
-: m_MpiComm(mpiComm), m_IsRowMajor(isRowMajor),
+DataManSerializer::DataManSerializer(helper::Comm const &comm,
+                                     const bool isRowMajor)
+: m_Comm(comm), m_IsRowMajor(isRowMajor),
   m_IsLittleEndian(helper::IsLittleEndian()),
   m_DeferredRequestsToSend(std::make_shared<DeferredRequestMap>())
 {
-    MPI_Comm_size(m_MpiComm, &m_MpiSize);
-    MPI_Comm_rank(m_MpiComm, &m_MpiRank);
+    MPI_Comm_size(m_Comm, &m_MpiSize);
+    MPI_Comm_rank(m_Comm, &m_MpiRank);
 }
 
 void DataManSerializer::NewWriterBuffer(size_t bufferSize)
@@ -73,7 +74,7 @@ void DataManSerializer::AggregateMetadata()
     auto localJsonPack = SerializeJson(m_MetadataJson);
     unsigned int size = localJsonPack->size();
     unsigned int maxSize;
-    MPI_Allreduce(&size, &maxSize, 1, MPI_UNSIGNED, MPI_MAX, m_MpiComm);
+    MPI_Allreduce(&size, &maxSize, 1, MPI_UNSIGNED, MPI_MAX, m_Comm);
     maxSize += sizeof(uint64_t);
     localJsonPack->resize(maxSize, '\0');
     *(reinterpret_cast<uint64_t *>(localJsonPack->data() +
@@ -82,7 +83,7 @@ void DataManSerializer::AggregateMetadata()
 
     std::vector<char> globalJsonStr(m_MpiSize * maxSize);
     MPI_Allgather(localJsonPack->data(), maxSize, MPI_CHAR,
-                  globalJsonStr.data(), maxSize, MPI_CHAR, m_MpiComm);
+                  globalJsonStr.data(), maxSize, MPI_CHAR, m_Comm);
 
     nlohmann::json aggMetadata;
 
@@ -260,7 +261,8 @@ VecPtr DataManSerializer::GetAggregatedMetadataPack(const int64_t stepRequested,
     return ret;
 }
 
-void DataManSerializer::PutAggregatedMetadata(VecPtr input, MPI_Comm mpiComm)
+void DataManSerializer::PutAggregatedMetadata(VecPtr input,
+                                              helper::Comm const &comm)
 {
     TAU_SCOPED_TIMER_FUNC();
     if (input == nullptr)
@@ -271,7 +273,7 @@ void DataManSerializer::PutAggregatedMetadata(VecPtr input, MPI_Comm mpiComm)
         return;
     }
 
-    helper::BroadcastVector(*input, mpiComm);
+    helper::BroadcastVector(*input, comm);
 
     if (input->size() > 0)
     {
diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.h b/source/adios2/toolkit/format/dataman/DataManSerializer.h
index e1009526be..b7a45dad2a 100644
--- a/source/adios2/toolkit/format/dataman/DataManSerializer.h
+++ b/source/adios2/toolkit/format/dataman/DataManSerializer.h
@@ -13,6 +13,7 @@
 
 #include "adios2/common/ADIOSTypes.h"
 #include "adios2/core/IO.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
 
 #include <mutex>
@@ -83,7 +84,7 @@ using DeferredRequestMapPtr = std::shared_ptr<DeferredRequestMap>;
 class DataManSerializer
 {
 public:
-    DataManSerializer(MPI_Comm mpiComm, const bool isRowMajor);
+    DataManSerializer(helper::Comm const &comm, const bool isRowMajor);
 
     // clear and allocate new buffer for writer
     void NewWriterBuffer(size_t size);
@@ -146,7 +147,7 @@ class DataManSerializer
 
     const DmvVecPtrMap GetMetaData();
 
-    void PutAggregatedMetadata(VecPtr input, MPI_Comm mpiComm);
+    void PutAggregatedMetadata(VecPtr input, helper::Comm const &comm);
 
     int PutDeferredRequest(const std::string &variable, const size_t step,
                            const Dims &start, const Dims &count, void *data);
@@ -241,7 +242,7 @@ class DataManSerializer
     bool m_EnableStat = true;
     int m_MpiRank;
     int m_MpiSize;
-    MPI_Comm m_MpiComm;
+    helper::Comm const &m_Comm;
 
     int m_Verbosity = 0;
 };

From 7ea742dbbe3eced02d17df40b407692705dc1bea Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 18 Jun 2019 13:00:57 -0400
Subject: [PATCH 12/42] bp4: Rename m_MPIComm to m_Comm

---
 source/adios2/engine/bp4/BP4Reader.cpp             | 2 +-
 source/adios2/toolkit/format/bp4/BP4Base.cpp       | 8 ++++----
 source/adios2/toolkit/format/bp4/BP4Base.h         | 2 +-
 source/adios2/toolkit/format/bp4/BP4Serializer.cpp | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/source/adios2/engine/bp4/BP4Reader.cpp b/source/adios2/engine/bp4/BP4Reader.cpp
index 7e8f79b10f..8469405415 100644
--- a/source/adios2/engine/bp4/BP4Reader.cpp
+++ b/source/adios2/engine/bp4/BP4Reader.cpp
@@ -440,7 +440,7 @@ bool BP4Reader::CheckWriterActive()
         bool active = m_BP4Deserializer.ReadActiveFlag(header);
         flag = (active ? 1 : 0);
     }
-    flag = helper::BroadcastValue(flag, m_BP4Deserializer.m_MPIComm, 0);
+    flag = helper::BroadcastValue(flag, m_BP4Deserializer.m_Comm, 0);
     m_BP4Deserializer.m_WriterIsActive = (flag > 0);
     return m_BP4Deserializer.m_WriterIsActive;
 }
diff --git a/source/adios2/toolkit/format/bp4/BP4Base.cpp b/source/adios2/toolkit/format/bp4/BP4Base.cpp
index b9b6cd700a..4ad648b740 100644
--- a/source/adios2/toolkit/format/bp4/BP4Base.cpp
+++ b/source/adios2/toolkit/format/bp4/BP4Base.cpp
@@ -49,10 +49,10 @@ const std::map<int, std::string> BP4Base::m_TransformTypesToNames = {
 };
 
 BP4Base::BP4Base(MPI_Comm mpiComm, const bool debugMode)
-: m_MPIComm(mpiComm), m_DebugMode(debugMode)
+: m_Comm(mpiComm), m_DebugMode(debugMode)
 {
-    SMPI_Comm_rank(m_MPIComm, &m_RankMPI);
-    SMPI_Comm_size(m_MPIComm, &m_SizeMPI);
+    SMPI_Comm_rank(m_Comm, &m_RankMPI);
+    SMPI_Comm_size(m_Comm, &m_SizeMPI);
     m_Profiler.IsActive = true; // default
 }
 
@@ -913,7 +913,7 @@ void BP4Base::InitParameterSubStreams(const std::string value)
 
     if (subStreams < m_SizeMPI)
     {
-        m_Aggregator.Init(subStreams, m_MPIComm);
+        m_Aggregator.Init(subStreams, m_Comm);
     }
 }
 
diff --git a/source/adios2/toolkit/format/bp4/BP4Base.h b/source/adios2/toolkit/format/bp4/BP4Base.h
index eaf46e9108..54a4f26b5c 100644
--- a/source/adios2/toolkit/format/bp4/BP4Base.h
+++ b/source/adios2/toolkit/format/bp4/BP4Base.h
@@ -153,7 +153,7 @@ class BP4Base
         bool HasSubFiles = false;
     };
 
-    MPI_Comm m_MPIComm;  ///< MPI communicator from Engine
+    MPI_Comm m_Comm;     ///< MPI communicator from Engine
     int m_RankMPI = 0;   ///< current MPI rank process
     int m_SizeMPI = 1;   ///< current MPI processes size
     int m_Processes = 1; ///< number of aggregated MPI processes
diff --git a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
index 35ec66cc1c..0284f7120c 100644
--- a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
+++ b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
@@ -2898,7 +2898,7 @@ BP4Serializer::SetCollectiveProfilingJSON(const std::string &rankLog) const
     // Gather sizes
     const size_t rankLogSize = rankLog.size();
     std::vector<size_t> rankLogsSizes =
-        helper::GatherValues(rankLogSize, m_MPIComm);
+        helper::GatherValues(rankLogSize, m_Comm);
 
     // Gatherv JSON per rank
     std::vector<char> profilingJSON(3);
@@ -2919,7 +2919,7 @@ BP4Serializer::SetCollectiveProfilingJSON(const std::string &rankLog) const
 
     helper::GathervArrays(rankLog.c_str(), rankLog.size(), rankLogsSizes.data(),
                           rankLogsSizes.size(), &profilingJSON[position],
-                          m_MPIComm);
+                          m_Comm);
 
     if (m_RankMPI == 0) // add footer to close JSON
     {

From ff37bac94d2363922f71870824b236cc37a09492 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 18 Jun 2019 12:59:43 -0400
Subject: [PATCH 13/42] bp4: Use Comm encapsulation

---
 source/adios2/toolkit/format/bp4/BP4Base.cpp         |  4 ++--
 source/adios2/toolkit/format/bp4/BP4Base.h           | 11 ++++++-----
 source/adios2/toolkit/format/bp4/BP4Deserializer.cpp |  4 ++--
 source/adios2/toolkit/format/bp4/BP4Deserializer.h   |  4 ++--
 source/adios2/toolkit/format/bp4/BP4Serializer.cpp   |  4 ++--
 source/adios2/toolkit/format/bp4/BP4Serializer.h     |  4 ++--
 6 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/source/adios2/toolkit/format/bp4/BP4Base.cpp b/source/adios2/toolkit/format/bp4/BP4Base.cpp
index 4ad648b740..ae319f520a 100644
--- a/source/adios2/toolkit/format/bp4/BP4Base.cpp
+++ b/source/adios2/toolkit/format/bp4/BP4Base.cpp
@@ -48,8 +48,8 @@ const std::map<int, std::string> BP4Base::m_TransformTypesToNames = {
     //    {transform_lz4, "lz4"},
 };
 
-BP4Base::BP4Base(MPI_Comm mpiComm, const bool debugMode)
-: m_Comm(mpiComm), m_DebugMode(debugMode)
+BP4Base::BP4Base(helper::Comm const &comm, const bool debugMode)
+: m_Comm(comm), m_DebugMode(debugMode)
 {
     SMPI_Comm_rank(m_Comm, &m_RankMPI);
     SMPI_Comm_size(m_Comm, &m_SizeMPI);
diff --git a/source/adios2/toolkit/format/bp4/BP4Base.h b/source/adios2/toolkit/format/bp4/BP4Base.h
index 54a4f26b5c..ef4ad22c72 100644
--- a/source/adios2/toolkit/format/bp4/BP4Base.h
+++ b/source/adios2/toolkit/format/bp4/BP4Base.h
@@ -27,6 +27,7 @@
 #include "adios2/common/ADIOSTypes.h"
 #include "adios2/core/Engine.h"
 #include "adios2/core/VariableBase.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/aggregator/mpi/MPIChain.h"
 #include "adios2/toolkit/format/bpOperation/BPOperation.h"
 #include "adios2/toolkit/format/buffer/heap/BufferSTL.h"
@@ -153,10 +154,10 @@ class BP4Base
         bool HasSubFiles = false;
     };
 
-    MPI_Comm m_Comm;     ///< MPI communicator from Engine
-    int m_RankMPI = 0;   ///< current MPI rank process
-    int m_SizeMPI = 1;   ///< current MPI processes size
-    int m_Processes = 1; ///< number of aggregated MPI processes
+    helper::Comm const &m_Comm; ///< multi-process communicator from Engine
+    int m_RankMPI = 0;          ///< current MPI rank process
+    int m_SizeMPI = 1;          ///< current MPI processes size
+    int m_Processes = 1;        ///< number of aggregated MPI processes
 
     /** statistics verbosity, only 0/1 is supported now */
     unsigned int m_StatsLevel = 1;
@@ -262,7 +263,7 @@ class BP4Base
      * @param mpiComm for m_BP1Aggregator
      * @param debugMode true: exceptions checks
      */
-    BP4Base(MPI_Comm mpiComm, const bool debugMode);
+    BP4Base(helper::Comm const &comm, const bool debugMode);
 
     virtual ~BP4Base();
 
diff --git a/source/adios2/toolkit/format/bp4/BP4Deserializer.cpp b/source/adios2/toolkit/format/bp4/BP4Deserializer.cpp
index ecb63e586c..bb1fbbec26 100644
--- a/source/adios2/toolkit/format/bp4/BP4Deserializer.cpp
+++ b/source/adios2/toolkit/format/bp4/BP4Deserializer.cpp
@@ -30,8 +30,8 @@ namespace format
 
 std::mutex BP4Deserializer::m_Mutex;
 
-BP4Deserializer::BP4Deserializer(MPI_Comm mpiComm, const bool debugMode)
-: BP4Base(mpiComm, debugMode)
+BP4Deserializer::BP4Deserializer(helper::Comm const &comm, const bool debugMode)
+: BP4Base(comm, debugMode)
 {
 }
 
diff --git a/source/adios2/toolkit/format/bp4/BP4Deserializer.h b/source/adios2/toolkit/format/bp4/BP4Deserializer.h
index dc87af93a3..9621558fdc 100644
--- a/source/adios2/toolkit/format/bp4/BP4Deserializer.h
+++ b/source/adios2/toolkit/format/bp4/BP4Deserializer.h
@@ -44,10 +44,10 @@ class BP4Deserializer : public BP4Base
 
     /**
      * Unique constructor
-     * @param mpiComm
+     * @param comm multi-process communicator
      * @param debug true: extra checks
      */
-    BP4Deserializer(MPI_Comm mpiComm, const bool debugMode);
+    BP4Deserializer(helper::Comm const &comm, const bool debugMode);
 
     ~BP4Deserializer() = default;
 
diff --git a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
index 0284f7120c..4b3af3ef04 100644
--- a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
+++ b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
@@ -31,8 +31,8 @@ namespace format
 
 std::mutex BP4Serializer::m_Mutex;
 
-BP4Serializer::BP4Serializer(MPI_Comm mpiComm, const bool debugMode)
-: BP4Base(mpiComm, debugMode)
+BP4Serializer::BP4Serializer(helper::Comm const &comm, const bool debugMode)
+: BP4Base(comm, debugMode)
 {
 }
 
diff --git a/source/adios2/toolkit/format/bp4/BP4Serializer.h b/source/adios2/toolkit/format/bp4/BP4Serializer.h
index 2e0e23ef58..8f8b96cb48 100644
--- a/source/adios2/toolkit/format/bp4/BP4Serializer.h
+++ b/source/adios2/toolkit/format/bp4/BP4Serializer.h
@@ -28,10 +28,10 @@ class BP4Serializer : public BP4Base
 public:
     /**
      * Unique constructor
-     * @param mpiComm MPI communicator for BP1 Aggregator
+     * @param comm multi-process communicator for BP1 Aggregator
      * @param debug true: extra checks
      */
-    BP4Serializer(MPI_Comm mpiComm, const bool debugMode = false);
+    BP4Serializer(helper::Comm const &comm, const bool debugMode = false);
 
     ~BP4Serializer() = default;
 

From a4916755f692bf9661854b61b2a9853e36e43758 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 18 Jun 2019 13:08:08 -0400
Subject: [PATCH 14/42] bp3: Rename m_MPIComm to m_Comm

---
 source/adios2/toolkit/format/bp3/BP3Base.cpp       | 8 ++++----
 source/adios2/toolkit/format/bp3/BP3Base.h         | 2 +-
 source/adios2/toolkit/format/bp3/BP3Serializer.cpp | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/source/adios2/toolkit/format/bp3/BP3Base.cpp b/source/adios2/toolkit/format/bp3/BP3Base.cpp
index bd486d415c..e061a23e11 100644
--- a/source/adios2/toolkit/format/bp3/BP3Base.cpp
+++ b/source/adios2/toolkit/format/bp3/BP3Base.cpp
@@ -49,10 +49,10 @@ const std::map<int, std::string> BP3Base::m_TransformTypesToNames = {
 };
 
 BP3Base::BP3Base(MPI_Comm mpiComm, const bool debugMode)
-: m_MPIComm(mpiComm), m_DebugMode(debugMode)
+: m_Comm(mpiComm), m_DebugMode(debugMode)
 {
-    SMPI_Comm_rank(m_MPIComm, &m_RankMPI);
-    SMPI_Comm_size(m_MPIComm, &m_SizeMPI);
+    SMPI_Comm_rank(m_Comm, &m_RankMPI);
+    SMPI_Comm_size(m_Comm, &m_SizeMPI);
     m_Profiler.IsActive = true; // default
 }
 
@@ -857,7 +857,7 @@ void BP3Base::InitParameterSubStreams(const std::string value)
 
     if (subStreams < m_SizeMPI)
     {
-        m_Aggregator.Init(subStreams, m_MPIComm);
+        m_Aggregator.Init(subStreams, m_Comm);
     }
 }
 
diff --git a/source/adios2/toolkit/format/bp3/BP3Base.h b/source/adios2/toolkit/format/bp3/BP3Base.h
index ee28052dd5..fbbeb77b6b 100644
--- a/source/adios2/toolkit/format/bp3/BP3Base.h
+++ b/source/adios2/toolkit/format/bp3/BP3Base.h
@@ -125,7 +125,7 @@ class BP3Base
         bool HasSubFiles = false;
     };
 
-    MPI_Comm m_MPIComm;  ///< MPI communicator from Engine
+    MPI_Comm m_Comm;     ///< MPI communicator from Engine
     int m_RankMPI = 0;   ///< current MPI rank process
     int m_SizeMPI = 1;   ///< current MPI processes size
     int m_Processes = 1; ///< number of aggregated MPI processes
diff --git a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
index 7e34015d77..8f48e33112 100644
--- a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
+++ b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
@@ -1417,7 +1417,7 @@ BP3Serializer::SetCollectiveProfilingJSON(const std::string &rankLog) const
     // Gather sizes
     const size_t rankLogSize = rankLog.size();
     std::vector<size_t> rankLogsSizes =
-        helper::GatherValues(rankLogSize, m_MPIComm);
+        helper::GatherValues(rankLogSize, m_Comm);
 
     // Gatherv JSON per rank
     std::vector<char> profilingJSON(3);
@@ -1438,7 +1438,7 @@ BP3Serializer::SetCollectiveProfilingJSON(const std::string &rankLog) const
 
     helper::GathervArrays(rankLog.c_str(), rankLog.size(), rankLogsSizes.data(),
                           rankLogsSizes.size(), &profilingJSON[position],
-                          m_MPIComm);
+                          m_Comm);
 
     if (m_RankMPI == 0) // add footer to close JSON
     {

From be80eb7f0520d720fa9a034c2ee4cfb25f8fbebf Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 18 Jun 2019 13:06:49 -0400
Subject: [PATCH 15/42] bp3: Use Comm encapsulation

---
 source/adios2/toolkit/format/bp3/BP3Base.cpp        |  4 ++--
 source/adios2/toolkit/format/bp3/BP3Base.h          | 13 +++++++------
 .../adios2/toolkit/format/bp3/BP3Deserializer.cpp   |  4 ++--
 source/adios2/toolkit/format/bp3/BP3Deserializer.h  |  2 +-
 source/adios2/toolkit/format/bp3/BP3Serializer.cpp  |  4 ++--
 source/adios2/toolkit/format/bp3/BP3Serializer.h    |  2 +-
 6 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/source/adios2/toolkit/format/bp3/BP3Base.cpp b/source/adios2/toolkit/format/bp3/BP3Base.cpp
index e061a23e11..624620eca1 100644
--- a/source/adios2/toolkit/format/bp3/BP3Base.cpp
+++ b/source/adios2/toolkit/format/bp3/BP3Base.cpp
@@ -48,8 +48,8 @@ const std::map<int, std::string> BP3Base::m_TransformTypesToNames = {
     //
 };
 
-BP3Base::BP3Base(MPI_Comm mpiComm, const bool debugMode)
-: m_Comm(mpiComm), m_DebugMode(debugMode)
+BP3Base::BP3Base(helper::Comm const &comm, const bool debugMode)
+: m_Comm(comm), m_DebugMode(debugMode)
 {
     SMPI_Comm_rank(m_Comm, &m_RankMPI);
     SMPI_Comm_size(m_Comm, &m_SizeMPI);
diff --git a/source/adios2/toolkit/format/bp3/BP3Base.h b/source/adios2/toolkit/format/bp3/BP3Base.h
index fbbeb77b6b..ab39c6a658 100644
--- a/source/adios2/toolkit/format/bp3/BP3Base.h
+++ b/source/adios2/toolkit/format/bp3/BP3Base.h
@@ -27,6 +27,7 @@
 #include "adios2/common/ADIOSTypes.h"
 #include "adios2/core/Engine.h"
 #include "adios2/core/VariableBase.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/aggregator/mpi/MPIChain.h"
 #include "adios2/toolkit/format/bpOperation/BPOperation.h"
 #include "adios2/toolkit/format/buffer/heap/BufferSTL.h"
@@ -125,10 +126,10 @@ class BP3Base
         bool HasSubFiles = false;
     };
 
-    MPI_Comm m_Comm;     ///< MPI communicator from Engine
-    int m_RankMPI = 0;   ///< current MPI rank process
-    int m_SizeMPI = 1;   ///< current MPI processes size
-    int m_Processes = 1; ///< number of aggregated MPI processes
+    helper::Comm const &m_Comm; ///< multi-process communicator from Engine
+    int m_RankMPI = 0;          ///< current MPI rank process
+    int m_SizeMPI = 1;          ///< current MPI processes size
+    int m_Processes = 1;        ///< number of aggregated MPI processes
 
     /** statistics verbosity, only 0 is supported */
     unsigned int m_StatsLevel = 0;
@@ -199,10 +200,10 @@ class BP3Base
 
     /**
      * Unique constructor
-     * @param mpiComm for m_BP1Aggregator
+     * @param comm for m_BP1Aggregator
      * @param debugMode true: exceptions checks
      */
-    BP3Base(MPI_Comm mpiComm, const bool debugMode);
+    BP3Base(helper::Comm const &comm, const bool debugMode);
 
     virtual ~BP3Base();
 
diff --git a/source/adios2/toolkit/format/bp3/BP3Deserializer.cpp b/source/adios2/toolkit/format/bp3/BP3Deserializer.cpp
index 922571a544..b0574bc9b0 100644
--- a/source/adios2/toolkit/format/bp3/BP3Deserializer.cpp
+++ b/source/adios2/toolkit/format/bp3/BP3Deserializer.cpp
@@ -28,8 +28,8 @@ namespace format
 
 std::mutex BP3Deserializer::m_Mutex;
 
-BP3Deserializer::BP3Deserializer(MPI_Comm mpiComm, const bool debugMode)
-: BP3Base(mpiComm, debugMode)
+BP3Deserializer::BP3Deserializer(helper::Comm const &comm, const bool debugMode)
+: BP3Base(comm, debugMode)
 {
 }
 
diff --git a/source/adios2/toolkit/format/bp3/BP3Deserializer.h b/source/adios2/toolkit/format/bp3/BP3Deserializer.h
index b9531f8c6a..adfa0c5034 100644
--- a/source/adios2/toolkit/format/bp3/BP3Deserializer.h
+++ b/source/adios2/toolkit/format/bp3/BP3Deserializer.h
@@ -37,7 +37,7 @@ class BP3Deserializer : public BP3Base
      * @param mpiComm
      * @param debug true: extra checks
      */
-    BP3Deserializer(MPI_Comm mpiComm, const bool debugMode);
+    BP3Deserializer(helper::Comm const &comm, const bool debugMode);
 
     ~BP3Deserializer() = default;
 
diff --git a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
index 8f48e33112..23686881f5 100644
--- a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
+++ b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
@@ -31,8 +31,8 @@ namespace format
 
 std::mutex BP3Serializer::m_Mutex;
 
-BP3Serializer::BP3Serializer(MPI_Comm mpiComm, const bool debugMode)
-: BP3Base(mpiComm, debugMode)
+BP3Serializer::BP3Serializer(helper::Comm const &comm, const bool debugMode)
+: BP3Base(comm, debugMode)
 {
 }
 
diff --git a/source/adios2/toolkit/format/bp3/BP3Serializer.h b/source/adios2/toolkit/format/bp3/BP3Serializer.h
index 3b720df033..3a7c6facfa 100644
--- a/source/adios2/toolkit/format/bp3/BP3Serializer.h
+++ b/source/adios2/toolkit/format/bp3/BP3Serializer.h
@@ -31,7 +31,7 @@ class BP3Serializer : public BP3Base
      * @param mpiComm MPI communicator for BP1 Aggregator
      * @param debug true: extra checks
      */
-    BP3Serializer(MPI_Comm mpiComm, const bool debugMode = false);
+    BP3Serializer(helper::Comm const &comm, const bool debugMode = false);
 
     ~BP3Serializer() = default;
 

From 19b0910d7e114f327bc31058239f28c7eff72a2a Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 18 Jun 2019 10:39:10 -0400
Subject: [PATCH 16/42] helper: Port network functions to Comm encapsulation

---
 source/adios2/helper/adiosNetwork.cpp | 31 ++++++++++++++-------------
 source/adios2/helper/adiosNetwork.h   |  8 +++----
 2 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp
index 7061f313cf..b3339e637d 100644
--- a/source/adios2/helper/adiosNetwork.cpp
+++ b/source/adios2/helper/adiosNetwork.cpp
@@ -9,6 +9,7 @@
  */
 
 #include "adiosNetwork.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/helper/adiosMPIFunctions.h"
 #include "adios2/toolkit/transport/file/FileFStream.h"
 
@@ -70,7 +71,7 @@ std::vector<std::string> AvailableIpAddresses() noexcept
     return ips;
 }
 
-void HandshakeWriter(MPI_Comm mpiComm, size_t &appID,
+void HandshakeWriter(Comm const &comm, size_t &appID,
                      std::vector<std::string> &fullAddresses,
                      const std::string &name, const std::string &engineName,
                      const int basePort, const int channelsPerRank,
@@ -79,8 +80,8 @@ void HandshakeWriter(MPI_Comm mpiComm, size_t &appID,
 
     int mpiRank;
     int mpiSize;
-    MPI_Comm_rank(mpiComm, &mpiRank);
-    MPI_Comm_size(mpiComm, &mpiSize);
+    MPI_Comm_rank(comm, &mpiRank);
+    MPI_Comm_size(comm, &mpiSize);
 
     const std::string globalFilename = ".socket";
     const std::string globalLockFilename = ".socket.lock";
@@ -99,7 +100,7 @@ void HandshakeWriter(MPI_Comm mpiComm, size_t &appID,
     // Check total number of writer apps
     if (mpiRank == 0)
     {
-        transport::FileFStream lockCheck(mpiComm, false);
+        transport::FileFStream lockCheck(comm, false);
         while (true)
         {
             try
@@ -112,10 +113,10 @@ void HandshakeWriter(MPI_Comm mpiComm, size_t &appID,
                 break;
             }
         }
-        transport::FileFStream lockWrite(mpiComm, false);
+        transport::FileFStream lockWrite(comm, false);
         lockWrite.Open(globalLockFilename, Mode::Write);
 
-        transport::FileFStream numRead(mpiComm, false);
+        transport::FileFStream numRead(comm, false);
         try
         {
             numRead.Open(globalFilename, Mode::Read);
@@ -129,7 +130,7 @@ void HandshakeWriter(MPI_Comm mpiComm, size_t &appID,
         catch (...)
         {
         }
-        transport::FileFStream numWrite(mpiComm, false);
+        transport::FileFStream numWrite(comm, false);
         numWrite.Open(globalFilename, Mode::Write);
         std::string numAppsString = std::to_string(appID);
         numWrite.Write(numAppsString.data(), numAppsString.size());
@@ -139,7 +140,7 @@ void HandshakeWriter(MPI_Comm mpiComm, size_t &appID,
         remove(globalLockFilename.c_str());
     }
 
-    appID = helper::BroadcastValue(appID, mpiComm);
+    appID = helper::BroadcastValue(appID, comm);
 
     // Make full addresses
     for (int i = 0; i < channelsPerRank; ++i)
@@ -158,7 +159,7 @@ void HandshakeWriter(MPI_Comm mpiComm, size_t &appID,
                 localAddressesStr.size());
     std::vector<char> globalAddressesChar(64 * channelsPerRank * mpiSize, '\0');
     helper::GatherArrays(localAddressesChar.data(), 64 * channelsPerRank,
-                         globalAddressesChar.data(), mpiComm);
+                         globalAddressesChar.data(), comm);
 
     // Writing handshake file
     if (mpiRank == 0)
@@ -174,9 +175,9 @@ void HandshakeWriter(MPI_Comm mpiComm, size_t &appID,
             }
         }
         std::string globalAddressesStr = globalAddressesJson.dump();
-        transport::FileFStream lockstream(mpiComm, false);
+        transport::FileFStream lockstream(comm, false);
         lockstream.Open(engineLockFilename, Mode::Write);
-        transport::FileFStream ipstream(mpiComm, false);
+        transport::FileFStream ipstream(comm, false);
         ipstream.Open(engineFilename, Mode::Write);
         ipstream.Write(globalAddressesStr.data(), globalAddressesStr.size());
         ipstream.Close();
@@ -185,7 +186,7 @@ void HandshakeWriter(MPI_Comm mpiComm, size_t &appID,
     }
 }
 
-void HandshakeReader(MPI_Comm mpiComm, size_t &appID,
+void HandshakeReader(Comm const &comm, size_t &appID,
                      std::vector<std::string> &fullAddresses,
                      const std::string &name, const std::string &engineName)
 {
@@ -202,9 +203,9 @@ void HandshakeReader(MPI_Comm mpiComm, size_t &appID,
         std::hash<std::string> hash_fn;
         appID = hash_fn(ips[0]);
     }
-    helper::BroadcastValue(appID, mpiComm);
+    helper::BroadcastValue(appID, comm);
 
-    transport::FileFStream ipstream(mpiComm, false);
+    transport::FileFStream ipstream(comm, false);
     while (true)
     {
         try
@@ -218,7 +219,7 @@ void HandshakeReader(MPI_Comm mpiComm, size_t &appID,
         }
     }
 
-    transport::FileFStream lockstream(mpiComm, false);
+    transport::FileFStream lockstream(comm, false);
     while (true)
     {
         try
diff --git a/source/adios2/helper/adiosNetwork.h b/source/adios2/helper/adiosNetwork.h
index b3fef464fb..35088dbd57 100644
--- a/source/adios2/helper/adiosNetwork.h
+++ b/source/adios2/helper/adiosNetwork.h
@@ -20,27 +20,27 @@
 #include <vector>
 /// \endcond
 
-#include "adios2/common/ADIOSMPI.h"
-
 namespace adios2
 {
 namespace helper
 {
 
+class Comm;
+
 /**
  * returns a vector of strings with all available IP addresses on the node
  * @return vector of strings
  */
 std::vector<std::string> AvailableIpAddresses() noexcept;
 
-void HandshakeWriter(MPI_Comm mpiComm, size_t &appID,
+void HandshakeWriter(Comm const &comm, size_t &appID,
                      std::vector<std::string> &fullAddresses,
                      const std::string &name, const std::string &engineName,
                      const int basePort, const int channelsPerRank,
                      const int maxRanksPerNode = 100,
                      const int maxAppsPerNode = 10);
 
-void HandshakeReader(MPI_Comm mpiComm, size_t &appID,
+void HandshakeReader(Comm const &comm, size_t &appID,
                      std::vector<std::string> &fullAddresses,
                      const std::string &name, const std::string &engineName);
 

From 6bd190d2bf24d9fff6bf5287d04f2cefa9d48095 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Fri, 21 Jun 2019 13:10:50 -0400
Subject: [PATCH 17/42] helper: Encapsulate MPI_Comm_split as Comm::Split

---
 source/adios2/helper/adiosComm.cpp |  7 +++++++
 source/adios2/helper/adiosComm.h   | 12 ++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index e36629a790..eddc4137e2 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -69,5 +69,12 @@ void Comm::Free(const std::string &hint)
     }
 }
 
+Comm Comm::Split(int color, int key, const std::string &hint) const
+{
+    MPI_Comm newComm;
+    CheckMPIReturn(MPI_Comm_split(m_MPIComm, color, key, &newComm), hint);
+    return Comm(newComm);
+}
+
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 4a434c87b5..09ed83b182 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -78,6 +78,18 @@ class Comm
      */
     void Free(const std::string &hint = std::string());
 
+    /**
+     * @brief Split the communicator.
+     * @param color Control of subset assignment (nonnegative integer).
+     * @param key Control of rank assignment (integer).
+     * @param hint Description of std::runtime_error exception on error.
+     *
+     * Creates a new communicator covering the subset of the original
+     * processes that pass the same 'color'.  Ranks assigned by 'key' order.
+     */
+    Comm Split(int color, int key,
+               const std::string &hint = std::string()) const;
+
 private:
     /**
      * @brief Construct by taking ownership of a MPI communicator.

From 3d0481e0efa7b4644f6be1c9c59a2e8dfb814249 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Wed, 19 Jun 2019 14:23:10 -0400
Subject: [PATCH 18/42] aggregator: Simplify substream assignment logic

---
 .../toolkit/aggregator/mpi/MPIAggregator.cpp  | 42 +++++++++----------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
index 38e45effb4..2b24b1e8be 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
+++ b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
@@ -58,33 +58,33 @@ void MPIAggregator::InitComm(const size_t subStreams, MPI_Comm parentComm)
     MPI_Comm_rank(parentComm, &parentRank);
     MPI_Comm_size(parentComm, &parentSize);
 
+    const size_t process = static_cast<size_t>(parentRank);
     const size_t processes = static_cast<size_t>(parentSize);
-    size_t stride = processes / subStreams + 1;
-    const size_t remainder = processes % subStreams;
 
-    size_t consumer = 0;
+    // Divide the processes into S=subStreams groups.
+    const size_t q = processes / subStreams;
+    const size_t r = processes % subStreams;
 
-    for (auto s = 0; s < subStreams; ++s)
+    // Groups [0,r) have size q+1.  Groups [r,S) have size q.
+    const size_t first_in_small_groups = r * (q + 1);
+
+    // Within each group the first process becomes its consumer.
+    if (process >= first_in_small_groups)
+    {
+        m_SubStreamIndex = r + (process - first_in_small_groups) / q;
+        m_ConsumerRank = static_cast<int>(first_in_small_groups +
+                                          (m_SubStreamIndex - r) * q);
+    }
+    else
     {
-        if (s >= remainder)
-        {
-            stride = processes / subStreams;
-        }
-
-        if (static_cast<size_t>(parentRank) >= consumer &&
-            static_cast<size_t>(parentRank) < consumer + stride)
-        {
-            helper::CheckMPIReturn(
-                MPI_Comm_split(parentComm, static_cast<int>(consumer),
-                               parentRank, &m_Comm),
-                "creating aggregators comm with split at Open");
-            m_ConsumerRank = static_cast<int>(consumer);
-            m_SubStreamIndex = static_cast<size_t>(s);
-        }
-
-        consumer += stride;
+        m_SubStreamIndex = process / (q + 1);
+        m_ConsumerRank = static_cast<int>(m_SubStreamIndex * (q + 1));
     }
 
+    helper::CheckMPIReturn(
+        MPI_Comm_split(parentComm, m_ConsumerRank, parentRank, &m_Comm),
+        "creating aggregators comm with split at Open");
+
     MPI_Comm_rank(m_Comm, &m_Rank);
     MPI_Comm_size(m_Comm, &m_Size);
 

From d121f2b3c994f445185c6f7ea7fe29a1ad4d21fe Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 18 Jun 2019 13:48:40 -0400
Subject: [PATCH 19/42] aggregator: Refactor to use Comm encapsulation

---
 .../toolkit/aggregator/mpi/MPIAggregator.cpp  | 23 ++++++++++---------
 .../toolkit/aggregator/mpi/MPIAggregator.h    |  7 +++---
 .../toolkit/aggregator/mpi/MPIChain.cpp       |  2 +-
 .../adios2/toolkit/aggregator/mpi/MPIChain.h  |  2 +-
 4 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
index 2b24b1e8be..1c38a6e739 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
+++ b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
@@ -17,19 +17,21 @@ namespace adios2
 namespace aggregator
 {
 
-MPIAggregator::MPIAggregator() : m_Comm(MPI_COMM_NULL) {}
+MPIAggregator::MPIAggregator() {}
 
 MPIAggregator::~MPIAggregator()
 {
     if (m_IsActive)
     {
-        helper::CheckMPIReturn(MPI_Comm_free(&m_Comm),
-                               "freeing aggregators comm in MPIAggregator "
-                               "destructor, not recommended");
+        m_Comm.Free("freeing aggregators comm in MPIAggregator "
+                    "destructor, not recommended");
     }
 }
 
-void MPIAggregator::Init(const size_t subStreams, MPI_Comm parentComm) {}
+void MPIAggregator::Init(const size_t subStreams,
+                         helper::Comm const &parentComm)
+{
+}
 
 void MPIAggregator::SwapBuffers(const int step) noexcept {}
 
@@ -44,14 +46,14 @@ void MPIAggregator::Close()
 {
     if (m_IsActive)
     {
-        helper::CheckMPIReturn(MPI_Comm_free(&m_Comm),
-                               "freeing aggregators comm at Close\n");
+        m_Comm.Free("freeing aggregators comm at Close\n");
         m_IsActive = false;
     }
 }
 
 // PROTECTED
-void MPIAggregator::InitComm(const size_t subStreams, MPI_Comm parentComm)
+void MPIAggregator::InitComm(const size_t subStreams,
+                             helper::Comm const &parentComm)
 {
     int parentRank;
     int parentSize;
@@ -81,9 +83,8 @@ void MPIAggregator::InitComm(const size_t subStreams, MPI_Comm parentComm)
         m_ConsumerRank = static_cast<int>(m_SubStreamIndex * (q + 1));
     }
 
-    helper::CheckMPIReturn(
-        MPI_Comm_split(parentComm, m_ConsumerRank, parentRank, &m_Comm),
-        "creating aggregators comm with split at Open");
+    m_Comm = parentComm.Split(m_ConsumerRank, parentRank,
+                              "creating aggregators comm with split at Open");
 
     MPI_Comm_rank(m_Comm, &m_Rank);
     MPI_Comm_size(m_Comm, &m_Size);
diff --git a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h
index 63b261daf4..98f39cf6e6 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h
+++ b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h
@@ -15,6 +15,7 @@
 
 #include "adios2/common/ADIOSMPI.h"
 #include "adios2/common/ADIOSTypes.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/format/buffer/Buffer.h"
 
 namespace adios2
@@ -32,7 +33,7 @@ class MPIAggregator
     size_t m_SubStreamIndex = 0;
 
     /** split Communicator for a substream: producers and consumer (rank=0) */
-    MPI_Comm m_Comm;
+    helper::Comm m_Comm;
 
     /** rank from m_Comm */
     int m_Rank = 0;
@@ -57,7 +58,7 @@ class MPIAggregator
 
     virtual ~MPIAggregator();
 
-    virtual void Init(const size_t subStreams, MPI_Comm parentComm);
+    virtual void Init(const size_t subStreams, helper::Comm const &parentComm);
 
     virtual std::vector<std::vector<MPI_Request>>
     IExchange(format::Buffer &buffer, const int step) = 0;
@@ -84,7 +85,7 @@ class MPIAggregator
 protected:
     /** Init m_Comm splitting assigning ranks to subStreams (balanced except for
      * the last rank) */
-    void InitComm(const size_t subStreams, MPI_Comm parentComm);
+    void InitComm(const size_t subStreams, helper::Comm const &parentComm);
 
     /** handshakes a single rank with the rest of the m_Comm ranks */
     void HandshakeRank(const int rank = 0);
diff --git a/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp b/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp
index 91a2f4bb51..26bd176e95 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp
+++ b/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp
@@ -21,7 +21,7 @@ namespace aggregator
 
 MPIChain::MPIChain() : MPIAggregator() {}
 
-void MPIChain::Init(const size_t subStreams, MPI_Comm parentComm)
+void MPIChain::Init(const size_t subStreams, helper::Comm const &parentComm)
 {
     InitComm(subStreams, parentComm);
     HandshakeRank(0);
diff --git a/source/adios2/toolkit/aggregator/mpi/MPIChain.h b/source/adios2/toolkit/aggregator/mpi/MPIChain.h
index 83a1dd93c5..d6939ca56d 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIChain.h
+++ b/source/adios2/toolkit/aggregator/mpi/MPIChain.h
@@ -26,7 +26,7 @@ class MPIChain : public MPIAggregator
 
     ~MPIChain() = default;
 
-    void Init(const size_t subStreams, MPI_Comm parentComm) final;
+    void Init(const size_t subStreams, helper::Comm const &parentComm) final;
 
     std::vector<std::vector<MPI_Request>> IExchange(format::Buffer &buffer,
                                                     const int step) final;

From 22a44878033a85ea97f8f3d8e9381713759546ae Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 16 Jul 2019 13:21:12 -0400
Subject: [PATCH 20/42] bp3: Use Comm encapsulation in Aggregate methods

---
 source/adios2/toolkit/format/bp3/BP3Serializer.cpp | 9 +++++----
 source/adios2/toolkit/format/bp3/BP3Serializer.h   | 8 +++++---
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
index 23686881f5..c56e08b2ed 100644
--- a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
+++ b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
@@ -271,7 +271,7 @@ BP3Serializer::AggregateProfilingJSON(const std::string &rankProfilingLog)
     return SetCollectiveProfilingJSON(rankProfilingLog);
 }
 
-void BP3Serializer::AggregateCollectiveMetadata(MPI_Comm comm,
+void BP3Serializer::AggregateCollectiveMetadata(helper::Comm const &comm,
                                                 BufferSTL &bufferSTL,
                                                 const bool inMetadataBuffer)
 {
@@ -766,7 +766,7 @@ void BP3Serializer::PutMinifooter(const uint64_t pgIndexStart,
 }
 
 std::vector<size_t>
-BP3Serializer::AggregateCollectiveMetadataIndices(MPI_Comm comm,
+BP3Serializer::AggregateCollectiveMetadataIndices(helper::Comm const &comm,
                                                   BufferSTL &bufferSTL)
 {
     TAU_SCOPED_TIMER_FUNC();
@@ -819,7 +819,8 @@ BP3Serializer::AggregateCollectiveMetadataIndices(MPI_Comm comm,
         }
     };
 
-    auto lf_SerializeAllIndices = [&](MPI_Comm comm, const int rank) {
+    auto lf_SerializeAllIndices = [&](helper::Comm const &comm,
+                                      const int rank) {
         TAU_SCOPED_TIMER_FUNC();
         const size_t pgIndicesSize = m_MetadataSet.PGIndex.Buffer.size();
         const size_t variablesIndicesSize =
@@ -1051,7 +1052,7 @@ BP3Serializer::AggregateCollectiveMetadataIndices(MPI_Comm comm,
 void BP3Serializer::MergeSerializeIndices(
     const std::unordered_map<std::string, std::vector<SerialElementIndex>>
         &nameRankIndices,
-    MPI_Comm comm, BufferSTL &bufferSTL)
+    helper::Comm const &comm, BufferSTL &bufferSTL)
 {
     auto lf_GetCharacteristics = [&](const std::vector<char> &buffer,
                                      size_t &position, const uint8_t dataType,
diff --git a/source/adios2/toolkit/format/bp3/BP3Serializer.h b/source/adios2/toolkit/format/bp3/BP3Serializer.h
index 3a7c6facfa..eeeb4646e1 100644
--- a/source/adios2/toolkit/format/bp3/BP3Serializer.h
+++ b/source/adios2/toolkit/format/bp3/BP3Serializer.h
@@ -134,7 +134,8 @@ class BP3Serializer : public BP3Base
      * @param inMetadataBuffer collective metadata from absolute rank = 0, else
      *                         from aggregators
      */
-    void AggregateCollectiveMetadata(MPI_Comm comm, BufferSTL &bufferSTL,
+    void AggregateCollectiveMetadata(helper::Comm const &comm,
+                                     BufferSTL &bufferSTL,
                                      const bool inMetadataBuffer);
 
     /**
@@ -378,7 +379,8 @@ class BP3Serializer : public BP3Base
      * @return contains indices positions in buffer
      */
     std::vector<size_t>
-    AggregateCollectiveMetadataIndices(MPI_Comm comm, BufferSTL &bufferSTL);
+    AggregateCollectiveMetadataIndices(helper::Comm const &comm,
+                                       BufferSTL &bufferSTL);
 
     /**
      * Merge indices by time step (default) and write to m_HeapBuffer.m_Metadata
@@ -387,7 +389,7 @@ class BP3Serializer : public BP3Base
     void MergeSerializeIndices(
         const std::unordered_map<std::string, std::vector<SerialElementIndex>>
             &nameRankIndices,
-        MPI_Comm comm, BufferSTL &bufferSTL);
+        helper::Comm const &comm, BufferSTL &bufferSTL);
 
     std::vector<char>
     SetCollectiveProfilingJSON(const std::string &rankLog) const;

From 420c06128b9523f727f51d350fc4a13237abb85c Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 16 Jul 2019 13:21:14 -0400
Subject: [PATCH 21/42] bp4: Use Comm encapsulation in Aggregate methods

---
 .../toolkit/format/bp4/BP4Serializer.cpp      | 21 ++++++++++---------
 .../adios2/toolkit/format/bp4/BP4Serializer.h | 20 ++++++++++--------
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
index 4b3af3ef04..c2222b4c18 100644
--- a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
+++ b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
@@ -429,7 +429,7 @@ BP4Serializer::AggregateProfilingJSON(const std::string &rankProfilingLog)
     return SetCollectiveProfilingJSON(rankProfilingLog);
 }
 
-void BP4Serializer::AggregateCollectiveMetadata(MPI_Comm comm,
+void BP4Serializer::AggregateCollectiveMetadata(helper::Comm const &comm,
                                                 BufferSTL &bufferSTL,
                                                 const bool inMetadataBuffer)
 {
@@ -966,7 +966,7 @@ void BP4Serializer::PutMinifooter(const uint64_t pgIndexStart,
 }
 
 void BP4Serializer::AggregateIndex(const SerialElementIndex &index,
-                                   const size_t count, MPI_Comm comm,
+                                   const size_t count, helper::Comm const &comm,
                                    BufferSTL &bufferSTL)
 {
     auto &buffer = bufferSTL.m_Buffer;
@@ -1000,7 +1000,7 @@ void BP4Serializer::AggregateIndex(const SerialElementIndex &index,
 
 void BP4Serializer::AggregateMergeIndex(
     const std::unordered_map<std::string, SerialElementIndex> &indices,
-    MPI_Comm comm, BufferSTL &bufferSTL, const bool isRankConstant)
+    helper::Comm const &comm, BufferSTL &bufferSTL, const bool isRankConstant)
 {
     // first serialize index
     std::vector<char> serializedIndices = SerializeIndices(indices, comm);
@@ -1054,7 +1054,7 @@ void BP4Serializer::AggregateMergeIndex(
 
 std::vector<char> BP4Serializer::SerializeIndices(
     const std::unordered_map<std::string, SerialElementIndex> &indices,
-    MPI_Comm comm) const noexcept
+    helper::Comm const &comm) const noexcept
 {
     // pre-allocate
     size_t serializedIndicesSize = 0;
@@ -1099,7 +1099,7 @@ std::vector<char> BP4Serializer::SerializeIndices(
 
 std::unordered_map<std::string, std::vector<BP4Base::SerialElementIndex>>
 BP4Serializer::DeserializeIndicesPerRankSingleThread(
-    const std::vector<char> &serialized, MPI_Comm comm,
+    const std::vector<char> &serialized, helper::Comm const &comm,
     const bool isRankConstant) const noexcept
 {
     std::unordered_map<std::string, std::vector<SerialElementIndex>>
@@ -1239,7 +1239,7 @@ BP4Serializer::DeserializeIndicesPerRankSingleThread(
 
 std::unordered_map<std::string, std::vector<BP4Base::SerialElementIndex>>
 BP4Serializer::DeserializeIndicesPerRankThreads(
-    const std::vector<char> &serialized, MPI_Comm comm,
+    const std::vector<char> &serialized, helper::Comm const &comm,
     const bool isRankConstant) const noexcept
 {
     if (m_Threads == 1)
@@ -1403,7 +1403,7 @@ BP4Serializer::DeserializeIndicesPerRankThreads(
     return deserialized;
 }
 
-void BP4Serializer::AggregateCollectiveMetadataIndices(MPI_Comm comm,
+void BP4Serializer::AggregateCollectiveMetadataIndices(helper::Comm const &comm,
                                                        BufferSTL &outBufferSTL)
 {
     int rank, size;
@@ -1453,7 +1453,8 @@ void BP4Serializer::AggregateCollectiveMetadataIndices(MPI_Comm comm,
         }
     };
 
-    auto lf_SerializeAllIndices = [&](MPI_Comm comm, const int rank) {
+    auto lf_SerializeAllIndices = [&](helper::Comm const &comm,
+                                      const int rank) {
         const size_t pgIndicesSize = m_MetadataSet.PGIndex.Buffer.size();
         const size_t variablesIndicesSize =
             lf_IndicesSize(m_MetadataSet.VarsIndices);
@@ -2211,7 +2212,7 @@ void BP4Serializer::AggregateCollectiveMetadataIndices(MPI_Comm comm,
 void BP4Serializer::MergeSerializeIndicesPerStep(
     const std::unordered_map<std::string, std::vector<SerialElementIndex>>
         &nameRankIndices,
-    MPI_Comm comm, BufferSTL &bufferSTL)
+    helper::Comm const &comm, BufferSTL &bufferSTL)
 {
     auto lf_GetCharacteristics = [&](const std::vector<char> &buffer,
                                      size_t &position, const uint8_t dataType,
@@ -2536,7 +2537,7 @@ void BP4Serializer::MergeSerializeIndicesPerStep(
 void BP4Serializer::MergeSerializeIndices(
     const std::unordered_map<std::string, std::vector<SerialElementIndex>>
         &nameRankIndices,
-    MPI_Comm comm, BufferSTL &bufferSTL)
+    helper::Comm const &comm, BufferSTL &bufferSTL)
 {
     auto lf_GetCharacteristics = [&](const std::vector<char> &buffer,
                                      size_t &position, const uint8_t dataType,
diff --git a/source/adios2/toolkit/format/bp4/BP4Serializer.h b/source/adios2/toolkit/format/bp4/BP4Serializer.h
index 8f8b96cb48..f1376421c4 100644
--- a/source/adios2/toolkit/format/bp4/BP4Serializer.h
+++ b/source/adios2/toolkit/format/bp4/BP4Serializer.h
@@ -146,7 +146,8 @@ class BP4Serializer : public BP4Base
      * @param inMetadataBuffer collective metadata from absolute rank = 0, else
      *                         from aggregators
      */
-    void AggregateCollectiveMetadata(MPI_Comm comm, BufferSTL &bufferSTL,
+    void AggregateCollectiveMetadata(helper::Comm const &comm,
+                                     BufferSTL &bufferSTL,
                                      const bool inMetadataBuffer);
 
     /**
@@ -399,7 +400,7 @@ class BP4Serializer : public BP4Base
      * @param bufferSTL
      */
     void AggregateIndex(const SerialElementIndex &index, const size_t count,
-                        MPI_Comm comm, BufferSTL &bufferSTL);
+                        helper::Comm const &comm, BufferSTL &bufferSTL);
 
     /**
      * Collective operation to aggregate and merge (sort) indices (variables and
@@ -415,9 +416,10 @@ class BP4Serializer : public BP4Base
      */
     void AggregateMergeIndex(
         const std::unordered_map<std::string, SerialElementIndex> &indices,
-        MPI_Comm comm, BufferSTL &bufferSTL, const bool isRankConstant = false);
+        helper::Comm const &comm, BufferSTL &bufferSTL,
+        const bool isRankConstant = false);
 
-    void AggregateCollectiveMetadataIndices(MPI_Comm comm,
+    void AggregateCollectiveMetadataIndices(helper::Comm const &comm,
                                             BufferSTL &bufferSTL);
 
     /**
@@ -428,7 +430,7 @@ class BP4Serializer : public BP4Base
      */
     std::vector<char> SerializeIndices(
         const std::unordered_map<std::string, SerialElementIndex> &indices,
-        MPI_Comm comm) const noexcept;
+        helper::Comm const &comm) const noexcept;
 
     /**
      * In rank=0, deserialize gathered indices
@@ -440,7 +442,7 @@ class BP4Serializer : public BP4Base
      */
     std::unordered_map<std::string, std::vector<SerialElementIndex>>
     DeserializeIndicesPerRankThreads(const std::vector<char> &serializedIndices,
-                                     MPI_Comm comm,
+                                     helper::Comm const &comm,
                                      const bool isRankConstant) const noexcept;
 
     /** private function called by DeserializeIndicesPerRankThreads
@@ -448,7 +450,7 @@ class BP4Serializer : public BP4Base
      */
     std::unordered_map<std::string, std::vector<SerialElementIndex>>
     DeserializeIndicesPerRankSingleThread(const std::vector<char> &serialized,
-                                          MPI_Comm comm,
+                                          helper::Comm const &comm,
                                           const bool isRankConstant) const
         noexcept;
     /**
@@ -458,7 +460,7 @@ class BP4Serializer : public BP4Base
     void MergeSerializeIndices(
         const std::unordered_map<std::string, std::vector<SerialElementIndex>>
             &nameRankIndices,
-        MPI_Comm comm, BufferSTL &bufferSTL);
+        helper::Comm const &comm, BufferSTL &bufferSTL);
 
     /**
      * Only merge indices of each time step and write to
@@ -469,7 +471,7 @@ class BP4Serializer : public BP4Base
     void MergeSerializeIndicesPerStep(
         const std::unordered_map<std::string, std::vector<SerialElementIndex>>
             &nameRankIndices,
-        MPI_Comm comm, BufferSTL &bufferSTL);
+        helper::Comm const &comm, BufferSTL &bufferSTL);
 
     std::vector<char>
     SetCollectiveProfilingJSON(const std::string &rankLog) const;

From abd26d8848262a3e682e663855561327a791384d Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 18 Jun 2019 13:43:22 -0400
Subject: [PATCH 22/42] helper: Move Gather functions to Comm encapsulation

---
 source/adios2/helper/adiosComm.h              |  43 +++++++
 source/adios2/helper/adiosComm.inl            |  79 +++++++++++-
 source/adios2/helper/adiosComm.tcc            | 118 +++++++++++++++++-
 source/adios2/helper/adiosMPIFunctions.cpp    |  14 ---
 source/adios2/helper/adiosMPIFunctions.h      |  57 ---------
 source/adios2/helper/adiosMPIFunctions.inl    |  62 ---------
 source/adios2/helper/adiosMPIFunctions.tcc    |  99 ---------------
 source/adios2/helper/adiosNetwork.cpp         |   4 +-
 .../toolkit/format/bp3/BP3Serializer.cpp      |  12 +-
 .../toolkit/format/bp4/BP4Serializer.cpp      |  18 ++-
 10 files changed, 253 insertions(+), 253 deletions(-)

diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 09ed83b182..06c630f67c 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -11,6 +11,7 @@
 #include "adios2/common/ADIOSMPI.h"
 
 #include <string>
+#include <vector>
 
 namespace adios2
 {
@@ -90,6 +91,48 @@ class Comm
     Comm Split(int color, int key,
                const std::string &hint = std::string()) const;
 
+    /**
+     * Gather a single source value from each ranks and forms a vector in
+     * rankDestination.
+     * @param rankDestination root, where all sizes are gathered in the returned
+     * vector
+     * @return in rankDestination: aggregated vector<T>, others: empty
+     */
+    template <class T>
+    std::vector<T> GatherValues(T source, int rankDestination = 0) const;
+
+    /**
+     * Gather equal size arrays
+     * @param source
+     * @param sourceCount
+     * @param destination
+     * @param rankDestination
+     */
+    template <class T>
+    void GatherArrays(const T *source, size_t sourceCount, T *destination,
+                      int rankDestination = 0) const;
+
+    /**
+     * Gather arrays of the same type into a destination (must be pre-allocated)
+     * if countsSize == 1, calls MPI_Gather, otherwise calls MPI_Gatherv.
+     * This function must be specialized for each MPI_Type.
+     * @param source  input from each rank
+     * @param counts  counts for each source
+     * @param countsSize number of counts
+     * @param destination resulting gathered buffer in rankDestination, unchaged
+     * in others
+     * @param rankDestination rank in which arrays are gathered (root)
+     */
+    template <class T>
+    void GathervArrays(const T *source, size_t sourceCount,
+                       const size_t *counts, size_t countsSize, T *destination,
+                       int rankDestination = 0) const;
+
+    template <class T>
+    void GathervVectors(const std::vector<T> &in, std::vector<T> &out,
+                        size_t &position, int rankDestination = 0,
+                        size_t extraSize = 0) const;
+
 private:
     /**
      * @brief Construct by taking ownership of a MPI communicator.
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index b8ea0af00b..5ab0ccf850 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -11,12 +11,89 @@
 #error "Inline file should only be included from it's header, never on it's own"
 #endif
 
+#include <numeric>   //std::accumulate
+#include <stdexcept> //std::runtime_error
+
 namespace adios2
 {
 namespace helper
 {
 
-// Add template definitions here.
+// GatherArrays full specializations implemented in 'adiosComm.tcc'.
+template <>
+void Comm::GatherArrays(const char *source, size_t sourceCount,
+                        char *destination, int rankDestination) const;
+template <>
+void Comm::GatherArrays(const size_t *source, size_t sourceCount,
+                        size_t *destination, int rankDestination) const;
+
+template <class T>
+std::vector<T> Comm::GatherValues(T source, int rankDestination) const
+{
+    int rank, size;
+    SMPI_Comm_rank(m_MPIComm, &rank);
+    SMPI_Comm_size(m_MPIComm, &size);
+
+    std::vector<T> output;
+
+    if (rank == rankDestination) // pre-allocate in destination rank
+    {
+        output.resize(size);
+    }
+
+    T sourceCopy = source; // so we can have an address for rvalues
+    this->GatherArrays(&sourceCopy, 1, output.data(), rankDestination);
+
+    return output;
+}
+
+// GathervArrays full specializations implemented in 'adiosComm.tcc'.
+template <>
+void Comm::GathervArrays(const char *source, size_t sourceCount,
+                         const size_t *counts, size_t countsSize,
+                         char *destination, int rankDestination) const;
+template <>
+void Comm::GathervArrays(const size_t *source, size_t sourceCount,
+                         const size_t *counts, size_t countsSize,
+                         size_t *destination, int rankDestination) const;
+
+template <class T>
+void Comm::GathervVectors(const std::vector<T> &in, std::vector<T> &out,
+                          size_t &position, int rankDestination,
+                          size_t extraSize) const
+{
+    const size_t inSize = in.size();
+    const std::vector<size_t> counts =
+        this->GatherValues(inSize, rankDestination);
+
+    size_t gatheredSize = 0;
+
+    int rank;
+    SMPI_Comm_rank(m_MPIComm, &rank);
+
+    if (rank == rankDestination) // pre-allocate vector
+    {
+        gatheredSize = std::accumulate(counts.begin(), counts.end(), size_t(0));
+
+        const size_t newSize = position + gatheredSize;
+        try
+        {
+            out.reserve(newSize + extraSize); // to avoid power of 2 growth
+            out.resize(newSize + extraSize);
+        }
+        catch (...)
+        {
+            std::throw_with_nested(
+                std::runtime_error("ERROR: buffer overflow when resizing to " +
+                                   std::to_string(newSize) +
+                                   " bytes, in call to GathervVectors\n"));
+        }
+    }
+
+    this->GathervArrays(in.data(), in.size(), counts.data(), counts.size(),
+                        out.data() + position);
+    position += gatheredSize;
+}
 
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc
index 1b109bf4ca..2a64772f18 100644
--- a/source/adios2/helper/adiosComm.tcc
+++ b/source/adios2/helper/adiosComm.tcc
@@ -11,12 +11,128 @@
 
 #include "adiosComm.h"
 
+#include "adios2/common/ADIOSMPI.h"
+#include "adios2/common/ADIOSTypes.h"
+#include "adios2/helper/adiosType.h"
+
+#include <stdexcept> //std::runtime_error
+
 namespace adios2
 {
 namespace helper
 {
 
-// Add template specializations here.
+namespace
+{
+
+std::vector<int> GetGathervDisplacements(const size_t *counts,
+                                         const size_t countsSize)
+{
+    std::vector<int> displacements(countsSize);
+    displacements[0] = 0;
+
+    for (size_t i = 1; i < countsSize; ++i)
+    {
+        displacements[i] =
+            displacements[i - 1] + static_cast<int>(counts[i - 1]);
+    }
+    return displacements;
+}
+
+}
+
+// GatherArrays full specializations forward-declared in 'adiosComm.inl'.
+template <>
+void Comm::GatherArrays(const char *source, size_t sourceCount,
+                        char *destination, int rankDestination) const
+{
+    int countsInt = static_cast<int>(sourceCount);
+    int result = SMPI_Gather(const_cast<char *>(source), countsInt, MPI_CHAR,
+                             destination, countsInt, MPI_CHAR, rankDestination,
+                             m_MPIComm);
+
+    if (result != MPI_SUCCESS)
+    {
+        throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI "
+                                 "Gather type MPI_CHAR function\n");
+    }
+}
+
+template <>
+void Comm::GatherArrays(const size_t *source, size_t sourceCount,
+                        size_t *destination, int rankDestination) const
+{
+    int countsInt = static_cast<int>(sourceCount);
+    int result = SMPI_Gather(const_cast<size_t *>(source), countsInt,
+                             ADIOS2_MPI_SIZE_T, destination, countsInt,
+                             ADIOS2_MPI_SIZE_T, rankDestination, m_MPIComm);
+
+    if (result != MPI_SUCCESS)
+    {
+        throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI "
+                                 "Gather type size_t function\n");
+    }
+}
+
+// GathervArrays full specializations forward-declared in 'adiosComm.inl'.
+template <>
+void Comm::GathervArrays(const char *source, size_t sourceCount,
+                         const size_t *counts, size_t countsSize,
+                         char *destination, int rankDestination) const
+{
+    int result = 0;
+    int rank;
+    SMPI_Comm_rank(m_MPIComm, &rank);
+
+    std::vector<int> countsInt, displacementsInt;
+
+    if (rank == rankDestination)
+    {
+        countsInt = NewVectorTypeFromArray<size_t, int>(counts, countsSize);
+        displacementsInt = GetGathervDisplacements(counts, countsSize);
+    }
+
+    int sourceCountInt = static_cast<int>(sourceCount);
+    result =
+        SMPI_Gatherv(const_cast<char *>(source), sourceCountInt, MPI_CHAR,
+                     destination, countsInt.data(), displacementsInt.data(),
+                     MPI_CHAR, rankDestination, m_MPIComm);
+
+    if (result != MPI_SUCCESS)
+    {
+        throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI "
+                                 "Gatherv type MPI_CHAR function\n");
+    }
+}
+
+template <>
+void Comm::GathervArrays(const size_t *source, size_t sourceCount,
+                         const size_t *counts, size_t countsSize,
+                         size_t *destination, int rankDestination) const
+{
+    int result = 0;
+    int rank;
+    SMPI_Comm_rank(m_MPIComm, &rank);
+
+    std::vector<int> countsInt =
+        NewVectorTypeFromArray<size_t, int>(counts, countsSize);
+
+    std::vector<int> displacementsInt =
+        GetGathervDisplacements(counts, countsSize);
+
+    int sourceCountInt = static_cast<int>(sourceCount);
+
+    result = SMPI_Gatherv(const_cast<size_t *>(source), sourceCountInt,
+                          ADIOS2_MPI_SIZE_T, destination, countsInt.data(),
+                          displacementsInt.data(), ADIOS2_MPI_SIZE_T,
+                          rankDestination, m_MPIComm);
+
+    if (result != MPI_SUCCESS)
+    {
+        throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI "
+                                 "Gather type size_t function\n");
+    }
+}
 
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/helper/adiosMPIFunctions.cpp b/source/adios2/helper/adiosMPIFunctions.cpp
index 17f344c1a4..8d03b0aac0 100644
--- a/source/adios2/helper/adiosMPIFunctions.cpp
+++ b/source/adios2/helper/adiosMPIFunctions.cpp
@@ -20,20 +20,6 @@ namespace adios2
 namespace helper
 {
 
-std::vector<int> GetGathervDisplacements(const size_t *counts,
-                                         const size_t countsSize)
-{
-    std::vector<int> displacements(countsSize);
-    displacements[0] = 0;
-
-    for (size_t i = 1; i < countsSize; ++i)
-    {
-        displacements[i] =
-            displacements[i - 1] + static_cast<int>(counts[i - 1]);
-    }
-    return displacements;
-}
-
 void CheckMPIReturn(const int value, const std::string &hint)
 {
     if (value == MPI_SUCCESS)
diff --git a/source/adios2/helper/adiosMPIFunctions.h b/source/adios2/helper/adiosMPIFunctions.h
index e7da203067..70dcff7e6e 100644
--- a/source/adios2/helper/adiosMPIFunctions.h
+++ b/source/adios2/helper/adiosMPIFunctions.h
@@ -34,19 +34,6 @@ template <class T>
 T ReduceValues(const T source, MPI_Comm mpiComm, MPI_Op operation = MPI_SUM,
                const int rankDestination = 0);
 
-/**
- * Gather a single source value from each ranks and forms a vector in
- * rankDestination
- * @param sourceSize input from each rank
- * @param mpiComm MPI communicator defining all ranks and size domain
- * @param rankDestination root, where all sizes are gathered in the returned
- * vector
- * @return in rankDestination: aggregated vector<T>, others: empty
- */
-template <class T>
-std::vector<T> GatherValues(const T source, MPI_Comm mpiComm,
-                            const int rankDestination = 0);
-
 /**
  * Perform AllGather for source value
  * @param source input
@@ -56,18 +43,6 @@ std::vector<T> GatherValues(const T source, MPI_Comm mpiComm,
 template <class T>
 std::vector<T> AllGatherValues(const T source, MPI_Comm mpiComm);
 
-/**
- * Gather equal size arrays
- * @param source
- * @param sourceCount
- * @param destination
- * @param mpiComm
- * @param rankDestination
- */
-template <class T>
-void GatherArrays(const T *source, const size_t sourceCount, T *destination,
-                  MPI_Comm mpiComm, const int rankDestination = 0);
-
 /**
  * Perform AllGather for equal size arrays
  * @param source
@@ -79,38 +54,6 @@ template <class T>
 void AllGatherArrays(const T *source, const size_t sourceCount, T *destination,
                      MPI_Comm mpiComm);
 
-/**
- * Gather arrays of the same type into a destination (must be pre-allocated)
- * if countsSize == 1, calls MPI_Gather, otherwise calls MPI_Gatherv.
- * This function must be specialized for each MPI_Type.
- * @param source  input from each rank
- * @param counts  counts for each source
- * @param countsSize number of counts
- * @param destination resulting gathered buffer in rankDestination, unchaged in
- * others
- * @param mpiComm communicator establishing the domain
- * @param rankDestination rank in which arrays are gathered (root)
- */
-template <class T>
-void GathervArrays(const T *source, const size_t sourceCount,
-                   const size_t *counts, const size_t countsSize,
-                   T *destination, MPI_Comm mpiComm,
-                   const int rankDestination = 0);
-
-template <class T>
-void GathervVectors(const std::vector<T> &in, std::vector<T> &out,
-                    size_t &position, MPI_Comm mpiComm,
-                    const int rankDestination = 0, const size_t extraSize = 0);
-
-/**
- * Gets the displacements (offsets, start) for each
- * @param counts
- * @param countsSize
- * @return
- */
-std::vector<int> GetGathervDisplacements(const size_t *counts,
-                                         const size_t countsSize);
-
 void CheckMPIReturn(const int value, const std::string &hint);
 
 std::string BroadcastFile(const std::string &fileName, MPI_Comm mpiComm,
diff --git a/source/adios2/helper/adiosMPIFunctions.inl b/source/adios2/helper/adiosMPIFunctions.inl
index ed7322595f..04d47372cc 100644
--- a/source/adios2/helper/adiosMPIFunctions.inl
+++ b/source/adios2/helper/adiosMPIFunctions.inl
@@ -14,35 +14,11 @@
 #error "Inline file should only be included from it's header, never on it's own"
 #endif
 
-#include <numeric>   //std::accumulate
-#include <stdexcept> //std::runtime_error
-
 namespace adios2
 {
 namespace helper
 {
 
-template <class T>
-std::vector<T> GatherValues(const T source, MPI_Comm mpiComm,
-                            const int rankDestination)
-{
-    int rank, size;
-    SMPI_Comm_rank(mpiComm, &rank);
-    SMPI_Comm_size(mpiComm, &size);
-
-    std::vector<T> output;
-
-    if (rank == rankDestination) // pre-allocate in destination rank
-    {
-        output.resize(size);
-    }
-
-    T sourceCopy = source; // so we can have an address for rvalues
-    GatherArrays(&sourceCopy, 1, output.data(), mpiComm, rankDestination);
-
-    return output;
-}
-
 template <class T>
 std::vector<T> AllGatherValues(const T source, MPI_Comm mpiComm)
 {
@@ -55,44 +31,6 @@ std::vector<T> AllGatherValues(const T source, MPI_Comm mpiComm)
     return output;
 }
 
-template <class T>
-void GathervVectors(const std::vector<T> &in, std::vector<T> &out,
-                    size_t &position, MPI_Comm mpiComm,
-                    const int rankDestination, const size_t extraSize)
-{
-    const size_t inSize = in.size();
-    const std::vector<size_t> counts =
-        GatherValues(inSize, mpiComm, rankDestination);
-
-    size_t gatheredSize = 0;
-
-    int rank;
-    SMPI_Comm_rank(mpiComm, &rank);
-
-    if (rank == rankDestination) // pre-allocate vector
-    {
-        gatheredSize = std::accumulate(counts.begin(), counts.end(), size_t(0));
-
-        const size_t newSize = position + gatheredSize;
-        try
-        {
-            out.reserve(newSize + extraSize); // to avoid power of 2 growth
-            out.resize(newSize + extraSize);
-        }
-        catch (...)
-        {
-            std::throw_with_nested(
-                std::runtime_error("ERROR: buffer overflow when resizing to " +
-                                   std::to_string(newSize) +
-                                   " bytes, in call to GathervVectors\n"));
-        }
-    }
-
-    GathervArrays(in.data(), in.size(), counts.data(), counts.size(),
-                  out.data() + position, mpiComm);
-    position += gatheredSize;
-}
-
 } // end namespace helper
 } // end namespace adios2
 
diff --git a/source/adios2/helper/adiosMPIFunctions.tcc b/source/adios2/helper/adiosMPIFunctions.tcc
index df8236fcb3..4cc52a1876 100644
--- a/source/adios2/helper/adiosMPIFunctions.tcc
+++ b/source/adios2/helper/adiosMPIFunctions.tcc
@@ -15,11 +15,9 @@
 #include "adiosMPIFunctions.h"
 
 #include <algorithm> //std::foreach
-#include <numeric>   //std::accumulate
 
 #include "adios2/common/ADIOSMPI.h"
 #include "adios2/common/ADIOSTypes.h"
-#include "adios2/helper/adiosType.h"
 
 namespace adios2
 {
@@ -177,41 +175,6 @@ void BroadcastVector(std::vector<size_t> &vector, MPI_Comm mpiComm,
     }
 }
 
-// GatherArrays specializations
-template <>
-void GatherArrays(const char *source, const size_t sourceCount,
-                  char *destination, MPI_Comm mpiComm,
-                  const int rankDestination)
-{
-    int countsInt = static_cast<int>(sourceCount);
-    int result =
-        SMPI_Gather(const_cast<char *>(source), countsInt, MPI_CHAR,
-                    destination, countsInt, MPI_CHAR, rankDestination, mpiComm);
-
-    if (result != MPI_SUCCESS)
-    {
-        throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI "
-                                 "Gather type MPI_CHAR function\n");
-    }
-}
-
-template <>
-void GatherArrays(const size_t *source, const size_t sourceCount,
-                  size_t *destination, MPI_Comm mpiComm,
-                  const int rankDestination)
-{
-    int countsInt = static_cast<int>(sourceCount);
-    int result = SMPI_Gather(const_cast<size_t *>(source), countsInt,
-                             ADIOS2_MPI_SIZE_T, destination, countsInt,
-                             ADIOS2_MPI_SIZE_T, rankDestination, mpiComm);
-
-    if (result != MPI_SUCCESS)
-    {
-        throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI "
-                                 "Gather type size_t function\n");
-    }
-}
-
 // AllGatherArray specializations
 template <>
 void AllGatherArrays(const size_t *source, const size_t sourceCount,
@@ -229,68 +192,6 @@ void AllGatherArrays(const size_t *source, const size_t sourceCount,
     }
 }
 
-// GathervArrays specializations
-template <>
-void GathervArrays(const char *source, const size_t sourceCount,
-                   const size_t *counts, const size_t countsSize,
-                   char *destination, MPI_Comm mpiComm,
-                   const int rankDestination)
-{
-    int result = 0;
-    int rank;
-    SMPI_Comm_rank(mpiComm, &rank);
-
-    std::vector<int> countsInt, displacementsInt;
-
-    if (rank == rankDestination)
-    {
-        countsInt = NewVectorTypeFromArray<size_t, int>(counts, countsSize);
-        displacementsInt = GetGathervDisplacements(counts, countsSize);
-    }
-
-    int sourceCountInt = static_cast<int>(sourceCount);
-    result =
-        SMPI_Gatherv(const_cast<char *>(source), sourceCountInt, MPI_CHAR,
-                     destination, countsInt.data(), displacementsInt.data(),
-                     MPI_CHAR, rankDestination, mpiComm);
-
-    if (result != MPI_SUCCESS)
-    {
-        throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI "
-                                 "Gatherv type MPI_CHAR function\n");
-    }
-}
-
-template <>
-void GathervArrays(const size_t *source, const size_t sourceCount,
-                   const size_t *counts, const size_t countsSize,
-                   size_t *destination, MPI_Comm mpiComm,
-                   const int rankDestination)
-{
-    int result = 0;
-    int rank;
-    SMPI_Comm_rank(mpiComm, &rank);
-
-    std::vector<int> countsInt =
-        NewVectorTypeFromArray<size_t, int>(counts, countsSize);
-
-    std::vector<int> displacementsInt =
-        GetGathervDisplacements(counts, countsSize);
-
-    int sourceCountInt = static_cast<int>(sourceCount);
-
-    result = SMPI_Gatherv(const_cast<size_t *>(source), sourceCountInt,
-                          ADIOS2_MPI_SIZE_T, destination, countsInt.data(),
-                          displacementsInt.data(), ADIOS2_MPI_SIZE_T,
-                          rankDestination, mpiComm);
-
-    if (result != MPI_SUCCESS)
-    {
-        throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI "
-                                 "Gather type size_t function\n");
-    }
-}
-
 template <>
 std::vector<MPI_Request> Isend64<char>(const char *buffer, const size_t count,
                                        int dest, int tag, MPI_Comm mpiComm,
diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp
index b3339e637d..f616dfd9b6 100644
--- a/source/adios2/helper/adiosNetwork.cpp
+++ b/source/adios2/helper/adiosNetwork.cpp
@@ -158,8 +158,8 @@ void HandshakeWriter(Comm const &comm, size_t &appID,
     std::memcpy(localAddressesChar.data(), localAddressesStr.c_str(),
                 localAddressesStr.size());
     std::vector<char> globalAddressesChar(64 * channelsPerRank * mpiSize, '\0');
-    helper::GatherArrays(localAddressesChar.data(), 64 * channelsPerRank,
-                         globalAddressesChar.data(), comm);
+    comm.GatherArrays(localAddressesChar.data(), 64 * channelsPerRank,
+                      globalAddressesChar.data());
 
     // Writing handshake file
     if (mpiRank == 0)
diff --git a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
index c56e08b2ed..49ae86e4d8 100644
--- a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
+++ b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
@@ -985,8 +985,8 @@ BP3Serializer::AggregateCollectiveMetadataIndices(helper::Comm const &comm,
     // use bufferSTL (will resize) to GatherV
     const size_t extraSize = 16 + 12 + 12 + m_MetadataSet.MiniFooterSize;
 
-    helper::GathervVectors(m_SerializedIndices, bufferSTL.m_Buffer,
-                           bufferSTL.m_Position, comm, 0, extraSize);
+    comm.GathervVectors(m_SerializedIndices, bufferSTL.m_Buffer,
+                        bufferSTL.m_Position, 0, extraSize);
 
     // deserialize, it's all local inside rank 0
     if (rank == 0)
@@ -1417,8 +1417,7 @@ BP3Serializer::SetCollectiveProfilingJSON(const std::string &rankLog) const
 {
     // Gather sizes
     const size_t rankLogSize = rankLog.size();
-    std::vector<size_t> rankLogsSizes =
-        helper::GatherValues(rankLogSize, m_Comm);
+    std::vector<size_t> rankLogsSizes = m_Comm.GatherValues(rankLogSize);
 
     // Gatherv JSON per rank
     std::vector<char> profilingJSON(3);
@@ -1437,9 +1436,8 @@ BP3Serializer::SetCollectiveProfilingJSON(const std::string &rankLog) const
                              header.size());
     }
 
-    helper::GathervArrays(rankLog.c_str(), rankLog.size(), rankLogsSizes.data(),
-                          rankLogsSizes.size(), &profilingJSON[position],
-                          m_Comm);
+    m_Comm.GathervArrays(rankLog.c_str(), rankLog.size(), rankLogsSizes.data(),
+                         rankLogsSizes.size(), &profilingJSON[position]);
 
     if (m_RankMPI == 0) // add footer to close JSON
     {
diff --git a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
index c2222b4c18..97953abbe0 100644
--- a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
+++ b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
@@ -987,7 +987,7 @@ void BP4Serializer::AggregateIndex(const SerialElementIndex &index,
     }
 
     // write contents
-    helper::GathervVectors(index.Buffer, buffer, position, comm);
+    comm.GathervVectors(index.Buffer, buffer, position);
 
     // get total length and write it after count and before index
     if (rank == 0)
@@ -1008,8 +1008,8 @@ void BP4Serializer::AggregateMergeIndex(
     std::vector<char> gatheredSerialIndices;
     size_t gatheredSerialIndicesPosition = 0;
 
-    helper::GathervVectors(serializedIndices, gatheredSerialIndices,
-                           gatheredSerialIndicesPosition, comm);
+    comm.GathervVectors(serializedIndices, gatheredSerialIndices,
+                        gatheredSerialIndicesPosition);
 
     // deallocate local serialized Indices
     std::vector<char>().swap(serializedIndices);
@@ -2144,8 +2144,8 @@ void BP4Serializer::AggregateCollectiveMetadataIndices(helper::Comm const &comm,
     // use bufferSTL (will resize) to GatherV
     // const size_t extraSize = 16 + 12 + 12 + m_MetadataSet.MiniFooterSize;
 
-    helper::GathervVectors(m_SerializedIndices, inBufferSTL.m_Buffer,
-                           inBufferSTL.m_Position, comm, 0, 0);
+    comm.GathervVectors(m_SerializedIndices, inBufferSTL.m_Buffer,
+                        inBufferSTL.m_Position, 0, 0);
 
     // deserialize, it's all local inside rank 0
     if (rank == 0)
@@ -2898,8 +2898,7 @@ BP4Serializer::SetCollectiveProfilingJSON(const std::string &rankLog) const
 {
     // Gather sizes
     const size_t rankLogSize = rankLog.size();
-    std::vector<size_t> rankLogsSizes =
-        helper::GatherValues(rankLogSize, m_Comm);
+    std::vector<size_t> rankLogsSizes = m_Comm.GatherValues(rankLogSize);
 
     // Gatherv JSON per rank
     std::vector<char> profilingJSON(3);
@@ -2918,9 +2917,8 @@ BP4Serializer::SetCollectiveProfilingJSON(const std::string &rankLog) const
                              header.size());
     }
 
-    helper::GathervArrays(rankLog.c_str(), rankLog.size(), rankLogsSizes.data(),
-                          rankLogsSizes.size(), &profilingJSON[position],
-                          m_Comm);
+    m_Comm.GathervArrays(rankLog.c_str(), rankLog.size(), rankLogsSizes.data(),
+                         rankLogsSizes.size(), &profilingJSON[position]);
 
     if (m_RankMPI == 0) // add footer to close JSON
     {

From 71b826c73aa42fa9e9463e2c04804f46e81e1d06 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 2 Jul 2019 11:31:18 -0400
Subject: [PATCH 23/42] helper: Move Allgather functions to Comm encapsulation

---
 source/adios2/helper/adiosComm.h           | 18 ++++++++++++++++++
 source/adios2/helper/adiosComm.inl         | 17 +++++++++++++++++
 source/adios2/helper/adiosComm.tcc         | 17 +++++++++++++++++
 source/adios2/helper/adiosMPIFunctions.h   | 20 --------------------
 source/adios2/helper/adiosMPIFunctions.inl | 14 +-------------
 source/adios2/helper/adiosMPIFunctions.tcc | 17 -----------------
 6 files changed, 53 insertions(+), 50 deletions(-)

diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 06c630f67c..0aa6b0c97a 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -132,6 +132,24 @@ class Comm
     void GathervVectors(const std::vector<T> &in, std::vector<T> &out,
                         size_t &position, int rankDestination = 0,
                         size_t extraSize = 0) const;
+    /**
+     * Perform AllGather for source value
+     * @param source input
+     * @return in all ranks: a vector with gathered source values ordered per
+     * rank
+     */
+    template <class T>
+    std::vector<T> AllGatherValues(const T source) const;
+
+    /**
+     * Perform AllGather for equal size arrays
+     * @param source
+     * @param sourceCount
+     * @param destination
+     */
+    template <class T>
+    void AllGatherArrays(const T *source, const size_t sourceCount,
+                         T *destination) const;
 
 private:
     /**
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index 5ab0ccf850..dfc6e84714 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -95,6 +95,23 @@ void Comm::GathervVectors(const std::vector<T> &in, std::vector<T> &out,
     position += gatheredSize;
 }
 
+template <class T>
+std::vector<T> Comm::AllGatherValues(const T source) const
+{
+    int size;
+    SMPI_Comm_size(m_MPIComm, &size);
+    std::vector<T> output(size);
+
+    T sourceCopy = source; // so we can have an address for rvalues
+    this->AllGatherArrays(&sourceCopy, 1, output.data());
+    return output;
+}
+
+// AllGatherArrays full specializations implemented in 'adiosComm.tcc'.
+template <>
+void Comm::AllGatherArrays(const size_t *source, const size_t sourceCount,
+                           size_t *destination) const;
+
 } // end namespace helper
 } // end namespace adios2
 
diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc
index 2a64772f18..8d206a3c5f 100644
--- a/source/adios2/helper/adiosComm.tcc
+++ b/source/adios2/helper/adiosComm.tcc
@@ -134,6 +134,23 @@ void Comm::GathervArrays(const size_t *source, size_t sourceCount,
     }
 }
 
+// AllGatherArrays full specializations forward-declared in 'adiosComm.inl'.
+template <>
+void Comm::AllGatherArrays(const size_t *source, const size_t sourceCount,
+                           size_t *destination) const
+{
+    int countsInt = static_cast<int>(sourceCount);
+    int result = MPI_Allgather(const_cast<size_t *>(source), countsInt,
+                               ADIOS2_MPI_SIZE_T, destination, countsInt,
+                               ADIOS2_MPI_SIZE_T, m_MPIComm);
+
+    if (result != MPI_SUCCESS)
+    {
+        throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI "
+                                 "Allgather type size_t function\n");
+    }
+}
+
 } // end namespace helper
 } // end namespace adios2
 
diff --git a/source/adios2/helper/adiosMPIFunctions.h b/source/adios2/helper/adiosMPIFunctions.h
index 70dcff7e6e..f0cc00fef8 100644
--- a/source/adios2/helper/adiosMPIFunctions.h
+++ b/source/adios2/helper/adiosMPIFunctions.h
@@ -34,26 +34,6 @@ template <class T>
 T ReduceValues(const T source, MPI_Comm mpiComm, MPI_Op operation = MPI_SUM,
                const int rankDestination = 0);
 
-/**
- * Perform AllGather for source value
- * @param source input
- * @param mpiComm MPI communicator defining all ranks and size domain
- * @return in all ranks: a vector with gathered source values ordered per rank
- */
-template <class T>
-std::vector<T> AllGatherValues(const T source, MPI_Comm mpiComm);
-
-/**
- * Perform AllGather for equal size arrays
- * @param source
- * @param sourceCount
- * @param destination
- * @param mpiComm
- */
-template <class T>
-void AllGatherArrays(const T *source, const size_t sourceCount, T *destination,
-                     MPI_Comm mpiComm);
-
 void CheckMPIReturn(const int value, const std::string &hint);
 
 std::string BroadcastFile(const std::string &fileName, MPI_Comm mpiComm,
diff --git a/source/adios2/helper/adiosMPIFunctions.inl b/source/adios2/helper/adiosMPIFunctions.inl
index 04d47372cc..98fff83438 100644
--- a/source/adios2/helper/adiosMPIFunctions.inl
+++ b/source/adios2/helper/adiosMPIFunctions.inl
@@ -18,19 +18,7 @@ namespace adios2
 {
 namespace helper
 {
-
-template <class T>
-std::vector<T> AllGatherValues(const T source, MPI_Comm mpiComm)
-{
-    int size;
-    SMPI_Comm_size(mpiComm, &size);
-    std::vector<T> output(size);
-
-    T sourceCopy = source; // so we can have an address for rvalues
-    AllGatherArrays(&sourceCopy, 1, output.data(), mpiComm);
-    return output;
-}
-
+// placeholder
 } // end namespace helper
 } // end namespace adios2
 
diff --git a/source/adios2/helper/adiosMPIFunctions.tcc b/source/adios2/helper/adiosMPIFunctions.tcc
index 4cc52a1876..f8889cd604 100644
--- a/source/adios2/helper/adiosMPIFunctions.tcc
+++ b/source/adios2/helper/adiosMPIFunctions.tcc
@@ -175,23 +175,6 @@ void BroadcastVector(std::vector<size_t> &vector, MPI_Comm mpiComm,
     }
 }
 
-// AllGatherArray specializations
-template <>
-void AllGatherArrays(const size_t *source, const size_t sourceCount,
-                     size_t *destination, MPI_Comm mpiComm)
-{
-    int countsInt = static_cast<int>(sourceCount);
-    int result = MPI_Allgather(const_cast<size_t *>(source), countsInt,
-                               ADIOS2_MPI_SIZE_T, destination, countsInt,
-                               ADIOS2_MPI_SIZE_T, mpiComm);
-
-    if (result != MPI_SUCCESS)
-    {
-        throw std::runtime_error("ERROR: in ADIOS2 detected failure in MPI "
-                                 "Allgather type size_t function\n");
-    }
-}
-
 template <>
 std::vector<MPI_Request> Isend64<char>(const char *buffer, const size_t count,
                                        int dest, int tag, MPI_Comm mpiComm,

From 9360cd64c53140441d15401e66495967c1848dc6 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 2 Jul 2019 11:37:30 -0400
Subject: [PATCH 24/42] helper: Move ReduceValues function to Comm
 encapsulation

---
 source/adios2/helper/adiosComm.h              |  4 +++
 source/adios2/helper/adiosComm.inl            | 13 +++++++
 source/adios2/helper/adiosComm.tcc            | 36 +++++++++++++++++++
 source/adios2/helper/adiosMPIFunctions.h      |  4 ---
 source/adios2/helper/adiosMPIFunctions.tcc    | 35 ------------------
 .../toolkit/format/bp3/BP3Serializer.cpp      |  3 +-
 .../toolkit/format/bp4/BP4Serializer.cpp      |  5 ++-
 7 files changed, 56 insertions(+), 44 deletions(-)

diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 0aa6b0c97a..c0986c5db7 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -151,6 +151,10 @@ class Comm
     void AllGatherArrays(const T *source, const size_t sourceCount,
                          T *destination) const;
 
+    template <class T>
+    T ReduceValues(const T source, MPI_Op operation = MPI_SUM,
+                   const int rankDestination = 0) const;
+
 private:
     /**
      * @brief Construct by taking ownership of a MPI communicator.
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index dfc6e84714..945449faff 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -112,6 +112,19 @@ template <>
 void Comm::AllGatherArrays(const size_t *source, const size_t sourceCount,
                            size_t *destination) const;
 
+// ReduceValues full specializations implemented in 'adiosComm.tcc'.
+template <>
+unsigned int Comm::ReduceValues(const unsigned int source, MPI_Op operation,
+                                const int rankDestination) const;
+template <>
+unsigned long int Comm::ReduceValues(const unsigned long int source,
+                                     MPI_Op operation,
+                                     const int rankDestination) const;
+template <>
+unsigned long long int Comm::ReduceValues(const unsigned long long int source,
+                                          MPI_Op operation,
+                                          const int rankDestination) const;
+
 } // end namespace helper
 } // end namespace adios2
 
diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc
index 8d206a3c5f..5652776791 100644
--- a/source/adios2/helper/adiosComm.tcc
+++ b/source/adios2/helper/adiosComm.tcc
@@ -151,6 +151,42 @@ void Comm::AllGatherArrays(const size_t *source, const size_t sourceCount,
     }
 }
 
+// ReduceValues full specializations forward-declared in 'adiosComm.inl'.
+template <>
+unsigned int Comm::ReduceValues(const unsigned int source, MPI_Op operation,
+                                const int rankDestination) const
+{
+    unsigned int sourceLocal = source;
+    unsigned int reduceValue = 0;
+    SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED, operation,
+                rankDestination, m_MPIComm);
+    return reduceValue;
+}
+
+template <>
+unsigned long int Comm::ReduceValues(const unsigned long int source,
+                                     MPI_Op operation,
+                                     const int rankDestination) const
+{
+    unsigned long int sourceLocal = source;
+    unsigned long int reduceValue = 0;
+    SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG, operation,
+                rankDestination, m_MPIComm);
+    return reduceValue;
+}
+
+template <>
+unsigned long long int Comm::ReduceValues(const unsigned long long int source,
+                                          MPI_Op operation,
+                                          const int rankDestination) const
+{
+    unsigned long long int sourceLocal = source;
+    unsigned long long int reduceValue = 0;
+    SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG_LONG,
+                operation, rankDestination, m_MPIComm);
+    return reduceValue;
+}
+
 } // end namespace helper
 } // end namespace adios2
 
diff --git a/source/adios2/helper/adiosMPIFunctions.h b/source/adios2/helper/adiosMPIFunctions.h
index f0cc00fef8..f6af254e56 100644
--- a/source/adios2/helper/adiosMPIFunctions.h
+++ b/source/adios2/helper/adiosMPIFunctions.h
@@ -30,10 +30,6 @@ template <class T>
 void BroadcastVector(std::vector<T> &vector, MPI_Comm mpiComm,
                      const int rankSource = 0);
 
-template <class T>
-T ReduceValues(const T source, MPI_Comm mpiComm, MPI_Op operation = MPI_SUM,
-               const int rankDestination = 0);
-
 void CheckMPIReturn(const int value, const std::string &hint);
 
 std::string BroadcastFile(const std::string &fileName, MPI_Comm mpiComm,
diff --git a/source/adios2/helper/adiosMPIFunctions.tcc b/source/adios2/helper/adiosMPIFunctions.tcc
index f8889cd604..1feab94862 100644
--- a/source/adios2/helper/adiosMPIFunctions.tcc
+++ b/source/adios2/helper/adiosMPIFunctions.tcc
@@ -68,41 +68,6 @@ std::string BroadcastValue(const std::string &input, MPI_Comm mpiComm,
     return output;
 }
 
-// ReduceValue specializations
-template <>
-unsigned int ReduceValues(const unsigned int source, MPI_Comm mpiComm,
-                          MPI_Op operation, const int rankDestination)
-{
-    unsigned int sourceLocal = source;
-    unsigned int reduceValue = 0;
-    SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED, operation,
-                rankDestination, mpiComm);
-    return reduceValue;
-}
-
-template <>
-unsigned long int ReduceValues(const unsigned long int source, MPI_Comm mpiComm,
-                               MPI_Op operation, const int rankDestination)
-{
-    unsigned long int sourceLocal = source;
-    unsigned long int reduceValue = 0;
-    SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG, operation,
-                rankDestination, mpiComm);
-    return reduceValue;
-}
-
-template <>
-unsigned long long int ReduceValues(const unsigned long long int source,
-                                    MPI_Comm mpiComm, MPI_Op operation,
-                                    const int rankDestination)
-{
-    unsigned long long int sourceLocal = source;
-    unsigned long long int reduceValue = 0;
-    SMPI_Reduce(&sourceLocal, &reduceValue, 1, MPI_UNSIGNED_LONG_LONG,
-                operation, rankDestination, mpiComm);
-    return reduceValue;
-}
-
 // BroadcastVector specializations
 template <>
 void BroadcastVector(std::vector<char> &vector, MPI_Comm mpiComm,
diff --git a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
index 49ae86e4d8..9a6ba186df 100644
--- a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
+++ b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
@@ -16,8 +16,7 @@
 #include <string>
 #include <vector>
 
-#include "adios2/helper/adiosFunctions.h" //helper::GetType<T>, helper::ReadValue<T>,
-                                          // ReduceValue<T>
+#include "adios2/helper/adiosFunctions.h" //helper::GetType<T>, helper::ReadValue<T>
 #include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
 
 #ifdef _WIN32
diff --git a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
index 97953abbe0..144dc7a6fb 100644
--- a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
+++ b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
@@ -17,8 +17,7 @@
 #include <tuple>
 #include <vector>
 
-#include "adios2/helper/adiosFunctions.h" //helper::GetType<T>, helper::ReadValue<T>,
-                                          // ReduceValue<T>
+#include "adios2/helper/adiosFunctions.h" //helper::GetType<T>, helper::ReadValue<T>
 
 #ifdef _WIN32
 #pragma warning(disable : 4503) // Windows complains about SubFileInfoMap levels
@@ -975,7 +974,7 @@ void BP4Serializer::AggregateIndex(const SerialElementIndex &index,
     SMPI_Comm_rank(comm, &rank);
 
     size_t countPosition = position;
-    const size_t totalCount = helper::ReduceValues<size_t>(count, comm);
+    const size_t totalCount = comm.ReduceValues<size_t>(count);
 
     if (rank == 0)
     {

From 08e03c7d12619f0f3e54c990df7cd635f382b0f0 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 2 Jul 2019 14:15:37 -0400
Subject: [PATCH 25/42] helper: Move Broadcast functions to Comm encapsulation

---
 source/adios2/engine/bp3/BP3Reader.cpp        |   3 +-
 source/adios2/engine/bp4/BP4Reader.cpp        |  18 ++-
 source/adios2/engine/bp4/BP4Writer.cpp        |   2 +-
 source/adios2/helper/adiosComm.cpp            |  20 +++
 source/adios2/helper/adiosComm.h              |  11 ++
 source/adios2/helper/adiosComm.inl            |  15 +++
 source/adios2/helper/adiosComm.tcc            | 114 +++++++++++++++++
 source/adios2/helper/adiosMPIFunctions.cpp    |  18 ---
 source/adios2/helper/adiosMPIFunctions.h      |  11 --
 source/adios2/helper/adiosMPIFunctions.tcc    | 116 ------------------
 source/adios2/helper/adiosNetwork.cpp         |   5 +-
 source/adios2/helper/adiosXML.cpp             |   5 +-
 .../format/dataman/DataManSerializer.cpp      |   4 +-
 13 files changed, 175 insertions(+), 167 deletions(-)

diff --git a/source/adios2/engine/bp3/BP3Reader.cpp b/source/adios2/engine/bp3/BP3Reader.cpp
index 3a0b3c5095..9649e9951e 100644
--- a/source/adios2/engine/bp3/BP3Reader.cpp
+++ b/source/adios2/engine/bp3/BP3Reader.cpp
@@ -12,7 +12,6 @@
 #include "BP3Reader.tcc"
 
 #include "adios2/helper/adiosComm.h"
-#include "adios2/helper/adiosFunctions.h" // MPI BroadcastVector
 #include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
 
 namespace adios2
@@ -203,7 +202,7 @@ void BP3Reader::InitBuffer()
     }
 
     // broadcast metadata buffer to all ranks from zero
-    helper::BroadcastVector(m_BP3Deserializer.m_Metadata.m_Buffer, m_Comm);
+    m_Comm.BroadcastVector(m_BP3Deserializer.m_Metadata.m_Buffer);
 
     // fills IO with available Variables and Attributes
     m_BP3Deserializer.ParseMetadata(m_BP3Deserializer.m_Metadata, *this);
diff --git a/source/adios2/engine/bp4/BP4Reader.cpp b/source/adios2/engine/bp4/BP4Reader.cpp
index 8469405415..bf3ad88667 100644
--- a/source/adios2/engine/bp4/BP4Reader.cpp
+++ b/source/adios2/engine/bp4/BP4Reader.cpp
@@ -11,7 +11,6 @@
 #include "BP4Reader.h"
 #include "BP4Reader.tcc"
 
-#include "adios2/helper/adiosFunctions.h" // MPI BroadcastVector
 #include "adios2/toolkit/profiling/taustubs/tautimer.hpp"
 
 #include <errno.h>
@@ -216,7 +215,7 @@ void BP4Reader::OpenFiles()
         }
     }
 
-    flag = helper::BroadcastValue(flag, m_Comm, 0);
+    flag = m_Comm.BroadcastValue(flag, 0);
     if (flag == 2)
     {
         if (m_BP4Deserializer.m_RankMPI == 0 && !lasterrmsg.empty())
@@ -268,7 +267,7 @@ void BP4Reader::OpenFiles()
         }
     }
 
-    flag = helper::BroadcastValue(flag, m_Comm, 0);
+    flag = m_Comm.BroadcastValue(flag, 0);
     if (flag == 1)
     {
         throw std::runtime_error("ERROR: File " + m_Name +
@@ -316,10 +315,10 @@ void BP4Reader::InitBuffer()
                                  fileSize);
     }
     // broadcast buffer to all ranks from zero
-    helper::BroadcastVector(m_BP4Deserializer.m_Metadata.m_Buffer, m_Comm);
+    m_Comm.BroadcastVector(m_BP4Deserializer.m_Metadata.m_Buffer);
 
     // broadcast metadata index buffer to all ranks from zero
-    helper::BroadcastVector(m_BP4Deserializer.m_MetadataIndex.m_Buffer, m_Comm);
+    m_Comm.BroadcastVector(m_BP4Deserializer.m_MetadataIndex.m_Buffer);
 
     /* Parse metadata index table */
     m_BP4Deserializer.ParseMetadataIndex(m_BP4Deserializer.m_MetadataIndex);
@@ -384,17 +383,16 @@ size_t BP4Reader::UpdateBuffer()
         }
     }
 
-    helper::BroadcastVector(sizes, m_Comm, 0);
+    m_Comm.BroadcastVector(sizes, 0);
     size_t newIdxSize = sizes[0];
 
     if (newIdxSize > 0)
     {
         // broadcast buffer to all ranks from zero
-        helper::BroadcastVector(m_BP4Deserializer.m_Metadata.m_Buffer, m_Comm);
+        m_Comm.BroadcastVector(m_BP4Deserializer.m_Metadata.m_Buffer);
 
         // broadcast metadata index buffer to all ranks from zero
-        helper::BroadcastVector(m_BP4Deserializer.m_MetadataIndex.m_Buffer,
-                                m_Comm);
+        m_Comm.BroadcastVector(m_BP4Deserializer.m_MetadataIndex.m_Buffer);
 
         if (m_BP4Deserializer.m_RankMPI != 0)
         {
@@ -440,7 +438,7 @@ bool BP4Reader::CheckWriterActive()
         bool active = m_BP4Deserializer.ReadActiveFlag(header);
         flag = (active ? 1 : 0);
     }
-    flag = helper::BroadcastValue(flag, m_BP4Deserializer.m_Comm, 0);
+    flag = m_BP4Deserializer.m_Comm.BroadcastValue(flag, 0);
     m_BP4Deserializer.m_WriterIsActive = (flag > 0);
     return m_BP4Deserializer.m_WriterIsActive;
 }
diff --git a/source/adios2/engine/bp4/BP4Writer.cpp b/source/adios2/engine/bp4/BP4Writer.cpp
index bb5f71a03b..f992ee8b5d 100644
--- a/source/adios2/engine/bp4/BP4Writer.cpp
+++ b/source/adios2/engine/bp4/BP4Writer.cpp
@@ -255,7 +255,7 @@ void BP4Writer::InitBPBuffer()
             m_FileMetadataIndexManager.ReadFile(
                 preMetadataIndex.m_Buffer.data(), preMetadataIndexFileSize);
         }
-        helper::BroadcastVector(preMetadataIndex.m_Buffer, m_Comm);
+        m_Comm.BroadcastVector(preMetadataIndex.m_Buffer);
         preMetadataIndexFileSize = preMetadataIndex.m_Buffer.size();
         if (preMetadataIndexFileSize > 0)
         {
diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index eddc4137e2..1f19b7370e 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -12,6 +12,7 @@
 #include <utility>
 
 #include "adios2/common/ADIOSMPI.h"
+#include "adios2/helper/adiosString.h"
 
 #include "adiosMPIFunctions.h"
 
@@ -76,5 +77,24 @@ Comm Comm::Split(int color, int key, const std::string &hint) const
     return Comm(newComm);
 }
 
+std::string Comm::BroadcastFile(const std::string &fileName,
+                                const std::string hint,
+                                const int rankSource) const
+{
+    int rank;
+    MPI_Comm_rank(m_MPIComm, &rank);
+    std::string fileContents;
+
+    // Read the file on rank 0 and broadcast it to everybody else
+    if (rank == rankSource)
+    {
+        // load file contents
+        fileContents = FileToString(fileName, hint);
+    }
+    fileContents = this->BroadcastValue(fileContents, rankSource);
+
+    return fileContents;
+}
+
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index c0986c5db7..6d96d7c9fe 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -155,6 +155,17 @@ class Comm
     T ReduceValues(const T source, MPI_Op operation = MPI_SUM,
                    const int rankDestination = 0) const;
 
+    template <class T>
+    T BroadcastValue(const T &input, const int rankSource = 0) const;
+
+    template <class T>
+    void BroadcastVector(std::vector<T> &vector,
+                         const int rankSource = 0) const;
+
+    std::string BroadcastFile(const std::string &fileName,
+                              const std::string hint = "",
+                              const int rankSource = 0) const;
+
 private:
     /**
      * @brief Construct by taking ownership of a MPI communicator.
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index 945449faff..7a482c504f 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -125,6 +125,21 @@ unsigned long long int Comm::ReduceValues(const unsigned long long int source,
                                           MPI_Op operation,
                                           const int rankDestination) const;
 
+// BroadcastValue full specializations implemented in 'adiosComm.tcc'.
+template <>
+size_t Comm::BroadcastValue(const size_t &input, const int rankSource) const;
+template <>
+std::string Comm::BroadcastValue(const std::string &input,
+                                 const int rankSource) const;
+
+// BroadcastVector full specializations implemented in 'adiosComm.tcc'.
+template <>
+void Comm::BroadcastVector(std::vector<char> &vector,
+                           const int rankSource) const;
+template <>
+void Comm::BroadcastVector(std::vector<size_t> &vector,
+                           const int rankSource) const;
+
 } // end namespace helper
 } // end namespace adios2
 
diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc
index 5652776791..22e1953282 100644
--- a/source/adios2/helper/adiosComm.tcc
+++ b/source/adios2/helper/adiosComm.tcc
@@ -187,6 +187,120 @@ unsigned long long int Comm::ReduceValues(const unsigned long long int source,
     return reduceValue;
 }
 
+// BroadcastValue full specializations forward-declared in 'adiosComm.inl'.
+template <>
+size_t Comm::BroadcastValue(const size_t &input, const int rankSource) const
+{
+    int rank;
+    SMPI_Comm_rank(m_MPIComm, &rank);
+    size_t output = 0;
+
+    if (rank == rankSource)
+    {
+        output = input;
+    }
+
+    SMPI_Bcast(&output, 1, ADIOS2_MPI_SIZE_T, rankSource, m_MPIComm);
+
+    return output;
+}
+
+template <>
+std::string Comm::BroadcastValue(const std::string &input,
+                                 const int rankSource) const
+{
+    int rank;
+    SMPI_Comm_rank(m_MPIComm, &rank);
+    const size_t inputSize = input.size();
+    const size_t length = this->BroadcastValue(inputSize, rankSource);
+    std::string output;
+
+    if (rank == rankSource)
+    {
+        output = input;
+    }
+    else
+    {
+        output.resize(length);
+    }
+
+    SMPI_Bcast(const_cast<char *>(output.data()), static_cast<int>(length),
+               MPI_CHAR, rankSource, m_MPIComm);
+
+    return output;
+}
+
+// BroadcastVector full specializations forward-declared in 'adiosComm.inl'.
+template <>
+void Comm::BroadcastVector(std::vector<char> &vector,
+                           const int rankSource) const
+{
+    int size;
+    SMPI_Comm_size(m_MPIComm, &size);
+
+    if (size == 1)
+    {
+        return;
+    }
+
+    // First Broadcast the size, then the contents
+    size_t inputSize = this->BroadcastValue(vector.size(), rankSource);
+    int rank;
+    SMPI_Comm_rank(m_MPIComm, &rank);
+
+    if (rank != rankSource)
+    {
+        vector.resize(inputSize);
+    }
+
+    const int MAXBCASTSIZE = 1073741824;
+    size_t blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize);
+    char *buffer = vector.data();
+    while (inputSize > 0)
+    {
+        SMPI_Bcast(buffer, static_cast<int>(blockSize), MPI_CHAR, rankSource,
+                   m_MPIComm);
+        buffer += blockSize;
+        inputSize -= blockSize;
+        blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize);
+    }
+}
+
+template <>
+void Comm::BroadcastVector(std::vector<size_t> &vector,
+                           const int rankSource) const
+{
+    int size;
+    SMPI_Comm_size(m_MPIComm, &size);
+
+    if (size == 1)
+    {
+        return;
+    }
+
+    // First Broadcast the size, then the contents
+    size_t inputSize = this->BroadcastValue(vector.size(), rankSource);
+    int rank;
+    SMPI_Comm_rank(m_MPIComm, &rank);
+
+    if (rank != rankSource)
+    {
+        vector.resize(inputSize);
+    }
+
+    const int MAXBCASTSIZE = 1073741824 / sizeof(size_t);
+    size_t blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize);
+    size_t *buffer = vector.data();
+    while (inputSize > 0)
+    {
+        SMPI_Bcast(buffer, static_cast<int>(blockSize), ADIOS2_MPI_SIZE_T,
+                   rankSource, m_MPIComm);
+        buffer += blockSize;
+        inputSize -= blockSize;
+        blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize);
+    }
+}
+
 } // end namespace helper
 } // end namespace adios2
 
diff --git a/source/adios2/helper/adiosMPIFunctions.cpp b/source/adios2/helper/adiosMPIFunctions.cpp
index 8d03b0aac0..47c017d2f4 100644
--- a/source/adios2/helper/adiosMPIFunctions.cpp
+++ b/source/adios2/helper/adiosMPIFunctions.cpp
@@ -43,23 +43,5 @@ void CheckMPIReturn(const int value, const std::string &hint)
     throw std::runtime_error("ERROR: ADIOS2 detected " + error + ", " + hint);
 }
 
-std::string BroadcastFile(const std::string &fileName, MPI_Comm mpiComm,
-                          const std::string hint, const int rankSource)
-{
-    int rank;
-    MPI_Comm_rank(mpiComm, &rank);
-    std::string fileContents;
-
-    // Read the file on rank 0 and broadcast it to everybody else
-    if (rank == rankSource)
-    {
-        // load file contents
-        fileContents = FileToString(fileName, hint);
-    }
-    fileContents = BroadcastValue(fileContents, mpiComm, rankSource);
-
-    return fileContents;
-}
-
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/helper/adiosMPIFunctions.h b/source/adios2/helper/adiosMPIFunctions.h
index f6af254e56..a95bade778 100644
--- a/source/adios2/helper/adiosMPIFunctions.h
+++ b/source/adios2/helper/adiosMPIFunctions.h
@@ -23,19 +23,8 @@ namespace adios2
 namespace helper
 {
 
-template <class T>
-T BroadcastValue(const T &input, MPI_Comm mpiComm, const int rankSource = 0);
-
-template <class T>
-void BroadcastVector(std::vector<T> &vector, MPI_Comm mpiComm,
-                     const int rankSource = 0);
-
 void CheckMPIReturn(const int value, const std::string &hint);
 
-std::string BroadcastFile(const std::string &fileName, MPI_Comm mpiComm,
-                          const std::string hint = "",
-                          const int rankSource = 0);
-
 template <class T>
 std::vector<MPI_Request> Isend64(const T *buffer, const size_t count,
                                  int destination, int tag, MPI_Comm mpiComm,
diff --git a/source/adios2/helper/adiosMPIFunctions.tcc b/source/adios2/helper/adiosMPIFunctions.tcc
index 1feab94862..dc0fdff40c 100644
--- a/source/adios2/helper/adiosMPIFunctions.tcc
+++ b/source/adios2/helper/adiosMPIFunctions.tcc
@@ -24,122 +24,6 @@ namespace adios2
 namespace helper
 {
 
-// BroadcastValue specializations
-template <>
-size_t BroadcastValue(const size_t &input, MPI_Comm mpiComm,
-                      const int rankSource)
-{
-    int rank;
-    SMPI_Comm_rank(mpiComm, &rank);
-    size_t output = 0;
-
-    if (rank == rankSource)
-    {
-        output = input;
-    }
-
-    SMPI_Bcast(&output, 1, ADIOS2_MPI_SIZE_T, rankSource, mpiComm);
-
-    return output;
-}
-
-template <>
-std::string BroadcastValue(const std::string &input, MPI_Comm mpiComm,
-                           const int rankSource)
-{
-    int rank;
-    SMPI_Comm_rank(mpiComm, &rank);
-    const size_t inputSize = input.size();
-    const size_t length = BroadcastValue(inputSize, mpiComm, rankSource);
-    std::string output;
-
-    if (rank == rankSource)
-    {
-        output = input;
-    }
-    else
-    {
-        output.resize(length);
-    }
-
-    SMPI_Bcast(const_cast<char *>(output.data()), static_cast<int>(length),
-               MPI_CHAR, rankSource, mpiComm);
-
-    return output;
-}
-
-// BroadcastVector specializations
-template <>
-void BroadcastVector(std::vector<char> &vector, MPI_Comm mpiComm,
-                     const int rankSource)
-{
-    int size;
-    SMPI_Comm_size(mpiComm, &size);
-
-    if (size == 1)
-    {
-        return;
-    }
-
-    // First Broadcast the size, then the contents
-    size_t inputSize = BroadcastValue(vector.size(), mpiComm, rankSource);
-    int rank;
-    SMPI_Comm_rank(mpiComm, &rank);
-
-    if (rank != rankSource)
-    {
-        vector.resize(inputSize);
-    }
-
-    const int MAXBCASTSIZE = 1073741824;
-    size_t blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize);
-    char *buffer = vector.data();
-    while (inputSize > 0)
-    {
-        SMPI_Bcast(buffer, static_cast<int>(blockSize), MPI_CHAR, rankSource,
-                   mpiComm);
-        buffer += blockSize;
-        inputSize -= blockSize;
-        blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize);
-    }
-}
-
-// BroadcastVector specializations
-template <>
-void BroadcastVector(std::vector<size_t> &vector, MPI_Comm mpiComm,
-                     const int rankSource)
-{
-    int size;
-    SMPI_Comm_size(mpiComm, &size);
-
-    if (size == 1)
-    {
-        return;
-    }
-
-    // First Broadcast the size, then the contents
-    size_t inputSize = BroadcastValue(vector.size(), mpiComm, rankSource);
-    int rank;
-    SMPI_Comm_rank(mpiComm, &rank);
-
-    if (rank != rankSource)
-    {
-        vector.resize(inputSize);
-    }
-
-    const int MAXBCASTSIZE = 1073741824 / sizeof(size_t);
-    size_t blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize);
-    size_t *buffer = vector.data();
-    while (inputSize > 0)
-    {
-        SMPI_Bcast(buffer, static_cast<int>(blockSize), ADIOS2_MPI_SIZE_T,
-                   rankSource, mpiComm);
-        buffer += blockSize;
-        inputSize -= blockSize;
-        blockSize = (inputSize > MAXBCASTSIZE ? MAXBCASTSIZE : inputSize);
-    }
-}
-
 template <>
 std::vector<MPI_Request> Isend64<char>(const char *buffer, const size_t count,
                                        int dest, int tag, MPI_Comm mpiComm,
diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp
index f616dfd9b6..3bad0d18f7 100644
--- a/source/adios2/helper/adiosNetwork.cpp
+++ b/source/adios2/helper/adiosNetwork.cpp
@@ -10,7 +10,6 @@
 
 #include "adiosNetwork.h"
 #include "adios2/helper/adiosComm.h"
-#include "adios2/helper/adiosMPIFunctions.h"
 #include "adios2/toolkit/transport/file/FileFStream.h"
 
 #ifndef _WIN32
@@ -140,7 +139,7 @@ void HandshakeWriter(Comm const &comm, size_t &appID,
         remove(globalLockFilename.c_str());
     }
 
-    appID = helper::BroadcastValue(appID, comm);
+    appID = comm.BroadcastValue(appID);
 
     // Make full addresses
     for (int i = 0; i < channelsPerRank; ++i)
@@ -203,7 +202,7 @@ void HandshakeReader(Comm const &comm, size_t &appID,
         std::hash<std::string> hash_fn;
         appID = hash_fn(ips[0]);
     }
-    helper::BroadcastValue(appID, comm);
+    comm.BroadcastValue(appID);
 
     transport::FileFStream ipstream(comm, false);
     while (true)
diff --git a/source/adios2/helper/adiosXML.cpp b/source/adios2/helper/adiosXML.cpp
index 931af2f2e3..d41bd32916 100644
--- a/source/adios2/helper/adiosXML.cpp
+++ b/source/adios2/helper/adiosXML.cpp
@@ -20,7 +20,6 @@
 #include "adios2/common/ADIOSMPI.h"
 #include "adios2/common/ADIOSTypes.h"
 #include "adios2/core/IO.h"
-#include "adios2/helper/adiosMPIFunctions.h"
 #include "adios2/helper/adiosString.h"
 
 #include <pugixml.hpp>
@@ -150,8 +149,8 @@ void ParseConfigXML(
                            " in call to ADIOS constructor");
 
     auto lf_FileContents = [&](const std::string &configXML) -> std::string {
-        const std::string fileContents(helper::BroadcastFile(
-            configXML, adios.GetComm(),
+        const std::string fileContents(adios.GetComm().BroadcastFile(
+            configXML,
             "when parsing configXML file, in call to the ADIOS constructor"));
 
         if (adios.m_DebugMode)
diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
index 2923eb7b22..ac222968b7 100644
--- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
+++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
@@ -11,8 +11,6 @@
 #include "DataManSerializer.h"
 #include "DataManSerializer.tcc"
 
-#include "adios2/helper/adiosMPIFunctions.h"
-
 #include <cstring>
 #include <iostream>
 
@@ -273,7 +271,7 @@ void DataManSerializer::PutAggregatedMetadata(VecPtr input,
         return;
     }
 
-    helper::BroadcastVector(*input, comm);
+    comm.BroadcastVector(*input);
 
     if (input->size() > 0)
     {

From 323125b3501da22321b23685bc3e24baa8b1e64b Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Mon, 22 Jul 2019 17:57:25 -0400
Subject: [PATCH 26/42] aggregator: Name async request types

---
 source/adios2/engine/bp3/BP3Writer.cpp          |  9 +++++----
 source/adios2/engine/bp4/BP4Writer.cpp          |  9 +++++----
 .../toolkit/aggregator/mpi/MPIAggregator.h      | 16 ++++++++++------
 .../adios2/toolkit/aggregator/mpi/MPIChain.cpp  | 17 ++++++++---------
 source/adios2/toolkit/aggregator/mpi/MPIChain.h | 10 ++++------
 5 files changed, 32 insertions(+), 29 deletions(-)

diff --git a/source/adios2/engine/bp3/BP3Writer.cpp b/source/adios2/engine/bp3/BP3Writer.cpp
index c4a0925352..40eb785caa 100644
--- a/source/adios2/engine/bp3/BP3Writer.cpp
+++ b/source/adios2/engine/bp3/BP3Writer.cpp
@@ -358,12 +358,13 @@ void BP3Writer::AggregateWriteData(const bool isFinal, const int transportIndex)
     // async?
     for (int r = 0; r < m_BP3Serializer.m_Aggregator.m_Size; ++r)
     {
-        std::vector<std::vector<MPI_Request>> dataRequests =
+        aggregator::MPIAggregator::ExchangeRequests dataRequests =
             m_BP3Serializer.m_Aggregator.IExchange(m_BP3Serializer.m_Data, r);
 
-        std::vector<std::vector<MPI_Request>> absolutePositionRequests =
-            m_BP3Serializer.m_Aggregator.IExchangeAbsolutePosition(
-                m_BP3Serializer.m_Data, r);
+        aggregator::MPIAggregator::ExchangeAbsolutePositionRequests
+            absolutePositionRequests =
+                m_BP3Serializer.m_Aggregator.IExchangeAbsolutePosition(
+                    m_BP3Serializer.m_Data, r);
 
         if (m_BP3Serializer.m_Aggregator.m_IsConsumer)
         {
diff --git a/source/adios2/engine/bp4/BP4Writer.cpp b/source/adios2/engine/bp4/BP4Writer.cpp
index f992ee8b5d..570f9b2dc3 100644
--- a/source/adios2/engine/bp4/BP4Writer.cpp
+++ b/source/adios2/engine/bp4/BP4Writer.cpp
@@ -650,12 +650,13 @@ void BP4Writer::AggregateWriteData(const bool isFinal, const int transportIndex)
     // async?
     for (int r = 0; r < m_BP4Serializer.m_Aggregator.m_Size; ++r)
     {
-        std::vector<std::vector<MPI_Request>> dataRequests =
+        aggregator::MPIAggregator::ExchangeRequests dataRequests =
             m_BP4Serializer.m_Aggregator.IExchange(m_BP4Serializer.m_Data, r);
 
-        std::vector<std::vector<MPI_Request>> absolutePositionRequests =
-            m_BP4Serializer.m_Aggregator.IExchangeAbsolutePosition(
-                m_BP4Serializer.m_Data, r);
+        aggregator::MPIAggregator::ExchangeAbsolutePositionRequests
+            absolutePositionRequests =
+                m_BP4Serializer.m_Aggregator.IExchangeAbsolutePosition(
+                    m_BP4Serializer.m_Data, r);
 
         if (m_BP4Serializer.m_Aggregator.m_IsConsumer)
         {
diff --git a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h
index 98f39cf6e6..9a93396ec9 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h
+++ b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h
@@ -60,18 +60,22 @@ class MPIAggregator
 
     virtual void Init(const size_t subStreams, helper::Comm const &parentComm);
 
-    virtual std::vector<std::vector<MPI_Request>>
-    IExchange(format::Buffer &buffer, const int step) = 0;
+    using ExchangeRequests = std::vector<std::vector<MPI_Request>>;
 
-    virtual std::vector<std::vector<MPI_Request>>
+    virtual ExchangeRequests IExchange(format::Buffer &buffer,
+                                       const int step) = 0;
+
+    using ExchangeAbsolutePositionRequests =
+        std::vector<std::vector<MPI_Request>>;
+
+    virtual ExchangeAbsolutePositionRequests
     IExchangeAbsolutePosition(format::Buffer &buffer, const int step) = 0;
 
     virtual void
-    WaitAbsolutePosition(std::vector<std::vector<MPI_Request>> &requests,
+    WaitAbsolutePosition(ExchangeAbsolutePositionRequests &requests,
                          const int step) = 0;
 
-    virtual void Wait(std::vector<std::vector<MPI_Request>> &requests,
-                      const int step) = 0;
+    virtual void Wait(ExchangeRequests &requests, const int step) = 0;
 
     virtual void SwapBuffers(const int step) noexcept;
 
diff --git a/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp b/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp
index 26bd176e95..cf80c788b2 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp
+++ b/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp
@@ -34,12 +34,12 @@ void MPIChain::Init(const size_t subStreams, helper::Comm const &parentComm)
     }
 }
 
-std::vector<std::vector<MPI_Request>>
-MPIChain::IExchange(format::Buffer &buffer, const int step)
+MPIChain::ExchangeRequests MPIChain::IExchange(format::Buffer &buffer,
+                                               const int step)
 {
     if (m_Size == 1)
     {
-        return std::vector<std::vector<MPI_Request>>();
+        return {};
     }
 
     format::Buffer &sendBuffer = GetSender(buffer);
@@ -109,12 +109,12 @@ MPIChain::IExchange(format::Buffer &buffer, const int step)
     return requests;
 }
 
-std::vector<std::vector<MPI_Request>>
+MPIChain::ExchangeAbsolutePositionRequests
 MPIChain::IExchangeAbsolutePosition(format::Buffer &buffer, const int step)
 {
     if (m_Size == 1)
     {
-        return std::vector<std::vector<MPI_Request>>();
+        return {};
     }
 
     if (m_IsInExchangeAbsolutePosition)
@@ -157,8 +157,7 @@ MPIChain::IExchangeAbsolutePosition(format::Buffer &buffer, const int step)
     return requests;
 }
 
-void MPIChain::Wait(std::vector<std::vector<MPI_Request>> &requests,
-                    const int step)
+void MPIChain::Wait(ExchangeRequests &requests, const int step)
 {
     if (m_Size == 1)
     {
@@ -193,8 +192,8 @@ void MPIChain::Wait(std::vector<std::vector<MPI_Request>> &requests,
     }
 }
 
-void MPIChain::WaitAbsolutePosition(
-    std::vector<std::vector<MPI_Request>> &requests, const int step)
+void MPIChain::WaitAbsolutePosition(ExchangeAbsolutePositionRequests &requests,
+                                    const int step)
 {
     if (m_Size == 1)
     {
diff --git a/source/adios2/toolkit/aggregator/mpi/MPIChain.h b/source/adios2/toolkit/aggregator/mpi/MPIChain.h
index d6939ca56d..a26895d8d2 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIChain.h
+++ b/source/adios2/toolkit/aggregator/mpi/MPIChain.h
@@ -28,16 +28,14 @@ class MPIChain : public MPIAggregator
 
     void Init(const size_t subStreams, helper::Comm const &parentComm) final;
 
-    std::vector<std::vector<MPI_Request>> IExchange(format::Buffer &buffer,
-                                                    const int step) final;
+    ExchangeRequests IExchange(format::Buffer &buffer, const int step) final;
 
-    std::vector<std::vector<MPI_Request>>
+    ExchangeAbsolutePositionRequests
     IExchangeAbsolutePosition(format::Buffer &buffer, const int step) final;
 
-    void Wait(std::vector<std::vector<MPI_Request>> &request,
-              const int step) final;
+    void Wait(ExchangeRequests &requests, const int step) final;
 
-    void WaitAbsolutePosition(std::vector<std::vector<MPI_Request>> &requests,
+    void WaitAbsolutePosition(ExchangeAbsolutePositionRequests &requests,
                               const int step) final;
 
     void SwapBuffers(const int step) noexcept final;

From 5c24ca290e92c389585ae4b6edb327661e03630a Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 23 Jul 2019 11:42:37 -0400
Subject: [PATCH 27/42] helper: Encapsulate MPI_{Request,Status} as
 Comm::{Req,Status}

Encapsulate a pending async operation and its result status.  This may
require multiple `MPI_Request` instances in case we split large
operations into a batch of multiple MPI operations.  In the case of a
batched operation, present the results as a combined status.
---
 source/adios2/helper/adiosComm.cpp | 86 +++++++++++++++++++++++++++++
 source/adios2/helper/adiosComm.h   | 88 ++++++++++++++++++++++++++++++
 2 files changed, 174 insertions(+)

diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index 1f19b7370e..6eb7c223df 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -96,5 +96,91 @@ std::string Comm::BroadcastFile(const std::string &fileName,
     return fileContents;
 }
 
+Comm::Req::Req() = default;
+
+Comm::Req::Req(MPI_Datatype datatype) : m_MPIDatatype(datatype) {}
+
+Comm::Req::~Req() {}
+
+Comm::Req::Req(Req &&req)
+: m_MPIDatatype(req.m_MPIDatatype), m_MPIReqs(std::move(req.m_MPIReqs))
+{
+}
+
+Comm::Req &Comm::Req::operator=(Req &&req)
+{
+    Req(std::move(req)).swap(*this);
+    return *this;
+}
+
+void Comm::Req::swap(Req &req)
+{
+    std::swap(this->m_MPIDatatype, req.m_MPIDatatype);
+    std::swap(this->m_MPIReqs, req.m_MPIReqs);
+}
+
+Comm::Status Comm::Req::Wait(const std::string &hint)
+{
+    Comm::Status status;
+    if (m_MPIReqs.empty())
+    {
+        return status;
+    }
+
+#ifdef ADIOS2_HAVE_MPI
+    std::vector<MPI_Request> mpiRequests = std::move(m_MPIReqs);
+    std::vector<MPI_Status> mpiStatuses(mpiRequests.size());
+
+    if (mpiRequests.size() > 1)
+    {
+        int mpiReturn = MPI_Waitall(static_cast<int>(mpiRequests.size()),
+                                    mpiRequests.data(), mpiStatuses.data());
+        if (mpiReturn == MPI_ERR_IN_STATUS)
+        {
+            for (auto &mpiStatus : mpiStatuses)
+            {
+                if (mpiStatus.MPI_ERROR != MPI_SUCCESS)
+                {
+                    mpiReturn = mpiStatus.MPI_ERROR;
+                    break;
+                }
+            }
+        }
+        CheckMPIReturn(mpiReturn, hint);
+    }
+    else
+    {
+        CheckMPIReturn(MPI_Wait(mpiRequests.data(), mpiStatuses.data()), hint);
+    }
+
+    // Our batched operation should be from only one source and have one tag.
+    status.Source = mpiStatuses.front().MPI_SOURCE;
+    status.Tag = mpiStatuses.front().MPI_TAG;
+
+    // Accumulate the total count of our batched operation.
+    for (auto &mpiStatus : mpiStatuses)
+    {
+        int mpiCount = 0;
+        CheckMPIReturn(MPI_Get_count(&mpiStatus, m_MPIDatatype, &mpiCount),
+                       hint);
+        status.Count += mpiCount;
+    }
+
+    // Our batched operation was cancelled if any member was cancelled.
+    for (auto &mpiStatus : mpiStatuses)
+    {
+        int mpiCancelled = 0;
+        MPI_Test_cancelled(&mpiStatus, &mpiCancelled);
+        if (mpiCancelled)
+        {
+            status.Cancelled = true;
+            break;
+        }
+    }
+#endif
+
+    return status;
+}
+
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 6d96d7c9fe..3a18844843 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -22,6 +22,9 @@ namespace helper
 class Comm
 {
 public:
+    class Req;
+    class Status;
+
     /**
      * @brief Default constructor.  Produces an empty communicator.
      *
@@ -179,6 +182,91 @@ class Comm
     MPI_Comm m_MPIComm = MPI_COMM_NULL;
 };
 
+class Comm::Req
+{
+public:
+    /**
+     * @brief Default constructor.  Produces an empty request.
+     *
+     * An empty request may not be used.
+     */
+    Req();
+
+    /**
+     * @brief Move constructor.  Moves request state from that given.
+     *
+     * The moved-from request is left empty and may not be used.
+     */
+    Req(Req &&);
+
+    /**
+     * @brief Deleted copy constructor.  A request may not be copied.
+     */
+    Req(Req const &) = delete;
+
+    ~Req();
+
+    /**
+     * @brief Move assignment.  Moves request state from that given.
+     *
+     * The moved-from request is left empty and may not be used.
+     */
+    Req &operator=(Req &&);
+
+    /**
+     * @brief Deleted copy assignment.  A request may not be copied.
+     */
+    Req &operator=(Req const &) = delete;
+
+    /**
+     * @brief Swap request state with another.
+     */
+    void swap(Req &req);
+
+    /**
+     * @brief Wait for the request to finish.
+     *
+     * On return, the request is empty.
+     */
+    Comm::Status Wait(const std::string &hint = std::string());
+
+private:
+    friend class Comm;
+
+    Req(MPI_Datatype datatype);
+
+    /** Encapsulated MPI datatype of the requested operation.  */
+    MPI_Datatype m_MPIDatatype = MPI_DATATYPE_NULL;
+
+    /** Encapsulated MPI request instances.  There may be more than
+     *  one when we batch requests too large for MPI interfaces.  */
+    std::vector<MPI_Request> m_MPIReqs;
+};
+
+class Comm::Status
+{
+public:
+    /**
+     * @brief The index of the process from which a message was received.
+     */
+    int Source = -1;
+
+    /**
+     * @brief The tag distinguishing the message for the application.
+     */
+    int Tag = -1;
+
+    /**
+     * @brief The number of elements received in a message.
+     */
+    size_t Count = 0;
+
+    /**
+     * @brief True if this is the status of a cancelled operation.
+     */
+    bool Cancelled = false;
+};
+
 } // end namespace helper
 } // end namespace adios2
 

From b1da13e683d54e83c21ef622cfbf2cf014d1a543 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 23 Jul 2019 14:05:20 -0400
Subject: [PATCH 28/42] helper: Add internal Comm method to look up MPI data
 type

---
 source/adios2/helper/adiosComm.h   |   4 ++
 source/adios2/helper/adiosComm.inl |  39 +++++++++++
 source/adios2/helper/adiosComm.tcc | 109 +++++++++++++++++++++++++++++
 3 files changed, 152 insertions(+)

diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 3a18844843..6653391ccc 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -180,6 +180,10 @@ class Comm
 
     /** Encapsulated MPI communicator instance.  */
     MPI_Comm m_MPIComm = MPI_COMM_NULL;
+
+    /** Return MPI datatype id for type T.  */
+    template <typename T>
+    static MPI_Datatype Datatype();
 };
 
 class Comm::Req
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index 7a482c504f..954f15ded5 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -13,6 +13,7 @@
 
 #include <numeric>   //std::accumulate
 #include <stdexcept> //std::runtime_error
+#include <utility>   //std::pair
 
 namespace adios2
 {
@@ -140,6 +141,44 @@ template <>
 void Comm::BroadcastVector(std::vector<size_t> &vector,
                            const int rankSource) const;
 
+// Datatype full specializations implemented in 'adiosComm.tcc'.
+template <>
+MPI_Datatype Comm::Datatype<signed char>();
+template <>
+MPI_Datatype Comm::Datatype<char>();
+template <>
+MPI_Datatype Comm::Datatype<short>();
+template <>
+MPI_Datatype Comm::Datatype<int>();
+template <>
+MPI_Datatype Comm::Datatype<long>();
+template <>
+MPI_Datatype Comm::Datatype<unsigned char>();
+template <>
+MPI_Datatype Comm::Datatype<unsigned short>();
+template <>
+MPI_Datatype Comm::Datatype<unsigned int>();
+template <>
+MPI_Datatype Comm::Datatype<unsigned long>();
+template <>
+MPI_Datatype Comm::Datatype<unsigned long long>();
+template <>
+MPI_Datatype Comm::Datatype<long long>();
+template <>
+MPI_Datatype Comm::Datatype<double>();
+template <>
+MPI_Datatype Comm::Datatype<long double>();
+template <>
+MPI_Datatype Comm::Datatype<std::pair<int, int>>();
+template <>
+MPI_Datatype Comm::Datatype<std::pair<float, int>>();
+template <>
+MPI_Datatype Comm::Datatype<std::pair<double, int>>();
+template <>
+MPI_Datatype Comm::Datatype<std::pair<long double, int>>();
+template <>
+MPI_Datatype Comm::Datatype<std::pair<short, int>>();
+
 } // end namespace helper
 } // end namespace adios2
 
diff --git a/source/adios2/helper/adiosComm.tcc b/source/adios2/helper/adiosComm.tcc
index 22e1953282..084386369a 100644
--- a/source/adios2/helper/adiosComm.tcc
+++ b/source/adios2/helper/adiosComm.tcc
@@ -301,6 +301,115 @@ void Comm::BroadcastVector(std::vector<size_t> &vector,
     }
 }
 
+// Datatype full specializations forward-declared in 'adiosComm.inl'.
+template <>
+MPI_Datatype Comm::Datatype<signed char>()
+{
+    return MPI_SIGNED_CHAR;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<char>()
+{
+    return MPI_CHAR;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<short>()
+{
+    return MPI_SHORT;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<int>()
+{
+    return MPI_INT;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<long>()
+{
+    return MPI_LONG;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<unsigned char>()
+{
+    return MPI_UNSIGNED_CHAR;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<unsigned short>()
+{
+    return MPI_UNSIGNED_SHORT;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<unsigned int>()
+{
+    return MPI_UNSIGNED;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<unsigned long>()
+{
+    return MPI_UNSIGNED_LONG;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<unsigned long long>()
+{
+    return MPI_UNSIGNED_LONG_LONG;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<long long>()
+{
+    return MPI_LONG_LONG_INT;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<double>()
+{
+    return MPI_DOUBLE;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<long double>()
+{
+    return MPI_LONG_DOUBLE;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<std::pair<int, int>>()
+{
+    return MPI_2INT;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<std::pair<float, int>>()
+{
+    return MPI_FLOAT_INT;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<std::pair<double, int>>()
+{
+    return MPI_DOUBLE_INT;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<std::pair<long double, int>>()
+{
+    return MPI_LONG_DOUBLE_INT;
+}
+
+template <>
+MPI_Datatype Comm::Datatype<std::pair<short, int>>()
+{
+    return MPI_SHORT_INT;
+}
+
 } // end namespace helper
 } // end namespace adios2
 

From 5c1b6a1eee533cd68dc62f1820e7d6c562960ee8 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 23 Jul 2019 11:46:11 -0400
Subject: [PATCH 29/42] helper: Move async send/recv functions to Comm
 encapsulation

---
 source/adios2/helper/adiosComm.cpp            |  99 ++++++++++++++
 source/adios2/helper/adiosComm.h              |  14 ++
 source/adios2/helper/adiosComm.inl            |  14 ++
 source/adios2/helper/adiosMPIFunctions.h      |  10 --
 source/adios2/helper/adiosMPIFunctions.tcc    |  90 +------------
 .../toolkit/aggregator/mpi/MPIAggregator.h    |  14 +-
 .../toolkit/aggregator/mpi/MPIChain.cpp       | 126 +++++++-----------
 7 files changed, 186 insertions(+), 181 deletions(-)

diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index 6eb7c223df..7e3a0f613d 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -96,6 +96,105 @@ std::string Comm::BroadcastFile(const std::string &fileName,
     return fileContents;
 }
 
+Comm::Req Comm::IsendImpl(const void *buffer, size_t count,
+                          MPI_Datatype datatype, int dest, int tag,
+                          const std::string &hint) const
+{
+    Comm::Req req(datatype);
+
+    if (count > DefaultMaxFileBatchSize)
+    {
+        const size_t batches = count / DefaultMaxFileBatchSize;
+
+        size_t position = 0;
+        for (size_t b = 0; b < batches; ++b)
+        {
+            int batchSize = static_cast<int>(DefaultMaxFileBatchSize);
+            MPI_Request mpiReq;
+            CheckMPIReturn(
+                MPI_Isend(static_cast<char *>(const_cast<void *>(buffer)) +
+                              position,
+                          batchSize, datatype, dest, tag, m_MPIComm, &mpiReq),
+                "in call to Isend batch " + std::to_string(b) + " " + hint +
+                    "\n");
+            req.m_MPIReqs.emplace_back(mpiReq);
+
+            position += DefaultMaxFileBatchSize;
+        }
+        const size_t remainder = count % DefaultMaxFileBatchSize;
+        if (remainder > 0)
+        {
+            int batchSize = static_cast<int>(remainder);
+            MPI_Request mpiReq;
+            CheckMPIReturn(
+                MPI_Isend(static_cast<char *>(const_cast<void *>(buffer)) +
+                              position,
+                          batchSize, datatype, dest, tag, m_MPIComm, &mpiReq),
+                "in call to Isend remainder batch " + hint + "\n");
+            req.m_MPIReqs.emplace_back(mpiReq);
+        }
+    }
+    else
+    {
+        int batchSize = static_cast<int>(count);
+        MPI_Request mpiReq;
+        CheckMPIReturn(
+            MPI_Isend(static_cast<char *>(const_cast<void *>(buffer)),
+                      batchSize, datatype, dest, tag, m_MPIComm, &mpiReq),
+            " in call to Isend with single batch " + hint + "\n");
+        req.m_MPIReqs.emplace_back(mpiReq);
+    }
+    return req;
+}
+
+Comm::Req Comm::IrecvImpl(void *buffer, size_t count, MPI_Datatype datatype,
+                          int source, int tag, const std::string &hint) const
+{
+    Comm::Req req(datatype);
+
+    if (count > DefaultMaxFileBatchSize)
+    {
+        const size_t batches = count / DefaultMaxFileBatchSize;
+        size_t position = 0;
+        for (size_t b = 0; b < batches; ++b)
+        {
+            int batchSize = static_cast<int>(DefaultMaxFileBatchSize);
+            MPI_Request mpiReq;
+            CheckMPIReturn(MPI_Irecv(static_cast<char *>(buffer) + position,
+                                     batchSize, datatype, source, tag,
+                                     m_MPIComm, &mpiReq),
+                           "in call to Irecv batch " + std::to_string(b) + " " +
+                               hint + "\n");
+            req.m_MPIReqs.emplace_back(mpiReq);
+
+            position += DefaultMaxFileBatchSize;
+        }
+
+        const size_t remainder = count % DefaultMaxFileBatchSize;
+        if (remainder > 0)
+        {
+            int batchSize = static_cast<int>(remainder);
+            MPI_Request mpiReq;
+            CheckMPIReturn(MPI_Irecv(static_cast<char *>(buffer) + position,
+                                     batchSize, datatype, source, tag,
+                                     m_MPIComm, &mpiReq),
+                           "in call to Irecv remainder batch " + hint + "\n");
+            req.m_MPIReqs.emplace_back(mpiReq);
+        }
+    }
+    else
+    {
+        int batchSize = static_cast<int>(count);
+        MPI_Request mpiReq;
+        CheckMPIReturn(MPI_Irecv(buffer, batchSize, datatype, source, tag,
+                                 m_MPIComm, &mpiReq),
+                       " in call to Isend with single batch " + hint + "\n");
+        req.m_MPIReqs.emplace_back(mpiReq);
+    }
+
+    return req;
+}
+
 Comm::Req::Req() = default;
 
 Comm::Req::Req(MPI_Datatype datatype) : m_MPIDatatype(datatype) {}
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 6653391ccc..161a0910d6 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -169,6 +169,14 @@ class Comm
                               const std::string hint = "",
                               const int rankSource = 0) const;
 
+    template <typename T>
+    Req Isend(const T *buffer, const size_t count, int destination, int tag,
+              const std::string &hint = std::string()) const;
+
+    template <typename T>
+    Req Irecv(T *buffer, const size_t count, int source, int tag,
+              const std::string &hint = std::string()) const;
+
 private:
     /**
      * @brief Construct by taking ownership of a MPI communicator.
@@ -181,6 +189,12 @@ class Comm
     /** Encapsulated MPI communicator instance.  */
     MPI_Comm m_MPIComm = MPI_COMM_NULL;
 
+    Req IsendImpl(const void *buffer, size_t count, MPI_Datatype datatype,
+                  int dest, int tag, const std::string &hint) const;
+
+    Req IrecvImpl(void *buffer, size_t count, MPI_Datatype datatype, int source,
+                  int tag, const std::string &hint) const;
+
     /** Return MPI datatype id for type T.  */
     template <typename T>
     static MPI_Datatype Datatype();
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index 954f15ded5..8557d44098 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -141,6 +141,20 @@ template <>
 void Comm::BroadcastVector(std::vector<size_t> &vector,
                            const int rankSource) const;
 
+template <typename T>
+Comm::Req Comm::Isend(const T *buffer, const size_t count, int dest, int tag,
+                      const std::string &hint) const
+{
+    return IsendImpl(buffer, count, Datatype<T>(), dest, tag, hint);
+}
+
+template <typename T>
+Comm::Req Comm::Irecv(T *buffer, const size_t count, int source, int tag,
+                      const std::string &hint) const
+{
+    return IrecvImpl(buffer, count, Datatype<T>(), source, tag, hint);
+}
+
 // Datatype full specializations implemented in 'adiosComm.tcc'.
 template <>
 MPI_Datatype Comm::Datatype<signed char>();
diff --git a/source/adios2/helper/adiosMPIFunctions.h b/source/adios2/helper/adiosMPIFunctions.h
index a95bade778..35d70c4d97 100644
--- a/source/adios2/helper/adiosMPIFunctions.h
+++ b/source/adios2/helper/adiosMPIFunctions.h
@@ -25,16 +25,6 @@ namespace helper
 
 void CheckMPIReturn(const int value, const std::string &hint);
 
-template <class T>
-std::vector<MPI_Request> Isend64(const T *buffer, const size_t count,
-                                 int destination, int tag, MPI_Comm mpiComm,
-                                 const std::string &hint);
-
-template <class T>
-std::vector<MPI_Request> Irecv64(T *buffer, const size_t count, int source,
-                                 int tag, MPI_Comm mpiComm,
-                                 const std::string &hint);
-
 } // end namespace helper
 } // end namespace adios2
 
diff --git a/source/adios2/helper/adiosMPIFunctions.tcc b/source/adios2/helper/adiosMPIFunctions.tcc
index dc0fdff40c..75c8997fb1 100644
--- a/source/adios2/helper/adiosMPIFunctions.tcc
+++ b/source/adios2/helper/adiosMPIFunctions.tcc
@@ -24,95 +24,7 @@ namespace adios2
 namespace helper
 {
 
-template <>
-std::vector<MPI_Request> Isend64<char>(const char *buffer, const size_t count,
-                                       int dest, int tag, MPI_Comm mpiComm,
-                                       const std::string &hint)
-{
-
-    std::vector<MPI_Request> requests(1);
-
-    if (count > DefaultMaxFileBatchSize)
-    {
-        const size_t batches = count / DefaultMaxFileBatchSize;
-        requests.resize(batches);
-
-        size_t position = 0;
-        for (size_t b = 0; b < batches; ++b)
-        {
-            int batchSize = static_cast<int>(DefaultMaxFileBatchSize);
-            CheckMPIReturn(MPI_Isend(const_cast<char *>(buffer + position),
-                                     batchSize, MPI_CHAR, dest, tag, mpiComm,
-                                     &requests[b]),
-                           "in call to Isend64 batch " + std::to_string(b) +
-                               " " + hint + "\n");
-
-            position += DefaultMaxFileBatchSize;
-        }
-        const size_t remainder = count % DefaultMaxFileBatchSize;
-        if (remainder > 0)
-        {
-            requests.resize(batches + 1);
-            int batchSize = static_cast<int>(remainder);
-            CheckMPIReturn(MPI_Isend(const_cast<char *>(buffer + position),
-                                     batchSize, MPI_CHAR, dest, tag, mpiComm,
-                                     &requests[batches]),
-                           "in call to Isend64 remainder batch " + hint + "\n");
-        }
-    }
-    else
-    {
-        int batchSize = static_cast<int>(count);
-        CheckMPIReturn(MPI_Isend(const_cast<char *>(buffer), batchSize,
-                                 MPI_CHAR, dest, tag, mpiComm, &requests[0]),
-                       " in call to Isend64 with single batch " + hint + "\n");
-    }
-
-    return requests;
-}
-
-template <>
-std::vector<MPI_Request> Irecv64<char>(char *buffer, const size_t count,
-                                       int source, int tag, MPI_Comm mpiComm,
-                                       const std::string &hint)
-{
-    std::vector<MPI_Request> requests(1);
-
-    if (count > DefaultMaxFileBatchSize)
-    {
-        const size_t batches = count / DefaultMaxFileBatchSize;
-        requests.resize(batches);
-        size_t position = 0;
-        for (size_t b = 0; b < batches; ++b)
-        {
-            int batchSize = static_cast<int>(DefaultMaxFileBatchSize);
-            CheckMPIReturn(MPI_Irecv(buffer + position, batchSize, MPI_CHAR,
-                                     source, tag, mpiComm, &requests[b]),
-                           "in call to Irecv64 batch " + std::to_string(b) +
-                               " " + hint + "\n");
-
-            position += DefaultMaxFileBatchSize;
-        }
-
-        const size_t remainder = count % DefaultMaxFileBatchSize;
-        if (remainder > 0)
-        {
-            requests.resize(batches + 1);
-            int batchSize = static_cast<int>(remainder);
-            CheckMPIReturn(MPI_Irecv(buffer + position, batchSize, MPI_CHAR,
-                                     source, tag, mpiComm, &requests[batches]),
-                           "in call to Irecv64 remainder batch " + hint + "\n");
-        }
-    }
-    else
-    {
-        int batchSize = static_cast<int>(count);
-        CheckMPIReturn(MPI_Irecv(buffer, batchSize, MPI_CHAR, source, tag,
-                                 mpiComm, &requests[0]),
-                       " in call to Isend64 with single batch " + hint + "\n");
-    }
-    return requests;
-}
+// placeholder
 
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h
index 9a93396ec9..7ab8ae876f 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h
+++ b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.h
@@ -60,13 +60,21 @@ class MPIAggregator
 
     virtual void Init(const size_t subStreams, helper::Comm const &parentComm);
 
-    using ExchangeRequests = std::vector<std::vector<MPI_Request>>;
+    struct ExchangeRequests
+    {
+        helper::Comm::Req m_SendSize;
+        helper::Comm::Req m_SendData;
+        helper::Comm::Req m_RecvData;
+    };
 
     virtual ExchangeRequests IExchange(format::Buffer &buffer,
                                        const int step) = 0;
 
-    using ExchangeAbsolutePositionRequests =
-        std::vector<std::vector<MPI_Request>>;
+    struct ExchangeAbsolutePositionRequests
+    {
+        helper::Comm::Req m_Send;
+        helper::Comm::Req m_Recv;
+    };
 
     virtual ExchangeAbsolutePositionRequests
     IExchangeAbsolutePosition(format::Buffer &buffer, const int step) = 0;
diff --git a/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp b/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp
index cf80c788b2..9cf24b17b8 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp
+++ b/source/adios2/toolkit/aggregator/mpi/MPIChain.cpp
@@ -10,7 +10,6 @@
 #include "MPIChain.h"
 
 #include "adios2/common/ADIOSMPI.h"
-#include "adios2/helper/adiosFunctions.h" //helper::CheckMPIReturn
 
 #include "adios2/toolkit/format/buffer/heap/BufferSTL.h"
 
@@ -47,47 +46,36 @@ MPIChain::ExchangeRequests MPIChain::IExchange(format::Buffer &buffer,
     const bool sender = (m_Rank >= 1 && m_Rank <= endRank) ? true : false;
     const bool receiver = (m_Rank < endRank) ? true : false;
 
-    std::vector<std::vector<MPI_Request>> requests(2);
+    ExchangeRequests requests;
 
     if (sender) // sender
     {
-        requests[0].resize(1);
-
-        helper::CheckMPIReturn(MPI_Isend(&sendBuffer.m_Position, 1,
-                                         ADIOS2_MPI_SIZE_T, m_Rank - 1, 0,
-                                         m_Comm, &requests[0][0]),
-                               ", aggregation Isend size at iteration " +
-                                   std::to_string(step) + "\n");
+        requests.m_SendSize =
+            m_Comm.Isend(&sendBuffer.m_Position, 1, m_Rank - 1, 0,
+                         ", aggregation Isend size at iteration " +
+                             std::to_string(step) + "\n");
 
         // only send data if buffer larger than 0
         if (sendBuffer.m_Position > 0)
         {
-
-            const std::vector<MPI_Request> requestsISend64 = helper::Isend64(
-                sendBuffer.Data(), sendBuffer.m_Position, m_Rank - 1, 1, m_Comm,
-                ", aggregation Isend64 data at iteration " +
+            requests.m_SendData = m_Comm.Isend(
+                sendBuffer.Data(), sendBuffer.m_Position, m_Rank - 1, 1,
+                ", aggregation Isend data at iteration " +
                     std::to_string(step));
-
-            requests[0].insert(requests[0].end(), requestsISend64.begin(),
-                               requestsISend64.end());
         }
     }
     // receive size, resize receiving buffer and receive data
     if (receiver)
     {
         size_t bufferSize = 0;
-        MPI_Request receiveSizeRequest;
-        helper::CheckMPIReturn(MPI_Irecv(&bufferSize, 1, ADIOS2_MPI_SIZE_T,
-                                         m_Rank + 1, 0, m_Comm,
-                                         &receiveSizeRequest),
-                               ", aggregation Irecv size at iteration " +
-                                   std::to_string(step) + "\n");
-
-        MPI_Status receiveStatus;
-        helper::CheckMPIReturn(
-            MPI_Wait(&receiveSizeRequest, &receiveStatus),
+        helper::Comm::Req receiveSizeRequest =
+            m_Comm.Irecv(&bufferSize, 1, m_Rank + 1, 0,
+                         ", aggregation Irecv size at iteration " +
+                             std::to_string(step) + "\n");
+
+        receiveSizeRequest.Wait(
             ", aggregation waiting for receiver size at iteration " +
-                std::to_string(step) + "\n");
+            std::to_string(step) + "\n");
 
         format::Buffer &receiveBuffer = GetReceiver(buffer);
         ResizeUpdateBuffer(
@@ -98,11 +86,10 @@ MPIChain::ExchangeRequests MPIChain::IExchange(format::Buffer &buffer,
         // only receive data if buffer is larger than 0
         if (bufferSize > 0)
         {
-            requests[1] =
-                helper::Irecv64(receiveBuffer.Data(), receiveBuffer.m_Position,
-                                m_Rank + 1, 1, m_Comm,
-                                ", aggregation Irecv64 data at iteration " +
-                                    std::to_string(step));
+            requests.m_RecvData = m_Comm.Irecv(
+                receiveBuffer.Data(), receiveBuffer.m_Position, m_Rank + 1, 1,
+                ", aggregation Irecv data at iteration " +
+                    std::to_string(step));
         }
     }
 
@@ -124,8 +111,7 @@ MPIChain::IExchangeAbsolutePosition(format::Buffer &buffer, const int step)
     }
 
     const int destination = (step != m_Size - 1) ? step + 1 : 0;
-    std::vector<std::vector<MPI_Request>> requests(2,
-                                                   std::vector<MPI_Request>(1));
+    ExchangeAbsolutePositionRequests requests;
 
     if (step == 0)
     {
@@ -138,19 +124,17 @@ MPIChain::IExchangeAbsolutePosition(format::Buffer &buffer, const int step)
         m_ExchangeAbsolutePosition =
             (m_Rank == 0) ? m_SizeSend : m_SizeSend + buffer.m_AbsolutePosition;
 
-        helper::CheckMPIReturn(
-            MPI_Isend(&m_ExchangeAbsolutePosition, 1, ADIOS2_MPI_SIZE_T,
-                      destination, 0, m_Comm, &requests[0][0]),
-            ", aggregation Isend absolute position at iteration " +
-                std::to_string(step) + "\n");
+        requests.m_Send =
+            m_Comm.Isend(&m_ExchangeAbsolutePosition, 1, destination, 0,
+                         ", aggregation Isend absolute position at iteration " +
+                             std::to_string(step) + "\n");
     }
     else if (m_Rank == destination)
     {
-        helper::CheckMPIReturn(
-            MPI_Irecv(&buffer.m_AbsolutePosition, 1, ADIOS2_MPI_SIZE_T, step, 0,
-                      m_Comm, &requests[1][0]),
-            ", aggregation Irecv absolute position at iteration " +
-                std::to_string(step) + "\n");
+        requests.m_Recv =
+            m_Comm.Irecv(&buffer.m_AbsolutePosition, 1, step, 0,
+                         ", aggregation Irecv absolute position at iteration " +
+                             std::to_string(step) + "\n");
     }
 
     m_IsInExchangeAbsolutePosition = true;
@@ -168,27 +152,20 @@ void MPIChain::Wait(ExchangeRequests &requests, const int step)
     const bool sender = (m_Rank >= 1 && m_Rank <= endRank) ? true : false;
     const bool receiver = (m_Rank < endRank) ? true : false;
 
-    MPI_Status status;
     if (receiver)
     {
-        for (auto &req : requests[1])
-        {
-            helper::CheckMPIReturn(
-                MPI_Wait(&req, &status),
-                ", aggregation waiting for receiver request at iteration " +
-                    std::to_string(step) + "\n");
-        }
+        requests.m_RecvData.Wait(
+            ", aggregation waiting for receiver request at iteration " +
+            std::to_string(step) + "\n");
     }
 
     if (sender)
     {
-        for (auto &req : requests[0])
-        {
-            helper::CheckMPIReturn(
-                MPI_Wait(&req, &status),
-                ", aggregation waiting for sender request at iteration " +
-                    std::to_string(step) + "\n");
-        }
+        const std::string hint =
+            ", aggregation waiting for sender request at iteration " +
+            std::to_string(step) + "\n";
+        requests.m_SendSize.Wait(hint);
+        requests.m_SendData.Wait(hint);
     }
 }
 
@@ -206,23 +183,20 @@ void MPIChain::WaitAbsolutePosition(ExchangeAbsolutePositionRequests &requests,
                                  "existing exchange is not active.");
     }
 
-    MPI_Status status;
     const int destination = (step != m_Size - 1) ? step + 1 : 0;
 
     if (m_Rank == destination)
     {
-        helper::CheckMPIReturn(
-            MPI_Wait(&requests[1][0], &status),
+        requests.m_Recv.Wait(
             ", aggregation Irecv Wait absolute position at iteration " +
-                std::to_string(step) + "\n");
+            std::to_string(step) + "\n");
     }
 
     if (m_Rank == step)
     {
-        helper::CheckMPIReturn(
-            MPI_Wait(&requests[0][0], &status),
+        requests.m_Send.Wait(
             ", aggregation Isend Wait absolute position at iteration " +
-                std::to_string(step) + "\n");
+            std::to_string(step) + "\n");
     }
     m_IsInExchangeAbsolutePosition = false;
 }
@@ -244,33 +218,27 @@ void MPIChain::HandshakeLinks()
 {
     int link = -1;
 
-    MPI_Request sendRequest;
+    helper::Comm::Req sendRequest;
     if (m_Rank > 0) // send
     {
-        helper::CheckMPIReturn(
-            MPI_Isend(&m_Rank, 1, MPI_INT, m_Rank - 1, 0, m_Comm, &sendRequest),
+        sendRequest = m_Comm.Isend(
+            &m_Rank, 1, m_Rank - 1, 0,
             "Isend handshake with neighbor, MPIChain aggregator, at Open");
     }
 
     if (m_Rank < m_Size - 1) // receive
     {
-        MPI_Request receiveRequest;
-        helper::CheckMPIReturn(
-            MPI_Irecv(&link, 1, MPI_INT, m_Rank + 1, 0, m_Comm,
-                      &receiveRequest),
+        helper::Comm::Req receiveRequest = m_Comm.Irecv(
+            &link, 1, m_Rank + 1, 0,
             "Irecv handshake with neighbor, MPIChain aggregator, at Open");
 
-        MPI_Status receiveStatus;
-        helper::CheckMPIReturn(
-            MPI_Wait(&receiveRequest, &receiveStatus),
+        receiveRequest.Wait(
             "Irecv Wait handshake with neighbor, MPIChain aggregator, at Open");
     }
 
     if (m_Rank > 0)
     {
-        MPI_Status sendStatus;
-        helper::CheckMPIReturn(
-            MPI_Wait(&sendRequest, &sendStatus),
+        sendRequest.Wait(
             "Isend wait handshake with neighbor, MPIChain aggregator, at Open");
     }
 }

From b3876c6329729c723f0c2bcd980177f56ab995aa Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 23 Jul 2019 14:23:20 -0400
Subject: [PATCH 30/42] helper: Encapsulate MPI_Bcast as Comm::Bcast

---
 source/adios2/engine/insitumpi/InSituMPIReader.cpp     | 10 +++++-----
 source/adios2/engine/insitumpi/InSituMPIWriter.cpp     |  4 ++--
 source/adios2/helper/adiosComm.cpp                     |  8 ++++++++
 source/adios2/helper/adiosComm.h                       |  7 +++++++
 source/adios2/helper/adiosComm.inl                     |  7 +++++++
 source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp |  3 +--
 6 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.cpp b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
index fb450abe7c..7e0863e6e4 100644
--- a/source/adios2/engine/insitumpi/InSituMPIReader.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
@@ -245,10 +245,10 @@ StepStatus InSituMPIReader::BeginStep(const StepMode mode,
         }
 
         // broadcast metadata to every reader
-        MPI_Bcast(&mdLen, 1, MPI_UNSIGNED_LONG, m_ReaderRootRank, m_Comm);
+        m_Comm.Bcast(&mdLen, 1, m_ReaderRootRank);
         m_BP3Deserializer.m_Metadata.m_Buffer.resize(mdLen);
-        MPI_Bcast(m_BP3Deserializer.m_Metadata.m_Buffer.data(), mdLen, MPI_CHAR,
-                  m_ReaderRootRank, m_Comm);
+        m_Comm.Bcast(m_BP3Deserializer.m_Metadata.m_Buffer.data(), mdLen,
+                     m_ReaderRootRank);
 
         // Parse metadata into Variables and Attributes maps
         m_IO.RemoveAllVariables();
@@ -277,7 +277,7 @@ StepStatus InSituMPIReader::BeginStep(const StepMode mode,
         }
 
         // broadcast fixed schedule flag to every reader
-        MPI_Bcast(&fixed, 1, MPI_INT, m_ReaderRootRank, m_Comm);
+        m_Comm.Bcast(&fixed, 1, m_ReaderRootRank);
         m_RemoteDefinitionsLocked = (fixed ? true : false);
         if (m_ReaderRootRank == m_ReaderRank)
         {
@@ -510,7 +510,7 @@ void InSituMPIReader::ProcessReceives()
 
     // Send final acknowledgment to the Writer
     int dummy = 1;
-    MPI_Bcast(&dummy, 1, MPI_INT, m_ReaderRootRank, m_Comm);
+    m_Comm.Bcast(&dummy, 1, m_ReaderRootRank);
     if (m_ReaderRootRank == m_ReaderRank)
     {
         MPI_Send(&dummy, 1, MPI_INT, m_WriteRootGlobalRank,
diff --git a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
index 4d6967340b..cf75ba0f97 100644
--- a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
@@ -219,7 +219,7 @@ void InSituMPIWriter::PerformPuts()
                          &status);
             }
             // broadcast fixed schedule flag to every reader
-            MPI_Bcast(&fixed, 1, MPI_INT, 0, m_Comm);
+            m_Comm.Bcast(&fixed, 1, 0);
             m_RemoteDefinitionsLocked = (fixed ? true : false);
             if (m_BP3Serializer.m_RankMPI == 0)
             {
@@ -327,7 +327,7 @@ void InSituMPIWriter::EndStep()
         MPI_Recv(&dummy, 1, MPI_INT, m_RankDirectPeers[0],
                  insitumpi::MpiTags::ReadCompleted, m_CommWorld, &status);
     }
-    MPI_Bcast(&dummy, 1, MPI_INT, 0, m_Comm);
+    m_Comm.Bcast(&dummy, 1, 0);
     TAU_STOP("WaitForReaderAck");
 
     if (m_Verbosity == 5)
diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index 7e3a0f613d..840ddcb2c4 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -96,6 +96,14 @@ std::string Comm::BroadcastFile(const std::string &fileName,
     return fileContents;
 }
 
+void Comm::BcastImpl(void *buffer, size_t count, MPI_Datatype datatype,
+                     int root, const std::string &hint) const
+{
+    CheckMPIReturn(
+        SMPI_Bcast(buffer, static_cast<int>(count), datatype, root, m_MPIComm),
+        hint);
+}
+
 Comm::Req Comm::IsendImpl(const void *buffer, size_t count,
                           MPI_Datatype datatype, int dest, int tag,
                           const std::string &hint) const
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 161a0910d6..60f4417e98 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -169,6 +169,10 @@ class Comm
                               const std::string hint = "",
                               const int rankSource = 0) const;
 
+    template <typename T>
+    void Bcast(T *buffer, size_t count, int root,
+               const std::string &hint = std::string()) const;
+
     template <typename T>
     Req Isend(const T *buffer, const size_t count, int destination, int tag,
               const std::string &hint = std::string()) const;
@@ -189,6 +193,9 @@ class Comm
     /** Encapsulated MPI communicator instance.  */
     MPI_Comm m_MPIComm = MPI_COMM_NULL;
 
+    void BcastImpl(void *buffer, size_t count, MPI_Datatype datatype, int root,
+                   const std::string &hint) const;
+
     Req IsendImpl(const void *buffer, size_t count, MPI_Datatype datatype,
                   int dest, int tag, const std::string &hint) const;
 
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index 8557d44098..46d6362a75 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -141,6 +141,13 @@ template <>
 void Comm::BroadcastVector(std::vector<size_t> &vector,
                            const int rankSource) const;
 
+template <typename T>
+void Comm::Bcast(T *buffer, const size_t count, int root,
+                 const std::string &hint) const
+{
+    return BcastImpl(buffer, count, Datatype<T>(), root, hint);
+}
+
 template <typename T>
 Comm::Req Comm::Isend(const T *buffer, const size_t count, int dest, int tag,
                       const std::string &hint) const
diff --git a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
index 1c38a6e739..d673a95295 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
+++ b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
@@ -106,8 +106,7 @@ void MPIAggregator::HandshakeRank(const int rank)
         message = m_Rank;
     }
 
-    helper::CheckMPIReturn(MPI_Bcast(&message, 1, MPI_INT, rank, m_Comm),
-                           "handshake with aggregator rank 0 at Open");
+    m_Comm.Bcast(&message, 1, rank, "handshake with aggregator rank 0 at Open");
 }
 
 } // end namespace aggregator

From d0a2fb6e78ef9f1da71776c9e31314d76b59f9f6 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 23 Jul 2019 14:52:09 -0400
Subject: [PATCH 31/42] toolkit: Port transport and transportman to Comm
 encapsulation

---
 source/adios2/toolkit/transport/Transport.cpp |  4 ++--
 source/adios2/toolkit/transport/Transport.h   |  7 ++++---
 .../toolkit/transport/file/FileFStream.cpp    |  4 ++--
 .../toolkit/transport/file/FileFStream.h      |  3 ++-
 .../toolkit/transport/file/FilePOSIX.cpp      |  4 ++--
 .../adios2/toolkit/transport/file/FilePOSIX.h |  6 +++++-
 .../toolkit/transport/file/FileStdio.cpp      |  4 ++--
 .../adios2/toolkit/transport/file/FileStdio.h |  6 +++++-
 .../toolkit/transport/null/NullTransport.cpp  |  4 ++--
 .../toolkit/transport/null/NullTransport.h    |  6 +++++-
 .../toolkit/transport/shm/ShmSystemV.cpp      |  4 ++--
 .../adios2/toolkit/transport/shm/ShmSystemV.h |  6 +++++-
 .../toolkit/transportman/TransportMan.cpp     | 20 +++++++++----------
 .../toolkit/transportman/TransportMan.h       | 10 +++++++---
 14 files changed, 55 insertions(+), 33 deletions(-)

diff --git a/source/adios2/toolkit/transport/Transport.cpp b/source/adios2/toolkit/transport/Transport.cpp
index 801ca3a24d..26ac1b2e75 100644
--- a/source/adios2/toolkit/transport/Transport.cpp
+++ b/source/adios2/toolkit/transport/Transport.cpp
@@ -17,8 +17,8 @@ namespace adios2
 {
 
 Transport::Transport(const std::string type, const std::string library,
-                     MPI_Comm mpiComm, const bool debugMode)
-: m_Type(type), m_Library(library), m_MPIComm(mpiComm), m_DebugMode(debugMode)
+                     helper::Comm const &comm, const bool debugMode)
+: m_Type(type), m_Library(library), m_Comm(comm), m_DebugMode(debugMode)
 {
 }
 
diff --git a/source/adios2/toolkit/transport/Transport.h b/source/adios2/toolkit/transport/Transport.h
index 8f44a14a21..b715f54a7f 100644
--- a/source/adios2/toolkit/transport/Transport.h
+++ b/source/adios2/toolkit/transport/Transport.h
@@ -19,6 +19,7 @@
 #include "adios2/common/ADIOSConfig.h"
 #include "adios2/common/ADIOSMPI.h"
 #include "adios2/common/ADIOSTypes.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/profiling/iochrono/IOChrono.h"
 
 namespace adios2
@@ -33,7 +34,7 @@ class Transport
     std::string m_Name; ///< from Open, unique identifier (e.g. filename)
     Mode m_OpenMode = Mode::Undefined; ///< at Open from ADIOSTypes.h
     bool m_IsOpen = false; ///< true: open for communication, false: unreachable
-    MPI_Comm m_MPIComm;    ///< current MPI communicator
+    helper::Comm const &m_Comm;     ///< current multi-process communicator
     profiling::IOChrono m_Profiler; ///< profiles Open, Write/Read, Close
 
     struct Status
@@ -47,11 +48,11 @@ class Transport
     /**
      * Base constructor that all derived classes pass
      * @param type from derived class
-     * @param mpiComm passed to m_MPIComm
+     * @param comm passed to m_Comm
      * @param debugMode passed to m_DebugMode
      */
     Transport(const std::string type, const std::string library,
-              MPI_Comm mpiComm, const bool debugMode);
+              helper::Comm const &comm, const bool debugMode);
 
     virtual ~Transport() = default;
 
diff --git a/source/adios2/toolkit/transport/file/FileFStream.cpp b/source/adios2/toolkit/transport/file/FileFStream.cpp
index b372e50e69..3269745190 100644
--- a/source/adios2/toolkit/transport/file/FileFStream.cpp
+++ b/source/adios2/toolkit/transport/file/FileFStream.cpp
@@ -18,8 +18,8 @@ namespace adios2
 namespace transport
 {
 
-FileFStream::FileFStream(MPI_Comm mpiComm, const bool debugMode)
-: Transport("File", "fstream", mpiComm, debugMode)
+FileFStream::FileFStream(helper::Comm const &comm, const bool debugMode)
+: Transport("File", "fstream", comm, debugMode)
 {
 }
 
diff --git a/source/adios2/toolkit/transport/file/FileFStream.h b/source/adios2/toolkit/transport/file/FileFStream.h
index c74ed45aff..a83af203b2 100644
--- a/source/adios2/toolkit/transport/file/FileFStream.h
+++ b/source/adios2/toolkit/transport/file/FileFStream.h
@@ -14,6 +14,7 @@
 #include <fstream>
 
 #include "adios2/common/ADIOSConfig.h"
+#include "adios2/helper/adiosComm.h"
 #include "adios2/toolkit/transport/Transport.h"
 
 namespace adios2
@@ -26,7 +27,7 @@ class FileFStream : public Transport
 {
 
 public:
-    FileFStream(MPI_Comm mpiComm, const bool debugMode);
+    FileFStream(helper::Comm const &comm, const bool debugMode);
 
     ~FileFStream() = default;
 
diff --git a/source/adios2/toolkit/transport/file/FilePOSIX.cpp b/source/adios2/toolkit/transport/file/FilePOSIX.cpp
index d5a45fa08b..711c530390 100644
--- a/source/adios2/toolkit/transport/file/FilePOSIX.cpp
+++ b/source/adios2/toolkit/transport/file/FilePOSIX.cpp
@@ -24,8 +24,8 @@ namespace adios2
 namespace transport
 {
 
-FilePOSIX::FilePOSIX(MPI_Comm mpiComm, const bool debugMode)
-: Transport("File", "POSIX", mpiComm, debugMode)
+FilePOSIX::FilePOSIX(helper::Comm const &comm, const bool debugMode)
+: Transport("File", "POSIX", comm, debugMode)
 {
 }
 
diff --git a/source/adios2/toolkit/transport/file/FilePOSIX.h b/source/adios2/toolkit/transport/file/FilePOSIX.h
index 445bf431d0..33f0b558b2 100644
--- a/source/adios2/toolkit/transport/file/FilePOSIX.h
+++ b/source/adios2/toolkit/transport/file/FilePOSIX.h
@@ -16,6 +16,10 @@
 
 namespace adios2
 {
+namespace helper
+{
+class Comm;
+}
 namespace transport
 {
 
@@ -24,7 +28,7 @@ class FilePOSIX : public Transport
 {
 
 public:
-    FilePOSIX(MPI_Comm mpiComm, const bool debugMode);
+    FilePOSIX(helper::Comm const &comm, const bool debugMode);
 
     ~FilePOSIX();
 
diff --git a/source/adios2/toolkit/transport/file/FileStdio.cpp b/source/adios2/toolkit/transport/file/FileStdio.cpp
index dd330237e9..f95fed4511 100644
--- a/source/adios2/toolkit/transport/file/FileStdio.cpp
+++ b/source/adios2/toolkit/transport/file/FileStdio.cpp
@@ -24,8 +24,8 @@ namespace adios2
 namespace transport
 {
 
-FileStdio::FileStdio(MPI_Comm mpiComm, const bool debugMode)
-: Transport("File", "stdio", mpiComm, debugMode)
+FileStdio::FileStdio(helper::Comm const &comm, const bool debugMode)
+: Transport("File", "stdio", comm, debugMode)
 {
 }
 
diff --git a/source/adios2/toolkit/transport/file/FileStdio.h b/source/adios2/toolkit/transport/file/FileStdio.h
index 50f007312f..9b828df9d5 100644
--- a/source/adios2/toolkit/transport/file/FileStdio.h
+++ b/source/adios2/toolkit/transport/file/FileStdio.h
@@ -17,6 +17,10 @@
 
 namespace adios2
 {
+namespace helper
+{
+class Comm;
+}
 namespace transport
 {
 
@@ -25,7 +29,7 @@ class FileStdio : public Transport
 {
 
 public:
-    FileStdio(MPI_Comm mpiComm, const bool debugMode);
+    FileStdio(helper::Comm const &comm, const bool debugMode);
 
     ~FileStdio();
 
diff --git a/source/adios2/toolkit/transport/null/NullTransport.cpp b/source/adios2/toolkit/transport/null/NullTransport.cpp
index 0e480504fc..be6c154a9a 100644
--- a/source/adios2/toolkit/transport/null/NullTransport.cpp
+++ b/source/adios2/toolkit/transport/null/NullTransport.cpp
@@ -24,8 +24,8 @@ struct NullTransport::NullTransportImpl
     size_t Capacity = 0;
 };
 
-NullTransport::NullTransport(MPI_Comm mpiComm, const bool debugMode)
-: Transport("NULL", "NULL", mpiComm, debugMode), Impl(new NullTransportImpl)
+NullTransport::NullTransport(helper::Comm const &comm, const bool debugMode)
+: Transport("NULL", "NULL", comm, debugMode), Impl(new NullTransportImpl)
 {
 }
 
diff --git a/source/adios2/toolkit/transport/null/NullTransport.h b/source/adios2/toolkit/transport/null/NullTransport.h
index 268cc11fd6..2556488396 100644
--- a/source/adios2/toolkit/transport/null/NullTransport.h
+++ b/source/adios2/toolkit/transport/null/NullTransport.h
@@ -18,6 +18,10 @@
 
 namespace adios2
 {
+namespace helper
+{
+class Comm;
+}
 namespace transport
 {
 
@@ -26,7 +30,7 @@ class NullTransport : public Transport
 {
 
 public:
-    NullTransport(MPI_Comm mpiComm, const bool debugMode);
+    NullTransport(helper::Comm const &comm, const bool debugMode);
 
     virtual ~NullTransport();
 
diff --git a/source/adios2/toolkit/transport/shm/ShmSystemV.cpp b/source/adios2/toolkit/transport/shm/ShmSystemV.cpp
index 34b7e0d201..bc1b597bb6 100644
--- a/source/adios2/toolkit/transport/shm/ShmSystemV.cpp
+++ b/source/adios2/toolkit/transport/shm/ShmSystemV.cpp
@@ -22,9 +22,9 @@ namespace transport
 {
 
 ShmSystemV::ShmSystemV(const unsigned int projectID, const size_t size,
-                       MPI_Comm mpiComm, const bool debugMode,
+                       helper::Comm const &comm, const bool debugMode,
                        const bool removeAtClose)
-: Transport("Shm", "SystemV", mpiComm, debugMode), m_ProjectID(projectID),
+: Transport("Shm", "SystemV", comm, debugMode), m_ProjectID(projectID),
   m_Size(size), m_RemoveAtClose(removeAtClose)
 {
     if (m_DebugMode && projectID == 0)
diff --git a/source/adios2/toolkit/transport/shm/ShmSystemV.h b/source/adios2/toolkit/transport/shm/ShmSystemV.h
index 305edd873e..c21bbbff47 100644
--- a/source/adios2/toolkit/transport/shm/ShmSystemV.h
+++ b/source/adios2/toolkit/transport/shm/ShmSystemV.h
@@ -15,6 +15,10 @@
 
 namespace adios2
 {
+namespace helper
+{
+class Comm;
+}
 namespace transport
 {
 
@@ -31,7 +35,7 @@ class ShmSystemV : public Transport
      * @param debugMode true: extra checks
      */
     ShmSystemV(const unsigned int projectID, const size_t size,
-               MPI_Comm mpiComm, const bool debugMode = false,
+               helper::Comm const &comm, const bool debugMode = false,
                const bool removeAtClose = false);
 
     ~ShmSystemV();
diff --git a/source/adios2/toolkit/transportman/TransportMan.cpp b/source/adios2/toolkit/transportman/TransportMan.cpp
index 36e90dcd72..6c947efc28 100644
--- a/source/adios2/toolkit/transportman/TransportMan.cpp
+++ b/source/adios2/toolkit/transportman/TransportMan.cpp
@@ -32,8 +32,8 @@ namespace adios2
 namespace transportman
 {
 
-TransportMan::TransportMan(MPI_Comm mpiComm, const bool debugMode)
-: m_MPIComm(mpiComm), m_DebugMode(debugMode)
+TransportMan::TransportMan(helper::Comm const &comm, const bool debugMode)
+: m_Comm(comm), m_DebugMode(debugMode)
 {
 }
 
@@ -61,13 +61,13 @@ void TransportMan::MkDirsBarrier(const std::vector<std::string> &fileNames,
     else
     {
         int rank;
-        SMPI_Comm_rank(m_MPIComm, &rank);
+        SMPI_Comm_rank(m_Comm, &rank);
         if (rank == 0)
         {
             lf_CreateDirectories(fileNames);
         }
 
-        helper::CheckMPIReturn(SMPI_Barrier(m_MPIComm),
+        helper::CheckMPIReturn(SMPI_Barrier(m_Comm),
                                "Barrier in TransportMan.MkDirsBarrier");
     }
 }
@@ -339,24 +339,24 @@ TransportMan::OpenFileTransport(const std::string &fileName,
         if (library == "stdio")
         {
             transport =
-                std::make_shared<transport::FileStdio>(m_MPIComm, m_DebugMode);
+                std::make_shared<transport::FileStdio>(m_Comm, m_DebugMode);
         }
         else if (library == "fstream")
         {
-            transport = std::make_shared<transport::FileFStream>(m_MPIComm,
-                                                                 m_DebugMode);
+            transport =
+                std::make_shared<transport::FileFStream>(m_Comm, m_DebugMode);
         }
 #ifndef _WIN32
         else if (library == "POSIX" || library == "posix")
         {
             transport =
-                std::make_shared<transport::FilePOSIX>(m_MPIComm, m_DebugMode);
+                std::make_shared<transport::FilePOSIX>(m_Comm, m_DebugMode);
         }
 #endif
         else if (library == "NULL" || library == "null")
         {
-            transport = std::make_shared<transport::NullTransport>(m_MPIComm,
-                                                                   m_DebugMode);
+            transport =
+                std::make_shared<transport::NullTransport>(m_Comm, m_DebugMode);
         }
         else
         {
diff --git a/source/adios2/toolkit/transportman/TransportMan.h b/source/adios2/toolkit/transportman/TransportMan.h
index 5a5e3312b8..cb10a8b006 100644
--- a/source/adios2/toolkit/transportman/TransportMan.h
+++ b/source/adios2/toolkit/transportman/TransportMan.h
@@ -21,6 +21,10 @@
 
 namespace adios2
 {
+namespace helper
+{
+class Comm;
+}
 namespace transportman
 {
 
@@ -39,10 +43,10 @@ class TransportMan
 
     /**
      * Unique base constructor
-     * @param mpiComm
+     * @param comm
      * @param debugMode
      */
-    TransportMan(MPI_Comm mpiComm, const bool debugMode);
+    TransportMan(helper::Comm const &comm, const bool debugMode);
 
     virtual ~TransportMan() = default;
 
@@ -169,7 +173,7 @@ class TransportMan
     void SeekToFileBegin(const int transportIndex = 0);
 
 protected:
-    MPI_Comm m_MPIComm;
+    helper::Comm const &m_Comm;
     const bool m_DebugMode = false;
 
     std::shared_ptr<Transport> OpenFileTransport(const std::string &fileName,

From c6e6758fa714e2b33861eaff83e88cf0dae837bc Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 23 Jul 2019 14:44:35 -0400
Subject: [PATCH 32/42] helper: Encapsulate MPI_Barrier as Comm::Barrier

---
 source/adios2/engine/ssc/SscWriter.cpp              | 2 +-
 source/adios2/engine/table/TableWriter.cpp          | 2 +-
 source/adios2/helper/adiosComm.cpp                  | 5 +++++
 source/adios2/helper/adiosComm.h                    | 2 ++
 source/adios2/toolkit/transportman/TransportMan.cpp | 3 +--
 5 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp
index 7fdf3e398a..60fbff39e3 100644
--- a/source/adios2/engine/ssc/SscWriter.cpp
+++ b/source/adios2/engine/ssc/SscWriter.cpp
@@ -231,7 +231,7 @@ void SscWriter::ReplyThread(const std::string &address)
 
 void SscWriter::DoClose(const int transportIndex)
 {
-    MPI_Barrier(m_Comm);
+    m_Comm.Barrier();
     m_Listening = false;
     for (auto &i : m_ReplyThreads)
     {
diff --git a/source/adios2/engine/table/TableWriter.cpp b/source/adios2/engine/table/TableWriter.cpp
index c99717d0b4..5649b2856e 100644
--- a/source/adios2/engine/table/TableWriter.cpp
+++ b/source/adios2/engine/table/TableWriter.cpp
@@ -83,7 +83,7 @@ void TableWriter::EndStep()
         }
     }
 
-    MPI_Barrier(m_Comm);
+    m_Comm.Barrier();
 
     m_Listening = false;
     if (m_Verbosity >= 5)
diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index 840ddcb2c4..d7ba389cbe 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -77,6 +77,11 @@ Comm Comm::Split(int color, int key, const std::string &hint) const
     return Comm(newComm);
 }
 
+void Comm::Barrier(const std::string &hint) const
+{
+    CheckMPIReturn(SMPI_Barrier(m_MPIComm), hint);
+}
+
 std::string Comm::BroadcastFile(const std::string &fileName,
                                 const std::string hint,
                                 const int rankSource) const
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 60f4417e98..06729eb894 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -94,6 +94,8 @@ class Comm
     Comm Split(int color, int key,
                const std::string &hint = std::string()) const;
 
+    void Barrier(const std::string &hint = std::string()) const;
+
     /**
      * Gather a single source value from each ranks and forms a vector in
      * rankDestination.
diff --git a/source/adios2/toolkit/transportman/TransportMan.cpp b/source/adios2/toolkit/transportman/TransportMan.cpp
index 6c947efc28..124856744e 100644
--- a/source/adios2/toolkit/transportman/TransportMan.cpp
+++ b/source/adios2/toolkit/transportman/TransportMan.cpp
@@ -67,8 +67,7 @@ void TransportMan::MkDirsBarrier(const std::vector<std::string> &fileNames,
             lf_CreateDirectories(fileNames);
         }
 
-        helper::CheckMPIReturn(SMPI_Barrier(m_Comm),
-                               "Barrier in TransportMan.MkDirsBarrier");
+        m_Comm.Barrier("Barrier in TransportMan.MkDirsBarrier");
     }
 }
 

From ba930318a052e7b302404c7d9901b9165f3c04cf Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 23 Jul 2019 14:55:58 -0400
Subject: [PATCH 33/42] helper: Move CheckMPIReturn helper into Comm
 encapsulation

Make it a private implementation detail.
---
 source/adios2/helper/adiosComm.cpp         | 25 ++++++++++++++++++++--
 source/adios2/helper/adiosComm.h           |  2 ++
 source/adios2/helper/adiosMPIFunctions.cpp | 25 +---------------------
 source/adios2/helper/adiosMPIFunctions.h   |  4 +---
 4 files changed, 27 insertions(+), 29 deletions(-)

diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index d7ba389cbe..05165972d8 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -14,8 +14,6 @@
 #include "adios2/common/ADIOSMPI.h"
 #include "adios2/helper/adiosString.h"
 
-#include "adiosMPIFunctions.h"
-
 namespace adios2
 {
 namespace helper
@@ -61,6 +59,29 @@ Comm Comm::Duplicate(MPI_Comm mpiComm)
     return Comm(newComm);
 }
 
+void Comm::CheckMPIReturn(const int value, const std::string &hint)
+{
+    if (value == MPI_SUCCESS)
+    {
+        return;
+    }
+
+    std::string error;
+    switch (value)
+    {
+    case MPI_ERR_COMM:
+        error = "MPI_ERR_COMM";
+        break;
+    case MPI_ERR_INTERN:
+        error = "MPI_ERR_INTERN";
+        break;
+    default:
+        error = "MPI_ERR number: " + std::to_string(value);
+    }
+
+    throw std::runtime_error("ERROR: ADIOS2 detected " + error + ", " + hint);
+}
+
 void Comm::Free(const std::string &hint)
 {
     if (m_MPIComm != MPI_COMM_NULL && m_MPIComm != MPI_COMM_WORLD &&
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 06729eb894..1ea4051a3e 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -195,6 +195,8 @@ class Comm
     /** Encapsulated MPI communicator instance.  */
     MPI_Comm m_MPIComm = MPI_COMM_NULL;
 
+    static void CheckMPIReturn(const int value, const std::string &hint);
+
     void BcastImpl(void *buffer, size_t count, MPI_Datatype datatype, int root,
                    const std::string &hint) const;
 
diff --git a/source/adios2/helper/adiosMPIFunctions.cpp b/source/adios2/helper/adiosMPIFunctions.cpp
index 47c017d2f4..e3322281fd 100644
--- a/source/adios2/helper/adiosMPIFunctions.cpp
+++ b/source/adios2/helper/adiosMPIFunctions.cpp
@@ -19,29 +19,6 @@ namespace adios2
 {
 namespace helper
 {
-
-void CheckMPIReturn(const int value, const std::string &hint)
-{
-    if (value == MPI_SUCCESS)
-    {
-        return;
-    }
-
-    std::string error;
-    switch (value)
-    {
-    case MPI_ERR_COMM:
-        error = "MPI_ERR_COMM";
-        break;
-    case MPI_ERR_INTERN:
-        error = "MPI_ERR_INTERN";
-        break;
-    default:
-        error = "MPI_ERR number: " + std::to_string(value);
-    }
-
-    throw std::runtime_error("ERROR: ADIOS2 detected " + error + ", " + hint);
-}
-
+// placeholder
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/helper/adiosMPIFunctions.h b/source/adios2/helper/adiosMPIFunctions.h
index 35d70c4d97..14744038ee 100644
--- a/source/adios2/helper/adiosMPIFunctions.h
+++ b/source/adios2/helper/adiosMPIFunctions.h
@@ -22,9 +22,7 @@ namespace adios2
 {
 namespace helper
 {
-
-void CheckMPIReturn(const int value, const std::string &hint);
-
+// placeholder
 } // end namespace helper
 } // end namespace adios2
 

From af7198ef69c4a2e78c4c1bb1d18d7024ebc166a3 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Tue, 23 Jul 2019 15:09:18 -0400
Subject: [PATCH 34/42] helper: Drop empty adiosMPIFunctions.h

All of the functions have been migrated to the Comm encapsulation.
---
 source/adios2/CMakeLists.txt               |  1 -
 source/adios2/helper/adiosFunctions.h      |  1 -
 source/adios2/helper/adiosMPIFunctions.cpp | 24 ----------------
 source/adios2/helper/adiosMPIFunctions.h   | 31 ---------------------
 source/adios2/helper/adiosMPIFunctions.inl | 25 -----------------
 source/adios2/helper/adiosMPIFunctions.tcc | 32 ----------------------
 6 files changed, 114 deletions(-)
 delete mode 100644 source/adios2/helper/adiosMPIFunctions.cpp
 delete mode 100644 source/adios2/helper/adiosMPIFunctions.h
 delete mode 100644 source/adios2/helper/adiosMPIFunctions.inl
 delete mode 100644 source/adios2/helper/adiosMPIFunctions.tcc

diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt
index e92343945d..90b96b4e30 100644
--- a/source/adios2/CMakeLists.txt
+++ b/source/adios2/CMakeLists.txt
@@ -36,7 +36,6 @@ add_library(adios2
   helper/adiosDynamicBinder.h  helper/adiosDynamicBinder.cpp
   helper/adiosMath.cpp
   helper/adiosMemory.cpp
-  helper/adiosMPIFunctions.cpp
   helper/adiosNetwork.cpp
   helper/adiosString.cpp helper/adiosString.tcc
   helper/adiosSystem.cpp
diff --git a/source/adios2/helper/adiosFunctions.h b/source/adios2/helper/adiosFunctions.h
index 8fa684caa1..c834381806 100644
--- a/source/adios2/helper/adiosFunctions.h
+++ b/source/adios2/helper/adiosFunctions.h
@@ -12,7 +12,6 @@
 #ifndef ADIOS2_HELPER_ADIOSFUNCTIONS_H_
 #define ADIOS2_HELPER_ADIOSFUNCTIONS_H_
 
-#include "adios2/helper/adiosMPIFunctions.h" //MPI functions (Bcast, send/recv)
 #include "adios2/helper/adiosMath.h"         //math functions (cmath, algorithm)
 #include "adios2/helper/adiosMemory.h"       //memcpy, std::copy, insert, resize
 #include "adios2/helper/adiosNetwork.h"      //network and staging functions
diff --git a/source/adios2/helper/adiosMPIFunctions.cpp b/source/adios2/helper/adiosMPIFunctions.cpp
deleted file mode 100644
index e3322281fd..0000000000
--- a/source/adios2/helper/adiosMPIFunctions.cpp
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Distributed under the OSI-approved Apache License, Version 2.0.  See
- * accompanying file Copyright.txt for details.
- *
- * adiosMPIFunctions.cpp
- *
- *  Created on: Jul 20, 2017
- *      Author: William F Godoy godoywf@ornl.gov
- */
-#include "adiosMPIFunctions.h"
-#include "adiosMPIFunctions.tcc"
-
-#include "adios2/common/ADIOSMPI.h"
-#include "adios2/common/ADIOSTypes.h"
-
-#include "adios2/helper/adiosString.h"
-
-namespace adios2
-{
-namespace helper
-{
-// placeholder
-} // end namespace helper
-} // end namespace adios2
diff --git a/source/adios2/helper/adiosMPIFunctions.h b/source/adios2/helper/adiosMPIFunctions.h
deleted file mode 100644
index 14744038ee..0000000000
--- a/source/adios2/helper/adiosMPIFunctions.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Distributed under the OSI-approved Apache License, Version 2.0.  See
- * accompanying file Copyright.txt for details.
- *
- * adiosMPIFunctions.h : collection of MPI functions used across adios2
- *
- *  Created on: Jul 20, 2017
- *      Author: William F Godoy godoywf@ornl.gov
- */
-
-#ifndef ADIOS2_HELPER_ADIOSMPIFUNCTIONS_H_
-#define ADIOS2_HELPER_ADIOSMPIFUNCTIONS_H_
-
-/// \cond EXCLUDE_FROM_DOXYGEN
-#include <string>
-#include <vector>
-/// \endcond
-
-#include "adios2/common/ADIOSMPI.h"
-
-namespace adios2
-{
-namespace helper
-{
-// placeholder
-} // end namespace helper
-} // end namespace adios2
-
-#include "adiosMPIFunctions.inl"
-
-#endif /* ADIOS2_HELPER_ADIOMPIFUNCTIONS_H_ */
diff --git a/source/adios2/helper/adiosMPIFunctions.inl b/source/adios2/helper/adiosMPIFunctions.inl
deleted file mode 100644
index 98fff83438..0000000000
--- a/source/adios2/helper/adiosMPIFunctions.inl
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Distributed under the OSI-approved Apache License, Version 2.0.  See
- * accompanying file Copyright.txt for details.
- *
- * adiosMPIFunctions.inl
- *
- *  Created on: Sep 7, 2017
- *      Author: William F Godoy godoywf@ornl.gov
- */
-
-#ifndef ADIOS2_HELPER_ADIOSMPIFUNCTIONS_INL_
-#define ADIOS2_HELPER_ADIOSMPIFUNCTIONS_INL_
-#ifndef ADIOS2_HELPER_ADIOSMPIFUNCTIONS_H_
-#error "Inline file should only be included from it's header, never on it's own"
-#endif
-
-namespace adios2
-{
-namespace helper
-{
-// placeholder
-} // end namespace helper
-} // end namespace adios2
-
-#endif /* ADIOS2_HELPER_ADIOSMPIFUNCTIONS_INL_ */
diff --git a/source/adios2/helper/adiosMPIFunctions.tcc b/source/adios2/helper/adiosMPIFunctions.tcc
deleted file mode 100644
index 75c8997fb1..0000000000
--- a/source/adios2/helper/adiosMPIFunctions.tcc
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Distributed under the OSI-approved Apache License, Version 2.0.  See
- * accompanying file Copyright.txt for details.
- *
- * adiosMPIFunctions.tcc : specialization of template functions defined in
- * adiosMPIFunctions.h
- *
- *  Created on: Aug 30, 2017
- *      Author: William F Godoy godoywf@ornl.gov
- */
-
-#ifndef ADIOS2_HELPER_ADIOSMPIFUNCTIONS_TCC_
-#define ADIOS2_HELPER_ADIOSMPIFUNCTIONS_TCC_
-
-#include "adiosMPIFunctions.h"
-
-#include <algorithm> //std::foreach
-
-#include "adios2/common/ADIOSMPI.h"
-#include "adios2/common/ADIOSTypes.h"
-
-namespace adios2
-{
-namespace helper
-{
-
-// placeholder
-
-} // end namespace helper
-} // end namespace adios2
-
-#endif /* ADIOS2_HELPER_ADIOSMPIFUNCTIONS_TCC_ */

From b6a34da557ed3c236f17ad21be04c1b595e50ee8 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Wed, 31 Jul 2019 19:37:46 -0400
Subject: [PATCH 35/42] helper: Encapsulate MPI_Comm_{rank,size} as
 Comm::{Rank,Size}

---
 .../adios2/engine/dataman/DataManCommon.cpp   |  4 ++--
 source/adios2/engine/inline/InlineReader.cpp  |  2 +-
 source/adios2/engine/inline/InlineWriter.cpp  |  2 +-
 .../engine/insitumpi/InSituMPIReader.cpp      |  4 ++--
 .../engine/insitumpi/InSituMPIWriter.cpp      |  4 ++--
 .../adios2/engine/skeleton/SkeletonReader.cpp |  2 +-
 .../adios2/engine/skeleton/SkeletonWriter.cpp |  2 +-
 source/adios2/engine/ssc/SscReader.cpp        |  2 +-
 source/adios2/engine/ssc/SscWriter.cpp        |  4 ++--
 source/adios2/engine/table/TableWriter.cpp    |  4 ++--
 source/adios2/helper/adiosComm.cpp            | 14 +++++++++++
 source/adios2/helper/adiosComm.h              |  3 +++
 source/adios2/helper/adiosNetwork.cpp         |  6 ++---
 .../toolkit/aggregator/mpi/MPIAggregator.cpp  | 10 ++++----
 source/adios2/toolkit/format/bp3/BP3Base.cpp  |  4 ++--
 .../toolkit/format/bp3/BP3Serializer.cpp      |  8 +++----
 source/adios2/toolkit/format/bp4/BP4Base.cpp  |  4 ++--
 .../toolkit/format/bp4/BP4Serializer.cpp      | 23 +++++++------------
 .../format/dataman/DataManSerializer.cpp      |  4 ++--
 .../toolkit/transportman/TransportMan.cpp     |  3 +--
 20 files changed, 56 insertions(+), 53 deletions(-)

diff --git a/source/adios2/engine/dataman/DataManCommon.cpp b/source/adios2/engine/dataman/DataManCommon.cpp
index 9b399bec79..582ea2dda1 100644
--- a/source/adios2/engine/dataman/DataManCommon.cpp
+++ b/source/adios2/engine/dataman/DataManCommon.cpp
@@ -24,8 +24,8 @@ DataManCommon::DataManCommon(const std::string engineType, IO &io,
   m_IsRowMajor(helper::IsRowMajor(io.m_HostLanguage)),
   m_DataManSerializer(m_Comm, m_IsRowMajor)
 {
-    MPI_Comm_rank(m_Comm, &m_MpiRank);
-    MPI_Comm_size(m_Comm, &m_MpiSize);
+    m_MpiRank = m_Comm.Rank();
+    m_MpiSize = m_Comm.Size();
     GetParameter(m_IO.m_Parameters, "IPAddress", m_IPAddress);
     GetParameter(m_IO.m_Parameters, "Port", m_Port);
     GetParameter(m_IO.m_Parameters, "StagingMode", m_StagingMode);
diff --git a/source/adios2/engine/inline/InlineReader.cpp b/source/adios2/engine/inline/InlineReader.cpp
index 72d8f5e827..bdc616bc1c 100644
--- a/source/adios2/engine/inline/InlineReader.cpp
+++ b/source/adios2/engine/inline/InlineReader.cpp
@@ -27,7 +27,7 @@ InlineReader::InlineReader(IO &io, const std::string &name, const Mode mode,
 : Engine("InlineReader", io, name, mode, std::move(comm))
 {
     m_EndMessage = " in call to IO Open InlineReader " + m_Name + "\n";
-    MPI_Comm_rank(m_Comm, &m_ReaderRank);
+    m_ReaderRank = m_Comm.Rank();
     Init();
     Engine &writer = io.GetEngine(m_WriterID);
     if (m_Verbosity == 5)
diff --git a/source/adios2/engine/inline/InlineWriter.cpp b/source/adios2/engine/inline/InlineWriter.cpp
index ebcb081fd3..0ba65197e7 100644
--- a/source/adios2/engine/inline/InlineWriter.cpp
+++ b/source/adios2/engine/inline/InlineWriter.cpp
@@ -27,7 +27,7 @@ InlineWriter::InlineWriter(IO &io, const std::string &name, const Mode mode,
 : Engine("InlineWriter", io, name, mode, std::move(comm))
 {
     m_EndMessage = " in call to InlineWriter " + m_Name + " Open\n";
-    MPI_Comm_rank(m_Comm, &m_WriterRank);
+    m_WriterRank = m_Comm.Rank();
     Init();
     if (m_Verbosity == 5)
     {
diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.cpp b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
index 7e0863e6e4..56a9627d08 100644
--- a/source/adios2/engine/insitumpi/InSituMPIReader.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
@@ -40,8 +40,8 @@ InSituMPIReader::InSituMPIReader(IO &io, const std::string &name,
     m_RankAllPeers = insitumpi::FindPeers(m_Comm, m_Name, false, m_CommWorld);
     MPI_Comm_rank(m_CommWorld, &m_GlobalRank);
     MPI_Comm_size(m_CommWorld, &m_GlobalNproc);
-    MPI_Comm_rank(m_Comm, &m_ReaderRank);
-    MPI_Comm_size(m_Comm, &m_ReaderNproc);
+    m_ReaderRank = m_Comm.Rank();
+    m_ReaderNproc = m_Comm.Size();
     m_RankDirectPeers =
         insitumpi::AssignPeers(m_ReaderRank, m_ReaderNproc, m_RankAllPeers);
     if (m_Verbosity == 5)
diff --git a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
index cf75ba0f97..729df8f8c7 100644
--- a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
@@ -45,8 +45,8 @@ InSituMPIWriter::InSituMPIWriter(IO &io, const std::string &name,
     }
     MPI_Comm_rank(m_CommWorld, &m_GlobalRank);
     MPI_Comm_size(m_CommWorld, &m_GlobalNproc);
-    MPI_Comm_rank(m_Comm, &m_WriterRank);
-    MPI_Comm_size(m_Comm, &m_WriterNproc);
+    m_WriterRank = m_Comm.Rank();
+    m_WriterNproc = m_Comm.Size();
     m_RankDirectPeers =
         insitumpi::AssignPeers(m_WriterRank, m_WriterNproc, m_RankAllPeers);
     int primaryContact = insitumpi::ConnectDirectPeers(
diff --git a/source/adios2/engine/skeleton/SkeletonReader.cpp b/source/adios2/engine/skeleton/SkeletonReader.cpp
index 687c5d8a00..a1f08dc6ad 100644
--- a/source/adios2/engine/skeleton/SkeletonReader.cpp
+++ b/source/adios2/engine/skeleton/SkeletonReader.cpp
@@ -27,7 +27,7 @@ SkeletonReader::SkeletonReader(IO &io, const std::string &name, const Mode mode,
 : Engine("SkeletonReader", io, name, mode, std::move(comm))
 {
     m_EndMessage = " in call to IO Open SkeletonReader " + m_Name + "\n";
-    MPI_Comm_rank(m_Comm, &m_ReaderRank);
+    m_ReaderRank = m_Comm.Rank();
     Init();
     if (m_Verbosity == 5)
     {
diff --git a/source/adios2/engine/skeleton/SkeletonWriter.cpp b/source/adios2/engine/skeleton/SkeletonWriter.cpp
index 0d9d9427aa..c775423544 100644
--- a/source/adios2/engine/skeleton/SkeletonWriter.cpp
+++ b/source/adios2/engine/skeleton/SkeletonWriter.cpp
@@ -28,7 +28,7 @@ SkeletonWriter::SkeletonWriter(IO &io, const std::string &name, const Mode mode,
 : Engine("SkeletonWriter", io, name, mode, std::move(comm))
 {
     m_EndMessage = " in call to SkeletonWriter " + m_Name + " Open\n";
-    MPI_Comm_rank(m_Comm, &m_WriterRank);
+    m_WriterRank = m_Comm.Rank();
     Init();
     if (m_Verbosity == 5)
     {
diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp
index bdacb225fc..b7592fa4e3 100644
--- a/source/adios2/engine/ssc/SscReader.cpp
+++ b/source/adios2/engine/ssc/SscReader.cpp
@@ -33,7 +33,7 @@ SscReader::SscReader(IO &io, const std::string &name, const Mode mode,
 {
     TAU_SCOPED_TIMER_FUNC();
 
-    MPI_Comm_rank(m_Comm, &m_MpiRank);
+    m_MpiRank = m_Comm.Rank();
     srand(time(NULL));
 
     // initialize parameters from IO
diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp
index 60fbff39e3..15006dff2f 100644
--- a/source/adios2/engine/ssc/SscWriter.cpp
+++ b/source/adios2/engine/ssc/SscWriter.cpp
@@ -109,8 +109,8 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
 void SscWriter::Init()
 {
     TAU_SCOPED_TIMER_FUNC();
-    MPI_Comm_rank(m_Comm, &m_MpiRank);
-    MPI_Comm_size(m_Comm, &m_MpiSize);
+    m_MpiRank = m_Comm.Rank();
+    m_MpiSize = m_Comm.Size();
     srand(time(NULL));
     InitParameters();
     helper::HandshakeWriter(m_Comm, m_AppID, m_FullAddresses, m_Name, "ssc",
diff --git a/source/adios2/engine/table/TableWriter.cpp b/source/adios2/engine/table/TableWriter.cpp
index 5649b2856e..e55de3bab6 100644
--- a/source/adios2/engine/table/TableWriter.cpp
+++ b/source/adios2/engine/table/TableWriter.cpp
@@ -31,8 +31,8 @@ TableWriter::TableWriter(IO &io, const std::string &name, const Mode mode,
   m_SubAdios(MPI_COMM_WORLD, adios2::DebugOFF),
   m_SubIO(m_SubAdios.DeclareIO("SubIO"))
 {
-    MPI_Comm_rank(m_Comm, &m_MpiRank);
-    MPI_Comm_size(m_Comm, &m_MpiSize);
+    m_MpiRank = m_Comm.Rank();
+    m_MpiSize = m_Comm.Size();
     Init();
 }
 
diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index 05165972d8..54c4e52fa0 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -98,6 +98,20 @@ Comm Comm::Split(int color, int key, const std::string &hint) const
     return Comm(newComm);
 }
 
+int Comm::Rank() const
+{
+    int rank;
+    CheckMPIReturn(SMPI_Comm_rank(m_MPIComm, &rank), {});
+    return rank;
+}
+
+int Comm::Size() const
+{
+    int size;
+    CheckMPIReturn(SMPI_Comm_size(m_MPIComm, &size), {});
+    return size;
+}
+
 void Comm::Barrier(const std::string &hint) const
 {
     CheckMPIReturn(SMPI_Barrier(m_MPIComm), hint);
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 1ea4051a3e..215f0a263e 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -94,6 +94,9 @@ class Comm
     Comm Split(int color, int key,
                const std::string &hint = std::string()) const;
 
+    int Rank() const;
+    int Size() const;
+
     void Barrier(const std::string &hint = std::string()) const;
 
     /**
diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp
index 3bad0d18f7..6f1254bf03 100644
--- a/source/adios2/helper/adiosNetwork.cpp
+++ b/source/adios2/helper/adiosNetwork.cpp
@@ -77,10 +77,8 @@ void HandshakeWriter(Comm const &comm, size_t &appID,
                      const int maxRanksPerNode, const int maxAppsPerNode)
 {
 
-    int mpiRank;
-    int mpiSize;
-    MPI_Comm_rank(comm, &mpiRank);
-    MPI_Comm_size(comm, &mpiSize);
+    int mpiRank = comm.Rank();
+    int mpiSize = comm.Size();
 
     const std::string globalFilename = ".socket";
     const std::string globalLockFilename = ".socket.lock";
diff --git a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
index d673a95295..47de4c7c97 100644
--- a/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
+++ b/source/adios2/toolkit/aggregator/mpi/MPIAggregator.cpp
@@ -55,10 +55,8 @@ void MPIAggregator::Close()
 void MPIAggregator::InitComm(const size_t subStreams,
                              helper::Comm const &parentComm)
 {
-    int parentRank;
-    int parentSize;
-    MPI_Comm_rank(parentComm, &parentRank);
-    MPI_Comm_size(parentComm, &parentSize);
+    int parentRank = parentComm.Rank();
+    int parentSize = parentComm.Size();
 
     const size_t process = static_cast<size_t>(parentRank);
     const size_t processes = static_cast<size_t>(parentSize);
@@ -86,8 +84,8 @@ void MPIAggregator::InitComm(const size_t subStreams,
     m_Comm = parentComm.Split(m_ConsumerRank, parentRank,
                               "creating aggregators comm with split at Open");
 
-    MPI_Comm_rank(m_Comm, &m_Rank);
-    MPI_Comm_size(m_Comm, &m_Size);
+    m_Rank = m_Comm.Rank();
+    m_Size = m_Comm.Size();
 
     if (m_Rank != 0)
     {
diff --git a/source/adios2/toolkit/format/bp3/BP3Base.cpp b/source/adios2/toolkit/format/bp3/BP3Base.cpp
index 624620eca1..67337c3f81 100644
--- a/source/adios2/toolkit/format/bp3/BP3Base.cpp
+++ b/source/adios2/toolkit/format/bp3/BP3Base.cpp
@@ -51,8 +51,8 @@ const std::map<int, std::string> BP3Base::m_TransformTypesToNames = {
 BP3Base::BP3Base(helper::Comm const &comm, const bool debugMode)
 : m_Comm(comm), m_DebugMode(debugMode)
 {
-    SMPI_Comm_rank(m_Comm, &m_RankMPI);
-    SMPI_Comm_size(m_Comm, &m_SizeMPI);
+    m_RankMPI = m_Comm.Rank();
+    m_SizeMPI = m_Comm.Size();
     m_Profiler.IsActive = true; // default
 }
 
diff --git a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
index 9a6ba186df..46b0e386ac 100644
--- a/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
+++ b/source/adios2/toolkit/format/bp3/BP3Serializer.cpp
@@ -280,8 +280,7 @@ void BP3Serializer::AggregateCollectiveMetadata(helper::Comm const &comm,
     const std::vector<size_t> indicesPosition =
         AggregateCollectiveMetadataIndices(comm, bufferSTL);
 
-    int rank;
-    SMPI_Comm_rank(comm, &rank);
+    int rank = comm.Rank();
     if (rank == 0)
     {
         PutMinifooter(static_cast<uint64_t>(indicesPosition[0]),
@@ -769,9 +768,8 @@ BP3Serializer::AggregateCollectiveMetadataIndices(helper::Comm const &comm,
                                                   BufferSTL &bufferSTL)
 {
     TAU_SCOPED_TIMER_FUNC();
-    int rank, size;
-    SMPI_Comm_rank(comm, &rank);
-    SMPI_Comm_size(comm, &size);
+    int rank = comm.Rank();
+    int size = comm.Size();
 
     // pre-allocate with rank 0 data
     size_t pgCount = 0; //< tracks global PG count
diff --git a/source/adios2/toolkit/format/bp4/BP4Base.cpp b/source/adios2/toolkit/format/bp4/BP4Base.cpp
index ae319f520a..d8cd8c0daa 100644
--- a/source/adios2/toolkit/format/bp4/BP4Base.cpp
+++ b/source/adios2/toolkit/format/bp4/BP4Base.cpp
@@ -51,8 +51,8 @@ const std::map<int, std::string> BP4Base::m_TransformTypesToNames = {
 BP4Base::BP4Base(helper::Comm const &comm, const bool debugMode)
 : m_Comm(comm), m_DebugMode(debugMode)
 {
-    SMPI_Comm_rank(m_Comm, &m_RankMPI);
-    SMPI_Comm_size(m_Comm, &m_SizeMPI);
+    m_RankMPI = m_Comm.Rank();
+    m_SizeMPI = m_Comm.Size();
     m_Profiler.IsActive = true; // default
 }
 
diff --git a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
index 144dc7a6fb..1906e1449a 100644
--- a/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
+++ b/source/adios2/toolkit/format/bp4/BP4Serializer.cpp
@@ -468,8 +468,7 @@ void BP4Serializer::AggregateCollectiveMetadata(helper::Comm const &comm,
     // AggregateMergeIndex(m_MetadataSet.AttributesIndices, comm, bufferSTL,
     // true);
 
-    int rank;
-    SMPI_Comm_rank(comm, &rank);
+    int rank = comm.Rank();
     if (rank == 0)
     {
         /* no more minifooter in the global metadata*/
@@ -970,8 +969,7 @@ void BP4Serializer::AggregateIndex(const SerialElementIndex &index,
 {
     auto &buffer = bufferSTL.m_Buffer;
     auto &position = bufferSTL.m_Position;
-    int rank;
-    SMPI_Comm_rank(comm, &rank);
+    int rank = comm.Rank();
 
     size_t countPosition = position;
     const size_t totalCount = comm.ReduceValues<size_t>(count);
@@ -1021,8 +1019,7 @@ void BP4Serializer::AggregateMergeIndex(
     // deallocate gathered serial indices (full in rank 0 only)
     std::vector<char>().swap(gatheredSerialIndices);
 
-    int rank;
-    SMPI_Comm_rank(comm, &rank);
+    int rank = comm.Rank();
 
     if (rank == 0)
     {
@@ -1071,8 +1068,7 @@ std::vector<char> BP4Serializer::SerializeIndices(
     std::vector<char> serializedIndices;
     serializedIndices.reserve(serializedIndicesSize);
 
-    int rank;
-    SMPI_Comm_rank(comm, &rank);
+    int rank = comm.Rank();
 
     for (const auto &indexPair : indices)
     {
@@ -1160,8 +1156,7 @@ BP4Serializer::DeserializeIndicesPerRankSingleThread(
 
     // BODY OF FUNCTION starts here
     const size_t serializedSize = serialized.size();
-    int rank;
-    SMPI_Comm_rank(comm, &rank);
+    int rank = comm.Rank();
 
     if (rank != 0 || serializedSize < 8)
     {
@@ -1340,8 +1335,7 @@ BP4Serializer::DeserializeIndicesPerRankThreads(
 
     // BODY OF FUNCTION starts here
     const size_t serializedSize = serialized.size();
-    int rank;
-    SMPI_Comm_rank(comm, &rank);
+    int rank = comm.Rank();
 
     if (rank != 0 || serializedSize < 8)
     {
@@ -1405,9 +1399,8 @@ BP4Serializer::DeserializeIndicesPerRankThreads(
 void BP4Serializer::AggregateCollectiveMetadataIndices(helper::Comm const &comm,
                                                        BufferSTL &outBufferSTL)
 {
-    int rank, size;
-    SMPI_Comm_rank(comm, &rank);
-    SMPI_Comm_size(comm, &size);
+    int rank = comm.Rank();
+    int size = comm.Size();
 
     BufferSTL inBufferSTL;
 
diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
index ac222968b7..0cf15cfb78 100644
--- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
+++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
@@ -25,8 +25,8 @@ DataManSerializer::DataManSerializer(helper::Comm const &comm,
   m_IsLittleEndian(helper::IsLittleEndian()),
   m_DeferredRequestsToSend(std::make_shared<DeferredRequestMap>())
 {
-    MPI_Comm_size(m_Comm, &m_MpiSize);
-    MPI_Comm_rank(m_Comm, &m_MpiRank);
+    m_MpiRank = m_Comm.Rank();
+    m_MpiSize = m_Comm.Size();
 }
 
 void DataManSerializer::NewWriterBuffer(size_t bufferSize)
diff --git a/source/adios2/toolkit/transportman/TransportMan.cpp b/source/adios2/toolkit/transportman/TransportMan.cpp
index 124856744e..e8d862448c 100644
--- a/source/adios2/toolkit/transportman/TransportMan.cpp
+++ b/source/adios2/toolkit/transportman/TransportMan.cpp
@@ -60,8 +60,7 @@ void TransportMan::MkDirsBarrier(const std::vector<std::string> &fileNames,
     }
     else
     {
-        int rank;
-        SMPI_Comm_rank(m_Comm, &rank);
+        int rank = m_Comm.Rank();
         if (rank == 0)
         {
             lf_CreateDirectories(fileNames);

From 39696b4675f3c43cc75c74e40c8923437bdf2102 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Wed, 31 Jul 2019 20:18:18 -0400
Subject: [PATCH 36/42] helper: Encapsulate MPI_Reduce as Comm::Reduce

Also add a `Comm::ReduceInPlace` variant for `MPI_IN_PLACE`.
---
 .../adios2/engine/insitumpi/InSituMPIReader.cpp | 17 +++++++++--------
 source/adios2/helper/adiosComm.cpp              | 17 +++++++++++++++++
 source/adios2/helper/adiosComm.h                | 15 +++++++++++++++
 source/adios2/helper/adiosComm.inl              | 14 ++++++++++++++
 4 files changed, 55 insertions(+), 8 deletions(-)

diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.cpp b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
index 56a9627d08..fa7eb9a589 100644
--- a/source/adios2/engine/insitumpi/InSituMPIReader.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
@@ -418,13 +418,16 @@ void InSituMPIReader::SendReadSchedule(
     }
 
     // Accumulate nReaderPerWriter for all readers
-    void *sendBuf = nReaderPerWriter.data();
     if (m_ReaderRootRank == m_ReaderRank)
     {
-        sendBuf = MPI_IN_PLACE;
+        m_Comm.ReduceInPlace(nReaderPerWriter.data(), nReaderPerWriter.size(),
+                             MPI_SUM, m_ReaderRootRank);
+    }
+    else
+    {
+        m_Comm.Reduce(nReaderPerWriter.data(), nReaderPerWriter.data(),
+                      nReaderPerWriter.size(), MPI_SUM, m_ReaderRootRank);
     }
-    MPI_Reduce(sendBuf, nReaderPerWriter.data(), nReaderPerWriter.size(),
-               MPI_INT, MPI_SUM, m_ReaderRootRank, m_Comm);
 
     // Reader root sends nReaderPerWriter to writer root
     if (m_ReaderRootRank == m_ReaderRank)
@@ -588,10 +591,8 @@ void InSituMPIReader::DoClose(const int transportIndex)
     if (m_Verbosity > 2)
     {
         uint64_t inPlaceBytes, inTempBytes;
-        MPI_Reduce(&m_BytesReceivedInPlace, &inPlaceBytes, 1, MPI_LONG_LONG_INT,
-                   MPI_SUM, 0, m_Comm);
-        MPI_Reduce(&m_BytesReceivedInTemporary, &inTempBytes, 1,
-                   MPI_LONG_LONG_INT, MPI_SUM, 0, m_Comm);
+        m_Comm.Reduce(&m_BytesReceivedInPlace, &inPlaceBytes, 1, MPI_SUM, 0);
+        m_Comm.Reduce(&m_BytesReceivedInTemporary, &inTempBytes, 1, MPI_SUM, 0);
         if (m_ReaderRank == 0)
         {
             std::cout << "ADIOS InSituMPI Reader for " << m_Name << " received "
diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index 54c4e52fa0..7dcaef1e39 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -144,6 +144,23 @@ void Comm::BcastImpl(void *buffer, size_t count, MPI_Datatype datatype,
         hint);
 }
 
+void Comm::ReduceImpl(const void *sendbuf, void *recvbuf, size_t count,
+                      MPI_Datatype datatype, MPI_Op op, int root,
+                      const std::string &hint) const
+{
+    CheckMPIReturn(SMPI_Reduce(sendbuf, recvbuf, static_cast<int>(count),
+                               datatype, op, root, m_MPIComm),
+                   hint);
+}
+
+void Comm::ReduceInPlaceImpl(void *buf, size_t count, MPI_Datatype datatype,
+                             MPI_Op op, int root, const std::string &hint) const
+{
+    CheckMPIReturn(SMPI_Reduce(MPI_IN_PLACE, buf, static_cast<int>(count),
+                               datatype, op, root, m_MPIComm),
+                   hint);
+}
+
 Comm::Req Comm::IsendImpl(const void *buffer, size_t count,
                           MPI_Datatype datatype, int dest, int tag,
                           const std::string &hint) const
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 215f0a263e..481b7da847 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -178,6 +178,14 @@ class Comm
     void Bcast(T *buffer, size_t count, int root,
                const std::string &hint = std::string()) const;
 
+    template <typename T>
+    void Reduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op, int root,
+                const std::string &hint = std::string()) const;
+
+    template <typename T>
+    void ReduceInPlace(T *buf, size_t count, MPI_Op op, int root,
+                       const std::string &hint = std::string()) const;
+
     template <typename T>
     Req Isend(const T *buffer, const size_t count, int destination, int tag,
               const std::string &hint = std::string()) const;
@@ -203,6 +211,13 @@ class Comm
     void BcastImpl(void *buffer, size_t count, MPI_Datatype datatype, int root,
                    const std::string &hint) const;
 
+    void ReduceImpl(const void *sendbuf, void *recvbuf, size_t count,
+                    MPI_Datatype datatype, MPI_Op op, int root,
+                    const std::string &hint) const;
+
+    void ReduceInPlaceImpl(void *buf, size_t count, MPI_Datatype datatype,
+                           MPI_Op op, int root, const std::string &hint) const;
+
     Req IsendImpl(const void *buffer, size_t count, MPI_Datatype datatype,
                   int dest, int tag, const std::string &hint) const;
 
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index 46d6362a75..ba914b3c53 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -148,6 +148,20 @@ void Comm::Bcast(T *buffer, const size_t count, int root,
     return BcastImpl(buffer, count, Datatype<T>(), root, hint);
 }
 
+template <typename T>
+void Comm::Reduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op,
+                  int root, const std::string &hint) const
+{
+    return ReduceImpl(sendbuf, recvbuf, count, Datatype<T>(), op, root, hint);
+}
+
+template <typename T>
+void Comm::ReduceInPlace(T *buf, size_t count, MPI_Op op, int root,
+                         const std::string &hint) const
+{
+    return ReduceInPlaceImpl(buf, count, Datatype<T>(), op, root, hint);
+}
+
 template <typename T>
 Comm::Req Comm::Isend(const T *buffer, const size_t count, int dest, int tag,
                       const std::string &hint) const

From 4b07c8fda4b07f1f4e8fd2fd048e5b28c6a6ae86 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Mon, 5 Aug 2019 13:30:15 -0400
Subject: [PATCH 37/42] helper: Encapsulate MPI_{Send,Recv} as
 Comm::{Send,Recv}

---
 source/adios2/helper/adiosComm.cpp | 29 +++++++++++++++++++++++++++++
 source/adios2/helper/adiosComm.h   | 14 ++++++++++++++
 source/adios2/helper/adiosComm.inl | 14 ++++++++++++++
 3 files changed, 57 insertions(+)

diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index 7dcaef1e39..caec5c452d 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -161,6 +161,35 @@ void Comm::ReduceInPlaceImpl(void *buf, size_t count, MPI_Datatype datatype,
                    hint);
 }
 
+void Comm::SendImpl(const void *buf, size_t count, MPI_Datatype datatype,
+                    int dest, int tag, const std::string &hint) const
+{
+    CheckMPIReturn(
+        MPI_Send(buf, static_cast<int>(count), datatype, dest, tag, m_MPIComm),
+        hint);
+}
+
+Comm::Status Comm::RecvImpl(void *buf, size_t count, MPI_Datatype datatype,
+                            int source, int tag, const std::string &hint) const
+{
+    MPI_Status mpiStatus;
+    CheckMPIReturn(MPI_Recv(buf, static_cast<int>(count), datatype, source, tag,
+                            m_MPIComm, &mpiStatus),
+                   hint);
+
+    Status status;
+#ifdef ADIOS2_HAVE_MPI
+    status.Source = mpiStatus.MPI_SOURCE;
+    status.Tag = mpiStatus.MPI_TAG;
+    {
+        int mpiCount = 0;
+        CheckMPIReturn(MPI_Get_count(&mpiStatus, datatype, &mpiCount), hint);
+        status.Count = mpiCount;
+    }
+#endif
+    return status;
+}
+
 Comm::Req Comm::IsendImpl(const void *buffer, size_t count,
                           MPI_Datatype datatype, int dest, int tag,
                           const std::string &hint) const
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 481b7da847..fee0b03571 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -186,6 +186,14 @@ class Comm
     void ReduceInPlace(T *buf, size_t count, MPI_Op op, int root,
                        const std::string &hint = std::string()) const;
 
+    template <typename T>
+    void Send(const T *buf, size_t count, int dest, int tag,
+              const std::string &hint = std::string()) const;
+
+    template <typename T>
+    Status Recv(T *buf, size_t count, int source, int tag,
+                const std::string &hint = std::string()) const;
+
     template <typename T>
     Req Isend(const T *buffer, const size_t count, int destination, int tag,
               const std::string &hint = std::string()) const;
@@ -218,6 +226,12 @@ class Comm
     void ReduceInPlaceImpl(void *buf, size_t count, MPI_Datatype datatype,
                            MPI_Op op, int root, const std::string &hint) const;
 
+    void SendImpl(const void *buf, size_t count, MPI_Datatype datatype,
+                  int dest, int tag, const std::string &hint) const;
+
+    Status RecvImpl(void *buf, size_t count, MPI_Datatype datatype, int source,
+                    int tag, const std::string &hint) const;
+
     Req IsendImpl(const void *buffer, size_t count, MPI_Datatype datatype,
                   int dest, int tag, const std::string &hint) const;
 
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index ba914b3c53..5d30c43570 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -162,6 +162,20 @@ void Comm::ReduceInPlace(T *buf, size_t count, MPI_Op op, int root,
     return ReduceInPlaceImpl(buf, count, Datatype<T>(), op, root, hint);
 }
 
+template <typename T>
+void Comm::Send(const T *buf, size_t count, int dest, int tag,
+                const std::string &hint) const
+{
+    return SendImpl(buf, count, Datatype<T>(), dest, tag, hint);
+}
+
+template <typename T>
+Comm::Status Comm::Recv(T *buf, size_t count, int source, int tag,
+                        const std::string &hint) const
+{
+    return RecvImpl(buf, count, Datatype<T>(), source, tag, hint);
+}
+
 template <typename T>
 Comm::Req Comm::Isend(const T *buffer, const size_t count, int dest, int tag,
                       const std::string &hint) const

From b282a11a9c2cedf306b5f13b6e5b23a590677a05 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Mon, 5 Aug 2019 14:48:32 -0400
Subject: [PATCH 38/42] helper: Encapsulate MPI_Allgather as Comm::Allgather

---
 source/adios2/engine/dataman/DataManWriter.cpp         |  8 ++++----
 source/adios2/engine/insitumpi/InSituMPIReader.cpp     |  2 +-
 source/adios2/engine/table/TableWriter.cpp             |  3 +--
 source/adios2/helper/adiosComm.cpp                     | 10 ++++++++++
 source/adios2/helper/adiosComm.h                       |  9 +++++++++
 source/adios2/helper/adiosComm.inl                     |  8 ++++++++
 .../toolkit/format/dataman/DataManSerializer.cpp       |  4 ++--
 7 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/source/adios2/engine/dataman/DataManWriter.cpp b/source/adios2/engine/dataman/DataManWriter.cpp
index b125cff789..0903a33bba 100644
--- a/source/adios2/engine/dataman/DataManWriter.cpp
+++ b/source/adios2/engine/dataman/DataManWriter.cpp
@@ -43,10 +43,10 @@ DataManWriter::DataManWriter(IO &io, const std::string &name,
         std::vector<char> allDaVec(32 * m_MpiSize, '\0');
         std::vector<char> allCaVec(32 * m_MpiSize, '\0');
 
-        MPI_Allgather(m_DataAddress.data(), m_DataAddress.size(), MPI_CHAR,
-                      allDaVec.data(), 32, MPI_CHAR, m_Comm);
-        MPI_Allgather(m_ControlAddress.data(), m_ControlAddress.size(),
-                      MPI_CHAR, allCaVec.data(), 32, MPI_CHAR, m_Comm);
+        m_Comm.Allgather(m_DataAddress.data(), m_DataAddress.size(),
+                         allDaVec.data(), 32);
+        m_Comm.Allgather(m_ControlAddress.data(), m_ControlAddress.size(),
+                         allCaVec.data(), 32);
 
         std::vector<std::string> daVec;
         std::vector<std::string> caVec;
diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.cpp b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
index fa7eb9a589..0653709024 100644
--- a/source/adios2/engine/insitumpi/InSituMPIReader.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
@@ -76,7 +76,7 @@ InSituMPIReader::InSituMPIReader(IO &io, const std::string &name,
 
     // figure out who is the Reader Root
     std::vector<int> v(m_ReaderNproc);
-    MPI_Allgather(&m_ReaderRootRank, 1, MPI_INT, v.data(), 1, MPI_INT, m_Comm);
+    m_Comm.Allgather(&m_ReaderRootRank, 1, v.data(), 1);
     for (int i = 0; i < m_ReaderNproc; i++)
     {
         if (v[i] != -1)
diff --git a/source/adios2/engine/table/TableWriter.cpp b/source/adios2/engine/table/TableWriter.cpp
index e55de3bab6..3a839d910e 100644
--- a/source/adios2/engine/table/TableWriter.cpp
+++ b/source/adios2/engine/table/TableWriter.cpp
@@ -205,8 +205,7 @@ void TableWriter::InitParameters()
     std::vector<char> cv(128);
     std::vector<char> cvAll(128 * m_MpiSize);
     std::memcpy(cv.data(), a.c_str(), a.size());
-    MPI_Allgather(cv.data(), cv.size(), MPI_CHAR, cvAll.data(), cv.size(),
-                  MPI_CHAR, m_Comm);
+    m_Comm.Allgather(cv.data(), cv.size(), cvAll.data(), cv.size());
     for (int i = 0; i < m_MpiSize; ++i)
     {
         auto j = nlohmann::json::parse(cvAll.data() + i * 128);
diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index caec5c452d..d8bf65ca7d 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -136,6 +136,16 @@ std::string Comm::BroadcastFile(const std::string &fileName,
     return fileContents;
 }
 
+void Comm::AllgatherImpl(const void *sendbuf, size_t sendcount,
+                         MPI_Datatype sendtype, void *recvbuf, size_t recvcount,
+                         MPI_Datatype recvtype, const std::string &hint) const
+{
+    CheckMPIReturn(
+        SMPI_Allgather(sendbuf, static_cast<int>(sendcount), sendtype, recvbuf,
+                       static_cast<int>(recvcount), recvtype, m_MPIComm),
+        hint);
+}
+
 void Comm::BcastImpl(void *buffer, size_t count, MPI_Datatype datatype,
                      int root, const std::string &hint) const
 {
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index fee0b03571..a57737bac5 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -174,6 +174,11 @@ class Comm
                               const std::string hint = "",
                               const int rankSource = 0) const;
 
+    template <typename TSend, typename TRecv>
+    void Allgather(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf,
+                   size_t recvcount,
+                   const std::string &hint = std::string()) const;
+
     template <typename T>
     void Bcast(T *buffer, size_t count, int root,
                const std::string &hint = std::string()) const;
@@ -216,6 +221,10 @@ class Comm
 
     static void CheckMPIReturn(const int value, const std::string &hint);
 
+    void AllgatherImpl(const void *sendbuf, size_t sendcount,
+                       MPI_Datatype sendtype, void *recvbuf, size_t recvcount,
+                       MPI_Datatype recvtype, const std::string &hint) const;
+
     void BcastImpl(void *buffer, size_t count, MPI_Datatype datatype, int root,
                    const std::string &hint) const;
 
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index 5d30c43570..7b8926730c 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -141,6 +141,14 @@ template <>
 void Comm::BroadcastVector(std::vector<size_t> &vector,
                            const int rankSource) const;
 
+template <typename TSend, typename TRecv>
+void Comm::Allgather(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf,
+                     size_t recvcount, const std::string &hint) const
+{
+    return AllgatherImpl(sendbuf, sendcount, Datatype<TSend>(), recvbuf,
+                         recvcount, Datatype<TRecv>(), hint);
+}
+
 template <typename T>
 void Comm::Bcast(T *buffer, const size_t count, int root,
                  const std::string &hint) const
diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
index 0cf15cfb78..5a7c12e586 100644
--- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
+++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
@@ -80,8 +80,8 @@ void DataManSerializer::AggregateMetadata()
       1) = size;
 
     std::vector<char> globalJsonStr(m_MpiSize * maxSize);
-    MPI_Allgather(localJsonPack->data(), maxSize, MPI_CHAR,
-                  globalJsonStr.data(), maxSize, MPI_CHAR, m_Comm);
+    m_Comm.Allgather(localJsonPack->data(), maxSize, globalJsonStr.data(),
+                     maxSize);
 
     nlohmann::json aggMetadata;
 

From d0f520b0caaeeb43bac5bf1908b3474562ddde80 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Mon, 5 Aug 2019 14:54:17 -0400
Subject: [PATCH 39/42] helper: Encapsulate MPI_Allreduce as Comm::Allreduce

---
 source/adios2/engine/insitumpi/InSituMPIReader.cpp       | 2 +-
 source/adios2/helper/adiosComm.cpp                       | 9 +++++++++
 source/adios2/helper/adiosComm.h                         | 8 ++++++++
 source/adios2/helper/adiosComm.inl                       | 7 +++++++
 .../adios2/toolkit/format/dataman/DataManSerializer.cpp  | 2 +-
 5 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.cpp b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
index 0653709024..2e7dd3964c 100644
--- a/source/adios2/engine/insitumpi/InSituMPIReader.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
@@ -190,7 +190,7 @@ StepStatus InSituMPIReader::BeginStep(const StepMode mode,
         }
         /* Exchange steps */
         int maxstep;
-        MPI_Allreduce(&step, &maxstep, 1, MPI_INT, MPI_MAX, m_Comm);
+        m_Comm.Allreduce(&step, &maxstep, 1, MPI_MAX);
 
         if (m_Verbosity == 5 && !m_ReaderRank)
         {
diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index d8bf65ca7d..3c98b0a257 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -146,6 +146,15 @@ void Comm::AllgatherImpl(const void *sendbuf, size_t sendcount,
         hint);
 }
 
+void Comm::AllreduceImpl(const void *sendbuf, void *recvbuf, size_t count,
+                         MPI_Datatype datatype, MPI_Op op,
+                         const std::string &hint) const
+{
+    CheckMPIReturn(SMPI_Allreduce(sendbuf, recvbuf, static_cast<int>(count),
+                                  datatype, op, m_MPIComm),
+                   hint);
+}
+
 void Comm::BcastImpl(void *buffer, size_t count, MPI_Datatype datatype,
                      int root, const std::string &hint) const
 {
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index a57737bac5..dbd9ece54b 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -179,6 +179,10 @@ class Comm
                    size_t recvcount,
                    const std::string &hint = std::string()) const;
 
+    template <typename T>
+    void Allreduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op,
+                   const std::string &hint = std::string()) const;
+
     template <typename T>
     void Bcast(T *buffer, size_t count, int root,
                const std::string &hint = std::string()) const;
@@ -225,6 +229,10 @@ class Comm
                        MPI_Datatype sendtype, void *recvbuf, size_t recvcount,
                        MPI_Datatype recvtype, const std::string &hint) const;
 
+    void AllreduceImpl(const void *sendbuf, void *recvbuf, size_t count,
+                       MPI_Datatype datatype, MPI_Op op,
+                       const std::string &hint) const;
+
     void BcastImpl(void *buffer, size_t count, MPI_Datatype datatype, int root,
                    const std::string &hint) const;
 
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index 7b8926730c..d536495fcd 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -149,6 +149,13 @@ void Comm::Allgather(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf,
                          recvcount, Datatype<TRecv>(), hint);
 }
 
+template <typename T>
+void Comm::Allreduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op,
+                     const std::string &hint) const
+{
+    return AllreduceImpl(sendbuf, recvbuf, count, Datatype<T>(), op, hint);
+}
+
 template <typename T>
 void Comm::Bcast(T *buffer, const size_t count, int root,
                  const std::string &hint) const
diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
index 5a7c12e586..b412554fa9 100644
--- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
+++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp
@@ -72,7 +72,7 @@ void DataManSerializer::AggregateMetadata()
     auto localJsonPack = SerializeJson(m_MetadataJson);
     unsigned int size = localJsonPack->size();
     unsigned int maxSize;
-    MPI_Allreduce(&size, &maxSize, 1, MPI_UNSIGNED, MPI_MAX, m_Comm);
+    m_Comm.Allreduce(&size, &maxSize, 1, MPI_MAX);
     maxSize += sizeof(uint64_t);
     localJsonPack->resize(maxSize, '\0');
     *(reinterpret_cast<uint64_t *>(localJsonPack->data() +

From 38086447dd4675a50fa3a8d78243e3aeffaf5315 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Mon, 5 Aug 2019 15:10:52 -0400
Subject: [PATCH 40/42] helper: Encapsulate MPI_Scatter as Comm::Scatter

---
 source/adios2/engine/insitumpi/InSituMPIWriter.cpp |  3 +--
 source/adios2/helper/adiosComm.cpp                 | 11 +++++++++++
 source/adios2/helper/adiosComm.h                   | 10 ++++++++++
 source/adios2/helper/adiosComm.inl                 |  8 ++++++++
 4 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
index 729df8f8c7..0ec5593ab6 100644
--- a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
@@ -441,8 +441,7 @@ void InSituMPIWriter::ReceiveReadSchedule(
     }
 
     // Each writer receives the number of its peer readers
-    MPI_Scatter(nReaderPerWriter.data(), 1, MPI_INT, &nPeerReaders, 1, MPI_INT,
-                0, m_Comm);
+    m_Comm.Scatter(nReaderPerWriter.data(), 1, &nPeerReaders, 1, 0);
 
     std::vector<MPI_Request> requests(nPeerReaders);
     // Reader global rank -> length of serialized read schedule
diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index 3c98b0a257..ef5ee23e6c 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -209,6 +209,17 @@ Comm::Status Comm::RecvImpl(void *buf, size_t count, MPI_Datatype datatype,
     return status;
 }
 
+void Comm::ScatterImpl(const void *sendbuf, size_t sendcount,
+                       MPI_Datatype sendtype, void *recvbuf, size_t recvcount,
+                       MPI_Datatype recvtype, int root,
+                       const std::string &hint) const
+{
+    CheckMPIReturn(MPI_Scatter(sendbuf, static_cast<int>(sendcount), sendtype,
+                               recvbuf, static_cast<int>(recvcount), recvtype,
+                               root, m_MPIComm),
+                   hint);
+}
+
 Comm::Req Comm::IsendImpl(const void *buffer, size_t count,
                           MPI_Datatype datatype, int dest, int tag,
                           const std::string &hint) const
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index dbd9ece54b..2a9bd00ed5 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -203,6 +203,11 @@ class Comm
     Status Recv(T *buf, size_t count, int source, int tag,
                 const std::string &hint = std::string()) const;
 
+    template <typename TSend, typename TRecv>
+    void Scatter(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf,
+                 size_t recvcount, int root,
+                 const std::string &hint = std::string()) const;
+
     template <typename T>
     Req Isend(const T *buffer, const size_t count, int destination, int tag,
               const std::string &hint = std::string()) const;
@@ -249,6 +254,11 @@ class Comm
     Status RecvImpl(void *buf, size_t count, MPI_Datatype datatype, int source,
                     int tag, const std::string &hint) const;
 
+    void ScatterImpl(const void *sendbuf, size_t sendcount,
+                     MPI_Datatype sendtype, void *recvbuf, size_t recvcount,
+                     MPI_Datatype recvtype, int root,
+                     const std::string &hint) const;
+
     Req IsendImpl(const void *buffer, size_t count, MPI_Datatype datatype,
                   int dest, int tag, const std::string &hint) const;
 
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index d536495fcd..5969dfe80a 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -191,6 +191,14 @@ Comm::Status Comm::Recv(T *buf, size_t count, int source, int tag,
     return RecvImpl(buf, count, Datatype<T>(), source, tag, hint);
 }
 
+template <typename TSend, typename TRecv>
+void Comm::Scatter(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf,
+                   size_t recvcount, int root, const std::string &hint) const
+{
+    return ScatterImpl(sendbuf, sendcount, Datatype<TSend>(), recvbuf,
+                       recvcount, Datatype<TRecv>(), root, hint);
+}
+
 template <typename T>
 Comm::Req Comm::Isend(const T *buffer, const size_t count, int dest, int tag,
                       const std::string &hint) const

From f5357b4cdb5e3536e07089bffafd3b6abfac14a4 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Mon, 5 Aug 2019 15:23:26 -0400
Subject: [PATCH 41/42] helper: Encapsulate MPI_Gather as Comm::Gather

---
 source/adios2/helper/adiosComm.cpp | 11 +++++++++++
 source/adios2/helper/adiosComm.h   | 10 ++++++++++
 source/adios2/helper/adiosComm.inl |  8 ++++++++
 3 files changed, 29 insertions(+)

diff --git a/source/adios2/helper/adiosComm.cpp b/source/adios2/helper/adiosComm.cpp
index ef5ee23e6c..2033bd3796 100644
--- a/source/adios2/helper/adiosComm.cpp
+++ b/source/adios2/helper/adiosComm.cpp
@@ -163,6 +163,17 @@ void Comm::BcastImpl(void *buffer, size_t count, MPI_Datatype datatype,
         hint);
 }
 
+void Comm::GatherImpl(const void *sendbuf, size_t sendcount,
+                      MPI_Datatype sendtype, void *recvbuf, size_t recvcount,
+                      MPI_Datatype recvtype, int root,
+                      const std::string &hint) const
+{
+    CheckMPIReturn(SMPI_Gather(sendbuf, static_cast<int>(sendcount), sendtype,
+                               recvbuf, static_cast<int>(recvcount), recvtype,
+                               root, m_MPIComm),
+                   hint);
+}
+
 void Comm::ReduceImpl(const void *sendbuf, void *recvbuf, size_t count,
                       MPI_Datatype datatype, MPI_Op op, int root,
                       const std::string &hint) const
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 2a9bd00ed5..3230847f1b 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -187,6 +187,11 @@ class Comm
     void Bcast(T *buffer, size_t count, int root,
                const std::string &hint = std::string()) const;
 
+    template <typename TSend, typename TRecv>
+    void Gather(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf,
+                size_t recvcount, int root,
+                const std::string &hint = std::string()) const;
+
     template <typename T>
     void Reduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op, int root,
                 const std::string &hint = std::string()) const;
@@ -241,6 +246,11 @@ class Comm
     void BcastImpl(void *buffer, size_t count, MPI_Datatype datatype, int root,
                    const std::string &hint) const;
 
+    void GatherImpl(const void *sendbuf, size_t sendcount,
+                    MPI_Datatype sendtype, void *recvbuf, size_t recvcount,
+                    MPI_Datatype recvtype, int root,
+                    const std::string &hint) const;
+
     void ReduceImpl(const void *sendbuf, void *recvbuf, size_t count,
                     MPI_Datatype datatype, MPI_Op op, int root,
                     const std::string &hint) const;
diff --git a/source/adios2/helper/adiosComm.inl b/source/adios2/helper/adiosComm.inl
index 5969dfe80a..f18b58cfba 100644
--- a/source/adios2/helper/adiosComm.inl
+++ b/source/adios2/helper/adiosComm.inl
@@ -163,6 +163,14 @@ void Comm::Bcast(T *buffer, const size_t count, int root,
     return BcastImpl(buffer, count, Datatype<T>(), root, hint);
 }
 
+template <typename TSend, typename TRecv>
+void Comm::Gather(const TSend *sendbuf, size_t sendcount, TRecv *recvbuf,
+                  size_t recvcount, int root, const std::string &hint) const
+{
+    return GatherImpl(sendbuf, sendcount, Datatype<TSend>(), recvbuf, recvcount,
+                      Datatype<TRecv>(), root, hint);
+}
+
 template <typename T>
 void Comm::Reduce(const T *sendbuf, T *recvbuf, size_t count, MPI_Op op,
                   int root, const std::string &hint) const

From 3f711df63c9c72d6e5dfe2edd8cf1dfd822baad4 Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Mon, 5 Aug 2019 15:31:14 -0400
Subject: [PATCH 42/42] helper: Replace implicit Comm conversion to MPI with
 explicit Comm::AsMPI

---
 source/adios2/core/IO.cpp                            | 2 +-
 source/adios2/engine/dataspaces/DataSpacesReader.cpp | 2 +-
 source/adios2/engine/dataspaces/DataSpacesWriter.cpp | 2 +-
 source/adios2/engine/hdf5/HDF5ReaderP.cpp            | 2 +-
 source/adios2/engine/hdf5/HDF5WriterP.cpp            | 6 +++---
 source/adios2/engine/insitumpi/InSituMPIReader.cpp   | 3 ++-
 source/adios2/engine/insitumpi/InSituMPIWriter.cpp   | 3 ++-
 source/adios2/engine/sst/SstReader.cpp               | 2 +-
 source/adios2/engine/sst/SstWriter.cpp               | 2 +-
 source/adios2/helper/adiosComm.h                     | 7 ++++---
 10 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp
index 4817b7e9ce..304cd8ac42 100644
--- a/source/adios2/core/IO.cpp
+++ b/source/adios2/core/IO.cpp
@@ -695,7 +695,7 @@ Engine &IO::Open(const std::string &name, const Mode mode, MPI_Comm mpiComm)
 
 Engine &IO::Open(const std::string &name, const Mode mode)
 {
-    return Open(name, mode, m_ADIOS.GetComm());
+    return Open(name, mode, m_ADIOS.GetComm().AsMPI());
 }
 
 Engine &IO::GetEngine(const std::string &name)
diff --git a/source/adios2/engine/dataspaces/DataSpacesReader.cpp b/source/adios2/engine/dataspaces/DataSpacesReader.cpp
index e1eddfc8a4..79c698f00e 100644
--- a/source/adios2/engine/dataspaces/DataSpacesReader.cpp
+++ b/source/adios2/engine/dataspaces/DataSpacesReader.cpp
@@ -52,7 +52,7 @@ DataSpacesReader::DataSpacesReader(IO &io, const std::string &name,
     {
         m_ProvideLatest = false;
     }
-    MPI_Comm mpiComm = m_Comm;
+    MPI_Comm mpiComm = m_Comm.AsMPI();
     ret = adios_read_dataspaces_init(&mpiComm, &m_data);
     if (ret < 0)
     {
diff --git a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp
index 43b96d6637..11d97972a2 100644
--- a/source/adios2/engine/dataspaces/DataSpacesWriter.cpp
+++ b/source/adios2/engine/dataspaces/DataSpacesWriter.cpp
@@ -40,7 +40,7 @@ DataSpacesWriter::DataSpacesWriter(IO &io, const std::string &name,
     {
         m_data.appid = 0;
     }
-    MPI_Comm mpiComm = m_Comm;
+    MPI_Comm mpiComm = m_Comm.AsMPI();
     ret = adios_dataspaces_init(&mpiComm, &m_data);
     if (ret < 0)
         fprintf(stderr, "Unable to connect to DataSpaces. Err: %d\n", ret);
diff --git a/source/adios2/engine/hdf5/HDF5ReaderP.cpp b/source/adios2/engine/hdf5/HDF5ReaderP.cpp
index 6a9fbfe962..8d5638562b 100644
--- a/source/adios2/engine/hdf5/HDF5ReaderP.cpp
+++ b/source/adios2/engine/hdf5/HDF5ReaderP.cpp
@@ -62,7 +62,7 @@ void HDF5ReaderP::Init()
             ", in call to Open\n");
     }
 
-    m_H5File.Init(m_Name, m_Comm, false);
+    m_H5File.Init(m_Name, m_Comm.AsMPI(), false);
     m_H5File.ParseParameters(m_IO);
 
     /*
diff --git a/source/adios2/engine/hdf5/HDF5WriterP.cpp b/source/adios2/engine/hdf5/HDF5WriterP.cpp
index c9bd9d804a..d3394239b9 100644
--- a/source/adios2/engine/hdf5/HDF5WriterP.cpp
+++ b/source/adios2/engine/hdf5/HDF5WriterP.cpp
@@ -58,7 +58,7 @@ void HDF5WriterP::Init()
     }
 
 #ifdef NEVER
-    m_H5File.Init(m_Name, m_Comm, true);
+    m_H5File.Init(m_Name, m_Comm.AsMPI(), true);
 #else
     // enforce .h5 ending
     std::string suffix = ".h5";
@@ -71,11 +71,11 @@ void HDF5WriterP::Init()
     {
         // is a file with .bp ending
         std::string updatedName = m_Name.substr(0, wpos) + suffix;
-        m_H5File.Init(updatedName, m_Comm, true);
+        m_H5File.Init(updatedName, m_Comm.AsMPI(), true);
     }
     else
     {
-        m_H5File.Init(m_Name, m_Comm, true);
+        m_H5File.Init(m_Name, m_Comm.AsMPI(), true);
     }
     m_H5File.ParseParameters(m_IO);
 #endif
diff --git a/source/adios2/engine/insitumpi/InSituMPIReader.cpp b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
index 2e7dd3964c..f310fd9e8e 100644
--- a/source/adios2/engine/insitumpi/InSituMPIReader.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIReader.cpp
@@ -37,7 +37,8 @@ InSituMPIReader::InSituMPIReader(IO &io, const std::string &name,
     m_EndMessage = " in call to IO Open InSituMPIReader " + m_Name + "\n";
     Init();
 
-    m_RankAllPeers = insitumpi::FindPeers(m_Comm, m_Name, false, m_CommWorld);
+    m_RankAllPeers =
+        insitumpi::FindPeers(m_Comm.AsMPI(), m_Name, false, m_CommWorld);
     MPI_Comm_rank(m_CommWorld, &m_GlobalRank);
     MPI_Comm_size(m_CommWorld, &m_GlobalNproc);
     m_ReaderRank = m_Comm.Rank();
diff --git a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
index 0ec5593ab6..0a6c0912bd 100644
--- a/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
+++ b/source/adios2/engine/insitumpi/InSituMPIWriter.cpp
@@ -38,7 +38,8 @@ InSituMPIWriter::InSituMPIWriter(IO &io, const std::string &name,
     Init();
     m_BP3Serializer.InitParameters(m_IO.m_Parameters);
 
-    m_RankAllPeers = insitumpi::FindPeers(m_Comm, m_Name, true, m_CommWorld);
+    m_RankAllPeers =
+        insitumpi::FindPeers(m_Comm.AsMPI(), m_Name, true, m_CommWorld);
     for (int i = 0; i < m_RankAllPeers.size(); i++)
     {
         m_RankToPeerID[m_RankAllPeers[i]] = i;
diff --git a/source/adios2/engine/sst/SstReader.cpp b/source/adios2/engine/sst/SstReader.cpp
index 3158a6484c..8bae3e0484 100644
--- a/source/adios2/engine/sst/SstReader.cpp
+++ b/source/adios2/engine/sst/SstReader.cpp
@@ -35,7 +35,7 @@ SstReader::SstReader(IO &io, const std::string &name, const Mode mode,
 
     Init();
 
-    m_Input = SstReaderOpen(cstr, &Params, m_Comm);
+    m_Input = SstReaderOpen(cstr, &Params, m_Comm.AsMPI());
     if (!m_Input)
     {
         throw std::runtime_error(
diff --git a/source/adios2/engine/sst/SstWriter.cpp b/source/adios2/engine/sst/SstWriter.cpp
index 456171b9cd..44de279af6 100644
--- a/source/adios2/engine/sst/SstWriter.cpp
+++ b/source/adios2/engine/sst/SstWriter.cpp
@@ -109,7 +109,7 @@ SstWriter::SstWriter(IO &io, const std::string &name, const Mode mode,
 
     Init();
 
-    m_Output = SstWriterOpen(name.c_str(), &Params, m_Comm);
+    m_Output = SstWriterOpen(name.c_str(), &Params, m_Comm.AsMPI());
 
     if (m_MarshalMethod == SstMarshalBP)
     {
diff --git a/source/adios2/helper/adiosComm.h b/source/adios2/helper/adiosComm.h
index 3230847f1b..45094a48c1 100644
--- a/source/adios2/helper/adiosComm.h
+++ b/source/adios2/helper/adiosComm.h
@@ -65,9 +65,10 @@ class Comm
      */
     void swap(Comm &comm);
 
-    // FIXME: Remove conversion after clients transition to encapsulation.
-    /** Convert to a concrete MPI communicator.  */
-    operator MPI_Comm() const { return m_MPIComm; }
+    /**
+     * @brief Get the underlying raw MPI communicator.
+     */
+    MPI_Comm AsMPI() const { return m_MPIComm; }
 
     /**
      * @brief Create a communicator by duplicating a MPI communicator.