From 0d381f13f66efd924df4d6b8080478cec104ca74 Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Wed, 16 Aug 2023 13:31:00 -0400 Subject: [PATCH] Move MPI rank topology functions out of Impl --- cajita/src/Cajita_ParticleGridDistributor.hpp | 26 ++++++++++++------- core/src/Cabana_CommunicationPlan.hpp | 17 +++++++----- core/unit_test/tstCommunicationPlan.hpp | 2 +- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/cajita/src/Cajita_ParticleGridDistributor.hpp b/cajita/src/Cajita_ParticleGridDistributor.hpp index f6705aa13..2de653f5f 100644 --- a/cajita/src/Cajita_ParticleGridDistributor.hpp +++ b/cajita/src/Cajita_ParticleGridDistributor.hpp @@ -31,12 +31,12 @@ namespace Cajita // Particle Grid Distributor //---------------------------------------------------------------------------// -namespace Impl -{ -//! \cond Impl - -// Build neighbor topology of 27 nearest 3D neighbors. Some of the ranks in -// this list may be invalid. +/*! + \brief Build neighbor topology of 27 nearest 3D neighbors. Some of the ranks + in this list may be invalid. + \param local_grid Local grid from which MPI neighbors will be extracted. + \return MPI neighbor ranks in k,j,i order. +*/ template std::enable_if_t<3 == NSD, std::vector> getTopology( const LocalGridType& local_grid ) @@ -50,8 +50,12 @@ getTopology( const LocalGridType& local_grid ) return topology; } -// Build neighbor topology of 8 nearest 2D neighbors. Some of the ranks in -// this list may be invalid. +/*! + \brief Build neighbor topology of 8 nearest 2D neighbors. Some of the ranks + in this list may be invalid. + \param local_grid Local grid from which MPI neighbors will be extracted. + \return MPI neighbor ranks in k,j,i order. +*/ template std::enable_if_t<2 == NSD, std::vector> getTopology( const LocalGridType& local_grid ) @@ -64,6 +68,10 @@ getTopology( const LocalGridType& local_grid ) return topology; } +namespace Impl +{ +//! \cond Impl + // Locate the particles in the local grid and get their destination rank. // Particles are assumed to only migrate to a location in the nearest // neighbor halo or stay on this rank. If the particle crosses a global @@ -237,7 +245,7 @@ createParticleGridDistributor( const LocalGridType& local_grid, using device_type = typename PositionSliceType::device_type; // Get all 26 neighbor ranks. - auto topology = Impl::getTopology( local_grid ); + auto topology = getTopology( local_grid ); Kokkos::View topology_host( topology.data(), topology.size() ); diff --git a/core/src/Cabana_CommunicationPlan.hpp b/core/src/Cabana_CommunicationPlan.hpp index 380020d90..b57b1bdbb 100644 --- a/core/src/Cabana_CommunicationPlan.hpp +++ b/core/src/Cabana_CommunicationPlan.hpp @@ -356,7 +356,16 @@ auto countSendsAndCreateSteering( const ExportRankView element_export_ranks, } //---------------------------------------------------------------------------// -// Return unique neighbor ranks, with the current rank first. +//! \endcond +} // end namespace Impl + +//---------------------------------------------------------------------------// +/*! + \brief Return unique neighbor ranks, with the current rank first. + \param comm MPI communicator. + \param topology MPI neighbor ranks in any order with possible duplication. + \return Unique MPI neighbor ranks, with the current rank first. +*/ inline std::vector getUniqueTopology( MPI_Comm comm, std::vector topology ) { @@ -379,10 +388,6 @@ inline std::vector getUniqueTopology( MPI_Comm comm, return topology; } -//---------------------------------------------------------------------------// -//! \endcond -} // end namespace Impl - //---------------------------------------------------------------------------// /*! \brief Communication plan base class. @@ -596,7 +601,7 @@ class CommunicationPlan _num_export_element = element_export_ranks.size(); // Store the unique neighbors (this rank first). - _neighbors = Impl::getUniqueTopology( comm(), neighbor_ranks ); + _neighbors = getUniqueTopology( comm(), neighbor_ranks ); int num_n = _neighbors.size(); // Get the size of this communicator. diff --git a/core/unit_test/tstCommunicationPlan.hpp b/core/unit_test/tstCommunicationPlan.hpp index 8551f4370..27fb08fa8 100644 --- a/core/unit_test/tstCommunicationPlan.hpp +++ b/core/unit_test/tstCommunicationPlan.hpp @@ -586,7 +586,7 @@ void testTopology() std::vector neighbor_ranks = { 0, 2, 2, 1, 1, 1, 2, 3, 10, 27, my_rank }; auto unique_ranks = - Cabana::Impl::getUniqueTopology( MPI_COMM_WORLD, neighbor_ranks ); + Cabana::getUniqueTopology( MPI_COMM_WORLD, neighbor_ranks ); // Check this rank is first. EXPECT_EQ( my_rank, unique_ranks[0] );