Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move MPI rank topology functions out of Impl #666

Merged
merged 1 commit into from
Aug 24, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 17 additions & 9 deletions cajita/src/Cajita_ParticleGridDistributor.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,12 @@ namespace Cajita
// Particle Grid Distributor
//---------------------------------------------------------------------------//

namespace Impl
{
//! \cond Impl

// Build neighbor topology of 27 nearest 3D neighbors. Some of the ranks in
// this list may be invalid.
/*!
\brief Build neighbor topology of 27 nearest 3D neighbors. Some of the ranks
in this list may be invalid.
\param local_grid Local grid from which MPI neighbors will be extracted.
\return MPI neighbor ranks in k,j,i order.
*/
template <class LocalGridType, std::size_t NSD = LocalGridType::num_space_dim>
std::enable_if_t<3 == NSD, std::vector<int>>
getTopology( const LocalGridType& local_grid )
Expand All @@ -50,8 +50,12 @@ getTopology( const LocalGridType& local_grid )
return topology;
}

// Build neighbor topology of 8 nearest 2D neighbors. Some of the ranks in
// this list may be invalid.
/*!
\brief Build neighbor topology of 8 nearest 2D neighbors. Some of the ranks
in this list may be invalid.
\param local_grid Local grid from which MPI neighbors will be extracted.
\return MPI neighbor ranks in k,j,i order.
*/
template <class LocalGridType, std::size_t NSD = LocalGridType::num_space_dim>
std::enable_if_t<2 == NSD, std::vector<int>>
getTopology( const LocalGridType& local_grid )
Expand All @@ -64,6 +68,10 @@ getTopology( const LocalGridType& local_grid )
return topology;
}

namespace Impl
{
//! \cond Impl

// Locate the particles in the local grid and get their destination rank.
// Particles are assumed to only migrate to a location in the nearest
// neighbor halo or stay on this rank. If the particle crosses a global
Expand Down Expand Up @@ -237,7 +245,7 @@ createParticleGridDistributor( const LocalGridType& local_grid,
using device_type = typename PositionSliceType::device_type;

// Get all 26 neighbor ranks.
auto topology = Impl::getTopology( local_grid );
auto topology = getTopology( local_grid );

Kokkos::View<int*, Kokkos::HostSpace, Kokkos::MemoryUnmanaged>
topology_host( topology.data(), topology.size() );
Expand Down
17 changes: 11 additions & 6 deletions core/src/Cabana_CommunicationPlan.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,16 @@ auto countSendsAndCreateSteering( const ExportRankView element_export_ranks,
}

//---------------------------------------------------------------------------//
// Return unique neighbor ranks, with the current rank first.
//! \endcond
} // end namespace Impl

//---------------------------------------------------------------------------//
/*!
\brief Return unique neighbor ranks, with the current rank first.
\param comm MPI communicator.
\param topology MPI neighbor ranks in any order with possible duplication.
\return Unique MPI neighbor ranks, with the current rank first.
*/
inline std::vector<int> getUniqueTopology( MPI_Comm comm,
std::vector<int> topology )
{
Expand All @@ -379,10 +388,6 @@ inline std::vector<int> getUniqueTopology( MPI_Comm comm,
return topology;
}

//---------------------------------------------------------------------------//
//! \endcond
} // end namespace Impl

//---------------------------------------------------------------------------//
/*!
\brief Communication plan base class.
Expand Down Expand Up @@ -596,7 +601,7 @@ class CommunicationPlan
_num_export_element = element_export_ranks.size();

// Store the unique neighbors (this rank first).
_neighbors = Impl::getUniqueTopology( comm(), neighbor_ranks );
_neighbors = getUniqueTopology( comm(), neighbor_ranks );
int num_n = _neighbors.size();

// Get the size of this communicator.
Expand Down
2 changes: 1 addition & 1 deletion core/unit_test/tstCommunicationPlan.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -586,7 +586,7 @@ void testTopology()
std::vector<int> neighbor_ranks = { 0, 2, 2, 1, 1, 1,
2, 3, 10, 27, my_rank };
auto unique_ranks =
Cabana::Impl::getUniqueTopology( MPI_COMM_WORLD, neighbor_ranks );
Cabana::getUniqueTopology( MPI_COMM_WORLD, neighbor_ranks );

// Check this rank is first.
EXPECT_EQ( my_rank, unique_ranks[0] );
Expand Down
Loading