From f7d2d01915f9d3fd86f10651424b8f9f489cc48b Mon Sep 17 00:00:00 2001 From: Rodrigo Bartolomeu Date: Wed, 30 Oct 2024 18:56:57 +0100 Subject: [PATCH] Add heFFTe backends --- .../heffte_fast_fourier_transform_example.cpp | 12 +++-- grid/src/Cabana_Grid_FastFourierTransform.hpp | 52 +++++++++++++++---- grid/unit_test/tstFastFourierTransform.hpp | 3 +- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/example/grid_tutorial/10_fft_heffte/heffte_fast_fourier_transform_example.cpp b/example/grid_tutorial/10_fft_heffte/heffte_fast_fourier_transform_example.cpp index e3160abe6..0c65cf9c5 100644 --- a/example/grid_tutorial/10_fft_heffte/heffte_fast_fourier_transform_example.cpp +++ b/example/grid_tutorial/10_fft_heffte/heffte_fast_fourier_transform_example.cpp @@ -104,9 +104,15 @@ void fastFourierTransformHeffteExample() */ Cabana::Grid::Experimental::FastFourierTransformParams params; - // Set communication to use all-to-all MPI communication. - // Set this option to false for point-to-point communication - params.setAllToAll( true ); + // Set this option to Cabana::Grid::Experimental::FFTCommPattern::alltoallv + // or true use all-to-all communication.with padding + // Set this option to Cabana::Grid::Experimental::FFTCommPattern::p2p or + // false for point-to-point communication + // Set this option to Cabana::Grid::Experimental::FFTCommPattern::alltoall + // for all-to-all communication without padding + // Set this option to Cabana::Grid::Experimental::FFTCommPattern::p2p_plined + // for pipelined point-to-point communication + params.setAlltoAll( Cabana::Grid::Experimental::FFTCommPattern::alltoallv ); // Set data exchange type to use pencil decomposition // Set this option to false to use slab decomposition diff --git a/grid/src/Cabana_Grid_FastFourierTransform.hpp b/grid/src/Cabana_Grid_FastFourierTransform.hpp index 0888916d2..d03a3aa78 100644 --- a/grid/src/Cabana_Grid_FastFourierTransform.hpp +++ b/grid/src/Cabana_Grid_FastFourierTransform.hpp @@ -19,8 +19,6 @@ #include #include -#include // FIXME: remove after next release. - #include #include @@ -97,6 +95,13 @@ struct is_matching_array< : public std::true_type { }; +enum class FFTCommPattern : unsigned int +{ + alltoallv = 0u, + p2p = 1u, + alltoall = 2u, + p2p_plined = 3u +}; //---------------------------------------------------------------------------// /*! @@ -104,6 +109,7 @@ struct is_matching_array< */ class FastFourierTransformParams { + FFTCommPattern FFTcomm = FFTCommPattern::alltoallv; bool alltoall = true; bool pencils = true; bool reorder = true; @@ -113,7 +119,18 @@ class FastFourierTransformParams \brief Set MPI communication strategy. \param value Use all to all MPI communication. */ - void setAllToAll( bool value ) { alltoall = value; } + void setAlltoAll( FFTCommPattern value ) { FFTcomm = value; } + /*! + \brief Set MPI communication strategy. + \param value Use all to all MPI communication. + */ + void setAlltoAll( bool value ) + { + if ( value ) + FFTcomm = FFTCommPattern::alltoallv; + else + FFTcomm = FFTCommPattern::p2p; + } /*! \brief Set data exchange type (pencil or slab). \param value Use pencil (true) or slab (false) decomposition. @@ -129,7 +146,7 @@ class FastFourierTransformParams \brief Get MPI communication strategy. \return Using AllToAll or not. */ - bool getAllToAll() const { return alltoall; } + FFTCommPattern getAlltoAll() const { return FFTcomm; } /*! \brief Get data exchange type (pencil or slab). \return Using pencil (true) or slab (false) decomposition. @@ -538,12 +555,26 @@ class HeffteFastFourierTransform heffte::plan_options heffte_params = heffte::default_options(); - // TODO: use all three heffte options for algorithm - bool alltoall = params.getAllToAll(); - if ( alltoall ) - heffte_params.algorithm = heffte::reshape_algorithm::alltoallv; - else + auto FFTcomm = params.getAlltoAll(); + switch ( FFTcomm ) + { + case Cabana::Grid::Experimental::FFTCommPattern::p2p: heffte_params.algorithm = heffte::reshape_algorithm::p2p; + break; + case Cabana::Grid::Experimental::FFTCommPattern::alltoallv: + heffte_params.algorithm = heffte::reshape_algorithm::alltoallv; + break; + case Cabana::Grid::Experimental::FFTCommPattern::alltoall: + heffte_params.algorithm = heffte::reshape_algorithm::alltoall; + break; + case Cabana::Grid::Experimental::FFTCommPattern::p2p_plined: + heffte_params.algorithm = heffte::reshape_algorithm::p2p_plined; + break; + default: + heffte_params.algorithm = heffte::reshape_algorithm::alltoallv; + break; + } + heffte_params.use_pencils = params.getPencils(); heffte_params.use_reorder = params.getReorder(); @@ -698,8 +729,7 @@ auto createHeffteFastFourierTransform( const heffte::plan_options heffte_params = heffte::default_options(); FastFourierTransformParams params; - // TODO: set appropriate default for AllToAll - params.setAllToAll( true ); + params.setAlltoAll( FFTCommPattern::alltoallv ); params.setPencils( heffte_params.use_pencils ); params.setReorder( heffte_params.use_reorder ); diff --git a/grid/unit_test/tstFastFourierTransform.hpp b/grid/unit_test/tstFastFourierTransform.hpp index b2ca53085..ee6e85716 100644 --- a/grid/unit_test/tstFastFourierTransform.hpp +++ b/grid/unit_test/tstFastFourierTransform.hpp @@ -39,7 +39,8 @@ void calculateFFT( bool use_default, bool use_params, Experimental::FastFourierTransformParams params; // Set MPI communication - params.setAllToAll( true ); + params.setAlltoAll( Cabana::Grid::Experimental::FFTCommPattern::alltoallv ); + params.setAlltoAll( true ); // Set data exchange type (false uses slab decomposition) params.setPencils( true );