From ba2e05df9e1777ca6227a6a4ce70005af8053a18 Mon Sep 17 00:00:00 2001 From: Heather Pacella Date: Fri, 8 Sep 2023 16:23:19 -0600 Subject: [PATCH] STK: Snapshot 09-08-23 16:23 from Sierra 5.15.5-101-ge9dc2961 --- packages/stk/CMakeLists.txt | 130 ++- packages/stk/cmake/Dependencies.cmake | 6 +- packages/stk/cmake/STK_Trilinos_config.h.in | 2 +- packages/stk/cmake/stk_wrappers.cmake | 180 ++++ .../stk_balance/stk_balance/CMakeLists.txt | 7 +- .../stk/stk_balance/stk_balance/balance.cpp | 6 +- .../stk_balance/stk_balance/balanceUtils.cpp | 22 +- .../stk_balance/stk_balance/balanceUtils.hpp | 8 +- .../internal/Zoltan2ParallelGraph.cpp | 57 +- .../internal/privateDeclarations.cpp | 251 +++--- .../internal/privateDeclarations.hpp | 4 +- .../stk_balance/stk_balance/io/BalanceIO.cpp | 2 +- .../stk_balance/setup/LifeCycle.cpp | 2 +- .../stk_balance/setup/M2NParser.cpp | 4 +- .../stk_balance/stk_balance/setup/Parser.cpp | 2 +- packages/stk/stk_coupling/CMakeLists.txt | 6 +- packages/stk/stk_coupling/Jamfile | 11 + .../stk_coupling/stk_coupling/CMakeLists.txt | 23 +- .../stk_coupling/stk_coupling/SyncInfo.cpp | 2 - .../stk_coupling/stk_coupling/SyncInfo.hpp | 5 +- packages/stk/stk_doc_tests/CMakeLists.txt | 5 + .../stk_doc_tests/cmake/Dependencies.cmake | 2 +- .../stk_doc_tests/stk_balance/CMakeLists.txt | 3 - .../stk_coupling/BasicCommSplit.cpp | 233 +++++ .../stk_doc_tests/stk_coupling/CMakeLists.txt | 52 ++ .../stk_doc_tests/stk_expreval/CMakeLists.txt | 4 - .../stk/stk_doc_tests/stk_io/CMakeLists.txt | 8 +- .../stk/stk_doc_tests/stk_mesh/CMakeLists.txt | 9 +- .../stk_doc_tests/stk_search/CMakeLists.txt | 6 +- .../stk_search/howToUseCoarseSearch.cpp | 201 +++++ .../stk_search/howToUseFilterCoarseSearch.cpp | 634 +------------- .../stk_doc_tests/stk_search/searchHex8.hpp | 335 +++++++ .../stk_search/searchMockMesh.hpp | 370 ++++++++ .../stk/stk_doc_tests/stk_simd/CMakeLists.txt | 8 +- .../stk_doc_tests/stk_topology/CMakeLists.txt | 3 - .../stk_doc_tests/stk_transfer/CMakeLists.txt | 1 - .../stk/stk_doc_tests/stk_util/CMakeLists.txt | 6 +- .../stk_util/TimerHowToParallel.cpp | 8 +- .../stk_util/filenameSubstitution.cpp | 7 +- packages/stk/stk_expreval/CMakeLists.txt | 6 +- packages/stk/stk_expreval/Jamfile | 2 +- .../stk/stk_expreval/cmake/Dependencies.cmake | 2 +- .../stk_expreval/stk_expreval/CMakeLists.txt | 56 +- .../stk_expreval/unit_tests/CMakeLists.txt | 19 - .../build_stk_no_stk_io_using_cmake | 8 +- .../build_stk_serial_using_cmake | 14 +- .../build_stk_standalone_serial_using_cmake | 78 ++ .../build_stk_standalone_using_cmake | 78 ++ ...ld_stk_transfer_no_middle_mesh_using_cmake | 8 +- .../cmake_install_test/build_stk_user_facing | 31 +- .../cmake_install_test/build_stk_using_cmake | 36 +- .../cmake_install_test/create_workspace.sh | 32 + .../cmake_install_test/load_gcc_modules | 7 +- .../load_gcc_modules_no_boost | 30 + .../cmake_install_test/run_cmake_stk | 4 +- .../cmake_install_test/run_cmake_stk_serial | 9 +- .../run_cmake_stk_standalone | 45 + .../run_cmake_stk_standalone_serial | 42 + .../run_cmake_stk_user_facing | 5 +- .../cmake_install_test/stk-integrate.cmake | 51 ++ .../stk_test_app/src/CMakeLists.txt | 2 + .../stk_test_app/src/test_stk_coupling.cpp | 18 + .../stk_test_app/src/test_stk_coupling.hpp | 11 + .../stk_test_app/src/test_stk_search.cpp | 16 + .../stk_test_app/src/test_stk_search.hpp | 11 + .../stk_test_app/test_stk_app.cpp | 6 + .../IntegrationTestSpiderElements.cpp | 2 +- .../stk_middle_mesh/NonConformalInterface.cpp | 1 - packages/stk/stk_io/stk_io/CMakeLists.txt | 15 +- .../stk/stk_io/stk_io/StkMeshIoBroker.cpp | 30 +- .../stk/stk_io/stk_io/StkMeshIoBroker.hpp | 5 + .../stk/stk_io/stk_io/util/CMakeLists.txt | 28 +- packages/stk/stk_math/CMakeLists.txt | 4 +- packages/stk/stk_math/stk_math/CMakeLists.txt | 49 +- .../stk/stk_mesh/stk_mesh/base/BulkData.cpp | 127 +-- .../stk/stk_mesh/stk_mesh/base/BulkData.hpp | 43 +- .../stk/stk_mesh/stk_mesh/base/CMakeLists.txt | 23 +- .../stk_mesh/base/DestroyElements.cpp | 173 ++-- .../stk_mesh/base/FieldDataManager.cpp | 38 +- .../stk_mesh/base/FieldDataManager.hpp | 19 +- .../stk_mesh/stk_mesh/base/FieldParallel.cpp | 20 - .../stk_mesh/stk_mesh/base/FieldParallel.hpp | 24 - .../stk_mesh/stk_mesh/base/MeshBuilder.cpp | 16 +- .../stk_mesh/stk_mesh/base/MeshBuilder.hpp | 17 +- .../stk_mesh/base/NgpFieldParallel.hpp | 38 +- .../stk_mesh/baseImpl/MeshImplUtils.cpp | 50 +- .../stk_mesh/baseImpl/MeshImplUtils.hpp | 7 + .../stk_mesh/baseImpl/MeshModLogObserver.cpp | 5 +- .../stk_mesh/baseImpl/MeshModification.cpp | 16 +- .../stk_mesh/baseImpl/check_comm_list.cpp | 7 +- packages/stk/stk_middle_mesh/CMakeLists.txt | 4 +- .../stk_middle_mesh/cmake/Dependencies.cmake | 4 +- .../stk_middle_mesh/CMakeLists.txt | 34 +- .../stk_middle_mesh/adjacency_search.cpp | 5 + ...plication_interface_fake_parallel_impl.cpp | 5 +- .../stk_middle_mesh/bounding_box_search.hpp | 26 +- .../destination_field_gatherer.cpp | 16 +- .../element_mesh_classifier.cpp | 42 +- .../element_mesh_classifier.hpp | 4 +- .../entity_sorted_by_owner.cpp | 44 - .../entity_sorted_by_owner.hpp | 52 +- .../stk_middle_mesh/stk_middle_mesh/mesh.cpp | 52 +- .../stk_middle_mesh/stk_middle_mesh/mesh.hpp | 4 +- .../stk_middle_mesh/mesh_entity.hpp | 10 +- .../mesh_exchange_boundary_edges.cpp | 8 +- .../mesh_projection_calculator.cpp | 172 ++-- .../mesh_projection_calculator.hpp | 12 +- .../stk_middle_mesh/mesh_relational_data.hpp | 24 +- .../mesh_relational_data_scatter.cpp | 700 +++++++++++++++ .../mesh_relational_data_scatter.hpp | 124 +++ .../stk_middle_mesh/mesh_scatter.cpp | 264 ++++-- .../stk_middle_mesh/mesh_scatter.hpp | 51 +- .../middle_grid_constraint_generator.cpp | 169 +++- .../middle_grid_constraint_generator.hpp | 13 + .../middle_grid_triangulator.cpp | 8 +- .../stk_middle_mesh/nonconformal4.cpp | 2 +- .../predicates/average_normal_field.hpp | 203 +++++ .../predicates/intersection_common.hpp | 15 + .../point_classifier_normal_wrapper.cpp | 65 +- .../point_classifier_normal_wrapper.hpp | 12 +- ...p => search_mesh_element_bounding_box.hpp} | 70 +- .../stk_middle_mesh/search_mesh_vertex.hpp | 83 ++ packages/stk/stk_ngp_test/CMakeLists.txt | 4 +- .../stk_ngp_test/stk_ngp_test/CMakeLists.txt | 23 +- .../stk_mesh/NodalFieldPerf.cpp | 26 +- .../stk_util/perfTagManager.cpp | 8 +- packages/stk/stk_search/CMakeLists.txt | 4 +- .../stk/stk_search/stk_search/CMakeLists.txt | 50 +- .../stk_search/FilterCoarseSearch.hpp | 12 + .../stk_search/stk_search/SearchInterface.hpp | 155 ++-- packages/stk/stk_simd/CMakeLists.txt | 4 +- packages/stk/stk_simd/stk_math/CMakeLists.txt | 27 - packages/stk/stk_simd/stk_simd/CMakeLists.txt | 40 +- .../stk_simd/kokkos_simd/CMakeLists.txt | 26 +- .../stk/stk_simd/stk_simd_view/CMakeLists.txt | 27 - .../stk_simd/stk_simd_view/simd_parallel.hpp | 2 +- .../stk/stk_tools/stk_tools/CMakeLists.txt | 23 +- .../stk_tools/mesh_tools/DisconnectBlocks.cpp | 2 +- .../mesh_tools/DisconnectBlocksImpl.hpp | 8 +- .../stk_tools/pmesh_lib/makeparfiles.cpp | 14 +- packages/stk/stk_topology/CMakeLists.txt | 4 +- .../stk_topology/stk_topology/CMakeLists.txt | 47 +- packages/stk/stk_transfer/CMakeLists.txt | 6 +- .../stk_transfer/stk_transfer/CMakeLists.txt | 35 +- .../stk/stk_unit_test_utils/CMakeLists.txt | 4 +- .../stk_unit_test_utils/BulkDataTester.hpp | 4 +- .../stk_unit_test_utils/CMakeLists.txt | 75 +- .../ParallelGtestOutput.cpp | 15 + .../stk_unit_test_utils/TextMesh.cpp | 45 +- .../TextMeshStkTopologyMapping.hpp | 26 + .../stk_mesh_fixtures/BoxFixture.cpp | 10 +- .../unit_main_lib/UnitTestMain.cpp | 2 + packages/stk/stk_unit_tests/CMakeLists.txt | 76 +- .../stk_unit_tests/cmake/Dependencies.cmake | 2 +- .../UnitTestDiagnosticsComputation.cpp | 2 +- .../stk_balance/UnitTestRebalanceSpiders.cpp | 10 +- .../stk_balance/UnitTestSpiderElements.cpp | 100 ++- .../stk_balance/UnitTestSpiderMeshSetup.hpp | 71 ++ .../stk_coupling/CMakeLists.txt | 15 +- .../stk_expreval/CMakeLists.txt | 20 + .../stk_expreval}/UnitTestEvaluator.cpp | 0 .../stk/stk_unit_tests/stk_io/CMakeLists.txt | 4 - .../stk_io/UnitTestCustomMeshBuilder.cpp | 186 ++++ .../stk_unit_tests/stk_math/CMakeLists.txt | 26 +- .../stk_unit_tests/stk_mesh/CMakeLists.txt | 2 + .../stk_unit_tests/stk_mesh/UnitTestAura.cpp | 205 ++++- .../stk_mesh/UnitTestAuraToShared.cpp | 44 + .../stk_mesh/UnitTestDestroyElements.cpp | 36 + .../stk_unit_tests/stk_mesh/UnitTestField.cpp | 206 +++-- .../stk_mesh/UnitTestFieldDataManager.cpp | 92 +- .../stk_mesh/UnitTestMeshModLogObserver.cpp | 20 +- .../stk_mesh/UnitTestStkTextMesh.cpp | 157 ++++ .../stk_middle_mesh/CMakeLists.txt | 29 +- .../test_average_normal_field.cpp | 33 + .../test_destination_field_gather.cpp | 36 +- .../test_entity_sorted_by_owner.cpp | 20 +- .../stk_middle_mesh/test_mesh.cpp | 30 + .../test_mesh_projection_calculator.cpp | 2 +- .../test_mesh_relational_data_scatter.cpp | 803 +++++++++++++++++ .../stk_middle_mesh/test_mesh_scatter.cpp | 530 ++++++++++- .../test_middle_grid_constraint_generator.cpp | 2 +- ...dle_grid_constraint_generator_parallel.cpp | 821 ++++++++++++++++++ .../test_middle_grid_triangulator.cpp | 4 - ...test_middle_grid_triangulator_parallel.cpp | 793 +++++++++++++++++ .../stk_middle_mesh/test_parallel_search.cpp | 53 +- .../util/nonconformal_interface_helpers.cpp | 32 +- .../util/nonconformal_interface_helpers.hpp | 4 +- .../util/parallel_search_test_util.hpp | 4 +- .../stk_ngp_test/CMakeLists.txt | 16 +- .../stk_unit_tests/stk_search/CMakeLists.txt | 23 +- .../stk_search/UnitTestFilterCoarseSearch.cpp | 12 +- .../stk_unit_tests/stk_simd/CMakeLists.txt | 33 +- .../stk_simd_old/CMakeLists.txt | 34 +- .../stk_unit_tests/stk_tools/CMakeLists.txt | 1 - .../stk_topology/CMakeLists.txt | 24 +- .../stk_transfer/CMakeLists.txt | 29 +- .../stk_unit_tests/stk_util/CMakeLists.txt | 24 +- .../stk_util/parallel/UnitTestCommSparse.cpp | 121 ++- packages/stk/stk_util/CMakeLists.txt | 4 +- packages/stk/stk_util/stk_util/CMakeLists.txt | 9 +- .../stk_util/command_line/CMakeLists.txt | 40 +- .../stk/stk_util/stk_util/diag/CMakeLists.txt | 50 +- .../stk_util/environment/CMakeLists.txt | 45 +- .../stk_util/stk_util/environment/EnvData.cpp | 8 + .../stk_util/stk_util/environment/EnvData.hpp | 2 + .../stk_util/environment/RuntimeMessage.cpp | 3 +- .../stk/stk_util/stk_util/ngp/CMakeLists.txt | 24 +- .../stk_util/stk_util/parallel/CMakeLists.txt | 52 +- .../stk_util/stk_util/parallel/CommSparse.cpp | 21 +- .../stk_util/stk_util/parallel/CommSparse.hpp | 27 +- .../stk_util/parallel/ReceiveCounter.cpp | 2 + .../stk_util/stk_util/registry/CMakeLists.txt | 46 +- .../stk_util/registry/ProductRegistry.cpp | 2 +- packages/stk/stk_util/stk_util/stk_config.h | 5 +- .../stk/stk_util/stk_util/util/CMakeLists.txt | 44 +- 215 files changed, 9416 insertions(+), 2789 deletions(-) create mode 100644 packages/stk/cmake/stk_wrappers.cmake create mode 100644 packages/stk/stk_doc_tests/stk_coupling/BasicCommSplit.cpp create mode 100644 packages/stk/stk_doc_tests/stk_coupling/CMakeLists.txt create mode 100644 packages/stk/stk_doc_tests/stk_search/howToUseCoarseSearch.cpp create mode 100644 packages/stk/stk_doc_tests/stk_search/searchHex8.hpp create mode 100644 packages/stk/stk_doc_tests/stk_search/searchMockMesh.hpp delete mode 100644 packages/stk/stk_expreval/unit_tests/CMakeLists.txt create mode 100755 packages/stk/stk_integration_tests/cmake_install_test/build_stk_standalone_serial_using_cmake create mode 100755 packages/stk/stk_integration_tests/cmake_install_test/build_stk_standalone_using_cmake create mode 100755 packages/stk/stk_integration_tests/cmake_install_test/create_workspace.sh create mode 100644 packages/stk/stk_integration_tests/cmake_install_test/load_gcc_modules_no_boost create mode 100755 packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone create mode 100755 packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone_serial create mode 100644 packages/stk/stk_integration_tests/cmake_install_test/stk-integrate.cmake create mode 100644 packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_coupling.cpp create mode 100644 packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_coupling.hpp create mode 100644 packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_search.cpp create mode 100644 packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_search.hpp delete mode 100644 packages/stk/stk_middle_mesh/stk_middle_mesh/entity_sorted_by_owner.cpp create mode 100644 packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_relational_data_scatter.cpp create mode 100644 packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_relational_data_scatter.hpp create mode 100644 packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/average_normal_field.hpp rename packages/stk/stk_middle_mesh/stk_middle_mesh/{parallel_search.hpp => search_mesh_element_bounding_box.hpp} (88%) create mode 100644 packages/stk/stk_middle_mesh/stk_middle_mesh/search_mesh_vertex.hpp create mode 100644 packages/stk/stk_unit_tests/stk_expreval/CMakeLists.txt rename packages/stk/{stk_expreval/unit_tests => stk_unit_tests/stk_expreval}/UnitTestEvaluator.cpp (100%) create mode 100644 packages/stk/stk_unit_tests/stk_io/UnitTestCustomMeshBuilder.cpp create mode 100644 packages/stk/stk_unit_tests/stk_middle_mesh/test_average_normal_field.cpp create mode 100644 packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh_relational_data_scatter.cpp create mode 100644 packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_constraint_generator_parallel.cpp create mode 100644 packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_triangulator_parallel.cpp diff --git a/packages/stk/CMakeLists.txt b/packages/stk/CMakeLists.txt index 1b3afd911bc4..bf77c842c818 100644 --- a/packages/stk/CMakeLists.txt +++ b/packages/stk/CMakeLists.txt @@ -32,23 +32,101 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -# -# A) Define the package -# +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) -TRIBITS_PACKAGE_DECL( STK ) +IF(COMMAND TRIBITS_PACKAGE_DECL) + SET(HAVE_STK_Trilinos ON) + TRIBITS_PACKAGE_DECL(STK) + MESSAGE("*** Building STK as a Trilinos package. ***") +ELSE() + SET(HAVE_STK_Trilinos OFF) + project(STK CXX) -TRIBITS_ADD_DEBUG_OPTION() -TRIBITS_ADD_SHOW_DEPRECATED_WARNINGS_OPTION() + SET(PACKAGE_NAME "STK") -# -# B) Set up package-specific options -# + MESSAGE("*** Building STK as a stand-alone cmake package. ***") +ENDIF() + +SET(STK_TOPLEVEL_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +SET(STK_TOPLEVEL_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +INCLUDE(${STK_TOPLEVEL_SOURCE_DIR}/cmake/stk_wrappers.cmake) + +if(CMAKE_INSTALL_INCLUDE_DIR) + SET(STK_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDE_DIR}) +elseif(INSTALL_INCLUDE_DIR) + SET(STK_INSTALL_INCLUDE_DIR ${INSTALL_INCLUDE_DIR}) +else() + SET(STK_INSTALL_INCLUDE_DIR "include") +endif() + +if(CMAKE_INSTALL_LIBDIR) + SET(STK_INSTALL_LIBDIR ${CMAKE_INSTALL_LIBDIR}) +elseif(INSTALL_LIBDIR) + SET(STK_INSTALL_LIBDIR ${INSTALL_LIBDIR}) +else() + SET(STK_INSTALL_LIBDIR "lib") +endif() + +if(CMAKE_INSTALL_BINDIR) + SET(STK_INSTALL_BINDIR ${CMAKE_INSTALL_BINDIR}) +elseif(INSTALL_BINDIR) + SET(STK_INSTALL_BINDIR ${INSTALL_BINDIR}) +else() + SET(STK_INSTALL_BINDIR "bin") +endif() + +STK_ADD_DEBUG_AND_DEPRECATED_OPTIONS() + +MESSAGE("\nPROJECT_NAME: '${PROJECT_NAME}'") +MESSAGE("PACKAGE_NAME: '${PACKAGE_NAME}'") +MESSAGE("${PACKAGE_NAME}_SOURCE_DIR: '${${PACKAGE_NAME}_SOURCE_DIR}'\n") + +IF (HAVE_STK_Trilinos) + SET(STK_HAVE_KOKKOS ON) + if (TPL_ENABLE_MPI OR HAVE_MPI) + message("STK_HAS_MPI is true") + SET(STK_HAS_MPI ON) + else() + message("STK_HAS_MPI is false") + SET(STK_HAS_MPI OFF) + endif() +ELSE() + if(STK_ENABLE_TESTS) + enable_testing() + endif() -IF (${${PROJECT_NAME}_ENABLE_Kokkos}) + find_package(Kokkos REQUIRED) SET(STK_HAVE_KOKKOS ON) + + IF(DEFINED STK_ENABLE_MPI) + IF(STK_ENABLE_MPI) + MESSAGE("MPI requested via STK_ENABLE_MPI=ON") + ELSE() + MESSAGE("MPI disabled via STK_ENABLE_MPI=OFF") + ENDIF() + ELSE() + MESSAGE("MPI defaulting to off. (STK_ENABLE_MPI not defined)") + ENDIF() + + IF(STK_ENABLE_MPI) + find_package(MPI) + IF(MPI_FOUND) + SET(STK_HAS_MPI ON) + include_directories(SYSTEM ${MPI_INCLUDE_PATH}) + message("MPI_INCLUDE_PATH: ${MPI_INCLUDE_PATH}") + ELSE() + MESSAGE(FATAL_ERROR "MPI enabled by '-DSTK_ENABLE_MPI' but not found.") + ENDIF() + ELSE() + MESSAGE("Building serial without MPI. (To enable MPI, use '-DSTK_ENABLE_MPI:BOOL=ON')") + ENDIF() ENDIF() +if(NOT HAVE_STK_Trilinos) + stk_process_enables() +endif() + IF (TPL_ENABLE_Boost) SET(STK_HAVE_BOOST ON) ENDIF() @@ -72,13 +150,6 @@ IF (NOT ${PROJECT_NAME}_ENABLE_SEACASAprepro_lib) SET(NOT_HAVE_STK_SEACASAPREPRO_LIB TRUE) ENDIF() -TRIBITS_ADD_OPTION_AND_DEFINE( - ${PACKAGE_NAME}_Trilinos - HAVE_STK_Trilinos - "Building as a Trilinos package" - ON - ) - # set fortran mangling options IF("${FC_FN_UNDERSCORE}" STREQUAL "NO_UNDER") @@ -92,24 +163,17 @@ ELSE() SET(FORTRAN_ONE_UNDERSCORE ON) ENDIF() -TRIBITS_CONFIGURE_FILE(STK_Trilinos_config.h) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) +MESSAGE("\nCMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}") +MESSAGE("${PROJECT_NAME}_INSTALL_INCLUDE_DIR: ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}") +MESSAGE("${PROJECT_NAME}_INSTALL_LIBDIR: ${${PROJECT_NAME}_INSTALL_LIBDIR}") +MESSAGE("${PROJECT_NAME}_INSTALL_BINDIR: ${${PROJECT_NAME}_INSTALL_BINDIR}\n") -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/STK_Trilinos_config.h DESTINATION - ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/) -# -# C) Add the libraries, tests, and examples -TRIBITS_PROCESS_SUBPACKAGES() +STK_CONFIGURE_FILE(STK_Trilinos_config.h) -TRIBITS_PACKAGE_DEF() +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/STK_Trilinos_config.h DESTINATION + ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_util) -#need to find out what to exclude -#TRIBITS_EXCLUDE_FILES( -# ) - -# -# D) Do standard postprocessing -# +STK_SUBPACKAGES() -TRIBITS_PACKAGE_POSTPROCESS() +STK_PACKAGE_POSTPROCESS() diff --git a/packages/stk/cmake/Dependencies.cmake b/packages/stk/cmake/Dependencies.cmake index 9b1fea8f3fc6..efef65f9ec8f 100644 --- a/packages/stk/cmake/Dependencies.cmake +++ b/packages/stk/cmake/Dependencies.cmake @@ -6,16 +6,16 @@ SET(SUBPACKAGES_DIRS_CLASSIFICATIONS_OPTREQS Math stk_math PT OPTIONAL Simd stk_simd PT OPTIONAL NGP_TEST stk_ngp_test PT OPTIONAL + ExprEval stk_expreval PT OPTIONAL Topology stk_topology PT OPTIONAL - Mesh stk_mesh PT OPTIONAL - IO stk_io PT OPTIONAL Search stk_search PT OPTIONAL Middle_mesh stk_middle_mesh EX OPTIONAL Transfer stk_transfer PT OPTIONAL + Mesh stk_mesh PT OPTIONAL + IO stk_io PT OPTIONAL Tools stk_tools PT OPTIONAL Balance stk_balance PT OPTIONAL Unit_test_utils stk_unit_test_utils PT OPTIONAL - ExprEval stk_expreval PT OPTIONAL SearchUtil stk_search_util EX OPTIONAL TransferUtil stk_transfer_util EX OPTIONAL Unit_tests stk_unit_tests PT OPTIONAL diff --git a/packages/stk/cmake/STK_Trilinos_config.h.in b/packages/stk/cmake/STK_Trilinos_config.h.in index 353c3adfa30c..417a53fe375f 100644 --- a/packages/stk/cmake/STK_Trilinos_config.h.in +++ b/packages/stk/cmake/STK_Trilinos_config.h.in @@ -48,7 +48,7 @@ #cmakedefine STK_HAVE_STKNGP_TEST -#cmakedefine HAVE_MPI +#cmakedefine STK_HAS_MPI #cmakedefine STK_VOLATILE_SIMD diff --git a/packages/stk/cmake/stk_wrappers.cmake b/packages/stk/cmake/stk_wrappers.cmake new file mode 100644 index 000000000000..34957c95068b --- /dev/null +++ b/packages/stk/cmake/stk_wrappers.cmake @@ -0,0 +1,180 @@ + +macro(STK_ADD_DEBUG_AND_DEPRECATED_OPTIONS) + if(HAVE_STK_Trilinos) + TRIBITS_ADD_DEBUG_OPTION() + TRIBITS_ADD_SHOW_DEPRECATED_WARNINGS_OPTION() + else() + # what should we do here? + endif() +endmacro() + +macro(STK_CONFIGURE_FILE filename) + if(HAVE_STK_Trilinos) + TRIBITS_CONFIGURE_FILE(${filename}) + TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) + else() + configure_file(${STK_TOPLEVEL_SOURCE_DIR}/cmake/${filename}.in ${CMAKE_CURRENT_BINARY_DIR}/${filename}) + endif() +endmacro() + +function(stk_process_enables) + if(STK_ENABLE_ALL) + set(STK_ENABLE_STKUtil ON PARENT_SCOPE) + + if (DEFINED STK_ENABLE_STKCoupling AND NOT STK_ENABLE_STK_Coupling) + message("STKCoupling explicitly disabled.") + elseif(NOT STK_HAS_MPI) + message("STKCoupling disabled because MPI not enabled.") + else() + set(STK_ENABLE_STKCoupling ON PARENT_SCOPE) + endif() + + if (DEFINED STK_ENABLE_STKMath AND NOT STK_ENABLE_STK_Math) + message("STKMath explicitly disabled.") + else() + set(STK_ENABLE_STKMath ON PARENT_SCOPE) + endif() + + if (DEFINED STK_ENABLE_STKSimd AND NOT STK_ENABLE_STK_Simd) + message("STKSimd explicitly disabled.") + else() + set(STK_ENABLE_STKSimd ON PARENT_SCOPE) + endif() + + if (DEFINED STK_ENABLE_STKNGP_TEST AND NOT STK_ENABLE_STK_NGP_TEST) + message("STKNGP_TEST explicitly disabled.") + else() + set(STK_ENABLE_STKNGP_TEST ON PARENT_SCOPE) + endif() + + if (DEFINED STK_ENABLE_STKExprEval AND NOT STK_ENABLE_STKExprEval) + message("STKExprEval explicitly disabled.") + else() + set(STK_ENABLE_STKExprEval ON PARENT_SCOPE) + endif() + + if (DEFINED STK_ENABLE_STKTopology AND NOT STK_ENABLE_STK_Topology) + message("STKTopology explicitly disabled.") + else() + set(STK_ENABLE_STKTopology ON PARENT_SCOPE) + endif() + + if (DEFINED STK_ENABLE_STKSearch AND NOT STK_ENABLE_STK_Search) + message("STKSearch explicitly disabled.") + elseif(NOT STK_HAS_MPI) + message("STKSearch disabled because MPI not enabled.") + else() + set(STK_ENABLE_STKSearch ON PARENT_SCOPE) + endif() + + if (DEFINED STK_ENABLE_STKMiddle_mesh AND NOT STK_ENABLE_STK_Middle_mesh) + message("STKMiddle_mesh explicitly disabled.") + else() + set(STK_ENABLE_STKMiddle_mesh ON PARENT_SCOPE) + endif() + + if (DEFINED STK_ENABLE_STKTransfer AND NOT STK_ENABLE_STK_Transfer) + message("STKTransfer explicitly disabled.") + elseif(NOT STK_ENABLE_STKSearch) + message("STKTransfer disabled because STKSearch not enabled.") + else() + set(STK_ENABLE_STKTransfer ON PARENT_SCOPE) + endif() + endif() + + if(STK_ENABLE_TESTS) + set(STK_ENABLE_STKUnit_tests ON PARENT_SCOPE) + set(STK_ENABLE_STKDoc_tests ON PARENT_SCOPE) + endif() +endfunction() + +macro(STK_SUBPACKAGES) + if(HAVE_STK_Trilinos) + TRIBITS_PROCESS_SUBPACKAGES() + else() + add_subdirectory(stk_util) + message("STKUtil is always enabled.") + + if(STK_ENABLE_STKCoupling) + add_subdirectory(stk_coupling) + endif() + + if(STK_ENABLE_STKMath) + add_subdirectory(stk_math) + message("STKMath is enabled.") + endif() + + if(STK_ENABLE_STKSimd) + add_subdirectory(stk_simd) + message("STKSimd is enabled.") + endif() + + if(STK_ENABLE_STKNGP_TEST) + add_subdirectory(stk_ngp_test) + message("STKNGP_TEST is enabled.") + endif() + + if(STK_ENABLE_STKExprEval) + add_subdirectory(stk_expreval) + message("STKExprEval is enabled.") + endif() + + if(STK_ENABLE_STKTopology) + add_subdirectory(stk_topology) + message("STKTopology is enabled.") + endif() + + if(STK_ENABLE_STKSearch) + add_subdirectory(stk_search) + message("STKSearch is enabled.") + endif() + + if(STK_ENABLE_STKMiddle_mesh) + add_subdirectory(stk_middle_mesh) + message("STKMiddle_mesh is enabled.") + endif() + + if(STK_ENABLE_STKTransfer) + add_subdirectory(stk_transfer) + message("STKTransfer is enabled.") + endif() + + if(STK_ENABLE_TESTS) + add_subdirectory(stk_unit_test_utils) + message("STKUnit_test_utils is enabled.") + + add_subdirectory(stk_unit_tests) + message("STKUnit_tests is enabled.") + endif() + endif() +endmacro() + +macro(STK_SUBPACKAGE subpkg) + if(HAVE_STK_Trilinos) + TRIBITS_SUBPACKAGE($subpkg) + else() + endif() +endmacro() + +macro(STK_ADD_TEST_DIRECTORIES subdir) + if(HAVE_STK_Trilinos) + TRIBITS_ADD_TEST_DIRECTORIES(${subdir}) + else() + add_subdirectory(${subdir}) + endif() +endmacro() + +macro(STK_SUBPACKAGE_POSTPROCESS) + if(HAVE_STK_Trilinos) + TRIBITS_SUBPACKAGE_POSTPROCESS() + else() + endif() +endmacro() + +macro(STK_PACKAGE_POSTPROCESS) + if(HAVE_STK_Trilinos) + TRIBITS_PACKAGE_DEF() + TRIBITS_PACKAGE_POSTPROCESS() + else() + endif() +endmacro() diff --git a/packages/stk/stk_balance/stk_balance/CMakeLists.txt b/packages/stk/stk_balance/stk_balance/CMakeLists.txt index b8efbd36c239..a9b9db3185f0 100644 --- a/packages/stk/stk_balance/stk_balance/CMakeLists.txt +++ b/packages/stk/stk_balance/stk_balance/CMakeLists.txt @@ -51,8 +51,6 @@ SET(SOURCES "") SET(HEADERS_IMPL "") SET(SOURCES_IMPL "") -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) - # # Core headers # @@ -97,6 +95,11 @@ TRIBITS_ADD_LIBRARY( SOURCES ${SOURCES} ${SOURCES_INTERNAL} ${SOURCES_M2N} ${SOURCES_SETUP} ${SOURCES_SEARCH_TOLERANCE_ALGS} ) +target_include_directories(stk_balance_lib PUBLIC + $ + $ +) + TRIBITS_ADD_EXECUTABLE( stk_balance SOURCES ${SOURCES_MAIN} diff --git a/packages/stk/stk_balance/stk_balance/balance.cpp b/packages/stk/stk_balance/stk_balance/balance.cpp index d3b2de2edd0a..357d43658b69 100644 --- a/packages/stk/stk_balance/stk_balance/balance.cpp +++ b/packages/stk/stk_balance/stk_balance/balance.cpp @@ -241,7 +241,7 @@ bool colorMesh(const BalanceSettings& balanceSettings, stk::mesh::BulkData& bulk bool balanceStkMesh(const BalanceSettings& balanceSettings, stk::mesh::BulkData& stkMeshBulkData, const std::vector& selectors) { - stk::balance::internal::register_internal_fields(stkMeshBulkData, balanceSettings); + stk::balance::internal::register_internal_fields_and_parts(stkMeshBulkData, balanceSettings); if (balanceSettings.shouldPrintDiagnostics()) { set_up_diagnostics(balanceSettings); @@ -259,7 +259,7 @@ bool balanceStkMesh(const BalanceSettings& balanceSettings, stk::mesh::BulkData& bool balanceStkMesh(const BalanceSettings& balanceSettings, stk::mesh::BulkData& stkMeshBulkData) { - stk::balance::internal::register_internal_fields(stkMeshBulkData, balanceSettings); + stk::balance::internal::register_internal_fields_and_parts(stkMeshBulkData, balanceSettings); if (balanceSettings.shouldPrintDiagnostics()) { set_up_diagnostics(balanceSettings); @@ -279,7 +279,7 @@ bool balanceStkMeshNodes(const BalanceSettings& balanceSettings, stk::mesh::Bulk { if ((balanceSettings.getGraphOption() == BalanceSettings::LOAD_BALANCE) && balanceSettings.useNodeBalancer()) { - stk::balance::internal::register_internal_fields(stkMeshBulkData, balanceSettings); + stk::balance::internal::register_internal_fields_and_parts(stkMeshBulkData, balanceSettings); if (balanceSettings.shouldPrintDiagnostics()) { set_up_diagnostics(balanceSettings); diff --git a/packages/stk/stk_balance/stk_balance/balanceUtils.cpp b/packages/stk/stk_balance/stk_balance/balanceUtils.cpp index 86b2e9f09d48..8e9d7c9bd6e3 100644 --- a/packages/stk/stk_balance/stk_balance/balanceUtils.cpp +++ b/packages/stk/stk_balance/stk_balance/balanceUtils.cpp @@ -284,9 +284,9 @@ bool BalanceSettings::shouldFixSpiders() const return false; } -std::string BalanceSettings::getSpiderBeamConnectivityCountFieldName() const +std::string BalanceSettings::getSpiderPartName() const { - return "stk_balance_beam_connectivity_count"; + return "stk_balance_spider_elements"; } std::string BalanceSettings::getSpiderVolumeConnectivityCountFieldName() const @@ -309,7 +309,7 @@ std::string BalanceSettings::getVertexConnectivityWeightFieldName() const return "stk_balance_vertex_connectivity_weight"; } -const stk::mesh::Field * BalanceSettings::getSpiderBeamConnectivityCountField(const stk::mesh::BulkData & stkMeshBulkData) const +stk::mesh::Part * BalanceSettings::getSpiderPart(const stk::mesh::BulkData & stkMeshBulkData) const { return nullptr; } @@ -413,7 +413,7 @@ GraphCreationSettings::GraphCreationSettings() m_UseConstantToleranceForFaceSearch(false), m_shouldFixSpiders(DefaultSettings::fixSpiders), m_shouldFixMechanisms(DefaultSettings::fixMechanisms), - m_spiderBeamConnectivityCountField(nullptr), + m_spiderPart(nullptr), m_spiderVolumeConnectivityCountField(nullptr), m_outputSubdomainField(nullptr), m_includeSearchResultInGraph(DefaultSettings::useContactSearch), @@ -785,16 +785,14 @@ bool GraphCreationSettings::shouldFixSpiders() const return m_shouldFixSpiders; } -const stk::mesh::Field * GraphCreationSettings::getSpiderBeamConnectivityCountField(const stk::mesh::BulkData & stkMeshBulkData) const +stk::mesh::Part * GraphCreationSettings::getSpiderPart(const stk::mesh::BulkData & stkMeshBulkData) const { - if (m_spiderBeamConnectivityCountField == nullptr) { - m_spiderBeamConnectivityCountField = - stkMeshBulkData.mesh_meta_data().get_field(stk::topology::NODE_RANK, - getSpiderBeamConnectivityCountFieldName()); - STK_ThrowRequireMsg(m_spiderBeamConnectivityCountField != nullptr, - "Must create nodal spider beam connectivity count field when fixing spider elements."); + if (m_spiderPart == nullptr) { + m_spiderPart = stkMeshBulkData.mesh_meta_data().get_part(getSpiderPartName()); + STK_ThrowRequireMsg(m_spiderPart != nullptr, + "Must create spider part when fixing spider elements."); } - return m_spiderBeamConnectivityCountField; + return m_spiderPart; } const stk::mesh::Field * GraphCreationSettings::getSpiderVolumeConnectivityCountField(const stk::mesh::BulkData & stkMeshBulkData) const diff --git a/packages/stk/stk_balance/stk_balance/balanceUtils.hpp b/packages/stk/stk_balance/stk_balance/balanceUtils.hpp index 81afdf74bec0..99804f05c484 100644 --- a/packages/stk/stk_balance/stk_balance/balanceUtils.hpp +++ b/packages/stk/stk_balance/stk_balance/balanceUtils.hpp @@ -177,12 +177,12 @@ class BalanceSettings virtual bool shouldFixMechanisms() const; virtual bool shouldFixSpiders() const; - virtual std::string getSpiderBeamConnectivityCountFieldName() const; + virtual std::string getSpiderPartName() const; virtual std::string getSpiderVolumeConnectivityCountFieldName() const; virtual std::string getOutputSubdomainFieldName() const; virtual std::string getDiagnosticElementWeightFieldName() const; virtual std::string getVertexConnectivityWeightFieldName() const; - virtual const stk::mesh::Field * getSpiderBeamConnectivityCountField(const stk::mesh::BulkData & stkMeshBulkData) const; + virtual stk::mesh::Part * getSpiderPart(const stk::mesh::BulkData & stkMeshBulkData) const; virtual const stk::mesh::Field * getSpiderVolumeConnectivityCountField(const stk::mesh::BulkData & stkMeshBulkData) const; virtual const stk::mesh::Field * getOutputSubdomainField(const stk::mesh::BulkData & stkMeshBulkData) const; virtual const stk::mesh::Field * getDiagnosticElementWeightField(const stk::mesh::BulkData & stkMeshBulkData) const; @@ -283,7 +283,7 @@ class GraphCreationSettings : public BalanceSettings virtual bool shouldFixMechanisms() const override; virtual bool shouldFixSpiders() const override; - virtual const stk::mesh::Field * getSpiderBeamConnectivityCountField(const stk::mesh::BulkData & stkMeshBulkData) const override; + virtual stk::mesh::Part * getSpiderPart(const stk::mesh::BulkData & stkMeshBulkData) const override; virtual const stk::mesh::Field * getSpiderVolumeConnectivityCountField(const stk::mesh::BulkData & stkMeshBulkData) const override; virtual const stk::mesh::Field * getOutputSubdomainField(const stk::mesh::BulkData & stkMeshBulkData) const override; @@ -307,7 +307,7 @@ class GraphCreationSettings : public BalanceSettings bool m_UseConstantToleranceForFaceSearch; bool m_shouldFixSpiders; bool m_shouldFixMechanisms; - mutable const stk::mesh::Field * m_spiderBeamConnectivityCountField; + mutable stk::mesh::Part * m_spiderPart; mutable const stk::mesh::Field * m_spiderVolumeConnectivityCountField; mutable const stk::mesh::Field * m_outputSubdomainField; std::shared_ptr m_faceSearchToleranceFunction; diff --git a/packages/stk/stk_balance/stk_balance/internal/Zoltan2ParallelGraph.cpp b/packages/stk/stk_balance/stk_balance/internal/Zoltan2ParallelGraph.cpp index 74beef0242a2..dba7dde4d740 100644 --- a/packages/stk/stk_balance/stk_balance/internal/Zoltan2ParallelGraph.cpp +++ b/packages/stk/stk_balance/stk_balance/internal/Zoltan2ParallelGraph.cpp @@ -135,19 +135,6 @@ stk::balance::GraphEdge create_graph_edge(const stk::mesh::BulkData &bulk, return stk::balance::GraphEdge(element1, element2Id, vertex2ParallelOwner, edgeWeight); } -bool should_create_graph_edge(const stk::mesh::BulkData &bulk, - const stk::mesh::Selector& selector, - const stk::balance::BalanceSettings &balanceSettings, - const stk::mesh::Entity & element1, - const stk::mesh::Entity & element2) -{ - const bool isValidGraphEdge = is_valid_graph_connectivity(bulk, selector, balanceSettings, element1, element2); - const bool notSpiderElement1 = !stk::balance::internal::should_omit_spider_element(bulk, balanceSettings, element1); - const bool notSpiderElement2 = !stk::balance::internal::should_omit_spider_element(bulk, balanceSettings, element2); - - return isValidGraphEdge && notSpiderElement1 && notSpiderElement2; -} - void Zoltan2ParallelGraph::convertGraphEdgesToZoltanGraph(const stk::mesh::BulkData& stkMeshBulkData, const stk::balance::BalanceSettings &balanceSettings, const std::vector &graphEdges, @@ -231,7 +218,7 @@ class GraphEdgeCreator bool considerOnlySelectedOwnedElement = (m_bulk.bucket(possiblyConnectedElement).owned() && m_selector(m_bulk.bucket(possiblyConnectedElement))); if (considerOnlySelectedOwnedElement) { - if (should_create_graph_edge(m_bulk, m_selector, m_balanceSettings, elementOfConcern, possiblyConnectedElement)) { + if (is_valid_graph_connectivity(m_bulk, m_selector, m_balanceSettings, elementOfConcern, possiblyConnectedElement)) { graphEdges.push_back(create_graph_edge(m_bulk, m_balanceSettings, m_localIds, elementOfConcern, possiblyConnectedElement)); } } @@ -245,14 +232,14 @@ class GraphEdgeCreator const stk::mesh::EntityId elementOfConcernId = m_bulk.identifier(elementOfConcern); const stk::mesh::EntityId possiblyConnectedElementId = m_bulk.identifier(possiblyConnectedElement); if (elementOfConcernId < possiblyConnectedElementId) { - if (should_create_graph_edge(m_bulk, m_selector, m_balanceSettings, elementOfConcern, possiblyConnectedElement)) { + if (is_valid_graph_connectivity(m_bulk, m_selector, m_balanceSettings, elementOfConcern, possiblyConnectedElement)) { graphEdges.emplace_back(create_graph_edge(m_bulk, m_balanceSettings, elementOfConcern, possiblyConnectedElement)); graphEdges.emplace_back(create_graph_edge(m_bulk, m_balanceSettings, possiblyConnectedElement, elementOfConcern)); } } } else { - if (should_create_graph_edge(m_bulk, m_selector, m_balanceSettings, elementOfConcern, possiblyConnectedElement)) { + if (is_valid_graph_connectivity(m_bulk, m_selector, m_balanceSettings, elementOfConcern, possiblyConnectedElement)) { graphEdges.emplace_back(create_graph_edge(m_bulk, m_balanceSettings, elementOfConcern, possiblyConnectedElement)); } } @@ -311,27 +298,25 @@ void Zoltan2ParallelGraph::createGraphEdgesUsingNodeConnectivity(stk::mesh::Bulk unsigned local_id = stk::balance::internal::get_local_id(localIds, elementOfConcern); mVertexIds[local_id] = getAdjacencyId(stkMeshBulkData, elementOfConcern, useLocalIds, localIds); - if (not stk::balance::internal::should_omit_spider_element(stkMeshBulkData, balanceSettings, elementOfConcern)) { - if (balanceSettings.getVertexWeightMethod() == stk::balance::VertexWeightMethod::CONSTANT) { - mVertexWeights[local_id] = 1; - } - else if (balanceSettings.getVertexWeightMethod() == stk::balance::VertexWeightMethod::TOPOLOGY) { - mVertexWeights[local_id] = balanceSettings.getGraphVertexWeight(bucket.topology()); - } - else if (balanceSettings.getVertexWeightMethod() == stk::balance::VertexWeightMethod::CONNECTIVITY) { - const stk::mesh::Field & connectivityWeights = *balanceSettings.getVertexConnectivityWeightField(stkMeshBulkData); - mVertexWeights[local_id] = *stk::mesh::field_data(connectivityWeights, elementOfConcern); - } - else if (balanceSettings.getVertexWeightMethod() == stk::balance::VertexWeightMethod::FIELD) { - mVertexWeights[local_id] = 0; //real field weights set in adjust_vertex_weights - } - else { - STK_ThrowErrorMsg("Unknown vertex weight method: " << vertex_weight_method_name(balanceSettings.getVertexWeightMethod())); - } - - stk::balance::internal::fillEntityCentroid(stkMeshBulkData, coord, elementOfConcern, &mVertexCoordinates[local_id*spatialDimension]); - graphEdgeCreator.create_graph_edges_for_element(elementOfConcern, graphEdges); + if (balanceSettings.getVertexWeightMethod() == stk::balance::VertexWeightMethod::CONSTANT) { + mVertexWeights[local_id] = 1; + } + else if (balanceSettings.getVertexWeightMethod() == stk::balance::VertexWeightMethod::TOPOLOGY) { + mVertexWeights[local_id] = balanceSettings.getGraphVertexWeight(bucket.topology()); } + else if (balanceSettings.getVertexWeightMethod() == stk::balance::VertexWeightMethod::CONNECTIVITY) { + const stk::mesh::Field & connectivityWeights = *balanceSettings.getVertexConnectivityWeightField(stkMeshBulkData); + mVertexWeights[local_id] = *stk::mesh::field_data(connectivityWeights, elementOfConcern); + } + else if (balanceSettings.getVertexWeightMethod() == stk::balance::VertexWeightMethod::FIELD) { + mVertexWeights[local_id] = 0; //real field weights set in adjust_vertex_weights + } + else { + STK_ThrowErrorMsg("Unknown vertex weight method: " << vertex_weight_method_name(balanceSettings.getVertexWeightMethod())); + } + + stk::balance::internal::fillEntityCentroid(stkMeshBulkData, coord, elementOfConcern, &mVertexCoordinates[local_id*spatialDimension]); + graphEdgeCreator.create_graph_edges_for_element(elementOfConcern, graphEdges); } } } diff --git a/packages/stk/stk_balance/stk_balance/internal/privateDeclarations.cpp b/packages/stk/stk_balance/stk_balance/internal/privateDeclarations.cpp index 4a5b23160f8c..66fe3eabe4c0 100644 --- a/packages/stk/stk_balance/stk_balance/internal/privateDeclarations.cpp +++ b/packages/stk/stk_balance/stk_balance/internal/privateDeclarations.cpp @@ -245,7 +245,7 @@ void filterOutConnectedElements(const stk::mesh::BulkData & bulk, SearchElemPair int owningProcElement2 = searchResult.second.proc(); STK_ThrowRequireWithSierraHelpMsg(owningProcElement1 == bulk.parallel_rank() || - owningProcElement2 == bulk.parallel_rank()); + owningProcElement2 == bulk.parallel_rank()); bool anyIntersections = false; @@ -618,22 +618,13 @@ void fillEntityCentroid(const stk::mesh::BulkData &stkMeshBulkData, const stk::m } } -int num_beams_connected_to_node(const stk::mesh::BulkData& bulk, stk::mesh::Entity node) +bool is_not_part_of_spider(const stk::mesh::BulkData & bulk, const stk::mesh::Part & spiderPart, + stk::mesh::Entity element) { - const unsigned numElems = bulk.num_elements(node); - const stk::mesh::Entity* elems = bulk.begin_elements(node); - - int numBeams = 0; - for(unsigned elemIndex=0; elemIndex & beamField = meta.declare_field(stk::topology::NODE_RANK, - balanceSettings.getSpiderBeamConnectivityCountFieldName()); - stk::mesh::put_field_on_mesh(beamField, meta.universal_part(), nullptr); - stk::mesh::Field & volumeField = meta.declare_field(stk::topology::ELEM_RANK, balanceSettings.getSpiderVolumeConnectivityCountFieldName()); stk::mesh::put_field_on_mesh(volumeField, meta.universal_part(), nullptr); @@ -681,6 +668,8 @@ void register_internal_fields(stk::mesh::BulkData & bulk, const stk::balance::Ba stk::mesh::Field & outputSubdomainField = meta.declare_field(stk::topology::ELEM_RANK, balanceSettings.getOutputSubdomainFieldName()); stk::mesh::put_field_on_mesh(outputSubdomainField, meta.universal_part(), nullptr); + + meta.declare_part(balanceSettings.getSpiderPartName(), stk::topology::ELEM_RANK); } const bool hasWeightField = meta.get_field(stk::topology::ELEM_RANK, @@ -705,33 +694,78 @@ void register_internal_fields(stk::mesh::BulkData & bulk, const stk::balance::Ba } } -void fill_spider_connectivity_count_fields(const stk::mesh::BulkData & bulk, const BalanceSettings & balanceSettings) +unsigned number_of_connected_beams(const stk::mesh::BulkData & bulk, stk::mesh::Entity node) { - if (balanceSettings.shouldFixSpiders()) { - const stk::mesh::MetaData& meta = bulk.mesh_meta_data(); - - const stk::mesh::Field * beamConnectivityCountField = balanceSettings.getSpiderBeamConnectivityCountField(bulk); - stk::mesh::Selector selectBeamNodes(meta.locally_owned_part() & - meta.get_topology_root_part(stk::topology::BEAM_2)); - stk::mesh::EntityVector nodes; - mesh::get_entities(bulk, stk::topology::NODE_RANK, selectBeamNodes, nodes); - for(stk::mesh::Entity node : nodes) { - int * beamConnectivityCount = stk::mesh::field_data(*beamConnectivityCountField, node); - *beamConnectivityCount = num_beams_connected_to_node(bulk, node); + unsigned numBeams = 0; + + const stk::mesh::Entity * elements = bulk.begin_elements(node); + const unsigned numElems = bulk.num_elements(node); + for (unsigned i = 0; i < numElems; ++i) { + stk::topology elemTopology = bulk.bucket(elements[i]).topology(); + if (elemTopology == stk::topology::BEAM_2) { + ++numBeams; + } + } + + return numBeams; +} + +bool element_has_an_unconnected_node(const stk::mesh::BulkData & bulk, stk::mesh::Entity element) +{ + const stk::mesh::Entity * nodes = bulk.begin_nodes(element); + const unsigned numNodes = bulk.num_nodes(element); + for (unsigned i = 0; i < numNodes; ++i) { + const unsigned numElems = bulk.num_elements(nodes[i]); + if (numElems < 2) { // Only has self + return true; } + } + return false; +} + +void fill_spider_connectivity_count_fields_and_parts(stk::mesh::BulkData & bulk, + const BalanceSettings & balanceSettings) +{ + const stk::mesh::MetaData& meta = bulk.mesh_meta_data(); + + stk::mesh::Selector selectBeamsAndParticles(meta.locally_owned_part() & + (meta.get_topology_root_part(stk::topology::BEAM_2) | + meta.get_topology_root_part(stk::topology::PARTICLE))); - const stk::mesh::Field * volumeConnectivityCountField = balanceSettings.getSpiderVolumeConnectivityCountField(bulk); - stk::mesh::Selector selectBeamElements(meta.locally_owned_part() & - meta.get_topology_root_part(stk::topology::BEAM_2)); - stk::mesh::EntityVector beams; - mesh::get_entities(bulk, stk::topology::ELEM_RANK, selectBeamElements, beams); - for(stk::mesh::Entity beam : beams) { - int * volumeConnectivityCount = stk::mesh::field_data(*volumeConnectivityCountField, beam); - *volumeConnectivityCount = num_volume_elements_connected_to_beam(bulk, beam); + constexpr unsigned spiderConnectivityThreshold = 5; + + stk::mesh::EntityVector spiderElements; + const auto beamsAndParticles = mesh::get_entities(bulk, stk::topology::ELEM_RANK, selectBeamsAndParticles); + for (stk::mesh::Entity candidateElem : beamsAndParticles) { + if (element_has_an_unconnected_node(bulk, candidateElem)) { + continue; } - stk::mesh::communicate_field_data(bulk, {beamConnectivityCountField, volumeConnectivityCountField}); + const stk::mesh::Entity * nodes = bulk.begin_nodes(candidateElem); + const unsigned numNodes = bulk.num_nodes(candidateElem); + for (unsigned i = 0; i < numNodes; ++i) { + if (number_of_connected_beams(bulk, nodes[i]) > spiderConnectivityThreshold) { + spiderElements.push_back(candidateElem); + break; + } + } } + + stk::mesh::Part * spiderPart = balanceSettings.getSpiderPart(bulk); + bulk.batch_change_entity_parts(spiderElements, {spiderPart}, {}); + + + const stk::mesh::Field * volumeConnectivityCountField = balanceSettings.getSpiderVolumeConnectivityCountField(bulk); + stk::mesh::Selector selectBeamElements(meta.locally_owned_part() & + meta.get_topology_root_part(stk::topology::BEAM_2)); + stk::mesh::EntityVector beams; + mesh::get_entities(bulk, stk::topology::ELEM_RANK, selectBeamElements, beams); + for(stk::mesh::Entity beam : beams) { + int * volumeConnectivityCount = stk::mesh::field_data(*volumeConnectivityCountField, beam); + *volumeConnectivityCount = num_volume_elements_connected_to_beam(bulk, beam); + } + + stk::mesh::communicate_field_data(bulk, {volumeConnectivityCountField}); } void fill_output_subdomain_field(const stk::mesh::BulkData & bulk, const BalanceSettings & balanceSettings, @@ -850,14 +884,6 @@ void get_multicriteria_decomp_using_selectors_as_segregation(const stk::mesh::Bu if (num_entities_across_procs > 0) { stk::balance::internal::GeometricVertices vertexInfo(balanceSettings, bulk, entitiesToBalance, criterions); - if (balanceSettings.shouldFixSpiders()) { - const stk::mesh::Field & beamConnectivityCountField = *balanceSettings.getSpiderBeamConnectivityCountField(bulk); - for (unsigned i = 0; i < entitiesToBalance.size(); ++i) { - if (is_element_part_of_spider(bulk, beamConnectivityCountField, entitiesToBalance[i])) { - vertexInfo.set_vertex_weight(i, 0); - } - } - } store_diagnostic_element_weights(bulk, balanceSettings, unionSelector, vertexInfo); @@ -971,46 +997,6 @@ Teuchos::ParameterList getGraphBasedParameters(const BalanceSettings& balanceSet return params; } -bool is_node_part_of_spider(const stk::mesh::BulkData& stkMeshBulkData, - const stk::mesh::Field& beamConnectivityCountField, - stk::mesh::Entity node) -{ - const int spiderConnectivityThreshold = 5; - const int connectivityCount = *stk::mesh::field_data(beamConnectivityCountField, node); - return (connectivityCount > spiderConnectivityThreshold); -} - -bool is_element_part_of_spider(const stk::mesh::BulkData& stkMeshBulkData, - const stk::mesh::Field& beamConnectivityCountField, - stk::mesh::Entity element) -{ - const stk::mesh::Entity* nodes = stkMeshBulkData.begin_nodes(element); - const unsigned numNodes = stkMeshBulkData.num_nodes(element); - for (unsigned i = 0; i < numNodes; ++i) { - if (is_node_part_of_spider(stkMeshBulkData, beamConnectivityCountField, nodes[i])) { - return true; - } - } - return false; -} - -bool should_omit_spider_element(const stk::mesh::BulkData & stkMeshBulkData, - const stk::balance::BalanceSettings & balanceSettings, - stk::mesh::Entity elem) -{ - bool omitConnection = false; - if (balanceSettings.shouldFixSpiders()) { - const stk::mesh::Field & beamConnectivityCountField = *balanceSettings.getSpiderBeamConnectivityCountField(stkMeshBulkData); - stk::topology elemTopology = stkMeshBulkData.bucket(elem).topology(); - - if (elemTopology == stk::topology::BEAM_2 || elemTopology == stk::topology::PARTICLE) { - omitConnection = is_element_part_of_spider(stkMeshBulkData, beamConnectivityCountField, elem); - } - } - - return omitConnection; -} - bool found_valid_new_owner(const stk::mesh::BulkData & bulk, int newOwner) { return (newOwner < bulk.parallel_size()); @@ -1106,53 +1092,52 @@ stk::mesh::EntityProcMap determine_global_new_owner(const stk::mesh::BulkData & return unpack_new_spider_entity_owners(bulk, comm); } +unsigned count_connected_spider_elements(const stk::mesh::BulkData & bulk, + const stk::mesh::Part & spiderPart, + const stk::mesh::Entity node) +{ + const stk::mesh::Entity * connectedElems = bulk.begin_elements(node); + const unsigned numConnectedElems = bulk.num_elements(node); + unsigned numSpiderElems = 0; + for (unsigned i = 0; i < numConnectedElems; ++i) { + if (bulk.bucket(connectedElems[i]).member(spiderPart)) { + ++numSpiderElems; + } + } + return numSpiderElems; +} + std::pair get_foot_and_body_nodes(const stk::mesh::BulkData & bulk, - const stk::mesh::Entity spiderLeg, - const stk::mesh::Field & beamConnectivityCountField) + const stk::mesh::Part & spiderPart, + const stk::mesh::Entity spiderLeg) { const stk::mesh::Entity* nodes = bulk.begin_nodes(spiderLeg); - const int node1ConnectivityCount = *stk::mesh::field_data(beamConnectivityCountField, nodes[0]); - const int node2ConnectivityCount = *stk::mesh::field_data(beamConnectivityCountField, nodes[1]); + const int node1ConnectivityCount = count_connected_spider_elements(bulk, spiderPart, nodes[0]); + const int node2ConnectivityCount = count_connected_spider_elements(bulk, spiderPart, nodes[1]); return (node1ConnectivityCount < node2ConnectivityCount) ? std::make_pair(nodes[0], nodes[1]) : std::make_pair(nodes[1], nodes[0]); } -stk::mesh::Entity get_body_node(const stk::mesh::BulkData & bulk, const stk::mesh::Entity spiderLeg, - const stk::mesh::Field & beamConnectivityCountField) -{ - return get_foot_and_body_nodes(bulk, spiderLeg, beamConnectivityCountField).second; -} - stk::mesh::EntityVector get_local_spider_legs(const stk::mesh::BulkData & bulk, - const stk::mesh::Field & beamConnectivityCountField) + const stk::mesh::Part & spiderPart) { - stk::mesh::EntityVector spiderLegs; const stk::mesh::MetaData & meta = bulk.mesh_meta_data(); - stk::mesh::Part & beamPart = meta.get_topology_root_part(stk::topology::BEAM_2); - const stk::mesh::BucketVector & beamBuckets = bulk.get_buckets(stk::topology::ELEM_RANK, - beamPart & meta.locally_owned_part()); - for (const stk::mesh::Bucket * beamBucket : beamBuckets) { - for (const stk::mesh::Entity & beam : *beamBucket) { - if (is_element_part_of_spider(bulk, beamConnectivityCountField, beam)) { - spiderLegs.push_back(beam); - } - } - } - - return spiderLegs; + stk::mesh::Selector localSpiderLegsSelector = spiderPart & + meta.get_topology_root_part(stk::topology::BEAM_2) & + meta.locally_owned_part(); + return stk::mesh::get_entities(bulk, stk::topology::ELEM_RANK, localSpiderLegsSelector); } int get_new_spider_leg_owner(const stk::mesh::BulkData & bulk, const stk::mesh::Entity & footNode, - const stk::mesh::Field & outputSubdomainField) + const stk::mesh::Part & spiderPart, const stk::mesh::Field & outputSubdomainField) { const stk::mesh::Entity* footElements = bulk.begin_elements(footNode); const unsigned numElements = bulk.num_elements(footNode); int newLegOwner = std::numeric_limits::max(); for (unsigned i = 0; i < numElements; ++i) { - const stk::topology::topology_t elemTopology = bulk.bucket(footElements[i]).topology(); - if (is_not_part_of_spider(elemTopology)) { + if (is_not_part_of_spider(bulk, spiderPart, footElements[i])) { const int & outputSubdomain = *stk::mesh::field_data(outputSubdomainField, footElements[i]); newLegOwner = std::min(newLegOwner, outputSubdomain); } @@ -1202,18 +1187,18 @@ void update_output_subdomain_field(const stk::mesh::BulkData & bulk, void fix_spider_elements(const BalanceSettings & balanceSettings, stk::mesh::BulkData & bulk, DecompositionChangeList & changeList) { - const stk::mesh::Field & beamConnectivityCountField = *balanceSettings.getSpiderBeamConnectivityCountField(bulk); const stk::mesh::Field & outputSubdomainField = *balanceSettings.getOutputSubdomainField(bulk); + const stk::mesh::Part & spiderPart = *balanceSettings.getSpiderPart(bulk); - stk::mesh::EntityVector spiderLegs = get_local_spider_legs(bulk, beamConnectivityCountField); + stk::mesh::EntityVector spiderLegs = get_local_spider_legs(bulk, spiderPart); stk::mesh::EntityProcMap newSpiderComponentOwners; for (stk::mesh::Entity spiderLeg : spiderLegs) { stk::mesh::Entity footNode; stk::mesh::Entity bodyNode; - std::tie(footNode, bodyNode) = get_foot_and_body_nodes(bulk, spiderLeg, beamConnectivityCountField); + std::tie(footNode, bodyNode) = get_foot_and_body_nodes(bulk, spiderPart, spiderLeg); - const int newLegOwner = get_new_spider_leg_owner(bulk, footNode, outputSubdomainField); + const int newLegOwner = get_new_spider_leg_owner(bulk, footNode, spiderPart, outputSubdomainField); if (found_valid_new_owner(bulk, newLegOwner)) { changeList.set_entity_destination(spiderLeg, newLegOwner); @@ -1356,19 +1341,29 @@ void calculateGeometricOrGraphBasedDecomp(stk::mesh::BulkData & stkMeshBulkData, { STK_ThrowRequireWithSierraHelpMsg(numSubdomainsToCreate > 0); STK_ThrowRequireWithSierraHelpMsg(is_geometric_method(balanceSettings.getDecompMethod()) || - is_graph_based_method(balanceSettings.getDecompMethod())); + is_graph_based_method(balanceSettings.getDecompMethod())); + + std::vector balanceSelectors = selectors; + if (balanceSettings.shouldFixSpiders()) { + internal::fill_spider_connectivity_count_fields_and_parts(stkMeshBulkData, balanceSettings); + + const stk::mesh::Part & spiderPart = *balanceSettings.getSpiderPart(stkMeshBulkData); + for (stk::mesh::Selector & selector : balanceSelectors) { + selector = selector & !spiderPart; + } + } - internal::fill_spider_connectivity_count_fields(stkMeshBulkData, balanceSettings); compute_connectivity_weight(stkMeshBulkData, balanceSettings); if (numSubdomainsToCreate > 1) { if (is_geometric_method(balanceSettings.getDecompMethod())) { stk::mesh::impl::LocalIdMapper localIds(stkMeshBulkData, stk::topology::ELEM_RANK); - fill_decomp_using_geometric_method(stkMeshBulkData, selectors, decompCommunicator, numSubdomainsToCreate, balanceSettings, - localIds, decomp); + fill_decomp_using_geometric_method(stkMeshBulkData, balanceSelectors, decompCommunicator, numSubdomainsToCreate, + balanceSettings, localIds, decomp); } else if (is_graph_based_method(balanceSettings.getDecompMethod())) { - fill_decomp_using_graph_based_method(stkMeshBulkData, selectors, decompCommunicator, numSubdomainsToCreate, balanceSettings, decomp); + fill_decomp_using_graph_based_method(stkMeshBulkData, balanceSelectors, decompCommunicator, numSubdomainsToCreate, + balanceSettings, decomp); } } else { diff --git a/packages/stk/stk_balance/stk_balance/internal/privateDeclarations.hpp b/packages/stk/stk_balance/stk_balance/internal/privateDeclarations.hpp index 60f7ee473228..41221b5b9b46 100644 --- a/packages/stk/stk_balance/stk_balance/internal/privateDeclarations.hpp +++ b/packages/stk/stk_balance/stk_balance/internal/privateDeclarations.hpp @@ -106,8 +106,8 @@ bool should_omit_spider_element(const stk::mesh::BulkData & stkMeshBulkData, const stk::balance::BalanceSettings & balanceSettings, stk::mesh::Entity elem); -void register_internal_fields(stk::mesh::BulkData & bulk, const stk::balance::BalanceSettings & balanceSettings); -void fill_spider_connectivity_count_fields(const stk::mesh::BulkData & bulk, const BalanceSettings & balanceSettings); +void register_internal_fields_and_parts(stk::mesh::BulkData & bulk, const stk::balance::BalanceSettings & balanceSettings); +void fill_spider_connectivity_count_fields_and_parts(stk::mesh::BulkData & bulk, const BalanceSettings & balanceSettings); void fill_output_subdomain_field(const stk::mesh::BulkData & bulk, const BalanceSettings & balanceSettings, stk::mesh::EntityProcVec & decomp); void fix_spider_elements(const BalanceSettings & balanceSettings, stk::mesh::BulkData & stkMeshBulkData, diff --git a/packages/stk/stk_balance/stk_balance/io/BalanceIO.cpp b/packages/stk/stk_balance/stk_balance/io/BalanceIO.cpp index 7616ae7c8258..f458f26e60a8 100644 --- a/packages/stk/stk_balance/stk_balance/io/BalanceIO.cpp +++ b/packages/stk/stk_balance/stk_balance/io/BalanceIO.cpp @@ -59,7 +59,7 @@ void read_mesh_with_auto_decomp(stk::io::StkMeshIoBroker & stkIo, stkIo.create_input_mesh(); stkIo.add_all_mesh_fields_as_input_fields(); - internal::register_internal_fields(bulkData, balanceSettings); + internal::register_internal_fields_and_parts(bulkData, balanceSettings); stkIo.populate_bulk_data(); diff --git a/packages/stk/stk_balance/stk_balance/setup/LifeCycle.cpp b/packages/stk/stk_balance/stk_balance/setup/LifeCycle.cpp index 548433e3c3fa..69ec4a6df1b7 100644 --- a/packages/stk/stk_balance/stk_balance/setup/LifeCycle.cpp +++ b/packages/stk/stk_balance/stk_balance/setup/LifeCycle.cpp @@ -173,7 +173,7 @@ void LifeCycle::rebalance() stk::io::StkMeshIoBroker ioBroker; meta.set_coordinate_field_name(m_settings.getCoordinateFieldName()); - stk::balance::internal::register_internal_fields(*bulk, m_settings); + stk::balance::internal::register_internal_fields_and_parts(*bulk, m_settings); stk::io::fill_mesh_preexisting(ioBroker, m_settings.get_input_filename(), *bulk); if (rebalance_will_corrupt_data(ioBroker, meta)) { diff --git a/packages/stk/stk_balance/stk_balance/setup/M2NParser.cpp b/packages/stk/stk_balance/stk_balance/setup/M2NParser.cpp index 6e856ec30383..497c54827f46 100644 --- a/packages/stk/stk_balance/stk_balance/setup/M2NParser.cpp +++ b/packages/stk/stk_balance/stk_balance/setup/M2NParser.cpp @@ -131,7 +131,7 @@ void M2NParser::set_logfile(M2NBalanceSettings& settings) const settings.set_log_filename(m_commandLineParser.get_option_value(m_optionNames.logfile)); } else { - const int initialNumProcs = stk::parallel_machine_size(MPI_COMM_WORLD); + const int initialNumProcs = stk::parallel_machine_size(m_comm); const int finalNumProcs = settings.get_num_output_processors(); settings.set_log_filename(stk::basename(stk::tailname(settings.get_input_filename())) + "." + std::to_string(initialNumProcs) + "_to_" + std::to_string(finalNumProcs) + ".log"); @@ -144,7 +144,7 @@ void M2NParser::set_use_nested_decomp(M2NBalanceSettings& settings) const const bool useNestedDecomp = m_commandLineParser.is_option_provided(m_optionNames.useNestedDecomp); settings.set_use_nested_decomp(useNestedDecomp); if (useNestedDecomp) { - const int initialNumProcs = stk::parallel_machine_size(MPI_COMM_WORLD); + const int initialNumProcs = stk::parallel_machine_size(m_comm); const int finalNumProcs = settings.get_num_output_processors(); const bool isValidProcCount = (finalNumProcs % initialNumProcs) == 0; STK_ThrowRequireMsg(isValidProcCount, "Final number of processors (" << finalNumProcs << ") must be an integer " diff --git a/packages/stk/stk_balance/stk_balance/setup/Parser.cpp b/packages/stk/stk_balance/stk_balance/setup/Parser.cpp index ef62e0125a74..9c028bcc5436 100644 --- a/packages/stk/stk_balance/stk_balance/setup/Parser.cpp +++ b/packages/stk/stk_balance/stk_balance/setup/Parser.cpp @@ -413,7 +413,7 @@ void Parser::set_use_nested_decomp(BalanceSettings& settings) const settings.set_use_nested_decomp(useNestedDecomp); if (useNestedDecomp) { - const int inputNumProcs = stk::parallel_machine_size(MPI_COMM_WORLD); + const int inputNumProcs = stk::parallel_machine_size(m_comm); const int outputNumProcs = settings.get_num_output_processors(); const bool isValidProcCount = (outputNumProcs % inputNumProcs) == 0; STK_ThrowRequireMsg(isValidProcCount, "Output number of processors (" << outputNumProcs << ") must be an integer " diff --git a/packages/stk/stk_coupling/CMakeLists.txt b/packages/stk/stk_coupling/CMakeLists.txt index 9938d8b49c2a..b84ee304cd02 100644 --- a/packages/stk/stk_coupling/CMakeLists.txt +++ b/packages/stk/stk_coupling/CMakeLists.txt @@ -32,10 +32,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TRIBITS_SUBPACKAGE(Coupling) - -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) +STK_SUBPACKAGE(Coupling) ADD_SUBDIRECTORY(stk_coupling) -TRIBITS_SUBPACKAGE_POSTPROCESS() +STK_SUBPACKAGE_POSTPROCESS() diff --git a/packages/stk/stk_coupling/Jamfile b/packages/stk/stk_coupling/Jamfile index 8b28d14f8b33..acbb4cf2fdd4 100644 --- a/packages/stk/stk_coupling/Jamfile +++ b/packages/stk/stk_coupling/Jamfile @@ -67,6 +67,7 @@ local installed-end-user-files = # be installed for end-users. (This is only intended to be used for testing.) local installed-developer-files = stk_coupling_utest + stk_coupling_doc_tests ; # @@ -161,3 +162,13 @@ exe stk_coupling_utest : @sierra-exec-tag ; +exe stk_coupling_doc_tests + : + [ glob $(stk_coupling-root)/../stk_doc_tests/stk_coupling/*.cpp ] + /sierra/stk_coupling//stk_coupling + /sierra/stk_unit_test_utils//stk_unit_main + : @sierra-exec-tag + $(stk_coupling-root-inc) + $(stk_coupling-root)/../stk_doc_tests/stk_coupling + ; + diff --git a/packages/stk/stk_coupling/stk_coupling/CMakeLists.txt b/packages/stk/stk_coupling/stk_coupling/CMakeLists.txt index dc997f2c9453..a63080b7aedf 100644 --- a/packages/stk/stk_coupling/stk_coupling/CMakeLists.txt +++ b/packages/stk/stk_coupling/stk_coupling/CMakeLists.txt @@ -32,17 +32,28 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) - FILE(GLOB HEADERS *.hpp) FILE(GLOB SOURCES *.cpp) -TRIBITS_ADD_LIBRARY( - stk_coupling - NOINSTALLHEADERS ${HEADERS} - SOURCES ${SOURCES} +if(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY(stk_coupling + NOINSTALLHEADERS ${HEADERS} + SOURCES ${SOURCES} ) +else() + add_library(stk_coupling ${SOURCES}) + + target_link_libraries(stk_coupling PUBLIC stk_util_parallel) +endif() + +target_include_directories(stk_coupling PUBLIC + $ + $ +) INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_coupling) +if(NOT HAVE_STK_Trilinos) + INSTALL(TARGETS stk_coupling DESTINATION ${STK_INSTALL_LIBDIR}) +endif() diff --git a/packages/stk/stk_coupling/stk_coupling/SyncInfo.cpp b/packages/stk/stk_coupling/stk_coupling/SyncInfo.cpp index 815684d3995a..08537c673cab 100644 --- a/packages/stk/stk_coupling/stk_coupling/SyncInfo.cpp +++ b/packages/stk/stk_coupling/stk_coupling/SyncInfo.cpp @@ -76,8 +76,6 @@ SyncInfo SyncInfo::exchange(const SplitComms & splitComms, int otherColor) const } }); - comm.communicate(); - if (localRank != 0) { recvInfo.unpack(comm.recv_buffer()); diff --git a/packages/stk/stk_coupling/stk_coupling/SyncInfo.hpp b/packages/stk/stk_coupling/stk_coupling/SyncInfo.hpp index 221685a42f5c..9deb1376d876 100644 --- a/packages/stk/stk_coupling/stk_coupling/SyncInfo.hpp +++ b/packages/stk/stk_coupling/stk_coupling/SyncInfo.hpp @@ -25,6 +25,8 @@ namespace coupling class SyncInfo { public: + using ColorToSyncInfoMap = std::map; + SyncInfo(const std::string& name = "") : m_name(name) { } @@ -54,9 +56,8 @@ class SyncInfo return m_vals.has_value(parameterName); } - using ColorToSyncInfoMap = std::map; - SyncInfo exchange(const SplitComms & splitComms, int otherColor) const; + ColorToSyncInfoMap exchange(const SplitComms & splitComms) const; const std::string& get_name() const { return m_name; } diff --git a/packages/stk/stk_doc_tests/CMakeLists.txt b/packages/stk/stk_doc_tests/CMakeLists.txt index 2f2ff20c98c0..7afd9750a39a 100644 --- a/packages/stk/stk_doc_tests/CMakeLists.txt +++ b/packages/stk/stk_doc_tests/CMakeLists.txt @@ -32,4 +32,9 @@ IF (Trilinos_ENABLE_STKExprEval) TRIBITS_ADD_TEST_DIRECTORIES(stk_expreval) ENDIF() +IF (Trilinos_ENABLE_STKCoupling) + TRIBITS_ADD_TEST_DIRECTORIES(stk_coupling) +ENDIF() + + TRIBITS_SUBPACKAGE_POSTPROCESS() diff --git a/packages/stk/stk_doc_tests/cmake/Dependencies.cmake b/packages/stk/stk_doc_tests/cmake/Dependencies.cmake index 9528a86e9d89..573d75db6249 100644 --- a/packages/stk/stk_doc_tests/cmake/Dependencies.cmake +++ b/packages/stk/stk_doc_tests/cmake/Dependencies.cmake @@ -1,6 +1,6 @@ SET(LIB_REQUIRED_DEP_PACKAGES) SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES STKTopology STKMath STKSimd STKMesh STKIO STKSearch STKSearchUtil STKTransfer STKTransferUtil STKExprEval) +SET(TEST_OPTIONAL_DEP_PACKAGES STKTopology STKMath STKSimd STKCoupling STKMesh STKIO STKSearch STKSearchUtil STKTransfer STKTransferUtil STKExprEval) SET(TEST_REQUIRED_DEP_PACKAGES Gtest STKUtil STKNGP_TEST STKUnit_test_utils) SET(LIB_REQUIRED_DEP_TPLS) SET(LIB_OPTIONAL_DEP_TPLS) diff --git a/packages/stk/stk_doc_tests/stk_balance/CMakeLists.txt b/packages/stk/stk_doc_tests/stk_balance/CMakeLists.txt index 4428465a803c..92d9e70571e1 100644 --- a/packages/stk/stk_doc_tests/stk_balance/CMakeLists.txt +++ b/packages/stk/stk_doc_tests/stk_balance/CMakeLists.txt @@ -33,9 +33,6 @@ # -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - FILE(GLOB SOURCES *.cpp) TRIBITS_ADD_EXECUTABLE_AND_TEST( diff --git a/packages/stk/stk_doc_tests/stk_coupling/BasicCommSplit.cpp b/packages/stk/stk_doc_tests/stk_coupling/BasicCommSplit.cpp new file mode 100644 index 000000000000..432a81699dc7 --- /dev/null +++ b/packages/stk/stk_doc_tests/stk_coupling/BasicCommSplit.cpp @@ -0,0 +1,233 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include +#include +#include +#include +#include +#include + +//BEGINsplit_comms_example +TEST(StkCouplingDocTest, split_comms) +{ + auto commWorld = MPI_COMM_WORLD; + auto rank = stk::parallel_machine_rank(commWorld); + auto commSize = stk::parallel_machine_size(commWorld); + + if (commSize < 2) GTEST_SKIP(); + + auto color = rank % 2; + + stk::coupling::SplitComms splitComms(commWorld, color); + splitComms.set_free_comms_in_destructor(true); + + auto subComm = splitComms.get_split_comm(); + std::vector otherColors = splitComms.get_other_colors(); + EXPECT_EQ(1u, otherColors.size()); + + for (auto otherColor : otherColors) { + auto otherComm = splitComms.get_pairwise_comm(otherColor); + + int result; + MPI_Comm_compare(subComm, otherComm, &result); + if (color != otherColor) { + EXPECT_NE(MPI_IDENT, result); + } else { + EXPECT_EQ(MPI_IDENT, result); + } + + EXPECT_EQ(splitComms.get_parent_comm(), commWorld); + } +} +//ENDsplit_comms_example + +//BEGINsplit_comms_singleton_example +TEST(StkCouplingDocTest, split_comms_singleton) +{ + auto commWorld = MPI_COMM_WORLD; + auto rank = stk::parallel_machine_rank(commWorld); + auto color = rank % 2; + + stk::coupling::SplitComms splitComms(commWorld, color); + splitComms.set_free_comms_in_destructor(true); + stk::coupling::set_split_comms_singleton(splitComms); + + auto singletonComms = stk::coupling::get_split_comms_singleton(); + EXPECT_TRUE(singletonComms.is_initialized()); + + int result; + MPI_Comm_compare(splitComms.get_split_comm(), singletonComms.get_split_comm(), &result); + EXPECT_EQ(MPI_IDENT, result); +} +//ENDsplit_comms_singleton_example + +TEST(StkCouplingDocTest, sync_info_set_get_values) +{ + using stk::coupling::SyncInfo; + using stk::coupling::SyncMode; + + double value = -2.0; + + SyncInfo syncInfo("sync_info"); + syncInfo.set_value("SyncModeValue", SyncMode::Minimum); + syncInfo.set_value("someDoubleValue", value); + EXPECT_TRUE(syncInfo.has_value("someDoubleValue")); + EXPECT_DOUBLE_EQ(value, syncInfo.get_value("someDoubleValue")); +} + +//BEGINsync_info_exchange_two_colors +TEST(StkCouplingDocTest, sync_info_exchange_two_colors) +{ + using stk::coupling::SplitComms; + using stk::coupling::SyncInfo; + + auto commWorld = MPI_COMM_WORLD; + if (stk::parallel_machine_size(commWorld) != 4) GTEST_SKIP(); + + auto rank = stk::parallel_machine_rank(commWorld); + auto color = rank % 2; + + SplitComms splitComms(commWorld, color); + SyncInfo syncInfo("exchange_info"); + + std::string stringValue("DataFrom" + std::to_string(color)); + std::vector intVector = (color == 0) ? std::vector{1, 3, 5} : std::vector{2, 4, 6, 8}; + std::vector> color0_vectorPairStringDouble = {{"one", 1.0}, {"two", 2.0}}; + std::vector> color1_vectorPairStringDouble = {{"three", 3.0}}; + std::vector> vectorPairStringDouble = + (color == 0) ? color0_vectorPairStringDouble : color1_vectorPairStringDouble; + + syncInfo.set_value("stringToExchange", stringValue); + syncInfo.set_value("vectorOfIntToExchange", intVector); + syncInfo.set_value("vectorOfPairToExchange", vectorPairStringDouble); + + auto otherColors = splitComms.get_other_colors(); + SyncInfo exchangeInfo = syncInfo.exchange(splitComms, otherColors[0]); + + std::string exepctedStringValue("DataFrom" + std::to_string(otherColors[0])); + std::vector expectedIntVector = (color == 1) ? std::vector{1, 3, 5} : std::vector{2, 4, 6, 8}; + std::vector> expectedVectorPairStringDouble = + (color == 1) ? color0_vectorPairStringDouble : color1_vectorPairStringDouble; + + auto recvString = exchangeInfo.get_value("stringToExchange"); + auto recvVectorOfInt = exchangeInfo.get_value>("vectorOfIntToExchange"); + auto recvVectorOfPair = exchangeInfo.get_value>>("vectorOfPairToExchange"); + + EXPECT_EQ(exepctedStringValue, recvString); + EXPECT_EQ(expectedIntVector, recvVectorOfInt); + EXPECT_EQ(expectedVectorPairStringDouble, recvVectorOfPair); +} +//ENDsync_info_exchange_two_colors + +//BEGINsync_info_exchange_multi_colors +TEST(StkCouplingDocTest, sync_info_exchange_multi_colors) +{ + using stk::coupling::SplitComms; + using stk::coupling::SyncInfo; + + auto commWorld = MPI_COMM_WORLD; + auto commSize = stk::parallel_machine_size(commWorld); + + if (commSize % 3 != 0) GTEST_SKIP(); + + auto rank = stk::parallel_machine_rank(commWorld); + auto color = rank; + auto intToExchange = color * (commSize / 3); + + SyncInfo syncInfo("exchange_info"); + SplitComms splitComms(commWorld, color); + + syncInfo.set_value("value", intToExchange); + + SyncInfo::ColorToSyncInfoMap otherInfos = syncInfo.exchange(splitComms); + + std::for_each(otherInfos.begin(), otherInfos.end(), [&](const auto &mapElement) { + [[maybe_unused]] int otherColor = mapElement.first; + SyncInfo otherSyncInfo = mapElement.second; + + EXPECT_NE(intToExchange, otherSyncInfo.get_value("value")); + EXPECT_TRUE(otherSyncInfo.has_value("value")); + }); +} +//ENDsync_info_exchange_multi_colors + +//BEGINsync_info_choose_values +TEST(StkCouplingDocTest, sync_info_choose_values) +{ + using stk::coupling::SplitComms; + using stk::coupling::SyncInfo; + using stk::coupling::SyncMode; + + SyncInfo syncInfo("sync_info"); + SyncInfo otherInfo("other_sync_info"); + const std::string parameterName = "time_step"; + + syncInfo.set_value(parameterName, 1.0); + otherInfo.set_value(parameterName, 2.0); + + EXPECT_DOUBLE_EQ(stk::coupling::choose_value(syncInfo, otherInfo, parameterName, SyncMode::Send), 1.0); + EXPECT_DOUBLE_EQ(stk::coupling::choose_value(syncInfo, otherInfo, parameterName, SyncMode::Receive), 2.0); + EXPECT_DOUBLE_EQ(stk::coupling::choose_value(syncInfo, otherInfo, parameterName, SyncMode::Minimum), 1.0); + EXPECT_DOUBLE_EQ(stk::coupling::choose_value(syncInfo, otherInfo, parameterName, SyncMode::Any), 1.0); +} +//ENDsync_info_choose_values + +TEST(StkCouplingDocTest, sync_info_check_sync_mode) +{ + using stk::coupling::SplitComms; + using stk::coupling::SyncInfo; + using stk::coupling::SyncMode; + + auto commWorld = MPI_COMM_WORLD; + auto commSize = stk::parallel_machine_size(commWorld); + + if (commSize != 2) GTEST_SKIP(); + + auto rank = stk::parallel_machine_rank(commWorld); + auto color = rank; + SyncMode modeToCheck = (color == 0) ? stk::coupling::Send : stk::coupling::Receive; + + SplitComms splitComms(commWorld, color); + SyncInfo syncInfo("exchange_info"); + syncInfo.set_value(stk::coupling::TimeSyncMode, modeToCheck); + + SyncInfo exchangeInfo = syncInfo.exchange(splitComms, splitComms.get_other_colors()[0]); + EXPECT_NO_THROW(stk::coupling::check_sync_mode_consistency(syncInfo, exchangeInfo)); + + SyncMode modeFail = stk::coupling::Send; + syncInfo.set_value(stk::coupling::TimeSyncMode, modeFail); + exchangeInfo = syncInfo.exchange(splitComms, splitComms.get_other_colors()[0]); + EXPECT_ANY_THROW(stk::coupling::check_sync_mode_consistency(syncInfo, exchangeInfo)); +} diff --git a/packages/stk/stk_doc_tests/stk_coupling/CMakeLists.txt b/packages/stk/stk_doc_tests/stk_coupling/CMakeLists.txt new file mode 100644 index 000000000000..eae469ad85f7 --- /dev/null +++ b/packages/stk/stk_doc_tests/stk_coupling/CMakeLists.txt @@ -0,0 +1,52 @@ +# Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +# Solutions of Sandia, LLC (NTESS). Under the terms of Contract +# DE-NA0003525 with NTESS, the U.S. Government retains certain rights +# in this software. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * Neither the name of NTESS nor the names of its contributors +# may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + + +TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) +TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) +TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util/stk_util) +TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../stk_coupling) + +FILE(GLOB SOURCES *.cpp) + +TRIBITS_ADD_EXECUTABLE_AND_TEST( + stk_coupling_doc_tests + SOURCES ${SOURCES} + TESTONLYLIBS stk_unit_main + ARGS "--gtest_filter=StkCouplingHowTo.use*" + COMM mpi + NUM_MPI_PROCS 1 + NOEXEPREFIX + ) + diff --git a/packages/stk/stk_doc_tests/stk_expreval/CMakeLists.txt b/packages/stk/stk_doc_tests/stk_expreval/CMakeLists.txt index 0b69588b93ef..3bbe96470f8b 100644 --- a/packages/stk/stk_doc_tests/stk_expreval/CMakeLists.txt +++ b/packages/stk/stk_doc_tests/stk_expreval/CMakeLists.txt @@ -33,10 +33,6 @@ # -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util/stk_util) - FILE(GLOB SOURCES *.cpp) TRIBITS_ADD_EXECUTABLE_AND_TEST( diff --git a/packages/stk/stk_doc_tests/stk_io/CMakeLists.txt b/packages/stk/stk_doc_tests/stk_io/CMakeLists.txt index 6adff4e0c4bb..fb3e419aab09 100644 --- a/packages/stk/stk_doc_tests/stk_io/CMakeLists.txt +++ b/packages/stk/stk_doc_tests/stk_io/CMakeLists.txt @@ -32,10 +32,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util/stk_util) - SET(SOURCES "") FILE(GLOB SOURCES *.cpp) #disable because of dependency on data file not in Trilinos @@ -54,3 +50,7 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST( NOEXEPREFIX ) +target_include_directories(stk_io_doc_tests PRIVATE + $ + $ +) diff --git a/packages/stk/stk_doc_tests/stk_mesh/CMakeLists.txt b/packages/stk/stk_doc_tests/stk_mesh/CMakeLists.txt index a8aed2874ebd..ca97c0ec0e77 100644 --- a/packages/stk/stk_doc_tests/stk_mesh/CMakeLists.txt +++ b/packages/stk/stk_doc_tests/stk_mesh/CMakeLists.txt @@ -32,11 +32,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util/stk_util) - FILE(GLOB SOURCES *.cpp) TRIBITS_ADD_EXECUTABLE_AND_TEST( @@ -49,3 +44,7 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST( NOEXEPREFIX ) +target_include_directories(stk_mesh_doc_tests PRIVATE + $ + $ +) diff --git a/packages/stk/stk_doc_tests/stk_search/CMakeLists.txt b/packages/stk/stk_doc_tests/stk_search/CMakeLists.txt index dd309998dab5..478b3e60cb3e 100644 --- a/packages/stk/stk_doc_tests/stk_search/CMakeLists.txt +++ b/packages/stk/stk_doc_tests/stk_search/CMakeLists.txt @@ -33,9 +33,7 @@ # -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util/stk_util) TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../stk_search_util) TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../stk_search) @@ -51,3 +49,7 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST( NOEXEPREFIX ) +target_include_directories(stk_search_doc_tests PRIVATE + $ + $ +) diff --git a/packages/stk/stk_doc_tests/stk_search/howToUseCoarseSearch.cpp b/packages/stk/stk_doc_tests/stk_search/howToUseCoarseSearch.cpp new file mode 100644 index 000000000000..e76d487f6b8b --- /dev/null +++ b/packages/stk/stk_doc_tests/stk_search/howToUseCoarseSearch.cpp @@ -0,0 +1,201 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +#include +#include "searchMockMesh.hpp" +#include "stk_search/CoarseSearch.hpp" + +namespace doc_test +{ +template +struct CoarseSearchTrait { + using SendMesh = SENDMESH; + using RecvMesh = RECVMESH; + using SendEntityProc = typename SENDMESH::EntityProc; + using RecvEntityProc = typename RECVMESH::EntityProc; + using SendBoundingBox = typename SENDMESH::BoundingBox; + using RecvBoundingBox = typename RECVMESH::BoundingBox; + + using EntityProcRelation = std::pair; + using EntityProcRelationVec = std::vector; +}; + +template +struct BoundingBoxCompare { + bool operator()(const BoundingBoxType& a, const BoundingBoxType& b) const { return a.second.id() < b.second.id(); } +}; + +template +bool local_is_sorted(ForwardIterator first, ForwardIterator last, Compare comparison) +{ + if (first == last) return true; + ForwardIterator next = first; + while (++next != last) { + if (comparison(*next, *first)) return false; + ++first; + } + return true; +} + +template +inline void inflate_bounding_box(stk::search::Sphere& s, U const& mult_fact, U const& add_fact) +{ + s.set_radius(s.radius() * mult_fact + add_fact); +} + +template +inline void inflate_bounding_box(stk::search::Box& b, U const& mult_fact, U const& add_fact) +{ + const double zero = 0.0; + + STK_ThrowRequire(mult_fact >= zero); + STK_ThrowRequire(add_fact >= zero); + + stk::search::Point& min_corner = b.min_corner(); + stk::search::Point& max_corner = b.max_corner(); + + T diag = 0.0; + for (int i = 0; i < 3; ++i) { + const T dd = max_corner[i] - min_corner[i]; + diag += dd * dd; + } + diag = std::sqrt(diag); + + const T d = mult_fact * diag + add_fact; + + for (int i = 0; i < 3; ++i) { + min_corner[i] -= d; + max_corner[i] += d; + } +} + +//BEGINcoarse_search +template +void do_coarse_search(typename CoarseSearchType::SendMesh& sendMesh, + typename CoarseSearchType::RecvMesh& recvMesh, + const double expansionFactor, + const double expansionSum, + typename CoarseSearchType::EntityProcRelationVec& coarseSearchResult) +{ + using SendBoundingBox = typename CoarseSearchType::SendBoundingBox; + using RecvBoundingBox = typename CoarseSearchType::RecvBoundingBox; + + std::vector domain_vector; + std::vector range_vector; + + sendMesh.bounding_boxes(domain_vector); + recvMesh.bounding_boxes(range_vector); + + if (!local_is_sorted(domain_vector.begin(), domain_vector.end(), BoundingBoxCompare())) + std::sort(domain_vector.begin(), domain_vector.end(), BoundingBoxCompare()); + + if (!local_is_sorted(range_vector.begin(), range_vector.end(), BoundingBoxCompare())) + std::sort(range_vector.begin(), range_vector.end(), BoundingBoxCompare()); + + for (SendBoundingBox& i : domain_vector) { + inflate_bounding_box(i.first, expansionFactor, expansionSum); + } + + stk::search::coarse_search(range_vector, domain_vector, stk::search::KDTREE, sendMesh.comm(), coarseSearchResult); + + std::sort(coarseSearchResult.begin(), coarseSearchResult.end()); +} +//ENDcoarse_search + +TEST(StkSearchHowTo, useCoarseSearch) +{ +//BEGINuse_coarse_search + using CoarseSearchType = CoarseSearchTrait; + using Relation = typename CoarseSearchType::EntityProcRelation; + using RelationVec = typename CoarseSearchType::EntityProcRelationVec; + + MPI_Comm communicator = MPI_COMM_WORLD; + if (stk::parallel_machine_size(communicator) != 1) { + GTEST_SKIP(); + } + + // Build 8 element cube + const std::string meshSpec("generated:2x2x2"); + const unsigned spatialDim = 3; + + stk::mesh::MeshBuilder builder(communicator); + builder.set_spatial_dimension(spatialDim); + std::shared_ptr mesh = builder.create(); + stk::mesh::MetaData& meta = mesh->mesh_meta_data(); + meta.use_simple_fields(); + stk::io::fill_mesh(meshSpec, *mesh); + + // Point in element 1 + double x = 0.5, y = 1, z = 1; + double geometricTolerance = 0.1; + double parametricTolerance = 0.001; + stk::mesh::EntityKey expectedSendKey(stk::topology::ELEM_RANK, 1u); + + // Create recv mesh + auto recvMesh = std::make_shared(communicator, x, y, z, parametricTolerance, geometricTolerance); + + // Create send mesh + stk::mesh::Part* part = meta.get_part("block_1"); + STK_ThrowRequireMsg(nullptr != part, "Error: block_1 does not exist"); + stk::mesh::PartVector parts{part}; + auto sendMesh = std::make_shared(*mesh, parts, mesh->parallel(), parametricTolerance); + + RelationVec coarseSearchResult; + + // Get single recv point + SinglePointMesh::EntityKey expectedRecvKey(1); + SinglePointMesh::EntityProc rangeEntry(expectedRecvKey, 0); + + double expansionFactor = 0.01; + double expansionSum = 0.005; + + do_coarse_search(*sendMesh, *recvMesh, expansionFactor, expansionSum, coarseSearchResult); + + EXPECT_EQ(4u, coarseSearchResult.size()); +//ENDuse_coarse_search + + stk::mesh::EntityIdVector expectedSendIds{1, 3, 5, 7}; + + for (auto& result : coarseSearchResult) { + const Hex8SourceMesh::EntityKey sendEntityKey = result.second.id(); + EXPECT_EQ(stk::topology::ELEM_RANK, sendEntityKey.rank()); + + auto found = std::binary_search(expectedSendIds.begin(), expectedSendIds.end(), sendEntityKey.id()); + EXPECT_TRUE(found); + + const int recvEntityKey = result.first.id(); + EXPECT_EQ(1, recvEntityKey); + } +} + +} // namespace doc_test diff --git a/packages/stk/stk_doc_tests/stk_search/howToUseFilterCoarseSearch.cpp b/packages/stk/stk_doc_tests/stk_search/howToUseFilterCoarseSearch.cpp index dfa0958f3b97..1e14798e00cd 100644 --- a/packages/stk/stk_doc_tests/stk_search/howToUseFilterCoarseSearch.cpp +++ b/packages/stk/stk_doc_tests/stk_search/howToUseFilterCoarseSearch.cpp @@ -32,626 +32,14 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #include -#include -#include // for sqrt -#include // for size_t -#include // for int64_t, uint64_t -#include // for numeric_limits -#include // for logic_error -#include // for string, basic_s... -#include // for type_info -#include // for move, pair -#include // for vector, swap -#include -#include -#include -#include -#include // for sort, max, min -#include // for __shared_ptr_ac... -#include - -#include -#include -#include "stk_mesh/base/Entity.hpp" // for Entity -#include -#include -#include "stk_mesh/base/Part.hpp" // for Part -#include "stk_mesh/base/Selector.hpp" // for Selector, opera... -#include "stk_mesh/base/Types.hpp" // for EntityRank, Ent... -#include "stk_search/DistanceComparison.hpp" // for stk_distance -#include "stk_search/Box.hpp" // for Box -#include "stk_search/IdentProc.hpp" // for IdentProc -#include "stk_search/Point.hpp" // for Point -#include "stk_search/Sphere.hpp" -#include "stk_search/FilterCoarseSearch.hpp" -#include "stk_search/SearchInterface.hpp" -#include "stk_search_util/ObjectCoordinates.hpp" // for compute_entity_centroid -#include "stk_topology/topology.hpp" // for topology, topol... -#include "stk_util/parallel/Parallel.hpp" // for parallel_machin... -#include "stk_util/util/ReportHandler.hpp" // for ThrowRequireMsg +#include "searchMockMesh.hpp" namespace doc_test { -class SourceMesh; -class SinglePointMesh; -} - -namespace stk { namespace search { -template<> -struct MeshTraits -{ - using Entity = stk::mesh::Entity; - using EntityVec = std::vector; - using EntityKey = stk::mesh::EntityKey; - using EntityKeySet = std::set; - using EntityProc = stk::search::IdentProc; - using EntityProcVec = std::vector; - using Point = stk::search::Point; - using Box = stk::search::Box; - using BoundingBox = std::pair; - using CoordinateField = stk::mesh::Field; -}; -}} - -namespace stk { namespace search { -template<> -struct MeshTraits -{ - using Entity = int; - using EntityVec = std::vector; - using EntityKey = int; - using EntityKeySet = std::set ; - using EntityProc = stk::search::IdentProc ; - using EntityProcVec = std::vector ; - using Point = stk::search::Point ; - using Box = stk::search::Box ; - using Sphere = stk::search::Sphere; - using BoundingBox = std::pair; - using CoordinateField = double*; -}; -}} - -namespace doc_test { - -namespace Hex { -double invSqrt(double x) { - // use the bit-shifting magic of the fast inverse square root method for 64-bit numbers - - union { - double f; - std::int64_t i; - } conv; - - double x2 = 0.5 * x; - conv.f = x; - conv.i = 0x5fe6eb50c7b537a9 - ( conv.i >> 1 ); - conv.f = conv.f * ( 1.5 - ( x2 * conv.f * conv.f ) ); - return conv.f; -} - -template -T vector_norm(std::array &x) -{ - T norm_sq = 0.0; - for (std::size_t i = 0; i < N; ++i ) { - norm_sq += x[i] * x[i]; - } - return norm_sq; -} - -bool within_tol(const double& value, const double& tolerance) -{ - return ( std::fabs(value) < tolerance ); -} - -double parametric_distance(const std::array& x) -{ - std::array y = {{ std::fabs(x[0]), std::fabs(x[1]), std::fabs(x[2]) }}; - - double d = 0.0; - for(int i = 0; i < 3; ++i) { - if(d < y[i]) { - d = y[i]; - } - } - return d; -} - -double is_in_element(const double* elem_nodal_coor, // (8,3) - const double* point_coor, // (3) - double* par_coor) -{ - const double isInElemConverged = 1.0e-16; - // Translate element so that (x,y,z) coordinates of the first node are (0,0,0) - double x[] = { 0., 0.125 * (elem_nodal_coor[1] - elem_nodal_coor[0]), 0.125 * (elem_nodal_coor[2] - elem_nodal_coor[0]), - 0.125 * (elem_nodal_coor[3] - elem_nodal_coor[0]), 0.125 * (elem_nodal_coor[4] - elem_nodal_coor[0]), - 0.125 * (elem_nodal_coor[5] - elem_nodal_coor[0]), 0.125 * (elem_nodal_coor[6] - elem_nodal_coor[0]), - 0.125 * (elem_nodal_coor[7] - elem_nodal_coor[0]) }; - double y[] = { 0., 0.125 * (elem_nodal_coor[9] - elem_nodal_coor[8]), - 0.125 * (elem_nodal_coor[10] - elem_nodal_coor[8]), 0.125 * (elem_nodal_coor[11] - elem_nodal_coor[8]), - 0.125 * (elem_nodal_coor[12] - elem_nodal_coor[8]), 0.125 * (elem_nodal_coor[13] - elem_nodal_coor[8]), - 0.125 * (elem_nodal_coor[14] - elem_nodal_coor[8]), 0.125 * (elem_nodal_coor[15] - elem_nodal_coor[8]) }; - double z[] = { 0., 0.125 * (elem_nodal_coor[17] - elem_nodal_coor[16]), - 0.125 * (elem_nodal_coor[18] - elem_nodal_coor[16]), 0.125 * (elem_nodal_coor[19] - elem_nodal_coor[16]), - 0.125 * (elem_nodal_coor[20] - elem_nodal_coor[16]), 0.125 * (elem_nodal_coor[21] - elem_nodal_coor[16]), - 0.125 * (elem_nodal_coor[22] - elem_nodal_coor[16]), - 0.125 * (elem_nodal_coor[23] - elem_nodal_coor[16]) }; - - // (xp,yp,zp) is the point at which we're searching for (xi,eta,zeta) - // (must translate this also) - double xp = point_coor[0] - elem_nodal_coor[0]; - double yp = point_coor[1] - elem_nodal_coor[8]; - double zp = point_coor[2] - elem_nodal_coor[16]; - - // Newton-Raphson iteration for (xi,eta,zeta) - double j[9]; - double f[3]; - double shapefct[8]; - double xinew = 0.5; // initial guess - double etanew = 0.5; - double zetanew = 0.5; - double xicur = xinew; - double etacur = etanew; - double zetacur = zetanew; - std::array xidiff = {{1.0, 1.0, 1.0}}; - unsigned i = 1; - const unsigned MAX_NR_ITER = 100; - - double xp8 = 0.125 * xp; - double yp8 = 0.125 * yp; - double zp8 = 0.125 * zp; - - constexpr std::array,5> t2n = {{ - {0,1,3,4}, - {4,1,5,6}, - {7,3,6,4}, - {6,1,2,3}, - {6,1,3,4} - }}; - constexpr std::array,5> tmat = {{ - {2, 0, 0,-1, 0, 2, 0,-1, 0, 0, 2,-1}, - {2, 2, 2,-1, 0, 0, 2,-1,-2, 0, 0, 1}, - {0, 2, 0,-1, 0, 0,-2, 1,-2, 0, 0, 1}, - {0, 0,-2, 1,-2, 0, 0, 1,-2,-2,-2, 1}, - {0,-2,-2, 1,-2, 0,-2, 1,-2,-2, 0, 1} - }}; - - - // Break the hex into five tets, and search inside each - bool found = false; - for (int tindex = 0 ; tindex < 5 ; tindex++) { - double a11 = x[t2n[tindex][1]]-x[t2n[tindex][0]]; - double a21 = y[t2n[tindex][1]]-y[t2n[tindex][0]]; - double a31 = z[t2n[tindex][1]]-z[t2n[tindex][0]]; - double a12 = x[t2n[tindex][2]]-x[t2n[tindex][0]]; - double a22 = y[t2n[tindex][2]]-y[t2n[tindex][0]]; - double a32 = z[t2n[tindex][2]]-z[t2n[tindex][0]]; - double a13 = x[t2n[tindex][3]]-x[t2n[tindex][0]]; - double a23 = y[t2n[tindex][3]]-y[t2n[tindex][0]]; - double a33 = z[t2n[tindex][3]]-z[t2n[tindex][0]]; - double f1 = xp8-x[t2n[tindex][0]]; - double f2 = yp8-y[t2n[tindex][0]]; - double f3 = zp8-z[t2n[tindex][0]]; - double oden = 1.0 / ( a31*(a13*a22-a12*a23) + a32*(a11*a23-a13*a21) + a33*(a12*a21-a11*a22) ) ; - double myxi = ( f1*(a23*a32-a22*a33) + f2*(a12*a33-a13*a32) + f3*(a13*a22-a12*a23) ) * oden; - double myeta = - ( f1*(a23*a31-a21*a33) + f2*(a11*a33-a13*a31) + f3*(a13*a21-a11*a23) ) * oden; - double myzeta = ( f1*(a22*a31-a21*a32) + f2*(a11*a32-a12*a31) + f3*(a12*a21-a11*a22) ) * oden; - - if (myxi >= 0 && myeta >= 0 && myzeta >= 0 && myzeta <= 1.0 - myxi - myeta) { - xicur = tmat[tindex][0]*myxi+tmat[tindex][1]*myeta+tmat[tindex][2 ]*myzeta+tmat[tindex][3 ]; - etacur = tmat[tindex][4]*myxi+tmat[tindex][5]*myeta+tmat[tindex][6 ]*myzeta+tmat[tindex][7 ]; - zetacur = tmat[tindex][8]*myxi+tmat[tindex][9]*myeta+tmat[tindex][10]*myzeta+tmat[tindex][11]; - found = true; - break; - } - } - - // If the point is not found inside any of the tetrahedra, fall back to IDW - if (!found) { - double w0 = invSqrt((xp8-x[0])*(xp8-x[0]) + (yp8-y[0])*(yp8-y[0]) + (zp8-z[0])*(zp8-z[0])); - double w1 = invSqrt((xp8-x[1])*(xp8-x[1]) + (yp8-y[1])*(yp8-y[1]) + (zp8-z[1])*(zp8-z[1])); - double w2 = invSqrt((xp8-x[2])*(xp8-x[2]) + (yp8-y[2])*(yp8-y[2]) + (zp8-z[2])*(zp8-z[2])); - double w3 = invSqrt((xp8-x[3])*(xp8-x[3]) + (yp8-y[3])*(yp8-y[3]) + (zp8-z[3])*(zp8-z[3])); - double w4 = invSqrt((xp8-x[4])*(xp8-x[4]) + (yp8-y[4])*(yp8-y[4]) + (zp8-z[4])*(zp8-z[4])); - double w5 = invSqrt((xp8-x[5])*(xp8-x[5]) + (yp8-y[5])*(yp8-y[5]) + (zp8-z[5])*(zp8-z[5])); - double w6 = invSqrt((xp8-x[6])*(xp8-x[6]) + (yp8-y[6])*(yp8-y[6]) + (zp8-z[6])*(zp8-z[6])); - double w7 = invSqrt((xp8-x[7])*(xp8-x[7]) + (yp8-y[7])*(yp8-y[7]) + (zp8-z[7])*(zp8-z[7])); - - double wt = 1.0 / (w0 + w1 + w2 + w3 + w4 + w5 + w6 + w7); - double p6m0 = w6 - w0; - double p7m1 = w7 - w1; - double p2m4 = w2 - w4; - double p5m3 = w5 - w3; - xicur = (p6m0 - p7m1 + p2m4 + p5m3)*wt; - etacur = (p6m0 + p7m1 + p2m4 - p5m3)*wt; - zetacur = (p6m0 + p7m1 - p2m4 + p5m3)*wt; - } - - // Constants for the iteration - double x3mx2 = x[3]-x[2]; - double x4mx5 = x[4]-x[5]; - double x7mx6 = x[7]-x[6]; - double x1mx2 = x[1]-x[2]; - double x4mx7 = x[4]-x[7]; - double x5mx6 = x[5]-x[6]; - double x1mx5 = x[1]-x[5]; - double x2mx6 = x[2]-x[6]; - double x3mx7 = x[3]-x[7]; - - double y3my2 = y[3]-y[2]; - double y4my5 = y[4]-y[5]; - double y7my6 = y[7]-y[6]; - double y1my2 = y[1]-y[2]; - double y4my7 = y[4]-y[7]; - double y5my6 = y[5]-y[6]; - double y1my5 = y[1]-y[5]; - double y2my6 = y[2]-y[6]; - double y3my7 = y[3]-y[7]; - - double z3mz2 = z[3]-z[2]; - double z4mz5 = z[4]-z[5]; - double z7mz6 = z[7]-z[6]; - double z1mz2 = z[1]-z[2]; - double z4mz7 = z[4]-z[7]; - double z5mz6 = z[5]-z[6]; - double z1mz5 = z[1]-z[5]; - double z2mz6 = z[2]-z[6]; - double z3mz7 = z[3]-z[7]; - - // Actual NR iteration - do { - double one_minu_xi = 1.0 - xicur; - double one_plus_xi = 1.0 + xicur; - double one_minu_eta = 1.0 - etacur; - double one_plus_eta = 1.0 + etacur; - double one_minu_zeta = 1.0 - zetacur; - double one_plus_zeta = 1.0 + zetacur; - - double memz = one_minu_eta * one_minu_zeta; - double mepz = one_minu_eta * one_plus_zeta; - double pepz = one_plus_eta * one_plus_zeta; - double pemz = one_plus_eta * one_minu_zeta; - - double mxmz = one_minu_xi * one_minu_zeta; - double mxpz = one_minu_xi * one_plus_zeta; - double pxpz = one_plus_xi * one_plus_zeta; - double pxmz = one_plus_xi * one_minu_zeta; - - double mxme = one_minu_xi * one_minu_eta; - double mxpe = one_minu_xi * one_plus_eta; - double pxpe = one_plus_xi * one_plus_eta; - double pxme = one_plus_xi * one_minu_eta; - - j[0] = - memz * x[1] + pemz * x3mx2 + mepz * x4mx5 + pepz * x7mx6; - j[1] = pxmz * x1mx2 - mxmz * x[3] + mxpz * x4mx7 + pxpz * x5mx6; - j[2] = pxme * x1mx5 + pxpe * x2mx6 + mxpe * x3mx7 - mxme * x[4]; - j[3] = - memz * y[1] + pemz * y3my2 + mepz * y4my5 + pepz * y7my6; - j[4] = pxmz * y1my2 - mxmz * y[3] + mxpz * y4my7 + pxpz * y5my6; - j[5] = pxme * y1my5 + pxpe * y2my6 + mxpe * y3my7 - mxme * y[4]; - j[6] = - memz * z[1] + pemz * z3mz2 + mepz * z4mz5 + pepz * z7mz6; - j[7] = pxmz * z1mz2 - mxmz * z[3] + mxpz * z4mz7 + pxpz * z5mz6; - j[8] = pxme * z1mz5 + pxpe * z2mz6 + mxpe * z3mz7 - mxme * z[4]; - - double jdet = -(j[2] * j[4] * j[6]) + j[1] * j[5] * j[6] + j[2] * j[3] * j[7] - j[0] * j[5] * j[7] - - j[1] * j[3] * j[8] + j[0] * j[4] * j[8]; - double odet = 1.0 / jdet; - - if(!jdet) { - i = MAX_NR_ITER; - break; - } - - shapefct[0] = mxme * one_minu_zeta; - shapefct[1] = pxme * one_minu_zeta; - shapefct[2] = pxpe * one_minu_zeta; - shapefct[3] = mxpe * one_minu_zeta; - shapefct[4] = mxme * one_plus_zeta; - shapefct[5] = pxme * one_plus_zeta; - shapefct[6] = pxpe * one_plus_zeta; - shapefct[7] = mxpe * one_plus_zeta; - - f[0] = xp - shapefct[1] * x[1] - shapefct[2] * x[2] - shapefct[3] * x[3] - shapefct[4] * x[4] - shapefct[5] * x[5] - - shapefct[6] * x[6] - shapefct[7] * x[7]; - f[1] = yp - shapefct[1] * y[1] - shapefct[2] * y[2] - shapefct[3] * y[3] - shapefct[4] * y[4] - shapefct[5] * y[5] - - shapefct[6] * y[6] - shapefct[7] * y[7]; - f[2] = zp - shapefct[1] * z[1] - shapefct[2] * z[2] - shapefct[3] * z[3] - shapefct[4] * z[4] - shapefct[5] * z[5] - - shapefct[6] * z[6] - shapefct[7] * z[7]; - - double relax = 1.0; - xinew = xicur + relax * (f[2] * (j[2] * j[4] - j[1] * j[5]) + f[1] * (j[1] * j[8] - j[2] * j[7]) + - f[0] * (j[5] * j[7] - j[4] * j[8])) * odet; - etanew = etacur + relax * (f[2] * (- j[2] * j[3] + j[0] * j[5]) + f[1] * (j[2] * j[6] - j[0] * j[8]) + - f[0] * (j[3] * j[8] - j[5] * j[6])) * odet; - zetanew = zetacur + relax*(f[2] * (j[1] * j[3] - j[0] * j[4]) + f[1] * (j[0] * j[7] - j[1] * j[6]) + - f[0] * (j[4] * j[6] - j[3] * j[7])) * odet; - - xidiff[0] = xinew - xicur; - xidiff[1] = etanew - etacur; - xidiff[2] = zetanew - zetacur; - xicur = xinew; - etacur = etanew; - zetacur = zetanew; - } while(!within_tol(vector_norm(xidiff), isInElemConverged) && ++i < MAX_NR_ITER); - - par_coor[0] = par_coor[1] = par_coor[2] = std::numeric_limits::max(); - double dist = std::numeric_limits::max(); - - if(i < MAX_NR_ITER) { - par_coor[0] = xinew; - par_coor[1] = etanew; - par_coor[2] = zetanew; - - std::array xtmp = {{ par_coor[0], par_coor[1], par_coor[2] }}; - dist = parametric_distance(xtmp); - } - - return dist; -} -} - -class SourceMesh : public stk::search::SourceMeshInterface { - public: - using Entity = typename stk::search::MeshTraits::Entity; - using EntityVec = typename stk::search::MeshTraits::EntityVec; - using EntityKey = typename stk::search::MeshTraits::EntityKey; - using EntityKeySet = typename stk::search::MeshTraits::EntityKeySet; - using EntityProc = typename stk::search::MeshTraits::EntityProc; - using EntityProcVec = typename stk::search::MeshTraits::EntityProcVec; - using Point = typename stk::search::MeshTraits::Point; - using Box = typename stk::search::MeshTraits::Box; - using BoundingBox = typename stk::search::MeshTraits::BoundingBox; - using CoordinateField = typename stk::search::MeshTraits::CoordinateField; - - SourceMesh(stk::mesh::BulkData& bulkData, const stk::mesh::PartVector& sendParts, - const stk::ParallelMachine comm, const double parametricTolerance) - : m_meta(bulkData.mesh_meta_data()) - , m_bulk(bulkData) - , m_coordinateField(bulkData.mesh_meta_data().coordinate_field()) - , m_parts(sendParts) - , m_comm(comm) - , m_parametricTolerance(parametricTolerance) - , m_extrapolateOption(stk::search::ObjectOutsideDomainPolicy::UNDEFINED_OBJFLAG) - { - for(const stk::mesh::Part* part : sendParts) { - STK_ThrowRequireMsg(part->primary_entity_rank() == stk::topology::ELEM_RANK, - "All source parts must be {ELEM_RANK}"); - } - } - - stk::ParallelMachine comm() const { return m_comm; } - - std::string name() const { return "SourceMesh"; } - void set_extrapolate_option(stk::search::ObjectOutsideDomainPolicy option) { m_extrapolateOption = option; } - stk::search::ObjectOutsideDomainPolicy get_extrapolate_option() const { return m_extrapolateOption; } - - void bounding_boxes(std::vector& boxes) const - { - Point min_corner, max_corner; - - stk::mesh::Selector selector = stk::mesh::selectUnion(m_parts); - stk::mesh::BucketVector const& buckets = m_bulk.get_buckets(stk::topology::ELEM_RANK, selector); - - for(auto&& ib : buckets) { - stk::mesh::Bucket& b = *ib; - - for(auto elem : b) { - fill_bounding_box(elem, min_corner, max_corner); - - EntityProc theIdent(m_bulk.entity_key(elem), m_bulk.parallel_rank()); - BoundingBox theBox(Box(min_corner, max_corner), theIdent); - boxes.push_back(theBox); - } - } - std::sort(boxes.begin(), boxes.end(), - [](const BoundingBox& a, const BoundingBox& b) { return a.second.id() < b.second.id(); }); - } - - void find_parametric_coords(const EntityKey k, const double* toCoords, - std::vector& parametricCoords, - double& parametricDistance, - bool& isWithinParametricTolerance) const - { - stk::mesh::Entity elem = m_bulk.get_entity(k); - stk::topology topology = m_bulk.bucket(elem).topology(); - STK_ThrowRequireMsg(topology == stk::topology::HEX_8, "Invalid topology: " << topology.name()); - - // load nodal coordinates from element - stk::mesh::Entity const* nodes = m_bulk.begin_nodes(elem); - const auto numNodes = m_bulk.num_nodes(elem); - unsigned nDim = m_meta.spatial_dimension(); - - std::vector transposedElementCoords(nDim * numNodes); - - for(auto ni = 0u; ni < numNodes; ++ni) { - stk::mesh::Entity node = nodes[ni]; - - const double* fromCoords = static_cast(stk::mesh::field_data(*m_coordinateField, node)); - for(unsigned j = 0; j < nDim; ++j) { - const auto offSet = ni + j * numNodes; - transposedElementCoords[offSet] = fromCoords[j]; - } - } - - parametricCoords.assign(3, std::numeric_limits::max()); - parametricDistance = Hex::is_in_element(transposedElementCoords.data(), toCoords, parametricCoords.data()); - - isWithinParametricTolerance = parametricDistance <= (1 + m_parametricTolerance); - } - - bool modify_search_outside_parametric_tolerance(const EntityKey k, - const double* toCoords, - std::vector& parametricCoords, - double& geometricDistanceSquared, - bool& isWithinGeometricTolerance) const - { - return false; - } - - double get_distance_from_nearest_node(const EntityKey k, const double* point) const - { - const stk::mesh::Entity e = m_bulk.get_entity(k); - - STK_ThrowRequireMsg(m_bulk.entity_rank(e) == stk::topology::ELEM_RANK, - "Invalid entity rank for object: " << m_bulk.entity_rank(e)); - - double minDistance = std::numeric_limits::max(); - const unsigned nDim = m_meta.spatial_dimension(); - - const stk::mesh::Entity* const nodes = m_bulk.begin_nodes(e); - const int num_nodes = m_bulk.num_nodes(e); - - for(int i = 0; i < num_nodes; ++i) { - double d = 0.0; - double* node_coordinates = static_cast(stk::mesh::field_data(*m_coordinateField, nodes[i])); - - for(unsigned j = 0; j < nDim; ++j) { - const double t = point[j] - node_coordinates[j]; - d += t * t; - } - if(d < minDistance) minDistance = d; - } - - minDistance = std::sqrt(minDistance); - return minDistance; - } - - double get_closest_geometric_distance_squared(const EntityKey k, const double* toCoords) const - { - double distance = get_distance_from_nearest_node(k, toCoords); - return distance*distance; - } - - double get_distance_from_centroid(const EntityKey k, const double* toCoords) const - { - double distanceSquared = get_distance_squared_from_centroid(k, toCoords); - return std::sqrt(distanceSquared); - } - - double get_distance_squared_from_centroid(const EntityKey k, const double* toCoords) const - { - std::vector centroidVec; - centroid(k, centroidVec); - - const unsigned nDim = m_meta.spatial_dimension(); - return stk::search::distance_sq(nDim, centroidVec.data(), toCoords); - } - - void centroid(const EntityKey k, std::vector& centroidVec) const - { - const stk::mesh::Entity e = m_bulk.get_entity(k); - stk::search::compute_entity_centroid(e, *m_coordinateField, centroidVec); - } - - const double* coord(const EntityKey k) const - { - centroid(k, m_coordVector); - return m_coordVector.data(); - } - - protected: - stk::mesh::MetaData& m_meta; - stk::mesh::BulkData& m_bulk; - const stk::mesh::FieldBase* m_coordinateField{nullptr}; - - private: - stk::mesh::PartVector m_parts; - const stk::ParallelMachine m_comm; - const double m_parametricTolerance; - - mutable std::vector m_coordVector; - - stk::search::ObjectOutsideDomainPolicy m_extrapolateOption{stk::search::ObjectOutsideDomainPolicy::UNDEFINED_OBJFLAG}; - - SourceMesh(const SourceMesh&) = delete; - const SourceMesh& operator()(const SourceMesh&) = delete; - - void fill_bounding_box(stk::mesh::Entity elem, - stk::search::Point& min_corner, - stk::search::Point& max_corner) const - { - const unsigned nDim = m_meta.spatial_dimension(); - - STK_ThrowRequireMsg(m_bulk.is_valid(elem), "Invalid entity: " << m_bulk.entity_key(elem)); - - for(unsigned j = 0; j < nDim; ++j) { - min_corner[j] = std::numeric_limits::max(); - max_corner[j] = -std::numeric_limits::max(); - } - - stk::mesh::Entity const* nodes = m_bulk.begin_nodes(elem); - int numNodes = m_bulk.num_nodes(elem); - for(int ni = 0; ni < numNodes; ++ni) { - stk::mesh::Entity node = nodes[ni]; - - double* coords = static_cast(stk::mesh::field_data(*m_coordinateField, node)); - - for(unsigned j = 0; j < nDim; ++j) { - min_corner[j] = std::min(min_corner[j], coords[j]); - max_corner[j] = std::max(max_corner[j], coords[j]); - } - } - } -}; - -class SinglePointMesh : public stk::search::DestinationMeshInterface { - public: - using Entity = typename stk::search::MeshTraits::Entity; - using EntityVec = typename stk::search::MeshTraits::EntityVec; - using EntityKey = typename stk::search::MeshTraits::EntityKey; - using EntityKeySet = typename stk::search::MeshTraits::EntityKeySet; - using EntityProc = typename stk::search::MeshTraits::EntityProc; - using EntityProcVec = typename stk::search::MeshTraits::EntityProcVec; - using Point = typename stk::search::MeshTraits::Point; - using Box = typename stk::search::MeshTraits::Box; - using Sphere = typename stk::search::MeshTraits::Sphere; - using BoundingBox = typename stk::search::MeshTraits::BoundingBox; - using CoordinateField = typename stk::search::MeshTraits::CoordinateField; - - SinglePointMesh(const stk::ParallelMachine comm, double x, double y, double z, double paramTol, double geomTol) - : m_comm(comm) - , m_parametricTolerance(paramTol) - , m_geometricTolerance(geomTol) - { - m_coords[0] = x; - m_coords[1] = y; - m_coords[2] = z; - } - - stk::ParallelMachine comm() const { return m_comm; }; - - std::string name() const { return "SinglePointMesh"; } - - void bounding_boxes(std::vector& v) const - { - Point center(m_coords[0], m_coords[1], m_coords[2]); - - EntityKey key = 1; - EntityProc theIdent(key, stk::parallel_machine_rank(m_comm)); - BoundingBox theBox(Sphere(center, m_geometricTolerance), theIdent); - v.push_back(theBox); - } - - const double* coord(const EntityKey k) const { return m_coords; } - double get_search_tolerance() const { return m_geometricTolerance; } - double get_parametric_tolerance() const { return m_parametricTolerance; } - - void centroid(const EntityKey k, std::vector& centroidVec) const { centroidVec.assign(m_coords, m_coords + 3); } - double get_distance_from_nearest_node(const EntityKey k, const double* toCoords) const - { - return stk::search::distance(3, m_coords, toCoords); - } - - private: - const stk::ParallelMachine m_comm; - double m_coords[3]; - double m_parametricTolerance = 0.00001; - double m_geometricTolerance = 0.1; -}; - -//BEGIN +//BEGINfilter_coarse_search TEST(StkSearchHowTo, useFilterCoarseSearch) { - using Relation = std::pair; + using Relation = std::pair; using RelationVec = std::vector; MPI_Comm communicator = MPI_COMM_WORLD; @@ -681,7 +69,7 @@ TEST(StkSearchHowTo, useFilterCoarseSearch) stk::mesh::Part* part = meta.get_part("block_1"); STK_ThrowRequireMsg(nullptr != part, "Error: block_1 does not exist"); stk::mesh::PartVector parts{part}; - auto sendMesh = std::make_shared(*mesh, parts, mesh->parallel(), parametricTolerance); + auto sendMesh = std::make_shared(*mesh, parts, mesh->parallel(), parametricTolerance); RelationVec relationVec; @@ -696,7 +84,7 @@ TEST(StkSearchHowTo, useFilterCoarseSearch) for(auto elem : b) { stk::mesh::EntityKey domainKey = mesh->entity_key(elem); - SourceMesh::EntityProc domainEntry(domainKey, 0); + Hex8SourceMesh::EntityProc domainEntry(domainKey, 0); relationVec.emplace_back(rangeEntry, domainEntry); } @@ -707,10 +95,11 @@ TEST(StkSearchHowTo, useFilterCoarseSearch) bool useNearestNodeForClosestBoundingBox{false}; bool useCentroidForGeometricProximity{false}; bool verbose{false}; + auto extrapolateOption = stk::search::ObjectOutsideDomainPolicy::ABORT; - stk::search::FilterCoarseSearchOptions options(std::cout, sendMesh->get_extrapolate_option(), - useNearestNodeForClosestBoundingBox, - useCentroidForGeometricProximity, verbose); + stk::search::FilterCoarseSearchOptions options(std::cout, extrapolateOption, + useNearestNodeForClosestBoundingBox, + useCentroidForGeometricProximity, verbose); stk::search::FilterCoarseSearchResultVector searchResults; stk::search::filter_coarse_search("filter", relationVec, *sendMesh, *recvMesh, options, searchResults); @@ -718,12 +107,11 @@ TEST(StkSearchHowTo, useFilterCoarseSearch) auto relation = relationVec[0]; const SinglePointMesh::EntityKey recvEntityKey = relation.first.id(); - const SourceMesh::EntityKey sendEntityKey = relation.second.id(); + const Hex8SourceMesh::EntityKey sendEntityKey = relation.second.id(); EXPECT_EQ(expectedRecvKey, recvEntityKey); EXPECT_EQ(expectedSendKey, sendEntityKey); } -//END +//ENDfilter_coarse_search } - diff --git a/packages/stk/stk_doc_tests/stk_search/searchHex8.hpp b/packages/stk/stk_doc_tests/stk_search/searchHex8.hpp new file mode 100644 index 000000000000..766018dac8b5 --- /dev/null +++ b/packages/stk/stk_doc_tests/stk_search/searchHex8.hpp @@ -0,0 +1,335 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef STK_DOC_TEST_SEARCH_HEX_8_HPP +#define STK_DOC_TEST_SEARCH_HEX_8_HPP + +#include // for sqrt +#include +#include // for sort, max, min +#include +#include +#include // for size_t +#include // for int64_t, uint64_t +#include +#include +#include // for numeric_limits +#include // for __shared_ptr_ac... +#include +#include // for logic_error +#include // for string, basic_s... +#include // for type_info +#include // for move, pair +#include // for vector, swap + +namespace doc_test +{ + +class Hex8 +{ + public: + static double invSqrt(double x) + { + // use the bit-shifting magic of the fast inverse square root method for 64-bit numbers + + union { + double f; + std::int64_t i; + } conv; + + double x2 = 0.5 * x; + conv.f = x; + conv.i = 0x5fe6eb50c7b537a9 - (conv.i >> 1); + conv.f = conv.f * (1.5 - (x2 * conv.f * conv.f)); + return conv.f; + } + + template + static T vector_norm(std::array& x) + { + T norm_sq = 0.0; + for (std::size_t i = 0; i < N; ++i) { + norm_sq += x[i] * x[i]; + } + return norm_sq; + } + + static bool within_tol(const double& value, const double& tolerance) { return (std::fabs(value) < tolerance); } + + static double parametric_distance(const std::array& x) + { + std::array y = {{std::fabs(x[0]), std::fabs(x[1]), std::fabs(x[2])}}; + + double d = 0.0; + for (int i = 0; i < 3; ++i) { + if (d < y[i]) { + d = y[i]; + } + } + return d; + } + + static double is_in_element(const double* elem_nodal_coor, // (8,3) + const double* point_coor, // (3) + double* par_coor) + { + const double isInElemConverged = 1.0e-16; + // Translate element so that (x,y,z) coordinates of the first node are (0,0,0) + double x[] = {0., 0.125 * (elem_nodal_coor[1] - elem_nodal_coor[0]), + 0.125 * (elem_nodal_coor[2] - elem_nodal_coor[0]), 0.125 * (elem_nodal_coor[3] - elem_nodal_coor[0]), + 0.125 * (elem_nodal_coor[4] - elem_nodal_coor[0]), 0.125 * (elem_nodal_coor[5] - elem_nodal_coor[0]), + 0.125 * (elem_nodal_coor[6] - elem_nodal_coor[0]), 0.125 * (elem_nodal_coor[7] - elem_nodal_coor[0])}; + double y[] = {0., 0.125 * (elem_nodal_coor[9] - elem_nodal_coor[8]), + 0.125 * (elem_nodal_coor[10] - elem_nodal_coor[8]), 0.125 * (elem_nodal_coor[11] - elem_nodal_coor[8]), + 0.125 * (elem_nodal_coor[12] - elem_nodal_coor[8]), 0.125 * (elem_nodal_coor[13] - elem_nodal_coor[8]), + 0.125 * (elem_nodal_coor[14] - elem_nodal_coor[8]), 0.125 * (elem_nodal_coor[15] - elem_nodal_coor[8])}; + double z[] = {0., 0.125 * (elem_nodal_coor[17] - elem_nodal_coor[16]), + 0.125 * (elem_nodal_coor[18] - elem_nodal_coor[16]), 0.125 * (elem_nodal_coor[19] - elem_nodal_coor[16]), + 0.125 * (elem_nodal_coor[20] - elem_nodal_coor[16]), 0.125 * (elem_nodal_coor[21] - elem_nodal_coor[16]), + 0.125 * (elem_nodal_coor[22] - elem_nodal_coor[16]), 0.125 * (elem_nodal_coor[23] - elem_nodal_coor[16])}; + + // (xp,yp,zp) is the point at which we're searching for (xi,eta,zeta) + // (must translate this also) + double xp = point_coor[0] - elem_nodal_coor[0]; + double yp = point_coor[1] - elem_nodal_coor[8]; + double zp = point_coor[2] - elem_nodal_coor[16]; + + // Newton-Raphson iteration for (xi,eta,zeta) + double j[9]; + double f[3]; + double shapefct[8]; + double xinew = 0.5; // initial guess + double etanew = 0.5; + double zetanew = 0.5; + double xicur = xinew; + double etacur = etanew; + double zetacur = zetanew; + std::array xidiff = {{1.0, 1.0, 1.0}}; + unsigned i = 1; + const unsigned MAX_NR_ITER = 100; + + double xp8 = 0.125 * xp; + double yp8 = 0.125 * yp; + double zp8 = 0.125 * zp; + + constexpr std::array, 5> t2n = { + {{0, 1, 3, 4}, {4, 1, 5, 6}, {7, 3, 6, 4}, {6, 1, 2, 3}, {6, 1, 3, 4}}}; + constexpr std::array, 5> tmat = {{{2, 0, 0, -1, 0, 2, 0, -1, 0, 0, 2, -1}, + {2, 2, 2, -1, 0, 0, 2, -1, -2, 0, 0, 1}, {0, 2, 0, -1, 0, 0, -2, 1, -2, 0, 0, 1}, + {0, 0, -2, 1, -2, 0, 0, 1, -2, -2, -2, 1}, {0, -2, -2, 1, -2, 0, -2, 1, -2, -2, 0, 1}}}; + + // Break the hex into five tets, and search inside each + bool found = false; + for (int tindex = 0; tindex < 5; tindex++) { + double a11 = x[t2n[tindex][1]] - x[t2n[tindex][0]]; + double a21 = y[t2n[tindex][1]] - y[t2n[tindex][0]]; + double a31 = z[t2n[tindex][1]] - z[t2n[tindex][0]]; + double a12 = x[t2n[tindex][2]] - x[t2n[tindex][0]]; + double a22 = y[t2n[tindex][2]] - y[t2n[tindex][0]]; + double a32 = z[t2n[tindex][2]] - z[t2n[tindex][0]]; + double a13 = x[t2n[tindex][3]] - x[t2n[tindex][0]]; + double a23 = y[t2n[tindex][3]] - y[t2n[tindex][0]]; + double a33 = z[t2n[tindex][3]] - z[t2n[tindex][0]]; + double f1 = xp8 - x[t2n[tindex][0]]; + double f2 = yp8 - y[t2n[tindex][0]]; + double f3 = zp8 - z[t2n[tindex][0]]; + double oden = + 1.0 / (a31 * (a13 * a22 - a12 * a23) + a32 * (a11 * a23 - a13 * a21) + a33 * (a12 * a21 - a11 * a22)); + double myxi = (f1 * (a23 * a32 - a22 * a33) + f2 * (a12 * a33 - a13 * a32) + f3 * (a13 * a22 - a12 * a23)) * oden; + double myeta = + -(f1 * (a23 * a31 - a21 * a33) + f2 * (a11 * a33 - a13 * a31) + f3 * (a13 * a21 - a11 * a23)) * oden; + double myzeta = + (f1 * (a22 * a31 - a21 * a32) + f2 * (a11 * a32 - a12 * a31) + f3 * (a12 * a21 - a11 * a22)) * oden; + + if (myxi >= 0 && myeta >= 0 && myzeta >= 0 && myzeta <= 1.0 - myxi - myeta) { + xicur = tmat[tindex][0] * myxi + tmat[tindex][1] * myeta + tmat[tindex][2] * myzeta + tmat[tindex][3]; + etacur = tmat[tindex][4] * myxi + tmat[tindex][5] * myeta + tmat[tindex][6] * myzeta + tmat[tindex][7]; + zetacur = tmat[tindex][8] * myxi + tmat[tindex][9] * myeta + tmat[tindex][10] * myzeta + tmat[tindex][11]; + found = true; + break; + } + } + + // If the point is not found inside any of the tetrahedra, fall back to IDW + if (!found) { + double w0 = invSqrt((xp8 - x[0]) * (xp8 - x[0]) + (yp8 - y[0]) * (yp8 - y[0]) + (zp8 - z[0]) * (zp8 - z[0])); + double w1 = invSqrt((xp8 - x[1]) * (xp8 - x[1]) + (yp8 - y[1]) * (yp8 - y[1]) + (zp8 - z[1]) * (zp8 - z[1])); + double w2 = invSqrt((xp8 - x[2]) * (xp8 - x[2]) + (yp8 - y[2]) * (yp8 - y[2]) + (zp8 - z[2]) * (zp8 - z[2])); + double w3 = invSqrt((xp8 - x[3]) * (xp8 - x[3]) + (yp8 - y[3]) * (yp8 - y[3]) + (zp8 - z[3]) * (zp8 - z[3])); + double w4 = invSqrt((xp8 - x[4]) * (xp8 - x[4]) + (yp8 - y[4]) * (yp8 - y[4]) + (zp8 - z[4]) * (zp8 - z[4])); + double w5 = invSqrt((xp8 - x[5]) * (xp8 - x[5]) + (yp8 - y[5]) * (yp8 - y[5]) + (zp8 - z[5]) * (zp8 - z[5])); + double w6 = invSqrt((xp8 - x[6]) * (xp8 - x[6]) + (yp8 - y[6]) * (yp8 - y[6]) + (zp8 - z[6]) * (zp8 - z[6])); + double w7 = invSqrt((xp8 - x[7]) * (xp8 - x[7]) + (yp8 - y[7]) * (yp8 - y[7]) + (zp8 - z[7]) * (zp8 - z[7])); + + double wt = 1.0 / (w0 + w1 + w2 + w3 + w4 + w5 + w6 + w7); + double p6m0 = w6 - w0; + double p7m1 = w7 - w1; + double p2m4 = w2 - w4; + double p5m3 = w5 - w3; + xicur = (p6m0 - p7m1 + p2m4 + p5m3) * wt; + etacur = (p6m0 + p7m1 + p2m4 - p5m3) * wt; + zetacur = (p6m0 + p7m1 - p2m4 + p5m3) * wt; + } + + // Constants for the iteration + double x3mx2 = x[3] - x[2]; + double x4mx5 = x[4] - x[5]; + double x7mx6 = x[7] - x[6]; + double x1mx2 = x[1] - x[2]; + double x4mx7 = x[4] - x[7]; + double x5mx6 = x[5] - x[6]; + double x1mx5 = x[1] - x[5]; + double x2mx6 = x[2] - x[6]; + double x3mx7 = x[3] - x[7]; + + double y3my2 = y[3] - y[2]; + double y4my5 = y[4] - y[5]; + double y7my6 = y[7] - y[6]; + double y1my2 = y[1] - y[2]; + double y4my7 = y[4] - y[7]; + double y5my6 = y[5] - y[6]; + double y1my5 = y[1] - y[5]; + double y2my6 = y[2] - y[6]; + double y3my7 = y[3] - y[7]; + + double z3mz2 = z[3] - z[2]; + double z4mz5 = z[4] - z[5]; + double z7mz6 = z[7] - z[6]; + double z1mz2 = z[1] - z[2]; + double z4mz7 = z[4] - z[7]; + double z5mz6 = z[5] - z[6]; + double z1mz5 = z[1] - z[5]; + double z2mz6 = z[2] - z[6]; + double z3mz7 = z[3] - z[7]; + + // Actual NR iteration + do { + double one_minu_xi = 1.0 - xicur; + double one_plus_xi = 1.0 + xicur; + double one_minu_eta = 1.0 - etacur; + double one_plus_eta = 1.0 + etacur; + double one_minu_zeta = 1.0 - zetacur; + double one_plus_zeta = 1.0 + zetacur; + + double memz = one_minu_eta * one_minu_zeta; + double mepz = one_minu_eta * one_plus_zeta; + double pepz = one_plus_eta * one_plus_zeta; + double pemz = one_plus_eta * one_minu_zeta; + + double mxmz = one_minu_xi * one_minu_zeta; + double mxpz = one_minu_xi * one_plus_zeta; + double pxpz = one_plus_xi * one_plus_zeta; + double pxmz = one_plus_xi * one_minu_zeta; + + double mxme = one_minu_xi * one_minu_eta; + double mxpe = one_minu_xi * one_plus_eta; + double pxpe = one_plus_xi * one_plus_eta; + double pxme = one_plus_xi * one_minu_eta; + + j[0] = -memz * x[1] + pemz * x3mx2 + mepz * x4mx5 + pepz * x7mx6; + j[1] = pxmz * x1mx2 - mxmz * x[3] + mxpz * x4mx7 + pxpz * x5mx6; + j[2] = pxme * x1mx5 + pxpe * x2mx6 + mxpe * x3mx7 - mxme * x[4]; + j[3] = -memz * y[1] + pemz * y3my2 + mepz * y4my5 + pepz * y7my6; + j[4] = pxmz * y1my2 - mxmz * y[3] + mxpz * y4my7 + pxpz * y5my6; + j[5] = pxme * y1my5 + pxpe * y2my6 + mxpe * y3my7 - mxme * y[4]; + j[6] = -memz * z[1] + pemz * z3mz2 + mepz * z4mz5 + pepz * z7mz6; + j[7] = pxmz * z1mz2 - mxmz * z[3] + mxpz * z4mz7 + pxpz * z5mz6; + j[8] = pxme * z1mz5 + pxpe * z2mz6 + mxpe * z3mz7 - mxme * z[4]; + + double jdet = -(j[2] * j[4] * j[6]) + j[1] * j[5] * j[6] + j[2] * j[3] * j[7] - j[0] * j[5] * j[7] - + j[1] * j[3] * j[8] + j[0] * j[4] * j[8]; + double odet = 1.0 / jdet; + + if (!jdet) { + i = MAX_NR_ITER; + break; + } + + shapefct[0] = mxme * one_minu_zeta; + shapefct[1] = pxme * one_minu_zeta; + shapefct[2] = pxpe * one_minu_zeta; + shapefct[3] = mxpe * one_minu_zeta; + shapefct[4] = mxme * one_plus_zeta; + shapefct[5] = pxme * one_plus_zeta; + shapefct[6] = pxpe * one_plus_zeta; + shapefct[7] = mxpe * one_plus_zeta; + + f[0] = xp - shapefct[1] * x[1] - shapefct[2] * x[2] - shapefct[3] * x[3] - shapefct[4] * x[4] - + shapefct[5] * x[5] - shapefct[6] * x[6] - shapefct[7] * x[7]; + f[1] = yp - shapefct[1] * y[1] - shapefct[2] * y[2] - shapefct[3] * y[3] - shapefct[4] * y[4] - + shapefct[5] * y[5] - shapefct[6] * y[6] - shapefct[7] * y[7]; + f[2] = zp - shapefct[1] * z[1] - shapefct[2] * z[2] - shapefct[3] * z[3] - shapefct[4] * z[4] - + shapefct[5] * z[5] - shapefct[6] * z[6] - shapefct[7] * z[7]; + + double relax = 1.0; + xinew = xicur + relax * + (f[2] * (j[2] * j[4] - j[1] * j[5]) + f[1] * (j[1] * j[8] - j[2] * j[7]) + + f[0] * (j[5] * j[7] - j[4] * j[8])) * + odet; + etanew = etacur + relax * + (f[2] * (-j[2] * j[3] + j[0] * j[5]) + f[1] * (j[2] * j[6] - j[0] * j[8]) + + f[0] * (j[3] * j[8] - j[5] * j[6])) * + odet; + zetanew = zetacur + relax * + (f[2] * (j[1] * j[3] - j[0] * j[4]) + f[1] * (j[0] * j[7] - j[1] * j[6]) + + f[0] * (j[4] * j[6] - j[3] * j[7])) * + odet; + + xidiff[0] = xinew - xicur; + xidiff[1] = etanew - etacur; + xidiff[2] = zetanew - zetacur; + xicur = xinew; + etacur = etanew; + zetacur = zetanew; + } while (!within_tol(vector_norm(xidiff), isInElemConverged) && ++i < MAX_NR_ITER); + + par_coor[0] = par_coor[1] = par_coor[2] = std::numeric_limits::max(); + double dist = std::numeric_limits::max(); + + if (i < MAX_NR_ITER) { + par_coor[0] = xinew; + par_coor[1] = etanew; + par_coor[2] = zetanew; + + std::array xtmp = {{par_coor[0], par_coor[1], par_coor[2]}}; + dist = parametric_distance(xtmp); + } + + return dist; + } +}; + +} + +#endif \ No newline at end of file diff --git a/packages/stk/stk_doc_tests/stk_search/searchMockMesh.hpp b/packages/stk/stk_doc_tests/stk_search/searchMockMesh.hpp new file mode 100644 index 000000000000..a4a6281e2d95 --- /dev/null +++ b/packages/stk/stk_doc_tests/stk_search/searchMockMesh.hpp @@ -0,0 +1,370 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef STK_DOC_TEST_SEARCH_MOCK_MESH_HPP +#define STK_DOC_TEST_SEARCH_MOCK_MESH_HPP + +#include // for sqrt +#include +#include // for sort, max, min +#include +#include +#include // for size_t +#include // for int64_t, uint64_t +#include +#include +#include // for numeric_limits +#include // for __shared_ptr_ac... +#include +#include // for logic_error +#include // for string, basic_s... +#include // for type_info +#include // for move, pair +#include // for vector, swap + +#include +#include +#include +#include +#include "stk_mesh/base/Entity.hpp" // for Entity +#include "stk_mesh/base/Part.hpp" // for Part +#include "stk_mesh/base/Selector.hpp" // for Selector, opera... +#include "stk_mesh/base/Types.hpp" // for EntityRank, Ent... +#include "stk_search/Box.hpp" // for Box +#include "stk_search/DistanceComparison.hpp" // for stk_distance +#include "stk_search/FilterCoarseSearch.hpp" +#include "stk_search/IdentProc.hpp" // for IdentProc +#include "stk_search/Point.hpp" // for Point +#include "stk_search/SearchInterface.hpp" +#include "stk_search/Sphere.hpp" +#include "stk_search_util/ObjectCoordinates.hpp" // for compute_entity_centroid +#include "stk_topology/topology.hpp" // for topology, topol... +#include "stk_util/parallel/Parallel.hpp" // for parallel_machin... +#include "stk_util/util/ReportHandler.hpp" // for ThrowRequireMsg +#include "searchHex8.hpp" + +namespace doc_test +{ +class Hex8SourceMesh; +class SinglePointMesh; +} // namespace doc_test + +namespace stk +{ +namespace search +{ +template <> +struct MeshTraits { + using Entity = stk::mesh::Entity; + using EntityVec = std::vector; + using EntityKey = stk::mesh::EntityKey; + using EntityProc = stk::search::IdentProc; + using EntityProcVec = std::vector; + using Point = stk::search::Point; + using Box = stk::search::Box; + using BoundingBox = std::pair; +}; +} // namespace search +} // namespace stk + +namespace stk +{ +namespace search +{ +//BEGINMeshTrait +template <> +struct MeshTraits { + using Entity = int; + using EntityVec = std::vector; + using EntityKey = int; + using EntityProc = stk::search::IdentProc; + using EntityProcVec = std::vector; + using Point = stk::search::Point; + using Sphere = stk::search::Sphere; + using BoundingBox = std::pair; +}; +//ENDMeshTrait +} // namespace search +} // namespace stk + +namespace doc_test +{ + +class Hex8SourceMesh : public stk::search::SourceMeshInterface +{ + public: + Hex8SourceMesh(stk::mesh::BulkData& bulkData, + const stk::mesh::PartVector& sendParts, + const stk::ParallelMachine comm, + const double parametricTolerance) + : m_meta(bulkData.mesh_meta_data()), + m_bulk(bulkData), + m_coordinateField(bulkData.mesh_meta_data().coordinate_field()), + m_parts(sendParts), + m_comm(comm), + m_parametricTolerance(parametricTolerance) + { + for (const stk::mesh::Part* part : sendParts) { + STK_ThrowRequireMsg( + part->primary_entity_rank() == stk::topology::ELEM_RANK, "All source parts must be {ELEM_RANK}"); + } + } + + stk::ParallelMachine comm() const { return m_comm; } + + std::string name() const { return "Hex8SourceMesh"; } + + //BEGINSource_bounding_boxes + void bounding_boxes(std::vector& boxes) const + { + Point min_corner, max_corner; + + stk::mesh::Selector selector = stk::mesh::selectUnion(m_parts); + stk::mesh::BucketVector const& buckets = m_bulk.get_buckets(stk::topology::ELEM_RANK, selector); + + for (auto&& ib : buckets) { + stk::mesh::Bucket& b = *ib; + + for (auto elem : b) { + fill_bounding_box(elem, min_corner, max_corner); + + EntityProc theIdent(m_bulk.entity_key(elem), m_bulk.parallel_rank()); + BoundingBox theBox(Box(min_corner, max_corner), theIdent); + boxes.push_back(theBox); + } + } + std::sort(boxes.begin(), boxes.end(), + [](const BoundingBox& a, const BoundingBox& b) { return a.second.id() < b.second.id(); }); + } + //ENDSource_bounding_boxes + + //BEGINSource_find_parametric_coords + void find_parametric_coords(const EntityKey k, + const double* toCoords, + std::vector& parametricCoords, + double& parametricDistance, + bool& isWithinParametricTolerance) const + { + stk::mesh::Entity elem = m_bulk.get_entity(k); + stk::topology topology = m_bulk.bucket(elem).topology(); + STK_ThrowRequireMsg(topology == stk::topology::HEX_8, "Invalid topology: " << topology.name()); + + // load nodal coordinates from element + stk::mesh::Entity const* nodes = m_bulk.begin_nodes(elem); + const auto numNodes = m_bulk.num_nodes(elem); + unsigned nDim = m_meta.spatial_dimension(); + + std::vector transposedElementCoords(nDim * numNodes); + + for (auto ni = 0u; ni < numNodes; ++ni) { + stk::mesh::Entity node = nodes[ni]; + + const double* fromCoords = static_cast(stk::mesh::field_data(*m_coordinateField, node)); + for (unsigned j = 0; j < nDim; ++j) { + const auto offSet = ni + j * numNodes; + transposedElementCoords[offSet] = fromCoords[j]; + } + } + + parametricCoords.assign(3, std::numeric_limits::max()); + parametricDistance = Hex8::is_in_element(transposedElementCoords.data(), toCoords, parametricCoords.data()); + + isWithinParametricTolerance = parametricDistance <= (1 + m_parametricTolerance); + } + //ENDSource_find_parametric_coords + + bool modify_search_outside_parametric_tolerance(const EntityKey k, + const double* toCoords, + std::vector& parametricCoords, + double& geometricDistanceSquared, + bool& isWithinGeometricTolerance) const + { + return false; + } + + //BEGINSource_get_distance_from_nearest_node + double get_distance_from_nearest_node(const EntityKey k, const double* point) const + { + const stk::mesh::Entity e = m_bulk.get_entity(k); + + STK_ThrowRequireMsg( + m_bulk.entity_rank(e) == stk::topology::ELEM_RANK, "Invalid entity rank for object: " << m_bulk.entity_rank(e)); + + double minDistance = std::numeric_limits::max(); + const unsigned nDim = m_meta.spatial_dimension(); + + const stk::mesh::Entity* const nodes = m_bulk.begin_nodes(e); + const int num_nodes = m_bulk.num_nodes(e); + + for (int i = 0; i < num_nodes; ++i) { + double d = 0.0; + double* node_coordinates = static_cast(stk::mesh::field_data(*m_coordinateField, nodes[i])); + + for (unsigned j = 0; j < nDim; ++j) { + const double t = point[j] - node_coordinates[j]; + d += t * t; + } + if (d < minDistance) minDistance = d; + } + + minDistance = std::sqrt(minDistance); + return minDistance; + } + //ENDSource_get_distance_from_nearest_node + + double get_closest_geometric_distance_squared(const EntityKey k, const double* toCoords) const + { + double distance = get_distance_from_nearest_node(k, toCoords); + return distance * distance; + } + + double get_distance_from_centroid(const EntityKey k, const double* toCoords) const + { + double distanceSquared = get_distance_squared_from_centroid(k, toCoords); + return std::sqrt(distanceSquared); + } + + double get_distance_squared_from_centroid(const EntityKey k, const double* toCoords) const + { + std::vector centroidVec; + centroid(k, centroidVec); + + const unsigned nDim = m_meta.spatial_dimension(); + return stk::search::distance_sq(nDim, centroidVec.data(), toCoords); + } + + void centroid(const EntityKey k, std::vector& centroidVec) const + { + const stk::mesh::Entity e = m_bulk.get_entity(k); + stk::search::compute_entity_centroid(e, *m_coordinateField, centroidVec); + } + + const double* coord(const EntityKey k) const + { + centroid(k, m_coordVector); + return m_coordVector.data(); + } + + protected: + stk::mesh::MetaData& m_meta; + stk::mesh::BulkData& m_bulk; + const stk::mesh::FieldBase* m_coordinateField{nullptr}; + + private: + stk::mesh::PartVector m_parts; + const stk::ParallelMachine m_comm; + const double m_parametricTolerance; + + mutable std::vector m_coordVector; + + Hex8SourceMesh(const Hex8SourceMesh&) = delete; + const Hex8SourceMesh& operator()(const Hex8SourceMesh&) = delete; + + void fill_bounding_box( + stk::mesh::Entity elem, stk::search::Point& min_corner, stk::search::Point& max_corner) const + { + const unsigned nDim = m_meta.spatial_dimension(); + + STK_ThrowRequireMsg(m_bulk.is_valid(elem), "Invalid entity: " << m_bulk.entity_key(elem)); + + for (unsigned j = 0; j < nDim; ++j) { + min_corner[j] = std::numeric_limits::max(); + max_corner[j] = -std::numeric_limits::max(); + } + + stk::mesh::Entity const* nodes = m_bulk.begin_nodes(elem); + int numNodes = m_bulk.num_nodes(elem); + for (int ni = 0; ni < numNodes; ++ni) { + stk::mesh::Entity node = nodes[ni]; + + double* coords = static_cast(stk::mesh::field_data(*m_coordinateField, node)); + + for (unsigned j = 0; j < nDim; ++j) { + min_corner[j] = std::min(min_corner[j], coords[j]); + max_corner[j] = std::max(max_corner[j], coords[j]); + } + } + } +}; + + +class SinglePointMesh : public stk::search::DestinationMeshInterface +{ + public: + SinglePointMesh(const stk::ParallelMachine comm, double x, double y, double z, double paramTol, double geomTol) + : m_comm(comm), m_parametricTolerance(paramTol), m_geometricTolerance(geomTol) + { + m_coords[0] = x; + m_coords[1] = y; + m_coords[2] = z; + } + + stk::ParallelMachine comm() const { return m_comm; }; + + std::string name() const { return "SinglePointMesh"; } + + //BEGINDestination_bounding_boxes + void bounding_boxes(std::vector& v) const + { + Point center(m_coords[0], m_coords[1], m_coords[2]); + + EntityKey key = 1; + EntityProc theIdent(key, stk::parallel_machine_rank(m_comm)); + BoundingBox theBox(Sphere(center, m_geometricTolerance), theIdent); + v.push_back(theBox); + } + //ENDDestination_bounding_boxes + + const double* coord(const EntityKey k) const { return m_coords; } + double get_search_tolerance() const { return m_geometricTolerance; } + double get_parametric_tolerance() const { return m_parametricTolerance; } + + void centroid(const EntityKey k, std::vector& centroidVec) const + { + centroidVec.assign(m_coords, m_coords + 3); + } + double get_distance_from_nearest_node(const EntityKey k, const double* toCoords) const + { + return stk::search::distance(3, m_coords, toCoords); + } + + private: + const stk::ParallelMachine m_comm; + double m_coords[3]; + double m_parametricTolerance = 0.00001; + double m_geometricTolerance = 0.1; +}; + +} // namespace doc_test +#endif diff --git a/packages/stk/stk_doc_tests/stk_simd/CMakeLists.txt b/packages/stk/stk_doc_tests/stk_simd/CMakeLists.txt index 5ddae2f50d81..f02817479b27 100644 --- a/packages/stk/stk_doc_tests/stk_simd/CMakeLists.txt +++ b/packages/stk/stk_doc_tests/stk_simd/CMakeLists.txt @@ -1,9 +1,6 @@ ASSERT_DEFINED(PACKAGE_SOURCE_DIR CMAKE_CURRENT_SOURCE_DIR) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_simd) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}/stk_simd) - FILE(GLOB SOURCES *.cpp) TRIBITS_ADD_EXECUTABLE( @@ -14,6 +11,11 @@ TRIBITS_ADD_EXECUTABLE( NOEXEPREFIX ) +target_include_directories(stk_simd_doc_test PRIVATE + $ + $ +) + TRIBITS_ADD_TEST( stk_simd_doc_test ARGS "" diff --git a/packages/stk/stk_doc_tests/stk_topology/CMakeLists.txt b/packages/stk/stk_doc_tests/stk_topology/CMakeLists.txt index 73f85c12b03d..a91bc10a5249 100644 --- a/packages/stk/stk_doc_tests/stk_topology/CMakeLists.txt +++ b/packages/stk/stk_doc_tests/stk_topology/CMakeLists.txt @@ -1,9 +1,6 @@ - ASSERT_DEFINED(PACKAGE_SOURCE_DIR CMAKE_CURRENT_SOURCE_DIR) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_BINARY_DIR}/stk_topology) - FILE(GLOB SOURCES *.cpp) TRIBITS_ADD_EXECUTABLE_AND_TEST( diff --git a/packages/stk/stk_doc_tests/stk_transfer/CMakeLists.txt b/packages/stk/stk_doc_tests/stk_transfer/CMakeLists.txt index b26f8b719695..60b598b20ee6 100644 --- a/packages/stk/stk_doc_tests/stk_transfer/CMakeLists.txt +++ b/packages/stk/stk_doc_tests/stk_transfer/CMakeLists.txt @@ -35,7 +35,6 @@ TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util/stk_util) FILE(GLOB SOURCES *.cpp) diff --git a/packages/stk/stk_doc_tests/stk_util/CMakeLists.txt b/packages/stk/stk_doc_tests/stk_util/CMakeLists.txt index 0614a986f8ca..8a5911392dad 100644 --- a/packages/stk/stk_doc_tests/stk_util/CMakeLists.txt +++ b/packages/stk/stk_doc_tests/stk_util/CMakeLists.txt @@ -2,8 +2,6 @@ ASSERT_DEFINED(PACKAGE_SOURCE_DIR CMAKE_CURRENT_SOURCE_DIR) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_BINARY_DIR}/stk_util) - FILE(GLOB SOURCES *.cpp) IF(NOT TPL_ENABLE_MPI) @@ -12,8 +10,8 @@ IF(NOT TPL_ENABLE_MPI) LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/usingScheduler.cpp") ENDIF() -LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/TimerHowTo.cpp") -LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/TimerHowToParallel.cpp") +# LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/TimerHowTo.cpp") +# LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/TimerHowToParallel.cpp") TRIBITS_ADD_EXECUTABLE_AND_TEST( stk_util_doc_test diff --git a/packages/stk/stk_doc_tests/stk_util/TimerHowToParallel.cpp b/packages/stk/stk_doc_tests/stk_util/TimerHowToParallel.cpp index 2c0b69fa2bf4..6e1b7769370b 100644 --- a/packages/stk/stk_doc_tests/stk_util/TimerHowToParallel.cpp +++ b/packages/stk/stk_doc_tests/stk_util/TimerHowToParallel.cpp @@ -55,9 +55,8 @@ void doWork() TEST(StkDiagTimerHowTo, useTimersInParallel) { - MPI_Comm communicator = MPI_COMM_WORLD; - int numProcs = -1; - MPI_Comm_size(communicator, &numProcs); + stk::ParallelMachine communicator = MPI_COMM_WORLD; + int numProcs = stk::parallel_machine_size(communicator); if(numProcs == 2) { enum {CHILDMASK1 = 1}; @@ -76,8 +75,7 @@ TEST(StkDiagTimerHowTo, useTimersInParallel) bool printTimingsOnlySinceLastPrint = false; stk::diag::printTimersTable(outputStream, rootTimer, stk::diag::METRICS_ALL, printTimingsOnlySinceLastPrint, communicator); - int procId = -1; - MPI_Comm_rank(communicator, &procId); + int procId = stk::parallel_machine_rank(communicator); if(procId == 0) { std::string expectedOutput = " \ diff --git a/packages/stk/stk_doc_tests/stk_util/filenameSubstitution.cpp b/packages/stk/stk_doc_tests/stk_util/filenameSubstitution.cpp index a500d01de13c..a7fe5bcbf259 100644 --- a/packages/stk/stk_doc_tests/stk_util/filenameSubstitution.cpp +++ b/packages/stk/stk_doc_tests/stk_util/filenameSubstitution.cpp @@ -35,6 +35,7 @@ //BEGINFilenameSubstitution #include "gtest/gtest.h" #include "stk_util/environment/EnvData.hpp" // for EnvData +#include "stk_util/environment/Env.hpp" #include "stk_util/environment/FileUtils.hpp" // for filename_substitution #include "stk_util/environment/ParsedOptions.hpp" // for ParsedOptions #include "stk_util/environment/ProgramOptions.hpp" // for get_parsed_options @@ -45,7 +46,8 @@ namespace TEST(StkUtilHowTo, useFilenameSubstitutionWithNoCommandLineOptions) { const std::string default_base_filename = "stdin"; - const std::string numProcsString = "1"; + const int numProcs = stk::parallel_machine_size(sierra::Env::parallel_comm()); + const std::string numProcsString = std::to_string(numProcs); const std::string expected_filename = default_base_filename + "-" + numProcsString + ".e"; std::string file_name = "%B-%P.e"; @@ -65,7 +67,8 @@ TEST(StkUtilHowTo, useFilenameSubstitutionWithFileComingFromCommandLineOptions) const std::string full_filename = "/path/to/" + base_filename + ".g"; setFilenameInCommandLineOptions(full_filename); - const std::string numProcsString = "1"; + const int numProcs = stk::parallel_machine_size(sierra::Env::parallel_comm()); + const std::string numProcsString = std::to_string(numProcs); const std::string expected_filename = base_filename + "-" + numProcsString + ".e"; std::string file_name = "%B-%P.e"; diff --git a/packages/stk/stk_expreval/CMakeLists.txt b/packages/stk/stk_expreval/CMakeLists.txt index c26466f30b1f..cc177efc728e 100644 --- a/packages/stk/stk_expreval/CMakeLists.txt +++ b/packages/stk/stk_expreval/CMakeLists.txt @@ -32,11 +32,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TRIBITS_SUBPACKAGE(ExprEval) +STK_SUBPACKAGE(ExprEval) ADD_SUBDIRECTORY(stk_expreval) -TRIBITS_ADD_TEST_DIRECTORIES(unit_tests) - -TRIBITS_SUBPACKAGE_POSTPROCESS() +STK_SUBPACKAGE_POSTPROCESS() diff --git a/packages/stk/stk_expreval/Jamfile b/packages/stk/stk_expreval/Jamfile index e4a9aa9be042..3985cd90beba 100644 --- a/packages/stk/stk_expreval/Jamfile +++ b/packages/stk/stk_expreval/Jamfile @@ -140,7 +140,7 @@ alias install-exe-targets : ; # exe stk_expreval_utest : - [ glob $(stk_expreval-root)/unit_tests/*.cpp ] + [ glob $(stk_expreval-root)/../stk_unit_tests/stk_expreval/*.cpp ] /sierra/stk_unit_test_utils//stk_unit_main stk_expreval : @sierra-exec-tag diff --git a/packages/stk/stk_expreval/cmake/Dependencies.cmake b/packages/stk/stk_expreval/cmake/Dependencies.cmake index 3f4569ddbb29..0b53fba109fb 100644 --- a/packages/stk/stk_expreval/cmake/Dependencies.cmake +++ b/packages/stk/stk_expreval/cmake/Dependencies.cmake @@ -1,6 +1,6 @@ SET(LIB_REQUIRED_DEP_PACKAGES STKUtil STKMath Kokkos) SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES Gtest STKUnit_test_utils) +SET(TEST_REQUIRED_DEP_PACKAGES) SET(TEST_OPTIONAL_DEP_PACKAGES) SET(LIB_REQUIRED_DEP_TPLS) SET(LIB_OPTIONAL_DEP_TPLS) diff --git a/packages/stk/stk_expreval/stk_expreval/CMakeLists.txt b/packages/stk/stk_expreval/stk_expreval/CMakeLists.txt index 2b7cc86d8211..bbd65810dce9 100644 --- a/packages/stk/stk_expreval/stk_expreval/CMakeLists.txt +++ b/packages/stk/stk_expreval/stk_expreval/CMakeLists.txt @@ -30,52 +30,36 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -# -# A) Package-specific configuration options -# - -# TRIBITS_CONFIGURE_FILE(${PACKAGE_NAME}_config.h) - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - + SET(HEADERS "") SET(SOURCES "") SET(HEADERS_IMPL "") SET(SOURCES_IMPL "") -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../) - -# -# Core headers -# - -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) - FILE(GLOB HEADERS *.hpp) FILE(GLOB SOURCES *.cpp) -IF (TPL_ENABLE_MPI) -ENDIF() - -# -# C) Define the targets for package's library(s) -# - -TRIBITS_ADD_LIBRARY( - stk_expreval - NOINSTALLHEADERS ${HEADERS} ${HEADERS_IMPL} - SOURCES ${SOURCES} ${SOURCES_IMPL} +if(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY(stk_expreval + NOINSTALLHEADERS ${HEADERS} ${HEADERS_IMPL} + SOURCES ${SOURCES} ${SOURCES_IMPL} ) +else() + add_library(stk_expreval ${SOURCES} ${SOURCES_IMPL}) + target_link_libraries(stk_expreval PUBLIC stk_util_ngp) + target_link_libraries(stk_expreval PUBLIC stk_util_util) + target_link_libraries(stk_expreval PUBLIC Kokkos::kokkos) +endif() + +target_include_directories(stk_expreval PUBLIC + $ + $ +) INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_expreval) + +if(NOT HAVE_STK_Trilinos) + INSTALL(TARGETS stk_expreval DESTINATION ${STK_INSTALL_LIBDIR}) +endif() diff --git a/packages/stk/stk_expreval/unit_tests/CMakeLists.txt b/packages/stk/stk_expreval/unit_tests/CMakeLists.txt deleted file mode 100644 index e1db842d7b18..000000000000 --- a/packages/stk/stk_expreval/unit_tests/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ - - -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../stk_util) - -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/) - -FILE(GLOB SOURCES *.cpp) - -TRIBITS_ADD_EXECUTABLE_AND_TEST( - stk_expreval_unit_tests - SOURCES ${SOURCES} - TESTONLYLIBS stk_unit_main - ARGS "" - COMM serial mpi - NUM_MPI_PROCS 1 - ) - diff --git a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_no_stk_io_using_cmake b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_no_stk_io_using_cmake index faf6fefb8f6c..39c3a1164915 100755 --- a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_no_stk_io_using_cmake +++ b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_no_stk_io_using_cmake @@ -9,6 +9,7 @@ sierra_proj=${SIERRA_PROJ:-${PWD}} output_dir=${OUTPUT_DIR:-${PWD}/../stk-cmake-testing} trilinos_dir=${output_dir}/Trilinos cuda_on_or_off=${CUDA:-OFF} +clear_cache=${CLEAR_CACHE:-ON} build_type=${CMAKE_BUILD_TYPE:-release} date_suffix=`date +%F_%H-%M-%S` @@ -29,8 +30,10 @@ trilinos_install_dir=${output_dir}/trilinos_install_${build_type}_gcc exe rm -rf $trilinos_install_dir stk_build_dir=${output_dir}/stk_build_no_stk_io_${build_type}_gcc -exe rm -rf $stk_build_dir -exe mkdir -p $stk_build_dir +if [ "${clear_cache}" == "ON" ] ; then + exe rm -rf $stk_build_dir + exe mkdir -p $stk_build_dir +fi stk_cmake_testing_source_dir=${sierra_proj}/stk/stk_integration_tests/cmake_install_test @@ -63,6 +66,7 @@ exe "TRILINOS_DIR=${trilinos_dir} \ TRILINOS_INSTALL_DIR=${trilinos_install_dir} \ CMAKE_BUILD_TYPE=${build_type} \ CUDA=${cuda_on_or_off} \ + CLEAR_CACHE=${clear_cache} \ ./run_cmake_stk_no_stk_io >& ${stk_config_log}" if [ $? -ne 0 ] ; then echo "!! error in stk/trilinos config, check output in ${stk_config_log} !!"; diff --git a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_serial_using_cmake b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_serial_using_cmake index 6e253fc74621..79cee907ad46 100755 --- a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_serial_using_cmake +++ b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_serial_using_cmake @@ -9,6 +9,7 @@ sierra_proj=${SIERRA_PROJ:-${PWD}} output_dir=${OUTPUT_DIR:-${PWD}/../stk-cmake-testing} trilinos_dir=${output_dir}/Trilinos build_type=${CMAKE_BUILD_TYPE:-release} +clear_cache=${CLEAR_CACHE:-ON} date_suffix=`date +%F_%H-%M-%S` if [ ! -d ${output_dir} ] ; then @@ -28,8 +29,10 @@ trilinos_install_dir=${output_dir}/trilinos_install_${build_type}_gcc exe rm -rf $trilinos_install_dir stk_build_dir=${output_dir}/stk_build_serial_${build_type}_gcc -exe rm -rf $stk_build_dir -exe mkdir -p $stk_build_dir +if [ "${clear_cache}" == "ON" ] ; then + exe rm -rf $stk_build_dir + exe mkdir -p $stk_build_dir +fi stk_cmake_testing_source_dir=${sierra_proj}/stk/stk_integration_tests/cmake_install_test @@ -55,7 +58,12 @@ exe cp ${stk_cmake_testing_source_dir}/run_cmake_stk_serial ${stk_build_dir} exe cd ${stk_build_dir} exe source ${stk_cmake_testing_source_dir}/load_gcc_modules printf "Configuring trilinos/stk (running cmake)...\n"; -exe "TRILINOS_DIR=${trilinos_dir} TRILINOS_INSTALL_DIR=${trilinos_install_dir} CMAKE_BUILD_TYPE=${build_type} CMAKE_CXX_FLAGS='-Wno-pragmas -Wno-unknown-pragmas' ./run_cmake_stk_serial >& ${stk_config_log}" +exe "TRILINOS_DIR=${trilinos_dir} \ + TRILINOS_INSTALL_DIR=${trilinos_install_dir} \ + CMAKE_BUILD_TYPE=${build_type} \ + CLEAR_CACHE=${clear_cache} \ + CMAKE_CXX_FLAGS='-Wno-pragmas -Wno-unknown-pragmas' \ + ./run_cmake_stk_serial >& ${stk_config_log}" if [ $? -ne 0 ] ; then echo "!! error in stk/trilinos config, check output in ${stk_config_log} !!"; exit 1; diff --git a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_standalone_serial_using_cmake b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_standalone_serial_using_cmake new file mode 100755 index 000000000000..eda9afa4f852 --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_standalone_serial_using_cmake @@ -0,0 +1,78 @@ +#!/bin/bash + +exe() { + stdbuf -o0 -e0 echo "% $@" ; + eval "$@" ; +} + +sierra_proj=${SIERRA_PROJ:-${PWD}} +output_dir=${OUTPUT_DIR:-/fgs/$USER/stk-standalone-cmake-testing} +build_type=${CMAKE_BUILD_TYPE:-release} +clear_cache=${CLEAR_CACHE:-ON} +date_suffix=`date +%F_%H-%M-%S` + +if [ ! -d ${output_dir} ] ; then + exe mkdir -p ${output_dir}; +fi + +stk_build_dir=${output_dir}/stk_standalone_build_serial_${build_type}_gcc +if [ "${clear_cache}" == "ON" ] ; then + exe rm -rf $stk_build_dir +fi + +exe mkdir -p $stk_build_dir + +stk_cmake_testing_source_dir=${sierra_proj}/stk/stk_integration_tests/cmake_install_test + +printf "\nUsing sierra project: ${sierra_proj}\n"; +printf "Using build-type: ${build_type}\n"; +printf "Putting output and logs here: ${output_dir}\n"; + +exe cd $sierra_proj + +stk_config_log=${output_dir}/stk-standalone-serial-config.out.$date_suffix +stk_make_log=${output_dir}/stk-standalone-serial-make.out.$date_suffix +stk_install_log=${output_dir}/stk-standalone-serial-install.out.$date_suffix +stk_ctest_log=${output_dir}/stk-standalone-serial-ctest.out.$date_suffix + +exe cp ${stk_cmake_testing_source_dir}/run_cmake_stk_standalone_serial ${stk_build_dir} +exe cd ${stk_build_dir} +exe source ${stk_cmake_testing_source_dir}/load_gcc_modules_no_boost + +printf "Configuring stk (running cmake)...\n"; +exe "STK_INSTALL_DIR=${output_dir}/install_stk_serial \ + BUILD_DIR=${stk_build_dir} \ + CMAKE_BUILD_TYPE=${build_type} \ + CLEAR_CACHE=${clear_cache} \ + CMAKE_CXX_FLAGS='-Wno-pragmas -Wno-unknown-pragmas' \ + ./run_cmake_stk_standalone_serial >& ${stk_config_log}" + +if [ $? -ne 0 ] ; then + echo "!! error in stk config, check output in ${stk_config_log} !!"; + exit 1; +fi + +printf "Now building stk using make...\n"; +exe "make VERBOSE=1 -j8 >& ${stk_make_log}"; +if [ $? -ne 0 ] ; then + echo "!! error in make, check output in ${stk_make_log} !!"; + exit 1; +fi + +exe "make install >& ${stk_install_log}"; + +if [ $? -ne 0 ] ; then + printf "!! error installing, check output in ${stk_install_log}\n"; + exit 1; +fi + +exe "ctest >& ${stk_ctest_log}"; + +if [ $? -ne 0 ] ; then + printf "!! error running ctest, check output in ${stk_ctest_log}\n"; + exit 1; +fi + +echo "all done!!"; +exit 0; + diff --git a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_standalone_using_cmake b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_standalone_using_cmake new file mode 100755 index 000000000000..4391088bf4e0 --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_standalone_using_cmake @@ -0,0 +1,78 @@ +#!/bin/bash + +exe() { + stdbuf -o0 -e0 echo "% $@" ; + eval "$@" ; +} + +sierra_proj=${SIERRA_PROJ:-${PWD}} +output_dir=${OUTPUT_DIR:-/fgs/$USER/stk-standalone-cmake-testing} +build_type=${CMAKE_BUILD_TYPE:-release} +clear_cache=${CLEAR_CACHE:-ON} +date_suffix=`date +%F_%H-%M-%S` + +if [ ! -d ${output_dir} ] ; then + exe mkdir -p ${output_dir}; +fi + +stk_build_dir=${output_dir}/stk_standalone_build_${build_type}_gcc +if [ "${clear_cache}" == "ON" ] ; then + exe rm -rf $stk_build_dir +fi + +exe mkdir -p $stk_build_dir + +stk_cmake_testing_source_dir=${sierra_proj}/stk/stk_integration_tests/cmake_install_test + +printf "\nUsing sierra project: ${sierra_proj}\n"; +printf "Using build-type: ${build_type}\n"; +printf "Putting output and logs here: ${output_dir}\n"; + +exe cd $sierra_proj + +stk_config_log=${output_dir}/stk-standalone-config.out.$date_suffix +stk_make_log=${output_dir}/stk-standalone-make.out.$date_suffix +stk_install_log=${output_dir}/stk-standalone-install.out.$date_suffix +stk_ctest_log=${output_dir}/stk-standalone-ctest.out.$date_suffix + +exe cp ${stk_cmake_testing_source_dir}/run_cmake_stk_standalone ${stk_build_dir} +exe cd ${stk_build_dir} +exe source ${stk_cmake_testing_source_dir}/load_gcc_modules_no_boost + +printf "Configuring stk (running cmake)...\n"; +exe "STK_INSTALL_DIR=${output_dir}/install_stk \ + BUILD_DIR=${stk_build_dir} \ + CMAKE_BUILD_TYPE=${build_type} \ + CLEAR_CACHE=${clear_cache} \ + CMAKE_CXX_FLAGS='-Wno-pragmas -Wno-unknown-pragmas' \ + ./run_cmake_stk_standalone >& ${stk_config_log}" + +if [ $? -ne 0 ] ; then + echo "!! error in stk config, check output in ${stk_config_log} !!"; + exit 1; +fi + +printf "Now building stk using make...\n"; +exe "make VERBOSE=1 -j8 >& ${stk_make_log}"; +if [ $? -ne 0 ] ; then + echo "!! error in make, check output in ${stk_make_log} !!"; + exit 1; +fi + +exe "make install >& ${stk_install_log}"; + +if [ $? -ne 0 ] ; then + printf "!! error installing, check output in ${stk_install_log}\n"; + exit 1; +fi + +exe "ctest >& ${stk_ctest_log}"; + +if [ $? -ne 0 ] ; then + printf "!! error running ctest, check output in ${stk_ctest_log}\n"; + exit 1; +fi + +echo "all done!!"; +exit 0; + diff --git a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_transfer_no_middle_mesh_using_cmake b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_transfer_no_middle_mesh_using_cmake index ee755e9a79f7..d2bac8773024 100755 --- a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_transfer_no_middle_mesh_using_cmake +++ b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_transfer_no_middle_mesh_using_cmake @@ -39,10 +39,10 @@ printf "Putting output and logs here: ${output_dir}\n"; exe cd $sierra_proj -stk_config_log=${output_dir}/stk-no-stk-mesh-config.out.$date_suffix -stk_make_log=${output_dir}/stk-no-stk-mesh-make.out.$date_suffix -stk_install_log=${output_dir}/stk-no-stk-mesh-install.out.$date_suffix -stk_ctest_log=${output_dir}/stk-no-stk-mesh-ctest.out.$date_suffix +stk_config_log=${output_dir}/stk-no-middle-mesh-config.out.$date_suffix +stk_make_log=${output_dir}/stk-no-middle-mesh-make.out.$date_suffix +stk_install_log=${output_dir}/stk-no-middle-mesh-install.out.$date_suffix +stk_ctest_log=${output_dir}/stk-no-middle-mesh-ctest.out.$date_suffix if [ -d ${trilinos_dir}/packages/stk ] ; then exe rm -rf ${trilinos_dir}/packages/stk; diff --git a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_user_facing b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_user_facing index a0a0becf17aa..dca1fe7944d2 100755 --- a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_user_facing +++ b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_user_facing @@ -8,6 +8,7 @@ exe() { sierra_proj=${SIERRA_PROJ:-${PWD}} output_dir=${OUTPUT_DIR:-${PWD}/../stk-cmake-testing} trilinos_dir=${output_dir}/Trilinos +clear_cache=${CLEAR_CACHE:-ON} build_type=${CMAKE_BUILD_TYPE:-release} date_suffix=`date +%F_%H-%M-%S` @@ -28,8 +29,10 @@ trilinos_install_dir=${output_dir}/trilinos_install_${build_type}_gcc exe rm -rf $trilinos_install_dir stk_build_dir=${output_dir}/stk_build_user_facing_${build_type}_gcc -exe rm -rf $stk_build_dir -exe mkdir -p $stk_build_dir +if [ "${clear_cache}" == "ON" ] ; then + exe rm -rf $stk_build_dir + exe mkdir -p $stk_build_dir +fi stk_cmake_testing_source_dir=${sierra_proj}/stk/stk_integration_tests/cmake_install_test @@ -50,6 +53,7 @@ stk_install_log=${output_dir}/stk-user-facing-install.out.$date_suffix stk_ctest_log=${output_dir}/stk-user-facing-ctest.out.$date_suffix stk_app_config_log=${output_dir}/stk-test-app-user-facing-config.out.$date_suffix stk_app_make_log=${output_dir}/stk-test-app-user-facing-make.out.$date_suffix +stk_app_run_log=${output_dir}/stk-test-app-user-facing-run.out.$date_suffix if [ -d ${trilinos_dir}/packages/stk ] ; then exe rm -rf ${trilinos_dir}/packages/stk; @@ -64,7 +68,12 @@ exe cd ${stk_build_dir} exe source ${stk_cmake_testing_source_dir}/load_gcc_modules printf "Configuring trilinos/stk (running cmake)...\n"; -exe "TRILINOS_DIR=${trilinos_dir} TRILINOS_INSTALL_DIR=${trilinos_install_dir} CMAKE_BUILD_TYPE=${build_type} ./run_cmake_stk_user_facing >& ${stk_config_log}" +exe "TRILINOS_DIR=${trilinos_dir} \ + TRILINOS_INSTALL_DIR=${trilinos_install_dir} \ + CMAKE_BUILD_TYPE=${build_type} \ + CLEAR_CACHE=${clear_cache} \ + ./run_cmake_stk_user_facing >& ${stk_config_log}" + if [ $? -ne 0 ] ; then echo "!! error in stk/trilinos config, check output in ${stk_config_log} !!"; exit 1; @@ -102,23 +111,25 @@ if [ $? -ne 0 ] ; then exit 1; fi -exe "make >& ${stk_app_make_log}"; +exe "make VERBOSE=1 >& ${stk_app_make_log}"; if [ $? -ne 0 ] ; then printf "!! error building stk-test-app, check output in ${stk_app_make_log}\n"; exit 1; fi -#Follow-on work: run test_stk_app, improve testing of user-facing stk installation. -#But not by replicating the sierra testrun system! -#The fact that the stk-test-app built is at least a partial check that the installation -#is correct, but how do we know the stk_balance executable was built/installed? -#What about all the headers/libraries that stk-test-app doesn't use? - if [ ! -x ${trilinos_install_dir}/bin/stk_balance.exe ] ; then print "!! error, stk_balance executable not installed.\n"; exit 1; fi +printf "\nRunning the stk test-app...\n"; + +exe "mpirun --np 1 ${output_dir}/stk_test_app_build/test_stk_app >& ${stk_app_run_log}"; +if [ $? -ne 0 ] ; then + printf "!! error running the stk test-app, check output in ${stk_app_run_log}\n"; + exit 1; +fi + echo "all done, SUCCESS!"; exit 0; diff --git a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_using_cmake b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_using_cmake index 12a7e9b0a692..06c4c46b2baf 100755 --- a/packages/stk/stk_integration_tests/cmake_install_test/build_stk_using_cmake +++ b/packages/stk/stk_integration_tests/cmake_install_test/build_stk_using_cmake @@ -17,18 +17,13 @@ clear_cache=${CLEAR_CACHE:-ON} build_type=${CMAKE_BUILD_TYPE:-release} date_suffix=`date +%F_%H-%M-%S` + if [ ! -d ${output_dir} ] ; then exe mkdir -p ${output_dir}; fi -if [ ! -d ${trilinos_dir} ] ; then - exe git clone -b develop https://github.com/trilinos/Trilinos.git ${trilinos_dir} -else - exe cd "${trilinos_dir}" - exe git checkout develop - exe git reset --hard origin/develop - exe git pull -fi +stk_cmake_testing_source_dir=${sierra_proj}/stk/stk_integration_tests/cmake_install_test +${stk_cmake_testing_source_dir}/create_workspace.sh ${trilinos_dir} ${sierra_proj} trilinos_install_dir=${output_dir}/trilinos_install_${build_type}_gcc exe rm -rf $trilinos_install_dir @@ -39,15 +34,11 @@ if [ "${clear_cache}" == "ON" ] ; then exe mkdir -p $stk_build_dir fi -stk_cmake_testing_source_dir=${sierra_proj}/stk/stk_integration_tests/cmake_install_test - printf "\nUsing sierra project: ${sierra_proj}\n"; printf "Using build-type: ${build_type}\n"; printf "CUDA: ${cuda_on_or_off}\n"; printf "Putting output and logs here: ${output_dir}\n"; -exe cd $sierra_proj - bake_log=${output_dir}/bake-stk_balance.out.$date_suffix stk_config_log=${output_dir}/stk-config.out.$date_suffix stk_make_log=${output_dir}/stk-make.out.$date_suffix @@ -56,13 +47,6 @@ stk_ctest_log=${output_dir}/stk-ctest.out.$date_suffix stk_app_config_log=${output_dir}/stk-test-app-config.out.$date_suffix stk_app_make_log=${output_dir}/stk-test-app-make.out.$date_suffix -if [ -d ${trilinos_dir}/packages/stk ] ; then - exe rm -rf ${trilinos_dir}/packages/stk; -fi -if [ ! -L ${trilinos_dir}/packages/stk ] ; then - exe ln -s ${sierra_proj}/stk ${trilinos_dir}/packages -fi - exe cp ${stk_cmake_testing_source_dir}/run_cmake_stk ${stk_build_dir} exe cd ${stk_build_dir} @@ -75,11 +59,23 @@ exe "TRILINOS_DIR=${trilinos_dir} \ CUDA=${cuda_on_or_off} \ CLEAR_CACHE=${clear_cache} \ ./run_cmake_stk >& ${stk_config_log}" +if [ $? -ne 0 ] ; then + echo "!! error in stk/trilinos config, check output in ${stk_config_log} !!"; + exit 1; +fi printf "Now building trilinos/stk using make...\n"; exe "make VERBOSE=1 -j8 >& ${stk_make_log}"; +if [ $? -ne 0 ] ; then + echo "!! error in make, check output in ${stk_make_log} !!"; + exit 1; +fi + exe "ctest -j8 >& ${stk_ctest_log}"; +if [ $? -ne 0 ] ; then + printf "!! error running ctest, check output in ${stk_ctest_log}\n"; + exit 1; +fi echo "all done, SUCCESS!"; exit 0; - diff --git a/packages/stk/stk_integration_tests/cmake_install_test/create_workspace.sh b/packages/stk/stk_integration_tests/cmake_install_test/create_workspace.sh new file mode 100755 index 000000000000..bc1bc6926b22 --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/create_workspace.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +exe() { + stdbuf -o0 -e0 echo "% $@" ; + eval "$@" ; + if [ $? -ne 0 ] ; then + echo "'$@' failed."; + exit 1; + fi +} + +trilinos_dir=${1:?"Must pass directory to clone Trilinos into!"} +sierra_proj=${2:?"Must pass path to sierra!"} + +if [ ! -d ${trilinos_dir} ] ; then + exe git clone -b develop https://github.com/trilinos/Trilinos.git ${trilinos_dir} +else + exe cd "${trilinos_dir}" + exe git checkout develop + exe git reset --hard origin/develop + exe git pull +fi + +if [ -d ${trilinos_dir}/packages/stk ] ; then + exe rm -rf ${trilinos_dir}/packages/stk; +fi +if [ ! -L ${trilinos_dir}/packages/stk ] ; then + exe ln -s ${sierra_proj}/stk ${trilinos_dir}/packages +fi + +# we do not want to pay attention to the Trilinos configuration +rm -rf ${trilinos_dir}/CTestConfig.cmake diff --git a/packages/stk/stk_integration_tests/cmake_install_test/load_gcc_modules b/packages/stk/stk_integration_tests/cmake_install_test/load_gcc_modules index 656de831f022..97f25517f81d 100644 --- a/packages/stk/stk_integration_tests/cmake_install_test/load_gcc_modules +++ b/packages/stk/stk_integration_tests/cmake_install_test/load_gcc_modules @@ -2,12 +2,15 @@ source /projects/sems/modulefiles/utils/sems-modules-init.sh -module load sems-cmake/3.24.3 +#module load sems-cmake/3.24.3 +module load cde/v3/cmake/3.23.1 module load sems-gcc/8.3.0 module load sems-metis/5.1.0 export BLAS_LIBRARIES=${BLAS_LIBRARIES:-/usr/lib64/libblas.so} export LAPACK_LIBRARIES=${LAPACK_LIBRARIES:-/usr/lib64/liblapack.so} +export TPL_BLAS_LIBRARIES=${BLAS_LIBRARIES:-/usr/lib64/libblas.so} +export TPL_LAPACK_LIBRARIES=${LAPACK_LIBRARIES:-/usr/lib64/liblapack.so} module load sems-openmpi/1.10.7 module load sems-hdf5/1.10.7 @@ -16,9 +19,7 @@ module load sems-parmetis/4.0.3 module load sems-parallel-netcdf/1.12.1 - ### Corresponding CDE v3 modules -- as of 2/6/23 these gave link errors -#module load cde/v3/cmake/3.23.1 #module load cde/v3/gcc/10.3.0 #module load cde/v3/openmpi/4.1.2-gcc-10.3.0 #module load cde/v3/netlib-lapack/3.10.1-gcc-10.3.0 diff --git a/packages/stk/stk_integration_tests/cmake_install_test/load_gcc_modules_no_boost b/packages/stk/stk_integration_tests/cmake_install_test/load_gcc_modules_no_boost new file mode 100644 index 000000000000..76c40903fb63 --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/load_gcc_modules_no_boost @@ -0,0 +1,30 @@ +#!/bin/bash + +source /projects/sems/modulefiles/utils/sems-modules-init.sh + +module load cde/v3/cmake/3.23.1 +module load sems-gcc/8.3.0 +module load sems-metis/5.1.0 + +export BLAS_LIBRARIES=${BLAS_LIBRARIES:-/usr/lib64/libblas.so} +export LAPACK_LIBRARIES=${LAPACK_LIBRARIES:-/usr/lib64/liblapack.so} + +module load sems-openmpi/1.10.7 +module load sems-hdf5/1.10.7 +module load sems-netcdf-c/4.7.3 +module load sems-parmetis/4.0.3 +module load sems-parallel-netcdf/1.12.1 + + + +### Corresponding CDE v3 modules -- as of 2/6/23 these gave link errors +#module load cde/v3/cmake/3.23.1 +#module load cde/v3/gcc/10.3.0 +#module load cde/v3/openmpi/4.1.2-gcc-10.3.0 +#module load cde/v3/netlib-lapack/3.10.1-gcc-10.3.0 +#module load cde/v3/hdf5/1.10.6-gcc-10.3.0-openmpi-4.1.2 +#module load cde/v3/netcdf-c/4.8.1-gcc-10.3.0-openmpi-4.1.2 +#module load cde/v3/parallel-netcdf/1.12.2-gcc-10.3.0-openmpi-4.1.2 +#module load cde/v3/metis/5.1.0-gcc-10.3.0 +#module load cde/v3/parmetis/4.0.3-gcc-10.3.0-openmpi-4.1.2 + diff --git a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk index 97c1b4a471f8..35b276b30a43 100755 --- a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk +++ b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk @@ -82,9 +82,9 @@ cmake \ -DCMAKE_CXX_FLAGS:STRING="-D${fortran_macro} ${cmake_cxx_flags} -Werror=dangling-else" \ -DSTK_ENABLE_TESTS:BOOL=ON \ -DTrilinos_ENABLE_STK:BOOL=ON \ --DTrilinos_ENABLE_STKMiddle_mesh:BOOL=OFF \ +-DTrilinos_ENABLE_STKMiddle_mesh:BOOL=ON \ -DTrilinos_ENABLE_Gtest:BOOL=ON \ --DTPL_ENABLE_CDT:BOOL=OFF \ +-DTPL_ENABLE_CDT:BOOL=ON \ -DCDT_INCLUDE_DIRS="/fgs/william/CDT/install/include" \ -DCDT_ROOT="/fgs/william/CDT/install" \ -DTrilinos_ENABLE_SEACASExodus:BOOL=ON \ diff --git a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_serial b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_serial index 7beeb76ad18d..ea914b9cd42b 100755 --- a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_serial +++ b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_serial @@ -5,6 +5,7 @@ build_dir=${BUILD_DIR:-${PWD}} build_type=${CMAKE_BUILD_TYPE:-release} trilinos_install_dir=${TRILINOS_INSTALL_DIR:-${PWD}/../trilinos_install_dir} cmake_cxx_flags=${CMAKE_CXX_FLAGS} +clear_cache=${CLEAR_CACHE:-ON} printf "\nTRILINOS_DIR=${trilinos_src_dir}\n"; printf "BUILD_DIR=${build_dir}\n"; @@ -13,9 +14,15 @@ printf "CMAKE_CXX_FLAGS=${cmake_cxx_flags}\n"; printf "TRILINOS_INSTALL_DIR=${trilinos_install_dir}\n"; printf "\nTo change these vars, set as env vars or pass to this script like 'VAR=value run_cmake_stk_serial'\n\n"; +mkdir -p $trilinos_install_dir +mkdir -p $build_dir + cd ${build_dir} -mkdir -p $trilinos_install_dir +if [ "${clear_cache}" == "ON" ] ; then +# Cleanup old cache before we configure + rm -rf CMakeFiles CMakeCache.txt +fi rm -rf CMakeFiles CMakeCache.txt diff --git a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone new file mode 100755 index 000000000000..fd6f31085b7a --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone @@ -0,0 +1,45 @@ +#!/bin/bash + +build_dir=${BUILD_DIR:-${PWD}} +build_type=${CMAKE_BUILD_TYPE:-release} +stk_src_dir=${STK_SRC_DIR:-/fgs/william/code_votd/stk} +stk_install_dir=${STK_INSTALL_DIR:-${PWD}/../stk_install_dir} +cmake_cxx_flags=${CMAKE_CXX_FLAGS} +clear_cache=${CLEAR_CACHE:-ON} + +printf "BUILD_DIR=${build_dir}\n"; +printf "CMAKE_BUILD_TYPE=${build_type}\n"; +printf "CMAKE_CXX_FLAGS=${cmake_cxx_flags}\n"; +printf "STK_INSTALL_DIR=${stk_install_dir}\n"; +printf "\nTo change these vars, set as env vars or pass to this script like 'VAR=value run_pure_cmake_stk'\n\n"; + +# before running this script, load these modules: +# module load cde/v3/cmake/3.23.1 +# module load cde/v3/gcc/10.3.0 +# module load cde/v3/openmpi/4.1.2-gcc-10.3.0 + +mkdir -p $build_dir + +cd ${build_dir} + +if [ "${clear_cache}" == "ON" ] ; then + rm -rf stk_* STK_Trilinos_config.h Makefile cmake_install.cmake C* +fi + +cmake \ +-DCMAKE_INSTALL_PREFIX=${stk_install_dir} \ +-DCMAKE_BUILD_TYPE=${build_type} \ +-DCMAKE_CXX_STANDARD:STRING=17 \ +-DCMAKE_CXX_STANDARD_REQUIRED=ON \ +-DCMAKE_CXX_EXTENSIONS=OFF \ +-DCMAKE_CXX_COMPILER=${OMPI_CXX} \ +-DCMAKE_CXX_FLAGS:STRING="${cmake_cxx_flags}" \ +-DSTK_ENABLE_ALL:BOOL=ON \ +-DSTK_ENABLE_STKMiddle_mesh=OFF \ +-DSTK_ENABLE_MPI:BOOL=ON \ +-DSTK_ENABLE_TESTS:BOOL=ON \ +-DKokkos_DIR=/fgs/william/kokkos/install/lib64/cmake/Kokkos \ +-DKokkosKernels_DIR=/fgs/william/kokkos-kernels/install/lib64/cmake/KokkosKernels \ +-DGTest_DIR=/fgs/william/googletest/install/lib64/cmake/GTest \ +${stk_src_dir}/ + diff --git a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone_serial b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone_serial new file mode 100755 index 000000000000..0f2d688dbf73 --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_standalone_serial @@ -0,0 +1,42 @@ +#!/bin/bash + +build_dir=${BUILD_DIR:-${PWD}} +build_type=${CMAKE_BUILD_TYPE:-release} +stk_src_dir=${STK_SRC_DIR:-/fgs/william/code_votd/stk} +stk_install_dir=${STK_INSTALL_DIR:-${PWD}/../stk_install_dir} +cmake_cxx_flags=${CMAKE_CXX_FLAGS} +clear_cache=${CLEAR_CACHE:-ON} + +printf "BUILD_DIR=${build_dir}\n"; +printf "CMAKE_BUILD_TYPE=${build_type}\n"; +printf "CMAKE_CXX_FLAGS=${cmake_cxx_flags}\n"; +printf "STK_INSTALL_DIR=${stk_install_dir}\n"; +printf "\nTo change these vars, set as env vars or pass to this script like 'VAR=value run_pure_cmake_stk'\n\n"; + +# before running this script, load these modules: +# module load cde/v3/cmake/3.23.1 +# module load cde/v3/gcc/10.3.0 +# module load cde/v3/openmpi/4.1.2-gcc-10.3.0 + +mkdir -p $build_dir + +cd ${build_dir} + +if [ "${clear_cache}" == "ON" ] ; then + rm -rf stk_* STK_Trilinos_config.h Makefile cmake_install.cmake C* +fi + +cmake \ +-DCMAKE_INSTALL_PREFIX=${stk_install_dir} \ +-DCMAKE_BUILD_TYPE=${build_type} \ +-DCMAKE_CXX_STANDARD:STRING=17 \ +-DCMAKE_CXX_COMPILER=${OMPI_CXX} \ +-DCMAKE_CXX_FLAGS:STRING="${cmake_cxx_flags}" \ +-DSTK_ENABLE_ALL=ON \ +-DSTK_ENABLE_MPI:BOOL=OFF \ +-DSTK_ENABLE_STKMiddle_mesh=OFF \ +-DSTK_ENABLE_TESTS:BOOL=ON \ +-DKokkos_DIR=/fgs/william/kokkos/install/lib64/cmake/Kokkos \ +-DGTest_DIR=/fgs/william/googletest/install/lib64/cmake/GTest \ +${stk_src_dir}/ + diff --git a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_user_facing b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_user_facing index 74c756b43048..ea9b51a28e41 100755 --- a/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_user_facing +++ b/packages/stk/stk_integration_tests/cmake_install_test/run_cmake_stk_user_facing @@ -8,6 +8,7 @@ fortran_macro=${FORTRAN_MACRO:-FORTRAN_ONE_UNDERSCORE} cmake_cxx_flags=${CMAKE_CXX_FLAGS} cmake_exe_linker_flags=${CMAKE_EXE_LINKER_FLAGS} cuda_on_or_off=${CUDA:-OFF} +clear_cache=${CLEAR_CACHE:-ON} printf "\nTRILINOS_DIR=${trilinos_src_dir}\n"; printf "BUILD_DIR=${build_dir}\n"; @@ -24,8 +25,10 @@ mkdir -p $build_dir cd ${build_dir} +if [ "${clear_cache}" == "ON" ] ; then # Cleanup old cache before we configure -rm -rf CMakeFiles CMakeCache.txt + rm -rf CMakeFiles CMakeCache.txt +fi cmake \ -DCMAKE_INSTALL_PREFIX=$trilinos_install_dir \ diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk-integrate.cmake b/packages/stk/stk_integration_tests/cmake_install_test/stk-integrate.cmake new file mode 100644 index 000000000000..0547459e0ef1 --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk-integrate.cmake @@ -0,0 +1,51 @@ +message(STATUS "Integration build of STK in Trilinos") + +if(NOT DEFINED CONFIGURE_ARGS) + set(CONFIGURE_ARGS "") +endif() + +set($ENV{LC_MESSAGES} "en_EN") +message(STATUS ${DASHBOARD}) + +if(NOT DEFINED(CUDA)) + set(CUDA OFF) +endif() +if(NOT DEFINED(CLEAR_CACHE)) + set(CLEAR_CACHE ON) +endif() +if(NOT DEFINED CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE release) +endif() +if(NOT DEFINED NUM_JOBS) + set(NUM_JOBS 12) +endif() +if(NOT DEFINED CDASH_TRACK) + set(CDASH_TRACK "foobar") +endif() + +set(STK_SCRIPT_DIR "${SIERRA_PROJ}/stk/stk_integration_tests/cmake_install_test") +set(TRILINOS_DIR "${OUTPUT_DIR}/Trilinos") + +set(CTEST_BINARY_DIRECTORY "${OUTPUT_DIR}/build") +set(CTEST_SOURCE_DIRECTORY "${TRILINOS_DIR}") + + +set(CTEST_UPDATE_COMMAND "${STK_SCRIPT_DIR}/create_workspace.sh") +set(CTEST_UPDATE_OPTIONS "${TRILINOS_DIR} ${SIERRA_PROJ}") +set(CTEST_CHECKOUT_COMMAND "${CTEST_UPDATE_COMMAND} ${CTEST_UPDATE_OPTIONS}") + + +set(CTEST_CONFIGURE_COMMAND "${STK_SCRIPT_DIR}/run_cmake_stk") +set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +set(CTEST_TEST_TIMEOUT "600") + +message(STATUS " -- Start dashboard --") +ctest_start("Experimental" GROUP ${CDASH_TRACK}) +ctest_configure() +ctest_build(PARALLEL_LEVEL ${NUM_JOBS} RETURN_VALUE ESTAT) +ctest_test(TEST_LOAD ${NUM_JOBS}) + +message(STATUS " -- Finished --") +if(NOT ${ESTAT} EQUAL 0) + message(FATAL_ERROR "Testing failed") +endif() diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/CMakeLists.txt b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/CMakeLists.txt index 2732d4fbb185..32d9055d4ecd 100644 --- a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/CMakeLists.txt +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/CMakeLists.txt @@ -1,5 +1,7 @@ target_sources(test_stk_lib PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/test_stk_coupling.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test_stk_search.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_stk_simd.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_stk_io.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_stk_tools.cpp diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_coupling.cpp b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_coupling.cpp new file mode 100644 index 000000000000..fff03436d4fe --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_coupling.cpp @@ -0,0 +1,18 @@ +#include "test_stk_coupling.hpp" + +#include + +#include +#include +#include + +namespace test_stk_lib { + +void test_stk_coupling() +{ + std::cout << "stk_coupling installation test, STK version: " << stk::version_string() + << ", Coupling-Version: "<< stk::util::get_local_max_coupling_version() << std::endl; +} + +} + diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_coupling.hpp b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_coupling.hpp new file mode 100644 index 000000000000..a692134722a9 --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_coupling.hpp @@ -0,0 +1,11 @@ +#ifndef TEST_STK_COUPLING_HPP +#define TEST_STK_COUPLING_HPP + +namespace test_stk_lib { + +void test_stk_coupling(); + +} + +#endif + diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_search.cpp b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_search.cpp new file mode 100644 index 000000000000..2e321664abfe --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_search.cpp @@ -0,0 +1,16 @@ +#include "test_stk_search.hpp" + +#include + +#include +#include + +namespace test_stk_lib { + +void test_stk_search() +{ + std::cout << "stk_search installation test, SearchMethod: " << stk::search::SearchMethod::KDTREE << std::endl; +} + +} + diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_search.hpp b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_search.hpp new file mode 100644 index 000000000000..386a3ed4c141 --- /dev/null +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/src/test_stk_search.hpp @@ -0,0 +1,11 @@ +#ifndef TEST_STK_SEARCH_HPP +#define TEST_STK_SEARCH_HPP + +namespace test_stk_lib { + +void test_stk_search(); + +} + +#endif + diff --git a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/test_stk_app.cpp b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/test_stk_app.cpp index a5cebf6167ee..3627c32ba920 100644 --- a/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/test_stk_app.cpp +++ b/packages/stk/stk_integration_tests/cmake_install_test/stk_test_app/test_stk_app.cpp @@ -4,6 +4,8 @@ #include #include +#include "src/test_stk_coupling.hpp" +#include "src/test_stk_search.hpp" #include "src/test_stk_simd.hpp" #include "src/test_stk_io.hpp" #include "src/test_stk_tools.hpp" @@ -38,6 +40,10 @@ int main(int argc, char** argv) std::cout << "Test-STK-App" << std::endl; } + test_stk_lib::test_stk_coupling(); + + test_stk_lib::test_stk_search(); + test_stk_lib::test_stk_simd(MPI_COMM_WORLD); test_stk_lib::test_stk_io(MPI_COMM_WORLD, meshSource, useAutoDecomp); diff --git a/packages/stk/stk_integration_tests/stk_balance/IntegrationTestSpiderElements.cpp b/packages/stk/stk_integration_tests/stk_balance/IntegrationTestSpiderElements.cpp index 6ed4c2058c57..06b8857b1dce 100644 --- a/packages/stk/stk_integration_tests/stk_balance/IntegrationTestSpiderElements.cpp +++ b/packages/stk/stk_integration_tests/stk_balance/IntegrationTestSpiderElements.cpp @@ -27,7 +27,7 @@ class SpiderElementMesh : public stk::unit_test_util::simple_fields::MeshFixture stk::io::put_io_part_attribute(spherePart); stk::io::put_io_part_attribute(beamPart); - stk::balance::internal::register_internal_fields(get_bulk(), m_balanceSettings); + stk::balance::internal::register_internal_fields_and_parts(get_bulk(), m_balanceSettings); stk::io::fill_mesh(meshSpec, get_bulk()); diff --git a/packages/stk/stk_integration_tests/stk_middle_mesh/NonConformalInterface.cpp b/packages/stk/stk_integration_tests/stk_middle_mesh/NonConformalInterface.cpp index a3cc3f41a080..104805581782 100644 --- a/packages/stk/stk_integration_tests/stk_middle_mesh/NonConformalInterface.cpp +++ b/packages/stk/stk_integration_tests/stk_middle_mesh/NonConformalInterface.cpp @@ -267,7 +267,6 @@ TEST(Interface, EllipsoidFromCADNew) for (int i = 1; i < nmeshes; ++i) { - //ssssssdouble tStart = MPI_Wtime(); std::cout << "mesh " << i << " / " << nmeshes << std::endl; stk_interface::StkMeshCreator creator1(meshPath + fnames[0]); std::shared_ptr mesh1 = creator1.create_mesh_from_part("block_1").mesh; diff --git a/packages/stk/stk_io/stk_io/CMakeLists.txt b/packages/stk/stk_io/stk_io/CMakeLists.txt index f98866ae57b8..8e86c2bc2ff4 100644 --- a/packages/stk/stk_io/stk_io/CMakeLists.txt +++ b/packages/stk/stk_io/stk_io/CMakeLists.txt @@ -54,16 +54,6 @@ SET(SOURCES "") # Core headers # -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) -#TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../util/) -#TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/..) -#TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../) -#TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../../) -#TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../../stk_util) -#TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../../seacas/ioss/src) - FILE(GLOB HEADERS *.hpp) FILE(GLOB SOURCES *.cpp) @@ -77,6 +67,11 @@ TRIBITS_ADD_LIBRARY( SOURCES ${SOURCES} ) +target_include_directories(stk_io PUBLIC + $ + $ +) + INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_io/) diff --git a/packages/stk/stk_io/stk_io/StkMeshIoBroker.cpp b/packages/stk/stk_io/stk_io/StkMeshIoBroker.cpp index 58f96c4a2bae..d78b8470180f 100644 --- a/packages/stk/stk_io/stk_io/StkMeshIoBroker.cpp +++ b/packages/stk/stk_io/stk_io/StkMeshIoBroker.cpp @@ -143,6 +143,7 @@ stk::mesh::FieldVector get_fields_by_name(const stk::mesh::MetaData &meta, const StkMeshIoBroker::StkMeshIoBroker() : m_communicator(MPI_COMM_NULL), + m_meshBuilder(std::make_shared()), m_activeMeshIndex(0), m_sidesetFaceCreationBehavior(STK_IO_SIDE_CREATION_USING_GRAPH_TEST), m_autoLoadAttributes(true), @@ -156,6 +157,7 @@ StkMeshIoBroker::StkMeshIoBroker() StkMeshIoBroker::StkMeshIoBroker(stk::ParallelMachine comm) : m_communicator(comm), + m_meshBuilder(std::make_shared(comm)), m_activeMeshIndex(0), m_sidesetFaceCreationBehavior(STK_IO_SIDE_CREATION_USING_GRAPH_TEST), m_autoLoadAttributes(true), @@ -255,10 +257,16 @@ void StkMeshIoBroker::create_sideset_observer() } } +void StkMeshIoBroker::set_mesh_builder(std::shared_ptr meshBuilder) +{ + STK_ThrowErrorMsgIf(m_metaData || m_bulkData, + "Setting a MeshBuilder after as mesh has already been built has no effect."); + m_meshBuilder = meshBuilder; +} + void StkMeshIoBroker::set_bulk_data(std::shared_ptr arg_bulk_data) { - STK_ThrowErrorMsgIf( m_bulkData != nullptr, - "Bulk data already initialized" ); + STK_ThrowErrorMsgIf(m_bulkData, "BulkData already initialized."); m_bulkData = arg_bulk_data; if (m_metaData == nullptr) { @@ -275,14 +283,12 @@ void StkMeshIoBroker::set_bulk_data(std::shared_ptr arg_bul void StkMeshIoBroker::replace_bulk_data(std::shared_ptr arg_bulk_data) { - STK_ThrowErrorMsgIf( m_bulkData == nullptr, - "There is no bulk data to replace." ); - STK_ThrowErrorMsgIf( m_metaData == nullptr, - "Meta data must be non-null when calling StkMeshIoBroker::replace_bulk_data." ); + STK_ThrowRequireMsg(m_bulkData, "There is no BulkData to replace."); + STK_ThrowRequireMsg(m_metaData, "MetaData must be non-null when calling StkMeshIoBroker::replace_bulk_data."); std::shared_ptr new_meta_data(&(arg_bulk_data->mesh_meta_data()), [](auto pointerWeWontDelete){}); - STK_ThrowErrorMsgIf( m_metaData.get() != new_meta_data.get(), - "Meta data for both new and old bulk data must be the same." ); + STK_ThrowErrorMsgIf(m_metaData.get() != new_meta_data.get(), + "MetaData for both new and old BulkData must be the same."); m_bulkData = arg_bulk_data; @@ -478,7 +484,7 @@ void StkMeshIoBroker::create_input_mesh() // See if meta data is null, if so, create a new one... if (is_meta_data_null()) { - m_metaData = std::shared_ptr(new stk::mesh::MetaData()); + m_metaData = m_meshBuilder->create_meta_data(); } if (m_useSimpleFields) { @@ -826,9 +832,9 @@ void StkMeshIoBroker::create_bulk_data() "INTERNAL ERROR: Mesh Input Region pointer is NULL in populate_mesh."); if (is_bulk_data_null()) { - set_bulk_data(stk::mesh::MeshBuilder(region->get_database()->util().communicator()) - .set_aura_option(stk::mesh::BulkData::AUTO_AURA) - .create(meta_data_ptr())); + set_bulk_data(m_meshBuilder->set_communicator(region->get_database()->util().communicator()) + .set_aura_option(stk::mesh::BulkData::AUTO_AURA) + .create(meta_data_ptr())); } } diff --git a/packages/stk/stk_io/stk_io/StkMeshIoBroker.hpp b/packages/stk/stk_io/stk_io/StkMeshIoBroker.hpp index 49140038d4ef..597d18e8e7de 100644 --- a/packages/stk/stk_io/stk_io/StkMeshIoBroker.hpp +++ b/packages/stk/stk_io/stk_io/StkMeshIoBroker.hpp @@ -168,6 +168,10 @@ namespace stk { stk::mesh::Selector get_active_selector() const; void set_active_selector(stk::mesh::Selector my_selector); + // Override the default MeshBuilder if you need to have a custom STK Mesh + // auto-generated internally. + void set_mesh_builder(std::shared_ptr meshBuilder); + // Set bulk data directly with your own bulk data. If this is // not called prior to the populate_bulk_data() call, it will be // created automatically using the communicator of the m_input_region. @@ -742,6 +746,7 @@ namespace stk { stk::ParallelMachine m_communicator; std::vector m_rankNames; // Optional rank name vector. + std::shared_ptr m_meshBuilder; std::shared_ptr m_metaData; std::shared_ptr m_bulkData; diff --git a/packages/stk/stk_io/stk_io/util/CMakeLists.txt b/packages/stk/stk_io/stk_io/util/CMakeLists.txt index a352ec19f1b4..4c6dd10f5f3c 100644 --- a/packages/stk/stk_io/stk_io/util/CMakeLists.txt +++ b/packages/stk/stk_io/stk_io/util/CMakeLists.txt @@ -33,35 +33,12 @@ # -# -# A) Package-specific configuration options -# - -# TRIBITS_CONFIGURE_FILE(${PACKAGE_NAME}_config.h) - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - SET(HEADERS "") SET(SOURCES "") FILE(GLOB HEADERS *.hpp) FILE(GLOB SOURCES *.cpp) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -#TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../) -#TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../../../seacas/libraries/ioss/src) - -# -# Core headers -# - # # C) Define the targets for package's library(s) # @@ -73,5 +50,10 @@ TRIBITS_ADD_LIBRARY( DEPLIBS stk_io ) +target_include_directories(stk_io_util PUBLIC + $ + $ +) + INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_io/util/) diff --git a/packages/stk/stk_math/CMakeLists.txt b/packages/stk/stk_math/CMakeLists.txt index 4c6baf4d1c3d..458f8c351caa 100644 --- a/packages/stk/stk_math/CMakeLists.txt +++ b/packages/stk/stk_math/CMakeLists.txt @@ -1,6 +1,6 @@ -TRIBITS_SUBPACKAGE(Math) +STK_SUBPACKAGE(Math) ADD_SUBDIRECTORY(stk_math) -TRIBITS_SUBPACKAGE_POSTPROCESS() +STK_SUBPACKAGE_POSTPROCESS() diff --git a/packages/stk/stk_math/stk_math/CMakeLists.txt b/packages/stk/stk_math/stk_math/CMakeLists.txt index 84c0e27035db..c32b31ce8f11 100644 --- a/packages/stk/stk_math/stk_math/CMakeLists.txt +++ b/packages/stk/stk_math/stk_math/CMakeLists.txt @@ -32,40 +32,29 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -# -# A) Package-specific configuration options -# - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - -SET(HEADERS "") -SET(SOURCES "") - -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) - -# -# Core headers -# - FILE(GLOB HEADERS *.hpp) - FILE(GLOB SOURCES *.cpp) -# -# C) Define the targets for package's library(s) -# - -TRIBITS_ADD_LIBRARY( - stk_math - HEADERS ${HEADERS} - SOURCES ${SOURCES} +if(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY(stk_math + HEADERS ${HEADERS} + SOURCES ${SOURCES} ) +else() + add_library(stk_math ${SOURCES}) + + target_link_libraries(stk_math PUBLIC Kokkos::kokkos) + target_link_libraries(stk_math PUBLIC stk_util_util) +endif() + +target_include_directories(stk_math PUBLIC + $ + $ +) INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_math) + +if(NOT HAVE_STK_Trilinos) + INSTALL(TARGETS stk_math DESTINATION ${STK_INSTALL_LIBDIR}) +endif() diff --git a/packages/stk/stk_mesh/stk_mesh/base/BulkData.cpp b/packages/stk/stk_mesh/stk_mesh/base/BulkData.cpp index e1ed2c4597bb..1dbc65a32da2 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/BulkData.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/BulkData.cpp @@ -334,7 +334,7 @@ BulkData::BulkData(std::shared_ptr mesh_meta_data, #ifdef SIERRA_MIGRATION bool add_fmwk_data, #endif - FieldDataManager *field_data_manager, + std::unique_ptr field_data_manager, unsigned initialBucketCapacity, unsigned maximumBucketCapacity, std::shared_ptr auraGhosting, @@ -358,8 +358,7 @@ BulkData::BulkData(std::shared_ptr mesh_meta_data, m_owner(), m_comm_list_updater(m_entity_comm_list, m_entitycomm, m_removedGhosts), m_entity_keys(), - m_default_field_data_manager(mesh_meta_data->entity_rank_count()), - m_field_data_manager(field_data_manager), + m_field_data_manager(std::move(field_data_manager)), m_bucket_repository(*this, mesh_meta_data->entity_rank_count(), initialBucketCapacity, maximumBucketCapacity), #ifdef SIERRA_MIGRATION m_add_fmwk_data(add_fmwk_data), @@ -395,9 +394,8 @@ BulkData::BulkData(std::shared_ptr mesh_meta_data, m_entity_comm_map.setCommMapChangeListener(&m_comm_list_updater); - if (m_field_data_manager == nullptr) - { - m_field_data_manager = &m_default_field_data_manager; + if (not m_field_data_manager) { + m_field_data_manager = std::make_unique(mesh_meta_data->entity_rank_count()); } initialize_arrays(); @@ -418,13 +416,6 @@ BulkData::BulkData(std::shared_ptr mesh_meta_data, BulkData::~BulkData() { - -#ifdef STK_PROFILE_MEMORY - ParallelMachine world = MPI_COMM_WORLD; // HACK, but necessary to work with Fmwk - const int real_rank = parallel_machine_rank(world); - print_max_stk_memory_usage(world, real_rank, std::cout); -#endif - #ifdef SIERRA_MIGRATION for(size_t i=0; i(mesh_meta_data().entity_rank_count()); const EntityRank beginRank = static_cast(entity_rank(entity)+1); - + for(EntityRank rank=beginRank; rank 0) { @@ -826,10 +817,10 @@ Entity BulkData::create_and_connect_side(const stk::mesh::EntityId globalSideId, { stk::topology sideTop = bucket(elem).topology().side_topology(localSideId); Entity side = internal_declare_entity(sideTop.rank(), globalSideId, add_root_topology_part(parts, mesh_meta_data().get_topology_root_part(sideTop))); - if (has_face_adjacent_element_graph()) { + if (has_face_adjacent_element_graph()) { use_graph_to_connect_side(get_face_adjacent_element_graph(), side, elem, localSideId); } - else { + else { impl::connect_element_to_entity(*this, elem, side, localSideId, parts, sideTop); } if(this->num_edges(elem) != 0) { @@ -943,7 +934,7 @@ void BulkData::add_node_sharing(const EntityProcVec& nodesAndProcs) internal_mark_entity(node, IS_SHARED); std::pair result = entity_comm_map_insert(node, EntityCommInfo(stk::mesh::BulkData::SHARED, sharing_proc)); entity_comm_map_erase(entity_key(node), EntityCommInfo(stk::mesh::BulkData::AURA, sharing_proc)); - + const bool inserted = result.second; if (inserted) { EntityKey key = entity_key(node); @@ -1447,9 +1438,9 @@ void BulkData::declare_entities(stk::topology::rank_t rank, const IDVECTOR& newI require_good_rank_and_id(key.rank(), key.id()); m_modSummary.track_declare_entity(key.rank(), key.id(), stk::mesh::PartVector()); - + std::pair entityBoolPair = m_entityKeyMapping->internal_create_entity(key); - + STK_ThrowErrorMsgIf( ! entityBoolPair.second, "Generated id " << key.id() << " of rank " << key.rank() << " which was already used."); @@ -1493,7 +1484,7 @@ bool BulkData::in_shared(Entity entity, int proc) const const int entityCommIndex = m_entitycomm[entity.local_offset()]; if (entityCommIndex != -1) { PairIterEntityComm commInfo = internal_comm_db().comm(entityCommIndex); - + while(!commInfo.empty() && commInfo->ghost_id == SHARED) { if (commInfo->proc == proc) { return true; @@ -1553,7 +1544,7 @@ bool BulkData::in_ghost( const Ghosting & ghost , Entity entity , int proc ) con } PairIterEntityComm commInfo = internal_comm_db().comm(m_entitycomm[entity.local_offset()]); - + while(!commInfo.empty() && commInfo->ghost_id < ghost.ordinal()) { ++commInfo; } @@ -1575,10 +1566,10 @@ bool BulkData::in_ghost( const Ghosting & ghost , Entity entity ) const } PairIterEntityComm commInfo = internal_comm_db().comm(m_entitycomm[entity.local_offset()]); - + const EntityCommInfo* i = commInfo.begin(); const EntityCommInfo* end = commInfo.end(); - + while(i!=end && i->ghost_id < ghost.ordinal()) { ++i; } @@ -1604,10 +1595,10 @@ bool BulkData::in_send_ghost( const Ghosting & ghost , Entity entity , int proc const int entityCommIndex = m_entitycomm[entity.local_offset()]; if (entityCommIndex >= 0) { EntityCommInfo tmp( ghost.ordinal() , proc ); - + PairIterEntityComm ec = internal_comm_db().comm(m_entitycomm[entity.local_offset()]); const EntityCommInfo* i = std::lower_bound( ec.begin(), ec.end() , tmp ); - + ret_val = i != ec.end() && tmp == *i ; } } @@ -1622,10 +1613,10 @@ bool BulkData::is_communicated_with_proc(Entity entity, int proc) const } PairIterEntityComm commInfo = internal_comm_db().comm(m_entitycomm[entity.local_offset()]); - + const EntityCommInfo* i = commInfo.begin(); const EntityCommInfo* end = commInfo.end(); - + while(i != end) { if (i->proc == proc) { return true; @@ -2438,8 +2429,8 @@ void BulkData::get_entities_that_have_sharing(std::vector &en // this communicates states of the entities to all procs so that entity states are consistent stk::mesh::EntityVector entitiesNoLongerShared; stk::mesh::EntityProcVec entitiesToRemoveFromSharing; - m_meshModification.delete_shared_entities_which_are_no_longer_in_owned_closure(entitiesToRemoveFromSharing); - + m_meshModification.delete_shared_entities_which_are_no_longer_in_owned_closure(entitiesToRemoveFromSharing); + impl::CommEntityMods commEntityMods(*this, internal_comm_db(), internal_comm_list()); commEntityMods.communicate(impl::CommEntityMods::PACK_SHARED); m_meshModification.internal_resolve_shared_modify_delete(commEntityMods.get_shared_mods(), entitiesToRemoveFromSharing, entitiesNoLongerShared); @@ -3100,15 +3091,18 @@ bool BulkData::inputs_ok_and_need_ghosting(Ghosting & ghosts , return anyProcsHaveNewGhosts; } -void BulkData::batch_add_to_ghosting(Ghosting &ghosting, const EntityProcVec &entitiesAndDestinationProcs) +bool BulkData::batch_add_to_ghosting(Ghosting &ghosting, const EntityProcVec &entitiesAndDestinationProcs) { - std::vector filtered_add_send; - std::vector empty_vector; + std::vector filtered_add_send; + std::vector empty_vector; - if (inputs_ok_and_need_ghosting(ghosting , entitiesAndDestinationProcs , empty_vector, filtered_add_send, empty_vector)) - { - internal_batch_add_to_ghosting(ghosting, filtered_add_send); - } + bool needToGhostEntities = false; + if (inputs_ok_and_need_ghosting(ghosting , entitiesAndDestinationProcs , empty_vector, filtered_add_send, empty_vector)) + { + needToGhostEntities = true; + internal_batch_add_to_ghosting(ghosting, filtered_add_send); + } + return needToGhostEntities; } void BulkData::internal_batch_add_to_ghosting(Ghosting &ghosting, const EntityProcVec &entitiesAndDestinationProcs) @@ -3240,7 +3234,7 @@ void BulkData::ghost_entities_and_fields(Ghosting & ghosting, while(buf.remaining()) { buf.unpack( rankAndFlag[0] ); buf.unpack( rankAndFlag[1] ); - + STK_ThrowAssert(rankAndFlag[1] == 0); EntityKey key; buf.unpack(key); @@ -3297,7 +3291,7 @@ void BulkData::ghost_entities_and_fields(Ghosting & ghosting, for(const Part* currentPart : currentParts) { Ordinal currentPartOrdinal = currentPart->mesh_meta_data_ordinal(); bool excludedPart = ( (currentPartOrdinal == meta.locally_owned_part().mesh_meta_data_ordinal()) || - (currentPartOrdinal == meta.globally_shared_part().mesh_meta_data_ordinal()) || + (currentPartOrdinal == meta.globally_shared_part().mesh_meta_data_ordinal()) || (!meta.get_parts()[currentPartOrdinal]->entity_membership_is_parallel_consistent() )); if (!excludedPart && !contains_ordinal(partOrdinals, currentPartOrdinal)) { @@ -3345,7 +3339,7 @@ void BulkData::ghost_entities_and_fields(Ghosting & ghosting, OrdinalVector addParts, scratchOrdinalVec; removeParts = {ghosting_part(ghosting).mesh_meta_data_ordinal()}; EntityCommListInfoVector& commList = m_entity_comm_list; - + stk::util::sort_and_unique(removedRecvGhosts, EntityLess(*this)); for(unsigned i=0; ikey == key) { + itr->key = EntityKey(); + } + } + } + else { + internal_change_entity_parts(recvGhostEntity, addParts, removeParts, scratchOrdinalVec, scratchSpace); + } + } + } } delete_unneeded_entries_from_the_comm_list(); } @@ -3558,6 +3570,8 @@ void BulkData::internal_change_ghosting( const unsigned ghostingPartOrdinal = ghosting_part(ghosting).mesh_meta_data_ordinal(); OrdinalVector removeGhostingPart = {ghostingPartOrdinal}; + const unsigned ownedPartOrdinal = mesh_meta_data().locally_owned_part().mesh_meta_data_ordinal(); + OrdinalVector addOwnedPart = {ownedPartOrdinal}; OrdinalVector addParts; OrdinalVector scratchOrdinalVec, scratchSpace; @@ -3593,8 +3607,11 @@ void BulkData::internal_change_ghosting( entity_comm_map_erase(key, ghosting); if (internal_entity_comm_map(rmEntity).empty()) { - STK_ThrowRequireMsg( internal_destroy_entity_with_notification(rmEntity, true), "P"<(sharingProc); entity_comm_map_insert(entity, EntityCommInfo(SHARED, sharingProc)); } - + const EntityState localState = state(entity); if (!in_shared(entity,p)) { if (localState == Unchanged) { @@ -4742,7 +4759,7 @@ void BulkData::internal_update_all_sharing_procs() const if (m_all_sharing_procs_sync_count < synchronized_count()) { const EntityRank numRanks = static_cast(mesh_meta_data().entity_rank_count()); m_all_sharing_procs.resize(numRanks); - if (parallel_size() > 1) { + if (parallel_size() > 1) { for (EntityRank r = stk::topology::BEGIN_RANK; r < numRanks; ++r) { m_all_sharing_procs[r].clear(); } @@ -4821,7 +4838,7 @@ void BulkData::internal_update_fast_comm_maps() const EntityKey const key = all_comm[i].key; EntityRank const rank = key.rank(); - + PairIterEntityComm ec = commDB.comm(all_comm[i].entity_comm); for(; !ec.empty() && ec->ghost_id == BulkData::SHARED; ++ec) { add_bucket_and_ord(bucket_id, bucket_ord, m_volatile_fast_shared_comm_map[rank][ec->proc]); @@ -4831,7 +4848,7 @@ void BulkData::internal_update_fast_comm_maps() const } m_volatile_fast_shared_comm_map_sync_count = synchronized_count(); } - + template void internal_throw_error_if_manipulating_internal_part_memberships(const PARTVECTOR & parts) { @@ -4873,8 +4890,8 @@ void BulkData::change_entity_parts( const EntityVector& entities, internal_throw_error_if_manipulating_internal_part_memberships(remove_parts); } internal_verify_and_change_entity_parts(entities, add_parts, remove_parts); -} - +} + template void BulkData::change_entity_parts(const EntityVector&, const PartVector&, const PartVector&); template void BulkData::change_entity_parts(const EntityVector&, const ConstPartVector&, const ConstPartVector&); @@ -4937,14 +4954,14 @@ void BulkData::change_entity_parts(const Selector& selector, { if(m_runConsistencyCheck) { impl::check_matching_selectors_and_parts_across_procs(selector, add_parts, remove_parts, parallel()); - } + } bool stkMeshRunningUnderFramework = m_add_fmwk_data; if(!stkMeshRunningUnderFramework) - { + { internal_throw_error_if_manipulating_internal_part_memberships(add_parts); internal_throw_error_if_manipulating_internal_part_memberships(remove_parts); - } + } internal_verify_and_change_entity_parts(selector, rank, add_parts, remove_parts); } @@ -4953,7 +4970,7 @@ void BulkData::batch_change_entity_parts(const Selector& selector, EntityRank rank, const PartVector& add_parts, const PartVector& remove_parts, - ModEndOptimizationFlag opt) + ModEndOptimizationFlag opt) { const bool starting_modification = modification_begin(); STK_ThrowRequireMsg(starting_modification, "ERROR: BulkData already being modified,\n" @@ -5542,7 +5559,7 @@ void BulkData::remove_entities_from_sharing(const EntityProcVec& entitiesToRemov entitiesNoLongerShared.push_back(entityAndProc.first); this->internal_change_entity_parts(entityAndProc.first,{},{this->mesh_meta_data().globally_shared_part().mesh_meta_data_ordinal()}, scratchOrdinalVec, scratchSpace); this->internal_mark_entity(entityAndProc.first, NOT_SHARED); - } + } } stk::util::sort_and_unique(entitiesNoLongerShared); } @@ -5721,7 +5738,7 @@ void BulkData::internal_resolve_sharing_and_ghosting_for_sides(bool connectFaces stk::mesh::EntityVector entitiesNoLongerShared; stk::mesh::EntityProcVec entitiesToRemoveFromSharing; m_meshModification.delete_shared_entities_which_are_no_longer_in_owned_closure(entitiesToRemoveFromSharing); - + impl::CommEntityMods commEntityMods(*this, internal_comm_db(), internal_comm_list()); commEntityMods.communicate(impl::CommEntityMods::PACK_SHARED); m_meshModification.internal_resolve_shared_modify_delete(commEntityMods.get_shared_mods(), entitiesToRemoveFromSharing, entitiesNoLongerShared); diff --git a/packages/stk/stk_mesh/stk_mesh/base/BulkData.hpp b/packages/stk/stk_mesh/stk_mesh/base/BulkData.hpp index d3f253d7c884..f343819bab81 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/BulkData.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/BulkData.hpp @@ -117,13 +117,6 @@ namespace mesh { using ModEndOptimizationFlag = impl::MeshModification::modification_optimization; -#ifndef STK_HIDE_DEPRECATED_CODE //delete after Aug 2023 -STK_DEPRECATED_MSG("Unnecessary sync argument removed. Use communicate_field_data(ghosts, fields).") -void communicate_field_data(const Ghosting & ghosts, const std::vector & fields, bool syncOnlySharedOrGhosted); -STK_DEPRECATED_MSG("Unnecessary sync argument removed. Use communicate_field_data(mesh, fields).") -void communicate_field_data(const BulkData & mesh, const std::vector & fields, bool syncOnlySharedOrGhosted); -#endif - void communicate_field_data(const Ghosting & ghosts, const std::vector & fields); void communicate_field_data(const BulkData & mesh, const std::vector & fields); void parallel_sum_including_ghosts(const BulkData & mesh, const std::vector & fields); @@ -142,6 +135,7 @@ stk::mesh::Entity clone_element_side(stk::mesh::BulkData &bulk, stk::mesh::Entity elem, stk::mesh::ConnectivityOrdinal ord, const stk::mesh::PartVector &parts); +void remove_ghosts_from_remote_procs(stk::mesh::BulkData& bulk, EntityVector& recvGhostsToRemove); namespace impl { stk::mesh::Entity connect_side_to_element(stk::mesh::BulkData& bulkData, stk::mesh::Entity element, @@ -236,6 +230,7 @@ class BulkData { * parallel consistent state to the "ok to modify" state. * False if already in this state. */ + virtual bool modification_begin(const std::string description = std::string("UNSPECIFIED")) { ProfilingBlock block("mod begin:" + description); @@ -265,6 +260,7 @@ class BulkData { * a parallel-consistent exception will be thrown. */ + virtual bool modification_end(ModEndOptimizationFlag modEndOpt = ModEndOptimizationFlag::MOD_END_SORT) { bool endStatus = false; @@ -621,7 +617,10 @@ class BulkData { const std::vector & add_send , const std::vector & remove_receive = std::vector()); // Mod Mark - void batch_add_to_ghosting(Ghosting &ghosting, const EntityProcVec &entitiesAndDestinationProcs); // Mod Mark + /** \brief send ghost entities to the specified entity-proc locations + * return true if entities were ghosted. (false if specified ghosts were already ghosted) + */ + virtual bool batch_add_to_ghosting(Ghosting &ghosting, const EntityProcVec &entitiesAndDestinationProcs); // Clear all ghosts for a particular ghosting. void destroy_ghosting( Ghosting& ghost_layer ); // Mod Mark @@ -874,7 +873,7 @@ class BulkData { #ifdef SIERRA_MIGRATION bool add_fmwk_data = false, #endif - FieldDataManager *field_dataManager = nullptr, + std::unique_ptr field_dataManager = std::unique_ptr(), unsigned initialBucketCapacity = get_default_initial_bucket_capacity(), unsigned maximumBucketCapacity = get_default_maximum_bucket_capacity(), std::shared_ptr auraGhosting = std::shared_ptr(), @@ -1445,10 +1444,6 @@ class BulkData { template class NgpDebugger> friend class stk::mesh::DeviceField; // friends until it is decided what we're doing with Fields and Parallel and BulkData - #ifndef STK_HIDE_DEPRECATED_CODE //delete after Aug 2023 - friend void communicate_field_data(const Ghosting & ghosts, const std::vector & fields, bool syncOnlySharedOrGhosted); - friend void communicate_field_data(const BulkData & mesh, const std::vector & fields, bool syncOnlySharedOrGhosted); - #endif friend void communicate_field_data(const Ghosting & ghosts, const std::vector & fields); friend void communicate_field_data(const BulkData & mesh, const std::vector & fields); template friend void parallel_op_including_ghosts_impl(const BulkData & mesh, const std::vector & fields); @@ -1468,6 +1463,8 @@ class BulkData { stk::mesh::ConnectivityOrdinal ord, const stk::mesh::PartVector &parts); + friend void remove_ghosts_from_remote_procs(stk::mesh::BulkData& bulk, EntityVector& recvGhostsToRemove); + friend stk::mesh::Entity impl::connect_side_to_element(stk::mesh::BulkData& bulkData, stk::mesh::Entity element, stk::mesh::EntityId side_global_id, stk::mesh::ConnectivityOrdinal side_ordinal, stk::mesh::Permutation side_permutation, const stk::mesh::PartVector& parts); @@ -1523,9 +1520,7 @@ class BulkData { std::vector> m_removedGhosts; CommListUpdater m_comm_list_updater; std::vector m_entity_keys; //indexed by Entity - // ContiguousFieldDataManager m_default_field_data_manager; - DefaultFieldDataManager m_default_field_data_manager; - FieldDataManager *m_field_data_manager; + std::unique_ptr m_field_data_manager; impl::BucketRepository m_bucket_repository; #ifdef SIERRA_MIGRATION @@ -1565,7 +1560,6 @@ class BulkData { mutable std::vector m_selector_to_buckets_maps; bool m_use_identifiers_for_resolving_sharing; - std::string m_lastModificationDescription; stk::EmptyModificationSummary m_modSummary; // If needing debug info for modifications, comment out above line and uncomment line below //stk::ModificationSummary m_modSummary; @@ -1579,6 +1573,7 @@ class BulkData { bool m_runConsistencyCheck; protected: + std::string m_lastModificationDescription; stk::mesh::impl::SoloSideIdGenerator m_soloSideIdGenerator; bool m_supportsLargeIds = false; }; @@ -1973,16 +1968,16 @@ BulkData::in_send_ghost( EntityKey key) const inline bool BulkData::in_send_ghost( Entity entity) const { - const int owner_rank = parallel_owner_rank(entity); - for ( PairIterEntityComm ec = internal_entity_comm_map(entity); ! ec.empty() ; ++ec ) - { - if ( ec->ghost_id != 0 && - ec->proc != owner_rank) - { + const int owner_rank = parallel_owner_rank(entity); + if (owner_rank == parallel_rank()) { + for ( PairIterEntityComm ec = internal_entity_comm_map(entity); ! ec.empty() ; ++ec ) { + if (ec->ghost_id != 0 && ec->proc != owner_rank) { return true; } } - return false; + } + + return false; } inline void diff --git a/packages/stk/stk_mesh/stk_mesh/base/CMakeLists.txt b/packages/stk/stk_mesh/stk_mesh/base/CMakeLists.txt index 682aedd71605..96a22e5aea53 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/CMakeLists.txt +++ b/packages/stk/stk_mesh/stk_mesh/base/CMakeLists.txt @@ -32,30 +32,12 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - -# -# A) Package-specific configuration options -# - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - SET(HEADERS "") SET(SOURCES "") SET(HEADERS_IMPL "") SET(SOURCES_IMPL "") -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) -# -# Core headers -# - FILE(GLOB HEADERS *.hpp) FILE(GLOB SOURCES *.cpp) @@ -75,6 +57,11 @@ TRIBITS_ADD_LIBRARY( SOURCES ${SOURCES} ${SOURCES_IMPL} ${SOURCES_ELEMGRAPH} ) +target_include_directories(stk_mesh_base PUBLIC + $ + $ +) + INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_mesh/base/) diff --git a/packages/stk/stk_mesh/stk_mesh/base/DestroyElements.cpp b/packages/stk/stk_mesh/stk_mesh/base/DestroyElements.cpp index e85b3429a7a7..89af6b8d222a 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/DestroyElements.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/DestroyElements.cpp @@ -9,62 +9,24 @@ namespace stk { namespace mesh { -void destroy_upward_connected_aura_entities(stk::mesh::BulkData &bulk, stk::mesh::Entity connectedEntity, stk::mesh::EntityRank conRank, - EntityVector& scratchSpace) -{ - impl::StoreEntity storeEntity(bulk); - impl::VisitUpwardClosure(bulk, connectedEntity, storeEntity); - - storeEntity.store_visited_entities_in_vec(scratchSpace); - stk::util::sort_and_unique(scratchSpace, EntityLess(bulk)); - - for(unsigned i=0; i& downwardConnectivity, - const Selector& orphansToDelete) +void pack_ghosts(BulkData &bulk, EntityVector &elemsAndRelatedEntities, stk::CommSparse& commSparse) { - stk::CommSparse commSparse(bulk.parallel()); - std::vector commProcs; for(int phase = 0; phase < 2; ++phase) { - for(Entity elem : elementsToDestroy) { - if (!bulk.is_valid(elem) || !bulk.bucket(elem).owned()) { + for(Entity ent : elemsAndRelatedEntities) { + if (!bulk.is_valid(ent)) { continue; } - bulk.comm_procs(elem, commProcs); + bulk.comm_procs(ent, commProcs); for(int p : commProcs) { - if (!bulk.in_shared(elem, p)) { - commSparse.send_buffer(p).pack(bulk.entity_key(elem)); - } - } - } - - for(EntityVector& downward : downwardConnectivity) { - for(Entity entity : downward) { - if (bulk.is_valid(entity) && bulk.bucket(entity).owned() && orphansToDelete(bulk.bucket(entity))) { - bulk.comm_procs(entity, commProcs); - for(int p : commProcs) { - if (!bulk.in_shared(entity, p)) { - commSparse.send_buffer(p).pack(bulk.entity_key(entity)); - } - } - } + commSparse.send_buffer(p).pack(bulk.entity_key(ent)); } } @@ -72,28 +34,54 @@ void remove_ghosts_on_remote_procs(stk::mesh::BulkData &bulk, stk::mesh::EntityV commSparse.allocate_buffers(); } } +} - commSparse.communicate(); - - EntityVector recvGhostsToRemove; +void unpack_recv_ghosts(stk::mesh::BulkData& bulk, stk::CommSparse& commSparse, EntityVector& recvGhostsToRemove) +{ for(int p=0; p(key); Entity entity = bulk.get_entity(key); - if (bulk.in_receive_ghost(entity)) { + if (bulk.is_valid(entity) && bulk.in_receive_ghost(entity)) { recvGhostsToRemove.push_back(entity); } } } +} +void remove_ghosts_from_remote_procs(stk::mesh::BulkData &bulk, EntityVector& recvGhostsToRemove) +{ if (!recvGhostsToRemove.empty()) { stk::util::sort_and_unique(recvGhostsToRemove, EntityLess(bulk)); + + impl::StoreEntity storeEntity(bulk); + impl::VisitUpwardClosure(bulk, recvGhostsToRemove.begin(), recvGhostsToRemove.end(), storeEntity); + storeEntity.store_visited_entities_in_vec(recvGhostsToRemove); + + stk::util::sort_and_unique(recvGhostsToRemove, EntityLess(bulk)); + } + + std::vector emptyAdd; + EntityVector removesForThisGhosting; + removesForThisGhosting.reserve(recvGhostsToRemove.size()); + const bool notAddingSendGhosts = true; + + const std::vector& ghostings = bulk.ghostings(); + + for(unsigned ig=0; ig downwardConnectivity(bulk.mesh_meta_data().entity_rank_count()); + for(stk::mesh::Entity element : elementsToDestroy) { + if(!bulk.is_valid(element)) + continue; - for(stk::mesh::Entity element : elementsToDestroy) { - if(!bulk.is_valid(element)) - continue; + STK_ThrowRequireMsg(!impl::has_upward_connectivity(bulk, element), "Element to be destroyed cannot have upward connectivity"); + STK_ThrowRequireMsg(bulk.entity_rank(element) == stk::topology::ELEM_RANK, "Entity to be destroyed must be an element"); + } - STK_ThrowRequireMsg(!impl::has_upward_connectivity(bulk, element), "Element to be destroyed cannot have upward connectivity"); - STK_ThrowRequireMsg(bulk.entity_rank(element) == stk::topology::ELEM_RANK, "Entity to be destroyed must be an element"); + stk::mesh::EntityVector elemsAndRelatedEntities; + elemsAndRelatedEntities.reserve(2*elementsToDestroy.size()); + get_all_related_entities(bulk, elementsToDestroy, orphansToDelete, elemsAndRelatedEntities); - for(stk::mesh::EntityRank conRank = stk::topology::NODE_RANK; conRank < stk::topology::ELEM_RANK; ++conRank) { - unsigned numConnected = bulk.num_connectivity(element, conRank); - const stk::mesh::Entity* connectedEntities = bulk.begin(element, conRank); - for(unsigned i = 0; i < numConnected; i++) - downwardConnectivity[conRank].push_back(connectedEntities[i]); - } + stk::CommSparse commSparse(bulk.parallel()); - } + pack_ghosts(bulk, elemsAndRelatedEntities, commSparse); - remove_ghosts_on_remote_procs(bulk, elementsToDestroy, downwardConnectivity, orphansToDelete); + commSparse.communicate(); - for(stk::mesh::Entity element : elementsToDestroy) { - bulk.destroy_entity(element); + EntityVector localEntitiesToRemove; + EntityVector recvGhostsToRemove; + for(Entity ent : elemsAndRelatedEntities) { + if (bulk.is_valid(ent)) { + if (bulk.in_receive_ghost(ent)) { + recvGhostsToRemove.push_back(ent); + } + else { + localEntitiesToRemove.push_back(ent); + } } + } - EntityVector scratchSpace; - for(stk::mesh::EntityRank conRank : {stk::topology::FACE_RANK, stk::topology::EDGE_RANK, stk::topology::NODE_RANK}) - { - stk::util::sort_and_unique(downwardConnectivity[conRank], stk::mesh::EntityLess(bulk)); - - int numConnections = downwardConnectivity[conRank].size(); - for(int i = numConnections-1; i >= 0; --i) - { - stk::mesh::Entity connectedEntity = downwardConnectivity[conRank][i]; - if(bulk.is_valid(connectedEntity) && orphansToDelete(bulk.bucket(connectedEntity))) - { - if (bulk.in_shared(connectedEntity)) { - destroy_upward_connected_aura_entities(bulk, connectedEntity, conRank, scratchSpace); - } - if(!impl::has_upward_connectivity(bulk, connectedEntity)) - bulk.destroy_entity(connectedEntity); - } - } + unpack_recv_ghosts(bulk, commSparse, recvGhostsToRemove); + + remove_ghosts_from_remote_procs(bulk, recvGhostsToRemove); + + for(unsigned i=0; i allocatorAdaptor) + : m_fieldDataAllocator(std::move(allocatorAdaptor)), + alignment_increment_bytes(alignmentIncrementBytes) +{ + if (not m_fieldDataAllocator) { + m_fieldDataAllocator = std::make_unique>>(); + } +} + void DefaultFieldDataManager::allocate_bucket_field_data(const EntityRank rank, const std::vector & fields, const PartVector& superset_parts, @@ -170,7 +180,7 @@ void DefaultFieldDataManager::allocate_bucket_field_data(const EntityRank rank, // Allocate all field data for this bucket if (total_field_data_size > 0) { - unsigned char* all_data = (*field_data_allocator).allocate(total_field_data_size); + unsigned char* all_data = m_fieldDataAllocator->allocate(total_field_data_size); m_field_raw_data[rank].push_back(all_data); // Set data ptrs in field meta datas @@ -341,13 +351,13 @@ void DefaultFieldDataManager::reallocate_bucket_field_data(const EntityRank rank const BucketFieldSegment & lastFieldSegment = newOffsetForField[newOffsetForField.size()-2]; m_num_bytes_allocated_per_field.back() += lastFieldSegment.size; - unsigned char* newAllocationAllFields = (*field_data_allocator).allocate(newBucketAllocationSize); + unsigned char* newAllocationAllFields = m_fieldDataAllocator->allocate(newBucketAllocationSize); const unsigned char* oldAllocationAllFields = m_field_raw_data[rank][bucketId]; copy_field_data_from_old_to_new_bucket(rank, bucketSize, bucketId, allFields, oldOffsetForField, newOffsetForField, oldAllocationAllFields, newAllocationAllFields); - (*field_data_allocator).deallocate(m_field_raw_data[rank][bucketId], oldBucketAllocationSize); + m_fieldDataAllocator->deallocate(m_field_raw_data[rank][bucketId], oldBucketAllocationSize); m_field_raw_data[rank][bucketId] = newAllocationAllFields; m_bucketCapacity[rank][bucketId] = bucketCapacity; @@ -383,7 +393,7 @@ void DefaultFieldDataManager::deallocate_bucket_field_data(const EntityRank rank field_data.m_data = nullptr; } } - (*field_data_allocator).deallocate(m_field_raw_data[rank][bucket_id], bytes_to_delete); + m_fieldDataAllocator->deallocate(m_field_raw_data[rank][bucket_id], bytes_to_delete); m_field_raw_data[rank][bucket_id] = nullptr; m_bucketCapacity[rank][bucket_id] = 0; } @@ -490,13 +500,13 @@ void DefaultFieldDataManager::grow_bucket_capacity(const FieldVector & allFields } } - unsigned char* newAllocationAllFields = (*field_data_allocator).allocate(newBucketAllocationSize); + unsigned char* newAllocationAllFields = m_fieldDataAllocator->allocate(newBucketAllocationSize); const unsigned char* oldAllocationAllFields = m_field_raw_data[rank][bucketId]; copy_field_data_from_old_to_new_bucket(rank, bucketSize, bucketId, allFields, oldOffsetForField, newOffsetForField, oldAllocationAllFields, newAllocationAllFields); - (*field_data_allocator).deallocate(m_field_raw_data[rank][bucketId], oldBucketAllocationSize); + m_fieldDataAllocator->deallocate(m_field_raw_data[rank][bucketId], oldBucketAllocationSize); m_field_raw_data[rank][bucketId] = newAllocationAllFields; update_field_pointers_to_new_bucket(rank, bucketId, allFields, bucketCapacity, newAllocationAllFields); @@ -533,7 +543,7 @@ ContiguousFieldDataManager::~ContiguousFieldDataManager() { for (size_t i=0;ideallocate(m_field_raw_data[i], m_num_bytes_allocated_per_field[i]); m_field_raw_data[i] = nullptr; m_num_bytes_allocated_per_field[i] = 0; m_num_bytes_used_per_field[i] = 0; @@ -649,7 +659,7 @@ void ContiguousFieldDataManager::reorder_bucket_field_data(EntityRank rank, cons FieldMetaDataVector &oldMetaData = const_cast(fields[field_index]->get_meta_data_for_field()); unsigned field_ordinal = fields[field_index]->mesh_meta_data_ordinal(); const size_t newFieldSize = m_num_bytes_used_per_field[field_ordinal] + m_extra_capacity; - unsigned char* new_field_data = (*field_data_allocator).allocate(newFieldSize); + unsigned char* new_field_data = m_fieldDataAllocator->allocate(newFieldSize); FieldMetaData fieldMeta; FieldMetaDataVector newMetaData(reorderedBucketIds.size(), fieldMeta); @@ -677,7 +687,7 @@ void ContiguousFieldDataManager::reorder_bucket_field_data(EntityRank rank, cons } } - (*field_data_allocator).deallocate(m_field_raw_data[field_ordinal], m_num_bytes_allocated_per_field[field_ordinal]); + m_fieldDataAllocator->deallocate(m_field_raw_data[field_ordinal], m_num_bytes_allocated_per_field[field_ordinal]); STK_ThrowRequire(new_offset == m_num_bytes_used_per_field[field_ordinal]); m_num_bytes_allocated_per_field[field_ordinal] = newFieldSize; m_field_raw_data[field_ordinal] = new_field_data; @@ -763,7 +773,7 @@ void ContiguousFieldDataManager::allocate_field_data(EntityRank rank, } m_num_bytes_allocated_per_field[field_ordinal] += m_num_bytes_used_per_field[field_ordinal]; - m_field_raw_data[field_ordinal] = (*field_data_allocator).allocate(m_num_bytes_allocated_per_field[field_ordinal]); + m_field_raw_data[field_ordinal] = m_fieldDataAllocator->allocate(m_num_bytes_allocated_per_field[field_ordinal]); size_t offset = 0; for(size_t i = 0; i < buckets.size(); ++i) @@ -886,12 +896,12 @@ void ContiguousFieldDataManager::reallocate_field_data(EntityRank rank, const st const size_t newFieldBucketAllocationSize = newOffsetForBucket.back() + m_extra_capacity; if (newFieldBucketAllocationSize > oldFieldBucketAllocationSize) { - unsigned char* newAllocationAllBuckets = (*field_data_allocator).allocate(newFieldBucketAllocationSize); + unsigned char* newAllocationAllBuckets = m_fieldDataAllocator->allocate(newFieldBucketAllocationSize); const unsigned char* oldAllocationAllBuckets = m_field_raw_data[fieldOrdinal]; copy_bucket_data_from_old_to_new_field(oldOffsetForBucket, newOffsetForBucket, oldAllocationAllBuckets, newAllocationAllBuckets); - (*field_data_allocator).deallocate(m_field_raw_data[fieldOrdinal], oldFieldBucketAllocationSize); + m_fieldDataAllocator->deallocate(m_field_raw_data[fieldOrdinal], oldFieldBucketAllocationSize); m_field_raw_data[fieldOrdinal] = newAllocationAllBuckets; m_num_bytes_allocated_per_field[fieldOrdinal] = newFieldBucketAllocationSize; @@ -946,7 +956,7 @@ void ContiguousFieldDataManager::add_field_data_for_entity(const std::vectorallocate(newFieldSize); resetFieldMetaDataPointers(0, dst_bucket_id, field_meta_data_vector, m_field_raw_data[field_ordinal], new_field_data); } @@ -973,7 +983,7 @@ void ContiguousFieldDataManager::add_field_data_for_entity(const std::vectordeallocate(m_field_raw_data[field_ordinal], m_num_bytes_allocated_per_field[field_ordinal]); m_num_bytes_allocated_per_field[field_ordinal] = newFieldSize; } else diff --git a/packages/stk/stk_mesh/stk_mesh/base/FieldDataManager.hpp b/packages/stk/stk_mesh/stk_mesh/base/FieldDataManager.hpp index 4770f06019ba..1a51d9dd826c 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/FieldDataManager.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/FieldDataManager.hpp @@ -83,12 +83,9 @@ struct BucketFieldSegment { class FieldDataManager { public: - FieldDataManager(unsigned alignmentIncrementBytes, AllocatorAdaptorInterface * allocatorAdaptor = nullptr) - : m_defaultAllocator(), - field_data_allocator((allocatorAdaptor) ? allocatorAdaptor : &m_defaultAllocator), - alignment_increment_bytes(alignmentIncrementBytes) - {} - + FieldDataManager(unsigned alignmentIncrementBytes, + std::unique_ptr allocatorAdaptor = + std::unique_ptr()); virtual ~FieldDataManager() = default; virtual void allocate_bucket_field_data(const EntityRank rank, const std::vector< FieldBase * > & field_set, @@ -111,15 +108,15 @@ class FieldDataManager unsigned get_alignment_bytes() const { return alignment_increment_bytes; } protected: - AllocatorAdaptor<::stk::impl::FieldDataAllocator> m_defaultAllocator; - AllocatorAdaptorInterface * field_data_allocator; + std::unique_ptr m_fieldDataAllocator; size_t alignment_increment_bytes; }; class DefaultFieldDataManager : public FieldDataManager { public: - DefaultFieldDataManager(const unsigned num_ranks, unsigned alignmentIncrementBytes = stk::impl::DEFAULT_FIELD_ALIGNMENT_BYTES) + DefaultFieldDataManager(const unsigned num_ranks, + unsigned alignmentIncrementBytes = stk::impl::DEFAULT_FIELD_ALIGNMENT_BYTES) : FieldDataManager(alignmentIncrementBytes), m_field_raw_data(num_ranks), m_bucketCapacity(num_ranks), @@ -128,9 +125,9 @@ class DefaultFieldDataManager : public FieldDataManager } DefaultFieldDataManager(const unsigned num_ranks, - AllocatorAdaptorInterface* userSpecifiedAllocatorAdaptor, + std::unique_ptr userSpecifiedAllocatorAdaptor, unsigned alignmentIncrementBytes) - : FieldDataManager(alignmentIncrementBytes, userSpecifiedAllocatorAdaptor), + : FieldDataManager(alignmentIncrementBytes, std::move(userSpecifiedAllocatorAdaptor)), m_field_raw_data(num_ranks), m_bucketCapacity(num_ranks), m_num_bytes_allocated_per_field() diff --git a/packages/stk/stk_mesh/stk_mesh/base/FieldParallel.cpp b/packages/stk/stk_mesh/stk_mesh/base/FieldParallel.cpp index 931e4f2dce97..2a3816a7f2a6 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/FieldParallel.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/FieldParallel.cpp @@ -96,16 +96,6 @@ void do_initial_sync_to_host(const std::vector& fields, Select } //---------------------------------------------------------------------- -#ifndef STK_HIDE_DEPRECATED_CODE //delete after Aug 2023 -STK_DEPRECATED_MSG("Unnecessary sync argument removed. Use communicate_field_data(ghosts, fields).") -void communicate_field_data(const Ghosting& ghosts , - const std::vector< const FieldBase *> & fields, - bool syncOnlySharedOrGhosted) -{ - communicate_field_data(ghosts, fields); -} -#endif - void communicate_field_data(const Ghosting& ghosts, const std::vector& fields) { if ( fields.empty() ) { return; } @@ -232,16 +222,6 @@ void communicate_field_data(const Ghosting& ghosts, const std::vector & fields, - bool syncOnlySharedOrGhosted) -{ - communicate_field_data(mesh, fields); -} -#endif - void communicate_field_data(const BulkData& mesh , const std::vector< const FieldBase *> & fields) { diff --git a/packages/stk/stk_mesh/stk_mesh/base/FieldParallel.hpp b/packages/stk/stk_mesh/stk_mesh/base/FieldParallel.hpp index 80bd4e71ed7c..6b62f6b3adb0 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/FieldParallel.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/FieldParallel.hpp @@ -60,19 +60,6 @@ namespace mesh { * For entities that are ghosted, this function updates field-data from the * original entity to the ghosts. */ -#ifndef STK_HIDE_DEPRECATED_CODE //delete after Aug 2023 -STK_DEPRECATED_MSG("Unnecessary sync argument removed. Use communicate_field_data(ghosts, fields).") -void communicate_field_data( - const Ghosting & ghosts , - const std::vector< const FieldBase *> & fields , - bool syncOnlySharedOrGhosted); - -STK_DEPRECATED_MSG("Unnecessary sync argument removed. Use communicate_field_data(mesh, fields).") -void communicate_field_data(const BulkData& mesh , - const std::vector< const FieldBase *> & fields , - bool syncOnlySharedOrGhosted); -#endif - void communicate_field_data(const Ghosting& ghosts, const std::vector& fields); void communicate_field_data(const BulkData& mesh, const std::vector& fields); @@ -87,17 +74,6 @@ void copy_owned_to_shared( const BulkData& mesh, communicate_field_data(*mesh.ghostings()[BulkData::SHARED], fields); } -#ifndef STK_HIDE_DEPRECATED_CODE //delete after Aug 2023 -STK_DEPRECATED_MSG("Unnecessary sync argument removed. Use copy_owned_to_shared(mesh, fields).") -inline -void copy_owned_to_shared( const BulkData& mesh, - const std::vector< const FieldBase *> & fields, - bool syncOnlySharedOrGhosted) -{ - copy_owned_to_shared(mesh, fields); -} -#endif - //---------------------------------------------------------------------- /** Sum/Max/Min (assemble) field-data for the specified fields on shared entities such that each shared entity diff --git a/packages/stk/stk_mesh/stk_mesh/base/MeshBuilder.cpp b/packages/stk/stk_mesh/stk_mesh/base/MeshBuilder.cpp index 4807306da763..747a118571bb 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/MeshBuilder.cpp +++ b/packages/stk/stk_mesh/stk_mesh/base/MeshBuilder.cpp @@ -45,7 +45,6 @@ MeshBuilder::MeshBuilder() m_haveComm(false), m_auraOption(BulkData::AUTO_AURA), m_addFmwkData(false), - m_fieldDataManager(nullptr), m_initialBucketCapacity(stk::mesh::get_default_initial_bucket_capacity()), m_maximumBucketCapacity(stk::mesh::get_default_maximum_bucket_capacity()), m_spatialDimension(0), @@ -59,7 +58,6 @@ MeshBuilder::MeshBuilder(ParallelMachine comm) m_haveComm(true), m_auraOption(BulkData::AUTO_AURA), m_addFmwkData(false), - m_fieldDataManager(nullptr), m_initialBucketCapacity(stk::mesh::get_default_initial_bucket_capacity()), m_maximumBucketCapacity(stk::mesh::get_default_maximum_bucket_capacity()), m_spatialDimension(0), @@ -99,9 +97,17 @@ MeshBuilder& MeshBuilder::set_add_fmwk_data(bool addFmwkData) return *this; } -MeshBuilder& MeshBuilder::set_field_data_manager(FieldDataManager* fieldDataManager) +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after 2023-09-27 +STK_DEPRECATED MeshBuilder& MeshBuilder::set_field_data_manager(FieldDataManager* fieldDataManager) { - m_fieldDataManager = fieldDataManager; + m_fieldDataManager = std::unique_ptr(fieldDataManager); + return *this; +} +#endif + +MeshBuilder& MeshBuilder::set_field_data_manager(std::unique_ptr fieldDataManager) +{ + m_fieldDataManager = std::move(fieldDataManager); return *this; } @@ -157,7 +163,7 @@ std::unique_ptr MeshBuilder::create(std::shared_ptr metaData #ifdef SIERRA_MIGRATION m_addFmwkData, #endif - m_fieldDataManager, + std::move(m_fieldDataManager), m_initialBucketCapacity, m_maximumBucketCapacity, create_aura_ghosting(), diff --git a/packages/stk/stk_mesh/stk_mesh/base/MeshBuilder.hpp b/packages/stk/stk_mesh/stk_mesh/base/MeshBuilder.hpp index 35304021b82b..0a6aedfb72dc 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/MeshBuilder.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/MeshBuilder.hpp @@ -48,6 +48,7 @@ class MeshBuilder public: MeshBuilder(); explicit MeshBuilder(ParallelMachine comm); + virtual ~MeshBuilder() = default; MeshBuilder& set_spatial_dimension(unsigned spatialDimension); MeshBuilder& set_entity_rank_names(const std::vector& entityRankNames); @@ -55,7 +56,11 @@ class MeshBuilder MeshBuilder& set_communicator(ParallelMachine comm); MeshBuilder& set_aura_option(BulkData::AutomaticAuraOption auraOption); MeshBuilder& set_add_fmwk_data(bool addFmwkData); - MeshBuilder& set_field_data_manager(FieldDataManager* fieldDataManager); +#ifndef STK_HIDE_DEPRECATED_CODE // Delete after 2023-09-27 + // This deprecated function now claims ownership of the argument + STK_DEPRECATED MeshBuilder& set_field_data_manager(FieldDataManager* fieldDataManager); +#endif + MeshBuilder& set_field_data_manager(std::unique_ptr fieldDataManager); MeshBuilder& set_bucket_capacity(unsigned bucketCapacity); MeshBuilder& set_initial_bucket_capacity(unsigned initialCapacity); @@ -63,19 +68,19 @@ class MeshBuilder MeshBuilder& set_upward_connectivity(bool onOrOff); - std::unique_ptr create(); - std::unique_ptr create(std::shared_ptr metaData); + virtual std::unique_ptr create(); + virtual std::unique_ptr create(std::shared_ptr metaData); - std::shared_ptr create_meta_data(); + virtual std::shared_ptr create_meta_data(); -private: +protected: std::shared_ptr create_aura_ghosting(); ParallelMachine m_comm; bool m_haveComm; BulkData::AutomaticAuraOption m_auraOption; bool m_addFmwkData; - FieldDataManager* m_fieldDataManager; + std::unique_ptr m_fieldDataManager; unsigned m_initialBucketCapacity; unsigned m_maximumBucketCapacity; unsigned m_spatialDimension; diff --git a/packages/stk/stk_mesh/stk_mesh/base/NgpFieldParallel.hpp b/packages/stk/stk_mesh/stk_mesh/base/NgpFieldParallel.hpp index 76449891237c..52161e271ea1 100644 --- a/packages/stk/stk_mesh/stk_mesh/base/NgpFieldParallel.hpp +++ b/packages/stk/stk_mesh/stk_mesh/base/NgpFieldParallel.hpp @@ -75,18 +75,6 @@ void do_final_sync_to_device(const std::vector*>& ngpFields) } } -#ifndef STK_HIDE_DEPRECATED_CODE //delete after Aug 2023 -template -STK_DEPRECATED_MSG("Unnecessary sync argument removed. Use copy_owned_to_shared(bulk, ngpFields, doFinalSyncBackToDevice).") -void copy_owned_to_shared(const stk::mesh::BulkData & bulk, - const std::vector *> & ngpFields, - bool doFinalSyncBackToDevice, - bool syncOnlySharedOrGhosted) -{ - copy_owned_to_shared(bulk, ngpFields, doFinalSyncBackToDevice); -} -#endif - template void copy_owned_to_shared(const stk::mesh::BulkData & bulk, const std::vector *> & ngpFields, @@ -107,18 +95,6 @@ void copy_owned_to_shared(const stk::mesh::BulkData & bulk, } } -#ifndef STK_HIDE_DEPRECATED_CODE //delete after Aug 2023 -template -STK_DEPRECATED_MSG("Unnecessary sync argument removed. Use communicate_field_data(ghosts, ngpFields, doFinalSyncBackToDevice).") -void communicate_field_data(const stk::mesh::Ghosting & ghosting, - const std::vector *> & ngpFields, - bool doFinalSyncBackToDevice, - bool syncOnlySharedOrGhosted) -{ - communicate_field_data(ghosting, ngpFields, doFinalSyncBackToDevice); -} -#endif - template void communicate_field_data(const stk::mesh::Ghosting & ghosting, const std::vector *> & ngpFields, @@ -139,18 +115,6 @@ void communicate_field_data(const stk::mesh::Ghosting & ghosting, } } -#ifndef STK_HIDE_DEPRECATED_CODE //delete after Aug 2023 -template -STK_DEPRECATED_MSG("Unnecessary sync argument removed. Use communicate_field_data(bulk, ngpFields, doFinalSyncBackToDevice).") -void communicate_field_data(const stk::mesh::BulkData & bulk, - const std::vector *> & ngpFields, - bool doFinalSyncBackToDevice, - bool syncOnlySharedOrGhosted) -{ - communicate_field_data(bulk, ngpFields, doFinalSyncBackToDevice); -} -#endif - template void communicate_field_data(const stk::mesh::BulkData & bulk, const std::vector *> & ngpFields, @@ -297,7 +261,7 @@ void parallel_sum_device_mpi(const stk::mesh::NgpMesh& ngpMesh, const std::vecto } const bool deterministic = false; - stk::mesh::ngp_parallel_data_exchange_sym_pack_unpack(MPI_COMM_WORLD, + stk::mesh::ngp_parallel_data_exchange_sym_pack_unpack(bulk.parallel(), comm_procs, exchangeHandler, deterministic); diff --git a/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshImplUtils.cpp b/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshImplUtils.cpp index 02e15599b6d3..f4e2c55a94f2 100644 --- a/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshImplUtils.cpp +++ b/packages/stk/stk_mesh/stk_mesh/baseImpl/MeshImplUtils.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include @@ -1314,7 +1315,8 @@ void comm_sync_nonowned_sends( Entity const e = mesh.get_entity( entity_key ); STK_ThrowAssert(parallel_rank != proc); - STK_ThrowAssert(mesh.is_valid(e)); + STK_ThrowAssertMsg(mesh.is_valid(e), "comm_sync_nonowned_sends mod-cycle="<remote_owned_closure); if(remotely_destroyed || not_shared_remotely) { + if (remotely_destroyed && remote_proc == owner) { + m_bulkData.entity_comm_map_clear_ghosting(key); + } + m_bulkData.entity_comm_map_erase(key, EntityCommInfo(stk::mesh::BulkData::SHARED, remote_proc)); if (!locally_destroyed && not_shared_remotely) { entitiesToRemoveFromSharing.push_back(EntityProc(entity, remote_proc)); @@ -631,12 +633,18 @@ void MeshModification::internal_resolve_ghosted_modify_delete(const std::vector< void MeshModification::add_entity_to_same_ghosting(Entity entity, Entity connectedGhost) { + std::vector to_insert; for(PairIterEntityComm ec(m_bulkData.internal_entity_comm_map(connectedGhost)); ! ec.empty(); ++ec) { if (ec->ghost_id > BulkData::AURA) { - m_bulkData.entity_comm_map_insert(entity, EntityCommInfo(ec->ghost_id, ec->proc)); - m_bulkData.entity_comm_list_insert(entity); + to_insert.emplace_back(ec->ghost_id, ec->proc); } } + if(!to_insert.empty()) { + m_bulkData.entity_comm_list_insert(entity); + for(const auto & entry : to_insert) { + m_bulkData.entity_comm_map_insert(entity, EntityCommInfo(entry.ghost_id, entry.proc)); + } + } } void MeshModification::internal_resolve_formerly_shared_entities(const EntityVector& entitiesNoLongerShared) diff --git a/packages/stk/stk_mesh/stk_mesh/baseImpl/check_comm_list.cpp b/packages/stk/stk_mesh/stk_mesh/baseImpl/check_comm_list.cpp index 4a62d1c81f0e..7412548538d5 100644 --- a/packages/stk/stk_mesh/stk_mesh/baseImpl/check_comm_list.cpp +++ b/packages/stk/stk_mesh/stk_mesh/baseImpl/check_comm_list.cpp @@ -89,7 +89,10 @@ void fill_expected_recv_data(const stk::mesh::BulkData& mesh, { for(const EntityCommListInfo& commInfo : comm_list) { STK_ThrowAssertMsg(commInfo.entity_comm != -1,"P"< + $ +) -IF(CDT_INCLUDE_DIRS) +if(CDT_INCLUDE_DIRS) SET(CDT_INC_DIR "${CDT_INCLUDE_DIRS}") -ELSE() +else() message(STATUS "CDT_INCLUDE_DIRS not set, trying get_target_property...") get_target_property(CDT_INC_DIR CDT::CDT INTERFACE_INCLUDE_DIRECTORIES) -ENDIF() +endif() target_include_directories(stk_middle_mesh PUBLIC ${CDT_INC_DIR}) @@ -75,3 +80,6 @@ INSTALL(FILES ${HEADERS} DESTINATION INSTALL(FILES ${PRED_HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_middle_mesh/predicates) +if(NOT HAVE_STK_Trilinos) + INSTALL(TARGETS stk_middle_mesh DESTINATION ${STK_INSTALL_LIBDIR}) +endif() diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/adjacency_search.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/adjacency_search.cpp index 1362b9d130cb..f93f1c0cea37 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/adjacency_search.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/adjacency_search.cpp @@ -91,6 +91,11 @@ void AdjacencySearch::get_next(mesh::MeshEntityPtr el1, mesh::MeshEntityPtr el2, // get element that contains the centroid of el1 el2 = get_mesh2_element(el1, el2); + if (m_output) + { + std::cout << "new el2 = " << el2 << std::endl; + } + mesh::MeshEntityPtr v, edge2; if ((v = m_preds.any_vertices_contained(el1, el2))) { diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/application_interface_fake_parallel_impl.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/application_interface_fake_parallel_impl.cpp index fd561ca40993..7367c5c2eb5f 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/application_interface_fake_parallel_impl.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/application_interface_fake_parallel_impl.cpp @@ -397,15 +397,16 @@ std::pair get_comm_sizes_on_root(MPI_Comm comm1, MPI_Comm comm2, MPI_C MPI_Request sendReq1 = MPI_REQUEST_NULL, sendReq2 = MPI_REQUEST_NULL, recvReq1 = MPI_REQUEST_NULL, recvReq2 = MPI_REQUEST_NULL; + int mesh1CommSizeSend = 0, mesh2CommSizeSend = 0; if (comm1 != MPI_COMM_NULL && utils::impl::comm_rank(comm1) == 0) { - int mesh1CommSizeSend = utils::impl::comm_size(comm1); + mesh1CommSizeSend = utils::impl::comm_size(comm1); MPI_Isend(&mesh1CommSizeSend, 1, MPI_INT, rootRank, tag1, unionComm, &sendReq1); } if (comm2 != MPI_COMM_NULL && utils::impl::comm_rank(comm2) == 0) { - int mesh2CommSizeSend = utils::impl::comm_size(comm2); + mesh2CommSizeSend = utils::impl::comm_size(comm2); MPI_Isend(&mesh2CommSizeSend, 1, MPI_INT, rootRank, tag2, unionComm, &sendReq2); } diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/bounding_box_search.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/bounding_box_search.hpp index 4683c4a2e1f0..07c9a01d4b9e 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/bounding_box_search.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/bounding_box_search.hpp @@ -47,6 +47,8 @@ #include #include +#include "search_mesh_element_bounding_box.hpp" +#include "search_mesh_vertex.hpp" namespace stk { namespace middle_mesh { @@ -99,6 +101,12 @@ bool local_is_sorted(ForwardIterator first, ForwardIterator last, Compare compar } // namespace impl +enum class SplitCommColor { + RECV = 0, + SEND, + INVALID +}; + template class BoundingBoxSearchType { public: @@ -110,7 +118,6 @@ template class BoundingBoxSearchType using EntityProcB = typename MeshB::EntityProc; using EntityProcRelation = std::pair; using EntityProcRelationVec = std::vector; - using EntityKeyMap = std::multimap; }; template class BoundingBoxSearch @@ -125,7 +132,6 @@ public : using MeshB = typename SearchClass::MeshB; using EntityKeyA = typename SearchClass::EntityKeyA; using EntityKeyB = typename SearchClass::EntityKeyB; - using EntityKeyMap = typename SearchClass::EntityKeyMap; using EntityProcA = typename SearchClass::EntityProcA; using EntityProcB = typename SearchClass::EntityProcB; @@ -146,6 +152,7 @@ public : std::shared_ptr recvMesh, const std::string &name, stk::ParallelMachine pm, + const bool doParallelSearch=true, const double expansion_factor = 1.5, const double expansion_sum = 0.0, const stk::search::SearchMethod search_method = stk::search::KDTREE); @@ -168,6 +175,7 @@ private : std::shared_ptr m_recvMesh; const std::string m_name; + const bool m_doParallelSearch; const double m_expansion_factor; const double m_expansion_sum; const stk::search::SearchMethod m_search_method; @@ -204,10 +212,12 @@ template BoundingBoxSearch::BoundingBoxSearch std::shared_ptr recvMesh, const std::string &name, stk::ParallelMachine pm, + const bool doParallelSearch, const double expansion_factor, const double expansion_sum, const stk::search::SearchMethod search_method) : - m_sendMesh(sendMesh), m_recvMesh(recvMesh), m_name(name), m_expansion_factor(expansion_factor), + m_sendMesh(sendMesh), m_recvMesh(recvMesh), m_name(name), m_doParallelSearch(doParallelSearch), + m_expansion_factor(expansion_factor), m_expansion_sum(expansion_sum), m_search_method(search_method), m_shared_comm(pm) { //In an mpmd program, there's no guarantee that the types specified for the entity keys are honored by each program, @@ -279,7 +289,7 @@ template void BoundingBoxSearch::coarse_search() // in coarse_search call, but really, this is what we want. EntityProcRelationVec rng_to_dom; EntityProcRelationVec& rng_to_dom_vec = not_empty_count == 0 ? m_global_range_to_domain : rng_to_dom; - stk::search::coarse_search(range_vector, domain_vector, m_search_method, m_shared_comm, rng_to_dom_vec); + stk::search::coarse_search(range_vector, domain_vector, m_search_method, m_shared_comm, rng_to_dom_vec, m_doParallelSearch); if(not_empty_count > 0) { m_global_range_to_domain.insert(m_global_range_to_domain.end(), rng_to_dom_vec.begin(), rng_to_dom_vec.end()); @@ -328,6 +338,14 @@ template void BoundingBoxSearch::coarse_search() std::sort(m_global_range_to_domain.begin(), m_global_range_to_domain.end()); } + +using ElementToElementBoundingBoxSearch = + BoundingBoxSearch>; + +using ElementToVertBoundingBoxSearch = + BoundingBoxSearch>; } } } diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/destination_field_gatherer.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/destination_field_gatherer.cpp index c676a91b5749..08c1393f43a2 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/destination_field_gatherer.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/destination_field_gatherer.cpp @@ -40,19 +40,21 @@ VariableSizeFieldPtr DestinationFieldGatherer::gather_vert_and_edge_destina void DestinationFieldGatherer::get_local_destinations_and_pack_buffers(Exchanger& exchanger, int dim, VariableSizeFieldPtr fieldPtr) { - int myRank = utils::impl::comm_rank(m_mesh->get_comm()); std::vector destRanks; for (auto& entity : m_mesh->get_mesh_entities(dim)) if (entity) { get_destinations_from_scatterspec(entity, destRanks); - RemoteSharedEntity owner = get_owner_remote(m_mesh, entity); - if (owner.remoteRank != myRank) + + for (int i=0; i < entity->count_remote_shared_entities(); ++i) { - exchanger.get_send_buf(owner.remoteRank).pack(owner.remoteId); - exchanger.get_send_buf(owner.remoteRank).pack(destRanks); - } else if (fieldPtr) + RemoteSharedEntity remote = entity->get_remote_shared_entity(i); + exchanger.get_send_buf(remote.remoteRank).pack(remote.remoteId); + exchanger.get_send_buf(remote.remoteRank).pack(destRanks); + } + + if (fieldPtr) { for (auto& destRank : destRanks) fieldPtr->insert(entity, 0, destRank); @@ -98,8 +100,6 @@ void DestinationFieldGatherer::unpack_buffer(int rank, int dim, stk::CommBuffer& { field.insert(entity, 0, destRank); } - - } } diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/element_mesh_classifier.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/element_mesh_classifier.cpp index 65cb0237d7b2..222f999528f3 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/element_mesh_classifier.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/element_mesh_classifier.cpp @@ -11,32 +11,28 @@ namespace impl { using predicates::impl::PointClassification; -void ElementMeshClassifier::classify(ElementMeshData& elementMeshData, int numConstraintEdges) +void ElementMeshClassifier::classify(ElementMeshData& elementMeshData, int numConstraintEdges, const std::vector& mesh2Els) { m_elementMeshData = elementMeshData; + m_mesh2Els = mesh2Els; get_mesh2_element_classification(numConstraintEdges); } void ElementMeshClassifier::get_mesh2_element_classification(int numConstraintEdges) { - auto& mesh1ElToMesh2Els = *(m_relationalData->mesh1ElementsToMesh2Elements); - std::vector mesh2Els; - for (int i = 0; i < mesh1ElToMesh2Els.get_num_comp(m_elementMeshData.el1, 0); ++i) - mesh2Els.push_back(mesh1ElToMesh2Els(m_elementMeshData.el1, 0, i)); - if (M_OUTPUT) { mesh::impl::print_vert_edges("mesh1_el", {m_elementMeshData.el1}); - mesh::impl::print_vert_edges("mesh2_els", mesh2Els); + mesh::impl::print_vert_edges("mesh2_els", m_mesh2Els); std::cout << "\nclassifying mesh" << std::endl; } // printTIN("mesh_class", class_i.mesh_in); // handle special case - if (mesh2Els.size() == 1) + if (m_mesh2Els.size() == 1) { - classify_all(mesh2Els[0]); + classify_all(m_mesh2Els[0]); return; } @@ -45,7 +41,7 @@ void ElementMeshClassifier::get_mesh2_element_classification(int numConstraintEd if (!need_numerical_classification()) { std::vector els2List; - mesh::MeshEntityPtr el22 = classification_pass1(mesh2Els, agg, els2List); + mesh::MeshEntityPtr el22 = classification_pass1(m_mesh2Els, agg, els2List); if (el22) { @@ -65,7 +61,7 @@ void ElementMeshClassifier::get_mesh2_element_classification(int numConstraintEd //----------------------------------------------------------------- // if there are any remaining mesh_in elements not classified, fall back // to using PointClassifier to determine the status of the third vertex - classification_pass3(mesh2Els, agg); + classification_pass3(m_mesh2Els, agg); } // std::cout << "\nfinished classification" << std::endl; @@ -131,12 +127,8 @@ mesh::impl::MeshAgglomerator ElementMeshClassifier::setup_agglomerations(int num void ElementMeshClassifier::get_mesh2_edges(std::vector& edges2) { edges2.clear(); - mesh::MeshEntityPtr el1 = m_elementMeshData.el1; - - auto& mesh1ElToMesh2Els = *(m_relationalData->mesh1ElementsToMesh2Elements); - for (int i = 0; i < mesh1ElToMesh2Els.get_num_comp(el1, 0); ++i) + for (auto& el2 : m_mesh2Els) { - mesh::MeshEntityPtr el2 = mesh1ElToMesh2Els(el1, 0, i); for (int j = 0; j < el2->count_down(); ++j) edges2.push_back(el2->get_down(j)); } @@ -188,10 +180,13 @@ bool ElementMeshClassifier::need_numerical_classification() { FakeVert fv = edges2ToFakeVertsIn(edge2, 0, i).vert; mesh::MeshEntityPtr vertIn = fakeVertsToRealVertsIn[fv.id]; - const predicates::impl::PointRecord& record = vertsInClassOnMesh1(vertIn, 0, 0); - if (is_classified_on_element(record)) + if (vertIn) { - vertsFromEdges2.insert(vertIn); + const predicates::impl::PointRecord& record = vertsInClassOnMesh1(vertIn, 0, 0); + if (is_classified_on_element(record)) + { + vertsFromEdges2.insert(vertIn); + } } } } @@ -516,10 +511,13 @@ void ElementMeshClassifier::get_verts_in(mesh::MeshEntityPtr el1, mesh::MeshEnti { FakeVert fv = edges2ToFakeVertsIn(edgeI, 0, j).vert; mesh::MeshEntityPtr vertIn = fakeVertsToRealVertsIn[fv.id]; - const predicates::impl::PointRecord& record = vertsInClassOnMesh1(vertIn, 0, 0); + if (vertIn) + { + const predicates::impl::PointRecord& record = vertsInClassOnMesh1(vertIn, 0, 0); - if (is_classified_on_element(record)) - vertsIn.insert(vertIn); + if (is_classified_on_element(record)) + vertsIn.insert(vertIn); + } } } } diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/element_mesh_classifier.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/element_mesh_classifier.hpp index 797157dad11a..1f6c886271a2 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/element_mesh_classifier.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/element_mesh_classifier.hpp @@ -3,6 +3,7 @@ #include "element_mesh_extractor.hpp" #include "mesh_agglomerator.hpp" +#include "mesh_entity.hpp" #include "mesh_relational_data.hpp" #include "predicates/point_classifier_normal_wrapper.hpp" @@ -25,7 +26,7 @@ class ElementMeshClassifier , m_classifier(classifier) {} - void classify(ElementMeshData& elementMeshData, int numConstraintEdges); + void classify(ElementMeshData& elementMeshData, int numConstraintEdges, const std::vector& mesh2Els); private: template @@ -78,6 +79,7 @@ class ElementMeshClassifier std::shared_ptr m_relationalData; std::shared_ptr m_classifier; ElementMeshData m_elementMeshData; + std::vector m_mesh2Els; std::vector m_agglomerationsClassified; static constexpr bool M_OUTPUT = false; }; diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/entity_sorted_by_owner.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/entity_sorted_by_owner.cpp deleted file mode 100644 index ced3c80959fd..000000000000 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/entity_sorted_by_owner.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "entity_sorted_by_owner.hpp" - -namespace stk { -namespace middle_mesh { -namespace mesh { -namespace impl { - -EntitySortedByOwner::EntitySortedByOwner(int numOwnerRanks) : - m_ownerLocalIds(numOwnerRanks), - m_localEntities(numOwnerRanks) -{} - -void EntitySortedByOwner::insert(const RemoteSharedEntity& owner, MeshEntityPtr localEntity) -{ - auto& ownerLocalIds = m_ownerLocalIds[owner.remoteRank]; - auto& localEntities = m_localEntities[owner.remoteRank]; - - if (ownerLocalIds.size() > 0) - { - assert(ownerLocalIds.size() > 0 && owner.remoteId > ownerLocalIds.back()); - } - - ownerLocalIds.push_back(owner.remoteId); - localEntities.push_back(localEntity); -} - -// returns the entity associated with owner, or nullptr if no such entity; -MeshEntityPtr EntitySortedByOwner::get_entity(const RemoteSharedEntity& owner) -{ - auto& ownerLocalIds = m_ownerLocalIds[owner.remoteRank]; - auto& localEntities = m_localEntities[owner.remoteRank]; - - auto it = std::lower_bound(ownerLocalIds.begin(), ownerLocalIds.end(), owner.remoteId); - MeshEntityPtr entity = nullptr; - if (it != ownerLocalIds.end() && *it == owner.remoteId) - entity = localEntities[std::distance(ownerLocalIds.begin(), it)]; - - return entity; -} - -} -} -} -} diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/entity_sorted_by_owner.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/entity_sorted_by_owner.hpp index e25b5387a702..669c3eb4271b 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/entity_sorted_by_owner.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/entity_sorted_by_owner.hpp @@ -8,23 +8,61 @@ namespace middle_mesh { namespace mesh { namespace impl { -class EntitySortedByOwner +template +class ValuesSortedByOwner { public: - explicit EntitySortedByOwner(int numOwnerRanks); + explicit ValuesSortedByOwner(int numOwnerRanks); - void insert(const RemoteSharedEntity& owner, MeshEntityPtr localEntity); + void insert(const RemoteSharedEntity& owner, const T& val); - // returns the entity associated with owner, or nullptr if no such entity; - MeshEntityPtr get_entity(const RemoteSharedEntity& owner); + // returns the value associated with owner, or defaultVal if no such owner + const T& get_value(const RemoteSharedEntity& owner, const T& defaultVal = T{}); - const std::vector& get_entities(int rank) const { return m_localEntities[rank]; } + const std::vector& get_values(int rank) const { return m_values[rank]; } private: std::vector> m_ownerLocalIds; - std::vector> m_localEntities; + std::vector> m_values; }; + +template +ValuesSortedByOwner::ValuesSortedByOwner(int numOwnerRanks) : + m_ownerLocalIds(numOwnerRanks), + m_values(numOwnerRanks) +{} + +template +void ValuesSortedByOwner::insert(const RemoteSharedEntity& owner, const T& val) +{ + auto& ownerLocalIds = m_ownerLocalIds[owner.remoteRank]; + auto& localEntities = m_values[owner.remoteRank]; + + if (ownerLocalIds.size() > 0) + { + assert(ownerLocalIds.size() > 0 && owner.remoteId > ownerLocalIds.back()); + } + + ownerLocalIds.push_back(owner.remoteId); + localEntities.push_back(val); +} + +template +const T& ValuesSortedByOwner::get_value(const RemoteSharedEntity& owner, const T& defaultVal) +{ + auto& ownerLocalIds = m_ownerLocalIds[owner.remoteRank]; + auto& localEntities = m_values[owner.remoteRank]; + + auto it = std::lower_bound(ownerLocalIds.begin(), ownerLocalIds.end(), owner.remoteId); + if (it != ownerLocalIds.end() && *it == owner.remoteId) + return localEntities[std::distance(ownerLocalIds.begin(), it)]; + + return defaultVal; +} + +using EntitySortedByOwner = ValuesSortedByOwner; + } } } diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.cpp index 09771267a84b..ae20ae6ff663 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.cpp @@ -4,6 +4,7 @@ #include #include +#include "mesh_entity.hpp" #include "newton2.hpp" #include "parallel_exchange.hpp" #include "stk_util/parallel/DataExchangeUnknownPatternNonBlockingBuffer.hpp" @@ -530,19 +531,37 @@ int check_angles(std::shared_ptr mesh, const double thetaMin, const double return count; } -void check_topology(std::shared_ptr mesh) +void check_topology(std::shared_ptr mesh, int maxDim) { - check_topology_down(mesh->get_edges(), mesh->get_vertices()); - check_topology_down(mesh->get_elements(), mesh->get_edges()); + if (maxDim >= 1) + { + check_topology_down(mesh->get_edges(), mesh->get_vertices()); + } + + if (maxDim >= 2) + { + check_topology_down(mesh->get_elements(), mesh->get_edges()); + } // check every vertex has at least two upward adjacencies, and edges // have one - check_topology_up(mesh->get_vertices(), mesh->get_edges(), 2); - check_topology_up(mesh->get_edges(), mesh->get_elements(), 1); + if (maxDim >= 1) + { + check_topology_up(mesh->get_vertices(), mesh->get_edges(), 2); + } + + if (maxDim >= 2) + { + check_topology_up(mesh->get_edges(), mesh->get_elements(), 1); + } check_remotes_unique(mesh); check_remotes_symmetric(mesh); - check_edge_orientation_parallel(mesh); + + if (maxDim >= 2) + { + check_edge_orientation_parallel(mesh); + } } void check_topology_down(const std::vector& entities, const std::vector& entitiesDown) @@ -1037,6 +1056,25 @@ int get_owner(std::shared_ptr mesh, MeshEntityPtr entity) return owner; } +int get_local_id(MeshEntityPtr higherDimensionEntity, MeshEntityPtr lowerDimensionEntity) +{ + int outputDim = get_type_dimension(lowerDimensionEntity->get_type()); + assert(get_type_dimension(higherDimensionEntity->get_type()) > outputDim); + + std::array entities; + int nentities = get_downward(higherDimensionEntity, outputDim, entities.data()); + for (int i=0; i < nentities; ++i) + { + if (entities[i] == lowerDimensionEntity) + { + return i; + } + } + + throw std::runtime_error("unable to find entity"); +} + + RemoteSharedEntity get_owner_remote(std::shared_ptr mesh, MeshEntityPtr entity) { int myrank = utils::impl::comm_rank(mesh->get_comm()); @@ -1060,10 +1098,12 @@ bool check_is_entity_owner(std::shared_ptr mesh, MeshEntityPtr entity) RemoteSharedEntity get_remote_shared_entity(MeshEntityPtr entity, int rank) { for (int i = 0; i < entity->count_remote_shared_entities(); ++i) + { if (entity->get_remote_shared_entity(i).remoteRank == rank) { return entity->get_remote_shared_entity(i); } + } throw std::runtime_error("unable to find remote on given rank"); } diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.hpp index 9491f377b2d9..d63417c0bf2e 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh.hpp @@ -137,7 +137,7 @@ int count_valid(const std::vector entities); // [theta_min, theta_max], angles in degrees int check_angles(std::shared_ptr mesh, const double thetaMin, const double thetaMax); -void check_topology(std::shared_ptr mesh); +void check_topology(std::shared_ptr mesh, int maxDim=2); void check_topology_down(const std::vector& entities, const std::vector& entitiesDown); @@ -164,6 +164,8 @@ int get_bridge_adjacent(MeshEntityPtr e, const int viaDim, const int targetDim, int get_owner(std::shared_ptr mesh, MeshEntityPtr entity); +int get_local_id(MeshEntityPtr higherDimensionEntity, MeshEntityPtr lowerDimensionEntity); + RemoteSharedEntity get_owner_remote(std::shared_ptr mesh, MeshEntityPtr entity); bool check_is_entity_owner(std::shared_ptr mesh, MeshEntityPtr entity); diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_entity.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_entity.hpp index 08615886a3cf..444a5af36ae1 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_entity.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_entity.hpp @@ -169,6 +169,7 @@ class MeshEntity void delete_down(MeshEntityPtr e); + //TODO: move definition to header to allow inlining int count_up() const; MeshEntityPtr get_up(const int i) const; @@ -183,7 +184,14 @@ class MeshEntity void delete_up(MeshEntityPtr e) { delete_entity(e, m_up); } - void add_remote_shared_entity(const RemoteSharedEntity& remote) { m_remoteEntities.push_back(remote); } + void add_remote_shared_entity(const RemoteSharedEntity& remote) + { +#ifndef NDEBUG + auto it = std::find(m_remoteEntities.begin(), m_remoteEntities.end(), remote); + assert(it == m_remoteEntities.end()); +#endif + m_remoteEntities.push_back(remote); + } int count_remote_shared_entities() const { return m_remoteEntities.size(); } diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_exchange_boundary_edges.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_exchange_boundary_edges.cpp index 1956278961f5..d3e19087cb05 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_exchange_boundary_edges.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_exchange_boundary_edges.cpp @@ -105,8 +105,8 @@ void MeshExchangeBoundaryEdges::fill_updated_remote_vertices_info(VertexPointExc for (int rank=0; rank < nprocs; ++rank) { auto& sendBuf = exchanger.get_send_buf(rank); - sendBuf.reserve(m_sortedVerticesByOwner.get_entities(rank).size()); - for (auto& vertex : m_sortedVerticesByOwner.get_entities(rank)) + sendBuf.reserve(m_sortedVerticesByOwner.get_values(rank).size()); + for (auto& vertex : m_sortedVerticesByOwner.get_values(rank)) sendBuf.push_back(vertex->get_point_orig(0)); } } @@ -146,8 +146,8 @@ void MeshExchangeBoundaryEdges::add_boundary_edges_on_root(VertexExchange& bound { for (unsigned j = 0; j < recvEdgeBuffer.size(); ++j) { BoundaryEdgeInfo edgeInfo = recvEdgeBuffer[j]; - m_boundaryMesh->create_edge(m_sortedVerticesByOwner.get_entity(edgeInfo.vertex1Owner), - m_sortedVerticesByOwner.get_entity(edgeInfo.vertex2Owner)); + m_boundaryMesh->create_edge(m_sortedVerticesByOwner.get_value(edgeInfo.vertex1Owner), + m_sortedVerticesByOwner.get_value(edgeInfo.vertex2Owner)); } }; diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_projection_calculator.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_projection_calculator.cpp index a6fdab5b0bc6..01b19c0e0490 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_projection_calculator.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_projection_calculator.cpp @@ -2,6 +2,9 @@ #include "adjacency_search.hpp" #include "mesh_entity.hpp" +#include "bounding_box_search.hpp" +#include "variable_size_field.hpp" +#include #include namespace stk { @@ -11,7 +14,7 @@ namespace impl { using predicates::impl::PointClassification; -std::shared_ptr MeshProjectionCalculator::project() +void MeshProjectionCalculator::project() { create_mesh1_fake_verts(); project_mesh2_vertices_onto_mesh1(); @@ -19,8 +22,6 @@ std::shared_ptr MeshProjectionCalculator::project() // sortEdge2FakeVerts(); m_relationalData->fakeVertsToVertsIn.resize(m_fakeVertGen.get_num_verts()); - - return m_meshIn; } void MeshProjectionCalculator::create_mesh1_fake_verts() @@ -43,79 +44,98 @@ void MeshProjectionCalculator::create_mesh1_fake_verts() } } + +mesh::VariableSizeFieldPtr +MeshProjectionCalculator::compute_mesh2_to_mesh1_element_maps() +{ + using SearchMesh1 = mesh::impl::SearchMeshElementBoundingBox; + using SearchMesh2 = mesh::impl::SearchMeshVertex; + using CoarseSearch = search::ElementToVertBoundingBoxSearch; + auto mesh2VertsToMesh1Els = mesh::create_variable_size_field(m_mesh2, mesh::FieldShape(1, 0, 0)); + + + auto searchMesh1 = std::make_shared(m_mesh1, MPI_COMM_SELF); + auto searchMesh2 = std::make_shared(m_mesh2, MPI_COMM_SELF); + + bool doParallelSearch = false; + CoarseSearch coarseSearch(searchMesh1, searchMesh2, "local_coarse_search", MPI_COMM_SELF, doParallelSearch); + coarseSearch.coarse_search(); + + if (coarseSearch.get_unpaired_recv_entities().size() > 0) + { + throw std::runtime_error("local coarse search could not find candidate elements for some vertices"); + } + + const CoarseSearch::EntityProcRelationVec& mesh2To1Relations = coarseSearch.get_range_to_domain(); + for (const CoarseSearch::EntityProcRelation& relation : mesh2To1Relations) + { + SearchMesh2::EntityKey mesh2VertId = relation.first.id(); + SearchMesh1::EntityKey mesh1ElId = relation.second.id(); + mesh::MeshEntityPtr mesh1El = m_mesh1->get_elements()[mesh1ElId]; + mesh::MeshEntityPtr mesh2Vert = m_mesh2->get_vertices()[mesh2VertId]; + mesh2VertsToMesh1Els->insert(mesh2Vert, 0, mesh1El); + } + + + return mesh2VertsToMesh1Els; + +} + void MeshProjectionCalculator::project_mesh2_vertices_onto_mesh1() { if (m_output) std::cout << "\nprojecting mesh1 vertices onto mesh1 " << std::endl; - // TODO: for memory savings, could - // 1. put first result into the regular field - // 2. when second result is found, copy them both to variable size field - // 3. Need to initialize regular field to type == Exterior, so we can detect - // vertices that don't project to anywhere - mesh::VariableSizeFieldPtr verts2AllClassificationsPtr = - mesh::create_variable_size_field(m_mesh2, mesh::FieldShape(1, 0, 0)); - - mesh::MeshEntityPtr el1; - std::vector mesh2Els; - SetType seenVerts; // TODO: this isn't quite what we want: there can - // be multiple equivalent result if a vertex - // projects onto a vertex or edge - auto& verts2AllClassifications = *verts2AllClassificationsPtr; - auto& mesh1ElToMesh2Els = *(m_relationalData->mesh1ElementsToMesh2Elements); - mesh::impl::AdjacencySearch search(m_mesh1, m_mesh2); - while ((el1 = search.get_next(mesh2Els))) + + std::vector mesh1Els; + + std::vector vert2AllClassifications; + + auto mesh2VertsToMesh1ElsPtr = compute_mesh2_to_mesh1_element_maps(); + const auto& mesh2VertsToMesh1Els = *(mesh2VertsToMesh1ElsPtr); + + for (auto& vert : m_mesh2->get_vertices()) { - seenVerts.clear(); - std::array verts; - for (auto& el2 : mesh2Els) + if (vert) { - int nverts = get_downward(el2, 0, verts.data()); - for (int i = 0; i < nverts; ++i) - if (seenVerts.count(verts[i]) == 0) - { - predicates::impl::PointRecord record = m_pointClassifier->classify(el1, el2, verts[i]->get_point_orig(0)); + mesh::MeshEntityPtr el2 = vert->get_up(0)->get_up(0); + vert2AllClassifications.clear(); + for (auto& el1 : mesh2VertsToMesh1Els(vert, 0)) + { + predicates::impl::PointRecord record = m_pointClassifier->classify(el1, el2, vert->get_point_orig(0)); - if (record.type != PointClassification::Exterior) + if (record.type != PointClassification::Exterior) + { + vert2AllClassifications.push_back(record); + if (m_output && vert->get_id() == 1001) { - verts2AllClassifications.insert(verts[i], 0, record); - if (m_output && verts[i]->get_id() == 1001) - { - std::cout << "classified vertex with id " << verts[i]->get_id() << " on element " << el1 - << ", record = " << record << std::endl; - utils::Point ptOrig = verts[i]->get_point_orig(0); - utils::Point ptProj = m_pointClassifier->compute_xyz_coords(record); - std::cout << "vertex coords = " << ptOrig << ", projected coordinates = " << ptProj << std::endl; - std::cout << "distance = " << std::sqrt(dot(ptOrig - ptProj, ptOrig - ptProj)) << std::endl; - } + std::cout << "classified vertex with id " << vert->get_id() << " on element " << el1 + << ", record = " << record << std::endl; + utils::Point ptOrig = vert->get_point_orig(0); + utils::Point ptProj = m_pointClassifier->compute_xyz_coords(record); + std::cout << "vertex coords = " << ptOrig << ", projected coordinates = " << ptProj << std::endl; + std::cout << "distance = " << std::sqrt(dot(ptOrig - ptProj, ptOrig - ptProj)) << std::endl; } - - seenVerts.insert(verts[i]); } - } + } - for (auto& el2 : mesh2Els) - mesh1ElToMesh2Els.insert(el1, 0, el2); + choose_unique_vert_projection(vert, vert2AllClassifications); + } } - - choose_unique_vert_projection(verts2AllClassificationsPtr); } void MeshProjectionCalculator::choose_unique_vert_projection( - mesh::VariableSizeFieldPtr verts2AllClassificationsPtr) + mesh::MeshEntityPtr vert2, const std::vector& verts2AllClassifications) { - auto& verts2AllClassifications = *verts2AllClassificationsPtr; - for (auto& vert2 : m_mesh2->get_vertices()) - if (vert2) - { - int nprojections = verts2AllClassifications.get_num_comp(vert2, 0); - if (nprojections > 0) - { - utils::Point closestPt; - int minIdx = get_closest_projection(verts2AllClassificationsPtr, vert2, closestPt); - record_vert2_classification(vert2, verts2AllClassifications(vert2, 0, minIdx), closestPt); - } else - throw std::runtime_error("vertex does not projection onto any mesh1 element"); - } + + int nprojections = verts2AllClassifications.size(); + if (nprojections > 0) + { + utils::Point closestPt; + int minIdx = get_closest_projection(verts2AllClassifications, vert2, closestPt); + record_vert2_classification(vert2, verts2AllClassifications[minIdx], closestPt); + } else + throw std::runtime_error("vertex does not projection onto any mesh1 element"); + } void MeshProjectionCalculator::record_vert2_classification(mesh::MeshEntityPtr vert2, @@ -143,31 +163,12 @@ void MeshProjectionCalculator::record_vert2_classification(mesh::MeshEntityPtr v verts2ClassOnMesh1(vert2, 0, 0) = record; verts2ToFakeVerts(vert2, 0, 0) = fv; record_edge2_vertex_association(edges2ToFakeVertsIn, vert2, fv); - /* - for (int i=0; i < vert2->count_up(); ++i) - { - mesh::MeshEntityPtr edge2 = vert2->get_up(i); - double xi = edge2->get_down(0) == vert2 ? 0 : 1; - edges2_to_fake_verts_in.insert(edge2, 0, {fv, xi}); - - if (edges2_to_fake_verts_in.get_num_comp(edge2, 0) == 0) - { - edges2_to_fake_verts_in.insert(edge2, 0); - edges2_to_fake_verts_in.insert(edge2, 0); - } - - int idx = edge2->get_down(0) == vert2 ? 0 : 1; - assert(edge2->get_down(idx) == vert2); - edges2_to_fake_verts_in(edge2, 0, idx) = VertOnEdge{fv, xi}; - - } - */ if (record.type == PointClassification::Edge) { double xi = m_pointClassifier->get_edge_xi(record); mesh::MeshEntityPtr edge = get_entity(record); - mesh1EdgesToSplit.insert(edge, 0, EdgeSplitRecord{fv, xi}); + mesh1EdgesToSplit.insert(edge, 0, EdgeSplitRecord{fv, xi, vert2}); if (m_output && m_vertIds.count(fv.id) > 0) { @@ -200,16 +201,15 @@ void MeshProjectionCalculator::record_edge2_vertex_association(mesh::VariableSiz } int MeshProjectionCalculator::get_closest_projection( - mesh::VariableSizeFieldPtr verts2AllClassificationsPtr, mesh::MeshEntityPtr vert2, + const std::vector& verts2AllClassifications, mesh::MeshEntityPtr vert2, utils::Point& closestPt) { - auto& verts2AllClassifications = *verts2AllClassificationsPtr; - int nprojections = verts2AllClassifications.get_num_comp(vert2, 0); + int nprojections = verts2AllClassifications.size(); assert(nprojections >= 1); if (nprojections == 1) { - closestPt = m_pointClassifier->compute_xyz_coords(verts2AllClassifications(vert2, 0, 0)); + closestPt = m_pointClassifier->compute_xyz_coords(verts2AllClassifications[0]); return 0; } @@ -219,7 +219,7 @@ int MeshProjectionCalculator::get_closest_projection( int minIdx = -1; for (int i = 0; i < nprojections; ++i) { - utils::Point destPt = m_pointClassifier->compute_xyz_coords(verts2AllClassifications(vert2, 0, i)); + utils::Point destPt = m_pointClassifier->compute_xyz_coords(verts2AllClassifications[i]); utils::Point disp = destPt - srcPt; double distSquared = dot(disp, disp); if (distSquared < minDist) @@ -362,7 +362,7 @@ void MeshProjectionCalculator::record_mesh2_edge_intersects_mesh1_edge(mesh::Mes //std::cout << std::endl; } */ - mesh1EdgesToSplit.insert(edge1, 0, EdgeSplitRecord{fv, xi}); + mesh1EdgesToSplit.insert(edge1, 0, EdgeSplitRecord{fv, xi, edge2}); edges2ToFakeVertsIn.insert(edge2, 0, {fv, intersection.alpha}); } diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_projection_calculator.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_projection_calculator.hpp index 288116b68495..a80f5772a099 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_projection_calculator.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_projection_calculator.hpp @@ -18,18 +18,17 @@ class MeshProjectionCalculator public: MeshProjectionCalculator(std::shared_ptr mesh1, std::shared_ptr mesh2, - std::shared_ptr meshIn, std::shared_ptr relationalData, + std::shared_ptr relationalData, std::shared_ptr pointClassifier, const middle_mesh::impl::EdgeTracerTolerances& edgeTracerTolerances) : m_mesh1(mesh1) , m_mesh2(mesh2) - , m_meshIn(meshIn) , m_pointClassifier(pointClassifier) , m_edgeTracer(m_mesh2, m_pointClassifier->get_normal_field(), edgeTracerTolerances) , m_relationalData(relationalData) {} - std::shared_ptr project(); + void project(); private: template @@ -39,10 +38,12 @@ class MeshProjectionCalculator void create_mesh1_fake_verts(); + mesh::VariableSizeFieldPtr compute_mesh2_to_mesh1_element_maps(); + void project_mesh2_vertices_onto_mesh1(); void choose_unique_vert_projection( - mesh::VariableSizeFieldPtr verts2AllClassificationsPtr); + mesh::MeshEntityPtr vert2, const std::vector& verts2AllClassificationsPtr); void record_vert2_classification(mesh::MeshEntityPtr vert2, const predicates::impl::PointRecord& record, const utils::Point& pt); @@ -50,7 +51,7 @@ class MeshProjectionCalculator void record_edge2_vertex_association(mesh::VariableSizeField& edges2ToFakeVertsIn, mesh::MeshEntityPtr vert2, FakeVert fv); - int get_closest_projection(mesh::VariableSizeFieldPtr verts2AllClassificationsPtr, + int get_closest_projection(const std::vector& verts2AllClassificationsPtr, mesh::MeshEntityPtr vert2, utils::Point& closestPt); void intersect_mesh2_edges_with_mesh1(); @@ -63,7 +64,6 @@ class MeshProjectionCalculator std::shared_ptr m_mesh1; std::shared_ptr m_mesh2; - std::shared_ptr m_meshIn; std::shared_ptr m_pointClassifier; EdgeTracer m_edgeTracer; FakeVertGenerator m_fakeVertGen; diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_relational_data.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_relational_data.hpp index a017942bcfff..e9a84e83604c 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_relational_data.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_relational_data.hpp @@ -3,6 +3,7 @@ #include "field.hpp" #include "mesh.hpp" +#include "mesh_entity.hpp" #include "predicates/intersection_common.hpp" #include "projection.hpp" #include "variable_size_field.hpp" @@ -41,6 +42,7 @@ struct EdgeSplitRecord { FakeVert vert; double xi; + mesh::MeshEntityPtr otherMeshEntity; }; struct VertOnEdge @@ -55,17 +57,20 @@ struct MeshRelationalData { MeshRelationalData(std::shared_ptr mesh1, std::shared_ptr mesh2, std::shared_ptr meshIn) - : vertsInClassOnMesh1(mesh::create_field(meshIn, FieldShape(1, 0, 0), 1)) + : vertsInClassOnMesh1(meshIn ? mesh::create_field(meshIn, FieldShape(1, 0, 0), 1) : nullptr) , verts2ClassOnMesh1(mesh::create_field(mesh2, FieldShape(1, 0, 0), 1)) , verts1ToFakeVerts(mesh::create_field(mesh1, FieldShape(1, 0, 0), 1)) , verts2ToFakeVerts(mesh::create_field(mesh2, FieldShape(1, 0, 0), 1)) , edges2ToFakeVertsIn(mesh::create_variable_size_field(mesh2, FieldShape(0, 1, 0))) , mesh1EdgesToSplit(mesh::create_variable_size_field(mesh1, FieldShape(0, 1, 0))) - , mesh1ElsToVertsIn(mesh::create_variable_size_field(mesh1, FieldShape(0, 0, 1))) - , meshInElementsToMesh1Elements(mesh::create_field(meshIn, FieldShape(0, 0, 1), 1, nullptr)) - , meshInElementsToMesh2Elements(mesh::create_field(meshIn, FieldShape(0, 0, 1), 1, nullptr)) - , mesh1ElementsToMesh2Elements(mesh::create_variable_size_field(mesh1, FieldShape(0, 0, 1))) - // edges1_to_in(mesh::create_variable_size_field(mesh1, FieldShape(0, 1, 0))) + , mesh1ElsToVertsIn(meshIn ? mesh::create_variable_size_field(mesh1, FieldShape(0, 0, 1)): nullptr) + , meshInElementsToMesh1Elements(meshIn ? mesh::create_field(meshIn, FieldShape(0, 0, 1), 1, nullptr) : nullptr) + , meshInElementsToMesh2Elements(meshIn ? mesh::create_field(meshIn, FieldShape(0, 0, 1), 1, nullptr) : nullptr) + {} + + // this constructor only allocates the fields needed *before* the MeshRelationalDataScatter + MeshRelationalData(std::shared_ptr mesh1, std::shared_ptr mesh2) : + MeshRelationalData(mesh1, mesh2, nullptr) {} mesh::FieldPtr vertsInClassOnMesh1; // classification of mesh_in vertices on mesh1 @@ -79,7 +84,6 @@ struct MeshRelationalData mesh::VariableSizeFieldPtr mesh1EdgesToSplit; // edges on mesh1 that are split by mesh2 edges or vertices - // VariableSizeFieldPtr edges1_to_in; // edges on mesh_in that are on mesh1 edges mesh::VariableSizeFieldPtr mesh1ElsToVertsIn; // maps the elements of mesh1 to the vertices // on mesh_in that are inside them (or on their @@ -92,12 +96,6 @@ struct MeshRelationalData // are created mesh::FieldPtr meshInElementsToMesh1Elements; // maps mesh_in elements to mesh1 elements mesh::FieldPtr meshInElementsToMesh2Elements; // maps mesh_in elements to mesh2 elements - mesh::VariableSizeFieldPtr - mesh1ElementsToMesh2Elements; // stores the candidate set of mesh2 elements - // that *may* overlap a given mesh1 element. - // This set of elements is required to cover - // the mesh1 elements, but may contain additional - // elements that are nearby }; } // namespace impl diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_relational_data_scatter.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_relational_data_scatter.cpp new file mode 100644 index 000000000000..b5d2f779b0cd --- /dev/null +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_relational_data_scatter.cpp @@ -0,0 +1,700 @@ +#include "stk_middle_mesh/mesh_relational_data_scatter.hpp" +#include "mesh_entity.hpp" +#include "predicates/intersection_common.hpp" +#include "predicates/point_classifier_normal_wrapper.hpp" + +namespace stk { +namespace middle_mesh { +namespace nonconformal4 { +namespace impl { + +namespace { +using predicates::impl::PointRecord; +using predicates::impl::PointRecordForTriangle; +using predicates::impl::PointClassification; +} + + +MeshRelationalDataScatter::MeshRelationalDataScatter( + const MeshRelationalDataScatterInput& scatterInput, + std::shared_ptr meshRelationalData, + std::shared_ptr pointClassifierInput, + std::shared_ptr pointClassifierOutput, + MPI_Comm unionComm) : + m_mesh1(scatterInput.mesh1), + m_mesh2(scatterInput.mesh2), + m_mesh1ScatteredToMesh2(scatterInput.mesh1ScatteredToMesh2), + m_mesh2ScatteredToMesh1(scatterInput.mesh2ScatteredToMesh1), + m_mesh1EntityOrigins(scatterInput.mesh1EntityOrigins), + m_mesh1EntityDestinations(scatterInput.mesh1EntityDestinations), + m_mesh2EntityOrigins(scatterInput.mesh2EntityOrigins), + m_mesh2EntityDestinations(scatterInput.mesh2EntityDestinations), + m_meshRelationDataInput(meshRelationalData), + m_pointClassifierInput(pointClassifierInput), + m_pointClassifierOutput(pointClassifierOutput), + m_quadToTrianglesOrigin(pointClassifierInput ? &(pointClassifierInput->get_quad_to_triangles()) : nullptr), + m_quadToTrianglesDest(pointClassifierOutput ? &(pointClassifierOutput->get_quad_to_triangles()) : nullptr), + m_unionComm(unionComm) +{ + if (m_mesh1) + { + // TODO: a flexible CSR structure might be better for this + m_senderFakeVertsToLocal.resize(utils::impl::comm_size(m_unionComm)); + + m_meshIn = mesh::make_empty_mesh(m_mesh1->get_comm()); + m_meshRelationDataOutput = std::make_shared(m_mesh1, m_mesh2ScatteredToMesh1, m_meshIn); + } + check_fields_are_on_correct_meshes(); +} + +std::shared_ptr MeshRelationalDataScatter::scatter() +{ + { + Exchanger mesh1VertExchanger(m_unionComm); + pack_mesh1_vert_fields(mesh1VertExchanger); + size_recv_buffer_mesh1_vert_fields(mesh1VertExchanger); + unpack_mesh1_vert_fields(mesh1VertExchanger); + } + + { + ExchangerUnknown mesh2VertExchanger(m_unionComm); + pack_mesh2_vert_fields(mesh2VertExchanger); + unpack_mesh2_vert_fields(mesh2VertExchanger); + } + + { + ExchangerUnknown mesh1EdgeExchanger(m_unionComm); + pack_mesh1_edge_fields(mesh1EdgeExchanger); + unpack_mesh1_edge_fields(mesh1EdgeExchanger); + } + + { + ExchangerUnknown mesh2EdgeExchanger(m_unionComm); + pack_mesh2_edge_fields(mesh2EdgeExchanger); + unpack_mesh2_edge_fields(mesh2EdgeExchanger); + } + + if (m_meshRelationDataOutput) + { + m_meshRelationDataOutput->fakeVertsToVertsIn.resize(m_fakeVertGenerator.get_num_verts()); + } + + return m_meshRelationDataOutput; +} + + +void MeshRelationalDataScatter::pack_mesh1_vert_fields(Exchanger& exchanger) +{ + if (m_mesh1ScatteredToMesh2) + { + for (int phase=0; phase < 2; ++phase) + { + auto& verts1ToFakeVerts = *(m_meshRelationDataInput->verts1ToFakeVerts); + auto& dests = *m_mesh1EntityOrigins; + for (auto& vert : m_mesh1ScatteredToMesh2->get_vertices()) + if (vert) + { + FakeVert& fakeVert = verts1ToFakeVerts(vert, 0, 0); + for (mesh::RemoteSharedEntity remote : dests(vert, 0)) + { + exchanger.get_send_buf(remote.remoteRank).pack(remote.remoteId); + exchanger.get_send_buf(remote.remoteRank).pack(fakeVert.id); + exchanger.get_send_buf(remote.remoteRank).pack(fakeVert.pt); + } + } + + if (phase == 0) + exchanger.allocate_send_buffers(); + } + } +} + +void MeshRelationalDataScatter::size_recv_buffer_mesh1_vert_fields(Exchanger& exchanger) +{ + if (m_mesh1) + { + auto& senders = *m_mesh1EntityDestinations; + for (auto& vert : m_mesh1->get_vertices()) + if (vert) + { + for (mesh::RemoteSharedEntity sender : senders(vert, 0)) + { + int id = -1; + utils::Point pt; + exchanger.get_recv_buf(sender.remoteRank).pack(id); + exchanger.get_recv_buf(sender.remoteRank).pack(id); + exchanger.get_recv_buf(sender.remoteRank).pack(pt); + } + } + + for (int i=0; i < utils::impl::comm_size(m_unionComm); ++i) + exchanger.set_recv_buffer_size(i, exchanger.get_recv_buf(i).size()); + } + + exchanger.allocate_recv_buffers(); + +} + +void MeshRelationalDataScatter::unpack_mesh1_vert_fields(Exchanger& exchanger) +{ + exchanger.start_nonblocking(); + + // don't unpack data as it comes in: this would give the + // fakeVertIds a non-deterministic order + auto f = [](int rank, stk::CommBuffer& buf) {}; + exchanger.complete_receives(f); + + if (m_mesh1) + { + auto& verts1ToFakeVerts = *(m_meshRelationDataOutput->verts1ToFakeVerts); + + using Bool = int_least8_t; + auto seenVertsPtr = mesh::create_field(m_mesh1, FieldShape(1, 0, 0), 1, false); + auto& seenVerts = *seenVertsPtr; + for (int rank=0; rank < utils::impl::comm_size(m_unionComm); ++rank) + { + auto& buf = exchanger.get_recv_buf(rank); + while (buf.remaining() > 0) + { + int localVertId, senderFakeVertId; + utils::Point pt; + buf.unpack(localVertId); + buf.unpack(senderFakeVertId); + buf.unpack(pt); + + mesh::MeshEntityPtr vert = m_mesh1->get_vertices()[localVertId]; + + FakeVert fv; + if (seenVerts(vert, 0, 0)) + { + fv = verts1ToFakeVerts(vert, 0, 0); + } else + { + fv = m_fakeVertGenerator.get_vert(pt); + verts1ToFakeVerts(vert, 0, 0) = fv; + seenVerts(vert, 0, 0) = true; + } + + m_senderFakeVertsToLocal[rank].insert(std::make_pair(senderFakeVertId, fv)); + } + } + } +} + + +void MeshRelationalDataScatter::pack_mesh2_vert_fields(ExchangerUnknown& exchanger) +{ + if (m_mesh2) + { + auto& verts2ToFakeVerts = *(m_meshRelationDataInput->verts2ToFakeVerts); + auto& verts2ClassOnMesh1 = *(m_meshRelationDataInput->verts2ClassOnMesh1); + auto& dests = *m_mesh2EntityDestinations; + for (int phase=0; phase < 2; ++phase) + { + for (auto& vert : m_mesh2->get_vertices()) + if (vert) + { + for (mesh::RemoteSharedEntity dest : dests(vert, 0)) + { + auto& buf = exchanger.get_send_buf(dest.remoteRank); + FakeVert fv = verts2ToFakeVerts(vert, 0, 0); + const PointRecord& pointRecord = verts2ClassOnMesh1(vert, 0, 0); + + buf.pack(dest.remoteId); + buf.pack(fv.id); + buf.pack(fv.pt); + pack(buf, dest.remoteRank, pointRecord); + } + } + + if (phase == 0) + exchanger.allocate_send_buffers(); + } + } +} + + +void MeshRelationalDataScatter::pack(stk::CommBuffer& buf, + int destRank, const PointRecord& record) +{ + + assert(record.type != predicates::impl::PointClassification::Exterior); + buf.pack(record.type); + if (record.type == predicates::impl::PointClassification::Vert) + { + mesh::MeshEntityPtr vertLocal = predicates::impl::get_entity(record); + int vertIdOnDest = get_id_on_dest(vertLocal, destRank); + buf.pack(vertIdOnDest); + } else if (record.type == predicates::impl::PointClassification::Edge) + { + mesh::MeshEntityPtr edgeLocal = predicates::impl::get_entity(record); + int edgeIdOnDest = get_id_on_dest(edgeLocal, destRank); + buf.pack(edgeIdOnDest); + if (edgeIdOnDest == -1) + { + return; + } + buf.pack(m_pointClassifierInput->get_edge_xi(record)); + } else if (record.type == predicates::impl::PointClassification::Interior) + { + int elIdOnDest = get_id_on_dest(record.el, destRank); + + buf.pack(elIdOnDest); + if (elIdOnDest == -1) + { + return; + } + pack(buf, record.m_r1); + pack(buf, record.m_r2); + } +} + + +int MeshRelationalDataScatter::get_id_on_dest(mesh::MeshEntityPtr entity1, int destRank) +{ + auto& mesh1EntityOrigins = *m_mesh1EntityOrigins; + for (mesh::RemoteSharedEntity remote : mesh1EntityOrigins(entity1, 0)) + { + if (remote.remoteRank == destRank) + { + return remote.remoteId; + } + } + + return -1; +} + +void MeshRelationalDataScatter::pack(stk::CommBuffer& buf, + const PointRecordForTriangle& record) +{ + int elId; + if (record.el == m_quadToTrianglesOrigin->el1) + { + elId = 0; + } else if (record.el == m_quadToTrianglesOrigin->el2) + { + elId = 1; + } else if (record.el == nullptr) + { + elId = 2; + } else + { + elId = 3; + } + + buf.pack(record.type); + buf.pack(record.id); + buf.pack(elId); + buf.pack(record.m_ptXi); +} + + + +PointRecord MeshRelationalDataScatter::unpack_point_record(stk::CommBuffer& buf) +{ + predicates::impl::PointClassification type; + int entityId; + + buf.unpack(type); + buf.unpack(entityId); + + // elId can be -1 in the partial overlap case: if a mesh2 element spans two + // (or more) mesh1 elements that are on different procs, the mesh2 element + // will be set to both procs, but some of its verts will be classified on + // elements not on one of the destination procs + // Example: + // |-------/--------| + // | / | + // | P0 / P1 | + // |-------/--------| + // If the slash marks are the inter-processor boundary of mesh1, + // the top right vertex will be classified on a P1 element, which + // is not present on P0. + if (entityId == -1) + { + return PointRecord(); + } + + if (type == predicates::impl::PointClassification::Vert) + { + mesh::MeshEntityPtr vert = m_mesh1->get_vertices()[entityId]; + mesh::MeshEntityPtr el = vert->get_up(0)->get_up(0); + int vertIdOnEl = mesh::get_local_id(el, vert); + + return m_pointClassifierOutput->create_vert_record(el, vertIdOnEl); + } else if (type == predicates::impl::PointClassification::Edge) + { + double edgeXi; + buf.unpack(edgeXi); + + mesh::MeshEntityPtr edge = m_mesh1->get_edges()[entityId]; + mesh::MeshEntityPtr el = edge->get_up(0); + int edgeIdOnEl = mesh::get_local_id(el, edge); + double edgeXiOnReferenceEl = el->get_down_orientation(edgeIdOnEl) == mesh::EntityOrientation::Standard ? edgeXi : 1 - edgeXi; + + return m_pointClassifierOutput->create_edge_record(el, edgeIdOnEl, edgeXiOnReferenceEl); + } else if (type == predicates::impl::PointClassification::Interior) + { + mesh::MeshEntityPtr el = m_mesh1->get_elements()[entityId]; + + PointRecordForTriangle r1 = unpack_point_record_for_triangle(buf, el); + PointRecordForTriangle r2 = unpack_point_record_for_triangle(buf, el); + + return PointRecord(type, 0, el, r1, r2); + } else + { + throw std::runtime_error("type cannot be Exterior"); + } +} + +PointRecordForTriangle MeshRelationalDataScatter::unpack_point_record_for_triangle( + stk::CommBuffer& buf, mesh::MeshEntityPtr parentEl) +{ + predicates::impl::PointClassification type; + int entityId, elId; + utils::Point ptXi; + + buf.unpack(type); + buf.unpack(entityId); + buf.unpack(elId); + buf.unpack(ptXi); + + mesh::MeshEntityPtr el; + switch (elId) + { + case 0 : { el = m_quadToTrianglesDest->el1; break; } + case 1 : { el = m_quadToTrianglesDest->el2; break; } + case 2 : { el = nullptr; break; } + case 3 : { el = parentEl; break; } + default: + { + throw std::runtime_error("invalid value for elId"); + } + } + + return PointRecordForTriangle(type, entityId, el, ptXi); +} + +void MeshRelationalDataScatter::unpack_mesh2_vert_fields(ExchangerUnknown& exchanger) +{ + exchanger.start_nonblocking(); + exchanger.post_nonblocking_receives(); + // Don't unpack data as it arrives because that would lead to non-deterministic + // assignment of FakeVert ids + auto f = [](int rank, stk::CommBuffer& buf) {}; + exchanger.complete_receives(f); + + if (m_mesh2ScatteredToMesh1) + { + auto& verts2ToFakeVerts = *(m_meshRelationDataOutput->verts2ToFakeVerts); + auto& verts2ClassOnMesh1 = *(m_meshRelationDataOutput->verts2ClassOnMesh1); + verts2ToFakeVerts.set(FakeVert{-1, {0, 0, 0}}); + + for (int rank=0; rank < utils::impl::comm_size(m_unionComm); ++rank) + { + stk::CommBuffer& buf = exchanger.get_recv_buf(rank); + while (buf.remaining() > 0) + { + int vertLocalId, senderFakeVertId; + utils::Point pt; + buf.unpack(vertLocalId); + buf.unpack(senderFakeVertId); + buf.unpack(pt); + predicates::impl::PointRecord record = unpack_point_record(buf); + + mesh::MeshEntityPtr vert = m_mesh2ScatteredToMesh1->get_vertices()[vertLocalId]; + FakeVert fv = verts2ToFakeVerts(vert, 0, 0); + + if (fv.id == -1) + { + bool fakeVertExists = m_senderFakeVertsToLocal[rank].count(senderFakeVertId); + if (fakeVertExists) + { + fv = m_senderFakeVertsToLocal[rank][senderFakeVertId]; + } else{ + fv = m_fakeVertGenerator.get_vert(pt); + } + } + + m_senderFakeVertsToLocal[rank].insert(std::make_pair(senderFakeVertId, fv)); + verts2ToFakeVerts(vert, 0, 0) = fv; + verts2ClassOnMesh1(vert, 0, 0) = record; + } + } + } +} + +void MeshRelationalDataScatter::pack_mesh1_edge_fields(ExchangerUnknown& exchanger) +{ + if (m_mesh1ScatteredToMesh2) + { + auto& mesh1Dests = *m_mesh1EntityOrigins; + std::vector ownedSplits; + for (int phase=0; phase < 2; ++phase) + { + for (auto& edge : m_mesh1ScatteredToMesh2->get_edges()) + { + if (edge) + { + get_owned_splits(edge, ownedSplits); + if (ownedSplits.size() > 0) + { + for (mesh::RemoteSharedEntity dest : mesh1Dests(edge, 0)) + { + stk::CommBuffer& buf = exchanger.get_send_buf(dest.remoteRank); + pack_edge_splits(buf, dest, edge, ownedSplits); + } + } + } + } + + if (phase == 0) + exchanger.allocate_send_buffers(); + } + } +} + +void MeshRelationalDataScatter::get_owned_splits(mesh::MeshEntityPtr edge, std::vector& ownedSplits) +{ + int myrank = utils::impl::comm_rank(m_mesh1ScatteredToMesh2->get_comm()); + auto& mesh1EdgesToSplit = *(m_meshRelationDataInput->mesh1EdgesToSplit); + + ownedSplits.resize(0); + int nsplits = mesh1EdgesToSplit.get_num_comp(edge, 0); + for (int i=0; i < nsplits; ++i) + { + mesh::MeshEntityPtr mesh1Entity = mesh1EdgesToSplit(edge, 0, i).otherMeshEntity; + if (mesh::get_owner(m_mesh1ScatteredToMesh2, mesh1Entity) == myrank) + { + ownedSplits.push_back(i); + } + } +} + +void MeshRelationalDataScatter::pack_edge_splits(stk::CommBuffer& buf, const mesh::RemoteSharedEntity& dest, + mesh::MeshEntityPtr edge, const std::vector ownedSplits) +{ + auto& mesh1EdgesToSplit = *(m_meshRelationDataInput->mesh1EdgesToSplit); + auto& mesh2Dests = *m_mesh2EntityDestinations; + + buf.pack(dest.remoteId); + buf.pack(int(ownedSplits.size())); + for (int i : ownedSplits) + { + EdgeSplitRecord& edgeSplit = mesh1EdgesToSplit(edge, 0, i); + + int otherEntityIdOnDest = -1; + int otherEntityDim = mesh::get_type_dimension(edgeSplit.otherMeshEntity->get_type()); + for (auto& mesh2Dest : mesh2Dests(edgeSplit.otherMeshEntity, 0)) + if (mesh2Dest.remoteRank == dest.remoteRank) + { + otherEntityIdOnDest = mesh2Dest.remoteId; + break; + } + + buf.pack(edgeSplit.vert.id); + buf.pack(otherEntityDim); + buf.pack(otherEntityIdOnDest); + buf.pack(edgeSplit.vert.pt); + buf.pack(edgeSplit.xi); + } +} + +void MeshRelationalDataScatter::unpack_mesh1_edge_fields(ExchangerUnknown& exchanger) +{ + exchanger.start_nonblocking(); + exchanger.post_nonblocking_receives(); + auto f = [](int rank, stk::CommBuffer& buf) {}; + exchanger.complete_receives(f); + + if (m_mesh1) + { + for (int rank=0; rank < utils::impl::comm_size(m_unionComm); ++rank) + { + stk::CommBuffer& buf = exchanger.get_recv_buf(rank); + while (buf.remaining() > 0) + { + unpack_edge_split(buf, rank); + } + } + } +} + +void MeshRelationalDataScatter::unpack_edge_split(stk::CommBuffer& buf, int senderRank) +{ + int edgeLocalId, nsplits; + buf.unpack(edgeLocalId); + buf.unpack(nsplits); + + mesh::MeshEntityPtr edge = m_mesh1->get_edges()[edgeLocalId]; + auto& mesh1EdgesToSplit = *(m_meshRelationDataOutput->mesh1EdgesToSplit); + for (int i=0; i < nsplits; ++i) + { + int senderFakeVertId, otherEntityDim, otherEntityId; + utils::Point pt; + double xi; + + buf.unpack(senderFakeVertId); + buf.unpack(otherEntityDim); + buf.unpack(otherEntityId); + buf.unpack(pt); + buf.unpack(xi); + + bool vertExists = m_senderFakeVertsToLocal[senderRank].count(senderFakeVertId) > 0; + FakeVert fv; + if (vertExists) + { + fv = m_senderFakeVertsToLocal[senderRank][senderFakeVertId]; + } else + { + fv = m_fakeVertGenerator.get_vert(pt); + m_senderFakeVertsToLocal[senderRank].insert(std::make_pair(senderFakeVertId, fv)); + } + + mesh::MeshEntityPtr otherEntity = m_mesh2ScatteredToMesh1->get_mesh_entities(otherEntityDim)[otherEntityId]; + mesh1EdgesToSplit.insert(edge, 0, EdgeSplitRecord{fv, xi, otherEntity}); + } +} + +void MeshRelationalDataScatter::pack_mesh2_edge_fields(ExchangerUnknown& exchanger) +{ + if (m_mesh2) + { + auto& edges2ToFakeVertsIn = *(m_meshRelationDataInput->edges2ToFakeVertsIn); + auto& dests = *m_mesh2EntityDestinations; + int myrank = utils::impl::comm_rank(m_mesh2->get_comm()); + for (int phase=0; phase < 2; ++phase) + { + for (auto& edge : m_mesh2->get_edges()) + if (edge) + { + if (mesh::get_owner(m_mesh2, edge) != myrank) + continue; + + for (mesh::RemoteSharedEntity dest : dests(edge, 0)) + { + stk::CommBuffer& buf = exchanger.get_send_buf(dest.remoteRank); + int nverts = edges2ToFakeVertsIn.get_num_comp(edge, 0); + + buf.pack(dest.remoteId); + buf.pack(nverts); + for (VertOnEdge& vertOnEdge : edges2ToFakeVertsIn(edge, 0)) + { + buf.pack(vertOnEdge); + } + } + } + + if (phase == 0) + exchanger.allocate_send_buffers(); + } + } +} + +void MeshRelationalDataScatter::unpack_mesh2_edge_fields(ExchangerUnknown& exchanger) +{ + exchanger.start_nonblocking(); + exchanger.post_nonblocking_receives(); + auto f = [](int rank, stk::CommBuffer& buf) {}; + exchanger.complete_receives(f); + + if (m_mesh2ScatteredToMesh1) + { + for (int rank=0; rank < utils::impl::comm_size(m_unionComm); ++rank) + { + stk::CommBuffer& buf = exchanger.get_recv_buf(rank); + + while (buf.remaining() > 0) + { + unpack_vert_on_edge(buf, rank); + } + } + } +} + +void MeshRelationalDataScatter::unpack_vert_on_edge(stk::CommBuffer& buf, int senderRank) +{ + int edgeLocalId, nverts; + buf.unpack(edgeLocalId); + buf.unpack(nverts); + + mesh::MeshEntityPtr edge = m_mesh2ScatteredToMesh1->get_edges()[edgeLocalId]; + auto& edges2ToFakeVertsIn = *(m_meshRelationDataOutput->edges2ToFakeVertsIn); + for (int i=0; i < nverts; ++i) + { + VertOnEdge vertOnEdge; + buf.unpack(vertOnEdge); + + bool vertExists = m_senderFakeVertsToLocal[senderRank].count(vertOnEdge.vert.id) > 0; + FakeVert fv; + if (vertExists) + { + fv = m_senderFakeVertsToLocal[senderRank][vertOnEdge.vert.id]; + } else + { + fv = m_fakeVertGenerator.get_vert(vertOnEdge.vert.pt); + m_senderFakeVertsToLocal[senderRank].insert(std::make_pair(vertOnEdge.vert.id, fv)); + } + + vertOnEdge.vert = fv; + edges2ToFakeVertsIn.insert(edge, 0, vertOnEdge); + } +} + + +void MeshRelationalDataScatter::check_fields_are_on_correct_meshes() +{ + if (m_mesh1) + { + STK_ThrowAssertMsg(m_quadToTrianglesDest, + "quadToTrianglesDest must be provided on all procs where mesh1 is present"); + + STK_ThrowAssertMsg(m_mesh1EntityDestinations->get_mesh() == m_mesh1, + "mesh1EntityDestinations must be on mesh1"); + } + if (m_mesh1ScatteredToMesh2) + { + STK_ThrowRequireMsg(m_meshRelationDataInput->verts1ToFakeVerts->get_mesh() == m_mesh1ScatteredToMesh2, + "verts1ToFakeVerts must be defined on mesh1ScatteredToMesh2"); + + STK_ThrowRequireMsg(m_meshRelationDataInput->mesh1EdgesToSplit->get_mesh() == m_mesh1ScatteredToMesh2, + "mesh1EdgesToSplit must be defined on mesh1ScatteredToMesh2"); + + STK_ThrowRequireMsg(m_mesh1EntityOrigins->get_mesh() == m_mesh1ScatteredToMesh2, + "mesh1EntityOrigins must be defined on mesh1ScatteredToMesh2"); + } + + if (m_mesh2) + { + STK_ThrowRequireMsg(m_mesh1ScatteredToMesh2, + "mesh1ScatteredToMesh2 must be present on all procs where mesh2 is present"); + + STK_ThrowAssertMsg(m_quadToTrianglesOrigin, + "quadToTrianglesOrigin must be provided on all procs where mesh2 is present"); + + STK_ThrowRequireMsg(m_meshRelationDataInput->verts2ToFakeVerts->get_mesh() == m_mesh2, + "verts2ToFakeVerts must be defined on mesh2"); + + STK_ThrowRequireMsg(m_meshRelationDataInput->verts2ClassOnMesh1->get_mesh() == m_mesh2, + "verts2ClassOnMesh1 must be defined on mesh2"); + + STK_ThrowRequireMsg(m_meshRelationDataInput->edges2ToFakeVertsIn->get_mesh() == m_mesh2, + "edges2ToFakeVertsIn must be defined on mesh2"); + + STK_ThrowRequireMsg(m_mesh2EntityDestinations->get_mesh() == m_mesh2, + "mesh2EntityDestinations must be defined on mesh2"); + } + + if (m_mesh2ScatteredToMesh1) + { + STK_ThrowRequireMsg(m_mesh2EntityOrigins->get_mesh() == m_mesh2ScatteredToMesh1, + "mesh2EntityDestinations must be defined on mesh2ScatteredToMesh1"); + } +} + + + +} +} +} +} \ No newline at end of file diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_relational_data_scatter.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_relational_data_scatter.hpp new file mode 100644 index 000000000000..f8370779ee2d --- /dev/null +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_relational_data_scatter.hpp @@ -0,0 +1,124 @@ +#ifndef STK_MIDDLE_MESH_MESH_RELATIONAL_DATA_SCATTER_H +#define STK_MIDDLE_MESH_MESH_RELATIONAL_DATA_SCATTER_H + +#include "mesh_entity.hpp" +#include "mesh_relational_data.hpp" +#include "predicates/intersection_common.hpp" +#include "predicates/quad_to_triangles.hpp" +#include "stk_middle_mesh/mesh.hpp" +#include "stk_middle_mesh/variable_size_field.hpp" +#include "stk_middle_mesh/entity_sorted_by_owner.hpp" +#include "stk_util/parallel/CommBuffer.hpp" +#include "stk_util/parallel/DataExchangeKnownPatternNonBlockingBuffer.hpp" +#include "stk_util/parallel/DataExchangeUnknownPatternNonBlockingBuffer.hpp" +#include "stk_util/util/ReportHandler.hpp" +#include "stk_middle_mesh/utils.hpp" + + +namespace stk { +namespace middle_mesh { +namespace nonconformal4 { +namespace impl { + +struct MeshRelationalDataScatterInput +{ + std::shared_ptr mesh1; + std::shared_ptr mesh2; + std::shared_ptr mesh1ScatteredToMesh2; + std::shared_ptr mesh2ScatteredToMesh1; + mesh::VariableSizeFieldPtr mesh1EntityOrigins; + mesh::VariableSizeFieldPtr mesh1EntityDestinations; + mesh::VariableSizeFieldPtr mesh2EntityOrigins; + mesh::VariableSizeFieldPtr mesh2EntityDestinations; +}; + + +class MeshRelationalDataScatter +{ + public: + // given a MeshRelationalData, whose fields are defined on mesh2 and mesh1ScatteredToMesh2, send the fields to + // mesh1 and mesh2ScatteredToMesh1 + // mesh1EntityOrigins maps mesh1ScatteredToMesh2 to mesh1 + // mesh2EntityDestinations maps mesh2 to mesh2Scattered to mesh1 + MeshRelationalDataScatter(const MeshRelationalDataScatterInput& scatterInput, + std::shared_ptr meshRelationalData, + std::shared_ptr pointClassifierInput, + std::shared_ptr pointClassifierOutput, + MPI_Comm unionComm); + + std::shared_ptr get_middle_mesh() { return m_meshIn; } + + std::shared_ptr scatter(); + + private: + using Exchanger = stk::DataExchangeKnownPatternNonBlockingCommBuffer; + using ExchangerUnknown = stk::DataExchangeUnknownPatternNonBlockingCommBuffer; + + + void pack_mesh1_vert_fields(Exchanger& exchanger); + + void size_recv_buffer_mesh1_vert_fields(Exchanger& exchanger); + + void unpack_mesh1_vert_fields(Exchanger& exchanger); + + void pack_mesh2_vert_fields(ExchangerUnknown& exchanger); + + void pack(stk::CommBuffer& buf, int destRank, const predicates::impl::PointRecord& record); + + void pack(stk::CommBuffer& buf, const predicates::impl::PointRecordForTriangle& record); + + int get_id_on_dest(mesh::MeshEntityPtr el1, int destRank); + + predicates::impl::PointRecord unpack_point_record(stk::CommBuffer& buf); + + predicates::impl::PointRecordForTriangle unpack_point_record_for_triangle(stk::CommBuffer& buf, mesh::MeshEntityPtr parentEl); + + void unpack_mesh2_vert_fields(ExchangerUnknown& exchanger); + + void pack_mesh1_edge_fields(ExchangerUnknown& exchanger); + + void get_owned_splits(mesh::MeshEntityPtr edge, std::vector& ownedSplits); + + void pack_edge_splits(stk::CommBuffer& buf, const mesh::RemoteSharedEntity& dest, + mesh::MeshEntityPtr edge, const std::vector ownedSplits); + + void unpack_mesh1_edge_fields(ExchangerUnknown& exchanger); + + void unpack_edge_split(stk::CommBuffer& buf, int senderRank); + + void pack_mesh2_edge_fields(ExchangerUnknown& exchanger); + + void unpack_mesh2_edge_fields(ExchangerUnknown& exchanger); + + void unpack_vert_on_edge(stk::CommBuffer& buf, int senderRank); + + void check_fields_are_on_correct_meshes(); + + std::shared_ptr m_mesh1; + std::shared_ptr m_mesh2; + std::shared_ptr m_mesh1ScatteredToMesh2; + std::shared_ptr m_mesh2ScatteredToMesh1; + mesh::VariableSizeFieldPtr m_mesh1EntityOrigins; + mesh::VariableSizeFieldPtr m_mesh1EntityDestinations; + mesh::VariableSizeFieldPtr m_mesh2EntityOrigins; + mesh::VariableSizeFieldPtr m_mesh2EntityDestinations; + std::shared_ptr m_meshIn; + std::shared_ptr m_meshRelationDataInput; + std::shared_ptr m_meshRelationDataOutput; + std::shared_ptr m_pointClassifierInput; + std::shared_ptr m_pointClassifierOutput; + predicates::impl::QuadToTriangles* m_quadToTrianglesOrigin; + predicates::impl::QuadToTriangles* m_quadToTrianglesDest; + + FakeVertGenerator m_fakeVertGenerator; + std::vector> m_senderFakeVertsToLocal; + MPI_Comm m_unionComm; + +}; + +} +} +} +} + +#endif \ No newline at end of file diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_scatter.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_scatter.cpp index 79d37b45a34d..b0c2c2c06436 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_scatter.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_scatter.cpp @@ -1,4 +1,6 @@ #include "mesh_scatter.hpp" +#include "mesh_entity.hpp" +#include "variable_size_field.hpp" namespace stk { namespace middle_mesh { @@ -6,22 +8,32 @@ namespace mesh { namespace impl { MeshScatter::MeshScatter(std::shared_ptr scatterSpec, - std::shared_ptr inputMesh, MPI_Comm scatteredMeshComm) : + std::shared_ptr inputMesh, MPI_Comm scatteredMeshComm, + bool computeEntityCorrespondence) : m_unionComm(scatterSpec->get_comm()), m_scatterSpec(scatterSpec), - m_inputMesh(inputMesh) + m_inputMesh(inputMesh), + m_computeEntityCorrespondence(computeEntityCorrespondence) { if (scatteredMeshComm != MPI_COMM_NULL) { m_outputMesh = make_empty_mesh(scatteredMeshComm); m_elementOrigins = create_field(m_outputMesh, FieldShape(0, 0, 1), 1); - //m_vertOtherDestRank = create_variable_size_field(m_outputMesh, FieldShape(1, 0, 0)); } check_mesh_comm_is_subset_of_union_comm(); int dest_comm_size = compute_dest_comm_size(); m_vertsBySrcMeshOwner = std::make_shared(dest_comm_size); m_edgesBySrcMeshOwner = std::make_shared(dest_comm_size); + + if (m_computeEntityCorrespondence) + { + if (inputMesh) + m_entityDestinations = create_variable_size_field(inputMesh, FieldShape(1, 1, 1)); + + if (m_outputMesh) + m_entityOrigins = create_variable_size_field(m_outputMesh, FieldShape(1, 1, 1)); + } } std::shared_ptr MeshScatter::scatter() @@ -53,11 +65,22 @@ std::shared_ptr MeshScatter::scatter() return m_outputMesh; } -mesh::FieldPtr MeshScatter::get_element_origins() +mesh::FieldPtr MeshScatter::get_element_origins() const { return m_elementOrigins; } +VariableSizeFieldPtr MeshScatter::get_entity_origins() const +{ + assert(m_computeEntityCorrespondence); + return m_entityOrigins; +} + +VariableSizeFieldPtr MeshScatter::get_entity_destinations() const +{ + assert(m_computeEntityCorrespondence); + return m_entityDestinations; +} void MeshScatter::check_mesh_comm_is_subset_of_union_comm() { @@ -98,28 +121,33 @@ int MeshScatter::compute_dest_comm_size() void MeshScatter::send_verts(VariableSizeFieldPtr destRanksOnUnionCommPtr) { EntityExchanger entityExchanger(m_unionComm); - std::shared_ptr sharingExchanger; + EntityIdExchanger entityCorrespondenceExchanger(m_unionComm); + std::shared_ptr sharingExchanger; if (m_outputMesh) - sharingExchanger = std::make_shared(m_outputMesh->get_comm()); + sharingExchanger = std::make_shared(m_outputMesh->get_comm()); if (m_inputMesh) { - pack_verts(destRanksOnUnionCommPtr, entityExchanger); + pack_verts(destRanksOnUnionCommPtr, entityExchanger, entityCorrespondenceExchanger); } - unpack_verts_and_pack_sharing(entityExchanger, sharingExchanger); + unpack_verts_and_pack_sharing(entityExchanger, sharingExchanger, entityCorrespondenceExchanger); if (m_outputMesh) { unpack_vert_sharing(sharingExchanger); } + + if (m_computeEntityCorrespondence) + { + unpack_returned_entity_ids(destRanksOnUnionCommPtr, 0, entityCorrespondenceExchanger); + } } -void MeshScatter::pack_verts(VariableSizeFieldPtr destRanksOnUnionCommPtr, EntityExchanger& entityExchanger) +void MeshScatter::pack_verts(VariableSizeFieldPtr destRanksOnUnionCommPtr, EntityExchanger& entityExchanger, EntityIdExchanger& entityCorrespondenceExchanger) { assert(m_inputMesh); - int myRank = utils::impl::comm_rank(m_inputMesh->get_comm()); std::vector vertDestRanksOnUnionComm; auto& destRanksOnUnionComm = *destRanksOnUnionCommPtr; for (int phase=0; phase < 2; ++phase) @@ -129,17 +157,19 @@ void MeshScatter::pack_verts(VariableSizeFieldPtr destRanksOnUnionCommPtr, { RemoteSharedEntity owner = get_owner_remote(m_inputMesh, vert); - if (owner.remoteRank == myRank) + vertDestRanksOnUnionComm.assign(destRanksOnUnionComm(vert, 0).begin(), + destRanksOnUnionComm(vert, 0).end()); + for (const auto& destRankOnUnionComm : vertDestRanksOnUnionComm) { - vertDestRanksOnUnionComm.assign(destRanksOnUnionComm(vert, 0).begin(), - destRanksOnUnionComm(vert, 0).end()); - for (const auto& destRankOnUnionComm : vertDestRanksOnUnionComm) - { - auto& buf = entityExchanger.get_send_buf(destRankOnUnionComm); - buf.pack(vert->get_point_orig(0)); - buf.pack(owner); - buf.pack(vertDestRanksOnUnionComm); - } + // need to pack my id (so the receiver can setup their fields) + auto& buf = entityExchanger.get_send_buf(destRankOnUnionComm); + buf.pack(vert->get_point_orig(0)); + buf.pack(vert->get_id()); + buf.pack(owner); + buf.pack(vertDestRanksOnUnionComm); + + if (phase == 1 && m_computeEntityCorrespondence) + entityCorrespondenceExchanger.get_recv_buf(destRankOnUnionComm).push_back(-1); } } @@ -149,7 +179,8 @@ void MeshScatter::pack_verts(VariableSizeFieldPtr destRanksOnUnionCommPtr, } void MeshScatter::unpack_verts_and_pack_sharing(EntityExchanger& entityExchanger, - std::shared_ptr sharingExchanger) + std::shared_ptr sharingExchanger, + EntityIdExchanger& entityCorrespondenceExchanger) { entityExchanger.start_nonblocking(); entityExchanger.post_nonblocking_receives(); @@ -163,45 +194,57 @@ void MeshScatter::unpack_verts_and_pack_sharing(EntityExchanger& entityExchanger { auto& buf = entityExchanger.get_recv_buf(rank); if (buf.remaining() > 0) - unpack_vert_buffer(rank, buf, sharingExchanger); - } + unpack_vert_buffer(rank, buf, sharingExchanger, entityCorrespondenceExchanger); + } } void MeshScatter::unpack_vert_buffer(int rank, stk::CommBuffer& buf, - std::shared_ptr sharingExchanger) + std::shared_ptr sharingExchanger, + EntityIdExchanger& entityCorrespondenceExchanger) { int myRankOnOutputComm = utils::impl::comm_rank(m_outputMesh->get_comm()); std::vector destRanksOnUnionComm, destRanksOnOutputComm; while (buf.remaining() > 0) { utils::Point pt; + int idOnSender; RemoteSharedEntity ownerOnSrcMesh; destRanksOnUnionComm.clear(); buf.unpack(pt); + buf.unpack(idOnSender); buf.unpack(ownerOnSrcMesh); buf.unpack(destRanksOnUnionComm); + MeshEntityPtr v = m_vertsBySrcMeshOwner->get_value(ownerOnSrcMesh); + if (!v) + { + v = m_outputMesh->create_vertex(pt); + m_vertsBySrcMeshOwner->insert(ownerOnSrcMesh, v); - MeshEntityPtr v = m_outputMesh->create_vertex(pt); - m_vertsBySrcMeshOwner->insert(ownerOnSrcMesh, v); - - translate_union_comm_ranks_to_output_comm(destRanksOnUnionComm, destRanksOnOutputComm); + translate_union_comm_ranks_to_output_comm(destRanksOnUnionComm, destRanksOnOutputComm); - for (auto& destRank : destRanksOnOutputComm) - if (destRank != myRankOnOutputComm) - { - sharingExchanger->get_send_buf(destRank).push_back(ownerOnSrcMesh.remoteRank); - sharingExchanger->get_send_buf(destRank).push_back(ownerOnSrcMesh.remoteId); - sharingExchanger->get_send_buf(destRank).push_back(v->get_id()); - for (int i=0; i < 3; ++i) + for (auto& destRank : destRanksOnOutputComm) + if (destRank != myRankOnOutputComm) { - sharingExchanger->get_recv_buf(destRank).push_back(-1); + sharingExchanger->get_send_buf(destRank).push_back(ownerOnSrcMesh.remoteRank); + sharingExchanger->get_send_buf(destRank).push_back(ownerOnSrcMesh.remoteId); + sharingExchanger->get_send_buf(destRank).push_back(v->get_id()); + for (int i=0; i < 3; ++i) + { + sharingExchanger->get_recv_buf(destRank).push_back(-1); + } } - } + } + + if (m_computeEntityCorrespondence) + { + m_entityOrigins->insert(v, 0, RemoteSharedEntity(rank, idOnSender)); + entityCorrespondenceExchanger.get_send_buf(rank).push_back(v->get_id()); + } } } -void MeshScatter::unpack_vert_sharing(std::shared_ptr sharingExchanger) +void MeshScatter::unpack_vert_sharing(std::shared_ptr sharingExchanger) { auto unpackShared = [&](int rank, const std::vector& buf) { @@ -211,6 +254,35 @@ void MeshScatter::unpack_vert_sharing(std::shared_ptr shar sharingExchanger->complete_receives(unpackShared); } +void MeshScatter::unpack_returned_entity_ids(VariableSizeFieldPtr destRanksOnUnionCommPtr, int dim, + EntityIdExchanger& entityCorrespondenceExchanger) +{ + auto& destRanksOnUnionComm = *destRanksOnUnionCommPtr; + + entityCorrespondenceExchanger.start_nonblocking(); + + auto unpacker = [&](int rank, const std::vector& ids) + { + assert(m_inputMesh); + int idIdx = 0; + std::vector vertDestRanksOnUnionComm; + for (auto& entity : m_inputMesh->get_mesh_entities(dim)) + if (entity) + { + vertDestRanksOnUnionComm.assign(destRanksOnUnionComm(entity, 0).begin(), + destRanksOnUnionComm(entity, 0).end()); + + bool foundDest = std::find(vertDestRanksOnUnionComm.begin(), vertDestRanksOnUnionComm.end(), rank) != vertDestRanksOnUnionComm.end(); + if (foundDest) + { + m_entityDestinations->insert(entity, 0, RemoteSharedEntity(rank, ids[idIdx++])); + } + } + }; + + entityCorrespondenceExchanger.complete_receives(unpacker); +} + void MeshScatter::translate_union_comm_ranks_to_output_comm(const std::vector& unionCommRanks, std::vector& outputCommRanks) { assert(m_outputMesh); @@ -231,7 +303,7 @@ void MeshScatter::unpack_vert_sharing_buffer(int rank, const std::vector& b RemoteSharedEntity ownerOnSrcMesh{buf[i], buf[i+1]}; int remoteId = buf[i+2]; - MeshEntityPtr v = m_vertsBySrcMeshOwner->get_entity(ownerOnSrcMesh); + MeshEntityPtr v = m_vertsBySrcMeshOwner->get_value(ownerOnSrcMesh); v->add_remote_shared_entity(RemoteSharedEntity{rank, remoteId}); } } @@ -239,12 +311,19 @@ void MeshScatter::unpack_vert_sharing_buffer(int rank, const std::vector& b void MeshScatter::send_edges(VariableSizeFieldPtr destRanksOnUnionCommPtr) { EntityExchanger entityExchanger(m_unionComm); - pack_edges(destRanksOnUnionCommPtr, entityExchanger); - unpack_edges(entityExchanger); + EntityIdExchanger entityCorrespondenceExchanger(m_unionComm); + pack_edges(destRanksOnUnionCommPtr, entityExchanger, entityCorrespondenceExchanger); + unpack_edges(entityExchanger, entityCorrespondenceExchanger); + + if (m_computeEntityCorrespondence) + { + unpack_returned_entity_ids(destRanksOnUnionCommPtr, 1, entityCorrespondenceExchanger); + } } -void MeshScatter::pack_edges(VariableSizeFieldPtr destRanksOnUnionCommPtr, EntityExchanger& entityExchanger) +void MeshScatter::pack_edges(VariableSizeFieldPtr destRanksOnUnionCommPtr, EntityExchanger& entityExchanger, + EntityIdExchanger& entityCorrespondenceExchanger) { std::vector destRanks; auto& destRanksOnUnionComm = *destRanksOnUnionCommPtr; @@ -262,9 +341,15 @@ void MeshScatter::pack_edges(VariableSizeFieldPtr destRanksOnUnionCommPtr, for (int destRank : destRanksOnUnionComm(edge, 0)) { auto& buf = entityExchanger.get_send_buf(destRank); + buf.pack(edge->get_id()); buf.pack(edgeOwner); buf.pack(v1Owner); buf.pack(v2Owner); + + if (phase == 1 && m_computeEntityCorrespondence) + { + entityCorrespondenceExchanger.get_recv_buf(destRank).push_back(-1); + } } } @@ -274,7 +359,7 @@ void MeshScatter::pack_edges(VariableSizeFieldPtr destRanksOnUnionCommPtr, } } -void MeshScatter::unpack_edges(EntityExchanger& entityExchanger) +void MeshScatter::unpack_edges(EntityExchanger& entityExchanger, EntityIdExchanger& entityCorrespondenceExchanger) { // dont process the buffers as they arrive because that would make // the edge ids non-deterministic @@ -285,34 +370,52 @@ void MeshScatter::unpack_edges(EntityExchanger& entityExchanger) for (int rank=0; rank < utils::impl::comm_size(m_unionComm); ++rank) { - unpack_edge_buffer(rank, entityExchanger.get_recv_buf(rank)); + unpack_edge_buffer(rank, entityExchanger.get_recv_buf(rank), entityCorrespondenceExchanger); } } -void MeshScatter::unpack_edge_buffer(int rank, stk::CommBuffer& buf) +void MeshScatter::unpack_edge_buffer(int rank, stk::CommBuffer& buf, EntityIdExchanger& entityCorrespondenceExchanger) { while (buf.remaining() > 0) { + int idOnSender; RemoteSharedEntity edgeOwnerOnSrcMesh, v1OwnerOnSrcMesh, v2OwnerOnSrcMesh; + buf.unpack(idOnSender); buf.unpack(edgeOwnerOnSrcMesh); buf.unpack(v1OwnerOnSrcMesh); buf.unpack(v2OwnerOnSrcMesh); - MeshEntityPtr v1 = m_vertsBySrcMeshOwner->get_entity(v1OwnerOnSrcMesh); - MeshEntityPtr v2 = m_vertsBySrcMeshOwner->get_entity(v2OwnerOnSrcMesh); - MeshEntityPtr edge = m_outputMesh->create_edge(v1, v2); - m_edgesBySrcMeshOwner->insert(edgeOwnerOnSrcMesh, edge); + MeshEntityPtr v1 = m_vertsBySrcMeshOwner->get_value(v1OwnerOnSrcMesh); + MeshEntityPtr v2 = m_vertsBySrcMeshOwner->get_value(v2OwnerOnSrcMesh); + MeshEntityPtr edge = m_edgesBySrcMeshOwner->get_value(edgeOwnerOnSrcMesh); + if (!edge) + { + edge = m_outputMesh->create_edge(v1, v2); + m_edgesBySrcMeshOwner->insert(edgeOwnerOnSrcMesh, edge); + } + + if (m_computeEntityCorrespondence) + { + m_entityOrigins->insert(edge, 0, RemoteSharedEntity(rank, idOnSender)); + entityCorrespondenceExchanger.get_send_buf(rank).push_back(edge->get_id()); + } } } void MeshScatter::send_elements() { EntityExchanger exchanger(m_unionComm); - pack_elements(exchanger); - unpack_elements(exchanger); + EntityIdExchanger entityCorrespondenceExchanger(m_unionComm); + pack_elements(exchanger, entityCorrespondenceExchanger); + unpack_elements(exchanger, entityCorrespondenceExchanger); + + if (m_computeEntityCorrespondence) + { + unpack_returned_element_ids(entityCorrespondenceExchanger); + } } -void MeshScatter::pack_elements(EntityExchanger& entityExchanger) +void MeshScatter::pack_elements(EntityExchanger& entityExchanger, EntityIdExchanger& entityCorrespondenceExchanger) { std::vector destRanksOnUnionComm; if (m_inputMesh) @@ -334,12 +437,18 @@ void MeshScatter::pack_elements(EntityExchanger& entityExchanger) for (auto& destRank : destRanksOnUnionComm) { auto& buf = entityExchanger.get_send_buf(destRank); + buf.pack(el->get_id()); for (int i=0; i < MAX_DOWN; ++i) { buf.pack(owners[i]); } buf.pack(static_cast(edge1Orient)); buf.pack(el->get_id()); + + if (phase == 1 && m_computeEntityCorrespondence) + { + entityCorrespondenceExchanger.get_recv_buf(destRank).push_back(-1); + } } } @@ -349,7 +458,7 @@ void MeshScatter::pack_elements(EntityExchanger& entityExchanger) } } -void MeshScatter::unpack_elements(EntityExchanger& exchanger) +void MeshScatter::unpack_elements(EntityExchanger& exchanger, EntityIdExchanger& entityCorrespondenceExchanger) { exchanger.start_nonblocking(); exchanger.post_nonblocking_receives(); @@ -358,17 +467,18 @@ void MeshScatter::unpack_elements(EntityExchanger& exchanger) exchanger.complete_receives(f); for (int rank=0; rank < utils::impl::comm_size(m_unionComm); ++rank) { - unpack_element_buffer(rank, exchanger.get_recv_buf(rank)); + unpack_element_buffer(rank, exchanger.get_recv_buf(rank), entityCorrespondenceExchanger); } } -void MeshScatter::unpack_element_buffer(int rank, stk::CommBuffer& buf) +void MeshScatter::unpack_element_buffer(int rank, stk::CommBuffer& buf, EntityIdExchanger& entityCorrespondenceExchanger) { auto& elementOrigins = *m_elementOrigins; while(buf.remaining() > 0) { std::array edgeOwnersOnSrcMesh; - int edge1Orient, srcMeshElementId; + int idOnSender, edge1Orient, srcMeshElementId; + buf.unpack(idOnSender); for (int i=0; i < MAX_DOWN; ++i) { buf.unpack(edgeOwnersOnSrcMesh[i]); @@ -376,23 +486,55 @@ void MeshScatter::unpack_element_buffer(int rank, stk::CommBuffer& buf) buf.unpack(edge1Orient); buf.unpack(srcMeshElementId); - MeshEntityPtr edge1 = m_edgesBySrcMeshOwner->get_entity(edgeOwnersOnSrcMesh[0]); - MeshEntityPtr edge2 = m_edgesBySrcMeshOwner->get_entity(edgeOwnersOnSrcMesh[1]); - MeshEntityPtr edge3 = m_edgesBySrcMeshOwner->get_entity(edgeOwnersOnSrcMesh[2]); + MeshEntityPtr edge1 = m_edgesBySrcMeshOwner->get_value(edgeOwnersOnSrcMesh[0]); + MeshEntityPtr edge2 = m_edgesBySrcMeshOwner->get_value(edgeOwnersOnSrcMesh[1]); + MeshEntityPtr edge3 = m_edgesBySrcMeshOwner->get_value(edgeOwnersOnSrcMesh[2]); MeshEntityPtr el; if (edgeOwnersOnSrcMesh[3].remoteRank == -1) { el = m_outputMesh->create_triangle(edge1, edge2, edge3, static_cast(edge1Orient)); } else { - MeshEntityPtr edge4 = m_edgesBySrcMeshOwner->get_entity(edgeOwnersOnSrcMesh[3]); + MeshEntityPtr edge4 = m_edgesBySrcMeshOwner->get_value(edgeOwnersOnSrcMesh[3]); el = m_outputMesh->create_quad(edge1, edge2, edge3, edge4, static_cast(edge1Orient)); } - elementOrigins(el, 0, 0) = RemoteSharedEntity{rank, srcMeshElementId}; + elementOrigins(el, 0, 0) = RemoteSharedEntity{rank, srcMeshElementId}; + + if (m_computeEntityCorrespondence) + { + m_entityOrigins->insert(el, 0, RemoteSharedEntity(rank, idOnSender)); + entityCorrespondenceExchanger.get_send_buf(rank).push_back(el->get_id()); + } } } +void MeshScatter::unpack_returned_element_ids(EntityIdExchanger& entityCorrespondenceExchanger) +{ + entityCorrespondenceExchanger.start_nonblocking(); + + auto unpacker = [&](int rank, const std::vector& ids) + { + std::vector destRanksOnUnionComm; + int idIdx = 0; + for (auto el : m_inputMesh->get_elements()) + if (el) + { + destRanksOnUnionComm.clear(); + m_scatterSpec->get_destinations(el, destRanksOnUnionComm); + + auto it = std::find(destRanksOnUnionComm.begin(), destRanksOnUnionComm.end(), rank); + if (it != destRanksOnUnionComm.end()) + { + int remoteId = ids[idIdx++]; + m_entityDestinations->insert(el, 0, RemoteSharedEntity(rank, remoteId)); + } + } + }; + + entityCorrespondenceExchanger.complete_receives(unpacker); +} + } } diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_scatter.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_scatter.hpp index abd76c94006e..4181d893ec88 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_scatter.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/mesh_scatter.hpp @@ -1,6 +1,7 @@ #ifndef STK_MIDDLE_MESH_MESH_SCATTER #define STK_MIDDLE_MESH_MESH_SCATTER +#include "mesh_entity.hpp" #include "mesh_scatter_spec.hpp" #include "mesh.hpp" #include "stk_util/parallel/DataExchangeUnknownPatternNonBlockingBuffer.hpp" @@ -20,17 +21,25 @@ namespace impl { class MeshScatter { public: + // computeEntityCorrespondence: if true, get_entity_origins and get_entity_destinations will + // return fields describeing the corespondence between entities on the + // input mesh and the scattered mesh MeshScatter(std::shared_ptr scatterSpec, - std::shared_ptr inputMesh, MPI_Comm scatteredMeshComm); + std::shared_ptr inputMesh, MPI_Comm scatteredMeshComm, + bool computeEntityCorrespondence=false); - std::shared_ptr scatter(); + std::shared_ptr scatter(); - mesh::FieldPtr get_element_origins(); + FieldPtr get_element_origins() const; + + VariableSizeFieldPtr get_entity_origins() const; + + VariableSizeFieldPtr get_entity_destinations() const; private: using EntityExchanger = stk::DataExchangeUnknownPatternNonBlockingCommBuffer; - using VertSharingExchanger = stk::DataExchangeKnownPatternNonBlockingBuffer; + using EntityIdExchanger = stk::DataExchangeKnownPatternNonBlockingBuffer; void check_mesh_comm_is_subset_of_union_comm(); @@ -40,14 +49,20 @@ class MeshScatter void send_verts(VariableSizeFieldPtr destRanksOnUnionCommPtr); - void pack_verts(VariableSizeFieldPtr destRanksOnUnionCommPtr, EntityExchanger& entityExchanger); + void pack_verts(VariableSizeFieldPtr destRanksOnUnionCommPtr, EntityExchanger& entityExchanger, + EntityIdExchanger& entityCorrespondenceExchanger); void unpack_verts_and_pack_sharing(EntityExchanger& entityExchanger, - std::shared_ptr sharingExchanger); + std::shared_ptr sharingExchanger, + EntityIdExchanger& entityCorrespondenceExchanger); + + void unpack_vert_buffer(int rank, stk::CommBuffer& buf, std::shared_ptr sharingExchanger, + EntityIdExchanger& entityCorrespondenceExchanger); - void unpack_vert_buffer(int rank, stk::CommBuffer& buf, std::shared_ptr sharingExchanger); + void unpack_vert_sharing(std::shared_ptr sharingExchanger); - void unpack_vert_sharing(std::shared_ptr sharingExchanger); + void unpack_returned_entity_ids(VariableSizeFieldPtr destRanksOnUnionCommPtr, int dim, + EntityIdExchanger& entityCorrespondenceExchanger); void translate_union_comm_ranks_to_output_comm(const std::vector& unionCommRanks, std::vector& outputCommRanks); @@ -55,26 +70,32 @@ class MeshScatter void send_edges(VariableSizeFieldPtr destRanksOnUnionCommPtr); + void pack_edges(VariableSizeFieldPtr destRanksOnUnionCommPtr, EntityExchanger& entityExchanger, + EntityIdExchanger& entityCorrespondenceExchanger); - void pack_edges(VariableSizeFieldPtr destRanksOnUnionCommPtr, EntityExchanger& entityExchanger); + void unpack_edges(EntityExchanger& entityExchanger, EntityIdExchanger& entityCorrespondenceExchanger); - void unpack_edges(EntityExchanger& entityExchanger); - - void unpack_edge_buffer(int rank, stk::CommBuffer& buf); + void unpack_edge_buffer(int rank, stk::CommBuffer& buf, EntityIdExchanger& entityCorrespondenceExchanger); void send_elements(); - void pack_elements(EntityExchanger& entityExchanger); + void pack_elements(EntityExchanger& entityExchanger, EntityIdExchanger& entityCorrespondenceExchanger); + + void unpack_elements(EntityExchanger& exchanger, EntityIdExchanger& entityCorrespondenceExchanger); + + void unpack_element_buffer(int rank, stk::CommBuffer& buf, EntityIdExchanger& entityCorrespondenceExchanger); - void unpack_elements(EntityExchanger& exchanger); + void unpack_returned_element_ids(EntityIdExchanger& entityCorrespondenceExchanger); - void unpack_element_buffer(int rank, stk::CommBuffer& buf); MPI_Comm m_unionComm; std::shared_ptr m_scatterSpec; std::shared_ptr m_inputMesh; + bool m_computeEntityCorrespondence; std::shared_ptr m_outputMesh; FieldPtr m_elementOrigins; + VariableSizeFieldPtr m_entityOrigins; + VariableSizeFieldPtr m_entityDestinations; std::shared_ptr m_vertsBySrcMeshOwner; std::shared_ptr m_edgesBySrcMeshOwner; }; diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/middle_grid_constraint_generator.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/middle_grid_constraint_generator.cpp index a37b574c4297..80123f3a915c 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/middle_grid_constraint_generator.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/middle_grid_constraint_generator.cpp @@ -1,5 +1,8 @@ #include "middle_grid_constraint_generator.hpp" +#include "mesh.hpp" +#include "mesh_entity.hpp" #include "predicates/point_classifier_normal_wrapper.hpp" +#include "variable_size_field.hpp" namespace stk { namespace middle_mesh { @@ -26,6 +29,7 @@ void MiddleGridConstraintGenerator::create_mesh1_vertices() auto& mesh1ElsToVertsIn = *(m_relationalData->mesh1ElsToVertsIn); std::vector els1; + ExchangerKnown exchanger(m_mesh1->get_comm()); for (auto& vert1 : m_mesh1->get_vertices()) if (vert1) @@ -39,7 +43,6 @@ void MiddleGridConstraintGenerator::create_mesh1_vertices() mesh::MeshEntityPtr el1 = vert1->get_up(0)->get_up(0); int localId = predicates::impl::get_entity_id(el1, vert1); vertsInClassOnMesh1(vertIn, 0, 0) = m_pointClassifier->create_vert_record(el1, localId); - //predicates::impl::PointRecord(predicates::impl::PointClassification::Vert, localId, el1); fakeVertsToVertsIn[fv.id] = vertIn; int nels = get_upward(vert1, 2, els1); @@ -49,7 +52,45 @@ void MiddleGridConstraintGenerator::create_mesh1_vertices() if (m_output && m_vertIds.count(vertIn->get_id()) > 0) std::cout << "adding vert_in id " << vertIn->get_id() << " to el1 with id " << els1[i]->get_id() << std::endl; } + + for (int i=0; i < vert1->count_remote_shared_entities(); ++i) + { + const mesh::RemoteSharedEntity& remote = vert1->get_remote_shared_entity(i); + exchanger.get_send_buf(remote.remoteRank).push_back(remote.remoteId); + exchanger.get_send_buf(remote.remoteRank).push_back(vertIn->get_id()); + + exchanger.get_recv_buf(remote.remoteRank).push_back(-1); + exchanger.get_recv_buf(remote.remoteRank).push_back(-1); + } } + + set_mesh1_vert_shared_entities(exchanger); +} + +void MiddleGridConstraintGenerator::set_mesh1_vert_shared_entities(ExchangerKnown& exchanger) +{ + auto& verts1ToFakeVerts = *(m_relationalData->verts1ToFakeVerts); + auto& fakeVertsToVertsIn = m_relationalData->fakeVertsToVertsIn; + + exchanger.start_nonblocking(); + + auto unpacker = [&](int rank, const std::vector& buf) + { + assert(buf.size() % 2 == 0); + for (size_t i=0; i < buf.size(); i += 2) + { + int vert1Id = buf[i]; + int vertInRemoteId = buf[i+1]; + + mesh::MeshEntityPtr vert1 = m_mesh1->get_vertices()[vert1Id]; + FakeVert fv = verts1ToFakeVerts(vert1, 0, 0); + mesh::MeshEntityPtr vertIn = fakeVertsToVertsIn[fv.id]; + + vertIn->add_remote_shared_entity(mesh::RemoteSharedEntity(rank, vertInRemoteId)); + } + }; + + exchanger.complete_receives(unpacker); } void MiddleGridConstraintGenerator::create_mesh2_interior_vertices() @@ -74,7 +115,7 @@ void MiddleGridConstraintGenerator::create_mesh2_interior_vertices() if (m_output && m_vertIds.count(vertIn->get_id()) > 0) std::cout << "created vert_in id " << vertIn->get_id() << " from fakevert id " << fv.id << std::endl; - vertsInClassOnMesh1(vertIn, 0, 0) = record; + vertsInClassOnMesh1(vertIn, 0, 0) = record; fakeVertsToVertsIn[fv.id] = vertIn; mesh1ElsToVertsIn.insert(record.el, 0, vertIn); if (m_output && m_vertIds.count(vertIn->get_id()) > 0) @@ -87,6 +128,7 @@ void MiddleGridConstraintGenerator::create_mesh2_interior_vertices() void MiddleGridConstraintGenerator::create_mesh1_edges() { auto& verts1ToFakeVerts = *(m_relationalData->verts1ToFakeVerts); + ExchangerKnown exchanger(m_mesh1->get_comm()); for (auto& edge : m_mesh1->get_edges()) if (edge) { @@ -96,8 +138,47 @@ void MiddleGridConstraintGenerator::create_mesh1_edges() mesh::MeshEntityPtr v2In = m_relationalData->fakeVertsToVertsIn[fv2.id]; assert(!stk::middle_mesh::mesh::get_common_edge(v1In, v2In)); - m_meshIn->create_edge(v1In, v2In); + mesh::MeshEntityPtr edgeIn = m_meshIn->create_edge(v1In, v2In); + + for (int i=0; i < edge->count_remote_shared_entities(); ++i) + { + const mesh::RemoteSharedEntity& edge1Remote = edge->get_remote_shared_entity(i); + + auto& sendBuf = exchanger.get_send_buf(edge1Remote.remoteRank); + auto& recvBuf = exchanger.get_recv_buf(edge1Remote.remoteRank); + mesh::RemoteSharedEntity v1InRemote = mesh::get_remote_shared_entity(v1In, edge1Remote.remoteRank); + mesh::RemoteSharedEntity v2InRemote = mesh::get_remote_shared_entity(v2In, edge1Remote.remoteRank); + + sendBuf.push_back(v1InRemote.remoteId); recvBuf.push_back(-1); + sendBuf.push_back(v2InRemote.remoteId); recvBuf.push_back(-1); + sendBuf.push_back(edgeIn->get_id()); recvBuf.push_back(-1); + } } + + set_mesh1_edge_shared_entities(exchanger); +} + +void MiddleGridConstraintGenerator::set_mesh1_edge_shared_entities(ExchangerKnown& exchanger) +{ + exchanger.start_nonblocking(); + + auto unpacker = [&](int rank, const std::vector& buf) + { + assert(buf.size() % 3 == 0); + for (size_t i=0; i < buf.size(); i += 3) + { + int vert1Id = buf[i]; + int vert2Id = buf[i+1]; + int edgeRemoteId = buf[i+2]; + + mesh::MeshEntityPtr vert1 = m_meshIn->get_vertices()[vert1Id]; + mesh::MeshEntityPtr vert2 = m_meshIn->get_vertices()[vert2Id]; + mesh::MeshEntityPtr edge = mesh::get_common_edge(vert1, vert2); + edge->add_remote_shared_entity(mesh::RemoteSharedEntity(rank, edgeRemoteId)); + } + }; + + exchanger.complete_receives(unpacker); } void MiddleGridConstraintGenerator::split_edges() @@ -108,11 +189,16 @@ void MiddleGridConstraintGenerator::split_edges() auto& fakeVertsToVertsIn = m_relationalData->fakeVertsToVertsIn; auto& vertsInClassOnMesh1 = *(m_relationalData->vertsInClassOnMesh1); auto& mesh1ElsToVertsIn = *(m_relationalData->mesh1ElsToVertsIn); + ExchangerKnown exchanger(m_mesh1->get_comm()); + + auto sharedEntityInfoPtr = mesh::create_variable_size_field(m_mesh1, mesh::FieldShape(0, 2, 0)); for (auto& edge1 : m_mesh1->get_edges()) + { if (edge1 && mesh1EdgesToSplit.get_num_comp(edge1, 0) > 0) { sort_edge_splits(edge1); + bool edge1HasRemote = edge1->count_remote_shared_entities() > 0; mesh::MeshEntityPtr el1 = edge1->get_up(0); int localId = predicates::impl::get_entity_id(el1, edge1); @@ -121,13 +207,6 @@ void MiddleGridConstraintGenerator::split_edges() double xiStart = 0; mesh::MeshEntityPtr newEdges[2]; - // std::cout << "about to split edge at xi: "; - // for (int i=0; i < mesh1_edges_to_split.get_num_comp(edge1, 0); ++i) - //{ - // std::cout << mesh1_edges_to_split(edge1, 0, i).xi << ", "; - // } - // std::cout << std::endl; - for (int i = 0; i < mesh1EdgesToSplit.get_num_comp(edge1, 0); ++i) { double xi = mesh1EdgesToSplit(edge1, 0, i).xi; @@ -140,6 +219,7 @@ void MiddleGridConstraintGenerator::split_edges() if (m_output && (m_vertIds.count(newVert->get_id()) > 0)) std::cout << "created vert_in id " << newVert->get_id() << " from fakevert id " << fv.id << std::endl; + fakeVertsToVertsIn[fv.id] = newVert; double edgeXiOnReferenceEl = el1->get_down_orientation(localId) == mesh::EntityOrientation::Standard ? xi : 1 - xi; @@ -152,11 +232,78 @@ void MiddleGridConstraintGenerator::split_edges() << std::endl; } + if (edge1HasRemote) + { + sharedEntityInfoPtr->insert(edge1, 0, newVert->get_id()); + sharedEntityInfoPtr->insert(edge1, 1, newEdges[0]->get_id()); + } + // update for next iteration xiStart = xi; currEdgeIn = newEdges[1]; } + + if (edge1HasRemote) + { + sharedEntityInfoPtr->insert(edge1, 1, currEdgeIn->get_id()); + pack_edge_split_shared_info(exchanger, sharedEntityInfoPtr, edge1); + } + } + } + + set_edge_split_shared_entities(exchanger, sharedEntityInfoPtr); +} + +void MiddleGridConstraintGenerator::pack_edge_split_shared_info(ExchangerKnown& exchanger, mesh::VariableSizeFieldPtr sharedEntityInfoPtr, mesh::MeshEntityPtr edge1) +{ + auto& sharedEntityInfo = *sharedEntityInfoPtr; + for (int i=0; i < edge1->count_remote_shared_entities(); ++i) + { + const mesh::RemoteSharedEntity& remote = edge1->get_remote_shared_entity(i); + + auto& sendBuf = exchanger.get_send_buf(remote.remoteRank); + auto& recvBuf = exchanger.get_recv_buf(remote.remoteRank); + + sendBuf.push_back(remote.remoteId); recvBuf.push_back(-1); + for (int dim=0; dim < 2; ++dim) + { + for (int entityId : sharedEntityInfo(edge1, dim)) + { + sendBuf.push_back(entityId); + recvBuf.push_back(-1); + } } + } +} + +void MiddleGridConstraintGenerator::set_edge_split_shared_entities(ExchangerKnown& exchanger, mesh::VariableSizeFieldPtr sharedEntityInfoPtr) +{ + auto& sharedEntityInfo = *sharedEntityInfoPtr; + exchanger.start_nonblocking(); + + auto unpacker = [&](int rank, const std::vector& buf) + { + size_t idx = 0; + while (idx < buf.size()) + { + int edge1Id = buf[idx++]; + mesh::MeshEntityPtr edge1 = m_mesh1->get_edges()[edge1Id]; + for (int dim=0; dim < 2; ++dim) + { + for (int i=0; i < sharedEntityInfo.get_num_comp(edge1, dim); ++i) + { + assert(idx < buf.size()); + int remoteEntityId = buf[idx++]; + int localEntityId = sharedEntityInfo(edge1, dim, i); + + mesh::MeshEntityPtr localVert = m_meshIn->get_mesh_entities(dim)[localEntityId]; + localVert->add_remote_shared_entity(mesh::RemoteSharedEntity(rank, remoteEntityId)); + } + } + } + }; + + exchanger.complete_receives(unpacker); } void MiddleGridConstraintGenerator::sort_edge_splits(mesh::MeshEntityPtr edge1) @@ -209,7 +356,7 @@ void MiddleGridConstraintGenerator::create_internal_edges() << ", xi = " << edges2ToFakeVertsIn(edge2, 0, j).xi << std::endl; } - if (!get_common_edge(v1In, v2In)) + if (v1In && v2In && !get_common_edge(v1In, v2In)) m_meshIn->create_edge(v1In, v2In); } } diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/middle_grid_constraint_generator.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/middle_grid_constraint_generator.hpp index acb2803604c8..dc484976a596 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/middle_grid_constraint_generator.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/middle_grid_constraint_generator.hpp @@ -2,6 +2,7 @@ #define MIDDLE_GRID_CONSTRAINT_GENERATOR_H #include "mesh_relational_data.hpp" +#include "stk_util/parallel/DataExchangeKnownPatternNonBlockingBuffer.hpp" #include //TODO: DEBUGGING @@ -29,14 +30,26 @@ class MiddleGridConstraintGenerator void generate(); private: + using ExchangerKnown = stk::DataExchangeKnownPatternNonBlockingBuffer; + void create_mesh1_vertices(); + void set_mesh1_vert_shared_entities(ExchangerKnown& exchanger); + void create_mesh2_interior_vertices(); void create_mesh1_edges(); + void set_mesh1_edge_shared_entities(ExchangerKnown& exchanger); + void split_edges(); + void pack_edge_split_shared_info(ExchangerKnown& exchanger, + mesh::VariableSizeFieldPtr sharedEntityInfoPtr, mesh::MeshEntityPtr edge1); + + void set_edge_split_shared_entities(ExchangerKnown& exchanger, + mesh::VariableSizeFieldPtr sharedEntityInfoPtr); + void sort_edge_splits(mesh::MeshEntityPtr edge1); mesh::MeshEntityPtr get_mesh_in_edge_from_mesh1_edge(mesh::MeshEntityPtr edge1); diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/middle_grid_triangulator.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/middle_grid_triangulator.cpp index d9e55895dd49..0a110eac2f64 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/middle_grid_triangulator.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/middle_grid_triangulator.cpp @@ -4,6 +4,7 @@ #include "element_mesh_extractor.hpp" #include "element_mesh_triangulator.hpp" #include "mesh_io.hpp" +#include "adjacency_search.hpp" namespace stk { namespace middle_mesh { @@ -15,7 +16,10 @@ void MiddleGridTriangulator::triangulate() ElementMeshExtractor extractor(m_meshIn, m_relationalData); ElementMeshTriangulator triangulator(m_relationalData); ElementMeshClassifier elementClassifier(m_relationalData, m_classifier); - for (auto& el1 : m_mesh1->get_elements()) + mesh::impl::AdjacencySearch adjSearch(m_mesh1, m_mesh2); + mesh::MeshEntityPtr el1; + std::vector mesh2Els; + while ( (el1 = adjSearch.get_next(mesh2Els)) ) { if (el1) { @@ -38,7 +42,7 @@ void MiddleGridTriangulator::triangulate() extractor.write_elements_back_to_middle_grid(elementMeshData, el1); - elementClassifier.classify(elementMeshData, numConstraintEdges); + elementClassifier.classify(elementMeshData, numConstraintEdges, mesh2Els); } } } diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/nonconformal4.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/nonconformal4.cpp index 690f69d7ced6..c91a5615c1ff 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/nonconformal4.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/nonconformal4.cpp @@ -33,7 +33,7 @@ invert_classification_field(std::shared_ptr meshIn, std::shared_ptr< std::shared_ptr Nonconformal4::create() { { - MeshProjectionCalculator meshProjection(m_mesh1, m_mesh2, m_meshIn, m_relationalData, m_classifier, + MeshProjectionCalculator meshProjection(m_mesh1, m_mesh2, m_relationalData, m_classifier, m_edgeTracerTolerances); meshProjection.project(); } diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/average_normal_field.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/average_normal_field.hpp new file mode 100644 index 000000000000..5e24513ee562 --- /dev/null +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/average_normal_field.hpp @@ -0,0 +1,203 @@ +#ifndef STK_MIDDLE_MESH_AVERAGE_NORMAL_FIELD_H +#define STK_MIDDLE_MESH_AVERAGE_NORMAL_FIELD_H + +#include "stk_middle_mesh/mesh_entity.hpp" +#include "stk_middle_mesh/field.hpp" +#include "stk_middle_mesh/utils.hpp" +#include "stk_util/parallel/DataExchangeKnownPatternNonBlockingBuffer.hpp" + +namespace stk { +namespace middle_mesh { +namespace predicates { +namespace impl { + +class AveragedNormalField +{ + public: + AveragedNormalField(std::shared_ptr mesh) : + m_mesh(mesh), + m_normalField(mesh::create_field(mesh, mesh::FieldShape(1, 0, 0), 1, {0, 0, 0})) + { + auto edgeInfoFieldPtr = mesh::create_field(m_mesh, mesh::FieldShape(1, 0, 0), 1); + + compute_averaged_normal_vectors(edgeInfoFieldPtr); + compute_edge_lengths(edgeInfoFieldPtr); + parallel_sum_edge_info(edgeInfoFieldPtr); + apply_scale_factor(edgeInfoFieldPtr); + } + + mesh::FieldPtr get_field() const { return m_normalField; } + + private: + struct EdgeInfo + { + int count = 0; + utils::Point normal = {0, 0, 0}; + double edgeLen = 0; + + EdgeInfo& operator+=(const EdgeInfo& other) + { + count += other.count; + normal += other.normal; + edgeLen += other.edgeLen; + + return *this; + } + }; + + struct EntityAndLength + { + int entityId; + EdgeInfo lenAndCount; + }; + + + void compute_averaged_normal_vectors(mesh::FieldPtr edgeInfoFieldPtr) + { + auto& edgeInfoField = *edgeInfoFieldPtr; + + std::vector elements; + std::array verts; + for (auto& vert : m_mesh->get_vertices()) + { + if (vert) + { + elements.clear(); + get_upward(vert, 2, elements); + + for (auto el : elements) + { + // get index of vert + int nverts = get_downward(el, 0, verts.data()); + int vertIdx = get_vert_idx(verts, nverts, vert); + assert(vertIdx >= 0 && vertIdx < nverts); + + // get two adjacent vertices + mesh::MeshEntityPtr vertNext = verts[(vertIdx + 1) % nverts]; + mesh::MeshEntityPtr vertPrev = verts[(vertIdx - 1 + nverts) % nverts]; + + // form form vector v_i+1 - v_i and v_i-1 - v_i + utils::Point b1 = vertNext->get_point_orig(0) - vert->get_point_orig(0); + utils::Point b2 = vertPrev->get_point_orig(0) - vert->get_point_orig(0); + + // compute normal + utils::Point normal = cross(b1, b2); + edgeInfoField(vert, 0, 0).normal += normal; + } + } + } + } + + + int get_vert_idx(const std::array& verts, int nverts, mesh::MeshEntityPtr vert) + { + for (int i = 0; i < nverts; ++i) + if (verts[i] == vert) + return i; + + return -1; + } + + + void compute_edge_lengths(mesh::FieldPtr edgeInfoFieldPtr) + { + int myrank = utils::impl::comm_rank(m_mesh->get_comm()); + auto& edgeInfoField = *edgeInfoFieldPtr; + + for (auto& vert : m_mesh->get_vertices()) + { + if (vert) + { + double totalEdgeLength = 0; + int numOwnedEdges = 0; + for (int i = 0; i < vert->count_up(); ++i) + { + mesh::MeshEntityPtr edge = vert->get_up(i); + if (mesh::get_owner(m_mesh, edge) == myrank) + { + utils::Point disp = edge->get_down(1)->get_point_orig(0) - edge->get_down(0)->get_point_orig(0); + totalEdgeLength += std::sqrt(dot(disp, disp)); + numOwnedEdges++; + } + } + + edgeInfoField(vert, 0, 0).count = numOwnedEdges; + edgeInfoField(vert, 0, 0).edgeLen = totalEdgeLength; + } + } + } + + + void parallel_sum_edge_info(mesh::FieldPtr edgeInfoFieldPtr) + { + auto& normalField = *m_normalField; + stk::DataExchangeKnownPatternNonBlockingBuffer exchanger(m_mesh->get_comm()); + auto& edgeInfoField = *edgeInfoFieldPtr; + + for (auto& vert : m_mesh->get_vertices()) + { + if (vert) + { + for (int i=0; i < vert->count_remote_shared_entities(); ++i) + { + mesh::RemoteSharedEntity remote = vert->get_remote_shared_entity(i); + exchanger.get_send_buf(remote.remoteRank).push_back({remote.remoteId, edgeInfoField(vert, 0, 0)}); + exchanger.get_recv_buf(remote.remoteRank).push_back(EntityAndLength{}); + } + } + } + + exchanger.start_nonblocking(); + + auto unpacker = [&](int rank, const std::vector& buf) + { + for (EntityAndLength entityAndLength : buf) + { + EdgeInfo edgeInfo = entityAndLength.lenAndCount; + + mesh::MeshEntityPtr vert = m_mesh->get_vertices()[entityAndLength.entityId]; + edgeInfoField(vert, 0, 0) += edgeInfo; + normalField(vert, 0, 0) = edgeInfoField(vert, 0, 0).normal; + } + }; + + exchanger.complete_receives(unpacker); + } + + + void apply_scale_factor(mesh::FieldPtr edgeInfoFieldPtr) + { + auto& normalField = *m_normalField; + auto& edgeInfoField = *edgeInfoFieldPtr; + for (auto& vert : m_mesh->get_vertices()) + { + if (vert) + { + EdgeInfo edgeInfo = edgeInfoField(vert, 0, 0); + double avgEdgeLength = edgeInfo.edgeLen / edgeInfo.count; + + normalField(vert, 0, 0) = scale_normal(edgeInfo.normal, avgEdgeLength); + } + } + } + + utils::Point scale_normal(const utils::Point& normal, double length) + { + double normalLength = std::sqrt(dot(normal, normal)); + if (normalLength < 1e-13) + throw std::runtime_error("normal vector has zero length"); + + return length * normal / normalLength; + } + + + std::shared_ptr m_mesh; + mesh::FieldPtr m_normalField; +}; + +} +} +} +} + +#endif \ No newline at end of file diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/intersection_common.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/intersection_common.hpp index 1757693e2d0a..3b792ca28686 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/intersection_common.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/intersection_common.hpp @@ -5,6 +5,18 @@ #include #include +namespace stk { +namespace middle_mesh { +namespace nonconformal4 { +namespace impl { + +class MeshRelationalDataScatter; + +} +} +} +} + namespace stk { namespace middle_mesh { namespace predicates { @@ -91,6 +103,8 @@ struct PointRecordForTriangle friend PointClassifierForTriangle; friend TriangleCoordUtils; friend QuadToTriangles; + friend class stk::middle_mesh::nonconformal4::impl::MeshRelationalDataScatter; + friend std::ostream& operator<<(std::ostream& os, const PointRecordForTriangle& record); }; @@ -130,6 +144,7 @@ struct PointRecord friend PointClassifier; // TODO: maybe find a way to not use friends friend QuadToTriangles; friend PointClassifierNormalWrapper; + friend class stk::middle_mesh::nonconformal4::impl::MeshRelationalDataScatter; friend std::ostream& operator<<(std::ostream& os, const PointRecord& record); }; diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/point_classifier_normal_wrapper.cpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/point_classifier_normal_wrapper.cpp index 94e58be045d4..dabcfe07c21b 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/point_classifier_normal_wrapper.cpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/point_classifier_normal_wrapper.cpp @@ -148,15 +148,15 @@ PointRecord PointClassifierNormalWrapper::create_vert_record(mesh::MeshEntityPtr return m_quadToTriangles.create_record(el, vertId); } -PointRecord PointClassifierNormalWrapper::create_edge_record(mesh::MeshEntityPtr el, int edgeId, double edgeXi) +PointRecord PointClassifierNormalWrapper::create_edge_record(mesh::MeshEntityPtr el, int edgeId, double edgeXiOnReferenceEl) { assert(el->get_type() == mesh::MeshEntityType::Triangle || el->get_type() == mesh::MeshEntityType::Quad); if (el->get_type() == mesh::MeshEntityType::Triangle) { - PointRecordForTriangle record = m_triangleCoordUtils.create_record(el, edgeId, edgeXi); + PointRecordForTriangle record = m_triangleCoordUtils.create_record(el, edgeId, edgeXiOnReferenceEl); return PointRecord(PointClassification::Edge, edgeId, el, record); } else // quad - return m_quadToTriangles.create_record(el, edgeId, edgeXi); + return m_quadToTriangles.create_record(el, edgeId, edgeXiOnReferenceEl); } PointRecord PointClassifierNormalWrapper::classify_onto(const PointRecord& record, mesh::MeshEntityPtr el) @@ -313,70 +313,15 @@ utils::Point PointClassifierNormalWrapper::compute_triangle_xi_coords(std::array std::array vertsProjected; for (int i = 0; i < 3; ++i) + { vertsProjected[i] = utils::Point(verts[i][coord1], verts[i][coord2]); + } utils::Point ptProjected(pt[coord1], pt[coord2]); mesh::impl::ElementOperations2D elemOps; return elemOps.compute_tri_xi_coords(vertsProjected, ptProjected); } -namespace { -int get_vert_idx(const std::array& verts, int nverts, mesh::MeshEntityPtr vert) -{ - for (int i = 0; i < nverts; ++i) - if (verts[i] == vert) - return i; - - return -1; -} -} // namespace - -void PointClassifierNormalWrapper::compute_averaged_normal_vectors(std::shared_ptr mesh) -{ - m_normalField = mesh::create_field(mesh, mesh::FieldShape(1, 0, 0), 1, utils::Point(0, 0, 0)); - auto& field = *m_normalField; - - std::vector elements; - std::array verts; - for (auto& vert : mesh->get_vertices()) - if (vert) - { - elements.clear(); - get_upward(vert, 2, elements); - - for (auto el : elements) - { - // get index of vert - int nverts = get_downward(el, 0, verts.data()); - int vertIdx = get_vert_idx(verts, nverts, vert); - assert(vertIdx >= 0 && vertIdx < nverts); - - // get two adjacent vertices - mesh::MeshEntityPtr vertNext = verts[(vertIdx + 1) % nverts]; - mesh::MeshEntityPtr vertPrev = verts[(vertIdx - 1 + nverts) % nverts]; - - // form form vector v_i+1 - v_i and v_i-1 - v_i - utils::Point b1 = vertNext->get_point_orig(0) - vert->get_point_orig(0); - utils::Point b2 = vertPrev->get_point_orig(0) - vert->get_point_orig(0); - - // compute normal - field(vert, 0, 0) += cross(b1, b2); - } - - double totalEdgeLength = 0; - for (int i = 0; i < vert->count_up(); ++i) - { - mesh::MeshEntityPtr edge = vert->get_up(i); - utils::Point disp = edge->get_down(1)->get_point_orig(0) - edge->get_down(0)->get_point_orig(0); - totalEdgeLength += std::sqrt(dot(disp, disp)); - } - double avgEdgeLength = totalEdgeLength / vert->count_up(); - - utils::Point normal = field(vert, 0, 0); - normal = avgEdgeLength * normal / std::sqrt(dot(normal, normal)); - field(vert, 0, 0) = normal; - } -} bool PointClassifierNormalWrapper::is_entity_on_mesh(mesh::MeshEntityPtr entity) { diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/point_classifier_normal_wrapper.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/point_classifier_normal_wrapper.hpp index 145faa8bc0f1..e4d2a2f8d322 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/point_classifier_normal_wrapper.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/predicates/point_classifier_normal_wrapper.hpp @@ -8,6 +8,7 @@ #include "intersection_common.hpp" #include "quad_to_triangles.hpp" #include "triangle_coord_utils.hpp" +#include "average_normal_field.hpp" namespace stk { namespace middle_mesh { @@ -29,10 +30,11 @@ class PointClassifierNormalWrapper const PointClassifierNormalWrapperTolerances& tolerances = PointClassifierNormalWrapperTolerances()) : m_normalClassifier(tolerances.normalInterpolationTolerances) , m_mesh(mesh) + , m_normalField(AveragedNormalField(mesh).get_field()) , m_tolerances(tolerances) - { - compute_averaged_normal_vectors(mesh); - } + {} + + QuadToTriangles& get_quad_to_triangles() { return m_quadToTriangles; } // src_face must be a face on the mesh that was passed to the constructor PointRecord classify(mesh::MeshEntityPtr destFace, mesh::MeshEntityPtr srcFace, const utils::Point& pt); @@ -63,7 +65,7 @@ class PointClassifierNormalWrapper PointRecord create_vert_record(mesh::MeshEntityPtr el, int vertId); - PointRecord create_edge_record(mesh::MeshEntityPtr el, int edgeId, double edgeXi); + PointRecord create_edge_record(mesh::MeshEntityPtr el, int edgeId, double edgeXiOnReferenceEl); PointRecord classify_onto(const PointRecord& record, mesh::MeshEntityPtr el); @@ -80,8 +82,6 @@ class PointClassifierNormalWrapper utils::Point compute_triangle_xi_coords(std::array& verts, const utils::Point& pt); - void compute_averaged_normal_vectors(std::shared_ptr mesh); - bool is_entity_on_mesh(mesh::MeshEntityPtr entity); PointClassifierNormalInterpolation m_normalClassifier; diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/parallel_search.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/search_mesh_element_bounding_box.hpp similarity index 88% rename from packages/stk/stk_middle_mesh/stk_middle_mesh/parallel_search.hpp rename to packages/stk/stk_middle_mesh/stk_middle_mesh/search_mesh_element_bounding_box.hpp index cd0fb61478ff..ee15854b97ef 100644 --- a/packages/stk/stk_middle_mesh/stk_middle_mesh/parallel_search.hpp +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/search_mesh_element_bounding_box.hpp @@ -39,7 +39,8 @@ #include "create_mesh.hpp" #include "mesh_io.hpp" #include "predicates/point_classifier_normal_wrapper.hpp" -#include "bounding_box_search.hpp" +//#include "bounding_box_search.hpp" +#include #include "stk_search/Box.hpp" //#include "stk_transfer/GeometricTransfer.hpp" #include "stk_util/util/SortAndUnique.hpp" @@ -53,22 +54,47 @@ namespace mesh { namespace impl { -class SearchMesh { +class SearchMeshElementBoundingBox { public: using Entity = mesh::MeshEntity; using EntityKey = int64_t; using EntityProc = stk::search::IdentProc; using EntityProcVec = std::vector; - using Point = stk::search::Point; using Box = stk::search::Box; using BoundingBox = std::pair; - SearchMesh(std::shared_ptr inputMesh) + SearchMeshElementBoundingBox(std::shared_ptr inputMesh, MPI_Comm unionComm) : m_mesh(inputMesh) + , m_unionComm(unionComm) {} - MPI_Comm get_comm() const {return m_mesh->get_comm(); } + //MPI_Comm get_comm() const {return m_mesh->get_comm(); } + + void fill_bounding_boxes(std::vector& boundingBoxes) const + { + int proc; + MPI_Comm_rank(m_unionComm, &proc); + auto entities = m_mesh->get_elements(); + + stk::search::Point minCorner, maxCorner; + + for(auto entity : entities) { + fill_bounding_box(entity, minCorner, maxCorner); + EntityProc entityProc(entity->get_id(), proc); + + BoundingBox boundingBox(Box(minCorner, maxCorner), entityProc); + boundingBoxes.push_back(boundingBox); + } + + std::sort(boundingBoxes.begin(), boundingBoxes.end(), + [](const BoundingBox& a, const BoundingBox& b) { return a.second.id() < b.second.id(); }); + }; + + std::shared_ptr get_mesh() const { return m_mesh; } + + private: + using Point = stk::search::Point; void fill_bounding_box(mesh::MeshEntityPtr element, stk::search::Point& minCorner, stk::search::Point& maxCorner) const @@ -95,44 +121,12 @@ class SearchMesh { } } - void fill_bounding_boxes(std::vector& boundingBoxes) const - { - int proc; - MPI_Comm_rank(MPI_COMM_WORLD, &proc); - auto entities = m_mesh->get_elements(); - - stk::search::Point minCorner, maxCorner; - - for(auto entity : entities) { - fill_bounding_box(entity, minCorner, maxCorner); - EntityProc entityProc(entity->get_id(), proc); - - BoundingBox boundingBox(Box(minCorner, maxCorner), entityProc); - boundingBoxes.push_back(boundingBox); - } - - std::sort(boundingBoxes.begin(), boundingBoxes.end(), - [](const BoundingBox& a, const BoundingBox& b) { return a.second.id() < b.second.id(); }); - }; - - std::shared_ptr get_mesh() const { return m_mesh; } - - private: std::shared_ptr m_mesh; + MPI_Comm m_unionComm; }; -using BoundingBoxSearch = - stk::middle_mesh::search::BoundingBoxSearch< - stk::middle_mesh::search::BoundingBoxSearchType>; -using SearchRelationVec = BoundingBoxSearch::EntityProcRelationVec; -using UnpairedRelationVec = std::vector; -enum class SplitCommColor { - RECV = 0, - SEND, - INVALID -}; } } diff --git a/packages/stk/stk_middle_mesh/stk_middle_mesh/search_mesh_vertex.hpp b/packages/stk/stk_middle_mesh/stk_middle_mesh/search_mesh_vertex.hpp new file mode 100644 index 000000000000..d8e6c0b9a843 --- /dev/null +++ b/packages/stk/stk_middle_mesh/stk_middle_mesh/search_mesh_vertex.hpp @@ -0,0 +1,83 @@ +#ifndef STK_MIDDLE_MESH_VERTEX_SEARCH_MESH_H +#define STK_MIDDLE_MESH_VERTEX_SEARCH_MESH_H + +#include "mesh.hpp" +#include "field.hpp" +#include "predicates/average_normal_field.hpp" +#include "stk_search/Box.hpp" +#include "stk_search/IdentProc.hpp" +#include "utils.hpp" + +namespace stk { +namespace middle_mesh { +namespace mesh { +namespace impl { + +class SearchMeshVertex +{ + public: + using Entity = MeshEntity; + using EntityKey = int64_t; + + using EntityProc = stk::search::IdentProc; + using EntityProcVec = std::vector; + using Box = stk::search::Box; + using BoundingBox = std::pair; + + explicit SearchMeshVertex(std::shared_ptr mesh, MPI_Comm unionComm, double boxNormalFac=1) : + m_mesh(mesh), + m_normalField(predicates::impl::AveragedNormalField(mesh).get_field()), + m_boxNormalFac(boxNormalFac), + m_unionComm(unionComm) + {} + + + void fill_bounding_boxes(std::vector& boundingBoxes) const + { + int proc = utils::impl::comm_rank(m_unionComm); // Local search only + boundingBoxes.reserve(m_mesh->get_vertices().size()); + for (auto& vert : m_mesh->get_vertices()) + if (vert) + { + Box box = create_bounding_box(vert); + EntityProc entityProc(vert->get_id(), proc); + boundingBoxes.push_back(BoundingBox(box, entityProc)); + } + } + + + private: + using Point = stk::search::Point; + + //TODO: what we really want is a line segment class + Box create_bounding_box(MeshEntityPtr vert) const + { + utils::Point normal = (*m_normalField)(vert, 0, 0); + utils::Point pt1 = normal * m_boxNormalFac + vert->get_point_orig(0); + utils::Point pt2 = -normal * m_boxNormalFac + vert->get_point_orig(0); + + Point minCorner, maxCorner; + for (int d=0; d < 3; ++d) + { + minCorner[d] = std::min(pt1[d], pt2[d]); + maxCorner[d] = std::max(pt1[d], pt2[d]); + } + + return Box(minCorner, maxCorner); + } + + + std::shared_ptr m_mesh; + FieldPtr m_normalField; + double m_boxNormalFac; + MPI_Comm m_unionComm; + +}; + + +} +} +} +} + +#endif \ No newline at end of file diff --git a/packages/stk/stk_ngp_test/CMakeLists.txt b/packages/stk/stk_ngp_test/CMakeLists.txt index f233939e22a9..67ac03a282a3 100644 --- a/packages/stk/stk_ngp_test/CMakeLists.txt +++ b/packages/stk/stk_ngp_test/CMakeLists.txt @@ -32,8 +32,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TRIBITS_SUBPACKAGE(NGP_TEST) +STK_SUBPACKAGE(NGP_TEST) ADD_SUBDIRECTORY(stk_ngp_test) -TRIBITS_SUBPACKAGE_POSTPROCESS() +STK_SUBPACKAGE_POSTPROCESS() diff --git a/packages/stk/stk_ngp_test/stk_ngp_test/CMakeLists.txt b/packages/stk/stk_ngp_test/stk_ngp_test/CMakeLists.txt index 235b3cc27917..53db94d5e467 100644 --- a/packages/stk/stk_ngp_test/stk_ngp_test/CMakeLists.txt +++ b/packages/stk/stk_ngp_test/stk_ngp_test/CMakeLists.txt @@ -32,22 +32,29 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/..) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_ngp_test/stk_ngp_test) - FILE(GLOB HEADERS *.hpp) FILE(GLOB SOURCES ngp_test.cpp) -TRIBITS_ADD_LIBRARY( - stk_ngp_test +if(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY(stk_ngp_test NOINSTALLHEADERS ${HEADERS} SOURCES ${SOURCES} ${Gtest_NO_INSTALL_LIB_OR_HEADERS_ARG} ) +else() + add_library(stk_ngp_test ${SOURCES}) + target_link_libraries(stk_ngp_test PUBLIC stk_util_ngp) + target_link_libraries(stk_ngp_test PUBLIC Kokkos::kokkos) +endif() + +target_include_directories(stk_ngp_test PUBLIC + $ + $ +) INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_ngp_test/) +if(NOT HAVE_STK_Trilinos) + INSTALL(TARGETS stk_ngp_test DESTINATION ${STK_INSTALL_LIBDIR}) +endif() diff --git a/packages/stk/stk_performance_tests/stk_mesh/NodalFieldPerf.cpp b/packages/stk/stk_performance_tests/stk_mesh/NodalFieldPerf.cpp index bd6370e57ce9..e067078e2590 100644 --- a/packages/stk/stk_performance_tests/stk_mesh/NodalFieldPerf.cpp +++ b/packages/stk/stk_performance_tests/stk_mesh/NodalFieldPerf.cpp @@ -63,18 +63,15 @@ class NgpFieldAccessPerformance : public stk::unit_test_util::simple_fields::Mes using DoubleVecField = stk::mesh::Field; NgpFieldAccessPerformance() - : batchTimer(get_comm()), - m_fieldDataManager(nullptr) + : batchTimer(get_comm()) { } virtual ~NgpFieldAccessPerformance() { reset_mesh(); - delete m_fieldDataManager; - m_fieldDataManager = nullptr; } void setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::AutomaticAuraOption auraOption, - stk::mesh::FieldDataManager & fieldDataManager, + std::unique_ptr fieldDataManager, unsigned initialBucketCapacity = stk::mesh::get_default_initial_bucket_capacity(), unsigned maximumBucketCapacity = stk::mesh::get_default_maximum_bucket_capacity()) { @@ -82,7 +79,7 @@ class NgpFieldAccessPerformance : public stk::unit_test_util::simple_fields::Mes builder.set_spatial_dimension(m_spatialDim); builder.set_entity_rank_names(m_entityRankNames); builder.set_aura_option(auraOption); - builder.set_field_data_manager(&fieldDataManager); + builder.set_field_data_manager(std::move(fieldDataManager)); builder.set_initial_bucket_capacity(initialBucketCapacity); builder.set_maximum_bucket_capacity(maximumBucketCapacity); @@ -234,7 +231,6 @@ class NgpFieldAccessPerformance : public stk::unit_test_util::simple_fields::Mes static constexpr double gamma = 3.14159; stk::unit_test_util::BatchTimer batchTimer; - stk::mesh::FieldDataManager * m_fieldDataManager; DoubleVecField * dispField; DoubleVecField * velField; @@ -248,10 +244,10 @@ TEST_F(NgpFieldAccessPerformance, pureHost_vectorSum_DefaultFieldDataManager) unsigned numElemsPerDim = 100; const int weKnowThereAreFiveRanks = 5; - m_fieldDataManager = new stk::mesh::DefaultFieldDataManager(weKnowThereAreFiveRanks); + auto fieldDataManager = std::make_unique(weKnowThereAreFiveRanks); batchTimer.initialize_batch_timer(); - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, *m_fieldDataManager); + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); createNodalVectorFields(); stk::io::fill_mesh(stk::unit_test_util::simple_fields::get_mesh_spec(numElemsPerDim), *bulkData); @@ -273,10 +269,10 @@ TEST_F(NgpFieldAccessPerformance, host_vectorSum_DefaultFieldDataManager) unsigned numElemsPerDim = 100; const int weKnowThereAreFiveRanks = 5; - m_fieldDataManager = new stk::mesh::DefaultFieldDataManager(weKnowThereAreFiveRanks); + auto fieldDataManager = std::make_unique(weKnowThereAreFiveRanks); batchTimer.initialize_batch_timer(); - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, *m_fieldDataManager); + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); createNodalVectorFields(); stk::io::fill_mesh(stk::unit_test_util::simple_fields::get_mesh_spec(numElemsPerDim), *bulkData); @@ -310,10 +306,10 @@ TEST_F(NgpFieldAccessPerformance, vectorSum_DefaultFieldDataManager) unsigned numElemsPerDim = 100; const int weKnowThereAreFiveRanks = 5; - m_fieldDataManager = new stk::mesh::DefaultFieldDataManager(weKnowThereAreFiveRanks); + auto fieldDataManager = std::make_unique(weKnowThereAreFiveRanks); batchTimer.initialize_batch_timer(); - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, *m_fieldDataManager); + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); createNodalVectorFields(); fill_mesh(*bulkData, numElemsPerDim); @@ -334,10 +330,10 @@ TEST_F(NgpFieldAccessPerformance, vectorSum_ContiguousFieldDataManager) if (get_parallel_size() != 1) return; unsigned numElemsPerDim = 100; - m_fieldDataManager = new stk::mesh::ContiguousFieldDataManager; + auto fieldDataManager = std::make_unique(); batchTimer.initialize_batch_timer(); - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, *m_fieldDataManager); + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); createNodalVectorFields(); fill_mesh(*bulkData, numElemsPerDim); diff --git a/packages/stk/stk_performance_tests/stk_util/perfTagManager.cpp b/packages/stk/stk_performance_tests/stk_util/perfTagManager.cpp index a2f557ffde0e..60c43c1754e8 100644 --- a/packages/stk/stk_performance_tests/stk_util/perfTagManager.cpp +++ b/packages/stk/stk_performance_tests/stk_util/perfTagManager.cpp @@ -86,6 +86,8 @@ TEST(MPITagManager, timing) { const unsigned NUM_RUNS = 5; + perf_test(32, 16, 256000, 16384); + stk::unit_test_util::BatchTimer batchTimer(MPI_COMM_WORLD); batchTimer.initialize_batch_timer(); @@ -96,14 +98,12 @@ TEST(MPITagManager, timing) perf_test(32, 16, 1024, 96); perf_test(32, 16, 2048, 2048); perf_test(32, 16, 2048, 96); - perf_test(32, 16, 4096, 4096); - perf_test(32, 16, 4096, 96); perf_test(32, 16, 8192, 8192); perf_test(32, 16, 8192, 96); - perf_test(32, 16, 16384, 16384); - perf_test(32, 16, 16384, 96); perf_test(32, 16, 32768, 16384); perf_test(32, 16, 32768, 96); + perf_test(32, 16, 256000, 16384); + perf_test(32, 16, 256000, 96); batchTimer.stop_batch_timer(); } diff --git a/packages/stk/stk_search/CMakeLists.txt b/packages/stk/stk_search/CMakeLists.txt index 1d3f9f7782fc..726a1e51b083 100644 --- a/packages/stk/stk_search/CMakeLists.txt +++ b/packages/stk/stk_search/CMakeLists.txt @@ -1,6 +1,6 @@ -TRIBITS_SUBPACKAGE(Search) +STK_SUBPACKAGE(Search) ADD_SUBDIRECTORY(stk_search) -TRIBITS_SUBPACKAGE_POSTPROCESS() +STK_SUBPACKAGE_POSTPROCESS() diff --git a/packages/stk/stk_search/stk_search/CMakeLists.txt b/packages/stk/stk_search/stk_search/CMakeLists.txt index 1396b2a47d41..bc33712238c6 100644 --- a/packages/stk/stk_search/stk_search/CMakeLists.txt +++ b/packages/stk/stk_search/stk_search/CMakeLists.txt @@ -31,41 +31,35 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - -# -# A) Package-specific configuration options -# - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - SET(HEADERS "") SET(SOURCES "") -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) - -# -# Core headers -# - FILE(GLOB HEADERS *.hpp) - FILE(GLOB SOURCES *.cpp) -# -# C) Define the targets for package's library(s) -# - -TRIBITS_ADD_LIBRARY( - stk_search - NOINSTALLHEADERS ${HEADERS} - SOURCES ${SOURCES} +if(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY(stk_search + NOINSTALLHEADERS ${HEADERS} + SOURCES ${SOURCES} ) +else() + add_library(stk_search ${SOURCES}) + + target_link_libraries(stk_search PUBLIC stk_math) + target_link_libraries(stk_search PUBLIC stk_util_parallel) + + find_package(KokkosKernels REQUIRED) + target_link_libraries(stk_search PUBLIC Kokkos::kokkoskernels) +endif() + +target_include_directories(stk_search PUBLIC + $ + $ +) INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_search) + +if(NOT HAVE_STK_Trilinos) + INSTALL(TARGETS stk_search DESTINATION ${STK_INSTALL_LIBDIR}) +endif() diff --git a/packages/stk/stk_search/stk_search/FilterCoarseSearch.hpp b/packages/stk/stk_search/stk_search/FilterCoarseSearch.hpp index a1db95b133fb..926b5aaaf57a 100644 --- a/packages/stk/stk_search/stk_search/FilterCoarseSearch.hpp +++ b/packages/stk/stk_search/stk_search/FilterCoarseSearch.hpp @@ -62,7 +62,9 @@ using FilterCoarseSearchProcRelation = std::pair using FilterCoarseSearchProcRelationVec = std::vector>; +//BEGINObjectOutsideDomainPolicy enum class ObjectOutsideDomainPolicy { IGNORE, EXTRAPOLATE, TRUNCATE, PROJECT, ABORT, UNDEFINED_OBJFLAG = 0xff }; +//ENDObjectOutsideDomainPolicy inline ObjectOutsideDomainPolicy get_object_outside_domain_policy(const std::string& id) { @@ -95,6 +97,7 @@ inline std::string get_object_outside_domain_policy(const ObjectOutsideDomainPol return std::string(""); } +//BEGINFilterCoarseSearchOptions struct FilterCoarseSearchOptions { std::ostream& m_outputStream{std::cout}; @@ -102,6 +105,7 @@ struct FilterCoarseSearchOptions bool m_useNearestNodeForClosestBoundingBox{false}; bool m_useCentroidForGeometricProximity{false}; bool m_verbose{true}; +//ENDFilterCoarseSearchOptions FilterCoarseSearchOptions(std::ostream& out) : m_outputStream(out) {} @@ -119,6 +123,7 @@ struct FilterCoarseSearchOptions , m_verbose(verbose) {} }; +//BEGINFilterCoarseSearchResult template class FilterCoarseSearchResult { @@ -137,9 +142,12 @@ class FilterCoarseSearchResult virtual ~FilterCoarseSearchResult() {} }; +//ENDFilterCoarseSearchResult +//BEGINFilterCoarseSearchResultMap template class FilterCoarseSearchResultMap : public FilterCoarseSearchResult +//ENDFilterCoarseSearchResultMap { public: using EntityKey = typename RECVMESH::EntityKey; @@ -167,8 +175,10 @@ class FilterCoarseSearchResultMap : public FilterCoarseSearchResult std::map> m_searchFilterInfo; }; +//BEGINFilterCoarseSearchResultVector template class FilterCoarseSearchResultVector : public FilterCoarseSearchResult +//ENDFilterCoarseSearchResultVector { public: using EntityKey = typename RECVMESH::EntityKey; @@ -499,12 +509,14 @@ void output_summary_outside_tolerance(SENDMESH& sendMesh, RECVMESH& recvMesh, Ou } } +//BEGINfilter_coarse_search_impl template void filter_coarse_search(const std::string& name, FilterCoarseSearchProcRelationVec& rangeToDomain, SENDMESH& sendMesh, RECVMESH& recvMesh, FilterCoarseSearchOptions& filterOptions, FilterCoarseSearchResult& filterResult) +//ENDfilter_coarse_search_impl { const double parametricTol = recvMesh.get_parametric_tolerance(); const double geometricTol = recvMesh.get_search_tolerance(); diff --git a/packages/stk/stk_search/stk_search/SearchInterface.hpp b/packages/stk/stk_search/stk_search/SearchInterface.hpp index 29897c2cdefd..c0dae3f3173b 100644 --- a/packages/stk/stk_search/stk_search/SearchInterface.hpp +++ b/packages/stk/stk_search/stk_search/SearchInterface.hpp @@ -35,25 +35,26 @@ #ifndef STK_STK_SEARCH_STK_SEARCH_SEARCHINTERFACE_HPP_ #define STK_STK_SEARCH_STK_SEARCH_SEARCHINTERFACE_HPP_ -#include -#include -#include -#include +#include +#include #include -#include -#include #include -#include +#include +#include +#include #include -#include - -#include "stk_util/parallel/Parallel.hpp" // for parallel_machin... +#include +#include -namespace stk { -namespace search { +#include "stk_util/parallel/Parallel.hpp" // for parallel_machin... +namespace stk +{ +namespace search +{ template -class EvaluatePointsInterface { +class EvaluatePointsInterface +{ public: virtual size_t num_points(ENTITYKEY, TOPOLOGY) = 0; virtual const double* coords(ENTITYKEY, size_t) = 0; @@ -62,45 +63,48 @@ class EvaluatePointsInterface { }; template -class ExternalPointHandlerInterface { +class ExternalPointHandlerInterface +{ public: - virtual bool - handle_point(const ENTITYKEY k, const double* toCoords, - std::vector& parametricCoords, - double& geometricDistanceSquared, - bool& isWithinGeometricTolerance) const = 0; + virtual bool handle_point(const ENTITYKEY k, + const double* toCoords, + std::vector& parametricCoords, + double& geometricDistanceSquared, + bool& isWithinGeometricTolerance) const = 0; virtual ~ExternalPointHandlerInterface() {} }; template -class FindParametricCoordsInterface { +class FindParametricCoordsInterface +{ public: - virtual void - find_parametric_coords(const ENTITYKEY k, const double* toCoords, - std::vector& paramCoords, double& paramDistance, - bool& isWithinParametricTolerance) const = 0; - virtual void - evaluate_parametric_coords(const ENTITYKEY k, const std::vector& paramCoords, - std::vector& evalPoint) const = 0; + virtual void find_parametric_coords(const ENTITYKEY k, + const double* toCoords, + std::vector& paramCoords, + double& paramDistance, + bool& isWithinParametricTolerance) const = 0; + virtual void evaluate_parametric_coords( + const ENTITYKEY k, const std::vector& paramCoords, std::vector& evalPoint) const = 0; virtual ~FindParametricCoordsInterface() {} }; template -class ProvideMasterElementInterface { +class ProvideMasterElementInterface +{ public: virtual void evaluate_field(const TOPOLOGY& topo, - const std::vector& paramCoords, // (numParamCoords) - const unsigned numFieldComponents, - const std::vector& fieldData, // (numFieldComponents x numNodes) - std::vector& result) = 0; // (numFieldComponents) + const std::vector& paramCoords, // (numParamCoords) + const unsigned numFieldComponents, + const std::vector& fieldData, // (numFieldComponents x numNodes) + std::vector& result) = 0; // (numFieldComponents) virtual void find_parametric_coordinates(const TOPOLOGY& topo, - const unsigned numCoordComponents, - const std::vector& elemNodeCoords, // (numCoordComponents x numNodes) - const std::vector& inputCoords, // (numCoordComponents) - std::vector& paramCoords, - double& paramDistance) = 0; + const unsigned numCoordComponents, + const std::vector& elemNodeCoords, // (numCoordComponents x numNodes) + const std::vector& inputCoords, // (numCoordComponents) + std::vector& paramCoords, + double& paramDistance) = 0; virtual void coordinate_center(const TOPOLOGY& topo, std::vector&) = 0; virtual unsigned num_parametric_coordinates(const TOPOLOGY& topo) = 0; @@ -109,15 +113,22 @@ class ProvideMasterElementInterface { virtual ~ProvideMasterElementInterface() {} }; +//BEGINSearch_Interface template struct MeshTraits; template class SourceMeshInterface { -public: - using BOUNDINGBOX = typename MeshTraits::BoundingBox; - using ENTITYKEY = typename MeshTraits::EntityKey; + public: + using Entity = typename MeshTraits::Entity; + using EntityVec = typename MeshTraits::EntityVec; + using EntityKey = typename MeshTraits::EntityKey; + using EntityProc = typename MeshTraits::EntityProc; + using EntityProcVec = typename MeshTraits::EntityProcVec; + using Point = typename MeshTraits::Point; + using Box = typename MeshTraits::Box; + using BoundingBox = typename MeshTraits::BoundingBox; SourceMeshInterface() = default; virtual ~SourceMeshInterface() = default; @@ -126,38 +137,51 @@ class SourceMeshInterface virtual std::string name() const = 0; - virtual void bounding_boxes(std::vector& boxes) const = 0; + virtual void bounding_boxes(std::vector& boxes) const = 0; - virtual void find_parametric_coords(const ENTITYKEY k, const double* toCoords, - std::vector& parametricCoords, - double& parametricDistance, - bool& isWithinParametricTolerance) const = 0; + virtual void find_parametric_coords( + const EntityKey k, + const double* toCoords, + std::vector& parametricCoords, + double& parametricDistance, + bool& isWithinParametricTolerance) const = 0; - virtual bool modify_search_outside_parametric_tolerance(const ENTITYKEY k, - const double* toCoords, - std::vector& parametricCoords, - double& geometricDistanceSquared, - bool& isWithinGeometricTolerance) const = 0; + virtual bool modify_search_outside_parametric_tolerance( + const EntityKey k, + const double* toCoords, + std::vector& parametricCoords, + double& geometricDistanceSquared, + bool& isWithinGeometricTolerance) const = 0; - virtual double get_distance_from_nearest_node(const ENTITYKEY k, const double* point) const = 0; + virtual double get_distance_from_nearest_node( + const EntityKey k, const double* point) const = 0; - virtual double get_closest_geometric_distance_squared(const ENTITYKEY k, const double* toCoords) const = 0; + virtual double get_closest_geometric_distance_squared( + const EntityKey k, const double* toCoords) const = 0; - virtual double get_distance_from_centroid(const ENTITYKEY k, const double* toCoords) const = 0; + virtual double get_distance_from_centroid( + const EntityKey k, const double* toCoords) const = 0; - virtual double get_distance_squared_from_centroid(const ENTITYKEY k, const double* toCoords) const = 0; + virtual double get_distance_squared_from_centroid( + const EntityKey k, const double* toCoords) const = 0; - virtual void centroid(const ENTITYKEY k, std::vector& centroidVec) const = 0; + virtual void centroid(const EntityKey k, std::vector& centroidVec) const = 0; - virtual const double* coord(const ENTITYKEY k) const = 0; + virtual const double* coord(const EntityKey k) const = 0; }; template class DestinationMeshInterface { -public: - using BOUNDINGBOX = typename MeshTraits::BoundingBox; - using ENTITYKEY = typename MeshTraits::EntityKey; + public: + using Entity = typename MeshTraits::Entity; + using EntityVec = typename MeshTraits::EntityVec; + using EntityKey = typename MeshTraits::EntityKey; + using EntityProc = typename MeshTraits::EntityProc; + using EntityProcVec = typename MeshTraits::EntityProcVec; + using Point = typename MeshTraits::Point; + using Sphere = typename MeshTraits::Sphere; + using BoundingBox = typename MeshTraits::BoundingBox; DestinationMeshInterface() = default; virtual ~DestinationMeshInterface() = default; @@ -166,16 +190,19 @@ class DestinationMeshInterface virtual std::string name() const = 0; - virtual void bounding_boxes(std::vector& v) const = 0; + virtual void bounding_boxes(std::vector& v) const = 0; - virtual const double* coord(const ENTITYKEY k) const = 0; + virtual const double* coord(const EntityKey k) const = 0; virtual double get_search_tolerance() const = 0; virtual double get_parametric_tolerance() const = 0; - virtual void centroid(const ENTITYKEY k, std::vector& centroidVec) const = 0; - virtual double get_distance_from_nearest_node(const ENTITYKEY k, const double* toCoords) const = 0; + virtual void centroid(const EntityKey k, std::vector& centroidVec) const = 0; + virtual double get_distance_from_nearest_node( + const EntityKey k, const double* toCoords) const = 0; }; +//ENDSearch_Interface -}} +} // namespace search +} // namespace stk #endif /* STK_STK_SEARCH_STK_SEARCH_SEARCHINTERFACE_HPP_ */ diff --git a/packages/stk/stk_simd/CMakeLists.txt b/packages/stk/stk_simd/CMakeLists.txt index beb6a6f627b9..d6d650c7f799 100644 --- a/packages/stk/stk_simd/CMakeLists.txt +++ b/packages/stk/stk_simd/CMakeLists.txt @@ -1,5 +1,5 @@ -TRIBITS_SUBPACKAGE(Simd) +STK_SUBPACKAGE(Simd) ADD_SUBDIRECTORY(stk_math) @@ -7,5 +7,5 @@ ADD_SUBDIRECTORY(stk_simd) ADD_SUBDIRECTORY(stk_simd_view) -TRIBITS_SUBPACKAGE_POSTPROCESS() +STK_SUBPACKAGE_POSTPROCESS() diff --git a/packages/stk/stk_simd/stk_math/CMakeLists.txt b/packages/stk/stk_simd/stk_math/CMakeLists.txt index 934e9872b89c..5feb7423cbfd 100644 --- a/packages/stk/stk_simd/stk_math/CMakeLists.txt +++ b/packages/stk/stk_simd/stk_math/CMakeLists.txt @@ -32,38 +32,11 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - -# -# A) Package-specific configuration options -# - -#TRIBITS_CONFIGURE_FILE(STK_config.h) - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - SET(HEADERS "") SET(SOURCES "") -# -# Core headers -# - -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) - FILE(GLOB HEADERS *.hpp) -# -# C) Define the targets for package's library(s) -# - INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_math) diff --git a/packages/stk/stk_simd/stk_simd/CMakeLists.txt b/packages/stk/stk_simd/stk_simd/CMakeLists.txt index 4ab6a0c401ee..e6866751aac5 100644 --- a/packages/stk/stk_simd/stk_simd/CMakeLists.txt +++ b/packages/stk/stk_simd/stk_simd/CMakeLists.txt @@ -32,40 +32,28 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - -# -# A) Package-specific configuration options -# - -#TRIBITS_CONFIGURE_FILE(STK_config.h) - -# -# B) Define the header and source files (and directories) -# - ADD_SUBDIRECTORY(kokkos_simd) -# -# src -# - SET(HEADERS "") SET(SOURCES "") -# -# Core headers -# - -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) - FILE(GLOB HEADERS *.hpp) FILE(GLOB KOKKOS_SIMD_HEADERS kokkos_simd/*.hpp) -# -# C) Define the targets for package's library(s) -# +IF(HAVE_STK_Trilinos) + message("in stk_simd, PACKAGE_NAME: ${PACKAGE_NAME} PARENT_PACKAGE_NAME: ${PARENT_PACKAGE_NAME}") + TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/..) + TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_simd) +ELSE() + add_library(stk_simd INTERFACE ${HEADERS}) + target_link_libraries(stk_simd INTERFACE stk_math) + target_link_libraries(stk_simd INTERFACE stk_util_util) + + target_include_directories(stk_simd INTERFACE + $ + $ + ) +ENDIF() INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_simd/) diff --git a/packages/stk/stk_simd/stk_simd/kokkos_simd/CMakeLists.txt b/packages/stk/stk_simd/stk_simd/kokkos_simd/CMakeLists.txt index b356dd891bc3..ec9ce4be3d9a 100644 --- a/packages/stk/stk_simd/stk_simd/kokkos_simd/CMakeLists.txt +++ b/packages/stk/stk_simd/stk_simd/kokkos_simd/CMakeLists.txt @@ -32,24 +32,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - -# src -# - -SET(HEADERS "") -SET(SOURCES "") - -# -# Core headers -# - -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) - -FILE(GLOB HEADERS *.hpp) - -# -# C) Define the targets for package's library(s) -# - +if(HAVE_STK_Trilinos) + TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) + TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) +endif() diff --git a/packages/stk/stk_simd/stk_simd_view/CMakeLists.txt b/packages/stk/stk_simd/stk_simd_view/CMakeLists.txt index 772942dbfabb..a35b9626ce7b 100644 --- a/packages/stk/stk_simd/stk_simd_view/CMakeLists.txt +++ b/packages/stk/stk_simd/stk_simd_view/CMakeLists.txt @@ -32,38 +32,11 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - -# -# A) Package-specific configuration options -# - -#TRIBITS_CONFIGURE_FILE(STK_config.h) - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - SET(HEADERS "") SET(SOURCES "") -# -# Core headers -# - -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) - FILE(GLOB HEADERS *.hpp) -# -# C) Define the targets for package's library(s) -# - INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_simd_view) diff --git a/packages/stk/stk_simd/stk_simd_view/simd_parallel.hpp b/packages/stk/stk_simd/stk_simd_view/simd_parallel.hpp index 86e37163fd85..c699077dd2d4 100644 --- a/packages/stk/stk_simd/stk_simd_view/simd_parallel.hpp +++ b/packages/stk/stk_simd/stk_simd_view/simd_parallel.hpp @@ -65,8 +65,8 @@ struct DeduceFunctorExecutionSpace { template KOKKOS_INLINE_FUNCTION constexpr bool is_gpu() { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) using execution_space = typename internal::DeduceFunctorExecutionSpace::execution_space; -#ifdef KOKKOS_ENABLE_CUDA return std::is_same::value; #else return false; diff --git a/packages/stk/stk_tools/stk_tools/CMakeLists.txt b/packages/stk/stk_tools/stk_tools/CMakeLists.txt index c800efad80ad..3c52a9064cfb 100644 --- a/packages/stk/stk_tools/stk_tools/CMakeLists.txt +++ b/packages/stk/stk_tools/stk_tools/CMakeLists.txt @@ -33,18 +33,6 @@ # -# -# A) Package-specific configuration options -# - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - SET(HEADERS "") SET(SOURCES "") @@ -59,8 +47,6 @@ FILE(GLOB TRANSFER_UTILS_HEADERS transfer_utils/*.hpp) FILE(GLOB TRANSFER_UTILS_SOURCES transfer_utils/*.cpp) LIST(REMOVE_ITEM BLOCK_EXTRACTOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/block_extractor/main.cpp) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) - # # C) Define the targets for package's library(s) # @@ -77,6 +63,15 @@ TRIBITS_ADD_LIBRARY( SOURCES ${MESH_CLONE_SOURCES} ${MESH_TOOLS_SOURCES} ${BLOCK_EXTRACTOR_SOURCES} ) +target_include_directories(stk_transfer_utils_lib PUBLIC + $ + $ +) +target_include_directories(stk_tools_lib PUBLIC + $ + $ +) + INSTALL(FILES ${MESH_CLONE_HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_tools/mesh_clone) diff --git a/packages/stk/stk_tools/stk_tools/mesh_tools/DisconnectBlocks.cpp b/packages/stk/stk_tools/stk_tools/mesh_tools/DisconnectBlocks.cpp index 1c47efca6690..6ea261e82a70 100644 --- a/packages/stk/stk_tools/stk_tools/mesh_tools/DisconnectBlocks.cpp +++ b/packages/stk/stk_tools/stk_tools/mesh_tools/DisconnectBlocks.cpp @@ -67,7 +67,7 @@ bool has_nodes_in_part(const stk::mesh::BulkData& bulk, const stk::mesh::Part* p unsigned localCount = stk::mesh::count_selected_entities(*part, bulk.buckets(stk::topology::NODE_RANK)); unsigned globalCount; - MPI_Allreduce(&localCount, &globalCount, 1, MPI_UNSIGNED, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(&localCount, &globalCount, 1, MPI_UNSIGNED, MPI_SUM, bulk.parallel()); return (globalCount > 0); } diff --git a/packages/stk/stk_tools/stk_tools/mesh_tools/DisconnectBlocksImpl.hpp b/packages/stk/stk_tools/stk_tools/mesh_tools/DisconnectBlocksImpl.hpp index 283c62169406..55c476912ff4 100644 --- a/packages/stk/stk_tools/stk_tools/mesh_tools/DisconnectBlocksImpl.hpp +++ b/packages/stk/stk_tools/stk_tools/mesh_tools/DisconnectBlocksImpl.hpp @@ -164,10 +164,10 @@ struct LinkInfo flush(std::cerr); } - std::ostream& print_debug_msg(int userDebugLevel, bool prefixMsg = true) { + std::ostream& print_debug_msg(int userDebugLevel, int mpiRank, bool prefixMsg = true) { if(userDebugLevel <= debugLevel) { if(prefixMsg) { - os << "P" << stk::parallel_machine_rank(MPI_COMM_WORLD) << ": "; + os << "P" << mpiRank << ": "; } return os; } else { @@ -175,8 +175,8 @@ struct LinkInfo } } - std::ostream& print_debug_msg_p0(int userDebugLevel, bool prefixMsg = true) { - if(stk::parallel_machine_rank(MPI_COMM_WORLD) == 0) { + std::ostream& print_debug_msg_p0(int userDebugLevel, int mpiRank, bool prefixMsg = true) { + if(mpiRank == 0) { return print_debug_msg(userDebugLevel, prefixMsg); } return ns; diff --git a/packages/stk/stk_tools/stk_tools/pmesh_lib/makeparfiles.cpp b/packages/stk/stk_tools/stk_tools/pmesh_lib/makeparfiles.cpp index 8a1514ae9daa..0fa24c4698dc 100644 --- a/packages/stk/stk_tools/stk_tools/pmesh_lib/makeparfiles.cpp +++ b/packages/stk/stk_tools/stk_tools/pmesh_lib/makeparfiles.cpp @@ -3,12 +3,13 @@ #include "makeparfiles.H" #include // for assert #include // for ex_put_node_set_param, ex_put_node_set, ex_... -#include // for MPI_Abort, MPI_COMM_WORLD +#include #include // for ne_put_cmap_params, ne_put_eb_info_global #include // for sprintf, printf #include // for exit #include // for strcpy #include // for operator<<, basic_ostream, char_traits, cerr +#include // clang-format on // ####################### End Clang Header Tool Managed Headers ######################## @@ -523,9 +524,7 @@ void SetFileName(char dirloc[], int total_subdomains, int i, sprintf(fn, "%s%d.%.6d", base_fn, total_subdomains, i); break; default: - std::cerr << "no more than 6 digits are supported " << std::endl; - MPI_Abort(MPI_COMM_WORLD, digit); - exit(-1); + throw std::runtime_error("no more than 6 digits are supported"); } @@ -549,10 +548,9 @@ int OpenExoFile(char filename[300] ) // Check for errors in creating the Exodus file if(exoid < 0) { - int ierr=0; - std::cerr << "\nCould not create file " << filename << "\n. Aborting.\n\n"; - MPI_Abort(MPI_COMM_WORLD, ierr); - exit(-1); + std::ostringstream os; + os<<"\nCould not create file " << filename << "\n."; + throw std::runtime_error(os.str()); } return exoid; } diff --git a/packages/stk/stk_topology/CMakeLists.txt b/packages/stk/stk_topology/CMakeLists.txt index f48481bc1161..2f13523c99e5 100644 --- a/packages/stk/stk_topology/CMakeLists.txt +++ b/packages/stk/stk_topology/CMakeLists.txt @@ -1,6 +1,6 @@ -TRIBITS_SUBPACKAGE(Topology) +STK_SUBPACKAGE(Topology) ADD_SUBDIRECTORY(stk_topology) -TRIBITS_SUBPACKAGE_POSTPROCESS() +STK_SUBPACKAGE_POSTPROCESS() diff --git a/packages/stk/stk_topology/stk_topology/CMakeLists.txt b/packages/stk/stk_topology/stk_topology/CMakeLists.txt index 37a5eb9561f7..feea7dc428ae 100644 --- a/packages/stk/stk_topology/stk_topology/CMakeLists.txt +++ b/packages/stk/stk_topology/stk_topology/CMakeLists.txt @@ -32,48 +32,37 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - -# -# A) Package-specific configuration options -# - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - SET(HEADERS "") SET(SOURCES "") - FILE(GLOB HEADERS *.hpp *.tcc) FILE(GLOB SOURCES *.cpp) FILE(GLOB SOURCES_DETAIL topology_detail/*.cpp) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) - -# -# Core headers -# - FILE(GLOB DETAIL_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/topology_detail/*.hpp) -# -# C) Define the targets for package's library(s) -# +if(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY(stk_topology + NOINSTALLHEADERS ${HEADERS} + DEPLIBS + SOURCES ${SOURCES} ${SOURCES_DETAIL} + ) +else() + add_library(stk_topology ${SOURCES} ${SOURCES_DETAIL}) + target_link_libraries(stk_topology PUBLIC stk_util_util) +endif() -TRIBITS_ADD_LIBRARY( - stk_topology - NOINSTALLHEADERS ${HEADERS} - DEPLIBS - SOURCES ${SOURCES} ${SOURCES_DETAIL} - ) +target_include_directories(stk_topology PUBLIC + $ + $ +) INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_topology/) INSTALL(FILES ${DETAIL_HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_topology/topology_detail) + +if(NOT HAVE_STK_Trilinos) + INSTALL(TARGETS stk_topology DESTINATION ${STK_INSTALL_LIBDIR}) +endif() diff --git a/packages/stk/stk_transfer/CMakeLists.txt b/packages/stk/stk_transfer/CMakeLists.txt index a2cea50071d1..c34049dca0f6 100644 --- a/packages/stk/stk_transfer/CMakeLists.txt +++ b/packages/stk/stk_transfer/CMakeLists.txt @@ -32,10 +32,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TRIBITS_SUBPACKAGE(Transfer) - -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) +STK_SUBPACKAGE(Transfer) ADD_SUBDIRECTORY(stk_transfer) -TRIBITS_SUBPACKAGE_POSTPROCESS() +STK_SUBPACKAGE_POSTPROCESS() diff --git a/packages/stk/stk_transfer/stk_transfer/CMakeLists.txt b/packages/stk/stk_transfer/stk_transfer/CMakeLists.txt index d62c2978531e..87523b73a0a3 100644 --- a/packages/stk/stk_transfer/stk_transfer/CMakeLists.txt +++ b/packages/stk/stk_transfer/stk_transfer/CMakeLists.txt @@ -5,14 +5,14 @@ FILE(GLOB HEADERS *.hpp) FILE(GLOB SOURCES *.cpp) -IF(NOT STK_ENABLE_STKMiddle_mesh) -message(STATUS "STKTransfer: STKMiddle_mesh not enabled") -LIST(REMOVE_ITEM HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/ConservativeTransfer.hpp") -LIST(REMOVE_ITEM HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/ConservativeTransferUser.hpp") -LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/ConservativeTransfer.cpp") -ELSE() -message(STATUS "STKTransfer: STKMiddle_mesh is enabled, ConservativeTransfer is available") -ENDIF() +if(NOT STK_ENABLE_STKMiddle_mesh) + message(STATUS "STKTransfer: STKMiddle_mesh not enabled") + LIST(REMOVE_ITEM HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/ConservativeTransfer.hpp") + LIST(REMOVE_ITEM HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/ConservativeTransferUser.hpp") + LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/ConservativeTransfer.cpp") +else() + message(STATUS "STKTransfer: STKMiddle_mesh is enabled, ConservativeTransfer is available") +endif() INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_transfer/) @@ -23,9 +23,20 @@ FILE(GLOB SOURCES_IMPL ${CMAKE_CURRENT_SOURCE_DIR}/copy_by_id/*.cpp) INSTALL(FILES ${HEADERS_IMPL} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_transfer/copy_by_id) -TRIBITS_ADD_LIBRARY( - stk_transfer - NOINSTALLHEADERS ${HEADERS} ${HEADERS_IMPL} - SOURCES ${SOURCES} ${SOURCES_IMPL} +if(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY(stk_transfer + NOINSTALLHEADERS ${HEADERS} ${HEADERS_IMPL} + SOURCES ${SOURCES} ${SOURCES_IMPL} ) +else() + add_library(stk_transfer ${SOURCES}) + target_link_libraries(stk_transfer PUBLIC stk_search) + target_link_libraries(stk_transfer PUBLIC stk_util_env) + target_link_libraries(stk_transfer PUBLIC stk_util_parallel) +endif() + +target_include_directories(stk_transfer PUBLIC + $ + $ +) diff --git a/packages/stk/stk_unit_test_utils/CMakeLists.txt b/packages/stk/stk_unit_test_utils/CMakeLists.txt index b690e76145bb..50b26ac1e920 100644 --- a/packages/stk/stk_unit_test_utils/CMakeLists.txt +++ b/packages/stk/stk_unit_test_utils/CMakeLists.txt @@ -1,6 +1,6 @@ -TRIBITS_SUBPACKAGE(Unit_test_utils) +STK_SUBPACKAGE(Unit_test_utils) ADD_SUBDIRECTORY(stk_unit_test_utils) -TRIBITS_SUBPACKAGE_POSTPROCESS() +STK_SUBPACKAGE_POSTPROCESS() diff --git a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/BulkDataTester.hpp b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/BulkDataTester.hpp index 38e966f82496..897f9827b773 100644 --- a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/BulkDataTester.hpp +++ b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/BulkDataTester.hpp @@ -93,7 +93,7 @@ class BulkDataTester : public stk::mesh::BulkData MPI_Comm comm, enum stk::mesh::BulkData::AutomaticAuraOption auto_aura_option, bool _add_fmwk_data, - stk::mesh::FieldDataManager *field_data_manager, + std::unique_ptr field_data_manager = std::unique_ptr(), unsigned initial_bucket_capacity = stk::mesh::get_default_initial_bucket_capacity(), unsigned maximum_bucket_capacity = stk::mesh::get_default_maximum_bucket_capacity()) : stk::mesh::BulkData(std::shared_ptr(&mesh_meta_data, [](auto pointerWeWontDelete){}), @@ -102,7 +102,7 @@ class BulkDataTester : public stk::mesh::BulkData #ifdef SIERRA_MIGRATION _add_fmwk_data, #endif - field_data_manager, + std::move(field_data_manager), initial_bucket_capacity, maximum_bucket_capacity) { diff --git a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/CMakeLists.txt b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/CMakeLists.txt index 1600f9146476..5cf6a55553e4 100644 --- a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/CMakeLists.txt +++ b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/CMakeLists.txt @@ -1,9 +1,11 @@ +if(HAVE_STK_Trilinos) TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}) TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_ngp_test) TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_util) TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util) +endif() SET(HEADERS_NEED_IO ElemGraphMultipleSharedSidesUtils.hpp @@ -85,51 +87,72 @@ SET(SOURCES ) -IF(TPL_ENABLE_MPI) -LIST(APPEND SOURCES ${SOURCES_NEED_MPI}) -LIST(APPEND HEADERS ${HEADERS_NEED_MPI}) +IF(STK_HAS_MPI) + LIST(APPEND SOURCES ${SOURCES_NEED_MPI}) + LIST(APPEND HEADERS ${HEADERS_NEED_MPI}) ENDIF() -IF(Trilinos_ENABLE_STKMesh) -LIST(APPEND SOURCES ${SOURCES_NEED_MESH_BUT_NOT_IO}) -LIST(APPEND HEADERS ${HEADERS_NEED_MESH_BUT_NOT_IO}) +IF(${PROJECT_NAME}_ENABLE_STKMesh) + LIST(APPEND SOURCES ${SOURCES_NEED_MESH_BUT_NOT_IO}) + LIST(APPEND HEADERS ${HEADERS_NEED_MESH_BUT_NOT_IO}) ENDIF() -IF(Trilinos_ENABLE_STKIO) -LIST(APPEND SOURCES ${SOURCES_NEED_IO}) -LIST(APPEND HEADERS ${HEADERS_NEED_IO}) +IF(${PROJECT_NAME}_ENABLE_STKIO) + LIST(APPEND SOURCES ${SOURCES_NEED_IO}) + LIST(APPEND HEADERS ${HEADERS_NEED_IO}) ENDIF() -IF(Trilinos_ENABLE_STKMesh) +IF(${PROJECT_NAME}_ENABLE_STKMesh) ADD_SUBDIRECTORY(stk_mesh_fixtures) ENDIF() -IF(Trilinos_ENABLE_STKTransfer AND Trilinos_ENABLE_STKMiddle_mesh) +IF(${PROJECT_NAME}_ENABLE_STKTransfer AND ${PROJECT_NAME}_ENABLE_STKMiddle_mesh) ADD_SUBDIRECTORY(stk_transfer_fixtures) ENDIF() INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_unit_test_utils/) -TRIBITS_ADD_LIBRARY( - stk_unit_test_utils - HEADERS ${HEADERS} - SOURCES ${SOURCES} - ${Gtest_NO_INSTALL_LIB_OR_HEADERS_ARG} +if(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY(stk_unit_test_utils + HEADERS ${HEADERS} + SOURCES ${SOURCES} + ${Gtest_NO_INSTALL_LIB_OR_HEADERS_ARG} ) +else() + add_library(stk_unit_test_utils ${SOURCES}) -IF (NOT DEFINED STK_ENABLE_UnitMain) - SET(STK_ENABLE_UnitMain ON) -ENDIF() + target_link_libraries(stk_unit_test_utils PUBLIC stk_util_parallel) + + find_package(GTest REQUIRED) + target_link_libraries(stk_unit_test_utils PUBLIC GTest::gtest) +endif() + +target_include_directories(stk_unit_test_utils PUBLIC + $ + $ +) -IF (STK_ENABLE_UnitMain) +if (NOT DEFINED STK_ENABLE_UnitMain) + set(STK_ENABLE_UnitMain ON) +endif() + +if (STK_ENABLE_UnitMain) FILE(GLOB UNIT_MAIN_SOURCES unit_main_lib/*.cpp) - TRIBITS_ADD_LIBRARY( - stk_unit_main - TESTONLY - SOURCES ${UNIT_MAIN_SOURCES} - ) -ENDIF() + if(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY(stk_unit_main + TESTONLY + SOURCES ${UNIT_MAIN_SOURCES} + ) + else() + add_library(stk_unit_main ${UNIT_MAIN_SOURCES}) + target_link_libraries(stk_unit_main PUBLIC stk_unit_test_utils) + target_link_libraries(stk_unit_main PUBLIC stk_ngp_test) + target_link_libraries(stk_unit_main PUBLIC stk_util_env) + target_link_libraries(stk_unit_main PUBLIC stk_util_parallel) + target_link_libraries(stk_unit_main PUBLIC Kokkos::kokkos) + endif() +endif() diff --git a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/ParallelGtestOutput.cpp b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/ParallelGtestOutput.cpp index 2ed7dc343a35..27b76453c41f 100644 --- a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/ParallelGtestOutput.cpp +++ b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/ParallelGtestOutput.cpp @@ -126,7 +126,12 @@ class MinimalistPrinter : public ::testing::EmptyTestEventListener { if(numTotalFailures == 0) { +#ifdef STK_BUILT_IN_SIERRA ::testing::internal::ColoredPrintf(::testing::internal::COLOR_GREEN, "[ OK ] "); +#else +//newer versions of gtest don't allow external access to ColoredPrintf + printf("[ OK ] "); +#endif } else { @@ -209,13 +214,23 @@ class MinimalistPrinter : public ::testing::EmptyTestEventListener void print_failed(const std::string &message) { +#ifdef STK_BUILT_IN_SIERRA ::testing::internal::ColoredPrintf(::testing::internal::COLOR_RED, "[ FAILED ] "); +#else +//newer versions of gtest don't allow external access to ColoredPrintf + printf("[ FAILED ] "); +#endif printf("%s\n", message.c_str()); } void print_passed(const std::string &message) { +#ifdef STK_BUILT_IN_SIERRA ::testing::internal::ColoredPrintf(::testing::internal::COLOR_GREEN, "[ PASSED ] "); +#else +//newer versions of gtest don't allow external access to ColoredPrintf + printf("[ PASSED ] "); +#endif printf("%s\n", message.c_str()); } }; diff --git a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/TextMesh.cpp b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/TextMesh.cpp index 8b3cf2cde024..45089705abaa 100644 --- a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/TextMesh.cpp +++ b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/TextMesh.cpp @@ -31,6 +31,7 @@ #include "stk_mesh/base/FieldBase.hpp" // for field_data #include "stk_mesh/base/Types.hpp" // for EntityId, etc #include "stk_topology/topology.hpp" // for topology, etc +#include "stk_util/util/SortAndUnique.hpp" namespace stk { namespace mesh { class Part; } } // clang-format on @@ -56,8 +57,8 @@ using AssemblyType = text_mesh::AssemblyType; class MetaDataInitializer { public: - MetaDataInitializer(const TextMeshData& d, stk::mesh::MetaData& m) - : m_data(d), m_meta(m) + MetaDataInitializer(const StkTopologyMapping& t, const TextMeshData& d, stk::mesh::MetaData& m) + : m_topologyMapping(t), m_data(d), m_meta(m) { } void setup() @@ -187,13 +188,28 @@ class MetaDataInitializer void declare_sideblock_part(stk::mesh::Part& sidesetPart, const SideBlockInfo& sideBlock) { - stk::mesh::Part& sideBlockPart = m_meta.declare_part(sideBlock.name, m_meta.side_rank()); + stk::topology sideTopology = m_topologyMapping.topology(sideBlock.sideTopology).topology; - stk::io::put_io_part_attribute(sideBlockPart); - m_meta.set_part_id(sideBlockPart, sidesetPart.id()); + stk::mesh::Part* sideBlockPart = nullptr; - if (sidesetPart.mesh_meta_data_ordinal() != sideBlockPart.mesh_meta_data_ordinal()) { - m_meta.declare_part_subset(sidesetPart, sideBlockPart); + if(stk::topology::INVALID_TOPOLOGY == sideTopology) { + sideBlockPart = &m_meta.declare_part(sideBlock.name, m_meta.side_rank()); + } else { + sideBlockPart = &m_meta.declare_part_with_topology(sideBlock.name, sideTopology); + } + + STK_ThrowRequire(nullptr != sideBlockPart); + stk::io::put_io_part_attribute(*sideBlockPart); + m_meta.set_part_id(*sideBlockPart, sidesetPart.id()); + + if (sidesetPart.mesh_meta_data_ordinal() != sideBlockPart->mesh_meta_data_ordinal()) { + m_meta.declare_part_subset(sidesetPart, *sideBlockPart); + } + + if(!sideBlock.touchingBlock.empty()) { + const stk::mesh::Part* touchingBlockPart = m_meta.get_part(sideBlock.touchingBlock); + STK_ThrowRequire(nullptr != touchingBlockPart); + m_meta.set_surface_to_block_mapping(sideBlockPart, std::vector{touchingBlockPart}); } } @@ -207,9 +223,19 @@ class MetaDataInitializer std::vector sideBlocks = sidesetData.get_side_block_info(); + std::vector touchingBlockParts; + for (const auto& sideBlock : sideBlocks) { declare_sideblock_part(sidesetPart, sideBlock); + + if(!sideBlock.touchingBlock.empty()) { + const stk::mesh::Part* touchingBlockPart = m_meta.get_part(sideBlock.touchingBlock); + STK_ThrowRequire(nullptr != touchingBlockPart); + stk::util::insert_keep_sorted_and_unique(touchingBlockPart, touchingBlockParts, stk::mesh::PartLess()); + } } + + m_meta.set_surface_to_block_mapping(&sidesetPart, touchingBlockParts); } } @@ -281,6 +307,7 @@ class MetaDataInitializer stk::io::set_field_output_type(coordsField, stk::io::FieldOutputType::VECTOR_3D); } + const StkTopologyMapping &m_topologyMapping; const TextMeshData& m_data; stk::mesh::MetaData& m_meta; }; @@ -501,12 +528,13 @@ class TextMesh m_parser(m_meta.spatial_dimension()) { validate_spatial_dim(m_meta.spatial_dimension()); + m_topologyMapping.initialize_topology_map(); m_data = m_parser.parse(meshDesc); } void setup_mesh() { - MetaDataInitializer metaInit(m_data, m_meta); + MetaDataInitializer metaInit(m_topologyMapping, m_data, m_meta); metaInit.setup(); BulkDataInitializer bulkInit(m_data, m_bulk); @@ -527,6 +555,7 @@ class TextMesh TextMeshParser m_parser; TextMeshData m_data; + StkTopologyMapping m_topologyMapping; }; std::string get_full_text_mesh_desc(const std::string& textMeshDesc, const std::vector& coordVec) diff --git a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/TextMeshStkTopologyMapping.hpp b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/TextMeshStkTopologyMapping.hpp index 9c7f5cf68091..2ffabd6d6113 100644 --- a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/TextMeshStkTopologyMapping.hpp +++ b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/TextMeshStkTopologyMapping.hpp @@ -136,12 +136,19 @@ class StkTopologyMapping : public text_mesh::TopologyMapping(nullptr), + std::unique_ptr(), bucketCapacity, bucketCapacity) - : stk::unit_test_util::BulkDataTester(m_fem_meta, pm, autoAuraOption, false, - static_cast(nullptr))), + : stk::unit_test_util::BulkDataTester(m_fem_meta, pm, autoAuraOption, false) ), m_comm_rank( stk::parallel_machine_rank( pm ) ), m_comm_size( stk::parallel_machine_size( pm ) ), m_elem_part( m_fem_meta.declare_part_with_topology("elem_part", stk::topology::HEX_8) ), @@ -229,10 +228,9 @@ BoxFixture::BoxFixture( stk::ParallelMachine pm , : m_fem_meta ( spatial_dimension, entity_names ), m_bulk_data ( (bucketCapacity != stk::mesh::get_default_bucket_capacity()) ? stk::unit_test_util::BulkDataTester(m_fem_meta, pm, autoAuraOption, false, - static_cast(nullptr), + std::unique_ptr(), bucketCapacity, bucketCapacity) - : stk::unit_test_util::BulkDataTester(m_fem_meta, pm, autoAuraOption, false, - static_cast(nullptr))), + : stk::unit_test_util::BulkDataTester(m_fem_meta, pm, autoAuraOption, false) ), m_comm_rank( stk::parallel_machine_rank( pm ) ), m_comm_size( stk::parallel_machine_size( pm ) ), m_elem_part( m_fem_meta.declare_part_with_topology("elem_part", stk::topology::HEX_8) ), diff --git a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/unit_main_lib/UnitTestMain.cpp b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/unit_main_lib/UnitTestMain.cpp index dfa23eea41c6..a9ba45ae1ccd 100644 --- a/packages/stk/stk_unit_test_utils/stk_unit_test_utils/unit_main_lib/UnitTestMain.cpp +++ b/packages/stk/stk_unit_test_utils/stk_unit_test_utils/unit_main_lib/UnitTestMain.cpp @@ -48,6 +48,7 @@ #include #include #include +#include int main(int argc, char **argv) { @@ -69,6 +70,7 @@ int main(int argc, char **argv) #ifdef STK_HAS_MPI int procId = stk::parallel_machine_rank(MPI_COMM_WORLD); + stk::EnvData::instance().initialize(MPI_COMM_WORLD); stk::unit_test_util::create_parallel_output(procId); if (stk::unit_test_util::has_option("-stk_coupling_version")) { int version = stk::unit_test_util::get_command_line_option("-stk_coupling_version", -1); diff --git a/packages/stk/stk_unit_tests/CMakeLists.txt b/packages/stk/stk_unit_tests/CMakeLists.txt index b7ac624ecc9e..001b5aefe6a0 100644 --- a/packages/stk/stk_unit_tests/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/CMakeLists.txt @@ -1,66 +1,68 @@ -TRIBITS_SUBPACKAGE(Unit_tests) +STK_SUBPACKAGE(Unit_tests) -IF(Trilinos_ENABLE_STKUtil) -TRIBITS_ADD_TEST_DIRECTORIES(stk_util) -ENDIF() +STK_ADD_TEST_DIRECTORIES(stk_util) -IF(Trilinos_ENABLE_Kokkos) -TRIBITS_ADD_TEST_DIRECTORIES(stk_math) -ENDIF() +if(${PROJECT_NAME}_ENABLE_STKMath) + STK_ADD_TEST_DIRECTORIES(stk_math) +endif() -IF(Trilinos_ENABLE_STKSimd) -TRIBITS_ADD_TEST_DIRECTORIES(stk_simd) -TRIBITS_ADD_TEST_DIRECTORIES(stk_simd_old) +IF(${PROJECT_NAME}_ENABLE_STKSimd) + STK_ADD_TEST_DIRECTORIES(stk_simd) + STK_ADD_TEST_DIRECTORIES(stk_simd_old) ENDIF() -IF(Trilinos_ENABLE_STKNGP_TEST) +IF(${PROJECT_NAME}_ENABLE_STKNGP_TEST) - IF(Trilinos_ENABLE_STKMesh) - TRIBITS_ADD_TEST_DIRECTORIES(stk_mesh) - ENDIF() + IF(${PROJECT_NAME}_ENABLE_STKMesh) + STK_ADD_TEST_DIRECTORIES(stk_mesh) + ENDIF() - TRIBITS_ADD_TEST_DIRECTORIES(stk_ngp_test) + STK_ADD_TEST_DIRECTORIES(stk_ngp_test) - IF(Trilinos_ENABLE_STKTopology) - TRIBITS_ADD_TEST_DIRECTORIES(stk_topology) - ENDIF() + IF(${PROJECT_NAME}_ENABLE_STKTopology) + STK_ADD_TEST_DIRECTORIES(stk_topology) + ENDIF() ENDIF() -IF(Trilinos_ENABLE_STKCoupling) -TRIBITS_ADD_TEST_DIRECTORIES(stk_coupling) +IF(${PROJECT_NAME}_ENABLE_STKCoupling) + STK_ADD_TEST_DIRECTORIES(stk_coupling) +ENDIF() + +IF(${PROJECT_NAME}_ENABLE_STKExprEval) + STK_ADD_TEST_DIRECTORIES(stk_expreval) ENDIF() -IF(Trilinos_ENABLE_STKSearch) -TRIBITS_ADD_TEST_DIRECTORIES(stk_search) +IF(${PROJECT_NAME}_ENABLE_STKSearch) + STK_ADD_TEST_DIRECTORIES(stk_search) ENDIF() -IF(Trilinos_ENABLE_STKSearchUtil) -TRIBITS_ADD_TEST_DIRECTORIES(stk_search_util) +IF(${PROJECT_NAME}_ENABLE_STKSearchUtil) + STK_ADD_TEST_DIRECTORIES(stk_search_util) ENDIF() -IF(Trilinos_ENABLE_STKMiddle_mesh) -TRIBITS_ADD_TEST_DIRECTORIES(stk_middle_mesh) +IF(${PROJECT_NAME}_ENABLE_STKMiddle_mesh) + STK_ADD_TEST_DIRECTORIES(stk_middle_mesh) ENDIF() -IF(Trilinos_ENABLE_STKTransfer) -TRIBITS_ADD_TEST_DIRECTORIES(stk_transfer) +IF(${PROJECT_NAME}_ENABLE_STKTransfer) + STK_ADD_TEST_DIRECTORIES(stk_transfer) ENDIF() -IF(Trilinos_ENABLE_STKTransferUtil AND Trilinos_ENABLE_STKIO) -TRIBITS_ADD_TEST_DIRECTORIES(stk_transfer_util) +IF(${PROJECT_NAME}_ENABLE_STKTransferUtil AND ${PROJECT_NAME}_ENABLE_STKIO) + STK_ADD_TEST_DIRECTORIES(stk_transfer_util) ENDIF() -IF(Trilinos_ENABLE_STKBalance) -TRIBITS_ADD_TEST_DIRECTORIES(stk_balance) +IF(${PROJECT_NAME}_ENABLE_STKBalance) + STK_ADD_TEST_DIRECTORIES(stk_balance) ENDIF() -IF(Trilinos_ENABLE_STKIO) -TRIBITS_ADD_TEST_DIRECTORIES(stk_io) +IF(${PROJECT_NAME}_ENABLE_STKIO) + STK_ADD_TEST_DIRECTORIES(stk_io) ENDIF() -IF(Trilinos_ENABLE_STKTools) -TRIBITS_ADD_TEST_DIRECTORIES(stk_tools) +IF(${PROJECT_NAME}_ENABLE_STKTools) + STK_ADD_TEST_DIRECTORIES(stk_tools) ENDIF() -TRIBITS_SUBPACKAGE_POSTPROCESS() +STK_SUBPACKAGE_POSTPROCESS() diff --git a/packages/stk/stk_unit_tests/cmake/Dependencies.cmake b/packages/stk/stk_unit_tests/cmake/Dependencies.cmake index faade3f2a17d..09710ac94fb0 100644 --- a/packages/stk/stk_unit_tests/cmake/Dependencies.cmake +++ b/packages/stk/stk_unit_tests/cmake/Dependencies.cmake @@ -1,7 +1,7 @@ SET(LIB_REQUIRED_DEP_PACKAGES) SET(LIB_OPTIONAL_DEP_PACKAGES) SET(TEST_REQUIRED_DEP_PACKAGES Gtest STKUtil STKUnit_test_utils) -SET(TEST_OPTIONAL_DEP_PACKAGES Shards STKCoupling STKTopology STKMath STKSimd STKMiddle_mesh STKMesh STKIO STKBalance STKTransfer STKTransferUtil STKSearch STKTools) +SET(TEST_OPTIONAL_DEP_PACKAGES Shards STKCoupling STKTopology STKMath STKSimd STKExprEval STKMiddle_mesh STKMesh STKIO STKBalance STKTransfer STKTransferUtil STKSearch STKTools) SET(LIB_REQUIRED_DEP_TPLS) SET(LIB_OPTIONAL_DEP_TPLS) SET(TEST_REQUIRED_DEP_TPLS) diff --git a/packages/stk/stk_unit_tests/stk_balance/UnitTestDiagnosticsComputation.cpp b/packages/stk/stk_unit_tests/stk_balance/UnitTestDiagnosticsComputation.cpp index c3a0d1e3dc75..c0227f360e5e 100644 --- a/packages/stk/stk_unit_tests/stk_balance/UnitTestDiagnosticsComputation.cpp +++ b/packages/stk/stk_unit_tests/stk_balance/UnitTestDiagnosticsComputation.cpp @@ -130,7 +130,7 @@ class TestDiagnosticsComputation : public stk::unit_test_util::simple_fields::Me { stk::mesh::BulkData & bulk = ioBroker.bulk_data(); - stk::balance::internal::register_internal_fields(bulk, balanceSettings); + stk::balance::internal::register_internal_fields_and_parts(bulk, balanceSettings); stk::balance::set_up_diagnostics(balanceSettings); stk::balance::rebalance(ioBroker, balanceSettings); diff --git a/packages/stk/stk_unit_tests/stk_balance/UnitTestRebalanceSpiders.cpp b/packages/stk/stk_unit_tests/stk_balance/UnitTestRebalanceSpiders.cpp index a5639bc5b5ed..436122f715b5 100644 --- a/packages/stk/stk_unit_tests/stk_balance/UnitTestRebalanceSpiders.cpp +++ b/packages/stk/stk_unit_tests/stk_balance/UnitTestRebalanceSpiders.cpp @@ -65,7 +65,7 @@ class RebalanceSpiders : public MeshFixtureRebalance setup_empty_mesh(stk::mesh::BulkData::AUTO_AURA); get_meta().set_coordinate_field_name(m_balanceSettings.getCoordinateFieldName()); - stk::balance::internal::register_internal_fields(get_bulk(), m_balanceSettings); + stk::balance::internal::register_internal_fields_and_parts(get_bulk(), m_balanceSettings); stk::io::fill_mesh_preexisting(m_ioBroker, get_input_file_name(), get_bulk()); } @@ -143,14 +143,14 @@ class RebalanceSpiders : public MeshFixtureRebalance // // || | \ . // // || | \ . // // / | \ \ . -// (2:0)// | | \ \(6:2) +// (2:0)// | | \ \(6:3) // // | | | \ . // // | | |(5:3) // //(1:0) | | \ \ . // // / |(4:1) | \ . // // | | | \ . // 2/ | 4 | 6 -// / (3:1)| \ . +// / (3:2)| \ . // / | \ . // 1 3 5 // @@ -167,10 +167,10 @@ TEST_F(RebalanceSpiders, notASpider_EnoughLegsNoVolumeElements_PartitionedNormal // Beams not part of a spider, so partitioned with RCB check_spider_legs_ownership({{stk::mesh::EntityKey(stk::topology::ELEM_RANK, 1), 0}, {stk::mesh::EntityKey(stk::topology::ELEM_RANK, 2), 0}, - {stk::mesh::EntityKey(stk::topology::ELEM_RANK, 3), 1}, + {stk::mesh::EntityKey(stk::topology::ELEM_RANK, 3), 2}, {stk::mesh::EntityKey(stk::topology::ELEM_RANK, 4), 1}, {stk::mesh::EntityKey(stk::topology::ELEM_RANK, 5), 3}, - {stk::mesh::EntityKey(stk::topology::ELEM_RANK, 6), 2}}); + {stk::mesh::EntityKey(stk::topology::ELEM_RANK, 6), 3}}); } diff --git a/packages/stk/stk_unit_tests/stk_balance/UnitTestSpiderElements.cpp b/packages/stk/stk_unit_tests/stk_balance/UnitTestSpiderElements.cpp index 6e1d63ed0eb7..af9affe8b612 100644 --- a/packages/stk/stk_unit_tests/stk_balance/UnitTestSpiderElements.cpp +++ b/packages/stk/stk_unit_tests/stk_balance/UnitTestSpiderElements.cpp @@ -2,8 +2,12 @@ #include #include #include +#include #include #include "UnitTestSpiderMeshSetup.hpp" +#include "stk_balance/setup/DefaultSettings.hpp" +#include "stk_balance/io/BalanceIO.hpp" +#include "stk_unit_test_utils/getOption.h" namespace { @@ -15,21 +19,31 @@ class SpiderElement : public stk::unit_test_util::simple_fields::MeshFixture m_balanceSettings.setShouldFixSpiders(true); setup_empty_mesh(stk::mesh::BulkData::AUTO_AURA); - stk::balance::internal::register_internal_fields(get_bulk(), m_balanceSettings); + stk::balance::internal::register_internal_fields_and_parts(get_bulk(), m_balanceSettings); } void fill_decomp_list_from_current_ownership(stk::mesh::EntityProcVec & decomp) { - stk::mesh::EntityVector localElems; - stk::mesh::get_entities(get_bulk(), stk::topology::ELEM_RANK, get_meta().locally_owned_part(), localElems); + stk::mesh::Part & spiderPart = *m_balanceSettings.getSpiderPart(get_bulk()); + stk::mesh::EntityVector localElems = stk::mesh::get_entities(get_bulk(), stk::topology::ELEM_RANK, + get_meta().locally_owned_part() & !spiderPart); + + // Fill list as if this is what came back from the partitioner, with local offsets + // placing the results at the start of the (now oversized) array. + unsigned listOffset = 0; for (const stk::mesh::Entity & elem : localElems) { - decomp.push_back(std::make_pair(elem, get_bulk().parallel_owner_rank(elem))); + decomp[listOffset++] = std::make_pair(elem, get_bulk().parallel_owner_rank(elem)); } } void fix_spider_elements() { - stk::balance::internal::fill_spider_connectivity_count_fields(get_bulk(), m_balanceSettings); + stk::balance::internal::fill_spider_connectivity_count_fields_and_parts(get_bulk(), m_balanceSettings); + + const size_t numLocallyOwned = stk::mesh::count_entities(get_bulk(), stk::topology::ELEM_RANK, + get_meta().locally_owned_part()); stk::mesh::EntityProcVec decomp; + decomp.resize(numLocallyOwned, std::make_pair(stk::mesh::Entity(), get_bulk().parallel_rank())); + fill_decomp_list_from_current_ownership(decomp); stk::balance::internal::fill_output_subdomain_field(get_bulk(), m_balanceSettings, decomp); m_changeList = std::make_unique(get_bulk(), decomp); @@ -47,7 +61,7 @@ class SpiderElement : public stk::unit_test_util::simple_fields::MeshFixture check_entity_key_ownership(spiderLegs); } -private: +protected: void check_entity_key_ownership(const stk::mesh::EntityKeyProcVec & entityKeyProcs) { for (const stk::mesh::EntityKeyProc & entityKeyProc : entityKeyProcs) { @@ -97,6 +111,10 @@ TEST_F(SpiderElement, notASpider_EnoughLegsNoVolumeElements_NoMovement) fix_spider_elements(); + // Since these elements don't count as a spider, the normal balance code path + // will change the decomposition. The spider-fixing code will not touch it + // afterward. Since we are only testing the spider-fixing code, there should + // be no change. check_spider_body({{stk::mesh::EntityKey(stk::topology::NODE_RANK, 7), 2}}); check_spider_legs({{stk::mesh::EntityKey(stk::topology::ELEM_RANK, 1), 2}, @@ -545,4 +563,74 @@ TEST_F(SpiderElement, twoSpiders_ParticleBody) {stk::mesh::EntityKey(stk::topology::ELEM_RANK, 21), 3}}); } +void set_sd_flag_options(const std::string & fileName, int numProcessors, stk::balance::BalanceSettings & settings) { + settings.setVertexWeightMethod(stk::balance::DefaultSettings::sdVertexWeightMethod); + settings.setGraphEdgeWeightMultiplier(stk::balance::DefaultSettings::sdGraphEdgeWeightMultiplier); + settings.setVertexWeightMultiplierForVertexInSearch(stk::balance::DefaultSettings::sdFaceSearchVertexMultiplier); + settings.setEdgeWeightForSearch(stk::balance::DefaultSettings::sdFaceSearchEdgeWeight); + settings.setShouldFixSpiders(stk::balance::DefaultSettings::sdFixSpiders); + settings.set_input_filename(fileName); + settings.set_output_filename(fileName); + settings.set_num_input_processors(numProcessors); + settings.set_num_output_processors(numProcessors); + settings.set_is_rebalancing(false); +} + +void compare_identical_volume_decompositions(stk::balance::BalanceMesh & meshNode, + stk::balance::BalanceMesh & meshParticle) +{ + stk::mesh::BulkData & bulkNode = meshNode.get_bulk(); + stk::mesh::MetaData & metaNode = bulkNode.mesh_meta_data(); + stk::mesh::BulkData & bulkParticle = meshParticle.get_bulk(); + stk::mesh::MetaData & metaParticle = bulkParticle.mesh_meta_data(); + + const bool sortElems = true; + stk::mesh::EntityVector hexesNode = stk::mesh::get_entities(bulkNode, stk::topology::ELEM_RANK, + metaNode.locally_owned_part() & + metaNode.get_topology_root_part(stk::topology::HEX_8), + sortElems); + stk::mesh::EntityVector hexesParticle = stk::mesh::get_entities(bulkParticle, stk::topology::ELEM_RANK, + metaParticle.locally_owned_part() & + metaParticle.get_topology_root_part(stk::topology::HEX_8), + sortElems); + + ASSERT_EQ(hexesNode.size(), hexesParticle.size()); + + for (unsigned i = 0; i < hexesNode.size(); ++i) { + EXPECT_EQ(bulkNode.identifier(hexesNode[i]), bulkParticle.identifier(hexesParticle[i])); + } +} + +TEST_F(SpiderElement, cubeMeshWithSpider_ParticleBodyInsensitivity) +{ + const unsigned meshSize = stk::unit_test_util::simple_fields::get_command_line_option("--size", 2); + + bool addParticleBody = false; + const std::string fileNameNode = "cube_spider_node.g"; + write_serial_cube_mesh_with_spider(meshSize, addParticleBody, fileNameNode); + + stk::balance::StkBalanceSettings balanceSettingsNode; + set_sd_flag_options(fileNameNode, stk::parallel_machine_size(get_comm()), balanceSettingsNode); + stk::balance::BalanceIO ioNode(get_comm(), balanceSettingsNode); + const stk::balance::Balancer balancerNode(balanceSettingsNode); + + stk::balance::BalanceMesh& meshNode = ioNode.initial_decomp(); + balancerNode.balance(meshNode); + + + addParticleBody = true; + const std::string fileNameParticle = "cube_spider_particle.g"; + write_serial_cube_mesh_with_spider(meshSize, addParticleBody, fileNameParticle); + + stk::balance::StkBalanceSettings balanceSettingsParticle; + set_sd_flag_options(fileNameParticle, stk::parallel_machine_size(get_comm()), balanceSettingsParticle); + stk::balance::BalanceIO ioParticle(get_comm(), balanceSettingsParticle); + const stk::balance::Balancer balancerParticle(balanceSettingsParticle); + + stk::balance::BalanceMesh& meshParticle = ioParticle.initial_decomp(); + balancerParticle.balance(meshParticle); + + compare_identical_volume_decompositions(meshNode, meshParticle); +} + } diff --git a/packages/stk/stk_unit_tests/stk_balance/UnitTestSpiderMeshSetup.hpp b/packages/stk/stk_unit_tests/stk_balance/UnitTestSpiderMeshSetup.hpp index 20ab0b9196a3..d278c1c403da 100644 --- a/packages/stk/stk_unit_tests/stk_balance/UnitTestSpiderMeshSetup.hpp +++ b/packages/stk/stk_unit_tests/stk_balance/UnitTestSpiderMeshSetup.hpp @@ -2,7 +2,13 @@ #define UNITTESTSPIDERMESHSETUP_HPP #include "stk_mesh/base/BulkData.hpp" +#include "stk_mesh/base/MetaData.hpp" +#include "stk_mesh/base/GetEntities.hpp" +#include "stk_mesh/base/MeshBuilder.hpp" #include "stk_unit_test_utils/TextMesh.hpp" +#include "stk_io/FillMesh.hpp" +#include "stk_io/IossBridge.hpp" +#include "stk_io/WriteMesh.hpp" inline void make_mesh_non_spider_no_volume_elements(stk::mesh::BulkData & bulk) @@ -245,4 +251,69 @@ void make_mesh_two_spiders_particle_body(stk::mesh::BulkData & bulk) stk::unit_test_util::simple_fields::setup_text_mesh(bulk, stk::unit_test_util::simple_fields::get_full_text_mesh_desc(meshDesc, coordinates)); } +inline +void write_serial_cube_mesh_with_spider(unsigned meshSize, bool addParticleBody, const std::string & fileName) +{ + if (stk::parallel_machine_rank(MPI_COMM_WORLD) == 0) { + stk::mesh::MeshBuilder builder(MPI_COMM_SELF); + builder.set_spatial_dimension(3); + builder.set_aura_option(stk::mesh::BulkData::NO_AUTO_AURA); + auto bulk = builder.create(); + stk::mesh::MetaData & meta = bulk->mesh_meta_data(); + meta.use_simple_fields(); + + stk::mesh::Part & block2Part = meta.declare_part_with_topology("block_2", stk::topology::BEAM_2); + stk::mesh::Part & block3Part = meta.declare_part_with_topology("block_3", stk::topology::PARTICLE); + stk::io::put_io_part_attribute(block2Part); + stk::io::put_io_part_attribute(block3Part); + + unsigned newNodeId = (meshSize+1) * (meshSize+1) * (meshSize+1) + 1; + unsigned newElemId = meshSize * meshSize * meshSize + 1; + stk::io::fill_mesh("generated:" + std::to_string(meshSize) + "x" + std::to_string(meshSize) + + "x" + std::to_string(meshSize), *bulk); + + double maxCoord = meshSize; + const auto & coordinates = *static_cast*>(meta.coordinate_field()); + + bulk->modification_begin(); + + stk::mesh::Entity spiderNode = bulk->declare_node(newNodeId); + double * spiderCoords = stk::mesh::field_data(coordinates, spiderNode); + spiderCoords[0] = maxCoord + 1.0; + spiderCoords[1] = maxCoord / 2.0; + spiderCoords[2] = maxCoord / 2.0; + + const stk::mesh::BucketVector ownedBuckets = bulk->get_buckets(stk::topology::NODE_RANK, meta.locally_owned_part()); + const stk::mesh::ConstPartVector elemParts {&block2Part}; + + const auto allNodes = stk::mesh::get_entities(*bulk, stk::topology::NODE_RANK, meta.locally_owned_part()); + stk::mesh::EntityVector edgeNodes; + for (const stk::mesh::Entity & node : allNodes) { + const double * coords = stk::mesh::field_data(coordinates, node); + if (std::abs(coords[0] - maxCoord) < 0.1) { + if (bulk->num_elements(node) <= 2) { + edgeNodes.push_back(node); + } + } + } + + for (stk::mesh::Entity node : edgeNodes) { + stk::mesh::Entity elem = bulk->declare_element(newElemId++, elemParts); + bulk->declare_relation(elem, node, 0); + bulk->declare_relation(elem, spiderNode, 1); + } + + if (addParticleBody) { + const stk::mesh::ConstPartVector bodyParts {&block3Part}; + stk::mesh::Entity spiderBody = bulk->declare_element(newElemId++, bodyParts); + bulk->declare_relation(spiderBody, spiderNode, 0); + } + + bulk->modification_end(); + + stk::io::write_mesh(fileName, *bulk); + } + MPI_Barrier(MPI_COMM_WORLD); +} + #endif // UNITTESTSPIDERMESHSETUP_HPP diff --git a/packages/stk/stk_unit_tests/stk_coupling/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_coupling/CMakeLists.txt index 5101bd1a7934..d651b1a078a0 100644 --- a/packages/stk/stk_unit_tests/stk_coupling/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_coupling/CMakeLists.txt @@ -32,14 +32,12 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) - FILE(GLOB SOURCES *.cpp ) -TRIBITS_ADD_EXECUTABLE_AND_TEST( - stk_coupling_utest +if(HAVE_STK_Trilinos) + TRIBITS_ADD_EXECUTABLE_AND_TEST(stk_coupling_utest SOURCES ${SOURCES} TESTONLYLIBS stk_unit_main ARGS "" @@ -47,4 +45,13 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST( NUM_MPI_PROCS 1-4 NOEXEPREFIX ) +else() + add_executable(stk_coupling_utest ${SOURCES}) + target_link_libraries(stk_coupling_utest stk_coupling) + target_link_libraries(stk_coupling_utest stk_unit_test_utils) + target_link_libraries(stk_coupling_utest stk_util_parallel) + target_link_libraries(stk_coupling_utest stk_unit_main) + + add_test(NAME "stk_coupling_utest" COMMAND stk_coupling_utest) +endif() diff --git a/packages/stk/stk_unit_tests/stk_expreval/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_expreval/CMakeLists.txt new file mode 100644 index 000000000000..338d7125bd2a --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_expreval/CMakeLists.txt @@ -0,0 +1,20 @@ + +FILE(GLOB SOURCES *.cpp) + +if(HAVE_STK_Trilinos) + TRIBITS_ADD_EXECUTABLE_AND_TEST(stk_expreval_unit_tests + SOURCES ${SOURCES} + TESTONLYLIBS stk_unit_main + ARGS "" + COMM serial mpi + NUM_MPI_PROCS 1 + ) +else() + add_executable(stk_expreval_unit_tests ${SOURCES}) + target_link_libraries(stk_expreval_unit_tests stk_expreval) + target_link_libraries(stk_expreval_unit_tests stk_ngp_test) + target_link_libraries(stk_expreval_unit_tests stk_unit_main) + + add_test(NAME "stk_expreval_unit_tests" COMMAND stk_expreval_unit_tests) +endif() + diff --git a/packages/stk/stk_expreval/unit_tests/UnitTestEvaluator.cpp b/packages/stk/stk_unit_tests/stk_expreval/UnitTestEvaluator.cpp similarity index 100% rename from packages/stk/stk_expreval/unit_tests/UnitTestEvaluator.cpp rename to packages/stk/stk_unit_tests/stk_expreval/UnitTestEvaluator.cpp diff --git a/packages/stk/stk_unit_tests/stk_io/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_io/CMakeLists.txt index 45ba3ca82c1c..e96fde7a659a 100644 --- a/packages/stk/stk_unit_tests/stk_io/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_io/CMakeLists.txt @@ -33,10 +33,6 @@ # -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util/stk_util) - FILE(GLOB SOURCES *.cpp) LIST(REMOVE_ITEM SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/UnitTestMeshData.cpp) LIST(REMOVE_ITEM SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/UnitTestWriteSTKMesh.cpp) diff --git a/packages/stk/stk_unit_tests/stk_io/UnitTestCustomMeshBuilder.cpp b/packages/stk/stk_unit_tests/stk_io/UnitTestCustomMeshBuilder.cpp new file mode 100644 index 000000000000..5868ff8a9445 --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_io/UnitTestCustomMeshBuilder.cpp @@ -0,0 +1,186 @@ +// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering +// Solutions of Sandia, LLC (NTESS). Under the terms of Contract +// DE-NA0003525 with NTESS, the U.S. Government retains certain rights +// in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided +// with the distribution. +// +// * Neither the name of NTESS nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "gtest/gtest.h" // for AssertHelper, ASSERT_TRUE +#include // for is_part_io_part +#include // for StkMeshIoBroker +#include // for BulkData +#include +#include "stk_unit_test_utils/GeneratedMeshToFile.hpp" +#include "stk_unit_test_utils/ioUtils.hpp" +#include // for size_t +#include // for rand, srand, RAND_MAX + +namespace { + +class CustomBulkData : public stk::mesh::BulkData +{ +protected: + friend class CustomMeshBuilder; + + CustomBulkData(std::shared_ptr metaData, + stk::ParallelMachine parallel, + enum AutomaticAuraOption autoAuraOption = AUTO_AURA, + std::unique_ptr fieldDataManager = std::unique_ptr(), + unsigned initialBucketCapacity = stk::mesh::get_default_initial_bucket_capacity(), + unsigned maximumBucketCapacity = stk::mesh::get_default_maximum_bucket_capacity(), + std::shared_ptr auraGhosting = std::shared_ptr(), + bool createUpwardConnectivity = true) +#ifdef SIERRA_MIGRATION + : BulkData(metaData, parallel, autoAuraOption, false, std::move(fieldDataManager), initialBucketCapacity, + maximumBucketCapacity, auraGhosting, createUpwardConnectivity) +#else + : BulkData(metaData, parallel, autoAuraOption, std::move(fieldDataManager), initialBucketCapacity, + maximumBucketCapacity, auraGhosting, createUpwardConnectivity) +#endif + { + } +}; + +class CustomMeshBuilder : public stk::mesh::MeshBuilder +{ +public: + CustomMeshBuilder() = default; + CustomMeshBuilder(stk::ParallelMachine comm) + : stk::mesh::MeshBuilder(comm) + {} + + virtual ~CustomMeshBuilder() override = default; + + //using statement to avoid compile-warning about 'only partially overridden' + using stk::mesh::MeshBuilder::create; + + virtual std::unique_ptr create(std::shared_ptr metaData) override + { + STK_ThrowRequireMsg(m_haveComm, "MeshBuilder must be given an MPI communicator before creating BulkData"); + + return std::unique_ptr(new CustomBulkData(metaData, + m_comm, + m_auraOption, + std::move(m_fieldDataManager), + m_initialBucketCapacity, + m_maximumBucketCapacity)); + } + + virtual std::shared_ptr create_meta_data() override + { + std::shared_ptr meta; + if (m_spatialDimension > 0 || !m_entityRankNames.empty()) { + meta = std::make_shared(m_spatialDimension, m_entityRankNames); + } + else { + meta = std::make_shared(); + } + + meta->declare_part("CUSTOM_PART"); + + return meta; + } +}; + + +TEST(StkMeshIoBroker, useExternalBulkData) +{ + stk::mesh::MeshBuilder meshBuilder(MPI_COMM_WORLD); + std::shared_ptr bulk = meshBuilder.set_spatial_dimension(3).create(); + bulk->mesh_meta_data().use_simple_fields(); + + stk::io::StkMeshIoBroker stkMeshIoBroker(MPI_COMM_WORLD); + stkMeshIoBroker.set_bulk_data(bulk); + stkMeshIoBroker.add_mesh_database("generated:1x1x8", stk::io::READ_MESH); + stkMeshIoBroker.create_input_mesh(); + stkMeshIoBroker.populate_bulk_data(); + + EXPECT_EQ(stkMeshIoBroker.bulk_data_ptr().get(), bulk.get()); +} + +TEST(StkMeshIoBroker, useDefaultMeshBuilder) +{ + stk::io::StkMeshIoBroker stkMeshIoBroker(MPI_COMM_WORLD); + stkMeshIoBroker.use_simple_fields(); + stkMeshIoBroker.add_mesh_database("generated:1x1x8", stk::io::READ_MESH); + stkMeshIoBroker.create_input_mesh(); + stkMeshIoBroker.populate_bulk_data(); + + ASSERT_TRUE(stkMeshIoBroker.meta_data_ptr()); + ASSERT_TRUE(stkMeshIoBroker.bulk_data_ptr()); +} + +TEST(StkMeshIoBroker, useCustomMeshBuilder) +{ + stk::io::StkMeshIoBroker stkMeshIoBroker(MPI_COMM_WORLD); + stkMeshIoBroker.set_mesh_builder(std::make_shared(MPI_COMM_WORLD)); + stkMeshIoBroker.use_simple_fields(); + stkMeshIoBroker.add_mesh_database("generated:1x1x8", stk::io::READ_MESH); + stkMeshIoBroker.create_input_mesh(); + stkMeshIoBroker.populate_bulk_data(); + + ASSERT_TRUE(stkMeshIoBroker.meta_data_ptr()); + ASSERT_TRUE(stkMeshIoBroker.bulk_data_ptr()); + + EXPECT_TRUE(stkMeshIoBroker.meta_data().get_part("CUSTOM_PART") != nullptr); + EXPECT_TRUE(dynamic_cast(stkMeshIoBroker.bulk_data_ptr().get()) != nullptr); +} + +TEST(StkMeshIoBroker, useCustomMeshBuilder_afterInternalAlreadyGenerated) +{ + { + stk::io::StkMeshIoBroker stkMeshIoBroker(MPI_COMM_WORLD); + stkMeshIoBroker.use_simple_fields(); + stkMeshIoBroker.add_mesh_database("generated:1x1x8", stk::io::READ_MESH); + stkMeshIoBroker.create_input_mesh(); + EXPECT_ANY_THROW(stkMeshIoBroker.set_mesh_builder(std::make_shared(MPI_COMM_WORLD))); + } + { + stk::io::StkMeshIoBroker stkMeshIoBroker(MPI_COMM_WORLD); + stkMeshIoBroker.use_simple_fields(); + stkMeshIoBroker.add_mesh_database("generated:1x1x8", stk::io::READ_MESH); + stkMeshIoBroker.create_input_mesh(); + stkMeshIoBroker.populate_bulk_data(); + EXPECT_ANY_THROW(stkMeshIoBroker.set_mesh_builder(std::make_shared(MPI_COMM_WORLD))); + } +} + +TEST(StkMeshIoBroker, useCustomMeshBuilder_afterSetExternal) +{ + stk::mesh::MeshBuilder meshBuilder(MPI_COMM_WORLD); + std::shared_ptr bulk = meshBuilder.set_spatial_dimension(3).create(); + bulk->mesh_meta_data().use_simple_fields(); + + stk::io::StkMeshIoBroker stkMeshIoBroker(MPI_COMM_WORLD); + stkMeshIoBroker.set_bulk_data(bulk); + EXPECT_ANY_THROW(stkMeshIoBroker.set_mesh_builder(std::make_shared(MPI_COMM_WORLD))); +} + +} diff --git a/packages/stk/stk_unit_tests/stk_math/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_math/CMakeLists.txt index 0092fc04516d..2c119c74644d 100644 --- a/packages/stk/stk_unit_tests/stk_math/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_math/CMakeLists.txt @@ -32,15 +32,12 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util/stk_util) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_math) - FILE(GLOB SOURCES *.cpp) -TRIBITS_ADD_EXECUTABLE_AND_TEST( - stk_math_unit_tests +if(HAVE_STK_Trilinos) + TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + + TRIBITS_ADD_EXECUTABLE_AND_TEST(stk_math_unit_tests SOURCES ${SOURCES} TESTONLYLIBS stk_unit_main ARGS "" @@ -48,4 +45,19 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST( NUM_MPI_PROCS 1 NOEXEPREFIX ) +else() + add_executable(stk_math_unit_tests ${SOURCES}) + target_link_libraries(stk_math_unit_tests stk_math) + target_link_libraries(stk_math_unit_tests stk_util_util) + target_link_libraries(stk_math_unit_tests Kokkos::kokkos) + target_link_libraries(stk_math_unit_tests stk_unit_test_utils) + target_link_libraries(stk_math_unit_tests stk_unit_main) + + target_include_directories(stk_math_unit_tests PRIVATE + $ + $ + ) + + add_test(NAME "stk_math_unit_tests" COMMAND stk_math_unit_tests) +endif() diff --git a/packages/stk/stk_unit_tests/stk_mesh/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_mesh/CMakeLists.txt index 3ec85b403433..c41c385ecc64 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_mesh/CMakeLists.txt @@ -62,6 +62,7 @@ LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestBulkDataSharing.cp LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestCheckOwnedOrphanedSidesOrEdges.cpp") LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestCEO.cpp") LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestCEOME.cpp") +LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestCEONodes.cpp") LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestChangeEntityOwnerCommMaps.cpp") LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestCheckUniqueGlobalIds.cpp") LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestChangeParts.cpp") @@ -88,6 +89,7 @@ LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestInducedPart.cpp") LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestKeyhole.cpp") LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestLocalIds.cpp") LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestMeshImplUtils.cpp") +LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestMeshModLogObserver.cpp") LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestModificationEnd.cpp") LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestModificationSummary.cpp") LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestNoUpwardConnectivity.cpp") diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestAura.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestAura.cpp index 00c59b29200c..add711a9bba4 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestAura.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestAura.cpp @@ -1285,21 +1285,6 @@ class AuraTetSide : public TestTextMeshAura } } - void destroy_elems(const std::vector>& procElemIds) - { - stk::mesh::EntityVector elemsToDestroy; - for(const std::pair& procElemId : procElemIds) { - if (procElemId.first == get_bulk().parallel_rank()) { - stk::mesh::Entity elem = get_bulk().get_entity(stk::topology::ELEM_RANK, procElemId.second); - ASSERT_TRUE(get_bulk().is_valid(elem)); - elemsToDestroy.push_back(elem); - } - } - - get_bulk().modification_begin(); - stk::mesh::destroy_elements_no_mod_cycle(get_bulk(), elemsToDestroy, get_meta().universal_part()); - get_bulk().modification_end(); - } }; TEST_F(AuraTetSide, removeAuraSideOnOwner_checkMarking) @@ -1375,7 +1360,69 @@ TEST_F(AuraTetSide, removeAuraElemOnP2_auraNodesDeletedOnP1) delete_elem_on_p1_check_aura_nodes_deleted(); } -TEST_F(AuraTetSide, destroy_elems) +class AuraTetGhosting : public TestTextMeshAura +{ +public: + + void make_node110_owned_by_p1() + { + stk::mesh::EntityProcVec nodeToMove; + { + stk::mesh::Entity node110 = get_bulk().get_entity(stk::topology::NODE_RANK,110); + if (get_bulk().parallel_rank() == 0) { + EXPECT_TRUE(get_bulk().bucket(node110).owned()); + int destProc = 1; + nodeToMove.push_back(stk::mesh::EntityProc(node110, destProc)); + } + } + + get_bulk().change_entity_owner(nodeToMove); + + { + stk::mesh::Entity node110 = get_bulk().get_entity(stk::topology::NODE_RANK,110); + if (get_bulk().parallel_rank() == 1) { + EXPECT_TRUE(get_bulk().bucket(node110).owned()); + } + } + } + + void destroy_elems(const std::vector>& procElemIds) + { + stk::mesh::EntityVector elemsToDestroy; + for(const std::pair& procElemId : procElemIds) { + if (procElemId.first == get_bulk().parallel_rank()) { + stk::mesh::Entity elem = get_bulk().get_entity(stk::topology::ELEM_RANK, procElemId.second); + ASSERT_TRUE(get_bulk().is_valid(elem)); + elemsToDestroy.push_back(elem); + } + } + + get_bulk().modification_begin(); + stk::mesh::destroy_elements_no_mod_cycle(get_bulk(), elemsToDestroy, get_meta().universal_part()); + get_bulk().modification_end(); + } + + void custom_ghost(stk::mesh::EntityRank rank, stk::mesh::EntityId entityId, const std::vector& destProcs) + { + get_bulk().modification_begin(); + + stk::mesh::Ghosting& customGhosting = get_bulk().create_ghosting("myCustomGhosting"); + + stk::mesh::Entity entity = get_bulk().get_entity(rank, entityId); + stk::mesh::EntityProcVec entityToGhost; + if (get_bulk().is_valid(entity) && get_bulk().parallel_owner_rank(entity)==get_bulk().parallel_rank()) { + for(int destProc : destProcs) { + entityToGhost.push_back(stk::mesh::EntityProc(entity, destProc)); + } + } + + get_bulk().change_ghosting(customGhosting, entityToGhost); + + get_bulk().modification_end(); + } +}; + +TEST_F(AuraTetGhosting, destroy_elems) { if (get_parallel_size() != 8) { GTEST_SKIP(); } @@ -1449,5 +1496,131 @@ TEST_F(AuraTetSide, destroy_elems) {6, 1165} }); } +TEST_F(AuraTetGhosting, destroy_elems_custom_ghosting) +{ + if (get_parallel_size() != 8) { GTEST_SKIP(); } + + std::string meshDesc = "0, 1049, TET_4, 101,140,138,137\n" + "0, 1050, TET_4, 140,110,139,136\n" + "0, 1051, TET_4, 138,139,108,135\n" + "0, 1052, TET_4, 137,136,135,107\n" + "0, 1053, TET_4, 138,137,140,136\n" + "0, 1054, TET_4, 138,135,137,136\n" + "0, 1055, TET_4, 138,139,135,136\n" + "0, 1056, TET_4, 138,140,139,136\n" + "1, 2000, TET_4, 200,110,162,201\n" + "2, 1065, TET_4, 147,108,103,109\n" + "2, 1066, TET_4, 105,147,103,109\n" + "3, 1108, TET_4, 134,106,133,104\n" + "3, 1109, TET_4, 132,133,105,104\n" + "3, 1110, TET_4, 132,134,133,104\n" + "3, 1111, TET_4, 107,134,132,104\n" + "4, 1081, TET_4, 134,136,107,135\n" + "4, 1145, TET_4, 135,107,136,134\n" + "4, 1146, TET_4, 139,136,110,162\n" + "4, 1148, TET_4, 136,135,134,159\n" + "4, 1149, TET_4, 136,139,135,159\n" + "4, 1150, TET_4, 136,162,139,159\n" + "4, 1151, TET_4, 136,134,162,159\n" + "5, 1089, TET_4, 108,147,135,159\n" + "5, 1090, TET_4, 147,105,132,133\n" + "5, 1091, TET_4, 135,132,107,134\n" + "5, 1092, TET_4, 159,133,134,106\n" + "5, 1093, TET_4, 132,147,133,159\n" + "5, 1094, TET_4, 132,135,147,159\n" + "5, 1095, TET_4, 132,134,135,159\n" + "5, 1096, TET_4, 132,133,134,159\n" + "6, 1160, TET_4, 159,133,160,147\n" + "6, 1161, TET_4, 159,160,133,106\n" + "6, 1162, TET_4, 109,159,160,147\n" + "6, 1163, TET_4, 109,159,147,108\n" + "6, 1164, TET_4, 105,160,133,147\n" + "6, 1165, TET_4, 105,160,147,109\n" + "7, 2001, TET_4, 202,162,110,203|sideset:data=1151,4,1146,2,1052,1,1050,1,1081,4,1053,2\n"; + + setup_text_mesh(meshDesc); + + //This mesh setup provides coverage for some dark corners of BulkData::modification_end. + //Specifically, node 110 is both aura-ghosted and custom-ghosted from proc 0 to procs 4 and 7, + //and it is both shared with, and custom-ghosted to proc 1. + //Similarly, node 135 is shared by procs 0 and 4 and 5, as well as being custom-ghosted + //from proc 0 to procs 4 and 5. + //This test is a success if no exception is thrown from BulkData::modification_end. + + custom_ghost(stk::topology::NODE_RANK, 110, {1, 4, 7}); + custom_ghost(stk::topology::NODE_RANK, 135, {4, 5}); + + destroy_elems({ {0, 1050}, + {4, 1081}, + {4, 1145}, + {4, 1146}, + {4, 1148}, + {4, 1149}, + {1, 2000} }); +} + +TEST_F(AuraTetGhosting, destroy_elems_sharing_and_aura_ghosting) +{ + if (get_parallel_size() != 4) { GTEST_SKIP(); } + + std::string meshDesc = "0, 1049, TET_4, 101,140,138,137\n" + "0, 1050, TET_4, 140,110,139,136\n" + "0, 1051, TET_4, 138,139,108,135\n" + "0, 1052, TET_4, 137,136,135,107\n" + "1, 2000, TET_4, 200,110,162,201\n" + "1, 2001, TET_4, 200,110,165,202\n" + "1, 2002, TET_4, 200,203,204,202\n" + "2, 1065, TET_4, 200,108,103,104\n" + "2, 1066, TET_4, 200,101,105,106\n" + "3, 1085, TET_4, 136,189,183,156\n" + "3, 1086, TET_4, 136,157,153,186|sideset:data=1050,1,1085,1,1086,2\n"; + + //This mesh setup provides coverage for some dark corners of BulkData::modification_end. + //Specifically, we make node 110 shared by procs 0 and 1, and aura-ghosting to proc 2 and 3. + //Additionally, we move the ownership of node 110 to proc 1, while the face that it is + //attached to (face 10501, i.e., the face on side 1 of element 1050) remains owned by proc 0. + //This test is a success if no exception is thrown from BulkData::modification_end. + + setup_text_mesh(meshDesc); + + make_node110_owned_by_p1(); + + destroy_elems({ {2, 1065} }); +} + +TEST_F(AuraTetGhosting, destroy_elems_sharing_and_aura_and_custom_ghosting) +{ + if (get_parallel_size() != 4) { GTEST_SKIP(); } + + std::string meshDesc = "0, 1049, TET_4, 101,140,138,137\n" + "0, 1050, TET_4, 140,110,139,136\n" + "0, 1051, TET_4, 138,139,108,135\n" + "0, 1052, TET_4, 137,136,135,107\n" + "1, 2000, TET_4, 200,110,162,201\n" + "1, 2001, TET_4, 200,110,165,202\n" + "1, 2002, TET_4, 200,203,204,202\n" + "2, 1065, TET_4, 200,108,103,104\n" + "2, 1066, TET_4, 200,101,105,106\n" + "3, 1085, TET_4, 136,189,183,156\n" + "3, 1086, TET_4, 136,157,153,186|sideset:data=1050,1,1085,1,1086,2\n"; + + //This mesh setup provides coverage for some dark corners of BulkData::modification_end. + //Specifically, we make node 110 shared by procs 0 and 1, and aura-ghosting to proc 2 and 3. + //Additionally, we move the ownership of node 110 to proc 1, while the face that it is + //attached to (face 10501, i.e., the face on side 1 of element 1050) remains owned by proc 0. + //Finally, we also custom-ghost elems 1050 and 1051 to procs 2 and 3, as well as node 110 to procs 2 and 3. + //This test is a success if no exception is thrown from BulkData::modification_end. + + setup_text_mesh(meshDesc); + + make_node110_owned_by_p1(); + + custom_ghost(stk::topology::ELEM_RANK, 1050, {2, 3}); + custom_ghost(stk::topology::ELEM_RANK, 1051, {2, 3}); + custom_ghost(stk::topology::NODE_RANK, 110, {2, 3}); + + destroy_elems({ {2, 1065} }); +} + } // empty namespace diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestAuraToShared.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestAuraToShared.cpp index dbdc3811c759..03f8e74e7794 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestAuraToShared.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestAuraToShared.cpp @@ -43,6 +43,7 @@ #include "stk_mesh/base/Bucket.hpp" // for Bucket #include "stk_mesh/base/Entity.hpp" // for Entity #include "stk_mesh/base/Comm.hpp" +#include "stk_mesh/base/DestroyElements.hpp" #include "stk_topology/topology.hpp" // for topology, etc #include "stk_io/FillMesh.hpp" #include "stk_mesh/base/MeshBuilder.hpp" @@ -77,6 +78,22 @@ class TestAura2D : public TestTextMeshAura2d get_bulk().modification_end(); } + + void destroy_elems(const std::vector>& procElemIds) + { + stk::mesh::EntityVector elemsToDestroy; + for(const std::pair& procElemId : procElemIds) { + if (procElemId.first == get_bulk().parallel_rank()) { + stk::mesh::Entity elem = get_bulk().get_entity(stk::topology::ELEM_RANK, procElemId.second); + ASSERT_TRUE(get_bulk().is_valid(elem)); + elemsToDestroy.push_back(elem); + } + } + + get_bulk().modification_begin(); + stk::mesh::destroy_elements_no_mod_cycle(get_bulk(), elemsToDestroy, get_meta().universal_part()); + get_bulk().modification_end(); + } }; TEST_F(TestAura2D, quadKeyhole) @@ -107,6 +124,33 @@ TEST_F(TestAura2D, quadKeyhole) } } +TEST_F(TestAura2D, triSharingGhosting) +{ + if (get_parallel_size() != 4) { GTEST_SKIP(); } + + std::string meshDesc = "0, 1, TRI_3_2D, 1,2,3\n" + "0, 2, TRI_3_2D, 3,2,4\n" + "0, 8, TRI_3_2D, 3,1,10\n" + "1, 3, TRI_3_2D, 2,5,4\n" + "1, 4, TRI_3_2D, 2,6,5\n" + "1, 9, TRI_3_2D, 4,5,11\n" + "2, 5, TRI_3_2D, 2,7,6\n" + "2, 6, TRI_3_2D, 1,7,2\n" + "3, 7, TRI_3_2D, 6,8,5|sideset:data=3,1,4,1,5,1\n"; + + //This test provides coverage for some dark corners of BulkData::modification_end. + //Specifically, it sets up a combination of sharing and aura-ghosting such that + //proc 3 has an aura-ghost of node 2, and doesn't know that procs 1 and 2 share + //that node. Correspondingly, procs 1 and 2 know about each other, but don't know + //that proc 3 has a recv-ghost of node 2. + //The handling of this type of situation will get easier if/when we make ghosting + //info symmetric in BulkData. + + setup_text_mesh(meshDesc); + + destroy_elems({ {0,8}, {1,9} }); +} + class AuraToSharedToAura : public TestTextMeshAura2d { public: diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestDestroyElements.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestDestroyElements.cpp index 833e13167889..99f471eadf9d 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestDestroyElements.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestDestroyElements.cpp @@ -195,6 +195,42 @@ TEST_F(TetMesh, DeleteOneElement) } } +TEST_F(TetMesh, DeleteOneElement_DontDeleteAnyOrphans) +{ + if(stk::parallel_machine_size(MPI_COMM_WORLD) == 1) + { + initialize_my_mesh(stk::mesh::BulkData::NO_AUTO_AURA); + stk::mesh::create_all_sides(get_bulk(), get_meta().universal_part(), {}, false); + + stk::mesh::EntityVector orphanedNodes{ + get_bulk().get_entity(stk::topology::NODE_RANK, 5) + }; + + stk::mesh::EntityVector elementToDestroy{get_bulk().get_entity(stk::topology::ELEMENT_RANK, 2)}; + + stk::mesh::EntityVector facesOfDestroyedElement = get_faces_for_entity(get_bulk(), elementToDestroy[0]); + EXPECT_EQ(facesOfDestroyedElement.size(), 4u); + for(stk::mesh::Entity face : facesOfDestroyedElement) + EXPECT_TRUE(get_bulk().is_valid(face)); + + expect_valid(get_bulk(), orphanedNodes, __LINE__); + expect_valid(get_bulk(), elementToDestroy, __LINE__); + + stk::mesh::Selector selectNoOrphans; + stk::mesh::destroy_elements(get_bulk(), elementToDestroy, selectNoOrphans); + + expect_valid(get_bulk(), orphanedNodes, __LINE__); + expect_invalid(get_bulk(), elementToDestroy, __LINE__); + + unsigned numValid = 0; + for(stk::mesh::Entity face : facesOfDestroyedElement) + if(get_bulk().is_valid(face)) + numValid++; + + EXPECT_EQ(4u, numValid); + } +} + TEST_F(TetMesh, DeleteElementOnProcBoundaryWithOwnedFace) { if(stk::parallel_machine_size(MPI_COMM_WORLD) == 2) diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestField.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestField.cpp index 613dc79702a8..76732751145e 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestField.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestField.cpp @@ -711,13 +711,9 @@ class LateFieldFixtureNoTest : public stk::unit_test_util::simple_fields::MeshFi LateFieldFixtureNoTest(unsigned spatial_dim, const std::vector& entityRankNames) : MeshFixtureNoTest(spatial_dim,entityRankNames) {} - stk::mesh::FieldDataManager * fieldDataManager = nullptr; - virtual ~LateFieldFixtureNoTest() { reset_mesh(); - delete fieldDataManager; - fieldDataManager = nullptr; } }; @@ -726,27 +722,29 @@ class LateFieldFixture : public LateFieldFixtureNoTest, public ::ngp_testing::Te { protected: void custom_allocate_bulk(stk::mesh::BulkData::AutomaticAuraOption auraOption, - stk::mesh::FieldDataManager * fieldDataManager) + std::unique_ptr fieldDataManager) { stk::mesh::MeshBuilder builder(communicator); builder.set_spatial_dimension(m_spatialDim); builder.set_entity_rank_names(m_entityRankNames); builder.set_aura_option(auraOption); - builder.set_field_data_manager(fieldDataManager); + builder.set_field_data_manager(std::move(fieldDataManager)); bulkData = builder.create(); metaData = bulkData->mesh_meta_data_ptr(); } - void setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::AutomaticAuraOption auraOption, stk::mesh::FieldDataManager * fieldDataManager) + void setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::AutomaticAuraOption auraOption, + std::unique_ptr fieldDataManager) { - custom_allocate_bulk(auraOption, fieldDataManager); + custom_allocate_bulk(auraOption, std::move(fieldDataManager)); metaData->enable_late_fields(); } - virtual void setup_empty_mesh(stk::mesh::BulkData::AutomaticAuraOption auraOption, stk::mesh::FieldDataManager * fieldDataManager) + virtual void setup_empty_mesh(stk::mesh::BulkData::AutomaticAuraOption auraOption, + std::unique_ptr fieldDataManager) { - custom_allocate_bulk(auraOption, fieldDataManager); + custom_allocate_bulk(auraOption, std::move(fieldDataManager)); metaData->enable_late_fields(); } @@ -798,8 +796,11 @@ class LateFieldFixture : public LateFieldFixtureNoTest, public ::ngp_testing::Te } template - void setup_add_late_first_field(stk::mesh::EntityRank rank, stk::mesh::FieldDataManager * fieldDataManager = nullptr) { - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, fieldDataManager); + void setup_add_late_first_field(stk::mesh::EntityRank rank, + std::unique_ptr fieldDataManager = + std::unique_ptr()) + { + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); // Note that we still have a nodal coordinates field stk::mesh::Field & lateField = declare_field("late_field", rank); @@ -811,8 +812,11 @@ class LateFieldFixture : public LateFieldFixtureNoTest, public ::ngp_testing::Te } template - void setup_add_late_field(stk::mesh::EntityRank rank, stk::mesh::FieldDataManager * fieldDataManager = nullptr) { - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, fieldDataManager); + void setup_add_late_field(stk::mesh::EntityRank rank, + std::unique_ptr fieldDataManager = + std::unique_ptr()) + { + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); stk::mesh::Field & earlyField = declare_field("early_field", rank); put_field(earlyField, get_meta().universal_part()); stk::io::fill_mesh("generated:1x1x2", *bulkData); @@ -828,8 +832,11 @@ class LateFieldFixture : public LateFieldFixtureNoTest, public ::ngp_testing::Te } template - void setup_add_late_field_multiple_buckets(stk::mesh::EntityRank rank, stk::mesh::FieldDataManager * fieldDataManager = nullptr) { - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, fieldDataManager); + void setup_add_late_field_multiple_buckets(stk::mesh::EntityRank rank, + std::unique_ptr fieldDataManager = + std::unique_ptr()) + { + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); create_part("block_2", stk::topology::ELEM_RANK); stk::mesh::Field & earlyField = declare_field("early_field", rank); put_field(earlyField, get_meta().universal_part()); @@ -856,8 +863,11 @@ class LateFieldFixture : public LateFieldFixtureNoTest, public ::ngp_testing::Te } template - void setup_add_late_field_multiple_duplicate_put_field(stk::mesh::EntityRank rank, stk::mesh::FieldDataManager * fieldDataManager = nullptr) { - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, fieldDataManager); + void setup_add_late_field_multiple_duplicate_put_field(stk::mesh::EntityRank rank, + std::unique_ptr fieldDataManager = + std::unique_ptr()) + { + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); stk::mesh::Field & earlyField = declare_field("early_field", rank); put_field(earlyField, get_meta().universal_part()); stk::io::fill_mesh("generated:1x1x2", *bulkData); @@ -874,8 +884,11 @@ class LateFieldFixture : public LateFieldFixtureNoTest, public ::ngp_testing::Te } template - void setup_add_late_field_multiple_different_put_field(stk::mesh::EntityRank rank, stk::mesh::FieldDataManager * fieldDataManager = nullptr) { - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, fieldDataManager); + void setup_add_late_field_multiple_different_put_field(stk::mesh::EntityRank rank, + std::unique_ptr fieldDataManager = + std::unique_ptr()) + { + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); create_part("block_2", stk::topology::ELEM_RANK); stk::mesh::Field & earlyField = declare_field("early_field", rank); put_field(earlyField, get_meta().universal_part()); @@ -893,8 +906,11 @@ class LateFieldFixture : public LateFieldFixtureNoTest, public ::ngp_testing::Te } template - void setup_add_two_late_fields_sequential(stk::mesh::EntityRank rank, stk::mesh::FieldDataManager * fieldDataManager = nullptr) { - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, fieldDataManager); + void setup_add_two_late_fields_sequential(stk::mesh::EntityRank rank, + std::unique_ptr fieldDataManager = + std::unique_ptr()) + { + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); stk::mesh::Field & earlyField = declare_field("early_field", rank); put_field(earlyField, get_meta().universal_part()); stk::io::fill_mesh("generated:1x1x2", *bulkData); @@ -914,8 +930,11 @@ class LateFieldFixture : public LateFieldFixtureNoTest, public ::ngp_testing::Te } template - void setup_add_two_late_fields_interleaved(stk::mesh::EntityRank rank, stk::mesh::FieldDataManager * fieldDataManager = nullptr) { - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, fieldDataManager); + void setup_add_two_late_fields_interleaved(stk::mesh::EntityRank rank, + std::unique_ptr fieldDataManager = + std::unique_ptr()) + { + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); stk::mesh::Field & earlyField = declare_field("early_field", rank); put_field(earlyField, get_meta().universal_part()); stk::io::fill_mesh("generated:1x1x2", *bulkData); @@ -936,8 +955,11 @@ class LateFieldFixture : public LateFieldFixtureNoTest, public ::ngp_testing::Te } template - void setup_add_two_late_fields_out_of_order(stk::mesh::EntityRank rank, stk::mesh::FieldDataManager * fieldDataManager = nullptr) { - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, fieldDataManager); + void setup_add_two_late_fields_out_of_order(stk::mesh::EntityRank rank, + std::unique_ptr fieldDataManager = + std::unique_ptr()) + { + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); stk::mesh::Field & earlyField = declare_field("early_field", rank); put_field(earlyField, get_meta().universal_part()); stk::io::fill_mesh("generated:1x1x2", *bulkData); @@ -958,8 +980,11 @@ class LateFieldFixture : public LateFieldFixtureNoTest, public ::ngp_testing::Te } template - void setup_add_two_late_fields_different_type_out_of_order(stk::mesh::EntityRank rank, stk::mesh::FieldDataManager * fieldDataManager = nullptr) { - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, fieldDataManager); + void setup_add_two_late_fields_different_type_out_of_order(stk::mesh::EntityRank rank, + std::unique_ptr fieldDataManager = + std::unique_ptr()) + { + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); stk::mesh::Field & earlyField = declare_field("early_field", rank); put_field(earlyField, get_meta().universal_part()); stk::io::fill_mesh("generated:1x1x2", *bulkData); @@ -980,8 +1005,11 @@ class LateFieldFixture : public LateFieldFixtureNoTest, public ::ngp_testing::Te } template - void setup_add_two_late_fields_different_rank_out_of_order(stk::mesh::EntityRank rank1, stk::mesh::EntityRank rank2, stk::mesh::FieldDataManager * fieldDataManager = nullptr) { - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, fieldDataManager); + void setup_add_two_late_fields_different_rank_out_of_order(stk::mesh::EntityRank rank1, stk::mesh::EntityRank rank2, + std::unique_ptr fieldDataManager = + std::unique_ptr()) + { + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); stk::mesh::Field & earlyField = declare_field("early_field", stk::topology::ELEM_RANK); put_field(earlyField, get_meta().universal_part()); stk::io::fill_mesh("generated:1x1x2", *bulkData); @@ -1002,8 +1030,11 @@ class LateFieldFixture : public LateFieldFixtureNoTest, public ::ngp_testing::Te } template - void setup_add_early_field_to_late_part(stk::mesh::EntityRank rank, stk::mesh::FieldDataManager * fieldDataManager = nullptr) { - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, fieldDataManager); + void setup_add_early_field_to_late_part(stk::mesh::EntityRank rank, + std::unique_ptr fieldDataManager = + std::unique_ptr()) + { + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); const bool addIoPartAttribute = false; create_part("block_1", stk::topology::ELEM_RANK, addIoPartAttribute); create_part("block_2", stk::topology::ELEM_RANK); @@ -1039,8 +1070,11 @@ class LateFieldFixture : public LateFieldFixtureNoTest, public ::ngp_testing::Te } template - void setup_add_late_field_to_late_part(stk::mesh::EntityRank rank, stk::mesh::FieldDataManager * fieldDataManager = nullptr) { - setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, fieldDataManager); + void setup_add_late_field_to_late_part(stk::mesh::EntityRank rank, + std::unique_ptr fieldDataManager = + std::unique_ptr()) + { + setup_empty_mesh_with_field_data_manager(stk::mesh::BulkData::NO_AUTO_AURA, std::move(fieldDataManager)); const bool addIoPartAttribute = false; create_part("block_1", stk::topology::ELEM_RANK, addIoPartAttribute); stk::mesh::Field & earlyField = declare_field("early_field", rank); @@ -1220,162 +1254,162 @@ TEST_F(LateFieldFixture, addLateElementFieldToLatePart) TEST_F(LateFieldFixture, addLateIntFirstElementFieldContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_late_first_field(stk::topology::ELEM_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_late_first_field(stk::topology::ELEM_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addLateIntNodalFieldContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_late_field(stk::topology::NODE_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_late_field(stk::topology::NODE_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addLateIntElementFieldContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_late_field(stk::topology::ELEM_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_late_field(stk::topology::ELEM_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addLateIntNodalField_multipleBucketsContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_late_field_multiple_buckets(stk::topology::NODE_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_late_field_multiple_buckets(stk::topology::NODE_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addLateIntNodalField_multipleDuplicatePutFieldContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_late_field_multiple_duplicate_put_field(stk::topology::NODE_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_late_field_multiple_duplicate_put_field(stk::topology::NODE_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addLateIntElementField_multipleDuplicatePutFieldContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_late_field_multiple_duplicate_put_field(stk::topology::ELEM_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_late_field_multiple_duplicate_put_field(stk::topology::ELEM_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addLateIntNodalField_multipleDifferentPutFieldContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_late_field_multiple_different_put_field(stk::topology::NODE_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_late_field_multiple_different_put_field(stk::topology::NODE_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addLateIntElementField_multipleDifferentPutFieldContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_late_field_multiple_different_put_field(stk::topology::ELEM_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_late_field_multiple_different_put_field(stk::topology::ELEM_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addTwoLateIntNodalFields_sequentialContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_two_late_fields_sequential(stk::topology::NODE_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_two_late_fields_sequential(stk::topology::NODE_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addTwoLateIntElementFields_sequentialContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_two_late_fields_sequential(stk::topology::ELEM_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_two_late_fields_sequential(stk::topology::ELEM_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addTwoLateIntNodalFields_interleavedContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_two_late_fields_interleaved(stk::topology::NODE_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_two_late_fields_interleaved(stk::topology::NODE_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addTwoLateIntElementFields_interleavedContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_two_late_fields_interleaved(stk::topology::ELEM_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_two_late_fields_interleaved(stk::topology::ELEM_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addTwoLateIntNodalFields_outOfOrderContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_two_late_fields_out_of_order(stk::topology::NODE_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_two_late_fields_out_of_order(stk::topology::NODE_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addTwoLateIntElementFields_outOfOrderContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_two_late_fields_out_of_order(stk::topology::ELEM_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_two_late_fields_out_of_order(stk::topology::ELEM_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addTwoLateIntAndDoubleNodalFields_outOfOrderContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_two_late_fields_different_type_out_of_order(stk::topology::NODE_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_two_late_fields_different_type_out_of_order(stk::topology::NODE_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addTwoLateIntAndDoubleElementFields_outOfOrderContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_two_late_fields_different_type_out_of_order(stk::topology::ELEM_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_two_late_fields_different_type_out_of_order(stk::topology::ELEM_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addTwoLateShortAndDoubleNodalFields_outOfOrderContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_two_late_fields_different_type_out_of_order(stk::topology::NODE_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_two_late_fields_different_type_out_of_order(stk::topology::NODE_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addTwoLateShortAndDoubleElementFields_outOfOrderContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_two_late_fields_different_type_out_of_order(stk::topology::ELEM_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_two_late_fields_different_type_out_of_order(stk::topology::ELEM_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addTwoLateNodalAndElementFields_outOfOrderContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_two_late_fields_different_rank_out_of_order(stk::topology::NODE_RANK, stk::topology::ELEM_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_two_late_fields_different_rank_out_of_order(stk::topology::NODE_RANK, stk::topology::ELEM_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addEarlyNodalFieldToLatePartContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_early_field_to_late_part(stk::topology::NODE_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_early_field_to_late_part(stk::topology::NODE_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addEarlyElementFieldToLatePartContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_early_field_to_late_part(stk::topology::ELEM_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_early_field_to_late_part(stk::topology::ELEM_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addLateNodalFieldToLatePartContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_late_field_to_late_part(stk::topology::NODE_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_late_field_to_late_part(stk::topology::NODE_RANK, std::move(fieldDataManager)); } TEST_F(LateFieldFixture, addLateElementFieldToLatePartContiguous) { if (stk::parallel_machine_size(MPI_COMM_WORLD) > 2) return; - fieldDataManager = new stk::mesh::ContiguousFieldDataManager; - setup_add_late_field_to_late_part(stk::topology::ELEM_RANK, fieldDataManager); + auto fieldDataManager = std::make_unique(); + setup_add_late_field_to_late_part(stk::topology::ELEM_RANK, std::move(fieldDataManager)); } TEST(SharedSidesetField, verifySidesetFieldAfterMeshRead) { @@ -1854,27 +1888,31 @@ class VariableCapacityFieldData : public ::testing::TestWithParam fieldDataManager; if (GetParam() == FieldDataManagerType::DEFAULT) { const unsigned numRanks = 1; const unsigned alignment = 4; - m_fieldDataManager = std::make_unique(numRanks, alignment); + fieldDataManager = std::make_unique(numRanks, alignment); } else if (GetParam() == FieldDataManagerType::CONTIGUOUS) { const unsigned extraCapacity = 0; const unsigned alignment = 4; - m_fieldDataManager = std::make_unique(extraCapacity, alignment); + fieldDataManager = std::make_unique(extraCapacity, alignment); } + m_fieldDataManager = fieldDataManager.get(); + stk::mesh::MeshBuilder builder(MPI_COMM_WORLD); builder.set_spatial_dimension(3); builder.set_initial_bucket_capacity(initialBucketCapacity); builder.set_maximum_bucket_capacity(maximumBucketCapacity); - builder.set_field_data_manager(m_fieldDataManager.get()); + builder.set_field_data_manager(std::move(fieldDataManager)); m_bulk = builder.create(); m_meta = &m_bulk->mesh_meta_data(); m_meta->use_simple_fields(); @@ -1897,7 +1935,7 @@ class VariableCapacityFieldData : public ::testing::TestWithParam m_fieldDataManager; + stk::mesh::FieldDataManager * m_fieldDataManager; std::unique_ptr m_bulk; stk::mesh::MetaData * m_meta; }; diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestFieldDataManager.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestFieldDataManager.cpp index bb99b5483b2a..058e3547530d 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestFieldDataManager.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestFieldDataManager.cpp @@ -96,7 +96,7 @@ void initializeTestField(stk::mesh::MetaData& meshMetaData) } template -void testAllocateFieldData(stk::mesh::BulkData& bulkData, stk::mesh::FieldDataManager* fieldDataManager, +void testAllocateFieldData(stk::mesh::BulkData& bulkData, stk::mesh::FieldDataManager * fieldDataManager, const size_t extraCapacityInBytes, const size_t numNodes) { const stk::mesh::MetaData& meshMetaData = bulkData.mesh_meta_data(); @@ -140,7 +140,7 @@ void testAllocateFieldData(stk::mesh::BulkData& bulkData, stk::mesh::FieldDataMa } template -void testReorderBucketFieldData(stk::mesh::BulkData& bulkData, stk::mesh::FieldDataManager* fieldDataManager, +void testReorderBucketFieldData(stk::mesh::BulkData& bulkData, stk::mesh::FieldDataManager * fieldDataManager, stk::mesh::EntityRank rank, const stk::mesh::FieldVector& fields, const std::vector &reorderedBucketIds) { @@ -180,7 +180,7 @@ void testReorderBucketFieldData(stk::mesh::BulkData& bulkData, stk::mesh::FieldD } } -void testTwoEntitiesTwoBuckets(stk::mesh::BulkData &bulkData, stk::mesh::FieldDataManager* fieldDataManager) +void testTwoEntitiesTwoBuckets(stk::mesh::BulkData &bulkData, stk::mesh::FieldDataManager * fieldDataManager) { bulkData.deactivate_field_updating(); @@ -238,11 +238,11 @@ void testTwoEntitiesTwoBuckets(stk::mesh::BulkData &bulkData, stk::mesh::FieldDa std::shared_ptr build_mesh(unsigned spatialDim, stk::ParallelMachine comm, - stk::mesh::FieldDataManager* fieldDataManager) + std::unique_ptr fieldDataManager) { stk::mesh::MeshBuilder builder(comm); builder.set_spatial_dimension(spatialDim); - builder.set_field_data_manager(fieldDataManager); + builder.set_field_data_manager(std::move(fieldDataManager)); return builder.create(); } @@ -262,15 +262,16 @@ TYPED_TEST(TestDefaultFieldDataManager, AllocateFieldData) const size_t spatialDim = 3; const size_t numRanks = stk::mesh::entity_rank_names().size(); - stk::mesh::DefaultFieldDataManager fieldDataManager(numRanks); - std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, &fieldDataManager); + auto fieldDataManager = std::make_unique(numRanks); + auto * localFieldDataManager = fieldDataManager.get(); + std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, std::move(fieldDataManager)); stk::mesh::MetaData& meshMetaData = bulkDataPtr->mesh_meta_data(); meshMetaData.use_simple_fields(); initializeTestField(meshMetaData); size_t numNodes = 20; size_t extraCapacity = 0; - testAllocateFieldData(*bulkDataPtr, &fieldDataManager, extraCapacity, numNodes); + testAllocateFieldData(*bulkDataPtr, localFieldDataManager, extraCapacity, numNodes); } TYPED_TEST(TestDefaultFieldDataManager, AllocateFieldDataTwoBuckets) @@ -281,15 +282,16 @@ TYPED_TEST(TestDefaultFieldDataManager, AllocateFieldDataTwoBuckets) const size_t spatialDim = 3; const size_t numRanks = stk::mesh::entity_rank_names().size(); - stk::mesh::DefaultFieldDataManager fieldDataManager(numRanks); - std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, &fieldDataManager); + auto fieldDataManager = std::make_unique(numRanks); + auto * localFieldDataManager = fieldDataManager.get(); + std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, std::move(fieldDataManager)); stk::mesh::MetaData& meshMetaData = bulkDataPtr->mesh_meta_data(); meshMetaData.use_simple_fields(); initializeTestField(meshMetaData); const size_t numNodes = 700; const size_t extraCapacity = 0; - testAllocateFieldData(*bulkDataPtr, &fieldDataManager, extraCapacity, numNodes); + testAllocateFieldData(*bulkDataPtr, localFieldDataManager, extraCapacity, numNodes); } TYPED_TEST(TestDefaultFieldDataManager, TwoEntitiesTwoBuckets) @@ -300,14 +302,15 @@ TYPED_TEST(TestDefaultFieldDataManager, TwoEntitiesTwoBuckets) const size_t spatialDim = 3; const size_t numRanks = stk::mesh::entity_rank_names().size(); - stk::mesh::DefaultFieldDataManager fieldDataManager(numRanks); - std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, &fieldDataManager); + auto fieldDataManager = std::make_unique(numRanks); + auto * localFieldDataManager = fieldDataManager.get(); + std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, std::move(fieldDataManager)); stk::mesh::MetaData& meshMetaData = bulkDataPtr->mesh_meta_data(); meshMetaData.use_simple_fields(); createPart(meshMetaData); initializeTestField(meshMetaData); - testTwoEntitiesTwoBuckets(*bulkDataPtr, &fieldDataManager); + testTwoEntitiesTwoBuckets(*bulkDataPtr, localFieldDataManager); } template @@ -321,15 +324,16 @@ TYPED_TEST(TestContiguousFieldDataManager, AllocateFieldData) using FieldDataType = TypeParam; const size_t spatialDim = 3; - stk::mesh::ContiguousFieldDataManager fieldDataManager; - std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, &fieldDataManager); + auto fieldDataManager = std::make_unique(); + auto * localFieldDataManager = fieldDataManager.get(); + std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, std::move(fieldDataManager)); stk::mesh::MetaData& meshMetaData = bulkDataPtr->mesh_meta_data(); meshMetaData.use_simple_fields(); initializeTestField(meshMetaData); size_t numNodes = 20; - const size_t extraCapacity = fieldDataManager.get_extra_capacity(); + const size_t extraCapacity = localFieldDataManager->get_extra_capacity(); - testAllocateFieldData(*bulkDataPtr, &fieldDataManager, extraCapacity, numNodes); + testAllocateFieldData(*bulkDataPtr, localFieldDataManager, extraCapacity, numNodes); } TYPED_TEST(TestContiguousFieldDataManager, AllocateFieldDataAndReorderBuckets) @@ -339,21 +343,22 @@ TYPED_TEST(TestContiguousFieldDataManager, AllocateFieldDataAndReorderBuckets) using FieldDataType = TypeParam; const size_t spatialDim = 3; - stk::mesh::ContiguousFieldDataManager fieldDataManager; - std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, &fieldDataManager); + auto fieldDataManager = std::make_unique(); + auto * localFieldDataManager = fieldDataManager.get(); + std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, std::move(fieldDataManager)); stk::mesh::MetaData& meshMetaData = bulkDataPtr->mesh_meta_data(); meshMetaData.use_simple_fields(); initializeTestField(meshMetaData); size_t numNodes = 10000; - const size_t extraCapacity = fieldDataManager.get_extra_capacity(); - testAllocateFieldData(*bulkDataPtr, &fieldDataManager, extraCapacity, numNodes); + const size_t extraCapacity = localFieldDataManager->get_extra_capacity(); + testAllocateFieldData(*bulkDataPtr, localFieldDataManager, extraCapacity, numNodes); const int num_buckets = static_cast(numNodes/stk::mesh::get_default_maximum_bucket_capacity() + 1); std::vector reorderedBucketIds(num_buckets,0); for (size_t i=0; i(*bulkDataPtr, &fieldDataManager, stk::topology::NODE_RANK, + testReorderBucketFieldData(*bulkDataPtr, localFieldDataManager, stk::topology::NODE_RANK, meshMetaData.get_fields(), reorderedBucketIds); } @@ -364,14 +369,15 @@ TYPED_TEST(TestContiguousFieldDataManager, TwoEntitiesTwoBuckets) using FieldDataType = TypeParam; const size_t spatialDim = 3; - stk::mesh::ContiguousFieldDataManager fieldDataManager; - std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, &fieldDataManager); + auto fieldDataManager = std::make_unique(); + auto * localFieldDataManager = fieldDataManager.get(); + std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, std::move(fieldDataManager)); stk::mesh::MetaData& meshMetaData = bulkDataPtr->mesh_meta_data(); meshMetaData.use_simple_fields(); createPart(meshMetaData); initializeTestField(meshMetaData); - testTwoEntitiesTwoBuckets(*bulkDataPtr, &fieldDataManager); + testTwoEntitiesTwoBuckets(*bulkDataPtr, localFieldDataManager); } template @@ -421,8 +427,8 @@ TYPED_TEST(TestContiguousFieldDataManager, nodalFieldNotOnAllNodeBuckets) using FieldDataType = TypeParam; const size_t spatialDim = 3; - stk::mesh::ContiguousFieldDataManager fieldDataManager; - std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, &fieldDataManager); + auto fieldDataManager = std::make_unique(); + std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, std::move(fieldDataManager)); stk::mesh::BulkData& bulkData = *bulkDataPtr; stk::mesh::MetaData& meshMetaData = bulkData.mesh_meta_data(); meshMetaData.use_simple_fields(); @@ -483,8 +489,9 @@ TYPED_TEST(TestContiguousFieldDataManager, allocate_bucket_field_data) using FieldDataType = TypeParam; const size_t spatialDim = 3; - stk::mesh::ContiguousFieldDataManager fieldDataManager; - std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, &fieldDataManager); + auto fieldDataManager = std::make_unique(); + auto * localFieldDataManager = fieldDataManager.get(); + std::shared_ptr bulkDataPtr = build_mesh(spatialDim, MPI_COMM_WORLD, std::move(fieldDataManager)); stk::mesh::BulkData& bulkData = *bulkDataPtr; stk::mesh::MetaData& meshMetaData = bulkData.mesh_meta_data(); meshMetaData.use_simple_fields(); @@ -504,7 +511,7 @@ TYPED_TEST(TestContiguousFieldDataManager, allocate_bucket_field_data) stk::mesh::PartVector parts(1, meshMetaData.get_part("part1")); size_t unusedSize = 0; size_t unusedCapacity = 0; - fieldDataManager.allocate_bucket_field_data(stk::topology::NODE_RANK, allFields, parts, unusedSize, unusedCapacity); + localFieldDataManager->allocate_bucket_field_data(stk::topology::NODE_RANK, allFields, parts, unusedSize, unusedCapacity); size_t expectedNumBucketsInField = 1; ASSERT_EQ(expectedNumBucketsInField, part1FieldMetaDataVector.size()); @@ -518,7 +525,7 @@ TYPED_TEST(TestContiguousFieldDataManager, allocate_bucket_field_data) EXPECT_EQ(expectedNumBytesPerEntity, part2FieldMetaDataVector[0].m_bytesPerEntity); - fieldDataManager.allocate_bucket_field_data(stk::topology::NODE_RANK, allFields, parts, unusedSize, unusedCapacity); + localFieldDataManager->allocate_bucket_field_data(stk::topology::NODE_RANK, allFields, parts, unusedSize, unusedCapacity); expectedNumBucketsInField = 2; ASSERT_EQ(expectedNumBucketsInField, part1FieldMetaDataVector.size()); @@ -537,7 +544,7 @@ TYPED_TEST(TestContiguousFieldDataManager, allocate_bucket_field_data) parts.push_back(meshMetaData.get_part("part2")); - fieldDataManager.allocate_bucket_field_data(stk::topology::NODE_RANK, allFields, parts, unusedSize, unusedCapacity); + localFieldDataManager->allocate_bucket_field_data(stk::topology::NODE_RANK, allFields, parts, unusedSize, unusedCapacity); expectedNumBucketsInField = 3; ASSERT_EQ(expectedNumBucketsInField, part1FieldMetaDataVector.size()); @@ -563,7 +570,7 @@ TYPED_TEST(TestContiguousFieldDataManager, allocate_bucket_field_data) size_t allocateAndTestNodeBucketFieldData(const std::vector &partsTable, const std::vector > &bytesPerEntityForField, - stk::mesh::FieldDataManager *fieldDataManager, + stk::mesh::FieldDataManager & fieldDataManager, const stk::mesh::FieldVector &allFields, const stk::mesh::FieldVector &fields) { @@ -575,8 +582,8 @@ size_t allocateAndTestNodeBucketFieldData(const std::vectorallocate_bucket_field_data(stk::topology::NODE_RANK, allFields, partsTable[i], - bucketSize, bucketCapacity); + fieldDataManager.allocate_bucket_field_data(stk::topology::NODE_RANK, allFields, partsTable[i], + bucketSize, bucketCapacity); size_t expectedNumBucketsInField = i+1; EXPECT_EQ(expectedNumBucketsInField, part1FieldMetaDataVector.size()); @@ -593,17 +600,17 @@ size_t allocateAndTestNodeBucketFieldData(const std::vectordeallocate_bucket_field_data(stk::topology::NODE_RANK,bucket_id,bucketCapacity,fields); + fieldDataManager.deallocate_bucket_field_data(stk::topology::NODE_RANK,bucket_id,bucketCapacity,fields); } } template -void allocate_bucket_field_data_tableBased(stk::mesh::FieldDataManager *fieldDataManager) +void allocate_bucket_field_data_tableBased(stk::mesh::FieldDataManager & fieldDataManager) { if (stk::parallel_machine_size(MPI_COMM_WORLD) != 1) return; @@ -653,7 +660,7 @@ TYPED_TEST(TestContiguousFieldDataManager, allocate_bucket_field_data_tableBased { using FieldDataType = TypeParam; stk::mesh::ContiguousFieldDataManager fieldDataManager; - allocate_bucket_field_data_tableBased(&fieldDataManager); + allocate_bucket_field_data_tableBased(fieldDataManager); } TYPED_TEST(TestDefaultFieldDataManager, allocate_bucket_field_data_tableBased) @@ -661,11 +668,12 @@ TYPED_TEST(TestDefaultFieldDataManager, allocate_bucket_field_data_tableBased) using FieldDataType = TypeParam; const int numRanks = 5; stk::mesh::DefaultFieldDataManager fieldDataManager(numRanks); - allocate_bucket_field_data_tableBased(&fieldDataManager); + allocate_bucket_field_data_tableBased(fieldDataManager); } template -void testAddingSingleEntity(stk::mesh::MetaData &meshMetaData, stk::mesh::ContiguousFieldDataManager &fieldDataManager) +void testAddingSingleEntity(stk::mesh::MetaData &meshMetaData, + stk::mesh::ContiguousFieldDataManager & fieldDataManager) { initialize2Parts2Fields(meshMetaData); diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestMeshModLogObserver.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestMeshModLogObserver.cpp index bf8e149219c9..234bfcff86d6 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestMeshModLogObserver.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestMeshModLogObserver.cpp @@ -116,9 +116,9 @@ TEST_F(MeshModLogTest, creation) "P0 mod-cycle=1, declare_relation (ELEMENT_RANK,1) -> (FACE_RANK,11) ordinal=0\n" "P0 mod-cycle=1, declare_relation (FACE_RANK,11) -> (ELEMENT_RANK,1) ordinal=0\n" "start modification_end mod-cycle=1\n" - "P0 (FACE_RANK,11) {P0,s=0,a=0}, elems{1} edges{} nodes{1 2 6 5} \n" + "P0 (FACE_RANK,11) {P0Created}, elems{1} edges{} nodes{1 2 6 5} \n" "finish modification_end mod-cycle=1\n" - "P0 (FACE_RANK,11) {P0,s=0,a=0}, elems{1} edges{} nodes{1 2 6 5} \n" + "P0 (FACE_RANK,11) {P0Created}, elems{1} edges{} nodes{1 2 6 5} \n" ); EXPECT_EQ(expected, str); } @@ -141,8 +141,8 @@ TEST_F(MeshModLogTest, deletion) if (bulk.parallel_rank() == 0) { const std::string expected( "modification_begin mod-cycle=1\n" - "P0 (ELEMENT_RANK,2) {P0,s=0,a=0}, faces{21 22 23 24} edges{} nodes{5 6 8 7 9 10 12 11} \n" - "P0 (ELEMENT_RANK,1) {P0,s=0,a=0}, faces{11 12 13 14 15} edges{} nodes{1 2 4 3 5 6 8 7} \n" + "P0 (ELEMENT_RANK,2) {P0Created}, faces{21 22 23 24} edges{} nodes{5 6 8 7 9 10 12 11} \n" + "P0 (ELEMENT_RANK,1) {P0Created}, faces{11 12 13 14 15} edges{} nodes{1 2 4 3 5 6 8 7} \n" "P0 mod-cycle=2, destroy_entity (ELEMENT_RANK,1)\n" "P0 mod-cycle=2, destroy_relation (ELEMENT_RANK,1) -> (FACE_RANK,15) ordinal=4\n" "P0 mod-cycle=2, destroy_relation (ELEMENT_RANK,1) -> (FACE_RANK,14) ordinal=3\n" @@ -158,9 +158,9 @@ TEST_F(MeshModLogTest, deletion) "P0 mod-cycle=2, destroy_relation (ELEMENT_RANK,1) -> (NODE_RANK,2) ordinal=1\n" "P0 mod-cycle=2, destroy_relation (ELEMENT_RANK,1) -> (NODE_RANK,1) ordinal=0\n" "start modification_end mod-cycle=2\n" - "P0 (ELEMENT_RANK,2) {P0,s=0,a=0}, faces{21 22 23 24} edges{} nodes{5 6 8 7 9 10 12 11} \n" + "P0 (ELEMENT_RANK,2) {P0Modified}, faces{21 22 23 24} edges{} nodes{5 6 8 7 9 10 12 11} \n" "finish modification_end mod-cycle=2\n" - "P0 (ELEMENT_RANK,2) {P0,s=0,a=0}, faces{21 22 23 24} edges{} nodes{5 6 8 7 9 10 12 11} \n" + "P0 (ELEMENT_RANK,2) {P0Modified}, faces{21 22 23 24} edges{} nodes{5 6 8 7 9 10 12 11} \n" ); EXPECT_EQ(expected, str); } @@ -181,7 +181,7 @@ TEST_F(MeshModLogTest, deletion_modcycle) if (bulk.parallel_rank() == 0) { const std::string expected( "modification_begin mod-cycle=1\n" - "P0 (ELEMENT_RANK,1) {P0,s=0,a=0}, faces{11 12 13 14 15} edges{} nodes{1 2 4 3 5 6 8 7} \n" + "P0 (ELEMENT_RANK,1) {P0Created}, faces{11 12 13 14 15} edges{} nodes{1 2 4 3 5 6 8 7} \n" "P0 mod-cycle=2, destroy_entity (ELEMENT_RANK,1)\n" "P0 mod-cycle=2, destroy_relation (ELEMENT_RANK,1) -> (FACE_RANK,15) ordinal=4\n" "P0 mod-cycle=2, destroy_relation (ELEMENT_RANK,1) -> (FACE_RANK,14) ordinal=3\n" @@ -219,14 +219,14 @@ TEST_F(MeshModLogTest, relations) if (bulk.parallel_rank() == 0) { const std::string expected( "modification_begin mod-cycle=1\n" - "P0 (ELEMENT_RANK,1) {P0,s=0,a=0}, faces{11 12 13 14 15} edges{} nodes{1 2 4 3 5 6 8 7} \n" + "P0 (ELEMENT_RANK,1) {P0Created}, faces{11 12 13 14 15} edges{} nodes{1 2 4 3 5 6 8 7} \n" "P0 mod-cycle=2, destroy_relation (ELEMENT_RANK,1) -> (NODE_RANK,1) ordinal=0\n" "P0 mod-cycle=2, declare_relation (ELEMENT_RANK,1) -> (NODE_RANK,100) ordinal=0\n" "P0 mod-cycle=2, declare_relation (NODE_RANK,100) -> (ELEMENT_RANK,1) ordinal=0\n" "start modification_end mod-cycle=2\n" - "P0 (ELEMENT_RANK,1) {P0,s=0,a=0}, faces{11 12 13 14 15} edges{} nodes{100 2 4 3 5 6 8 7} \n" + "P0 (ELEMENT_RANK,1) {P0Modified}, faces{11 12 13 14 15} edges{} nodes{100 2 4 3 5 6 8 7} \n" "finish modification_end mod-cycle=2\n" - "P0 (ELEMENT_RANK,1) {P0,s=0,a=0}, faces{11 12 13 14 15} edges{} nodes{100 2 4 3 5 6 8 7} \n" + "P0 (ELEMENT_RANK,1) {P0Modified}, faces{11 12 13 14 15} edges{} nodes{100 2 4 3 5 6 8 7} \n" ); EXPECT_EQ(expected, str); } diff --git a/packages/stk/stk_unit_tests/stk_mesh/UnitTestStkTextMesh.cpp b/packages/stk/stk_unit_tests/stk_mesh/UnitTestStkTextMesh.cpp index b08d378711ea..3156c9aa7c67 100644 --- a/packages/stk/stk_unit_tests/stk_mesh/UnitTestStkTextMesh.cpp +++ b/packages/stk/stk_unit_tests/stk_mesh/UnitTestStkTextMesh.cpp @@ -37,6 +37,163 @@ TEST_F(TestTextMeshAura, twoQuadShellWithCoordinates) if (rank == 1) verify_shared_nodes(EntityIdVector{2,5}, 0); } +TEST_F(TestTextMeshAura, sideBlockTopologyForConnectedHexes) +{ + if (get_parallel_size() != 1) return; + + std::string meshDesc = + "0,1,HEX_8,1, 2, 3, 4, 5, 6, 7, 8,block_1\n" + "0,2,HEX_8,5, 6, 7, 8, 9,10,11,12,block_2\n" + "|sideset:name=surface_1; data=1,1, 2,1; split=block"; + + setup_text_mesh(meshDesc); + + verify_num_elements(2); + + std::vector sidesetSubsets; + std::string prefix("surface_"); + sidesetSubsets.push_back(prefix + std::string("block_1") + "_" + get_topology_name("QUAD_4") + "_1"); + sidesetSubsets.push_back(prefix + std::string("block_2") + "_" + get_topology_name("QUAD_4") + "_1"); + + verify_num_sidesets(1); + verify_single_sideset("surface_1", 1, sidesetSubsets, SideVector{{1, 1}, {2, 1}}); + + for (const std::string& subsetName : sidesetSubsets) { + stk::mesh::Part* subsetPart = get_meta().get_part(subsetName); + EXPECT_TRUE(nullptr != subsetPart); + EXPECT_EQ(stk::topology::QUAD_4, subsetPart->topology()); + } +} + +TEST_F(TestTextMeshAura2d, surfaceToBlockMapping) +{ + if (get_parallel_size() != 1) return; + + std::string meshDesc = + "0,1,TRI_3_2D,3,1,4,block_1\n" + "0,2,TRI_3_2D,1,2,4,block_1\n" + "0,3,TRI_3_2D,2,5,4,block_1\n" + "0,4,TRI_3_2D,5,7,4,block_2\n" + "0,5,TRI_3_2D,7,6,4,block_2\n" + "0,6,TRI_3_2D,6,3,4,block_2\n" + "|coordinates: 0,0,0.1,0,0,0.1,0.05,0.1,0.1,0.1,0,0.2,0.1,0.2" + "|dimension:2" + "|sideset:name=skinned_surf; skin=all; split=block" + "|sideset:name=internal_surf; data=1,3,3,2,4,3,6,2; split=block" + "|sideset:name=shared_surf; data=1,1,6,1; split=block" + "|sideset:name=owned_surf; data=5,1; split=block"; + + setup_text_mesh(meshDesc); + + verify_num_elements(6); + verify_num_sidesets(4); + + const stk::mesh::Part* block_1 = get_meta().get_part("block_1"); + const stk::mesh::Part* block_2 = get_meta().get_part("block_2"); + + EXPECT_TRUE(nullptr != block_1); + EXPECT_TRUE(nullptr != block_2); + + { + std::string sidesetName("skinned_surf"); + + stk::mesh::Part* sidesetPart = get_meta().get_part(sidesetName); + EXPECT_TRUE(nullptr != sidesetPart); + std::vector goldParts{block_1, block_2}; + EXPECT_EQ(goldParts, get_meta().get_blocks_touching_surface(sidesetPart)); + } + + { + std::string sidesetName("skinned_surf_" + std::string("block_1") + "_" + get_topology_name("LINE_2")); + + stk::mesh::Part* sidesetPart = get_meta().get_part(sidesetName); + EXPECT_TRUE(nullptr != sidesetPart); + std::vector goldParts{block_1}; + EXPECT_EQ(goldParts, get_meta().get_blocks_touching_surface(sidesetPart)); + } + + { + std::string sidesetName("skinned_surf_" + std::string("block_2") + "_" + get_topology_name("LINE_2")); + + stk::mesh::Part* sidesetPart = get_meta().get_part(sidesetName); + EXPECT_TRUE(nullptr != sidesetPart); + std::vector goldParts{block_2}; + EXPECT_EQ(goldParts, get_meta().get_blocks_touching_surface(sidesetPart)); + } + + { + std::string sidesetName("internal_surf"); + + stk::mesh::Part* sidesetPart = get_meta().get_part(sidesetName); + EXPECT_TRUE(nullptr != sidesetPart); + std::vector goldParts{block_1, block_2}; + EXPECT_EQ(goldParts, get_meta().get_blocks_touching_surface(sidesetPart)); + } + + { + std::string sidesetName("internal_surf_" + std::string("block_1") + "_" + get_topology_name("LINE_2")); + + stk::mesh::Part* sidesetPart = get_meta().get_part(sidesetName); + EXPECT_TRUE(nullptr != sidesetPart); + std::vector goldParts{block_1}; + EXPECT_EQ(goldParts, get_meta().get_blocks_touching_surface(sidesetPart)); + } + + { + std::string sidesetName("internal_surf_" + std::string("block_2") + "_" + get_topology_name("LINE_2")); + + stk::mesh::Part* sidesetPart = get_meta().get_part(sidesetName); + EXPECT_TRUE(nullptr != sidesetPart); + std::vector goldParts{block_2}; + EXPECT_EQ(goldParts, get_meta().get_blocks_touching_surface(sidesetPart)); + } + + { + std::string sidesetName("shared_surf"); + + stk::mesh::Part* sidesetPart = get_meta().get_part(sidesetName); + EXPECT_TRUE(nullptr != sidesetPart); + std::vector goldParts{block_1, block_2}; + EXPECT_EQ(goldParts, get_meta().get_blocks_touching_surface(sidesetPart)); + } + + { + std::string sidesetName("shared_surf_" + std::string("block_1") + "_" + get_topology_name("LINE_2")); + + stk::mesh::Part* sidesetPart = get_meta().get_part(sidesetName); + EXPECT_TRUE(nullptr != sidesetPart); + std::vector goldParts{block_1}; + EXPECT_EQ(goldParts, get_meta().get_blocks_touching_surface(sidesetPart)); + } + + { + std::string sidesetName("shared_surf_" + std::string("block_2") + "_" + get_topology_name("LINE_2")); + + stk::mesh::Part* sidesetPart = get_meta().get_part(sidesetName); + EXPECT_TRUE(nullptr != sidesetPart); + std::vector goldParts{block_2}; + EXPECT_EQ(goldParts, get_meta().get_blocks_touching_surface(sidesetPart)); + } + + { + std::string sidesetName("owned_surf"); + + stk::mesh::Part* sidesetPart = get_meta().get_part(sidesetName); + EXPECT_TRUE(nullptr != sidesetPart); + std::vector goldParts{block_2}; + EXPECT_EQ(goldParts, get_meta().get_blocks_touching_surface(sidesetPart)); + } + + { + std::string sidesetName("owned_surf_" + std::string("block_2") + "_" + get_topology_name("LINE_2")); + + stk::mesh::Part* sidesetPart = get_meta().get_part(sidesetName); + EXPECT_TRUE(nullptr != sidesetPart); + std::vector goldParts{block_2}; + EXPECT_EQ(goldParts, get_meta().get_blocks_touching_surface(sidesetPart)); + } +} + void test_get_mesh_spec(unsigned blockCountToDist, const std::vector& numProcs, const std::vector>& expectedDist) { diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_middle_mesh/CMakeLists.txt index a0e27428e110..0f96faa56128 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/CMakeLists.txt @@ -1,25 +1,30 @@ -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util/stk_util) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_middle_mesh) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_middle_mesh/stk_middle_mesh) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_middle_mesh/stk_middle_mesh/predicates) - FILE(GLOB SOURCES *.cpp util/*.cpp) - -TRIBITS_ADD_EXECUTABLE( - stk_middle_mesh_utest +if(HAVE_STK_Trilinos) + TRIBITS_ADD_EXECUTABLE(stk_middle_mesh_utest SOURCES ${SOURCES} TESTONLYLIBS stk_unit_main COMM serial mpi NOEXEPREFIX ) -TRIBITS_ADD_TEST( - stk_middle_mesh_utest + TRIBITS_ADD_TEST(stk_middle_mesh_utest ARGS "" COMM serial mpi NUM_MPI_PROCS 1 NOEXEPREFIX ) +else() + add_executable(stk_middle_mesh_utest ${SOURCES}) + target_link_libraries(stk_middle_mesh_utest stk_middle_mesh) + target_link_libraries(stk_coupling_utest stk_unit_test_utils) + target_link_libraries(stk_coupling_utest stk_util_parallel) + target_link_libraries(stk_coupling_utest stk_unit_main) + + add_test(NAME "stk_middle_mesh_utest" COMMAND stk_middle_mesh_utest) +endif() + +target_include_directories(stk_middle_mesh_utest PRIVATE + $ + $ +) diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/test_average_normal_field.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/test_average_normal_field.cpp new file mode 100644 index 000000000000..43e273b90361 --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/test_average_normal_field.cpp @@ -0,0 +1,33 @@ +#include "gtest/gtest.h" +#include "stk_middle_mesh/predicates/average_normal_field.hpp" +#include "stk_middle_mesh/create_mesh.hpp" + +using namespace stk::middle_mesh; + +TEST(AveragedNormalField, Plane) +{ + mesh::impl::MeshSpec spec; + spec.xmin = 0; spec.ymin = 0; + spec.xmax = 1; spec.ymax = 1; + spec.numelX = 4, spec.numelY = 4; + + auto f = [](const utils::Point& pt) { return pt; }; + auto mesh = mesh::impl::create_mesh(spec, f); + + predicates::impl::AveragedNormalField averagedNormalField(mesh); + auto& normalField = *(averagedNormalField.get_field()); + + double edgeLength = (spec.xmax - spec.xmin)/spec.numelX; + for (auto& vert : mesh->get_vertices()) + { + if (vert) + { + utils::Point normal = normalField(vert, 0, 0); + + EXPECT_NEAR(normal.x, 0.0, 1e-13); + EXPECT_NEAR(normal.y, 0.0, 1e-13); + EXPECT_NEAR(normal.z, edgeLength, 1e-13); + } + } + +} \ No newline at end of file diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/test_destination_field_gather.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/test_destination_field_gather.cpp index acbfb301f464..ccce25f06260 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/test_destination_field_gather.cpp +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/test_destination_field_gather.cpp @@ -112,21 +112,21 @@ TEST(DestinationFieldGatherer, 2Procs) auto el2 = find_closest_entity(mesh, 2, {0.75, 0.75, 0}); - expect_eq(get_values(gatheredDestField, v1, 0), {}); + expect_eq(get_values(gatheredDestField, v1, 0), {0, 2}); expect_eq(get_values(gatheredDestField, v2, 0), {2}); - expect_eq(get_values(gatheredDestField, v3, 0), {}); + expect_eq(get_values(gatheredDestField, v3, 0), {0, 1, 2, 3}); expect_eq(get_values(gatheredDestField, v4, 0), {2, 3}); - expect_eq(get_values(gatheredDestField, v5, 0), {}); + expect_eq(get_values(gatheredDestField, v5, 0), {1, 3}); expect_eq(get_values(gatheredDestField, v6, 0), {3}); expect_eq(get_values(gatheredDestField, el1->get_down(0), 0), {2}); expect_eq(get_values(gatheredDestField, el1->get_down(1), 0), {2}); expect_eq(get_values(gatheredDestField, el1->get_down(2), 0), {2, 3}); - expect_eq(get_values(gatheredDestField, el1->get_down(3), 0), {}); + expect_eq(get_values(gatheredDestField, el1->get_down(3), 0), {0, 2}); expect_eq(get_values(gatheredDestField, el2->get_down(1), 0), {3}); expect_eq(get_values(gatheredDestField, el2->get_down(2), 0), {3}); - expect_eq(get_values(gatheredDestField, el2->get_down(3), 0), {}); + expect_eq(get_values(gatheredDestField, el2->get_down(3), 0), {1, 3}); } } @@ -195,15 +195,15 @@ TEST(DestinationFieldGatherer, 4ProcsMultipleDestinations) auto v4 = find_closest_entity(mesh, 0, {1.0, 0.5, 0}); auto el1 = find_closest_entity(mesh, 2, {0.75, 0.25, 0}); - expect_eq(get_values(gatheredDestField, v1, 0), {}); + expect_eq(get_values(gatheredDestField, v1, 0), {0, 1, 2}); expect_eq(get_values(gatheredDestField, v2, 0), {1, 2}); - expect_eq(get_values(gatheredDestField, v3, 0), {}); + expect_eq(get_values(gatheredDestField, v3, 0), {0, 1, 2, 3}); expect_eq(get_values(gatheredDestField, v4, 0), {1, 2, 3}); expect_eq(get_values(gatheredDestField, el1->get_down(0), 0), {1, 2}); expect_eq(get_values(gatheredDestField, el1->get_down(1), 0), {1, 2}); expect_eq(get_values(gatheredDestField, el1->get_down(2), 0), {1, 2, 3}); - expect_eq(get_values(gatheredDestField, el1->get_down(3), 0), {}); + expect_eq(get_values(gatheredDestField, el1->get_down(3), 0), {0, 1, 2}); } else if (myrank == 2) { auto v1 = find_closest_entity(mesh, 0, {0.0, 0.5, 0}); @@ -212,12 +212,12 @@ TEST(DestinationFieldGatherer, 4ProcsMultipleDestinations) auto v4 = find_closest_entity(mesh, 0, {0.5, 1.0, 0}); auto el1 = find_closest_entity(mesh, 2, {0.25, 0.75, 0}); - expect_eq(get_values(gatheredDestField, v1, 0), {}); - expect_eq(get_values(gatheredDestField, v2, 0), {}); + expect_eq(get_values(gatheredDestField, v1, 0), {0, 1, 2, 3}); + expect_eq(get_values(gatheredDestField, v2, 0), {0, 1, 2, 3}); expect_eq(get_values(gatheredDestField, v3, 0), {2, 3}); expect_eq(get_values(gatheredDestField, v4, 0), {1, 2, 3}); - expect_eq(get_values(gatheredDestField, el1->get_down(0), 0), {}); + expect_eq(get_values(gatheredDestField, el1->get_down(0), 0), {0, 1, 2, 3}); expect_eq(get_values(gatheredDestField, el1->get_down(1), 0), {1, 2, 3}); expect_eq(get_values(gatheredDestField, el1->get_down(2), 0), {2, 3}); expect_eq(get_values(gatheredDestField, el1->get_down(3), 0), {2, 3}); @@ -229,16 +229,14 @@ TEST(DestinationFieldGatherer, 4ProcsMultipleDestinations) auto v4 = find_closest_entity(mesh, 0, {1.0, 1.0, 0}); auto el1 = find_closest_entity(mesh, 2, {0.75, 0.75, 0}); - expect_eq(get_values(gatheredDestField, v1, 0), {}); - expect_eq(get_values(gatheredDestField, v2, 0), {}); - expect_eq(get_values(gatheredDestField, v3, 0), {}); + expect_eq(get_values(gatheredDestField, v1, 0), {0, 1, 2, 3}); + expect_eq(get_values(gatheredDestField, v2, 0), {1, 2, 3}); + expect_eq(get_values(gatheredDestField, v3, 0), {1, 2, 3}); expect_eq(get_values(gatheredDestField, v4, 0), {1, 3}); - expect_eq(get_values(gatheredDestField, el1->get_down(0), 0), {}); + expect_eq(get_values(gatheredDestField, el1->get_down(0), 0), {1, 2, 3}); expect_eq(get_values(gatheredDestField, el1->get_down(1), 0), {1, 3}); expect_eq(get_values(gatheredDestField, el1->get_down(2), 0), {1, 3}); - expect_eq(get_values(gatheredDestField, el1->get_down(3), 0), {}); + expect_eq(get_values(gatheredDestField, el1->get_down(3), 0), {1, 2, 3}); } -} - -//TODO: test 4 procs, test multiple destinations \ No newline at end of file +} \ No newline at end of file diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/test_entity_sorted_by_owner.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/test_entity_sorted_by_owner.cpp index c2967c876fa3..9316a365185d 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/test_entity_sorted_by_owner.cpp +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/test_entity_sorted_by_owner.cpp @@ -36,19 +36,19 @@ TEST(EntitySortedByOwner, Access) container.insert({0, 1}, v0); container.insert({0, 2}, v2); - EXPECT_EQ(container.get_entity({0, 2}), v2); - EXPECT_EQ(container.get_entity({0, 1}), v0); - EXPECT_EQ(container.get_entity({0, 0}), v1); + EXPECT_EQ(container.get_value({0, 2}), v2); + EXPECT_EQ(container.get_value({0, 1}), v0); + EXPECT_EQ(container.get_value({0, 0}), v1); container.insert({1, 1}, v4); container.insert({1, 3}, v3); container.insert({1, 5}, v5); - EXPECT_EQ(container.get_entity({1, 5}), v5); - EXPECT_EQ(container.get_entity({1, 3}), v3); - EXPECT_EQ(container.get_entity({1, 1}), v4); - EXPECT_EQ(container.get_entity({1, 0}), nullptr); - EXPECT_EQ(container.get_entity({1, 2}), nullptr); - EXPECT_EQ(container.get_entity({1, 4}), nullptr); - EXPECT_EQ(container.get_entity({1, 6}), nullptr); + EXPECT_EQ(container.get_value({1, 5}), v5); + EXPECT_EQ(container.get_value({1, 3}), v3); + EXPECT_EQ(container.get_value({1, 1}), v4); + EXPECT_EQ(container.get_value({1, 0}), nullptr); + EXPECT_EQ(container.get_value({1, 2}), nullptr); + EXPECT_EQ(container.get_value({1, 4}), nullptr); + EXPECT_EQ(container.get_value({1, 6}), nullptr); } \ No newline at end of file diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh.cpp index 1787e210f845..5dfe30f3db19 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh.cpp +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh.cpp @@ -1267,6 +1267,9 @@ TEST(Mesh, ErrorRemotesNotUnique) if (utils::impl::comm_size(MPI_COMM_WORLD) != 1) GTEST_SKIP(); +#ifndef NDEBUG + GTEST_SKIP(); +#endif auto mesh = make_empty_mesh(MPI_COMM_WORLD); auto v1 = mesh->create_vertex(0, 0, 0); auto v2 = mesh->create_vertex(0, 1, 0); @@ -1289,6 +1292,33 @@ TEST(Mesh, AnnulusRemotes) mesh::check_topology(mesh1); } +TEST(Mesh, getLocalId) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 1) + GTEST_SKIP(); + + MeshSpec spec; + spec.numelX = 1; + spec.numelY = 1; + spec.xmin = 0; + spec.xmax = 1; + spec.ymin = 0; + spec.ymax = 1; + + auto func = [&](const utils::Point& pt) { return pt; }; + + std::shared_ptr mesh = create_mesh(spec, func); + + mesh::MeshEntityPtr el = mesh->get_elements()[0]; + std::array verts; + mesh::get_downward(el, 0, verts.data()); + + for (int i=0; i < 4; ++i) + { + EXPECT_EQ(mesh::get_local_id(el, verts[i]), i); + } +} + } // namespace impl } // namespace middle_mesh } // namespace stk diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh_projection_calculator.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh_projection_calculator.cpp index 17750a13840a..53ac400a5e67 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh_projection_calculator.cpp +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh_projection_calculator.cpp @@ -37,7 +37,7 @@ class MeshProjectionCalculatorTester : public ::testing::Test m_mesh2, PointClassifierNormalWrapperTolerances(eps)); mRelationalData = std::make_shared(m_mesh1, m_mesh2, m_meshIn); mProjector = std::make_shared( - m_mesh1, m_mesh2, m_meshIn, mRelationalData, mClassifier, middle_mesh::impl::EdgeTracerTolerances(eps)); + m_mesh1, m_mesh2, mRelationalData, mClassifier, middle_mesh::impl::EdgeTracerTolerances(eps)); mProjector->project(); } diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh_relational_data_scatter.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh_relational_data_scatter.cpp new file mode 100644 index 000000000000..d5d8ed4b4c1a --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh_relational_data_scatter.cpp @@ -0,0 +1,803 @@ +#include "gtest/gtest.h" +#include "stk_middle_mesh/predicates/intersection_common.hpp" +#include "stk_middle_mesh/mesh_relational_data_scatter.hpp" +#include "stk_middle_mesh/mesh.hpp" +#include "stk_middle_mesh/mesh_relational_data.hpp" +#include "stk_middle_mesh/create_mesh.hpp" +#include "stk_middle_mesh/mesh_scatter_spec.hpp" +#include "stk_middle_mesh/mesh_scatter.hpp" +#include "stk_middle_mesh/predicates/point_classifier_normal_wrapper.hpp" +#include "stk_util/parallel/ParallelReduce.hpp" + +namespace { + +using namespace stk::middle_mesh; + +class MeshRelationalDataScatterTesterBase : public ::testing::Test +{ + protected: + MeshRelationalDataScatterTesterBase() : + m_comm1(MPI_COMM_NULL), + m_comm2(MPI_COMM_NULL), + m_unionComm(MPI_COMM_WORLD) + {} + + virtual ~MeshRelationalDataScatterTesterBase() + { + if (m_comm1 != MPI_COMM_NULL) + MPI_Comm_free(&m_comm1); + + if (m_comm2 != MPI_COMM_NULL) + MPI_Comm_free(&m_comm2); + } + + void SetUp() override + { + int commsize = utils::impl::comm_size(m_unionComm); + if (!(commsize == 1 || commsize == 2 || commsize == 4)) + { + GTEST_SKIP(); + } + + split_comm(); + create_meshes(); + auto scatterSpec1 = create_scatter_spec1(); + auto scatterSpec2 = create_scatter_spec2(); + + mesh::impl::MeshScatter scatter1(scatterSpec1, m_mesh1, m_comm2, true); + m_mesh1ScatteredToMesh2 = scatter1.scatter(); + auto mesh1EntityDestinations = scatter1.get_entity_destinations(); + auto mesh1EntityOrigins = scatter1.get_entity_origins(); + + mesh::impl::MeshScatter scatter2(scatterSpec2, m_mesh2, m_comm1, true); + m_mesh2ScatteredToMesh1 = scatter2.scatter(); + auto mesh2EntityDestinations = scatter2.get_entity_destinations(); + auto mesh2EntityOrigins = scatter2.get_entity_origins(); + + create_mesh_relational_data_input(); + + nonconformal4::impl::MeshRelationalDataScatterInput scatterInput; + scatterInput.mesh1 = m_mesh1; + scatterInput.mesh2 = m_mesh2; + scatterInput.mesh1ScatteredToMesh2 = m_mesh1ScatteredToMesh2; + scatterInput.mesh2ScatteredToMesh1 = m_mesh2ScatteredToMesh1; + scatterInput.mesh1EntityOrigins = mesh1EntityOrigins; + scatterInput.mesh1EntityDestinations = mesh1EntityDestinations; + scatterInput.mesh2EntityOrigins = mesh2EntityOrigins; + scatterInput.mesh2EntityDestinations = mesh2EntityDestinations; + + m_meshRelationalDataScatter = std::make_shared( + scatterInput, + m_meshRelationalDataInput, + m_pointClassifierOrigin, m_pointClassifierDest, + m_unionComm); + } + + + void split_comm() + { + int myrank = utils::impl::comm_rank(m_unionComm); + int commsize = utils::impl::comm_size(m_unionComm); + + if (commsize == 1) + { + MPI_Comm_dup(m_unionComm, &m_comm1); + MPI_Comm_dup(m_unionComm, &m_comm2); + } else if (commsize == 2 || commsize == 4) + { + int color = myrank < commsize/2 ? 0 : 1; + MPI_Comm* newcomm = myrank < commsize/2 ? &m_comm1 : &m_comm2; + MPI_Comm_split(m_unionComm, color, 0, newcomm); + } else + { + throw std::runtime_error("MeshRelationalDataScatter tests only run with 1, 2, or 4 procs"); + } + } + + virtual void create_meshes() = 0; + + virtual std::shared_ptr create_scatter_spec1() = 0; + + virtual std::shared_ptr create_scatter_spec2() = 0; + + void create_mesh_relational_data_input() + { + if (m_mesh2) + { + + m_meshRelationalDataInput = std::make_shared(m_mesh1ScatteredToMesh2, m_mesh2, nullptr); + m_pointClassifierOrigin = std::make_shared(m_mesh2); + + // put enough data into the fields so the MeshRelationalScatter will complete + // without segfaulting due to nullptrs + // Each test can overwrite this data as needed + auto& verts1ToFakeVerts = *(m_meshRelationalDataInput->verts1ToFakeVerts); + for (auto& vert1 : m_mesh1ScatteredToMesh2->get_vertices()) + if (vert1) + { + verts1ToFakeVerts(vert1, 0, 0) = m_vertGenerator.get_vert(vert1->get_point_orig(0)); + } + + auto& verts2ToFakeVerts = *(m_meshRelationalDataInput->verts2ToFakeVerts); + auto& verts2ClassOnMesh1 = *(m_meshRelationalDataInput->verts2ClassOnMesh1); + mesh::MeshEntityPtr el1 = m_mesh1ScatteredToMesh2->get_elements()[0]; + for (auto& vert2 : m_mesh2->get_vertices()) + if (vert2) + { + verts2ToFakeVerts(vert2, 0, 0) = m_vertGenerator.get_vert(vert2->get_point_orig(0)); + verts2ClassOnMesh1(vert2, 0, 0) = predicates::impl::PointRecord(predicates::impl::PointClassification::Vert, 0, el1); + } + } + + if (m_mesh1) + { + m_pointClassifierDest = std::make_shared(m_mesh2ScatteredToMesh1); + } + } + + std::shared_ptr m_mesh1; + std::shared_ptr m_mesh1ScatteredToMesh2; + std::shared_ptr m_mesh2; + std::shared_ptr m_mesh2ScatteredToMesh1; + nonconformal4::impl::FakeVertGenerator m_vertGenerator; + std::shared_ptr m_meshRelationalDataInput; + std::shared_ptr m_pointClassifierOrigin; + std::shared_ptr m_pointClassifierDest; + std::shared_ptr m_meshRelationalDataScatter; + MPI_Comm m_comm1; + MPI_Comm m_comm2; + MPI_Comm m_unionComm; +}; + +class MeshRelationalDataScatterTester : public MeshRelationalDataScatterTesterBase +{ + protected: + + void create_meshes() override + { + mesh::impl::MeshSpec spec1, spec2; + spec1.xmin = 0; spec1.xmax = 1; + spec1.ymin = 0; spec1.ymax = 1; + spec1.numelX = 2; spec1.numelY = 2; + + spec2.xmin = 0; spec2.xmax = 1; + spec2.ymin = 0; spec2.ymax = 1; + spec2.numelX = 3; spec2.numelY = 3; + + auto f = [](const utils::Point& pt) { return pt; }; + if (m_comm1 != MPI_COMM_NULL) + { + m_mesh1 = create_mesh(spec1, f, m_comm1); + } + + if (m_comm2 != MPI_COMM_NULL) + { + m_mesh2 = create_mesh(spec2, f, m_comm2); + } + } + + std::shared_ptr create_scatter_spec1() override + { + int myrank = utils::impl::comm_rank(m_unionComm); + int commsize = utils::impl::comm_size(m_unionComm); + auto scatterSpec1 = std::make_shared(m_unionComm, m_mesh1); + if (m_mesh1) + { + if (commsize == 1) + { + for (auto el : m_mesh1->get_elements()) + if (el) + scatterSpec1->add_destination(el, 0); + } else if (commsize == 2) + { + for (auto el : m_mesh1->get_elements()) + if (el) + scatterSpec1->add_destination(el, 1); + } else + { + for (auto el : m_mesh1->get_elements()) + if (el) + { + scatterSpec1->add_destination(el, 2); + if (myrank == 1) + scatterSpec1->add_destination(el, 3); + } + } + } + + return scatterSpec1; + } + + std::shared_ptr create_scatter_spec2() override + { + int myrank = utils::impl::comm_rank(m_unionComm); + int commsize = utils::impl::comm_size(m_unionComm); + auto scatterSpec2 = std::make_shared(m_unionComm, m_mesh2); + if (m_mesh2) + { + if (commsize == 1) + { + for (auto el : m_mesh2->get_elements()) + if (el) + scatterSpec2->add_destination(el, 0); + } else if (commsize == 2) + { + for (auto el : m_mesh2->get_elements()) + if (el) + scatterSpec2->add_destination(el, 0); + } else + { + for (auto el : m_mesh2->get_elements()) + if (el) + { + scatterSpec2->add_destination(el, myrank - 2); + if (std::abs(mesh::compute_centroid(el).x - 0.5) < 1e-12) + scatterSpec2->add_destination(el, 1); + } + } + } + + return scatterSpec2; + } +}; + +mesh::MeshEntityPtr find_entity(std::shared_ptr mesh, int dim, const utils::Point& centroid, double tol) +{ + double minDist = std::numeric_limits::max(); + mesh::MeshEntityPtr minEntity = nullptr; + + for (auto& entity : mesh->get_mesh_entities(dim)) + if (entity) + { + auto disp = mesh::compute_centroid(entity) - centroid; + double dist = std::sqrt(dot(disp, disp)); + if (dist < tol && dist < minDist) + { + minDist = dist; + minEntity = entity; + } + } + + return minEntity; +} + +} // namespace + + +TEST_F(MeshRelationalDataScatterTester, Mesh1Verts) +{ + if (m_mesh1ScatteredToMesh2) + { + auto& verts1ToFakeVerts = *(m_meshRelationalDataInput->verts1ToFakeVerts); + + for (auto& vert : m_mesh1ScatteredToMesh2->get_vertices()) + if (vert) + verts1ToFakeVerts(vert, 0, 0) = m_vertGenerator.get_vert(vert->get_point_orig(0)); + } + + std::shared_ptr meshRelationalDataOutput = m_meshRelationalDataScatter->scatter(); + + if (m_mesh1) + { + std::set fakeVertIds; + auto& verts1ToFakeVerts = *(meshRelationalDataOutput->verts1ToFakeVerts); + int minFakeVertId = std::numeric_limits::max(); + int maxFakeVertId = std::numeric_limits::min(); + int nverts = 0; + for (auto& vert : m_mesh1->get_vertices()) + if (vert) + { + nonconformal4::impl::FakeVert fv = verts1ToFakeVerts(vert, 0, 0); + utils::Point pt = vert->get_point_orig(0); + + EXPECT_EQ(fakeVertIds.count(fv.id), 0u); + fakeVertIds.insert(fv.id); + + double dist = std::sqrt(dot(fv.pt - pt, fv.pt - pt)); + EXPECT_NEAR(dist, 0, 1e-13); + + minFakeVertId = std::min(minFakeVertId, fv.id); + maxFakeVertId = std::max(maxFakeVertId, fv.id); + nverts++; + } + + EXPECT_EQ(fakeVertIds.size(), size_t(mesh::count_valid(m_mesh1->get_vertices()))); + EXPECT_EQ(maxFakeVertId - minFakeVertId + 1, nverts); + } +} + + +TEST_F(MeshRelationalDataScatterTester, Mesh2Verts) +{ + if (m_mesh2) + { + auto& verts2ToFakeVerts = *(m_meshRelationalDataInput->verts2ToFakeVerts); + auto& verts2ClassOnMesh1 = *(m_meshRelationalDataInput->verts2ClassOnMesh1); + + for (auto& vert : m_mesh2->get_vertices()) + if (vert) + { + verts2ToFakeVerts(vert, 0, 0) = m_vertGenerator.get_vert(vert->get_point_orig(0)); + + for (auto& el : m_mesh1ScatteredToMesh2->get_elements()) + if (el) + { + mesh::MeshEntityPtr el2 = vert->get_up(0)->get_up(0); + predicates::impl::PointRecord record = m_pointClassifierOrigin->classify(el, el2, vert->get_point_orig(0)); + if (record.type != predicates::impl::PointClassification::Exterior) + { + verts2ClassOnMesh1(vert, 0, 0) = record; + } + } + } + } + + std::shared_ptr meshRelationalDataOutput = m_meshRelationalDataScatter->scatter(); + + if (m_mesh2ScatteredToMesh1) + { + std::set fakeVertIds; + auto& verts2ToFakeVerts = *(meshRelationalDataOutput->verts2ToFakeVerts); + auto& verts2ClassOnMesh1 = *(meshRelationalDataOutput->verts2ClassOnMesh1); + + int minFakeVertId = std::numeric_limits::max(); + int maxFakeVertId = std::numeric_limits::min(); + int nverts = 0; + for (auto& vert : m_mesh2ScatteredToMesh1->get_vertices()) + { + if (vert) + { + nonconformal4::impl::FakeVert fv = verts2ToFakeVerts(vert, 0, 0); + utils::Point pt = vert->get_point_orig(0); + + EXPECT_EQ(fakeVertIds.count(fv.id), 0u); + fakeVertIds.insert(fv.id); + + double dist = std::sqrt(dot(fv.pt - pt, fv.pt - pt)); + EXPECT_NEAR(dist, 0, 1e-13); + + minFakeVertId = std::min(minFakeVertId, fv.id); + maxFakeVertId = std::max(maxFakeVertId, fv.id); + nverts++; + + predicates::impl::PointRecord& record = verts2ClassOnMesh1(vert, 0, 0); + if (record.el) + { + mesh::MeshEntityPtr el2 = vert->get_up(0)->get_up(0); + predicates::impl::PointRecord recordLocal = m_pointClassifierDest->classify(record.el, el2, vert->get_point_orig(0)); + + EXPECT_EQ(record.type, recordLocal.type); + EXPECT_EQ(record.el, recordLocal.el); + EXPECT_EQ(record.id, recordLocal.id); + EXPECT_EQ(m_pointClassifierDest->compute_xyz_coords(record), m_pointClassifierDest->compute_xyz_coords(recordLocal)); + } + + } + } + + EXPECT_EQ(fakeVertIds.size(), size_t(mesh::count_valid(m_mesh2ScatteredToMesh1->get_vertices()))); + EXPECT_EQ(maxFakeVertId - minFakeVertId + 1, nverts); + + size_t numFakeVertsExpected = mesh::count_valid(m_mesh2ScatteredToMesh1->get_vertices()) + + mesh::count_valid(m_mesh1->get_vertices()); + EXPECT_EQ(meshRelationalDataOutput->fakeVertsToVertsIn.size(), numFakeVertsExpected); + } +} + +TEST_F(MeshRelationalDataScatterTester, Mesh2ClassificationNearBoundary) +{ + if (m_mesh2) + { + auto& verts2ClassOnMesh1 = *(m_meshRelationalDataInput->verts2ClassOnMesh1); + + auto vert2 = find_entity(m_mesh2, 0, {2.0/3.0, 2.0/3.0, 0}, 1e-13); + if (vert2) + { + auto el1 = find_entity(m_mesh1ScatteredToMesh2, 2, {0.75, 0.25}, 1e-13); + verts2ClassOnMesh1(vert2, 0, 0) = predicates::impl::PointRecord(predicates::impl::PointClassification::Vert, 0, el1); + } + } + + std::shared_ptr meshRelationalDataOutput = m_meshRelationalDataScatter->scatter(); + + if (m_mesh2ScatteredToMesh1) + { + auto& verts2ClassOnMesh1 = *(meshRelationalDataOutput->verts2ClassOnMesh1); + auto vert2 = find_entity(m_mesh2ScatteredToMesh1, 0, {2.0/3.0, 2.0/3.0, 0}, 1e-13); + predicates::impl::PointRecord record = verts2ClassOnMesh1(vert2, 0, 0); + EXPECT_NE(record.el, nullptr); + } +} + +TEST_F(MeshRelationalDataScatterTester, Mesh1VertOnMesh2Vert) +{ + if (m_mesh2) + { + auto& verts1ToFakeVerts = *(m_meshRelationalDataInput->verts1ToFakeVerts); + auto& verts2ToFakeVerts = *(m_meshRelationalDataInput->verts2ToFakeVerts); + + for (auto& vert : m_mesh1ScatteredToMesh2->get_vertices()) + if (vert) + { + verts1ToFakeVerts(vert, 0, 0) = m_vertGenerator.get_vert(vert->get_point_orig(0)); + } + + for (auto& vert : m_mesh2->get_vertices()) + if (vert) + { + if (utils::impl::comm_rank(m_mesh2->get_comm()) == 0 && vert->get_id() == 0) + { + verts2ToFakeVerts(vert, 0, 0) = verts1ToFakeVerts(m_mesh1ScatteredToMesh2->get_vertices()[0], 0, 0); + } else + { + verts2ToFakeVerts(vert, 0, 0) = m_vertGenerator.get_vert(vert->get_point_orig(0)); + } + } + } + + std::shared_ptr meshRelationalDataOutput = m_meshRelationalDataScatter->scatter(); + + if (m_mesh2ScatteredToMesh1) + { + std::set fakeVertIds; + auto& verts1ToFakeVerts = *(meshRelationalDataOutput->verts1ToFakeVerts); + auto& verts2ToFakeVerts = *(meshRelationalDataOutput->verts2ToFakeVerts); + for (auto& vert : m_mesh1->get_vertices()) + if (vert) + { + nonconformal4::impl::FakeVert fv = verts1ToFakeVerts(vert, 0, 0); + utils::Point pt = vert->get_point_orig(0); + fakeVertIds.insert(fv.id); + + double dist = std::sqrt(dot(fv.pt - pt, fv.pt - pt)); + EXPECT_NEAR(dist, 0, 1e-13); + } + + for (auto& vert : m_mesh2ScatteredToMesh1->get_vertices()) + if (vert) + { + nonconformal4::impl::FakeVert fv = verts2ToFakeVerts(vert, 0, 0); + utils::Point pt = vert->get_point_orig(0); + + fakeVertIds.insert(fv.id); + + double dist = std::sqrt(dot(fv.pt - pt, fv.pt - pt)); + EXPECT_NEAR(dist, 0, 1e-13); + } + + int numUniqueIds = fakeVertIds.size(); + int numUniqueIdsExpected = mesh::count_valid(m_mesh1->get_vertices()) + + mesh::count_valid(m_mesh2ScatteredToMesh1->get_vertices()); + + if (utils::impl::comm_rank(m_mesh1->get_comm()) == 0) + --numUniqueIdsExpected; + + numUniqueIds = stk::get_global_sum(m_mesh1->get_comm(), numUniqueIds); + numUniqueIdsExpected = stk::get_global_sum(m_mesh1->get_comm(), numUniqueIdsExpected); + EXPECT_EQ(numUniqueIds, numUniqueIdsExpected); + } +} + + +TEST_F(MeshRelationalDataScatterTester, Mesh2VertOnMesh1Vert) +{ + if (m_mesh2) + { + auto& verts1ToFakeVerts = *(m_meshRelationalDataInput->verts1ToFakeVerts); + auto& verts2ToFakeVerts = *(m_meshRelationalDataInput->verts2ToFakeVerts); + + for (auto& vert : m_mesh2->get_vertices()) + if (vert) + { + verts2ToFakeVerts(vert, 0, 0) = m_vertGenerator.get_vert(vert->get_point_orig(0)); + } + + for (auto& vert : m_mesh1ScatteredToMesh2->get_vertices()) + if (vert) + { + if (utils::impl::comm_rank(m_mesh1ScatteredToMesh2->get_comm()) == 0 && vert->get_id() == 0) + { + verts1ToFakeVerts(vert, 0, 0) = verts2ToFakeVerts(m_mesh2->get_vertices()[0], 0, 0); + } else + { + verts1ToFakeVerts(vert, 0, 0) = m_vertGenerator.get_vert(vert->get_point_orig(0)); + } + } + } + + std::shared_ptr meshRelationalDataOutput = m_meshRelationalDataScatter->scatter(); + + if (m_mesh2ScatteredToMesh1) + { + std::set fakeVertIds; + auto& verts1ToFakeVerts = *(meshRelationalDataOutput->verts1ToFakeVerts); + auto& verts2ToFakeVerts = *(meshRelationalDataOutput->verts2ToFakeVerts); + for (auto& vert : m_mesh1->get_vertices()) + if (vert) + { + nonconformal4::impl::FakeVert fv = verts1ToFakeVerts(vert, 0, 0); + utils::Point pt = vert->get_point_orig(0); + fakeVertIds.insert(fv.id); + + double dist = std::sqrt(dot(fv.pt - pt, fv.pt - pt)); + EXPECT_NEAR(dist, 0, 1e-13); + } + + for (auto& vert : m_mesh2ScatteredToMesh1->get_vertices()) + if (vert) + { + nonconformal4::impl::FakeVert fv = verts2ToFakeVerts(vert, 0, 0); + utils::Point pt = vert->get_point_orig(0); + + fakeVertIds.insert(fv.id); + + double dist = std::sqrt(dot(fv.pt - pt, fv.pt - pt)); + EXPECT_NEAR(dist, 0, 1e-13); + } + + int numUniqueIds = fakeVertIds.size(); + int numUniqueIdsExpected = mesh::count_valid(m_mesh1->get_vertices()) + + mesh::count_valid(m_mesh2ScatteredToMesh1->get_vertices()); + + if (utils::impl::comm_rank(m_mesh1->get_comm()) == 0) + --numUniqueIdsExpected; + + numUniqueIds = stk::get_global_sum(m_mesh1->get_comm(), numUniqueIds); + numUniqueIdsExpected = stk::get_global_sum(m_mesh1->get_comm(), numUniqueIdsExpected); + EXPECT_EQ(numUniqueIds, numUniqueIdsExpected); + } +} + + +TEST_F(MeshRelationalDataScatterTester, mesh1EdgeFields) +{ + utils::Point pt(0, 1.0/3, 0); + double xi = 2.0/3.0; + if (m_mesh1ScatteredToMesh2) + { + auto& mesh1EdgesToSplit = *(m_meshRelationalDataInput->mesh1EdgesToSplit); + mesh::MeshEntityPtr edge1 = find_entity(m_mesh1ScatteredToMesh2, 1, {0.0, 0.25, 0}, 1e-13); + mesh::MeshEntityPtr edge2 = find_entity(m_mesh2, 1, {0, 1.0/6, 0}, 1e-12); + + if (edge1) + { + nonconformal4::impl::FakeVert fv = m_vertGenerator.get_vert(pt); + nonconformal4::impl::EdgeSplitRecord edgeSplit{fv, xi, edge2}; + mesh1EdgesToSplit.insert(edge1, 0, edgeSplit); + } + } + + auto meshRelationalDataOutput = m_meshRelationalDataScatter->scatter(); + + if (m_mesh1) + { + auto& mesh1EdgesToSplit = *(meshRelationalDataOutput->mesh1EdgesToSplit); + mesh::MeshEntityPtr edge1 = find_entity(m_mesh1, 1, {0, 0.25, 0}, 1e-13); + mesh::MeshEntityPtr edge2 = find_entity(m_mesh2ScatteredToMesh1, 1, {0, 1.0/6, 0}, 1e-12); + + if (edge1) + { + EXPECT_EQ(mesh1EdgesToSplit.get_num_comp(edge1, 0), 1); + nonconformal4::impl::EdgeSplitRecord edgeSplit = mesh1EdgesToSplit(edge1, 0, 0); + + double dist = std::sqrt(dot(edgeSplit.vert.pt - pt, edgeSplit.vert.pt - pt)); + EXPECT_NEAR(dist, 0, 1e-12); + EXPECT_NEAR(edgeSplit.xi, xi, 1e-13); + EXPECT_EQ(edgeSplit.otherMeshEntity, edge2); + } + + } +} + +TEST_F(MeshRelationalDataScatterTester, mesh1EdgeFieldsSenderEntityShared) +{ + utils::Point pt(0, 1.0/3, 0); + double xi = 0.5; + if (m_mesh1ScatteredToMesh2) + { + auto& mesh1EdgesToSplit = *(m_meshRelationalDataInput->mesh1EdgesToSplit); + mesh::MeshEntityPtr edge1 = find_entity(m_mesh1ScatteredToMesh2, 1, {0.75, 0.5, 0}, 1e-13); + mesh::MeshEntityPtr edge2 = find_entity(m_mesh2, 1, {2.0/3.0, 0.5, 0}, 1e-13); + + if (edge1) + { + nonconformal4::impl::FakeVert fv = m_vertGenerator.get_vert(pt); + nonconformal4::impl::EdgeSplitRecord edgeSplit{fv, xi, edge2}; + mesh1EdgesToSplit.insert(edge1, 0, edgeSplit); + } + } + + auto meshRelationalDataOutput = m_meshRelationalDataScatter->scatter(); + + if (m_mesh1) + { + auto& mesh1EdgesToSplit = *(meshRelationalDataOutput->mesh1EdgesToSplit); + mesh::MeshEntityPtr edge1 = find_entity(m_mesh1, 1, {0.75, 0.5, 0}, 1e-13); + mesh::MeshEntityPtr edge2 = find_entity(m_mesh2ScatteredToMesh1, 1, {2.0/3.0, 0.5, 0}, 1e-13); + + if (edge1) + { + EXPECT_EQ(mesh1EdgesToSplit.get_num_comp(edge1, 0), 1); + nonconformal4::impl::EdgeSplitRecord edgeSplit = mesh1EdgesToSplit(edge1, 0, 0); + + double dist = std::sqrt(dot(edgeSplit.vert.pt - pt, edgeSplit.vert.pt - pt)); + EXPECT_NEAR(dist, 0, 1e-12); + EXPECT_NEAR(edgeSplit.xi, xi, 1e-13); + EXPECT_EQ(edgeSplit.otherMeshEntity, edge2); + } + + } +} + + +TEST_F(MeshRelationalDataScatterTester, mesh1EdgeFieldsWithRepeatedVert) +{ + utils::Point pt(1.0/3, 0, 0); + double xi = 2.0/3.0; + if (m_mesh1ScatteredToMesh2) + { + auto& mesh1EdgesToSplit = *(m_meshRelationalDataInput->mesh1EdgesToSplit); + mesh::MeshEntityPtr edge1 = find_entity(m_mesh1ScatteredToMesh2, 1, {0.25, 0, 0}, 1e-13); + mesh::MeshEntityPtr edge2 = find_entity(m_mesh2, 1, {1.0/6, 0, 0}, 1e-12); + auto& verts2ToFakeVerts = *(m_meshRelationalDataInput->verts2ToFakeVerts); + + if (edge1) + { + nonconformal4::impl::FakeVert fv = verts2ToFakeVerts(edge2->get_down(1), 0, 0); + nonconformal4::impl::EdgeSplitRecord edgeSplit{fv, xi, edge2}; + mesh1EdgesToSplit.insert(edge1, 0, edgeSplit); + } + } + + auto meshRelationalDataOutput = m_meshRelationalDataScatter->scatter(); + + if (m_mesh1) + { + auto& mesh1EdgesToSplit = *(meshRelationalDataOutput->mesh1EdgesToSplit); + mesh::MeshEntityPtr edge1 = find_entity(m_mesh1, 1, {0.25, 0, 0}, 1e-13); + mesh::MeshEntityPtr edge2 = find_entity(m_mesh2ScatteredToMesh1, 1, {1.0/6, 0, 0}, 1e-12); + auto& verts2ToFakeVerts = *(meshRelationalDataOutput->verts2ToFakeVerts); + + if (edge1) + { + EXPECT_EQ(mesh1EdgesToSplit.get_num_comp(edge1, 0), 1); + nonconformal4::impl::EdgeSplitRecord edgeSplit = mesh1EdgesToSplit(edge1, 0, 0); + + double dist = std::sqrt(dot(edgeSplit.vert.pt - pt, edgeSplit.vert.pt - pt)); + EXPECT_NEAR(dist, 0, 1e-12); + EXPECT_NEAR(edgeSplit.xi, xi, 1e-13); + EXPECT_EQ(edgeSplit.otherMeshEntity, edge2); + EXPECT_EQ(edgeSplit.vert.id, verts2ToFakeVerts(edge2->get_down(1), 0, 0).id); + } + + } +} + + +TEST_F(MeshRelationalDataScatterTester, mesh2EdgeFields) +{ + utils::Point pt1(1.0/3, 0, 0), pt2(1.0/3, 1.0/3, 0); + if (m_mesh2) + { + auto& edges2ToFakeVertsIn = *(m_meshRelationalDataInput->edges2ToFakeVertsIn); + mesh::MeshEntityPtr edge2 = find_entity(m_mesh2, 1, {1.0/3, 1.0/6, 0}, 1e-12); + + if (edge2) + { + nonconformal4::impl::FakeVert fv1 = m_vertGenerator.get_vert(edge2->get_down(0)->get_point_orig(0)); + nonconformal4::impl::FakeVert fv2 = m_vertGenerator.get_vert(edge2->get_down(1)->get_point_orig(0)); + edges2ToFakeVertsIn.insert(edge2, 0, {fv1, 0}); + edges2ToFakeVertsIn.insert(edge2, 0, {fv2, 1}); + } + } + + auto meshRelationalDataOutput = m_meshRelationalDataScatter->scatter(); + + if (m_mesh2ScatteredToMesh1) + { + auto& edges2ToFakeVertsIn = *(meshRelationalDataOutput->edges2ToFakeVertsIn); + mesh::MeshEntityPtr edge2 = find_entity(m_mesh2ScatteredToMesh1, 1, {1.0/3, 1.0/6, 0}, 1e-12); + + if (edge2) + { + EXPECT_EQ(edges2ToFakeVertsIn.get_num_comp(edge2, 0), 2); + nonconformal4::impl::VertOnEdge vert1 = edges2ToFakeVertsIn(edge2, 0, 0); + nonconformal4::impl::VertOnEdge vert2 = edges2ToFakeVertsIn(edge2, 0, 1); + + double dist1 = std::sqrt(dot(vert1.vert.pt - pt1, vert1.vert.pt - pt1)); + double dist2 = std::sqrt(dot(vert2.vert.pt - pt2, vert2.vert.pt - pt2)); + + EXPECT_NEAR(dist1, 0, 1e-12); + EXPECT_NEAR(dist2, 0, 1e-12); + EXPECT_NEAR(vert1.xi, 0, 1e-13); + EXPECT_NEAR(vert2.xi, 1, 1e-13); + } + } +} + +TEST_F(MeshRelationalDataScatterTester, mesh2EdgeFieldsSenderEntityShared) +{ + utils::Point pt1(2.0/3, 1.0/3, 0), pt2(2.0/3, 2.0/3, 0); + if (m_mesh2) + { + auto& edges2ToFakeVertsIn = *(m_meshRelationalDataInput->edges2ToFakeVertsIn); + mesh::MeshEntityPtr edge2 = find_entity(m_mesh2, 1, {2.0/3, 0.5, 0}, 1e-12); + + if (edge2) + { + nonconformal4::impl::FakeVert fv1 = m_vertGenerator.get_vert(edge2->get_down(0)->get_point_orig(0)); + nonconformal4::impl::FakeVert fv2 = m_vertGenerator.get_vert(edge2->get_down(1)->get_point_orig(0)); + edges2ToFakeVertsIn.insert(edge2, 0, {fv1, 0}); + edges2ToFakeVertsIn.insert(edge2, 0, {fv2, 1}); + } + } + + auto meshRelationalDataOutput = m_meshRelationalDataScatter->scatter(); + + if (m_mesh2ScatteredToMesh1) + { + auto& edges2ToFakeVertsIn = *(meshRelationalDataOutput->edges2ToFakeVertsIn); + mesh::MeshEntityPtr edge2 = find_entity(m_mesh2ScatteredToMesh1, 1, {2.0/3, 0.5, 0}, 1e-12); + + if (edge2) + { + EXPECT_EQ(edges2ToFakeVertsIn.get_num_comp(edge2, 0), 2); + nonconformal4::impl::VertOnEdge vert1 = edges2ToFakeVertsIn(edge2, 0, 0); + nonconformal4::impl::VertOnEdge vert2 = edges2ToFakeVertsIn(edge2, 0, 1); + + double dist1 = std::sqrt(dot(vert1.vert.pt - pt1, vert1.vert.pt - pt1)); + double dist2 = std::sqrt(dot(vert2.vert.pt - pt2, vert2.vert.pt - pt2)); + + EXPECT_NEAR(dist1, 0, 1e-12); + EXPECT_NEAR(dist2, 0, 1e-12); + EXPECT_NEAR(vert1.xi, 0, 1e-13); + EXPECT_NEAR(vert2.xi, 1, 1e-13); + } + } +} + +TEST_F(MeshRelationalDataScatterTester, mesh2EdgeFieldsWithRepeatedVert) +{ + utils::Point pt1(0, 0, 0), pt2(1.0/3, 0, 0); + if (m_mesh2) + { + auto& verts2ToFakeVerts = *(m_meshRelationalDataInput->verts2ToFakeVerts); + auto& edges2ToFakeVertsIn = *(m_meshRelationalDataInput->edges2ToFakeVertsIn); + mesh::MeshEntityPtr edge2 = find_entity(m_mesh2, 1, {1.0/6, 0, 0}, 1e-12); + + if (edge2) + { + nonconformal4::impl::FakeVert fv1 = verts2ToFakeVerts(edge2->get_down(0), 0, 0); + nonconformal4::impl::FakeVert fv2 = verts2ToFakeVerts(edge2->get_down(1), 0, 0); + edges2ToFakeVertsIn.insert(edge2, 0, {fv1, 0}); + edges2ToFakeVertsIn.insert(edge2, 0, {fv2, 1}); + } + } + + auto meshRelationalDataOutput = m_meshRelationalDataScatter->scatter(); + + if (m_mesh2ScatteredToMesh1) + { + auto& verts2ToFakeVerts = *(meshRelationalDataOutput->verts2ToFakeVerts); + auto& edges2ToFakeVertsIn = *(meshRelationalDataOutput->edges2ToFakeVertsIn); + mesh::MeshEntityPtr edge2 = find_entity(m_mesh2ScatteredToMesh1, 1, {1.0/6, 0, 0}, 1e-12); + + if (edge2) + { + EXPECT_EQ(edges2ToFakeVertsIn.get_num_comp(edge2, 0), 2); + nonconformal4::impl::VertOnEdge vert1 = edges2ToFakeVertsIn(edge2, 0, 0); + nonconformal4::impl::VertOnEdge vert2 = edges2ToFakeVertsIn(edge2, 0, 1); + + double dist1 = std::sqrt(dot(vert1.vert.pt - pt1, vert1.vert.pt - pt1)); + double dist2 = std::sqrt(dot(vert2.vert.pt - pt2, vert2.vert.pt - pt2)); + + EXPECT_NEAR(dist1, 0, 1e-12); + EXPECT_NEAR(dist2, 0, 1e-12); + EXPECT_NEAR(vert1.xi, 0, 1e-13); + EXPECT_NEAR(vert2.xi, 1, 1e-13); + EXPECT_EQ(vert1.vert.id, verts2ToFakeVerts(edge2->get_down(0), 0, 0).id); + EXPECT_EQ(vert2.vert.id, verts2ToFakeVerts(edge2->get_down(1), 0, 0).id); + + } + } +} \ No newline at end of file diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh_scatter.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh_scatter.cpp index fff73f113822..6ca915315c74 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh_scatter.cpp +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/test_mesh_scatter.cpp @@ -1,9 +1,9 @@ #include "gtest/gtest.h" #include "stk_middle_mesh/mesh.hpp" #include "stk_middle_mesh/mesh_entity.hpp" -#include "stk_middle_mesh/parallel_search.hpp" #include "stk_middle_mesh/field.hpp" #include "stk_middle_mesh/mesh_scatter.hpp" +#include "stk_middle_mesh/variable_size_field.hpp" using namespace stk::middle_mesh; @@ -35,6 +35,16 @@ mesh::MeshEntityPtr get_closest_entity(std::shared_ptr mesh, int dim return minEntity; } +mesh::RemoteSharedEntity get_remote(mesh::VariableSizeFieldPtr fieldPtr, mesh::MeshEntityPtr entity, int rank) +{ + auto& field = *fieldPtr; + for (int i=0; i < field.get_num_comp(entity, 0); ++i) + if (field(entity, 0, i).remoteRank == rank) + return field(entity, 0, i); + + throw std::runtime_error("unable to find remote"); +} + class RemoteChecker { @@ -159,11 +169,138 @@ TEST(MeshScatter, 2x2FromOneProc) scatterSpec->add_destination(el4, 4); } - mesh::impl::MeshScatter scatter(scatterSpec, mesh, color == 1 ? meshComm : MPI_COMM_NULL); + mesh::impl::MeshScatter scatter(scatterSpec, mesh, color == 1 ? meshComm : MPI_COMM_NULL, true); auto meshScattered = scatter.scatter(); auto elementOrigins = scatter.get_element_origins(); + auto entityOrigins = scatter.get_entity_origins(); + auto entityDests = scatter.get_entity_destinations(); + + if (color == 0) + { + EXPECT_EQ(entityOrigins, nullptr); + EXPECT_NE(entityDests, nullptr); + + auto v1 = get_closest_entity(mesh, 0, {0, 0, 0}); + auto v2 = get_closest_entity(mesh, 0, {0.5, 0, 0}); + auto v3 = get_closest_entity(mesh, 0, {1.0, 0, 0}); + auto v4 = get_closest_entity(mesh, 0, {0, 0.5, 0}); + auto v5 = get_closest_entity(mesh, 0, {0.5, 0.5, 0}); + auto v6 = get_closest_entity(mesh, 0, {1.0, 0.5, 0}); + auto v7 = get_closest_entity(mesh, 0, {0, 1.0, 0}); + auto v8 = get_closest_entity(mesh, 0, {0.5, 1.0, 0}); + auto v9 = get_closest_entity(mesh, 0, {1.0, 1.0, 0}); + + auto edge1 = get_closest_entity(mesh, 1, {0.25, 0, 0}); + auto edge2 = get_closest_entity(mesh, 1, {0.50, 0.25, 0}); + auto edge3 = get_closest_entity(mesh, 1, {0.25, 0.50, 0}); + auto edge4 = get_closest_entity(mesh, 1, {0.0, 0.25, 0}); + + auto edge5 = get_closest_entity(mesh, 1, {0.75, 0, 0}); + auto edge6 = get_closest_entity(mesh, 1, {1.0, 0.25, 0}); + auto edge7 = get_closest_entity(mesh, 1, {0.75, 0.50, 0}); + + auto edge8 = get_closest_entity(mesh, 1, {0.50, 0.75, 0}); + auto edge9 = get_closest_entity(mesh, 1, {0.25, 1.0, 0}); + auto edge10 = get_closest_entity(mesh, 1, {0, 0.75, 0}); + + auto edge11 = get_closest_entity(mesh, 1, {1.0, 0.75, 0}); + auto edge12 = get_closest_entity(mesh, 1, {0.75, 1.0, 0}); + + auto el1 = get_closest_entity(mesh, 2, {0.25, 0.25, 0}); + auto el2 = get_closest_entity(mesh, 2, {0.75, 0.25, 0}); + auto el3 = get_closest_entity(mesh, 2, {0.25, 0.75, 0}); + auto el4 = get_closest_entity(mesh, 2, {0.75, 0.75, 0}); + + + EXPECT_EQ(entityDests->get_num_comp(v1, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(v2, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(v3, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(v4, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(v5, 0), 4); + EXPECT_EQ(entityDests->get_num_comp(v6, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(v7, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(v8, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(v9, 0), 1); + + EXPECT_EQ(get_remote(entityDests, v1, 1), mesh::RemoteSharedEntity(1, 0)); + + EXPECT_EQ(get_remote(entityDests, v2, 1), mesh::RemoteSharedEntity(1, 1)); + EXPECT_EQ(get_remote(entityDests, v2, 2), mesh::RemoteSharedEntity(2, 0)); + + EXPECT_EQ(get_remote(entityDests, v3, 2), mesh::RemoteSharedEntity(2, 1)); + + EXPECT_EQ(get_remote(entityDests, v4, 1), mesh::RemoteSharedEntity(1, 2)); + EXPECT_EQ(get_remote(entityDests, v4, 3), mesh::RemoteSharedEntity(3, 0)); + + EXPECT_EQ(get_remote(entityDests, v5, 1), mesh::RemoteSharedEntity(1, 3)); + EXPECT_EQ(get_remote(entityDests, v5, 2), mesh::RemoteSharedEntity(2, 2)); + EXPECT_EQ(get_remote(entityDests, v5, 3), mesh::RemoteSharedEntity(3, 1)); + EXPECT_EQ(get_remote(entityDests, v5, 4), mesh::RemoteSharedEntity(4, 0)); + + EXPECT_EQ(get_remote(entityDests, v6, 2), mesh::RemoteSharedEntity(2, 3)); + EXPECT_EQ(get_remote(entityDests, v6, 4), mesh::RemoteSharedEntity(4, 1)); + + EXPECT_EQ(get_remote(entityDests, v7, 3), mesh::RemoteSharedEntity(3, 2)); + + EXPECT_EQ(get_remote(entityDests, v8, 3), mesh::RemoteSharedEntity(3, 3)); + EXPECT_EQ(get_remote(entityDests, v8, 4), mesh::RemoteSharedEntity(4, 2)); + + EXPECT_EQ(get_remote(entityDests, v9, 4), mesh::RemoteSharedEntity(4, 3)); + + EXPECT_EQ(entityDests->get_num_comp(edge1, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge2, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(edge3, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(edge4, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge5, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge6, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge7, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(edge8, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(edge9, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge10, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge11, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge12, 0), 1); + + + EXPECT_EQ(get_remote(entityDests, edge1, 1), mesh::RemoteSharedEntity(1, 0)); + + EXPECT_EQ(get_remote(entityDests, edge2, 1), mesh::RemoteSharedEntity(1, 1)); + EXPECT_EQ(get_remote(entityDests, edge2, 2), mesh::RemoteSharedEntity(2, 0)); + + EXPECT_EQ(get_remote(entityDests, edge3, 1), mesh::RemoteSharedEntity(1, 2)); + EXPECT_EQ(get_remote(entityDests, edge3, 3), mesh::RemoteSharedEntity(3, 0)); + + EXPECT_EQ(get_remote(entityDests, edge4, 1), mesh::RemoteSharedEntity(1, 3)); + + EXPECT_EQ(get_remote(entityDests, edge5, 2), mesh::RemoteSharedEntity(2, 1)); + + EXPECT_EQ(get_remote(entityDests, edge6, 2), mesh::RemoteSharedEntity(2, 2)); + + EXPECT_EQ(get_remote(entityDests, edge7, 2), mesh::RemoteSharedEntity(2, 3)); + EXPECT_EQ(get_remote(entityDests, edge7, 4), mesh::RemoteSharedEntity(4, 0)); + + EXPECT_EQ(get_remote(entityDests, edge8, 3), mesh::RemoteSharedEntity(3, 1)); + EXPECT_EQ(get_remote(entityDests, edge8, 4), mesh::RemoteSharedEntity(4, 1)); - if (color == 1) + EXPECT_EQ(get_remote(entityDests, edge9, 3), mesh::RemoteSharedEntity(3, 2)); + + EXPECT_EQ(get_remote(entityDests, edge10, 3), mesh::RemoteSharedEntity(3, 3)); + + EXPECT_EQ(get_remote(entityDests, edge11, 4), mesh::RemoteSharedEntity(4, 2)); + + EXPECT_EQ(get_remote(entityDests, edge12, 4), mesh::RemoteSharedEntity(4, 3)); + + EXPECT_EQ(entityDests->get_num_comp(el1, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(el2, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(el3, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(el4, 0), 1); + + EXPECT_EQ(get_remote(entityDests, el1, 1), mesh::RemoteSharedEntity(1, 0)); + EXPECT_EQ(get_remote(entityDests, el2, 2), mesh::RemoteSharedEntity(2, 0)); + EXPECT_EQ(get_remote(entityDests, el3, 3), mesh::RemoteSharedEntity(3, 0)); + EXPECT_EQ(get_remote(entityDests, el4, 4), mesh::RemoteSharedEntity(4, 0)); + + + } else if (color == 1) { EXPECT_EQ(mesh::count_valid(meshScattered->get_vertices()), 4); EXPECT_EQ(mesh::count_valid(meshScattered->get_edges()), 4); @@ -209,6 +346,29 @@ TEST(MeshScatter, 2x2FromOneProc) EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(1), 1)); EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(2), 2)); + EXPECT_EQ(entityOrigins->get_num_comp(verts[0], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[1], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[2], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[3], 0), 1); + + EXPECT_EQ(get_remote(entityOrigins, verts[0], 0), mesh::RemoteSharedEntity(0, 0)); + EXPECT_EQ(get_remote(entityOrigins, verts[1], 0), mesh::RemoteSharedEntity(0, 1)); + EXPECT_EQ(get_remote(entityOrigins, verts[2], 0), mesh::RemoteSharedEntity(0, 4)); + EXPECT_EQ(get_remote(entityOrigins, verts[3], 0), mesh::RemoteSharedEntity(0, 3)); + + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(0), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(1), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(2), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(3), 0), 1); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(0), 0), mesh::RemoteSharedEntity(0, 0)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(1), 0), mesh::RemoteSharedEntity(0, 1)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(2), 0), mesh::RemoteSharedEntity(0, 2)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(3), 0), mesh::RemoteSharedEntity(0, 3)); + + EXPECT_EQ(entityOrigins->get_num_comp(el, 0), 1); + EXPECT_EQ(get_remote(entityOrigins, el, 0), mesh::RemoteSharedEntity(0, 0)); + } else if (myRank == 1) { EXPECT_EQ(verts[0]->count_remote_shared_entities(), 1); @@ -229,6 +389,29 @@ TEST(MeshScatter, 2x2FromOneProc) EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(2), 3)); EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(3), 0)); + + EXPECT_EQ(entityOrigins->get_num_comp(verts[0], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[1], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[2], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[3], 0), 1); + + EXPECT_EQ(get_remote(entityOrigins, verts[0], 0), mesh::RemoteSharedEntity(0, 1)); + EXPECT_EQ(get_remote(entityOrigins, verts[1], 0), mesh::RemoteSharedEntity(0, 2)); + EXPECT_EQ(get_remote(entityOrigins, verts[2], 0), mesh::RemoteSharedEntity(0, 5)); + EXPECT_EQ(get_remote(entityOrigins, verts[3], 0), mesh::RemoteSharedEntity(0, 4)); + + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(0), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(1), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(2), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(3), 0), 1); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(0), 0), mesh::RemoteSharedEntity(0, 4)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(1), 0), mesh::RemoteSharedEntity(0, 5)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(2), 0), mesh::RemoteSharedEntity(0, 6)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(3), 0), mesh::RemoteSharedEntity(0, 1)); + + EXPECT_EQ(entityOrigins->get_num_comp(el, 0), 1); + EXPECT_EQ(get_remote(entityOrigins, el, 0), mesh::RemoteSharedEntity(0, 1)); } else if (myRank == 2) { EXPECT_EQ(verts[0]->count_remote_shared_entities(), 1); @@ -248,7 +431,30 @@ TEST(MeshScatter, 2x2FromOneProc) EXPECT_EQ(el->get_down(3)->count_remote_shared_entities(), 0); EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(0), 0)); - EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(1), 3)); + EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(1), 3)); + + EXPECT_EQ(entityOrigins->get_num_comp(verts[0], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[1], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[2], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[3], 0), 1); + + EXPECT_EQ(get_remote(entityOrigins, verts[0], 0), mesh::RemoteSharedEntity(0, 3)); + EXPECT_EQ(get_remote(entityOrigins, verts[1], 0), mesh::RemoteSharedEntity(0, 4)); + EXPECT_EQ(get_remote(entityOrigins, verts[2], 0), mesh::RemoteSharedEntity(0, 7)); + EXPECT_EQ(get_remote(entityOrigins, verts[3], 0), mesh::RemoteSharedEntity(0, 6)); + + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(0), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(1), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(2), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(3), 0), 1); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(0), 0), mesh::RemoteSharedEntity(0, 2)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(1), 0), mesh::RemoteSharedEntity(0, 7)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(2), 0), mesh::RemoteSharedEntity(0, 8)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(3), 0), mesh::RemoteSharedEntity(0, 9)); + + EXPECT_EQ(entityOrigins->get_num_comp(el, 0), 1); + EXPECT_EQ(get_remote(entityOrigins, el, 0), mesh::RemoteSharedEntity(0, 2)); } else if (myRank == 3) { EXPECT_EQ(verts[0]->count_remote_shared_entities(), 3); @@ -268,7 +474,30 @@ TEST(MeshScatter, 2x2FromOneProc) EXPECT_EQ(el->get_down(3)->count_remote_shared_entities(), 1); EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(0), 1)); - EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(3), 2)); + EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(3), 2)); + + EXPECT_EQ(entityOrigins->get_num_comp(verts[0], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[1], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[2], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[3], 0), 1); + + EXPECT_EQ(get_remote(entityOrigins, verts[0], 0), mesh::RemoteSharedEntity(0, 4)); + EXPECT_EQ(get_remote(entityOrigins, verts[1], 0), mesh::RemoteSharedEntity(0, 5)); + EXPECT_EQ(get_remote(entityOrigins, verts[2], 0), mesh::RemoteSharedEntity(0, 8)); + EXPECT_EQ(get_remote(entityOrigins, verts[3], 0), mesh::RemoteSharedEntity(0, 7)); + + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(0), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(1), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(2), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(3), 0), 1); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(0), 0), mesh::RemoteSharedEntity(0, 6)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(1), 0), mesh::RemoteSharedEntity(0, 10)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(2), 0), mesh::RemoteSharedEntity(0, 11)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(3), 0), mesh::RemoteSharedEntity(0, 7)); + + EXPECT_EQ(entityOrigins->get_num_comp(el, 0), 1); + EXPECT_EQ(get_remote(entityOrigins, el, 0), mesh::RemoteSharedEntity(0, 3)); } mesh::check_topology(meshScattered); @@ -294,12 +523,12 @@ TEST(MeshScatter, 2x2FromTwoProcs) MPI_Comm meshComm; int color = utils::impl::comm_rank(unionComm) <= 1 ? 0 : 1; MPI_Comm_split(unionComm, color, 0, &meshComm); + double xStart = utils::impl::comm_rank(meshComm) * 0.5; if (color == 0) { mesh = mesh::make_empty_mesh(meshComm); int myRank = utils::impl::comm_rank(meshComm); - double xStart = myRank * 0.5; auto v1 = mesh->create_vertex(xStart + 0, 0, 0); auto v2 = mesh->create_vertex(xStart + 0.5, 0, 0); auto v3 = mesh->create_vertex(xStart + 0, 0.5, 0); @@ -332,16 +561,164 @@ TEST(MeshScatter, 2x2FromTwoProcs) scatterSpec->add_destination(el2, 2*myRank + 3); } - mesh::impl::MeshScatter scatter(scatterSpec, mesh, color == 1 ? meshComm : MPI_COMM_NULL); + mesh::impl::MeshScatter scatter(scatterSpec, mesh, color == 1 ? meshComm : MPI_COMM_NULL, true); auto meshScattered = scatter.scatter(); auto elementOrigins = scatter.get_element_origins(); + auto entityOrigins = scatter.get_entity_origins(); + auto entityDests = scatter.get_entity_destinations(); + + if (color == 0) + { + EXPECT_EQ(entityOrigins, nullptr); + EXPECT_NE(entityDests, nullptr); + + auto v1 = get_closest_entity(mesh, 0, {xStart, 0, 0}); + auto v2 = get_closest_entity(mesh, 0, {xStart + 0.5, 0, 0}); + auto v3 = get_closest_entity(mesh, 0, {xStart, 0.5, 0}); + auto v4 = get_closest_entity(mesh, 0, {xStart + 0.5, 0.5, 0}); + auto v5 = get_closest_entity(mesh, 0, {xStart, 1.0, 0}); + auto v6 = get_closest_entity(mesh, 0, {xStart + 0.5, 1.0, 0}); + + double x1 = xStart, x2 = xStart + 0.5; + auto edge1 = get_closest_entity(mesh, 1, {(x1 + x2)/2, 0, 0}); + auto edge2 = get_closest_entity(mesh, 1, {x1 + 0.5, 0.25, 0}); + auto edge3 = get_closest_entity(mesh, 1, {(x1 + x2)/2, 0.5, 0}); + auto edge4 = get_closest_entity(mesh, 1, {x1, 0.25, 0}); + auto edge5 = get_closest_entity(mesh, 1, {x1 + 0.5, 0.75, 0}); + auto edge6 = get_closest_entity(mesh, 1, {(x1 + x2)/2, 1.0, 0}); + auto edge7 = get_closest_entity(mesh, 1, {x1, 0.75, 0}); + + auto el1 = get_closest_entity(mesh, 2, {(x1 + x2)/2, 0.25, 0}); + auto el2 = get_closest_entity(mesh, 2, {(x1 + x2)/2, 0.75, 0}); + + int myRank = utils::impl::comm_rank(meshComm); + + if (myRank == 0) + { + EXPECT_EQ(entityDests->get_num_comp(v1, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(v2, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(v3, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(v4, 0), 4); + EXPECT_EQ(entityDests->get_num_comp(v5, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(v6, 0), 2); + + EXPECT_EQ(entityDests->get_num_comp(edge1, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge2, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(edge3, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(edge4, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge5, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(edge6, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge7, 0), 1); + + EXPECT_EQ(entityDests->get_num_comp(el1, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(el2, 0), 1); + + EXPECT_EQ(get_remote(entityDests, v1, 2), mesh::RemoteSharedEntity(2, 0)); + + EXPECT_EQ(get_remote(entityDests, v2, 2), mesh::RemoteSharedEntity(2, 1)); + EXPECT_EQ(get_remote(entityDests, v2, 4), mesh::RemoteSharedEntity(4, 0)); + + EXPECT_EQ(get_remote(entityDests, v3, 2), mesh::RemoteSharedEntity(2, 2)); + EXPECT_EQ(get_remote(entityDests, v3, 3), mesh::RemoteSharedEntity(3, 0)); + + EXPECT_EQ(get_remote(entityDests, v4, 2), mesh::RemoteSharedEntity(2, 3)); + EXPECT_EQ(get_remote(entityDests, v4, 3), mesh::RemoteSharedEntity(3, 1)); + EXPECT_EQ(get_remote(entityDests, v4, 4), mesh::RemoteSharedEntity(4, 1)); + EXPECT_EQ(get_remote(entityDests, v4, 5), mesh::RemoteSharedEntity(5, 0)); + + EXPECT_EQ(get_remote(entityDests, v5, 3), mesh::RemoteSharedEntity(3, 2)); + + EXPECT_EQ(get_remote(entityDests, v6, 3), mesh::RemoteSharedEntity(3, 3)); + EXPECT_EQ(get_remote(entityDests, v6, 5), mesh::RemoteSharedEntity(5, 1)); + + EXPECT_EQ(get_remote(entityDests, edge1, 2), mesh::RemoteSharedEntity(2, 0)); + + EXPECT_EQ(get_remote(entityDests, edge2, 2), mesh::RemoteSharedEntity(2, 1)); + EXPECT_EQ(get_remote(entityDests, edge2, 4), mesh::RemoteSharedEntity(4, 0)); + + EXPECT_EQ(get_remote(entityDests, edge3, 2), mesh::RemoteSharedEntity(2, 2)); + EXPECT_EQ(get_remote(entityDests, edge3, 3), mesh::RemoteSharedEntity(3, 0)); + + EXPECT_EQ(get_remote(entityDests, edge4, 2), mesh::RemoteSharedEntity(2, 3)); + + EXPECT_EQ(get_remote(entityDests, edge5, 3), mesh::RemoteSharedEntity(3, 1)); + EXPECT_EQ(get_remote(entityDests, edge5, 5), mesh::RemoteSharedEntity(5, 0)); + + EXPECT_EQ(get_remote(entityDests, edge6, 3), mesh::RemoteSharedEntity(3, 2)); + + EXPECT_EQ(get_remote(entityDests, edge7, 3), mesh::RemoteSharedEntity(3, 3)); + + EXPECT_EQ(get_remote(entityDests, el1, 2), mesh::RemoteSharedEntity(2, 0)); + EXPECT_EQ(get_remote(entityDests, el2, 3), mesh::RemoteSharedEntity(3, 0)); + + + } else if (myRank == 1) + { + EXPECT_EQ(entityDests->get_num_comp(v1, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(v2, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(v3, 0), 4); + EXPECT_EQ(entityDests->get_num_comp(v4, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(v5, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(v6, 0), 1); + + EXPECT_EQ(entityDests->get_num_comp(edge1, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge2, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge3, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(edge4, 0), 2); + EXPECT_EQ(entityDests->get_num_comp(edge5, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge6, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(edge7, 0), 2); - if (color == 1) + EXPECT_EQ(entityDests->get_num_comp(el1, 0), 1); + EXPECT_EQ(entityDests->get_num_comp(el2, 0), 1); + + EXPECT_EQ(get_remote(entityDests, v1, 2), mesh::RemoteSharedEntity(2, 1)); + EXPECT_EQ(get_remote(entityDests, v1, 4), mesh::RemoteSharedEntity(4, 0)); + + EXPECT_EQ(get_remote(entityDests, v2, 4), mesh::RemoteSharedEntity(4, 2)); + + EXPECT_EQ(get_remote(entityDests, v3, 2), mesh::RemoteSharedEntity(2, 3)); + EXPECT_EQ(get_remote(entityDests, v3, 3), mesh::RemoteSharedEntity(3, 1)); + EXPECT_EQ(get_remote(entityDests, v3, 4), mesh::RemoteSharedEntity(4, 1)); + EXPECT_EQ(get_remote(entityDests, v3, 5), mesh::RemoteSharedEntity(5, 0)); + + EXPECT_EQ(get_remote(entityDests, v4, 4), mesh::RemoteSharedEntity(4, 3)); + EXPECT_EQ(get_remote(entityDests, v4, 5), mesh::RemoteSharedEntity(5, 2)); + + EXPECT_EQ(get_remote(entityDests, v5, 3), mesh::RemoteSharedEntity(3, 3)); + EXPECT_EQ(get_remote(entityDests, v5, 5), mesh::RemoteSharedEntity(5, 1)); + + EXPECT_EQ(get_remote(entityDests, v6, 5), mesh::RemoteSharedEntity(5, 3)); + + EXPECT_EQ(get_remote(entityDests, edge1, 4), mesh::RemoteSharedEntity(4, 1)); + + EXPECT_EQ(get_remote(entityDests, edge2, 4), mesh::RemoteSharedEntity(4, 2)); + + EXPECT_EQ(get_remote(entityDests, edge3, 4), mesh::RemoteSharedEntity(4, 3)); + EXPECT_EQ(get_remote(entityDests, edge3, 5), mesh::RemoteSharedEntity(5, 1)); + + EXPECT_EQ(get_remote(entityDests, edge4, 2), mesh::RemoteSharedEntity(2, 1)); + EXPECT_EQ(get_remote(entityDests, edge4, 4), mesh::RemoteSharedEntity(4, 0)); + + EXPECT_EQ(get_remote(entityDests, edge5, 5), mesh::RemoteSharedEntity(5, 2)); + + EXPECT_EQ(get_remote(entityDests, edge6, 5), mesh::RemoteSharedEntity(5, 3)); + + EXPECT_EQ(get_remote(entityDests, edge7, 3), mesh::RemoteSharedEntity(3, 1)); + EXPECT_EQ(get_remote(entityDests, edge7, 5), mesh::RemoteSharedEntity(5, 0)); + + EXPECT_EQ(get_remote(entityDests, el1, 4), mesh::RemoteSharedEntity(4, 0)); + EXPECT_EQ(get_remote(entityDests, el2, 5), mesh::RemoteSharedEntity(5, 0)); + } + } else if (color == 1) { EXPECT_EQ(mesh::count_valid(meshScattered->get_vertices()), 4); EXPECT_EQ(mesh::count_valid(meshScattered->get_edges()), 4); EXPECT_EQ(mesh::count_valid(meshScattered->get_elements()), 1); + EXPECT_NE(entityOrigins, nullptr); + EXPECT_EQ(entityDests, nullptr); + int myRank = utils::impl::comm_rank(meshComm); utils::Point lowerLeftCorner; switch (myRank) @@ -382,6 +759,38 @@ TEST(MeshScatter, 2x2FromTwoProcs) EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(1), 2)); EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(2), 1)); + EXPECT_EQ(entityOrigins->get_num_comp(verts[0], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[1], 0), 2); + EXPECT_EQ(entityOrigins->get_num_comp(verts[2], 0), 2); + EXPECT_EQ(entityOrigins->get_num_comp(verts[3], 0), 1); + + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(0), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(1), 0), 2); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(2), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(3), 0), 1); + + EXPECT_EQ(entityOrigins->get_num_comp(el, 0), 1); + + EXPECT_EQ(get_remote(entityOrigins, verts[0], 0), mesh::RemoteSharedEntity(0, 0)); + + EXPECT_EQ(get_remote(entityOrigins, verts[1], 0), mesh::RemoteSharedEntity(0, 1)); + EXPECT_EQ(get_remote(entityOrigins, verts[1], 1), mesh::RemoteSharedEntity(1, 0)); + + EXPECT_EQ(get_remote(entityOrigins, verts[2], 0), mesh::RemoteSharedEntity(0, 3)); + EXPECT_EQ(get_remote(entityOrigins, verts[2], 1), mesh::RemoteSharedEntity(1, 2)); + + EXPECT_EQ(get_remote(entityOrigins, verts[3], 0), mesh::RemoteSharedEntity(0, 2)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(0), 0), mesh::RemoteSharedEntity(0, 0)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(1), 0), mesh::RemoteSharedEntity(0, 1)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(1), 1), mesh::RemoteSharedEntity(1, 3)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(2), 0), mesh::RemoteSharedEntity(0, 2)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(3), 0), mesh::RemoteSharedEntity(0, 3)); + + EXPECT_EQ(get_remote(entityOrigins, el, 0), mesh::RemoteSharedEntity(0, 0)); } else if (myRank == 1) { EXPECT_EQ(verts[0]->count_remote_shared_entities(), 1); @@ -402,6 +811,39 @@ TEST(MeshScatter, 2x2FromTwoProcs) EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(0), 0)); EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(1), 3)); + + EXPECT_EQ(entityOrigins->get_num_comp(verts[0], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[1], 0), 2); + EXPECT_EQ(entityOrigins->get_num_comp(verts[2], 0), 2); + EXPECT_EQ(entityOrigins->get_num_comp(verts[3], 0), 1); + + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(0), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(1), 0), 2); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(2), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(3), 0), 1); + + EXPECT_EQ(entityOrigins->get_num_comp(el, 0), 1); + + EXPECT_EQ(get_remote(entityOrigins, verts[0], 0), mesh::RemoteSharedEntity(0, 2)); + + EXPECT_EQ(get_remote(entityOrigins, verts[1], 0), mesh::RemoteSharedEntity(0, 3)); + EXPECT_EQ(get_remote(entityOrigins, verts[1], 1), mesh::RemoteSharedEntity(1, 2)); + + EXPECT_EQ(get_remote(entityOrigins, verts[2], 0), mesh::RemoteSharedEntity(0, 5)); + EXPECT_EQ(get_remote(entityOrigins, verts[2], 1), mesh::RemoteSharedEntity(1, 4)); + + EXPECT_EQ(get_remote(entityOrigins, verts[3], 0), mesh::RemoteSharedEntity(0, 4)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(0), 0), mesh::RemoteSharedEntity(0, 2)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(1), 0), mesh::RemoteSharedEntity(0, 4)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(1), 1), mesh::RemoteSharedEntity(1, 6)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(2), 0), mesh::RemoteSharedEntity(0, 5)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(3), 0), mesh::RemoteSharedEntity(0, 6)); + + EXPECT_EQ(get_remote(entityOrigins, el, 0), mesh::RemoteSharedEntity(0, 1)); } else if (myRank == 2) { EXPECT_EQ(verts[0]->count_remote_shared_entities(), 1); @@ -421,7 +863,40 @@ TEST(MeshScatter, 2x2FromTwoProcs) EXPECT_EQ(el->get_down(3)->count_remote_shared_entities(), 1); EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(2), 3)); - EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(3), 0)); + EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(3), 0)); + + EXPECT_EQ(entityOrigins->get_num_comp(verts[0], 0), 2); + EXPECT_EQ(entityOrigins->get_num_comp(verts[1], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[2], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[3], 0), 2); + + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(0), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(1), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(2), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(3), 0), 2); + + EXPECT_EQ(entityOrigins->get_num_comp(el, 0), 1); + + EXPECT_EQ(get_remote(entityOrigins, verts[0], 0), mesh::RemoteSharedEntity(0, 1)); + EXPECT_EQ(get_remote(entityOrigins, verts[0], 1), mesh::RemoteSharedEntity(1, 0)); + + EXPECT_EQ(get_remote(entityOrigins, verts[1], 1), mesh::RemoteSharedEntity(1, 1)); + + EXPECT_EQ(get_remote(entityOrigins, verts[2], 1), mesh::RemoteSharedEntity(1, 3)); + + EXPECT_EQ(get_remote(entityOrigins, verts[3], 0), mesh::RemoteSharedEntity(0, 3)); + EXPECT_EQ(get_remote(entityOrigins, verts[3], 1), mesh::RemoteSharedEntity(1, 2)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(0), 1), mesh::RemoteSharedEntity(1, 0)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(1), 1), mesh::RemoteSharedEntity(1, 1)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(2), 1), mesh::RemoteSharedEntity(1, 2)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(3), 0), mesh::RemoteSharedEntity(0, 1)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(3), 1), mesh::RemoteSharedEntity(1, 3)); + + EXPECT_EQ(get_remote(entityOrigins, el, 1), mesh::RemoteSharedEntity(1, 0)); } else if (myRank == 3) { EXPECT_EQ(verts[0]->count_remote_shared_entities(), 3); @@ -441,7 +916,40 @@ TEST(MeshScatter, 2x2FromTwoProcs) EXPECT_EQ(el->get_down(3)->count_remote_shared_entities(), 1); EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(0), 2)); - EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(3), 1)); + EXPECT_NO_THROW(mesh::get_remote_shared_entity(el->get_down(3), 1)); + + EXPECT_EQ(entityOrigins->get_num_comp(verts[0], 0), 2); + EXPECT_EQ(entityOrigins->get_num_comp(verts[1], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[2], 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(verts[3], 0), 2); + + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(0), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(1), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(2), 0), 1); + EXPECT_EQ(entityOrigins->get_num_comp(el->get_down(3), 0), 2); + + EXPECT_EQ(entityOrigins->get_num_comp(el, 0), 1); + + EXPECT_EQ(get_remote(entityOrigins, verts[0], 0), mesh::RemoteSharedEntity(0, 3)); + EXPECT_EQ(get_remote(entityOrigins, verts[0], 1), mesh::RemoteSharedEntity(1, 2)); + + EXPECT_EQ(get_remote(entityOrigins, verts[1], 1), mesh::RemoteSharedEntity(1, 3)); + + EXPECT_EQ(get_remote(entityOrigins, verts[2], 1), mesh::RemoteSharedEntity(1, 5)); + + EXPECT_EQ(get_remote(entityOrigins, verts[3], 0), mesh::RemoteSharedEntity(0, 5)); + EXPECT_EQ(get_remote(entityOrigins, verts[3], 1), mesh::RemoteSharedEntity(1, 4)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(0), 1), mesh::RemoteSharedEntity(1, 2)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(1), 1), mesh::RemoteSharedEntity(1, 4)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(2), 1), mesh::RemoteSharedEntity(1, 5)); + + EXPECT_EQ(get_remote(entityOrigins, el->get_down(3), 0), mesh::RemoteSharedEntity(0, 4)); + EXPECT_EQ(get_remote(entityOrigins, el->get_down(3), 1), mesh::RemoteSharedEntity(1, 6)); + + EXPECT_EQ(get_remote(entityOrigins, el, 1), mesh::RemoteSharedEntity(1, 1)); } check_remotes_centroids(meshScattered); @@ -904,4 +1412,4 @@ TEST(MeshScatter, ThreeDestinations) } -// test when a single edge is on 2 or more procs \ No newline at end of file +// test when a single edge is on 2 or more procs diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_constraint_generator.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_constraint_generator.cpp index c8bc521383d7..8295a6131b48 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_constraint_generator.cpp +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_constraint_generator.cpp @@ -37,7 +37,7 @@ class MiddleGridConstraintGeneratorTester : public ::testing::Test mMesh2, PointClassifierNormalWrapperTolerances(eps)); mRelationalData = std::make_shared(mMesh1, mMesh2, mMeshIn); mProjector = std::make_shared( - mMesh1, mMesh2, mMeshIn, mRelationalData, mClassifier, middle_mesh::impl::EdgeTracerTolerances(eps)); + mMesh1, mMesh2, mRelationalData, mClassifier, middle_mesh::impl::EdgeTracerTolerances(eps)); mConstraintGenerator = std::make_shared( mMesh1, mMesh2, mMeshIn, mRelationalData, mClassifier); mProjector->project(); diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_constraint_generator_parallel.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_constraint_generator_parallel.cpp new file mode 100644 index 000000000000..d7018aa6a856 --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_constraint_generator_parallel.cpp @@ -0,0 +1,821 @@ +#include "stk_middle_mesh/mesh_relational_data.hpp" +#include "stk_middle_mesh/edge_tracer_opts.hpp" +#include "gtest/gtest.h" +#include "stk_middle_mesh/mesh_relational_data_scatter.hpp" +#include "stk_middle_mesh/predicates/point_classifier_normal_wrapper.hpp" +#include "stk_middle_mesh/mesh_scatter.hpp" +#include "stk_middle_mesh/mesh_scatter_spec.hpp" +#include "stk_middle_mesh/bounding_box_search.hpp" +#include "stk_middle_mesh/middle_grid_constraint_generator.hpp" +#include "stk_middle_mesh/mesh.hpp" +#include "stk_middle_mesh/mesh_projection_calculator.hpp" +#include "stk_middle_mesh/create_mesh.hpp" +#include "util/parallel_search_test_util.hpp" +#include "util/nonconformal_interface_helpers.hpp" + +namespace { + +using namespace stk::middle_mesh; + +class MiddleGridConstraintGeneratorParallelTester : public ::testing::Test +{ + protected: + + using BoundingBoxSearch = search::ElementToElementBoundingBoxSearch; + + void setup_spmd(const mesh::impl::MeshSpec& spec1, const mesh::impl::MeshSpec& spec2) + { + auto f = [](const utils::Point& pt) { return pt; }; + + auto mesh1 = mesh::impl::create_mesh(spec1, f); + auto mesh2 = mesh::impl::create_mesh(spec1, f); + + setup(mesh1, mesh2); + } + + void setup_mpmd(const mesh::impl::MeshSpec& spec1, const mesh::impl::MeshSpec& spec2, + const SplitCommTestUtil& splitter) + { + auto f = [](const utils::Point& pt) { return pt; }; + + std::shared_ptr mesh1, mesh2; + + if (splitter.get_color() == SplitCommColor::SEND) + { + mesh1 = mesh::impl::create_mesh(spec1, f, splitter.get_comm()); + } + + if (splitter.get_color() == SplitCommColor::RECV) + { + mesh2 = mesh::impl::create_mesh(spec2, f, splitter.get_comm()); + } + + setup(mesh1, mesh2); + } + + void setup(std::shared_ptr mesh1, std::shared_ptr mesh2) + { + m_mesh1 = mesh1; + m_mesh2 = mesh2; + + auto scatterSpec = create_scatter_spec(mesh1, mesh2); + scatter_mesh_1to2(scatterSpec); + if (m_mesh2) + { + m_meshRelationalDataOnMesh2 = std::make_shared(m_mesh1ScatteredToMesh2, m_mesh2); + } + + do_mesh_projections(); + + scatterSpec = create_scatter_spec(mesh2, mesh1); + scatter_mesh_2to1(scatterSpec); + scatter_mesh_relational_data_2to1(); + + create_middle_mesh_verts_and_edges(); + } + + + std::shared_ptr create_scatter_spec( + std::shared_ptr sendMesh, std::shared_ptr recvMesh) + { + //TODO: add some tests where union communication is not MPI_COMM_WORLD + std::shared_ptr sendMeshSearchAdaptor, recvMeshSearchAdaptor; + if (sendMesh) + { + sendMeshSearchAdaptor = std::make_shared(sendMesh, MPI_COMM_WORLD); + } + + if (recvMesh) + { + recvMeshSearchAdaptor = std::make_shared(recvMesh, MPI_COMM_WORLD); + } + + auto search = std::make_shared(sendMeshSearchAdaptor, recvMeshSearchAdaptor, + "mesh1To2Search", MPI_COMM_WORLD); + + search->coarse_search(); + + auto scatterSpec = std::make_shared(MPI_COMM_WORLD, sendMesh); + if (sendMesh) + { + const std::vector&unpairedEntities = search->get_unpaired_recv_entities(); + BoundingBoxSearch::EntityProcRelationVec& mesh2To1Relations = search->get_range_to_domain(); + + if (unpairedEntities.size() > 0) + throw std::runtime_error("coarse search could not find a destination for some mesh1 entities"); + + for (const BoundingBoxSearch::EntityProcRelation& searchRelation : mesh2To1Relations) + { + int destProc = searchRelation.first.proc(); + int mesh1ElementLocalId = searchRelation.second.id(); + + mesh::MeshEntityPtr mesh1El = sendMesh->get_elements()[mesh1ElementLocalId]; + scatterSpec->add_destination(mesh1El, destProc); + } + } + + return scatterSpec; + + } + + void scatter_mesh_1to2(std::shared_ptr scatterSpec) + { + MPI_Comm scatteredMeshComm = m_mesh2 ? m_mesh2->get_comm() : MPI_COMM_NULL; + mesh::impl::MeshScatter scatterer(scatterSpec, m_mesh1, scatteredMeshComm, true); + m_mesh1ScatteredToMesh2 = scatterer.scatter(); + m_mesh1EntityOrigins = scatterer.get_entity_origins(); + m_mesh1EntityDestinations = scatterer.get_entity_destinations(); + } + + void scatter_mesh_2to1(std::shared_ptr scatterSpec) + { + MPI_Comm scatteredMeshComm = m_mesh1 ? m_mesh1->get_comm() : MPI_COMM_NULL; + mesh::impl::MeshScatter scatterer(scatterSpec, m_mesh2, scatteredMeshComm, true); + m_mesh2ScatteredToMesh1 = scatterer.scatter(); + m_mesh2EntityOrigins = scatterer.get_entity_origins(); + m_mesh2EntityDestinations = scatterer.get_entity_destinations(); + } + + + void do_mesh_projections() + { + if (m_mesh2) + { + m_pointClassifierOnMesh2 = std::make_shared(m_mesh2); + nonconformal4::impl::MeshProjectionCalculator meshProjection(m_mesh1ScatteredToMesh2, m_mesh2, m_meshRelationalDataOnMesh2, m_pointClassifierOnMesh2, + stk::middle_mesh::impl::EdgeTracerTolerances()); + meshProjection.project(); + } + } + + void scatter_mesh_relational_data_2to1() + { + if (m_mesh2ScatteredToMesh1) + { + m_pointClassifierOnMesh1 = std::make_shared(m_mesh2ScatteredToMesh1); + } + + nonconformal4::impl::MeshRelationalDataScatterInput scatterInput; + scatterInput.mesh1 = m_mesh1; + scatterInput.mesh2 = m_mesh2; + scatterInput.mesh1ScatteredToMesh2 = m_mesh1ScatteredToMesh2; + scatterInput.mesh2ScatteredToMesh1 = m_mesh2ScatteredToMesh1; + scatterInput.mesh1EntityOrigins = m_mesh1EntityOrigins; + scatterInput.mesh1EntityDestinations = m_mesh1EntityDestinations; + scatterInput.mesh2EntityOrigins = m_mesh2EntityOrigins; + scatterInput.mesh2EntityDestinations = m_mesh2EntityDestinations; + + nonconformal4::impl::MeshRelationalDataScatter scatterer(scatterInput, m_meshRelationalDataOnMesh2, + m_pointClassifierOnMesh2, + m_pointClassifierOnMesh1, + MPI_COMM_WORLD); + m_meshRelationalDataOnMesh1 = scatterer.scatter(); + m_meshInOnMesh1Procs = scatterer.get_middle_mesh(); + } + + void create_middle_mesh_verts_and_edges() + { + if (m_mesh1) + { + nonconformal4::impl::MiddleGridConstraintGenerator generator( + m_mesh1, m_mesh2ScatteredToMesh1, m_meshInOnMesh1Procs, m_meshRelationalDataOnMesh1, m_pointClassifierOnMesh1); + + generator.generate(); + } + } + + std::shared_ptr m_mesh1; + std::shared_ptr m_mesh2; + + std::shared_ptr m_mesh1ScatteredToMesh2; + std::shared_ptr m_mesh2ScatteredToMesh1; + + mesh::VariableSizeFieldPtr m_mesh1EntityOrigins; + mesh::VariableSizeFieldPtr m_mesh1EntityDestinations; + mesh::VariableSizeFieldPtr m_mesh2EntityOrigins; + mesh::VariableSizeFieldPtr m_mesh2EntityDestinations; + + std::shared_ptr m_pointClassifierOnMesh2; + std::shared_ptr m_pointClassifierOnMesh1; + std::shared_ptr m_meshRelationalDataOnMesh2; + std::shared_ptr m_meshRelationalDataOnMesh1; + std::shared_ptr m_meshInOnMesh1Procs; +}; + +mesh::MeshEntityPtr find_entity(std::shared_ptr mesh, int dim, const utils::Point& pt, double tol=1e-13) +{ + mesh::MeshEntityPtr minEntity = nullptr; + double minDist = std::numeric_limits::max(); + + for (auto entity : mesh->get_mesh_entities(dim)) + if (entity) + { + utils::Point centroid = mesh::compute_centroid(entity); + double dist = std::sqrt(dot(centroid - pt, centroid - pt)); + if (dist < minDist && dist < tol) + { + minDist = dist; + minEntity = entity; + } + } + + if (minEntity == nullptr) + throw std::runtime_error("unable to find entity"); + + return minEntity; +} +} + + +TEST_F(MiddleGridConstraintGeneratorParallelTester, 2x2IdenticalSPMD) +{ + int commSize = utils::impl::comm_size(MPI_COMM_WORLD); + if (!(commSize == 1 || commSize == 2 || commSize == 4)) + GTEST_SKIP(); + + + mesh::impl::MeshSpec spec1, spec2; + spec1.xmin = 0; spec2.xmin = 0; + spec1.xmax = 1; spec2.xmax = 1; + spec1.ymin = 0; spec2.ymin = 0; + spec1.ymax = 1; spec2.ymax = 1; + spec1.numelX = 2; spec2.numelX = 2; + spec1.numelY = 2; spec2.numelY = 2; + + setup_spmd(spec1, spec2); + + auto meshIn = m_meshInOnMesh1Procs; + if (meshIn) + { + EXPECT_EQ(mesh::count_valid(m_meshInOnMesh1Procs->get_vertices()), + mesh::count_valid(m_mesh1->get_vertices())); + + EXPECT_EQ(mesh::count_valid(m_meshInOnMesh1Procs->get_edges()), + mesh::count_valid(m_mesh1->get_edges())); + + if (utils::impl::comm_size(MPI_COMM_WORLD) == 2) + { + if (utils::impl::comm_rank(meshIn->get_comm()) == 0) + { + EXPECT_EQ(find_entity(meshIn, 0, {0, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0, 0.5, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 1.0, 0})->count_remote_shared_entities(), 1); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.25, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.25, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.25, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0, 0.25, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0, 0.75, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 0.25, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 0.75, 0})); + } else + { + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 0.5, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 1.0, 0})->count_remote_shared_entities(), 1); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.75, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.75, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.75, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 0.25, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 0.75, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 0.25, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 0.75, 0})); + } + } + + mesh::check_topology(meshIn, 1); + } +} + +TEST_F(MiddleGridConstraintGeneratorParallelTester, 2x3MPMD) +{ + int commSize = utils::impl::comm_size(MPI_COMM_WORLD); + if (!(commSize == 4 || commSize == 8)) + GTEST_SKIP(); + + + mesh::impl::MeshSpec spec1, spec2; + spec1.xmin = 0; spec2.xmin = 0; + spec1.xmax = 1; spec2.xmax = 1; + spec1.ymin = 0; spec2.ymin = 0; + spec1.ymax = 1; spec2.ymax = 1; + spec1.numelX = 2; spec2.numelX = 3; + spec1.numelY = 2; spec2.numelY = 3; + + SplitCommTestUtil splitter(commSize/2, commSize/2); + + setup_mpmd(spec1, spec2, splitter); + + auto meshIn = m_meshInOnMesh1Procs; + if (meshIn) + { + if (utils::impl::comm_size(MPI_COMM_WORLD) == 4) + { + EXPECT_EQ(mesh::count_valid(m_meshInOnMesh1Procs->get_vertices()), 15); + EXPECT_EQ(mesh::count_valid(m_meshInOnMesh1Procs->get_edges()), 22); + + if (utils::impl::comm_rank(meshIn->get_comm()) == 0) + { + EXPECT_EQ(find_entity(meshIn, 0, {0, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0/3, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 1.0/3.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0/3, 1.0/3.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 1.0/3.0, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 0.5, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0/3, 0.5, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.5, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 2.0/3, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0/3, 2.0/3, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 2.0/3, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0/3, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 1.0, 0})->count_remote_shared_entities(), 1); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/6, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/6, 1.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/6, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/6, 2.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/6, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/12, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/12, 1.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/12, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/12, 2.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/12, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0, 1.0/6, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0, 5.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0, 7.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0, 10.0/12, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/3, 1.0/6, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/3, 5.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/3, 7.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/3, 10.0/12, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 1.0/6, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 5.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 7.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 10.0/12, 0})); + + } else + { + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {2.0/3, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 0, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 1.0/3, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {2.0/3, 1.0/3, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 1.0/3, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {2.0/3, 0.5, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 0.5, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 2.0/3, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {2.0/3, 2.0/3, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 2.0/3, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 1.0, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {2.0/3, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 1.0, 0})->count_remote_shared_entities(), 0); + + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/12, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/12, 1.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/12, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/12, 2.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/12, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {10.0/12, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {10.0/12, 1.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {10.0/12, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {10.0/12, 2.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {10.0/12, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 1.0/6, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 5.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 7.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 10.0/12, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {2.0/3, 1.0/6, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {2.0/3, 5.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {2.0/3, 7.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {2.0/3, 10.0/12, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 1.0/6, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 5.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 7.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 10.0/12, 0})); + } + } else if (utils::impl::comm_size(MPI_COMM_WORLD) == 8) + { + EXPECT_EQ(mesh::count_valid(m_meshInOnMesh1Procs->get_vertices()), 9); + EXPECT_EQ(mesh::count_valid(m_meshInOnMesh1Procs->get_edges()), 12); + + int myrank = utils::impl::comm_rank(meshIn->get_comm()); + + if (myrank == 0) + { + EXPECT_EQ(find_entity(meshIn, 0, {0, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0/3, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 1.0/3, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0/3, 1.0/3, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 1.0/3, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {1.0/3, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.5, 0})->count_remote_shared_entities(), 3); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/6, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/6, 1.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/6, 0.5, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/12, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/12, 1.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/12, 0.5, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0, 1.0/6, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0, 5.0/12, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/3, 1.0/6, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/3, 5.0/12, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 1.0/6, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 5.0/12, 0})); + + } else if (myrank == 1) + { + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {2.0/3, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 0, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 1.0/3, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {2.0/3, 1.0/3, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 1.0/3, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.5, 0})->count_remote_shared_entities(), 3); + EXPECT_EQ(find_entity(meshIn, 0, {2.0/3, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 0.5, 0})->count_remote_shared_entities(), 1); + + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/12, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/12, 1.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/12, 0.5, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {10.0/12, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {10.0/12, 1.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {10.0/12, 0.5, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 1.0/6, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 5.0/12, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {2.0/3, 1.0/6, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {2.0/3, 5.0/12, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 1.0/6, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 5.0/12, 0})); + } else if (myrank == 2) + { + EXPECT_EQ(find_entity(meshIn, 0, {0, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {1.0/3, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.5, 0})->count_remote_shared_entities(), 3); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 2.0/3, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0/3, 2.0/3, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 2.0/3, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0/3, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 1.0, 0})->count_remote_shared_entities(), 1); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/6, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/6, 2.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/6, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/12, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/12, 2.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/12, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0, 7.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0, 10.0/12, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/3, 7.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/3, 10.0/12, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 7.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 10.0/12, 0})); + } else + { + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.5, 0})->count_remote_shared_entities(), 3); + EXPECT_EQ(find_entity(meshIn, 0, {2.0/3, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 0.5, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 2.0/3, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {2.0/3, 2.0/3, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 2.0/3, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 1.0, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {2.0/3, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 1.0, 0})->count_remote_shared_entities(), 0); + + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/12, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/12, 2.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/12, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {10.0/12, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {10.0/12, 2.0/3, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {10.0/12, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 7.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 10.0/12, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {2.0/3, 7.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {2.0/3, 10.0/12, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 7.0/12, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 10.0/12, 0})); + } + } + + mesh::check_topology(meshIn, 1); + } +} + + +TEST_F(MiddleGridConstraintGeneratorParallelTester, 2x4MPMD) +{ + int commSize = utils::impl::comm_size(MPI_COMM_WORLD); + if (!(commSize == 4 || commSize == 8)) + GTEST_SKIP(); + + + mesh::impl::MeshSpec spec1, spec2; + spec1.xmin = 0; spec2.xmin = 0; + spec1.xmax = 1; spec2.xmax = 1; + spec1.ymin = 0; spec2.ymin = 0; + spec1.ymax = 1; spec2.ymax = 1; + spec1.numelX = 2; spec2.numelX = 4; + spec1.numelY = 2; spec2.numelY = 4; + + SplitCommTestUtil splitter(commSize/2, commSize/2); + + setup_mpmd(spec1, spec2, splitter); + + auto meshIn = m_meshInOnMesh1Procs; + if (meshIn) + { + if (utils::impl::comm_size(MPI_COMM_WORLD) == 4) + { + EXPECT_EQ(mesh::count_valid(m_meshInOnMesh1Procs->get_vertices()), 15); + EXPECT_EQ(mesh::count_valid(m_meshInOnMesh1Procs->get_edges()), 22); + + if (utils::impl::comm_rank(meshIn->get_comm()) == 0) + { + EXPECT_EQ(find_entity(meshIn, 0, {0, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.25, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 0.25, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.25, 0.25, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.25, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 0.5, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.25, 0.5, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.5, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 0.75, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.25, 0.75, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.75, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.25, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 1.0, 0})->count_remote_shared_entities(), 1); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/8, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/8, 0.25, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/8, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/8, 0.75, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/8, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {3.0/8, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {3.0/8, 0.25, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {3.0/8, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {3.0/8, 0.75, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {3.0/8, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0, 1.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0, 3.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0, 5.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0, 7.0/8, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.25, 1.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.25, 3.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.25, 5.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.25, 7.0/8, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 1.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 3.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 5.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 7.0/8, 0})); + + } else + { + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.75, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 0, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.25, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.75, 0.25, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 0.25, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.75, 0.5, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 0.5, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.75, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.75, 0.75, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 0.75, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 1.0, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.75, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.0, 1.0, 0})->count_remote_shared_entities(), 0); + + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/8, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/8, 0.25, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/8, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/8, 0.75, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/8, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/8, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/8, 0.25, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/8, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/8, 0.75, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/8, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 1.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 3.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 5.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 7.0/8, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.75, 1.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.75, 3.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.75, 5.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.75, 7.0/8, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 1.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 3.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 5.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 7.0/8, 0})); + + } + } else if (utils::impl::comm_size(MPI_COMM_WORLD) == 8) + { + EXPECT_EQ(mesh::count_valid(m_meshInOnMesh1Procs->get_vertices()), 9); + EXPECT_EQ(mesh::count_valid(m_meshInOnMesh1Procs->get_edges()), 12); + + int myrank = utils::impl::comm_rank(meshIn->get_comm()); + + if (myrank == 0) + { + EXPECT_EQ(find_entity(meshIn, 0, {0, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.25, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 0.25, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.25, 0.25, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.25, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.25, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.5, 0})->count_remote_shared_entities(), 3); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/8, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/8, 0.25, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/8, 0.5, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {3.0/8, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {3.0/8, 0.25, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {3.0/8, 0.5, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0, 1.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0, 3.0/8, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.25, 1.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.25, 3.0/8, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 1.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 3.0/8, 0})); + + } else if (myrank == 1) + { + EXPECT_EQ(find_entity(meshIn, 0, {0.50, 0, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.75, 0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.00, 0, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.50, 0.25, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.75, 0.25, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.00, 0.25, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.50, 0.5, 0})->count_remote_shared_entities(), 3); + EXPECT_EQ(find_entity(meshIn, 0, {0.75, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {1.00, 0.5, 0})->count_remote_shared_entities(), 1); + + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/8, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/8, 0.25, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/8, 0.5, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/8, 0, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/8, 0.25, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/8, 0.5, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 1.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 3.0/8, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.75, 1.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.75, 3.0/8, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 1.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 3.0/8, 0})); + } else if (myrank == 2) + { + EXPECT_EQ(find_entity(meshIn, 0, {0, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.25, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.5, 0})->count_remote_shared_entities(), 3); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 0.75, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.25, 0.75, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 0.75, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.25, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {0.5, 1.0, 0})->count_remote_shared_entities(), 1); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/8, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/8, 0.75, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0/8, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {3.0/8, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {3.0/8, 0.75, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {3.0/8, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0, 5.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0, 7.0/8, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.25, 5.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.25, 7.0/8, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 5.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 7.0/8, 0})); + } else + { + EXPECT_EQ(find_entity(meshIn, 0, {0.50, 0.5, 0})->count_remote_shared_entities(), 3); + EXPECT_EQ(find_entity(meshIn, 0, {0.75, 0.5, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {1.00, 0.5, 0})->count_remote_shared_entities(), 1); + + EXPECT_EQ(find_entity(meshIn, 0, {0.50, 0.75, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.75, 0.75, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.00, 0.75, 0})->count_remote_shared_entities(), 0); + + EXPECT_EQ(find_entity(meshIn, 0, {0.50, 1.0, 0})->count_remote_shared_entities(), 1); + EXPECT_EQ(find_entity(meshIn, 0, {0.75, 1.0, 0})->count_remote_shared_entities(), 0); + EXPECT_EQ(find_entity(meshIn, 0, {1.00, 1.0, 0})->count_remote_shared_entities(), 0); + + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/8, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/8, 0.75, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {5.0/8, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/8, 0.5, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/8, 0.75, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {7.0/8, 1.0, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 5.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.5, 7.0/8, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {0.75, 5.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {0.75, 7.0/8, 0})); + + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 5.0/8, 0})); + EXPECT_TRUE(find_entity(meshIn, 1, {1.0, 7.0/8, 0})); + } + } + + mesh::check_topology(meshIn, 1); + } +} \ No newline at end of file diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_triangulator.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_triangulator.cpp index f20a11d24142..1cf6181807ab 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_triangulator.cpp +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_triangulator.cpp @@ -110,11 +110,7 @@ TEST_F(Nonconformal4Tester, El1ContainedInEl2) std::vector> mesh2Els = {{0, 1, 2, 3}}; create_mesh2(mesh2Verts, mesh2Els); - stk::middle_mesh::mesh::impl::print_vert_edges("mesh1", mMesh1); - stk::middle_mesh::mesh::impl::print_vert_edges("mesh2", mMesh2); - run(); - stk::middle_mesh::mesh::impl::print_vert_edges("mesh_in", mMeshIn); stk::middle_mesh::test_util::test_number_of_elements(mMeshIn, 10); } diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_triangulator_parallel.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_triangulator_parallel.cpp new file mode 100644 index 000000000000..bdb6b04749cc --- /dev/null +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/test_middle_grid_triangulator_parallel.cpp @@ -0,0 +1,793 @@ + +#include "gtest/gtest.h" +#include "stk_middle_mesh/mesh_relational_data_scatter.hpp" +#include "stk_middle_mesh/predicates/point_classifier_normal_wrapper.hpp" +#include "stk_middle_mesh/mesh_scatter.hpp" +#include "stk_middle_mesh/mesh_scatter_spec.hpp" +#include "stk_middle_mesh/bounding_box_search.hpp" +#include "stk_middle_mesh/middle_grid_constraint_generator.hpp" +#include "stk_middle_mesh/mesh.hpp" +#include "stk_middle_mesh/mesh_projection_calculator.hpp" +#include "stk_middle_mesh/create_mesh.hpp" +#include "stk_middle_mesh/middle_grid_triangulator.hpp" +#include "stk_middle_mesh/nonconformal4.hpp" +#include "stk_util/parallel/DataExchangeUnknownPatternNonBlockingBuffer.hpp" +#include "util/parallel_search_test_util.hpp" +#include "util/nonconformal_interface_helpers.hpp" +#include "stk_middle_mesh/variable_size_field.hpp" +#include "stk_middle_mesh/mesh_entity.hpp" +#include "stk_middle_mesh/mesh_relational_data.hpp" +#include "stk_middle_mesh/edge_tracer_opts.hpp" +#include "stk_middle_mesh/bounding_box_search.hpp" + +namespace { + +using namespace stk::middle_mesh; + +class MiddleGridTriangulatorParallelTester : public ::testing::Test +{ + protected: + + using BoundingBoxSearch = search::ElementToElementBoundingBoxSearch; + + void setup_spmd(const mesh::impl::MeshSpec& spec1, const mesh::impl::MeshSpec& spec2) + { + auto f = [](const utils::Point& pt) { return pt; }; + + auto mesh1 = mesh::impl::create_mesh(spec1, f); + auto mesh2 = mesh::impl::create_mesh(spec1, f); + + setup(mesh1, mesh2); + } + + void setup_mpmd(const mesh::impl::MeshSpec& spec1, const mesh::impl::MeshSpec& spec2, + const SplitCommTestUtil& splitter) + { + auto f = [](const utils::Point& pt) { return pt; }; + + std::shared_ptr mesh1, mesh2; + + if (splitter.get_color() == SplitCommColor::SEND) + { + mesh1 = mesh::impl::create_mesh(spec1, f, splitter.get_comm()); + } + + if (splitter.get_color() == SplitCommColor::RECV) + { + mesh2 = mesh::impl::create_mesh(spec2, f, splitter.get_comm()); + } + + setup(mesh1, mesh2); + } + + void setup_mpmd(const std::vector& vertCoords1, + const std::vector>& elementVerts1, + const std::vector& vertCoords2, + const std::vector>& elementVerts2) + { + assert(utils::impl::comm_size(MPI_COMM_WORLD) == 2); + std::shared_ptr mesh1, mesh2; + if (utils::impl::comm_rank(MPI_COMM_WORLD) == 0) + { + mesh1 = mesh::make_empty_mesh(MPI_COMM_SELF); + create_mesh(mesh1, vertCoords1, elementVerts1); + } else + { + mesh2 = mesh::make_empty_mesh(MPI_COMM_SELF); + create_mesh(mesh2, vertCoords2, elementVerts2); + } + + setup(mesh1, mesh2); + } + + void setup(std::shared_ptr mesh1, std::shared_ptr mesh2) + { + m_mesh1 = mesh1; + m_mesh2 = mesh2; + + auto scatterSpec = create_scatter_spec(mesh1, mesh2); + scatter_mesh_1to2(scatterSpec); + if (m_mesh2) + { + m_meshRelationalDataOnMesh2 = std::make_shared(m_mesh1ScatteredToMesh2, m_mesh2); + } + + do_mesh_projections(); + + scatterSpec = create_scatter_spec(mesh2, mesh1); + scatter_mesh_2to1(scatterSpec); + scatter_mesh_relational_data_2to1(); + + create_middle_mesh_verts_and_edges(); + create_middle_mesh_triangles(); + } + + void run_tests() + { + if (m_mesh1) + { + m_meshInElementsToMesh1 = m_meshRelationalDataOnMesh1->meshInElementsToMesh1Elements; + m_meshInElementsToMesh2ScatteredToMesh1 = m_meshRelationalDataOnMesh1->meshInElementsToMesh2Elements; + m_mesh1InverseClassification = nonconformal4::impl::invert_classification_field( + m_meshInOnMesh1Procs, m_mesh1, m_meshInElementsToMesh1); + m_mesh2ScatteredToMesh1InverseClassification = nonconformal4::impl::invert_classification_field( + m_meshInOnMesh1Procs, m_mesh2ScatteredToMesh1, + m_meshInElementsToMesh2ScatteredToMesh1); + + stk::middle_mesh::test_util::test_areas_positive(m_meshInOnMesh1Procs); + stk::middle_mesh::test_util::test_every_element_classified(m_meshInOnMesh1Procs, m_meshInElementsToMesh1); + stk::middle_mesh::test_util::test_every_element_classified(m_meshInOnMesh1Procs, m_meshInElementsToMesh2ScatteredToMesh1); + stk::middle_mesh::test_util::test_every_element_classified_inverse(m_mesh1, m_mesh1InverseClassification); + stk::middle_mesh::test_util::test_area_per_element(m_mesh1, m_mesh1InverseClassification); + + mesh::check_topology(m_meshInOnMesh1Procs); + } + + stk::middle_mesh::test_util::test_total_areas_same(m_mesh1, m_mesh2, m_meshInOnMesh1Procs); + test_mesh2_area_per_element(m_mesh2ScatteredToMesh1InverseClassification); + + } + + + std::shared_ptr create_scatter_spec( + std::shared_ptr sendMesh, std::shared_ptr recvMesh) + { + std::shared_ptr sendMeshSearchAdaptor, recvMeshSearchAdaptor; + if (sendMesh) + { + sendMeshSearchAdaptor = std::make_shared(sendMesh, MPI_COMM_WORLD); + } + + if (recvMesh) + { + recvMeshSearchAdaptor = std::make_shared(recvMesh, MPI_COMM_WORLD); + } + + auto search = std::make_shared(sendMeshSearchAdaptor, recvMeshSearchAdaptor, + "mesh1To2Search", MPI_COMM_WORLD); + + search->coarse_search(); + + auto scatterSpec = std::make_shared(MPI_COMM_WORLD, sendMesh); + if (sendMesh) + { + const std::vector&unpairedEntities = search->get_unpaired_recv_entities(); + BoundingBoxSearch::EntityProcRelationVec& mesh2To1Relations = search->get_range_to_domain(); + + if (unpairedEntities.size() > 0) + throw std::runtime_error("coarse search could not find a destination for some mesh1 entities"); + + for (const BoundingBoxSearch::EntityProcRelation& searchRelation : mesh2To1Relations) + { + int destProc = searchRelation.first.proc(); + int mesh1ElementLocalId = searchRelation.second.id(); + + mesh::MeshEntityPtr mesh1El = sendMesh->get_elements()[mesh1ElementLocalId]; + scatterSpec->add_destination(mesh1El, destProc); + } + } + + return scatterSpec; + + } + + void scatter_mesh_1to2(std::shared_ptr scatterSpec) + { + MPI_Comm scatteredMeshComm = m_mesh2 ? m_mesh2->get_comm() : MPI_COMM_NULL; + mesh::impl::MeshScatter scatterer(scatterSpec, m_mesh1, scatteredMeshComm, true); + m_mesh1ScatteredToMesh2 = scatterer.scatter(); + m_mesh1EntityOrigins = scatterer.get_entity_origins(); + m_mesh1EntityDestinations = scatterer.get_entity_destinations(); + } + + void scatter_mesh_2to1(std::shared_ptr scatterSpec) + { + MPI_Comm scatteredMeshComm = m_mesh1 ? m_mesh1->get_comm() : MPI_COMM_NULL; + mesh::impl::MeshScatter scatterer(scatterSpec, m_mesh2, scatteredMeshComm, true); + m_mesh2ScatteredToMesh1 = scatterer.scatter(); + m_mesh2EntityOrigins = scatterer.get_entity_origins(); + m_mesh2EntityDestinations = scatterer.get_entity_destinations(); + } + + + void do_mesh_projections() + { + if (m_mesh2) + { + m_pointClassifierOnMesh2 = std::make_shared(m_mesh2); + nonconformal4::impl::MeshProjectionCalculator meshProjection(m_mesh1ScatteredToMesh2, m_mesh2, m_meshRelationalDataOnMesh2, m_pointClassifierOnMesh2, + stk::middle_mesh::impl::EdgeTracerTolerances()); + meshProjection.project(); + } + } + + void scatter_mesh_relational_data_2to1() + { + if (m_mesh2ScatteredToMesh1) + { + m_pointClassifierOnMesh1 = std::make_shared(m_mesh2ScatteredToMesh1); + } + + nonconformal4::impl::MeshRelationalDataScatterInput scatterInput; + scatterInput.mesh1 = m_mesh1; + scatterInput.mesh2 = m_mesh2; + scatterInput.mesh1ScatteredToMesh2 = m_mesh1ScatteredToMesh2; + scatterInput.mesh2ScatteredToMesh1 = m_mesh2ScatteredToMesh1; + scatterInput.mesh1EntityOrigins = m_mesh1EntityOrigins; + scatterInput.mesh1EntityDestinations = m_mesh1EntityDestinations; + scatterInput.mesh2EntityOrigins = m_mesh2EntityOrigins; + scatterInput.mesh2EntityDestinations = m_mesh2EntityDestinations; + + nonconformal4::impl::MeshRelationalDataScatter scatterer(scatterInput, m_meshRelationalDataOnMesh2, + m_pointClassifierOnMesh2, + m_pointClassifierOnMesh1, + MPI_COMM_WORLD); + m_meshRelationalDataOnMesh1 = scatterer.scatter(); + m_meshInOnMesh1Procs = scatterer.get_middle_mesh(); + } + + void create_middle_mesh_verts_and_edges() + { + if (m_mesh1) + { + nonconformal4::impl::MiddleGridConstraintGenerator generator( + m_mesh1, m_mesh2ScatteredToMesh1, m_meshInOnMesh1Procs, m_meshRelationalDataOnMesh1, m_pointClassifierOnMesh1); + + generator.generate(); + } + } + + void create_middle_mesh_triangles() + { + if (m_mesh1) + { + nonconformal4::impl::MiddleGridTriangulator triangulator(m_mesh1, m_mesh2ScatteredToMesh1, + m_meshInOnMesh1Procs, m_meshRelationalDataOnMesh1, m_pointClassifierOnMesh1); + + triangulator.triangulate(); + } + } + + void test_mesh2_area_per_element(mesh::VariableSizeFieldPtr mesh2ScatteredToMesh1InverseClassificationPtr) + { + stk::DataExchangeUnknownPatternNonBlockingCommBuffer exchanger(MPI_COMM_WORLD); + + if (m_mesh2ScatteredToMesh1) + { + auto& mesh2EntityOrigins = *m_mesh2EntityOrigins; + auto& invClassification = *mesh2ScatteredToMesh1InverseClassificationPtr; + + mesh::impl::ElementOperations2D elemOps; + for (int phase=0; phase < 2; ++phase) + { + for (auto& el2 : m_mesh2ScatteredToMesh1->get_elements()) + if (el2) + { + double area = 0; + for (auto& elIn : invClassification(el2, 0)) + { + area += elemOps.compute_area(elIn); + } + + mesh::RemoteSharedEntity remote = mesh2EntityOrigins(el2, 0, 0); + exchanger.get_send_buf(remote.remoteRank).pack(remote.remoteId); + exchanger.get_send_buf(remote.remoteRank).pack(area); + } + + if (phase == 0) + exchanger.allocate_send_buffers(); + } + } + + exchanger.start_nonblocking(); + exchanger.post_nonblocking_receives(); + + mesh::FieldPtr elAreasPtr; + if (m_mesh2) + { + elAreasPtr = mesh::create_field(m_mesh2, mesh::FieldShape(0, 0, 1), 1, 0); + } + + auto unpacker = [&](int rank, stk::CommBuffer& buf) + { + auto& elAreas = *elAreasPtr; + while (buf.remaining() > 0) + { + int elId; + double area; + buf.unpack(elId); + buf.unpack(area); + mesh::MeshEntityPtr el = m_mesh2->get_elements()[elId]; + elAreas(el, 0, 0) += area; + } + }; + + exchanger.complete_receives(unpacker); + + if (m_mesh2) + { + auto& elAreas = *elAreasPtr; + mesh::impl::ElementOperations2D elemOps; + for (auto& el : m_mesh2->get_elements()) + if (el) + { + double areaEx = elemOps.compute_area(el); + EXPECT_NEAR(areaEx, elAreas(el, 0, 0), 1e-13); + } + } + } + + void create_mesh(std::shared_ptr mesh, const std::vector& vertCoords, + const std::vector>& elementVerts) + { + for (auto& vert : vertCoords) + mesh->create_vertex(vert); + + auto& verts = mesh->get_vertices(); + for (auto& elVertIds : elementVerts) + { + if (elVertIds[3] == -1) + { + auto v0 = verts[elVertIds[0]]; + auto v1 = verts[elVertIds[1]]; + auto v2 = verts[elVertIds[2]]; + mesh->create_triangle_from_verts(v0, v1, v2); + } else + { + auto v0 = verts[elVertIds[0]]; + auto v1 = verts[elVertIds[1]]; + auto v2 = verts[elVertIds[2]]; + auto v3 = verts[elVertIds[3]]; + mesh->create_quad_from_verts(v0, v1, v2, v3); + } + } + } + + std::shared_ptr m_mesh1; + std::shared_ptr m_mesh2; + + std::shared_ptr m_mesh1ScatteredToMesh2; + std::shared_ptr m_mesh2ScatteredToMesh1; + + mesh::VariableSizeFieldPtr m_mesh1EntityOrigins; + mesh::VariableSizeFieldPtr m_mesh1EntityDestinations; + mesh::VariableSizeFieldPtr m_mesh2EntityOrigins; + mesh::VariableSizeFieldPtr m_mesh2EntityDestinations; + + std::shared_ptr m_pointClassifierOnMesh2; + std::shared_ptr m_pointClassifierOnMesh1; + std::shared_ptr m_meshRelationalDataOnMesh2; + std::shared_ptr m_meshRelationalDataOnMesh1; + std::shared_ptr m_meshInOnMesh1Procs; + + mesh::FieldPtr m_meshInElementsToMesh1; + mesh::FieldPtr m_meshInElementsToMesh2ScatteredToMesh1; + mesh::VariableSizeFieldPtr m_mesh1InverseClassification; + mesh::VariableSizeFieldPtr m_mesh2ScatteredToMesh1InverseClassification; +}; + +//TODO: for serial tests, do them in parallel in MPMD + +} // namespace + + +TEST_F(MiddleGridTriangulatorParallelTester, SingleEdgeOverlap) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 2) + { + GTEST_SKIP(); + } + + // The top edge of el1 overlaps with the top edge of el2. No other + // edges itnersect + std::vector mesh1Verts = {utils::Point(0, 0), utils::Point(1, 0), utils::Point(1, 1), + utils::Point(0, 1), utils::Point(-0.25, -0.25), utils::Point(1.25, -0.25), + utils::Point(1.25, 1), utils::Point(-0.25, 1)}; + std::vector> mesh1Els = {{0, 1, 2, 3}, {4, 5, 1, 0}, {5, 6, 2, 1}, {0, 3, 7, 4}}; + + std::vector mesh2Verts = {utils::Point(-0.25, -0.25), utils::Point(1.25, -0.25), utils::Point(1.25, 1), + utils::Point(-0.25, 1)}; + std::vector> mesh2Els = {{0, 1, 2, 3}}; + + setup_mpmd(mesh1Verts, mesh1Els, mesh2Verts, mesh2Els); + + run_tests(); + + if (m_meshInOnMesh1Procs) + stk::middle_mesh::test_util::test_number_of_elements(m_meshInOnMesh1Procs, 8); +} + +TEST_F(MiddleGridTriangulatorParallelTester, TwoEdgeOverlapBisection) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 2) + { + GTEST_SKIP(); + } + + // One edge of mesh2 cuts el1 in half. + std::vector mesh1Verts = {utils::Point(0, 0), utils::Point(1, 0), utils::Point(1, 1), + utils::Point(0, 1), utils::Point(-1, -1), utils::Point(2, -1), + utils::Point(2, 2), utils::Point(-1, 2)}; + std::vector> mesh1Els = {{0, 1, 2, 3}, {4, 5, 1, 0}, {5, 6, 2, 1}, {3, 2, 6, 7}, {4, 0, 3, 7}}; + + std::vector mesh2Verts = {utils::Point(-1, -1), utils::Point(2, -1), utils::Point(2, 0.5), + utils::Point(-1, 0.5), utils::Point(2, 2), utils::Point(-1, 2)}; + std::vector> mesh2Els = {{0, 1, 2, 3}, {3, 2, 4, 5}}; + + setup_mpmd(mesh1Verts, mesh1Els, mesh2Verts, mesh2Els); + + run_tests(); + + if (m_meshInOnMesh1Procs) + stk::middle_mesh::test_util::test_number_of_elements(m_meshInOnMesh1Procs, 16); +} + +TEST_F(MiddleGridTriangulatorParallelTester, TwoEdgeOverlapSharedCorner) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 2) + { + GTEST_SKIP(); + } + + // The top left corner of el1 overlaps the top left corner of el2, but el2 + // is larger than el1 + std::vector mesh1Verts = {utils::Point(0, -1), utils::Point(1, -1), utils::Point(2, -1), + utils::Point(0, 0), utils::Point(1, 0), utils::Point(2, 0), + utils::Point(0, 1), utils::Point(1, 1), utils::Point(2, 1)}; + std::vector> mesh1Els = {{0, 1, 4, 3}, {1, 2, 5, 4}, {3, 4, 7, 6}, {4, 5, 8, 7}}; + + std::vector mesh2Verts = {utils::Point(0, -1), utils::Point(2, -1), utils::Point(2, 1), + utils::Point(0, 1)}; + std::vector> mesh2Els = {{0, 1, 2, 3}}; + + setup_mpmd(mesh1Verts, mesh1Els, mesh2Verts, mesh2Els); + + run_tests(); + + if (m_meshInOnMesh1Procs) + stk::middle_mesh::test_util::test_number_of_elements(m_meshInOnMesh1Procs, 8); +} + +TEST_F(MiddleGridTriangulatorParallelTester, TwoEdgeOverlapCutCorner) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 2) + { + GTEST_SKIP(); + } + + // An edge of the first element on mesh2 cuts off the top left corner of el1. + std::vector mesh1Verts = {utils::Point(0, 0), utils::Point(1, 0), utils::Point(1, 1), + utils::Point(0, 1), utils::Point(0.75, 2), utils::Point(1, 3), + utils::Point(-0.5, 3), utils::Point(-0.5, -1), utils::Point(2, -1)}; + std::vector> mesh1Els = {{0, 1, 2, 3}, {7, 8, 1, 0}, {1, 8, 2, -1}, {2, 8, 4, -1}, + {2, 4, 3, -1}, {3, 4, 5, -1}, {3, 5, 6, -1}, {0, 3, 6, 7}}; + + std::vector mesh2Verts = {utils::Point(-0.5, 0), utils::Point(0.75, 2), utils::Point(1, 3), + utils::Point(-0.5, 3), utils::Point(-0.5, -1), utils::Point(2, -1)}; + std::vector> mesh2Els = {{0, 1, 2, 3}, {4, 5, 1, 0}}; + + setup_mpmd(mesh1Verts, mesh1Els, mesh2Verts, mesh2Els); + + run_tests(); + + if (m_meshInOnMesh1Procs) + stk::middle_mesh::test_util::test_number_of_elements(m_meshInOnMesh1Procs, 16); +} + +TEST_F(MiddleGridTriangulatorParallelTester, ThreeEdgeOverlapCutCorners) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 2) + { + GTEST_SKIP(); + } + + // edges of an element on mesh2 cut off the top left and top right corners + // of el1. No other edges are intersected + std::vector mesh1Verts = {utils::Point(0, 0), utils::Point(1, 0), utils::Point(1, 1), + utils::Point(0, 1), utils::Point(-0.5, -1), utils::Point(1.25, -0.5), + utils::Point(2, 3), utils::Point(-0.5, 3)}; + std::vector> mesh1Els = {{0, 1, 2, 3}, {4, 5, 1, 0}, {1, 5, 6, 2}, {3, 2, 6, 7}, {0, 3, 7, 4}}; + + std::vector mesh2Verts = {utils::Point(-0.5, 0), utils::Point(0.75, 2), utils::Point(1, 3), + utils::Point(-0.5, 3), utils::Point(-0.5, -1), utils::Point(1.25, -.5), + utils::Point(2, 3)}; + std::vector> mesh2Els = {{0, 1, 2, 3}, {4, 5, 1, 0}, {5, 6, 2, 1}}; + + setup_mpmd(mesh1Verts, mesh1Els, mesh2Verts, mesh2Els); + + run_tests(); + + if (m_meshInOnMesh1Procs) + stk::middle_mesh::test_util::test_number_of_elements(m_meshInOnMesh1Procs, 22); +} + +TEST_F(MiddleGridTriangulatorParallelTester, TwoEdgeOverlapBisection2) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 2) + { + GTEST_SKIP(); + } + + // One edge of mesh2 cuts el1 in half. The top edges of mesh2 elements overlap + // the top edge of el1, and similarly for the bottom edges + std::vector mesh1Verts = {utils::Point(-1, 0), utils::Point(0, 0), utils::Point(1, 0), + utils::Point(3, 0), utils::Point(-1, 1), utils::Point(0, 1), + utils::Point(1, 1), utils::Point(3, 1)}; + std::vector> mesh1Els = {{0, 1, 5, 4}, {1, 2, 6, 5}, {2, 3, 7, 6}}; + + std::vector mesh2Verts = {utils::Point(-1, 0), utils::Point(0.5, 0), utils::Point(0.5, 1), + utils::Point(-1, 1), utils::Point(3, 0), utils::Point(3, 1)}; + std::vector> mesh2Els = {{0, 1, 2, 3}, {1, 4, 5, 2}}; + + setup_mpmd(mesh1Verts, mesh1Els, mesh2Verts, mesh2Els); + + run_tests(); + + if (m_meshInOnMesh1Procs) + stk::middle_mesh::test_util::test_number_of_elements(m_meshInOnMesh1Procs, 8); +} + +TEST_F(MiddleGridTriangulatorParallelTester, TwoVertexOverlapDiagonal) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 2) + { + GTEST_SKIP(); + } + + // One edge of mesh2 cuts el1 in half. The top edges of mesh2 elements overlap + // the top edge of el1, and similarly for the bottom edges + std::vector mesh1Verts = { + utils::Point(0, 0), utils::Point(1, 0), utils::Point(1, 1), utils::Point(0, 1), utils::Point(-1, -1), + utils::Point(2, -1), utils::Point(3, 0), utils::Point(3, 1.5), utils::Point(-1, 2), utils::Point(-2, 0.5)}; + std::vector> mesh1Els = {{0, 1, 2, 3}, {4, 5, 1, 0}, {1, 5, 6, -1}, {1, 6, 7, 2}, + {2, 7, 8, 3}, {0, 3, 8, 9}, {4, 0, 9, -1}}; + + std::vector mesh2Verts = {utils::Point(-1, -1), utils::Point(2, -1), utils::Point(-1, 2), + utils::Point(-2, 0.5), utils::Point(3, 0), utils::Point(3, 1.5)}; + std::vector> mesh2Els = {{0, 1, 2, 3}, {1, 4, 5, 2}}; + + setup_mpmd(mesh1Verts, mesh1Els, mesh2Verts, mesh2Els); + + run_tests(); + + if (m_meshInOnMesh1Procs) + stk::middle_mesh::test_util::test_number_of_elements(m_meshInOnMesh1Procs, 12); +} + +TEST_F(MiddleGridTriangulatorParallelTester, FourVertexOverlapDiagonals) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 2) + { + GTEST_SKIP(); + } + + // mesh2 edges cross both diagonals of element 1, dividing it into 4 triangles + std::vector mesh1Verts = {utils::Point(0, 0), utils::Point(1, 0), utils::Point(1, 1), + utils::Point(0, 1), utils::Point(0.5, -3), utils::Point(2, -1), + utils::Point(3, 0.5), utils::Point(2, 2), utils::Point(0.5, 3), + utils::Point(-1, 2), utils::Point(-3, 0.5), utils::Point(-1, -1)}; + std::vector> mesh1Els = {{0, 1, 2, 3}, {4, 0, 11, -1}, {4, 5, 1, 0}, + {5, 6, 2, 1}, {6, 7, 2, -1}, {2, 7, 8, -1}, + {2, 8, 9, 3}, {3, 9, 10, -1}, {0, 3, 10, 11}}; + + std::vector mesh2Verts = {utils::Point(0.5, 0.5), utils::Point(-1, -1), utils::Point(2, -1), + utils::Point(2, 2), utils::Point(-1, 2), utils::Point(0.5, -3), + utils::Point(3, 0.5), utils::Point(0.5, 3), utils::Point(-3, 0.5)}; + std::vector> mesh2Els = {{1, 5, 2, 0}, {2, 6, 3, 0}, {0, 3, 7, 4}, {1, 0, 4, 8}}; + + setup_mpmd(mesh1Verts, mesh1Els, mesh2Verts, mesh2Els); + + run_tests(); + + if (m_meshInOnMesh1Procs) + stk::middle_mesh::test_util::test_number_of_elements(m_meshInOnMesh1Procs, 16); +} + +TEST_F(MiddleGridTriangulatorParallelTester, ThreeVertexOverlapDiagonals) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 2) + { + GTEST_SKIP(); + } + + // mesh2 edges cross 3 vertices and one edge of el1 + std::vector mesh1Verts = {utils::Point(0, 0), utils::Point(1, 0), utils::Point(1, 1), + utils::Point(0, 1), utils::Point(0.5, -3), utils::Point(2, -1), + utils::Point(3, 0.5), utils::Point(2, 2), utils::Point(0.5, 3), + utils::Point(-1, 2), utils::Point(-3, 0.5), utils::Point(-1, 0.25)}; + std::vector> mesh1Els = {{0, 1, 2, 3}, {4, 0, 11, -1}, {4, 5, 1, 0}, + {5, 6, 2, 1}, {6, 7, 2, -1}, {2, 7, 8, -1}, + {2, 8, 9, 3}, {3, 9, 10, -1}, {0, 3, 10, 11}}; + + std::vector mesh2Verts = {utils::Point(0.5, 0.5), utils::Point(-1, 0.25), utils::Point(2, -1), + utils::Point(2, 2), utils::Point(-1, 2), utils::Point(0.5, -3), + utils::Point(3, 0.5), utils::Point(0.5, 3), utils::Point(-3, 0.5)}; + std::vector> mesh2Els = {{1, 5, 2, 0}, {2, 6, 3, 0}, {0, 3, 7, 4}, {1, 0, 4, 8}}; + + setup_mpmd(mesh1Verts, mesh1Els, mesh2Verts, mesh2Els); + + run_tests(); + + if (m_meshInOnMesh1Procs) + stk::middle_mesh::test_util::test_number_of_elements(m_meshInOnMesh1Procs, 18); +} + +TEST_F(MiddleGridTriangulatorParallelTester, ThreeVertexNonCentroid) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 2) + { + GTEST_SKIP(); + } + + // mesh2 edges cross 3 vertices and one edge of el1, but the interior point is not + // on the centroid of el1 + std::vector mesh1Verts = {utils::Point(0, 0), utils::Point(1, 0), utils::Point(1, 1), + utils::Point(0, 1), utils::Point(0.5, -3), utils::Point(3, -2.0 / 3.0), + utils::Point(3, 0.5), utils::Point(2, 2), utils::Point(0.5, 3.5), + utils::Point(-1, 4), utils::Point(-3, 2), utils::Point(-1, 0.125)}; + std::vector> mesh1Els = {{0, 1, 2, 3}, {4, 0, 11, -1}, {4, 5, 1, 0}, + {5, 6, 2, 1}, {6, 7, 2, -1}, {2, 7, 8, -1}, + {2, 8, 9, 3}, {3, 9, 10, -1}, {0, 3, 10, 11}}; + + std::vector mesh2Verts = { + utils::Point(0.25, 0.25), utils::Point(-1, 0.125), utils::Point(3, -2.0 / 3.0), + utils::Point(2, 2), utils::Point(-1, 4), utils::Point(0.5, -3), + utils::Point(3, 0.5), utils::Point(0.5, 3.5), utils::Point(-3, 2)}; + std::vector> mesh2Els = {{1, 5, 2, 0}, {2, 6, 3, 0}, {0, 3, 7, 4}, {1, 0, 4, 8}}; + + setup_mpmd(mesh1Verts, mesh1Els, mesh2Verts, mesh2Els); + + run_tests(); + + if (m_meshInOnMesh1Procs) + stk::middle_mesh::test_util::test_number_of_elements(m_meshInOnMesh1Procs, 18); +} + +TEST_F(MiddleGridTriangulatorParallelTester, DoubleEdgeIntersection) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 2) + { + GTEST_SKIP(); + } + + // Two edges of a mesh2 element intersect the same edge of the mesh1 element + std::vector mesh1Verts = {utils::Point(0, 0), utils::Point(1, 0), utils::Point(1, 1), + utils::Point(0, 1), utils::Point(-0.25, -1), utils::Point(1.5, -1), + utils::Point(1.5, 2.0), utils::Point(0.75, 2), utils::Point(0.25, 2.5), + utils::Point(-1, 0.75), utils::Point(-1.5, 0.25), utils::Point(-1, 0.25)}; + std::vector> mesh1Els = {{0, 1, 2, 3}, {4, 5, 1, 0}, {5, 6, 2, 1}, {2, 6, 7, -1}, + {2, 7, 8, 3}, {3, 8, 9, -1}, {3, 9, 11, -1}, {9, 10, 11, -1}, + {0, 3, 11, -1}, {0, 11, 4, -1}}; + + std::vector mesh2Verts = { + utils::Point(0.5, 0.5), utils::Point(-0.25, -1), utils::Point(0.5, -1), utils::Point(-1, 0.25), + utils::Point(1.5, -1), utils::Point(1.5, 0.5), utils::Point(1.5, 2), utils::Point(0.75, 2), + utils::Point(0.25, 2.5), utils::Point(-1, 0.75), utils::Point(-1.5, 0.25), + }; + std::vector> mesh2Els = {{1, 2, 0, 3}, {2, 4, 5, 0}, {0, 5, 6, 7}, {0, 7, 8, 9}, {3, 0, 9, 10}}; + + setup_mpmd(mesh1Verts, mesh1Els, mesh2Verts, mesh2Els); + + run_tests(); + + if (m_meshInOnMesh1Procs) + stk::middle_mesh::test_util::test_number_of_elements(m_meshInOnMesh1Procs, 30); +} + +TEST_F(MiddleGridTriangulatorParallelTester, EdgeCrossThroughCornerVertex) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 2) + { + GTEST_SKIP(); + } + + // Edge starts on midpoint of one edge of el1 and passes through a vertex that + // is not adjacent to the edge + std::vector mesh1Verts = {utils::Point(0, 0), utils::Point(1, 0), utils::Point(1, 1), + utils::Point(0, 1), utils::Point(0, -1), utils::Point(2, -2), + utils::Point(3, -1), utils::Point(3, 2), utils::Point(0, 2)}; + std::vector> mesh1Els = {{0, 1, 2, 3}, {4, 5, 1, 0}, {5, 6, 1, -1}, {1, 6, 7, 2}, {2, 7, 8, 3}}; + + std::vector mesh2Verts = {utils::Point(0, -1), utils::Point(2, -2), utils::Point(0.5, 1), + utils::Point(0, 1), utils::Point(3, -1), utils::Point(3, 2), + utils::Point(0, 2)}; + std::vector> mesh2Els = {{0, 1, 2, 3}, {1, 4, 5, 2}, {3, 2, 5, 6}}; + + setup_mpmd(mesh1Verts, mesh1Els, mesh2Verts, mesh2Els); + + run_tests(); + + if (m_meshInOnMesh1Procs) + stk::middle_mesh::test_util::test_number_of_elements(m_meshInOnMesh1Procs, 11); +} + +TEST_F(MiddleGridTriangulatorParallelTester, ThreeElementsWithCutCorner) +{ + if (utils::impl::comm_size(MPI_COMM_WORLD) != 2) + { + GTEST_SKIP(); + } + + // two elements cover most of el1, but a third element cuts off a corner + std::vector mesh1Verts = {utils::Point(0, 0), utils::Point(1, 0), utils::Point(1, 1), + utils::Point(0, 1), utils::Point(-1, 0), utils::Point(0.5, -0.25), + utils::Point(1.5, -0.25), utils::Point(1.5, 1.25), utils::Point(-0.25, 1.25), + utils::Point(-0.25, 0.5), utils::Point(-0.5, 0.5)}; + std::vector> mesh1Els = {{0, 1, 2, 3}, {5, 6, 1, 0}, {1, 6, 7, 2}, {2, 7, 8, 3}, + {0, 3, 8, 9}, {0, 9, 10, 4}, {0, 4, 5, -1}}; + + std::vector mesh2Verts = {utils::Point(0.5, -0.25), utils::Point(-0.25, 0.5), utils::Point(-0.5, 0.5), + utils::Point(-1, 0), utils::Point(1.5, -0.25), utils::Point(1.5, 1.25), + utils::Point(0.5, 1.25), utils::Point(-0.25, 1.25)}; + std::vector> mesh2Els = {{0, 1, 2, 3}, {0, 4, 5, 6}, {0, 6, 7, 1}}; + + setup_mpmd(mesh1Verts, mesh1Els, mesh2Verts, mesh2Els); + + run_tests(); + + if (m_meshInOnMesh1Procs) + stk::middle_mesh::test_util::test_number_of_elements(m_meshInOnMesh1Procs, 22); +} + + +TEST_F(MiddleGridTriangulatorParallelTester, 2x2IdenticalSPMD) +{ + int commSize = utils::impl::comm_size(MPI_COMM_WORLD); + if (!(commSize == 1 || commSize == 2 || commSize == 4)) + GTEST_SKIP(); + + + mesh::impl::MeshSpec spec1, spec2; + spec1.xmin = 0; spec2.xmin = 0; + spec1.xmax = 1; spec2.xmax = 1; + spec1.ymin = 0; spec2.ymin = 0; + spec1.ymax = 1; spec2.ymax = 1; + spec1.numelX = 2; spec2.numelX = 2; + spec1.numelY = 2; spec2.numelY = 2; + + setup_spmd(spec1, spec2); + run_tests(); +} + +TEST_F(MiddleGridTriangulatorParallelTester, 2x3MPMD) +{ + int commSize = utils::impl::comm_size(MPI_COMM_WORLD); + if (!(commSize == 4 || commSize == 8)) + GTEST_SKIP(); + + + mesh::impl::MeshSpec spec1, spec2; + spec1.xmin = 0; spec2.xmin = 0; + spec1.xmax = 1; spec2.xmax = 1; + spec1.ymin = 0; spec2.ymin = 0; + spec1.ymax = 1; spec2.ymax = 1; + spec1.numelX = 2; spec2.numelX = 3; + spec1.numelY = 2; spec2.numelY = 3; + + SplitCommTestUtil splitter(commSize/2, commSize/2); + + setup_mpmd(spec1, spec2, splitter); + run_tests(); +} + + +TEST_F(MiddleGridTriangulatorParallelTester, 2x4MPMD) +{ + int commSize = utils::impl::comm_size(MPI_COMM_WORLD); + if (!(commSize == 4 || commSize == 8)) + GTEST_SKIP(); + + + mesh::impl::MeshSpec spec1, spec2; + spec1.xmin = 0; spec2.xmin = 0; + spec1.xmax = 1; spec2.xmax = 1; + spec1.ymin = 0; spec2.ymin = 0; + spec1.ymax = 1; spec2.ymax = 1; + spec1.numelX = 2; spec2.numelX = 4; + spec1.numelY = 2; spec2.numelY = 4; + + SplitCommTestUtil splitter(commSize/2, commSize/2); + + setup_mpmd(spec1, spec2, splitter); + run_tests(); +} \ No newline at end of file diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/test_parallel_search.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/test_parallel_search.cpp index c0ebf14edb5d..f6a961f49bdf 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/test_parallel_search.cpp +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/test_parallel_search.cpp @@ -36,6 +36,8 @@ #include "util/parallel_search_test_util.hpp" #include "stk_middle_mesh/mesh_scatter_spec.hpp" +#include "stk_middle_mesh/bounding_box_search.hpp" + namespace { @@ -97,8 +99,8 @@ TEST(SearchBoundingBox, CreateBoundingBox) auto elements = mesh->get_elements(); EXPECT_EQ(1u, elements.size()); - std::vector boundingBoxVec; - stk::middle_mesh::mesh::impl::SearchMesh meshA(mesh); + std::vector boundingBoxVec; + stk::middle_mesh::mesh::impl::SearchMeshElementBoundingBox meshA(mesh, MPI_COMM_WORLD); meshA.fill_bounding_boxes(boundingBoxVec); EXPECT_EQ(1u, boundingBoxVec.size()); @@ -112,6 +114,7 @@ TEST(SearchBoundingBox, CreateBoundingBox) class ParallelSearch : public ::testing::Test { protected: + using BoundingBoxSearch = stk::middle_mesh::search::ElementToElementBoundingBoxSearch; ParallelSearch() {} void setup_comm() @@ -141,22 +144,22 @@ class ParallelSearch : public ::testing::Test { auto func = [&](const stk::middle_mesh::utils::Point& pt) { return stk::middle_mesh::utils::Point(pt.x, pt.y, 0); }; - if (m_color == stk::middle_mesh::mesh::impl::SplitCommColor::SEND) { + if (m_color == stk::middle_mesh::search::SplitCommColor::SEND) { m_mesh = stk::middle_mesh::mesh::impl::create_mesh(m_sendMeshSpec, func, m_splitComm, false); - } else if (m_color == stk::middle_mesh::mesh::impl::SplitCommColor::RECV) { + } else if (m_color == stk::middle_mesh::search::SplitCommColor::RECV) { m_mesh = stk::middle_mesh::mesh::impl::create_mesh(m_recvMeshSpec, func, m_splitComm, false); } } void setup_search() { - if (m_color == stk::middle_mesh::mesh::impl::SplitCommColor::SEND) { - m_searchSendMesh = std::make_shared(m_mesh); - m_search = std::make_shared( + if (m_color == stk::middle_mesh::search::SplitCommColor::SEND) { + m_searchSendMesh = std::make_shared(m_mesh, MPI_COMM_WORLD); + m_search = std::make_shared( m_searchSendMesh, m_searchRecvMesh, "BoundingBoxSearch", MPI_COMM_WORLD); - } else if (m_color == stk::middle_mesh::mesh::impl::SplitCommColor::RECV) { - m_searchRecvMesh = std::make_shared(m_mesh); - m_search = std::make_shared( + } else if (m_color == stk::middle_mesh::search::SplitCommColor::RECV) { + m_searchRecvMesh = std::make_shared(m_mesh, MPI_COMM_WORLD); + m_search = std::make_shared( m_searchSendMesh, m_searchRecvMesh, "BoundingBoxSearch", MPI_COMM_WORLD); } else { STK_ThrowRequireMsg(false, "Invalid SplitComm color"); @@ -182,10 +185,10 @@ class ParallelSearch : public ::testing::Test { int myRankWorld; MPI_Comm_rank(MPI_COMM_WORLD, &myRankWorld); - const stk::middle_mesh::mesh::impl::SearchRelationVec& pairedEntities = m_search->get_range_to_domain(); - const stk::middle_mesh::mesh::impl::UnpairedRelationVec& unpairedEntities = m_search->get_unpaired_recv_entities(); + const BoundingBoxSearch::EntityProcRelationVec& pairedEntities = m_search->get_range_to_domain(); + const std::vector& unpairedEntities = m_search->get_unpaired_recv_entities(); - if(m_color == stk::middle_mesh::mesh::impl::SplitCommColor::SEND) { + if(m_color == stk::middle_mesh::search::SplitCommColor::SEND) { EXPECT_EQ(expectedNumSearchPairs, pairedEntities.size()); EXPECT_EQ(0u, unpairedEntities.size()); } @@ -193,10 +196,10 @@ class ParallelSearch : public ::testing::Test { unsigned expectedGlobalRank = 0; for(const auto &pairedEntity : pairedEntities) { - const stk::middle_mesh::mesh::impl::SearchMesh::EntityProc& recvEntityProc = pairedEntity.first; - const stk::middle_mesh::mesh::impl::SearchMesh::EntityProc& sendEntityProc = pairedEntity.second; + const stk::middle_mesh::mesh::impl::SearchMeshElementBoundingBox::EntityProc& recvEntityProc = pairedEntity.first; + const stk::middle_mesh::mesh::impl::SearchMeshElementBoundingBox::EntityProc& sendEntityProc = pairedEntity.second; - if(m_color == stk::middle_mesh::mesh::impl::SplitCommColor::RECV) { + if(m_color == stk::middle_mesh::search::SplitCommColor::RECV) { bool isGhost = myRankWorld != (int)recvEntityProc.proc(); if(!isGhost) { @@ -212,16 +215,16 @@ class ParallelSearch : public ::testing::Test { void fill_mesh_scatter(stk::middle_mesh::mesh::impl::MeshScatterSpec& scatter) { - const stk::middle_mesh::mesh::impl::SearchRelationVec& pairedEntities = m_search->get_range_to_domain(); - const stk::middle_mesh::mesh::impl::UnpairedRelationVec& unpairedEntities = m_search->get_unpaired_recv_entities(); + const BoundingBoxSearch::EntityProcRelationVec& pairedEntities = m_search->get_range_to_domain(); + const std::vector& unpairedEntities = m_search->get_unpaired_recv_entities(); EXPECT_EQ(0u, unpairedEntities.size()); - if(m_color == stk::middle_mesh::mesh::impl::SplitCommColor::SEND) { + if(m_color == stk::middle_mesh::search::SplitCommColor::SEND) { auto elements = m_search->send_mesh()->get_mesh()->get_elements(); for(const auto &pairedEntity : pairedEntities) { - const stk::middle_mesh::mesh::impl::SearchMesh::EntityProc& recvEntityProc = pairedEntity.first; - const stk::middle_mesh::mesh::impl::SearchMesh::EntityProc& sendEntityProc = pairedEntity.second; + const stk::middle_mesh::mesh::impl::SearchMeshElementBoundingBox::EntityProc& recvEntityProc = pairedEntity.first; + const stk::middle_mesh::mesh::impl::SearchMeshElementBoundingBox::EntityProc& sendEntityProc = pairedEntity.second; int globalRank = recvEntityProc.proc(); int splitRank = m_splitCommUtil->get_local_comm_rank(globalRank); @@ -237,19 +240,19 @@ class ParallelSearch : public ::testing::Test { int m_numProcsSendMesh{0}; int m_numProcsRecvMesh{0}; - stk::middle_mesh::mesh::impl::SplitCommColor m_color{stk::middle_mesh::mesh::impl::SplitCommColor::INVALID}; + stk::middle_mesh::search::SplitCommColor m_color{stk::middle_mesh::search::SplitCommColor::INVALID}; MPI_Comm m_splitComm{MPI_COMM_NULL}; stk::middle_mesh::mesh::impl::MeshSpec m_sendMeshSpec; stk::middle_mesh::mesh::impl::MeshSpec m_recvMeshSpec; - std::shared_ptr m_search; + std::shared_ptr m_search; std::shared_ptr m_mesh; - std::shared_ptr m_searchSendMesh; - std::shared_ptr m_searchRecvMesh; + std::shared_ptr m_searchSendMesh; + std::shared_ptr m_searchRecvMesh; std::shared_ptr m_splitCommUtil; }; diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/util/nonconformal_interface_helpers.cpp b/packages/stk/stk_unit_tests/stk_middle_mesh/util/nonconformal_interface_helpers.cpp index 1aa6c8eb7faa..1bfe632d511f 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/util/nonconformal_interface_helpers.cpp +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/util/nonconformal_interface_helpers.cpp @@ -38,18 +38,38 @@ void test_areas_positive(std::shared_ptr mesh) } } + + + +double compute_mesh_area(std::shared_ptr mesh, MPI_Comm unionComm) +{ + double area = 0; + double areaLocal = 0; + + if (mesh) + { + mesh::impl::ElementOperations2D elemOps; + areaLocal = elemOps.compute_area(mesh); + } + + MPI_Allreduce(&areaLocal, &area, 1, MPI_DOUBLE, MPI_SUM, unionComm); + + return area; +} + double compute_mesh_area(std::shared_ptr mesh) { - mesh::impl::ElementOperations2D elemOps; - return elemOps.compute_area(mesh); + return compute_mesh_area(mesh, mesh->get_comm()); } void test_total_areas_same(std::shared_ptr mesh1, std::shared_ptr mesh2, - std::shared_ptr meshIn) + std::shared_ptr meshIn, MPI_Comm unionComm) { - double area1 = compute_mesh_area(mesh1); - double area2 = compute_mesh_area(mesh2); - double areaIn = compute_mesh_area(meshIn); + double area1 = 0, area2 = 2, areaIn = 0; + area1 = compute_mesh_area(mesh1, unionComm); + area2 = compute_mesh_area(mesh2, unionComm); + areaIn = compute_mesh_area(meshIn, unionComm); + EXPECT_NEAR(area1, area2, 1e-13); EXPECT_NEAR(areaIn, area1, 1e-13); diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/util/nonconformal_interface_helpers.hpp b/packages/stk/stk_unit_tests/stk_middle_mesh/util/nonconformal_interface_helpers.hpp index 9f14f821b405..5c9eca2c46eb 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/util/nonconformal_interface_helpers.hpp +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/util/nonconformal_interface_helpers.hpp @@ -12,10 +12,12 @@ void test_area_per_element(std::shared_ptr mesh, void test_areas_positive(std::shared_ptr mesh); +double compute_mesh_area(std::shared_ptr mesh, MPI_Comm unionComm); + double compute_mesh_area(std::shared_ptr mesh); void test_total_areas_same(std::shared_ptr mesh1, std::shared_ptr mesh2, - std::shared_ptr meshIn); + std::shared_ptr meshIn, MPI_Comm unionComm=MPI_COMM_WORLD); void test_number_of_elements(std::shared_ptr mesh, int numel); diff --git a/packages/stk/stk_unit_tests/stk_middle_mesh/util/parallel_search_test_util.hpp b/packages/stk/stk_unit_tests/stk_middle_mesh/util/parallel_search_test_util.hpp index bec7b0a6ff14..8c46c8e40c0d 100644 --- a/packages/stk/stk_unit_tests/stk_middle_mesh/util/parallel_search_test_util.hpp +++ b/packages/stk/stk_unit_tests/stk_middle_mesh/util/parallel_search_test_util.hpp @@ -35,11 +35,11 @@ #ifndef STK_MIDDLE_MESH_PARALLEL_SEARCH_TEST_UTIL_HPP #define STK_MIDDLE_MESH_PARALLEL_SEARCH_TEST_UTIL_HPP -#include "stk_middle_mesh/parallel_search.hpp" +#include "stk_middle_mesh/bounding_box_search.hpp" namespace { -using namespace stk::middle_mesh::mesh::impl; +using namespace stk::middle_mesh::search; class SplitCommTestUtil { diff --git a/packages/stk/stk_unit_tests/stk_ngp_test/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_ngp_test/CMakeLists.txt index 1ee95dd2b8d7..f37e0a72b3a9 100644 --- a/packages/stk/stk_unit_tests/stk_ngp_test/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_ngp_test/CMakeLists.txt @@ -32,13 +32,10 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - FILE(GLOB SOURCES *.cpp) -TRIBITS_ADD_EXECUTABLE_AND_TEST( - stk_ngp_test_unit_tests +if(HAVE_STK_Trilinos) + TRIBITS_ADD_EXECUTABLE_AND_TEST(stk_ngp_test_unit_tests SOURCES ${SOURCES} TESTONLYLIBS stk_unit_main ARGS "--gtest_filter=-*Device*" @@ -46,4 +43,13 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST( NUM_MPI_PROCS 1-4 NOEXEPREFIX ) +else() + add_executable(stk_ngp_test_unit_tests ${SOURCES}) + target_link_libraries(stk_ngp_test_unit_tests stk_unit_test_utils) + target_link_libraries(stk_ngp_test_unit_tests stk_util_parallel) + target_link_libraries(stk_ngp_test_unit_tests stk_util_ngp) + target_link_libraries(stk_ngp_test_unit_tests stk_unit_main) + + add_test(NAME "stk_ngp_test_unit_tests" COMMAND stk_ngp_test_unit_tests) +endif() diff --git a/packages/stk/stk_unit_tests/stk_search/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_search/CMakeLists.txt index 8bb5c05d4df5..5ee4b571ac61 100644 --- a/packages/stk/stk_unit_tests/stk_search/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_search/CMakeLists.txt @@ -32,24 +32,31 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util/stk_util) - FILE(GLOB SOURCES *.cpp) -TRIBITS_ADD_EXECUTABLE( - stk_search_utest +if(HAVE_STK_Trilinos) + TRIBITS_ADD_EXECUTABLE(stk_search_utest SOURCES ${SOURCES} TESTONLYLIBS stk_unit_main COMM serial mpi NOEXEPREFIX - ) + ) -TRIBITS_ADD_TEST( - stk_search_utest + TRIBITS_ADD_TEST(stk_search_utest ARGS "" COMM serial mpi PASS_REGULAR_EXPRESSION "PASS" FAIL_REGULAR_EXPRESSION "FAIL" NUM_MPI_PROCS 1 NOEXEPREFIX -) + ) +else() + add_executable(stk_search_utest ${SOURCES}) + target_link_libraries(stk_search_utest stk_search) + target_link_libraries(stk_search_utest stk_unit_test_utils) + target_link_libraries(stk_search_utest stk_util_diag) + target_link_libraries(stk_search_utest stk_util_parallel) + target_link_libraries(stk_search_utest stk_unit_main) + + add_test(NAME "stk_search_utest" COMMAND stk_search_utest) +endif() diff --git a/packages/stk/stk_unit_tests/stk_search/UnitTestFilterCoarseSearch.cpp b/packages/stk/stk_unit_tests/stk_search/UnitTestFilterCoarseSearch.cpp index af28c0ecfbc8..74a6d0de92cc 100644 --- a/packages/stk/stk_unit_tests/stk_search/UnitTestFilterCoarseSearch.cpp +++ b/packages/stk/stk_unit_tests/stk_search/UnitTestFilterCoarseSearch.cpp @@ -165,10 +165,12 @@ TEST_F(SearchFilterTester, checkEarlyGeometricMatch) // mix of points within and out of parametric and geo tol add_coarse_results({ 2.5, 1.1, 2.0, 0.5 }, { 2.0, 0.2, 2.5, 0.0 }); + auto extrapolateOption = stk::search::ObjectOutsideDomainPolicy::ABORT; + stk::search::impl::FilterCoarseSearchStats stats = stk::search::impl::filter_coarse_search_by_range(coarse_results, sendMesh, recvMesh, false, false, - sendMesh.get_extrapolate_option(), + extrapolateOption, filteredSearchResults); EXPECT_EQ(1u, stats.numEntitiesWithinTolerance); @@ -189,10 +191,12 @@ TEST_F(SearchFilterTester, checkAllGeometric) // all points outside parametric tol - pick closest geo add_coarse_results({ 2.5, 1.6, 2.0, 3.5 }, { 2.0, 0.2, 2.5, 0.7 }); + auto extrapolateOption = stk::search::ObjectOutsideDomainPolicy::ABORT; + stk::search::impl::FilterCoarseSearchStats stats = stk::search::impl::filter_coarse_search_by_range(coarse_results, sendMesh, recvMesh, false, false, - sendMesh.get_extrapolate_option(), + extrapolateOption, filteredSearchResults); EXPECT_EQ(1u, stats.numEntitiesWithinTolerance); @@ -213,10 +217,12 @@ TEST_F(SearchFilterTester, checkAllOutsideTolerance) // all points outside both parametric and geometric tolerance add_coarse_results({ 2.5, 1.6, 2.0, 3.5 }, { 2.0, 0.75, 2.5, 0.7 }); + auto extrapolateOption = stk::search::ObjectOutsideDomainPolicy::ABORT; + stk::search::impl::FilterCoarseSearchStats stats = stk::search::impl::filter_coarse_search_by_range(coarse_results, sendMesh, recvMesh, false, false, - sendMesh.get_extrapolate_option(), + extrapolateOption, filteredSearchResults); EXPECT_EQ(0u, stats.numEntitiesWithinTolerance); diff --git a/packages/stk/stk_unit_tests/stk_simd/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_simd/CMakeLists.txt index 52022fdd9785..eaaf0e067b1b 100644 --- a/packages/stk/stk_unit_tests/stk_simd/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_simd/CMakeLists.txt @@ -1,27 +1,36 @@ - -ASSERT_DEFINED(PACKAGE_SOURCE_DIR CMAKE_CURRENT_SOURCE_DIR) - -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_simd) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}/stk_simd) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_BINARY_DIR}/stk_simd) - FILE(GLOB SOURCES *.cpp) -TRIBITS_ADD_EXECUTABLE( - stk_simd_unit_tests +if(HAVE_STK_Trilinos) + message("in stk_unit_tests/stk_simd, PACKAGE_NAME: ${PACKAGE_NAME}, PARENT_PACKAGE_NAME: ${PARENT_PACKAGE_NAME}") + TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_simd) + TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}/stk_simd) + + TRIBITS_ADD_EXECUTABLE(stk_simd_unit_tests SOURCES ${SOURCES} TESTONLYLIBS stk_unit_main COMM serial mpi NOEXEPREFIX ) -TRIBITS_ADD_TEST( - stk_simd_unit_tests + TRIBITS_ADD_TEST(stk_simd_unit_tests COMM serial mpi PASS_REGULAR_EXPRESSION "PASS" FAIL_REGULAR_EXPRESSION "FAIL" NUM_MPI_PROCS 1 NOEXEPREFIX ) +else() + add_executable(stk_simd_unit_tests ${SOURCES}) + + target_include_directories(stk_simd_unit_tests PRIVATE + $ + $ + ) + + target_link_libraries(stk_simd_unit_tests stk_simd) + target_link_libraries(stk_simd_unit_tests stk_unit_test_utils) + target_link_libraries(stk_simd_unit_tests stk_unit_main) +endif() + diff --git a/packages/stk/stk_unit_tests/stk_simd_old/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_simd_old/CMakeLists.txt index aa00e491b238..d767d804dd42 100644 --- a/packages/stk/stk_unit_tests/stk_simd_old/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_simd_old/CMakeLists.txt @@ -1,24 +1,20 @@ - -ASSERT_DEFINED(PACKAGE_SOURCE_DIR CMAKE_CURRENT_SOURCE_DIR) - -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_simd) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}/stk_simd) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_BINARY_DIR}/stk_simd) - FILE(GLOB SOURCES *.cpp) -TRIBITS_ADD_EXECUTABLE( - simd_old_unit_tests +if(HAVE_STK_Trilinos) + message("in stk_unit_tests/stk_simd, PACKAGE_NAME: ${PACKAGE_NAME}, PARENT_PACKAGE_NAME: ${PARENT_PACKAGE_NAME}") + TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_simd) + TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}/stk_simd) + + TRIBITS_ADD_EXECUTABLE(stk_simd_old_unit_tests SOURCES ${SOURCES} TESTONLYLIBS stk_unit_main COMM serial mpi NOEXEPREFIX ) -TRIBITS_ADD_TEST( - simd_old_unit_tests + TRIBITS_ADD_TEST(stk_simd_old_unit_tests ARGS "--gtest_filter=StkSimd.SimdAddSubtractMultDivide" COMM serial mpi PASS_REGULAR_EXPRESSION "PASS" @@ -26,3 +22,17 @@ TRIBITS_ADD_TEST( NUM_MPI_PROCS 1 NOEXEPREFIX ) +else() + add_executable(stk_simd_old_unit_tests ${SOURCES}) + + target_include_directories(stk_simd_old_unit_tests PRIVATE + $ + $ + ) + + target_link_libraries(stk_simd_old_unit_tests stk_simd) + target_link_libraries(stk_simd_old_unit_tests stk_math) + target_link_libraries(stk_simd_old_unit_tests stk_unit_test_utils) + target_link_libraries(stk_simd_old_unit_tests stk_unit_main) +endif() + diff --git a/packages/stk/stk_unit_tests/stk_tools/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_tools/CMakeLists.txt index e7b3e6f0e384..63be18948481 100644 --- a/packages/stk/stk_unit_tests/stk_tools/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_tools/CMakeLists.txt @@ -32,7 +32,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util/stk_util) TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_tools) FILE(GLOB SOURCES mesh_clone/*.cpp mesh_tools/*.cpp block_extractor/*.cpp) diff --git a/packages/stk/stk_unit_tests/stk_topology/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_topology/CMakeLists.txt index f804abe8bb6e..d8562ee8b3a7 100644 --- a/packages/stk/stk_unit_tests/stk_topology/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_topology/CMakeLists.txt @@ -1,12 +1,8 @@ -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util/stk_util) - FILE(GLOB SOURCES *.cpp) -TRIBITS_ADD_EXECUTABLE_AND_TEST( - stk_topology_unit_tests +if(HAVE_STK_Trilinos) + TRIBITS_ADD_EXECUTABLE_AND_TEST(stk_topology_unit_tests SOURCES ${SOURCES} TESTONLYLIBS stk_unit_main ARGS "" @@ -14,4 +10,20 @@ TRIBITS_ADD_EXECUTABLE_AND_TEST( NUM_MPI_PROCS 1 NOEXEPREFIX ) +else() + add_executable(stk_topology_unit_tests ${SOURCES}) + target_link_libraries(stk_topology_unit_tests stk_topology) + target_link_libraries(stk_topology_unit_tests stk_unit_test_utils) + target_link_libraries(stk_topology_unit_tests stk_util_env) + target_link_libraries(stk_topology_unit_tests stk_util_ngp) + target_link_libraries(stk_topology_unit_tests stk_util_util) + target_link_libraries(stk_topology_unit_tests stk_unit_main) + + add_test(NAME "stk_topology_unit_tests" COMMAND stk_topology_unit_tests) +endif() + +target_include_directories(stk_topology_unit_tests PUBLIC + $ + $ +) diff --git a/packages/stk/stk_unit_tests/stk_transfer/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_transfer/CMakeLists.txt index 38192a3681a5..271d98084d48 100644 --- a/packages/stk/stk_unit_tests/stk_transfer/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_transfer/CMakeLists.txt @@ -32,27 +32,21 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_BINARY_DIR}/stk_util/stk_util) -TRIBITS_INCLUDE_DIRECTORIES(${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/stk_transfer) - FILE(GLOB SOURCES *.cpp) -IF(NOT STK_ENABLE_STKMiddle_mesh) -LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestConservativeTransfer.cpp") -ENDIF() +if(NOT STK_ENABLE_STKMiddle_mesh) + LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/UnitTestConservativeTransfer.cpp") +endif() -TRIBITS_ADD_EXECUTABLE( - stk_transfer_utest +if(HAVE_STK_Trilinos) + TRIBITS_ADD_EXECUTABLE(stk_transfer_utest SOURCES ${SOURCES} TESTONLYLIBS stk_unit_main COMM serial mpi NOEXEPREFIX - ) + ) -TRIBITS_ADD_TEST( - stk_transfer_utest + TRIBITS_ADD_TEST(stk_transfer_utest ARGS "--gtest_filter=StkMeshHowTo.useCopyTransfer" COMM serial mpi NUM_MPI_PROCS 2 @@ -60,4 +54,13 @@ TRIBITS_ADD_TEST( FAIL_REGULAR_EXPRESSION "FAIL" NOEXEPREFIX ) +else() + add_executable(stk_transfer_utest ${SOURCES}) + target_link_libraries(stk_transfer_utest stk_transfer) + target_link_libraries(stk_transfer_utest stk_unit_test_utils) + target_link_libraries(stk_transfer_utest stk_util_parallel) + target_link_libraries(stk_transfer_utest stk_unit_main) + + add_test(NAME "stk_transfer_utest" COMMAND stk_transfer_utest) +endif() diff --git a/packages/stk/stk_unit_tests/stk_util/CMakeLists.txt b/packages/stk/stk_unit_tests/stk_util/CMakeLists.txt index fe5dc3e39d20..bbe5d6e4c10f 100644 --- a/packages/stk/stk_unit_tests/stk_util/CMakeLists.txt +++ b/packages/stk/stk_unit_tests/stk_util/CMakeLists.txt @@ -32,7 +32,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # +if(HAVE_STK_Trilinos) ASSERT_DEFINED(PACKAGE_SOURCE_DIR CMAKE_CURRENT_SOURCE_DIR) +endif() FILE(GLOB SOURCES *.cpp command_line/*.cpp @@ -43,7 +45,7 @@ FILE(GLOB SOURCES *.cpp registry/*.cpp util/*.cpp) -IF (NOT TPL_ENABLE_MPI) +IF (NOT STK_HAS_MPI) LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/parallel/UnitTestDistributedIndex.cpp") LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/parallel/UnitTestGenerateParallelConsistentIDs.cpp") LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/parallel/UnitTestGenerateUniqueConsistentIDs.cpp") @@ -58,15 +60,23 @@ LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/parallel/UnitTestImprobeSe LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/parallel/UnitTestMPIFinalizationCallback.cpp") ENDIF() -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_BINARY_DIR}/stk_util) - -TRIBITS_ADD_EXECUTABLE_AND_TEST( - stk_util_unit_tests +if(HAVE_STK_Trilinos) + TRIBITS_ADD_EXECUTABLE_AND_TEST(stk_util_unit_tests SOURCES ${SOURCES} TESTONLYLIBS stk_unit_main COMM serial mpi NUM_MPI_PROCS 1-6 NOEXEPREFIX - ) - + ) +else() + add_executable(stk_util_unit_tests ${SOURCES}) + target_link_libraries(stk_util_unit_tests stk_unit_test_utils) + target_link_libraries(stk_util_unit_tests stk_util_registry) + target_link_libraries(stk_util_unit_tests stk_util_diag) + target_link_libraries(stk_util_unit_tests stk_util_env) + target_link_libraries(stk_util_unit_tests stk_util_parallel) + target_link_libraries(stk_util_unit_tests stk_util_command_line) + target_link_libraries(stk_util_unit_tests stk_unit_main) + add_test(NAME "stk_util_unit_tests" COMMAND stk_util_unit_tests) +endif() diff --git a/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestCommSparse.cpp b/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestCommSparse.cpp index 2e89ee6411fd..b7480684761f 100644 --- a/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestCommSparse.cpp +++ b/packages/stk/stk_unit_tests/stk_util/parallel/UnitTestCommSparse.cpp @@ -44,72 +44,6 @@ #include // for basic_ostream::operator<<, operator<<, bas... #include // for vector -#ifndef STK_HIDE_DEPRECATED_CODE // delete after August 2022 -TEST(ParallelComm, comm_recv_msg_sizes) -{ - MPI_Comm comm = MPI_COMM_WORLD; - int numProcs = stk::parallel_machine_size(comm); - if (numProcs == 1) { - GTEST_SKIP(); - } - int myProc = stk::parallel_machine_rank(comm); - - std::vector send_bufs(numProcs), recv_bufs(numProcs); - std::vector send_procs, recv_procs; - - for(int p=myProc-2; p<=myProc+2; ++p) { - if (p >= 0 && p < numProcs) { - send_bufs[p].set_size(myProc+1); - send_procs.push_back(p); - recv_procs.push_back(p); - } - } - - stk::comm_recv_msg_sizes(comm, send_procs, recv_procs, send_bufs, recv_bufs); - - for(int p=0; p (myProc+2)) { - EXPECT_EQ(0u, recv_bufs[p].size()); - } - else { - EXPECT_EQ((unsigned)p+1, recv_bufs[p].size()); - } - } -} - -TEST(ParallelComm, comm_recv_procs_and_msg_sizes) -{ - MPI_Comm comm = MPI_COMM_WORLD; - int numProcs = stk::parallel_machine_size(comm); - if (numProcs == 1) { - GTEST_SKIP(); - } - int myProc = stk::parallel_machine_rank(comm); - - std::vector send_bufs(numProcs), recv_bufs(numProcs); - - for(int p=myProc-2; p<=myProc+2; ++p) { - if (p >= 0 && p < numProcs) { - send_bufs[p].set_size(myProc+1); - } - } - - std::vector send_procs, recv_procs; - stk::comm_recv_procs_and_msg_sizes(comm, send_bufs, recv_bufs, send_procs, recv_procs); - - for(int p=0; p (myProc+2)) { - EXPECT_EQ(0u, recv_bufs[p].size()); - } - else { - unsigned expected = p+1; - STK_ThrowRequireMsg( recv_bufs[p].size() == expected, "proc "< + $ +) INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_util/command_line/) +IF(NOT HAVE_STK_Trilinos) + INSTALL(TARGETS stk_util_command_line DESTINATION ${STK_INSTALL_LIBDIR}) +ENDIF() diff --git a/packages/stk/stk_util/stk_util/diag/CMakeLists.txt b/packages/stk/stk_util/stk_util/diag/CMakeLists.txt index 247c6f997ae8..1c11624d638d 100644 --- a/packages/stk/stk_util/stk_util/diag/CMakeLists.txt +++ b/packages/stk/stk_util/stk_util/diag/CMakeLists.txt @@ -32,48 +32,32 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -# -# A) Package-specific configuration options -# - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - SET(HEADERS "") SET(SOURCES "") -# -# Core headers -# - -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) - FILE(GLOB HEADERS *.hpp *.h) -#removing references to mpih which is not available in Trilinos -LIST(REMOVE_ITEM HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/mpih.hpp") - FILE(GLOB SOURCES *.cpp) -#removing references to mpih which is not available in Trilinos -LIST(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/mpih.cpp") -LIST(REMOVE_ITEM HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/Exception.hpp") -# -# C) Define the targets for package's library(s) -# - -TRIBITS_ADD_LIBRARY( - stk_util_diag - DEPLIBS stk_util_env - NOINSTALLHEADERS ${HEADERS} - SOURCES ${SOURCES} +IF(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY( + stk_util_diag + NOINSTALLHEADERS ${HEADERS} + SOURCES ${SOURCES} ) +ELSE() + add_library(stk_util_diag ${SOURCES}) + + target_link_libraries(stk_util_diag PUBLIC stk_util_env) +ENDIF() +target_include_directories(stk_util_diag PUBLIC + $ + $ +) INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_util/diag/) +IF(NOT HAVE_STK_Trilinos) + INSTALL(TARGETS stk_util_diag DESTINATION ${STK_INSTALL_LIBDIR}) +ENDIF() diff --git a/packages/stk/stk_util/stk_util/environment/CMakeLists.txt b/packages/stk/stk_util/stk_util/environment/CMakeLists.txt index faacac0b7e37..9f1f4aab5e1d 100644 --- a/packages/stk/stk_util/stk_util/environment/CMakeLists.txt +++ b/packages/stk/stk_util/stk_util/environment/CMakeLists.txt @@ -32,42 +32,31 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - -# -# A) Package-specific configuration options -# - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - SET(HEADERS "") SET(SOURCES "") -# -# Core headers -# - FILE(GLOB HEADERS *.hpp *.h) FILE(GLOB SOURCES *.cpp) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) +IF(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY(stk_util_env + NOINSTALLHEADERS ${HEADERS} + SOURCES ${SOURCES} + ) +ELSE() + add_library(stk_util_env ${SOURCES}) -# -# C) Define the targets for package's library(s) -# + target_link_libraries(stk_util_env PUBLIC stk_util_util) +ENDIF() -TRIBITS_ADD_LIBRARY( - stk_util_env - DEPLIBS stk_util_util stk_util_parallel - NOINSTALLHEADERS ${HEADERS} - SOURCES ${SOURCES} - DEPLIBS stk_util_util - ) +target_include_directories(stk_util_env PUBLIC + $ + $ +) INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_util/environment/) + +IF(NOT HAVE_STK_Trilinos) + INSTALL(TARGETS stk_util_env DESTINATION ${STK_INSTALL_LIBDIR}) +ENDIF() diff --git a/packages/stk/stk_util/stk_util/environment/EnvData.cpp b/packages/stk/stk_util/stk_util/environment/EnvData.cpp index 20bf43fac9e2..b09aff4f1650 100644 --- a/packages/stk/stk_util/stk_util/environment/EnvData.cpp +++ b/packages/stk/stk_util/stk_util/environment/EnvData.cpp @@ -90,6 +90,14 @@ namespace stk { static_cast(sierra::dwout().rdbuf())->redirect(sierra::dout().rdbuf()); } + void EnvData::initialize(MPI_Comm worldComm) + { + m_worldComm = worldComm; + m_parallelComm = worldComm; + m_parallelSize = stk::parallel_machine_size(m_parallelComm); + m_parallelRank = stk::parallel_machine_rank(m_parallelComm); + } + EnvData::~EnvData() { static_cast(sierra::dwout().rdbuf())->redirect(std::cout.rdbuf()); diff --git a/packages/stk/stk_util/stk_util/environment/EnvData.hpp b/packages/stk/stk_util/stk_util/environment/EnvData.hpp index 97b9cbd0f3ed..21ba461baa46 100644 --- a/packages/stk/stk_util/stk_util/environment/EnvData.hpp +++ b/packages/stk/stk_util/stk_util/environment/EnvData.hpp @@ -82,6 +82,8 @@ struct EnvData static EnvData &instance(); EnvData(); + void initialize(MPI_Comm worldComm); + ~EnvData(); static void setInputFileName(std::string value); diff --git a/packages/stk/stk_util/stk_util/environment/RuntimeMessage.cpp b/packages/stk/stk_util/stk_util/environment/RuntimeMessage.cpp index c847484d305e..3f0b7b0888f5 100644 --- a/packages/stk/stk_util/stk_util/environment/RuntimeMessage.cpp +++ b/packages/stk/stk_util/stk_util/environment/RuntimeMessage.cpp @@ -33,6 +33,7 @@ // #include "stk_util/environment/RuntimeMessage.hpp" +#include "stk_util/environment/Env.hpp" #include "stk_util/parallel/Parallel.hpp" // for parallel_machine_rank, MPI_Gather, MPI_Gatherv #include "stk_util/stk_config.h" // for STK_HAS_MPI #include "stk_util/util/Bootstrap.hpp" // for Bootstrap @@ -186,7 +187,7 @@ Marshal &operator>>(Marshal &min, DeferredMessage &s) { bool should_increment_message_count(unsigned messageType) { - return !(messageType & MSG_SYMMETRIC) || (stk::parallel_machine_rank(MPI_COMM_WORLD) == 0); + return !(messageType & MSG_SYMMETRIC) || (stk::parallel_machine_rank(sierra::Env::parallel_comm()) == 0); } CutoffStatus diff --git a/packages/stk/stk_util/stk_util/ngp/CMakeLists.txt b/packages/stk/stk_util/stk_util/ngp/CMakeLists.txt index 7ef4509f30b0..4958f365f4b9 100644 --- a/packages/stk/stk_util/stk_util/ngp/CMakeLists.txt +++ b/packages/stk/stk_util/stk_util/ngp/CMakeLists.txt @@ -32,25 +32,21 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - -# -# A) Package-specific configuration options -# - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - SET(HEADERS "") SET(SOURCES "") FILE(GLOB HEADERS *.hpp *.h) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) +IF(HAVE_STK_Trilinos) + TRIBITS_INCLUDE_DIRECTORIES(${STK_TOPLEVEL_SOURCE_DIR}/stk_util) +ELSE() + add_library(stk_util_ngp INTERFACE ${HEADERS}) + + target_include_directories(stk_util_ngp INTERFACE + $ + $ + ) +ENDIF() INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_util/ngp/) diff --git a/packages/stk/stk_util/stk_util/parallel/CMakeLists.txt b/packages/stk/stk_util/stk_util/parallel/CMakeLists.txt index 2fa5e307739b..4c6c229c8e19 100644 --- a/packages/stk/stk_util/stk_util/parallel/CMakeLists.txt +++ b/packages/stk/stk_util/stk_util/parallel/CMakeLists.txt @@ -32,26 +32,10 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - -# -# A) Package-specific configuration options -# - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - SET(HEADERS "") SET(SOURCES "") -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) - - -IF (TPL_ENABLE_MPI) +IF (STK_HAS_MPI) FILE(GLOB HEADERS *.hpp *.h) FILE(GLOB SOURCES *.cpp) @@ -77,16 +61,32 @@ FILE(GLOB SOURCES ENDIF() -# -# C) Define the targets for package's library(s) -# - -TRIBITS_ADD_LIBRARY( - stk_util_parallel - NOINSTALLHEADERS ${HEADERS} - SOURCES ${SOURCES} - DEPLIBS stk_util_util +IF(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY(stk_util_parallel + NOINSTALLHEADERS ${HEADERS} + SOURCES ${SOURCES} + DEPLIBS stk_util_util ) +ELSE() + add_library(stk_util_parallel ${SOURCES}) + + IF(STK_HAS_MPI) + target_link_libraries(stk_util_parallel PUBLIC ${MPI_LIBRARIES}) + message("MPI_C_LIBRARIES: ${MPI_C_LIBRARIES}") + message("MPI_LIBRARIES: ${MPI_LIBRARIES}") + ENDIF() + + target_link_libraries(stk_util_parallel PUBLIC stk_util_util) +ENDIF() + +target_include_directories(stk_util_parallel PUBLIC + $ + $ +) INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_util/parallel/) + +IF(NOT HAVE_STK_Trilinos) + INSTALL(TARGETS stk_util_parallel DESTINATION ${STK_INSTALL_LIBDIR}) +ENDIF() diff --git a/packages/stk/stk_util/stk_util/parallel/CommSparse.cpp b/packages/stk/stk_util/stk_util/parallel/CommSparse.cpp index c4e3374cef35..ab6bcf02f025 100644 --- a/packages/stk/stk_util/stk_util/parallel/CommSparse.cpp +++ b/packages/stk/stk_util/stk_util/parallel/CommSparse.cpp @@ -347,8 +347,9 @@ bool CommSparse::communicate(bool deallocateSendBuffers) if (stk::util::get_common_coupling_version() >= 6) { if (m_exchanger) { - auto f = [](int rank, stk::CommBuffer& buf) {}; - communicate_with_unpacker(f); + auto noExtraWork = [](){}; + auto noUnpacker = [](int rank, stk::CommBuffer& buf) {}; + communicate_with_extra_work_and_unpacker(noExtraWork, noUnpacker, false); } } else { verify_send_buffers_filled(); @@ -374,7 +375,10 @@ bool CommSparse::communicate(bool deallocateSendBuffers) return returnValue; } -void CommSparse::communicate_with_unpacker(const std::function& functor) +void CommSparse::communicate_with_extra_work_and_unpacker( + const std::function& workFunctor, + const std::function& unpackFunctor, + bool deallocateSendBuffers) { #ifdef STK_HAS_MPI stk::util::print_unsupported_version_warning(5, __LINE__, __FILE__); @@ -385,17 +389,24 @@ void CommSparse::communicate_with_unpacker(const std::functionstart_nonblocking(m_num_recvs); + workFunctor(); m_exchanger->post_nonblocking_receives(); - m_exchanger->complete_receives(functor); + m_exchanger->complete_receives(unpackFunctor); m_exchanger->complete_sends(); } } else { verify_send_buffers_filled(); if (1 < m_size) { - communicate_unpack(m_comm , m_send , m_recv, m_send_procs, m_recv_procs, functor); + communicate_unpack(m_comm , m_send , m_recv, m_send_procs, m_recv_procs, unpackFunctor); } } + + if (deallocateSendBuffers) { + if (m_exchanger) { + m_exchanger->deallocate_send_bufs(); + } + } #endif } diff --git a/packages/stk/stk_util/stk_util/parallel/CommSparse.hpp b/packages/stk/stk_util/stk_util/parallel/CommSparse.hpp index 2b957e4957b7..740c75da05eb 100644 --- a/packages/stk/stk_util/stk_util/parallel/CommSparse.hpp +++ b/packages/stk/stk_util/stk_util/parallel/CommSparse.hpp @@ -211,9 +211,28 @@ class CommSparse { /** Communicate send buffers to receive buffers, interleave unpacking with * caller-provided functor. */ template - void communicate_with_unpack(const UNPACK_ALGORITHM & alg) + void communicate_with_unpack(const UNPACK_ALGORITHM & unpacker) { - communicate_with_unpacker(alg); + auto noExtraWork = [](){}; + communicate_with_extra_work_and_unpacker(noExtraWork, unpacker, true); + } + + /** Communicate send buffers to receive buffers, interleave extra work. + */ + template + void communicate_with_extra_work(const EXTRA_WORK & extraWork) + { + auto noUnpacker = [](int, CommBuffer&){}; + communicate_with_extra_work_and_unpacker(extraWork, noUnpacker, true); + } + + /** Communicate send buffers to receive buffers, interleave extra work and + * also unpacking with caller-provided functor. + */ + template + void communicate_with_extra_work_and_unpack(const EXTRA_WORK & extraWork, const UNPACK_ALGORITHM & unpacker) + { + communicate_with_extra_work_and_unpacker(extraWork, unpacker, true); } /** Reset, but do not reallocate, message buffers for reprocessing. @@ -227,7 +246,9 @@ class CommSparse { void allocate_data(std::vector& bufs, std::vector& data); #endif void verify_send_buffers_filled(); - void communicate_with_unpacker(const std::function& functor); + void communicate_with_extra_work_and_unpacker(const std::function& workFunctor, + const std::function& unpackFunctor, + bool deallocateSendBuffers = false); ParallelMachine m_comm ; int m_size ; diff --git a/packages/stk/stk_util/stk_util/parallel/ReceiveCounter.cpp b/packages/stk/stk_util/stk_util/parallel/ReceiveCounter.cpp index 67017da981cf..b38b1d44a97b 100644 --- a/packages/stk/stk_util/stk_util/parallel/ReceiveCounter.cpp +++ b/packages/stk/stk_util/stk_util/parallel/ReceiveCounter.cpp @@ -18,7 +18,9 @@ void ReceiveCounter::start_receive_count(const std::vector &sendCounts) m_recvFinished = false; m_sendCount.resize(sendCounts.size()); for (unsigned int i=0; i < sendCounts.size(); ++i) + { m_sendCount[i] = sendCounts[i] > 0 ? 1 : 0; + } MPI_Ireduce_scatter_block(m_sendCount.data(), &m_nrecv, 1, MPI_INT, MPI_SUM, m_comm, &m_recvReq); } diff --git a/packages/stk/stk_util/stk_util/registry/CMakeLists.txt b/packages/stk/stk_util/stk_util/registry/CMakeLists.txt index 4c34bcb48eb9..c741aa6d17c2 100644 --- a/packages/stk/stk_util/stk_util/registry/CMakeLists.txt +++ b/packages/stk/stk_util/stk_util/registry/CMakeLists.txt @@ -32,42 +32,32 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -# -# A) Package-specific configuration options -# - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - SET(HEADERS "") SET(SOURCES "") -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) - -# -# Core headers -# - FILE(GLOB HEADERS *.hpp *.h) - FILE(GLOB SOURCES *.cpp) -# -# C) Define the targets for package's library(s) -# - -TRIBITS_ADD_LIBRARY( - stk_util_registry - NOINSTALLHEADERS ${HEADERS} - SOURCES ${SOURCES} - DEPLIBS stk_util_env +IF(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY( + stk_util_registry + NOINSTALLHEADERS ${HEADERS} + SOURCES ${SOURCES} ) +ELSE() + add_library(stk_util_registry ${SOURCES}) + + target_link_libraries(stk_util_registry PUBLIC stk_util_env) +ENDIF() + +target_include_directories(stk_util_registry PUBLIC + $ + $ +) INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_util/registry/) +IF(NOT HAVE_STK_Trilinos) + INSTALL(TARGETS stk_util_registry DESTINATION ${STK_INSTALL_LIBDIR}) +ENDIF() diff --git a/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp b/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp index 6f01ca289619..39b363e61368 100644 --- a/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp +++ b/packages/stk/stk_util/stk_util/registry/ProductRegistry.cpp @@ -42,7 +42,7 @@ //In Sierra, STK_VERSION_STRING is provided on the compile line by bake. //For Trilinos stk snapshots, the following macro definition gets populated with //the real version string by the trilinos_snapshot.sh script. -#define STK_VERSION_STRING "5.15.2-455-gca4dade0" +#define STK_VERSION_STRING "5.15.5-101-ge9dc2961" #endif namespace stk { diff --git a/packages/stk/stk_util/stk_util/stk_config.h b/packages/stk/stk_util/stk_util/stk_config.h index 1827bc9ad13d..0ae153f55814 100644 --- a/packages/stk/stk_util/stk_util/stk_config.h +++ b/packages/stk/stk_util/stk_util/stk_config.h @@ -48,9 +48,8 @@ // This file gets created by cmake during a Trilinos build // and will not be present in a sierra build using bjam or associated wrappers #include "STK_Trilinos_config.h" -#ifdef HAVE_MPI -#define STK_HAS_MPI -#else + +#ifndef STK_HAS_MPI #ifndef MPI_Comm #define MPI_Comm int diff --git a/packages/stk/stk_util/stk_util/util/CMakeLists.txt b/packages/stk/stk_util/stk_util/util/CMakeLists.txt index b3f14ce07e63..7dfc917393b9 100644 --- a/packages/stk/stk_util/stk_util/util/CMakeLists.txt +++ b/packages/stk/stk_util/stk_util/util/CMakeLists.txt @@ -32,36 +32,36 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - -# -# A) Package-specific configuration options -# - -# -# B) Define the header and source files (and directories) -# - -# -# src -# - SET(HEADERS "") SET(SOURCES "") FILE(GLOB HEADERS *.hpp *.h) FILE(GLOB SOURCES *.cpp) -TRIBITS_INCLUDE_DIRECTORIES(${${PACKAGE_NAME}_SOURCE_DIR}) +IF(HAVE_STK_Trilinos) + TRIBITS_ADD_LIBRARY(stk_util_util + NOINSTALLHEADERS ${HEADERS} + SOURCES ${SOURCES} + ) +ELSE() + add_library(stk_util_util ${SOURCES}) -# -# C) Define the targets for package's library(s) -# + target_link_libraries(stk_util_util PUBLIC Kokkos::kokkos) +ENDIF() -TRIBITS_ADD_LIBRARY( - stk_util_util - NOINSTALLHEADERS ${HEADERS} - SOURCES ${SOURCES} - ) +target_include_directories(stk_util_util PUBLIC + $ + $ +) +target_include_directories(stk_util_util PUBLIC + $ + $ +) INSTALL(FILES ${HEADERS} DESTINATION ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}/stk_util/util/) + +IF(NOT HAVE_STK_Trilinos) + INSTALL(TARGETS stk_util_util DESTINATION ${STK_INSTALL_LIBDIR}) +ENDIF() +