From 3d6bf51962385978a4ab95958737cef1673fc147 Mon Sep 17 00:00:00 2001 From: Sebastian Grimberg Date: Wed, 29 May 2024 12:12:01 -0700 Subject: [PATCH 1/6] Cross-platform compatibility for MAGMA build --- cmake/ExternalMAGMA.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/ExternalMAGMA.cmake b/cmake/ExternalMAGMA.cmake index 53d427c71..dd212f59c 100644 --- a/cmake/ExternalMAGMA.cmake +++ b/cmake/ExternalMAGMA.cmake @@ -71,6 +71,9 @@ if(PALACE_WITH_HIP) endif() endif() set(MAKE_GENERATE_INC "${MAKE_GENERATE_INC}FORT = true\\n") +file(WRITE ${CMAKE_BINARY_DIR}/extern/magma-cmake/make.inc.cmake + "file(WRITE make.inc \"${MAKE_GENERATE_INC}\")\n" +) string(REPLACE ";" "; " MAGMA_OPTIONS_PRINT "${MAGMA_OPTIONS}") message(STATUS "MAGMA_OPTIONS: ${MAGMA_OPTIONS_PRINT}") @@ -86,7 +89,7 @@ ExternalProject_Add(magma PREFIX ${CMAKE_BINARY_DIR}/extern/magma-cmake UPDATE_COMMAND "" PATCH_COMMAND - echo -e "${MAKE_GENERATE_INC}" > make.inc && + ${CMAKE_COMMAND} -P ../magma-cmake/make.inc.cmake && ${CMAKE_MAKE_PROGRAM} generate CONFIGURE_COMMAND ${CMAKE_COMMAND} "${MAGMA_OPTIONS}" TEST_COMMAND "" From ac5e7f478c13130ba21d7b9622192b9b6e9b0a06 Mon Sep 17 00:00:00 2001 From: Sebastian Grimberg Date: Thu, 30 May 2024 15:13:16 -0700 Subject: [PATCH 2/6] Patch MFEM bug for old CUDA architectures and update MFEM CUDA/HIP dependencies --- cmake/ExternalMFEM.cmake | 25 ++++++++----------- .../patch/mfem/patch_cuda_atomicadd_fix.diff | 16 ++++++++++++ 2 files changed, 27 insertions(+), 14 deletions(-) create mode 100644 extern/patch/mfem/patch_cuda_atomicadd_fix.diff diff --git a/cmake/ExternalMFEM.cmake b/cmake/ExternalMFEM.cmake index 0c0592fb9..ecff9dfff 100644 --- a/cmake/ExternalMFEM.cmake +++ b/cmake/ExternalMFEM.cmake @@ -164,22 +164,18 @@ if(PALACE_BUILD_EXTERNAL_DEPS) ) endif() - # HYPRE is built with cusparse, curand (or HIP counterparts). + # HYPRE is built with cusparse, curand (or HIP counterparts), and these are added to + # HYPRE_LIBRARIES by the MFEM CMake build. However, this ignores the include directories + # (for #include , for example), which we can add this way via CMake defining + # CUDAToolkit_INCLUDE_DIRS (and the HIP counterpart). if(PALACE_WITH_CUDA) - find_package(CUDAToolkit REQUIRED) - get_target_property(HYPRE_CURAND_LIBRARY CUDA::curand LOCATION) - get_target_property(HYPRE_CUSPARSE_LIBRARY CUDA::cusparse LOCATION) list(APPEND MFEM_OPTIONS - "-DHYPRE_REQUIRED_LIBRARIES=${HYPRE_CURAND_LIBRARY}$${HYPRE_CUSPARSE_LIBRARY}" + "-DHYPRE_REQUIRED_PACKAGES=CUDAToolkit" ) endif() if(PALACE_WITH_HIP) - find_package(rocrand REQUIRED) - find_package(rocsparse REQUIRED) - get_target_property(HYPRE_ROCRAND_LIBRARY roc::rocrand LOCATION) - get_target_property(HYPRE_ROCSPARSE_LIBRARY roc::rocsparse LOCATION) list(APPEND MFEM_OPTIONS - "-DHYPRE_REQUIRED_LIBRARIES=${HYPRE_ROCRAND_LIBRARY}$${HYPRE_ROCSPARSE_LIBRARY}" + "-DHYPRE_REQUIRED_PACKAGES=rocsparse" ) endif() @@ -241,11 +237,11 @@ Intel C++ compiler for MUMPS and STRUMPACK dependencies") list(APPEND SUPERLU_REQUIRED_PACKAGES "OpenMP") endif() if(PALACE_WITH_CUDA) - list(APPEND SUPERLU_REQUIRED_PACKAGES "CUDA") + list(APPEND SUPERLU_REQUIRED_PACKAGES "CUDAToolkit") list(APPEND SUPERLU_REQUIRED_LIBRARIES ${SUPERLU_STRUMPACK_CUDA_LIBRARIES}) endif() if(PALACE_WITH_HIP) - list(APPEND SUPERLU_REQUIRED_PACKAGES "HIP") + list(APPEND SUPERLU_REQUIRED_PACKAGES "hipblas$rocblas") list(APPEND SUPERLU_REQUIRED_LIBRARIES ${SUPERLU_STRUMPACK_ROCM_LIBRARIES}) endif() string(REPLACE ";" "$" SUPERLU_REQUIRED_PACKAGES "${SUPERLU_REQUIRED_PACKAGES}") @@ -272,11 +268,11 @@ Intel C++ compiler for MUMPS and STRUMPACK dependencies") list(APPEND STRUMPACK_REQUIRED_PACKAGES "OpenMP") endif() if(PALACE_WITH_CUDA) - list(APPEND STRUMPACK_REQUIRED_PACKAGES "CUDA") + list(APPEND STRUMPACK_REQUIRED_PACKAGES "CUDAToolkit") list(APPEND STRUMPACK_REQUIRED_LIBRARIES ${SUPERLU_STRUMPACK_CUDA_LIBRARIES}) endif() if(PALACE_WITH_HIP) - list(APPEND STRUMPACK_REQUIRED_PACKAGES "HIP") + list(APPEND STRUMPACK_REQUIRED_PACKAGES "hipblas$rocblas") list(APPEND STRUMPACK_REQUIRED_LIBRARIES ${SUPERLU_STRUMPACK_ROCM_LIBRARIES}) endif() string(REPLACE ";" "$" STRUMPACK_REQUIRED_PACKAGES "${STRUMPACK_REQUIRED_PACKAGES}") @@ -365,6 +361,7 @@ set(MFEM_PATCH_FILES "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_par_tet_mesh_fix_dev.diff" "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_gmsh_parser_performance.diff" "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_nc_internal_bdr_project_fix.diff" + "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_cuda_atomicadd_fix.diff" ) include(ExternalProject) diff --git a/extern/patch/mfem/patch_cuda_atomicadd_fix.diff b/extern/patch/mfem/patch_cuda_atomicadd_fix.diff new file mode 100644 index 000000000..ae8467a8f --- /dev/null +++ b/extern/patch/mfem/patch_cuda_atomicadd_fix.diff @@ -0,0 +1,16 @@ +diff --git a/general/backends.hpp b/general/backends.hpp +index 0c2f3a531..dda9af53b 100644 +--- a/general/backends.hpp ++++ b/general/backends.hpp +@@ -63,9 +63,9 @@ + #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i Date: Thu, 30 May 2024 15:13:39 -0700 Subject: [PATCH 3/6] Some CMake policy updates --- CMakeLists.txt | 2 ++ cmake/ExternalPalace.cmake | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 00546f040..cf0e6802c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,8 @@ if(PALACE_WITH_CUDA AND PALACE_WITH_HIP) message(FATAL_ERROR "PALACE_WITH_CUDA is not compatible with PALACE_WITH_HIP") endif() if(PALACE_WITH_CUDA) + # Note: The new behavior of CMake policy CMP0104 will initialize CMAKE_CUDA_ARCHITECTURES + # to an (old) compatible value even when not set by the user. enable_language(CUDA) get_filename_component(NVCC_DIR ${CMAKE_CUDA_COMPILER} DIRECTORY) get_filename_component(BIN_NVCC_DIR ${NVCC_DIR} DIRECTORY) diff --git a/cmake/ExternalPalace.cmake b/cmake/ExternalPalace.cmake index 41a801c4a..9e7150a0c 100644 --- a/cmake/ExternalPalace.cmake +++ b/cmake/ExternalPalace.cmake @@ -80,9 +80,6 @@ string(REPLACE ";" "; " PALACE_OPTIONS_PRINT "${PALACE_OPTIONS}") message(STATUS "PALACE_OPTIONS: ${PALACE_OPTIONS_PRINT}") include(ExternalProject) -if(POLICY CMP0114) - cmake_policy(SET CMP0114 NEW) -endif() ExternalProject_Add(palace DEPENDS ${PALACE_DEPENDENCIES} SOURCE_DIR ${CMAKE_SOURCE_DIR}/palace From dceaf7ad84265f226aa86f0285c048a01095ae69 Mon Sep 17 00:00:00 2001 From: Sebastian Grimberg Date: Mon, 10 Jun 2024 13:52:35 -0700 Subject: [PATCH 4/6] Fix CMake warning for ParMETIS build when not using MPI compiler wrappers --- cmake/ExternalMETIS.cmake | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmake/ExternalMETIS.cmake b/cmake/ExternalMETIS.cmake index 593f2a339..abc56c558 100644 --- a/cmake/ExternalMETIS.cmake +++ b/cmake/ExternalMETIS.cmake @@ -99,9 +99,11 @@ if(PALACE_WITH_SUPERLU OR PALACE_WITH_STRUMPACK) message(FATAL_ERROR "MPI is not found when trying to build ParMETIS") endif() if(NOT CMAKE_C_COMPILER STREQUAL MPI_C_COMPILER) + string(REPLACE ";" "$" PARMETIS_MPI_LIBRARIES "${MPI_C_LIBRARIES}") + string(REPLACE ";" "$" PARMETIS_MPI_INCLUDE_DIRS "${MPI_C_INCLUDE_DIRS}") list(APPEND PARMETIS_OPTIONS - "-DMPI_LIBRARIES=${MPI_C_LIBRARIES}" - "-DMPI_INCLUDE_PATH=${MPI_C_INCLUDE_DIRS}" + "-DMPI_LIBRARIES=${PARMETIS_MPI_LIBRARIES}" + "-DMPI_INCLUDE_PATH=${PARMETIS_MPI_INCLUDE_DIRS}" ) endif() From e5562d8ef4e2b9f90b9e5aa9be19a07c8177c07b Mon Sep 17 00:00:00 2001 From: Sebastian Grimberg Date: Mon, 10 Jun 2024 14:50:17 -0700 Subject: [PATCH 5/6] Add comment on MAGMA build --- cmake/ExternalMAGMA.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/ExternalMAGMA.cmake b/cmake/ExternalMAGMA.cmake index dd212f59c..f15ea3aca 100644 --- a/cmake/ExternalMAGMA.cmake +++ b/cmake/ExternalMAGMA.cmake @@ -78,6 +78,7 @@ file(WRITE ${CMAKE_BINARY_DIR}/extern/magma-cmake/make.inc.cmake string(REPLACE ";" "; " MAGMA_OPTIONS_PRINT "${MAGMA_OPTIONS}") message(STATUS "MAGMA_OPTIONS: ${MAGMA_OPTIONS_PRINT}") +# Note: MAGMA requires Python < 3.12 for patch step as of June 2024 include(ExternalProject) ExternalProject_Add(magma DEPENDS ${MAGMA_DEPENDENCIES} From 08399fd456507a7e39bd4b95ec1609bde1754217 Mon Sep 17 00:00:00 2001 From: Sebastian Grimberg Date: Mon, 10 Jun 2024 15:58:46 -0700 Subject: [PATCH 6/6] Update MFEM and remove no longer required patch files --- cmake/ExternalGitTags.cmake | 2 +- cmake/ExternalMFEM.cmake | 2 - .../patch/mfem/patch_cuda_atomicadd_fix.diff | 16 - .../patch_nc_internal_bdr_project_fix.diff | 273 ------------------ 4 files changed, 1 insertion(+), 292 deletions(-) delete mode 100644 extern/patch/mfem/patch_cuda_atomicadd_fix.diff delete mode 100644 extern/patch/mfem/patch_nc_internal_bdr_project_fix.diff diff --git a/cmake/ExternalGitTags.cmake b/cmake/ExternalGitTags.cmake index b90f6e7f9..d37e45e4d 100644 --- a/cmake/ExternalGitTags.cmake +++ b/cmake/ExternalGitTags.cmake @@ -132,7 +132,7 @@ set(EXTERN_MFEM_GIT_BRANCH "Git branch for external MFEM build" ) set(EXTERN_MFEM_GIT_TAG - "c444b17c973cc301590a6ac186fb33587b5881e6" CACHE STRING # master @ 05/18/2024 + "9a327eeca6472254e8bdefc1527aa130250e528e" CACHE STRING # master @ 05/18/2024 "Git tag for external MFEM build" ) diff --git a/cmake/ExternalMFEM.cmake b/cmake/ExternalMFEM.cmake index ecff9dfff..0f3896ef2 100644 --- a/cmake/ExternalMFEM.cmake +++ b/cmake/ExternalMFEM.cmake @@ -360,8 +360,6 @@ set(MFEM_PATCH_FILES "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_mesh_part_const.diff" "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_par_tet_mesh_fix_dev.diff" "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_gmsh_parser_performance.diff" - "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_nc_internal_bdr_project_fix.diff" - "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_cuda_atomicadd_fix.diff" ) include(ExternalProject) diff --git a/extern/patch/mfem/patch_cuda_atomicadd_fix.diff b/extern/patch/mfem/patch_cuda_atomicadd_fix.diff deleted file mode 100644 index ae8467a8f..000000000 --- a/extern/patch/mfem/patch_cuda_atomicadd_fix.diff +++ /dev/null @@ -1,16 +0,0 @@ -diff --git a/general/backends.hpp b/general/backends.hpp -index 0c2f3a531..dda9af53b 100644 ---- a/general/backends.hpp -+++ b/general/backends.hpp -@@ -63,9 +63,9 @@ - #define MFEM_FOREACH_THREAD(i,k,N) for(int i=0; i &attr, - Array &values_counter) - { -- int i, j, fdof, d, ind, vdim; -- real_t val; -- const FiniteElement *fe; -- ElementTransformation *transf; - Array vdofs; - Vector vc; - - values_counter.SetSize(Size()); - values_counter = 0; - -- vdim = fes->GetVDim(); -- -+ const int vdim = fes->GetVDim(); - HostReadWrite(); - -- for (i = 0; i < fes->GetNBE(); i++) -+ for (int i = 0; i < fes->GetNBE(); i++) - { - if (attr[fes->GetBdrAttribute(i) - 1] == 0) { continue; } - -- fe = fes->GetBE(i); -- fdof = fe->GetDof(); -- transf = fes->GetBdrElementTransformation(i); -+ const FiniteElement *fe = fes->GetBE(i); -+ const int fdof = fe->GetDof(); -+ ElementTransformation *transf = fes->GetBdrElementTransformation(i); - const IntegrationRule &ir = fe->GetNodes(); - fes->GetBdrElementVDofs(i, vdofs); - -- for (j = 0; j < fdof; j++) -+ for (int j = 0; j < fdof; j++) - { - const IntegrationPoint &ip = ir.IntPoint(j); - transf->SetIntPoint(&ip); - if (vcoeff) { vcoeff->Eval(vc, *transf, ip); } -- for (d = 0; d < vdim; d++) -+ for (int d = 0; d < vdim; d++) - { - if (!vcoeff && !coeff[d]) { continue; } - -- val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip); -- if ( (ind = vdofs[fdof*d+j]) < 0 ) -+ real_t val = vcoeff ? vc(d) : coeff[d]->Eval(*transf, ip); -+ int ind = vdofs[fdof*d+j]; -+ if ( ind < 0 ) - { - val = -val, ind = -1-ind; - } -@@ -2117,10 +2113,11 @@ void GridFunction::AccumulateAndCountBdrValues( - // iff A_ij != 0. It is sufficient to resolve just the first level of - // dependency, since A is a projection matrix: A^n = A due to cR.cP = I. - // Cases like these arise in 3D when boundary edges are constrained by -- // (depend on) internal faces/elements. We use the virtual method -- // GetBoundaryClosure from NCMesh to resolve the dependencies. -- -- if (fes->Nonconforming() && fes->GetMesh()->Dimension() == 3) -+ // (depend on) internal faces/elements, or for internal boundaries in 2 or -+ // 3D. We use the virtual method GetBoundaryClosure from NCMesh to resolve -+ // the dependencies. -+ if (fes->Nonconforming() && (fes->GetMesh()->Dimension() == 2 || -+ fes->GetMesh()->Dimension() == 3)) - { - Vector vals; - Mesh *mesh = fes->GetMesh(); -@@ -2128,26 +2125,19 @@ void GridFunction::AccumulateAndCountBdrValues( - Array bdr_edges, bdr_vertices, bdr_faces; - ncmesh->GetBoundaryClosure(attr, bdr_vertices, bdr_edges, bdr_faces); - -- for (i = 0; i < bdr_edges.Size(); i++) -+ auto mark_dofs = [&](ElementTransformation &transf, const FiniteElement &fe) - { -- int edge = bdr_edges[i]; -- fes->GetEdgeVDofs(edge, vdofs); -- if (vdofs.Size() == 0) { continue; } -- -- transf = mesh->GetEdgeTransformation(edge); -- transf->Attribute = -1; // TODO: set the boundary attribute -- fe = fes->GetEdgeElement(edge); - if (!vcoeff) - { -- vals.SetSize(fe->GetDof()); -- for (d = 0; d < vdim; d++) -+ vals.SetSize(fe.GetDof()); -+ for (int d = 0; d < vdim; d++) - { - if (!coeff[d]) { continue; } - -- fe->Project(*coeff[d], *transf, vals); -+ fe.Project(*coeff[d], transf, vals); - for (int k = 0; k < vals.Size(); k++) - { -- ind = vdofs[d*vals.Size()+k]; -+ const int ind = vdofs[d*vals.Size()+k]; - if (++values_counter[ind] == 1) - { - (*this)(ind) = vals(k); -@@ -2161,11 +2151,11 @@ void GridFunction::AccumulateAndCountBdrValues( - } - else // vcoeff != NULL - { -- vals.SetSize(vdim*fe->GetDof()); -- fe->Project(*vcoeff, *transf, vals); -+ vals.SetSize(vdim*fe.GetDof()); -+ fe.Project(*vcoeff, transf, vals); - for (int k = 0; k < vals.Size(); k++) - { -- ind = vdofs[k]; -+ const int ind = vdofs[k]; - if (++values_counter[ind] == 1) - { - (*this)(ind) = vals(k); -@@ -2176,6 +2166,26 @@ void GridFunction::AccumulateAndCountBdrValues( - } - } - } -+ }; -+ -+ for (auto edge : bdr_edges) -+ { -+ fes->GetEdgeVDofs(edge, vdofs); -+ if (vdofs.Size() == 0) { continue; } -+ -+ ElementTransformation *transf = mesh->GetEdgeTransformation(edge); -+ const FiniteElement *fe = fes->GetEdgeElement(edge); -+ mark_dofs(*transf, *fe); -+ } -+ -+ for (auto face : bdr_faces) -+ { -+ fes->GetFaceVDofs(face, vdofs); -+ if (vdofs.Size() == 0) { continue; } -+ -+ ElementTransformation *transf = mesh->GetFaceTransformation(face); -+ const FiniteElement *fe = fes->GetFaceElement(face); -+ mark_dofs(*transf, *fe); - } - } - } -@@ -2228,26 +2238,37 @@ void GridFunction::AccumulateAndCountBdrTangentValues( - accumulate_dofs(dofs, lvec, *this, values_counter); - } - -- if (fes->Nonconforming() && fes->GetMesh()->Dimension() == 3) -+ if (fes->Nonconforming() && (fes->GetMesh()->Dimension() == 2 || -+ fes->GetMesh()->Dimension() == 3)) - { - Mesh *mesh = fes->GetMesh(); - NCMesh *ncmesh = mesh->ncmesh; - Array bdr_edges, bdr_vertices, bdr_faces; - ncmesh->GetBoundaryClosure(bdr_attr, bdr_vertices, bdr_edges, bdr_faces); - -- for (int i = 0; i < bdr_edges.Size(); i++) -+ for (auto edge : bdr_edges) - { -- int edge = bdr_edges[i]; - fes->GetEdgeDofs(edge, dofs); - if (dofs.Size() == 0) { continue; } - - T = mesh->GetEdgeTransformation(edge); -- T->Attribute = -1; // TODO: set the boundary attribute - fe = fes->GetEdgeElement(edge); - lvec.SetSize(fe->GetDof()); - fe->Project(vcoeff, *T, lvec); - accumulate_dofs(dofs, lvec, *this, values_counter); - } -+ -+ for (auto face : bdr_faces) -+ { -+ fes->GetFaceDofs(face, dofs); -+ if (dofs.Size() == 0) { continue; } -+ -+ T = mesh->GetFaceTransformation(face); -+ fe = fes->GetFaceElement(face); -+ lvec.SetSize(fe->GetDof()); -+ fe->Project(vcoeff, *T, lvec); -+ accumulate_dofs(dofs, lvec, *this, values_counter); -+ } - } - } - -diff --git a/tests/unit/mesh/test_ncmesh.cpp b/tests/unit/mesh/test_ncmesh.cpp -index 1f6dd65fa..49f71739b 100644 ---- a/tests/unit/mesh/test_ncmesh.cpp -+++ b/tests/unit/mesh/test_ncmesh.cpp -@@ -2811,4 +2811,80 @@ TEST_CASE("RP=I", "[NCMesh]") - } - } - -+ -+TEST_CASE("InternalBoundaryProjectBdrCoefficient", "[NCMesh]") -+{ -+ auto test_project_H1 = [](Mesh &mesh, int order, double coef) -+ { -+ H1_FECollection fe_collection(order, mesh.SpaceDimension()); -+ FiniteElementSpace fe_space(&mesh, &fe_collection); -+ GridFunction x(&fe_space); -+ x = -coef; -+ ConstantCoefficient c(coef); -+ -+ // Check projecting on the internal face sets essential dof. -+ Array ess_bdr(mesh.bdr_attributes.Max()); -+ ess_bdr = 0; -+ ess_bdr.Last() = 1; // internal boundary -+ x.ProjectBdrCoefficient(c, ess_bdr); -+ -+ Array ess_vdofs_list, ess_vdofs_marker; -+ fe_space.GetEssentialVDofs(ess_bdr, ess_vdofs_marker); -+ fe_space.MarkerToList(ess_vdofs_marker, ess_vdofs_list); -+ for (auto ess_dof : ess_vdofs_list) -+ { -+ CHECK(x[ess_dof] == Approx(coef).epsilon(1e-8)); -+ } -+ -+ int iess = 0; -+ for (int i = 0; i < x.Size(); i++) -+ { -+ if (iess < ess_vdofs_list.Size() && i == ess_vdofs_list[iess]) -+ { -+ iess++; -+ continue; -+ } -+ CHECK(x[i] == Approx(-coef).epsilon(1e-8)); -+ } -+ -+ }; -+ -+ auto OneSidedNCRefine = [](Mesh &mesh) -+ { -+ // Pick one element attached to the new boundary attribute and refine. -+ const auto interface_attr = mesh.bdr_attributes.Max(); -+ Array el_to_ref; -+ for (int nbe = 0; nbe < mesh.GetNBE(); nbe++) -+ { -+ if (mesh.GetBdrAttribute(nbe) == interface_attr) -+ { -+ int f, o, e1, e2; -+ mesh.GetBdrElementFace(nbe, &f, &o); -+ mesh.GetFaceElements(f, &e1, &e2); -+ el_to_ref.Append(e1); -+ } -+ } -+ mesh.GeneralRefinement(el_to_ref); -+ return; -+ }; -+ -+ SECTION("Hex") -+ { -+ auto smesh = DividingPlaneMesh(false, true); -+ smesh.EnsureNCMesh(true); -+ OneSidedNCRefine(smesh); -+ test_project_H1(smesh, 2, 0.25); -+ } -+ -+ SECTION("Tet") -+ { -+ auto smesh = DividingPlaneMesh(true, true); -+ smesh.EnsureNCMesh(true); -+ OneSidedNCRefine(smesh); -+ test_project_H1(smesh, 3, 0.25); -+ } -+} -+ -+ -+ - } // namespace mfem