Use ctest more often in CI and tweak defaults (#1420)

* Uncomment ctest section in CI * Update ctest, use exe for windows, no mpi for gcc * Add wildcard for win32 ctest * Make ctest tolerance looser to account for mac diffs * Try 2 ranks for gcc mpi tests * Make fewer regression test executables ci flag * Fix syntax * Remove unused unit test ctest functions --------- Co-authored-by: Aaron M. Lattanzi <[email protected]>
erf-model · Feb 3, 2024 · a048073 · a048073
1 parent c1ed778
commit a048073
Show file tree

Hide file tree

Showing 8 changed files with 70 additions and 36 deletions.
diff --git a/.github/workflows/cuda-ci.yml b/.github/workflows/cuda-ci.yml
@@ -46,5 +46,6 @@ jobs:
             -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON           \
             -DERF_DIM:STRING=3 \
             -DERF_ENABLE_MPI:BOOL=ON \
-            -DERF_ENABLE_CUDA:BOOL=ON .
+            -DERF_ENABLE_CUDA:BOOL=ON \
+            -DERF_ENABLE_REGRESSION_TESTS_ONLY:BOOL=ON .
           cmake --build build-${{matrix.cuda_pkg}} -- -j $(nproc)
diff --git a/.github/workflows/gcc.yml b/.github/workflows/gcc.yml
@@ -36,7 +36,7 @@ jobs:
       run: |
         cmake --build ${{runner.workspace}}/ERF/build --parallel 2 --verbose
 
-  # - name: CMake Tests # see file ERF/Tests/CTestList.cmake
-  #   run: |
-  #     ctest
-  #   working-directory: ${{runner.workspace}}/ERF/build
+    - name: CMake Tests # see file ERF/Tests/CTestList.cmake
+      run: |
+        ctest -L regression -VV
+      working-directory: ${{runner.workspace}}/ERF/build
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -37,7 +37,7 @@ jobs:
       run: |
         cmake --build ${{runner.workspace}}/ERF/build --parallel 2 --verbose
 
- #  - name: CMake Tests # see file ERF/Tests/CTestList.cmake
- #    run: |
- #      ctest
- #    working-directory: ${{runner.workspace}}/ERF/build
+    - name: CMake Tests # see file ERF/Tests/CTestList.cmake
+      run: |
+        ctest -L regression -VV
+      working-directory: ${{runner.workspace}}/ERF/build
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -31,7 +31,7 @@ jobs:
       run: |
         cmake --build build --parallel 2 --verbose
 
- #  - name: CMake Tests # see file ERF/Tests/CTestList.cmake
- #    run: |
- #      ctest
- #    working-directory: ${{runner.workspace}}/ERF/build
+    - name: CMake Tests # see file ERF/Tests/CTestList.cmake
+      run: |
+        ctest -L regression -VV
+      working-directory: ${{runner.workspace}}/ERF/build
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -12,6 +12,7 @@ set(ERF_DIM "3" CACHE STRING "Number of physical dimensions")
 option(ERF_ENABLE_DOCUMENTATION "Build documentation" OFF)
 option(ERF_ENABLE_ALL_WARNINGS "Enable all compiler warnings" OFF)
 option(ERF_ENABLE_TESTS "Enable regression and unit tests" OFF)
+option(ERF_ENABLE_REGRESSION_TESTS_ONLY "Enable only regression tests" OFF)
 option(ERF_ENABLE_NETCDF "Enable NetCDF IO" OFF)
 option(ERF_ENABLE_HDF5 "Enable HDF5 IO" ${ERF_ENABLE_NETCDF})
 option(ERF_ENABLE_PARTICLES "Enable Lagrangian particles" OFF)

diff --git a/Exec/CMakeLists.txt b/Exec/CMakeLists.txt
@@ -4,6 +4,17 @@ include(${CMAKE_SOURCE_DIR}/CMake/BuildERFExe.cmake)
 build_erf_lib(${erf_lib_name})
 if (ERF_ENABLE_MULTIBLOCK)
   add_subdirectory(DevTests/MultiBlock)
+elseif (ERF_ENABLE_REGRESSION_TESTS_ONLY)
+#  add_subdirectory(RegTests/Bubble)
+  add_subdirectory(RegTests/CouetteFlow)
+  add_subdirectory(RegTests/DensityCurrent)
+  add_subdirectory(RegTests/DynamicRefinement)
+  add_subdirectory(RegTests/EkmanSpiral_custom)
+  add_subdirectory(RegTests/IsentropicVortex)
+  add_subdirectory(RegTests/PoiseuilleFlow)
+  add_subdirectory(RegTests/ScalarAdvDiff)
+  add_subdirectory(RegTests/TaylorGreenVortex)
+  add_subdirectory(DevTests/MovingTerrain)
 else ()
   add_subdirectory(ABL)
   add_subdirectory(SuperCell)

diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt
@@ -1,4 +1,7 @@
-
-set(FCOMPARE_EXE ${CMAKE_BINARY_DIR}/Submodules/AMReX/Tools/Plotfile/amrex_fcompare CACHE INTERNAL "Path to fcompare executable for regression tests")
-set(ERF_TEST_NRANKS 4 CACHE STRING  "Number of MPI ranks to use for each test")
+if(WIN32)
+    set(FCOMPARE_EXE ${CMAKE_BINARY_DIR}/Submodules/AMReX/Tools/Plotfile/*/amrex_fcompare.exe CACHE INTERNAL "Path to fcompare executable for regression tests")
+else()
+    set(FCOMPARE_EXE ${CMAKE_BINARY_DIR}/Submodules/AMReX/Tools/Plotfile/amrex_fcompare CACHE INTERNAL "Path to fcompare executable for regression tests")
+endif()
+set(ERF_TEST_NRANKS 2 CACHE STRING  "Number of MPI ranks to use for each test")
 include(${CMAKE_CURRENT_SOURCE_DIR}/CTestList.cmake)
diff --git a/Tests/CTestList.cmake b/Tests/CTestList.cmake
@@ -38,7 +38,7 @@ function(add_test_r TEST_NAME TEST_EXE PLTFILE)
     setup_test()
 
     set(TEST_EXE ${CMAKE_BINARY_DIR}/Exec/${TEST_EXE})
-    set(FCOMPARE_TOLERANCE "-r 1e-12 --abs_tol 1.0e-12")
+    set(FCOMPARE_TOLERANCE "-r 1e-10 --abs_tol 1.0e-10")
     set(FCOMPARE_FLAGS "-a ${FCOMPARE_TOLERANCE}")
     set(test_command sh -c "${MPI_COMMANDS} ${TEST_EXE} ${CURRENT_TEST_BINARY_DIR}/${TEST_NAME}.i ${RUNTIME_OPTIONS} > ${TEST_NAME}.log && ${MPI_FCOMP_COMMANDS} ${FCOMPARE_EXE} ${FCOMPARE_FLAGS} ${PLOT_GOLD} ${CURRENT_TEST_BINARY_DIR}/${PLTFILE}")
 
@@ -73,27 +73,45 @@ function(add_test_0 TEST_NAME TEST_EXE PLTFILE)
     )
 endfunction(add_test_0)
 
-# Standard unit test
-function(add_test_u TEST_NAME)
-    setup_test()
-    add_test(${TEST_NAME} sh -c "${MPI_COMMANDS} ${CMAKE_BINARY_DIR}/${amr_wind_unit_test_exe_name}")
-    set_tests_properties(${TEST_NAME}
-        PROPERTIES
-        TIMEOUT 500
-        PROCESSORS ${NP}
-        WORKING_DIRECTORY "${CURRENT_TEST_BINARY_DIR}/"
-        LABELS "unit"
-    )
-endfunction(add_test_u)
-
-#=============================================================================
-# Unit tests
-#=============================================================================
-# add_test_u(unit_tests)
-
 #=============================================================================
 # Regression tests
 #=============================================================================
+if(WIN32)
+#add_test_r(Bubble_DensityCurrent             "Bubble/bubble.exe" "plt00010")
+add_test_r(CouetteFlow                       "RegTests/CouetteFlow/*/erf_couette_flow.exe" "plt00050")
+add_test_r(DensityCurrent                    "RegTests/DensityCurrent/*/density_current.exe" "plt00010")
+add_test_r(DensityCurrent_detJ2              "RegTests/DensityCurrent/*/density_current.exe" "plt00010")
+add_test_r(DensityCurrent_detJ2_nosub        "RegTests/DensityCurrent/*/density_current.exe" "plt00020")
+add_test_r(DensityCurrent_detJ2_MT           "RegTests/DensityCurrent/*/density_current.exe" "plt00010")
+add_test_r(EkmanSpiral                       "RegTests/EkmanSpiral_custom/*/ekman_spiral_custom.exe" "plt00010")
+add_test_r(IsentropicVortexStationary        "RegTests/IsentropicVortex/*/erf_isentropic_vortex.exe" "plt00010")
+add_test_r(IsentropicVortexAdvecting         "RegTests/IsentropicVortex/*/erf_isentropic_vortex.exe" "plt00010")
+add_test_r(MovingTerrain_nosub               "DevTests/MovingTerrain/*/moving_terrain.exe"   "plt00020")
+add_test_r(MovingTerrain_sub                 "DevTests/MovingTerrain/*/moving_terrain.exe"   "plt00010")
+add_test_r(PoiseuilleFlow                    "RegTests/PoiseuilleFlow/*/erf_poiseuille_flow.exe" "plt00010")
+add_test_r(RayleighDamping                   "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00100")
+add_test_r(ScalarAdvectionUniformU           "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00020")
+add_test_r(ScalarAdvectionShearedU           "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00080")
+add_test_r(ScalarAdvDiff_order2              "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00020")
+add_test_r(ScalarAdvDiff_order3              "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00020")
+add_test_r(ScalarAdvDiff_order4              "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00020")
+add_test_r(ScalarAdvDiff_order5              "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00020")
+add_test_r(ScalarAdvDiff_order6              "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00020")
+add_test_r(ScalarAdvDiff_weno3               "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00020")
+add_test_r(ScalarAdvDiff_weno3z              "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00020")
+add_test_r(ScalarAdvDiff_weno5               "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00020")
+add_test_r(ScalarAdvDiff_weno5z              "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00020")
+add_test_r(ScalarAdvDiff_wenomzq3            "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00020")
+add_test_r(ScalarDiffusionGaussian           "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00020")
+add_test_r(ScalarDiffusionSine               "RegTests/ScalarAdvDiff/*/erf_scalar_advdiff.exe" "plt00020")
+add_test_r(TaylorGreenAdvecting              "RegTests/TaylorGreenVortex/*/taylor_green.exe" "plt00010")
+add_test_r(TaylorGreenAdvectingDiffusing     "RegTests/TaylorGreenVortex/*/taylor_green.exe" "plt00010")
+add_test_r(MSF_NoSub_IsentropicVortexAdv     "RegTests/IsentropicVortex/*/erf_isentropic_vortex.exe" "plt00010")
+add_test_r(MSF_Sub_IsentropicVortexAdv       "RegTests/IsentropicVortex/*/erf_isentropic_vortex.exe" "plt00010")
+
+add_test_0(Deardorff_stationary              "ABL/*/erf_abl.exe" "plt00010")
+
+else()
 #add_test_r(Bubble_DensityCurrent             "Bubble/bubble" "plt00010")
 add_test_r(CouetteFlow                       "RegTests/CouetteFlow/erf_couette_flow" "plt00050")
 add_test_r(DensityCurrent                    "RegTests/DensityCurrent/density_current" "plt00010")
@@ -127,7 +145,7 @@ add_test_r(MSF_NoSub_IsentropicVortexAdv     "RegTests/IsentropicVortex/erf_isen
 add_test_r(MSF_Sub_IsentropicVortexAdv       "RegTests/IsentropicVortex/erf_isentropic_vortex" "plt00010")
 
 add_test_0(Deardorff_stationary              "ABL/erf_abl" "plt00010")
-
+endif()
 #=============================================================================
 # Performance tests
 #=============================================================================