From 09cc7d19529d4c6a2f5f69197ddf803867268d99 Mon Sep 17 00:00:00 2001
From: Sebastian Grimberg <sjg@amazon.com>
Date: Mon, 25 Sep 2023 12:34:16 -0700
Subject: [PATCH] Add CI jobs to run Palace unit tests

---
 .github/workflows/build-and-test-linux.yml | 36 +++++++++++++++++-----
 .github/workflows/build-and-test-macos.yml | 22 +++++++++++--
 2 files changed, 48 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/build-and-test-linux.yml b/.github/workflows/build-and-test-linux.yml
index 7f4f60e0d..0d243144c 100644
--- a/.github/workflows/build-and-test-linux.yml
+++ b/.github/workflows/build-and-test-linux.yml
@@ -174,7 +174,7 @@ jobs:
           [[ "${{ matrix.with-eigensolver }}" == 'slepc' ]] && WITH_SLEPC='ON' || WITH_SLEPC='OFF'
           [[ "${{ matrix.with-eigensolver }}" == 'arpack' ]] && WITH_ARPACK='ON' || WITH_ARPACK='OFF'
 
-          # Build and install
+          # Build and install (with unit tests)
           mkdir palace-build && cd palace-build
           cmake .. \
             -DCMAKE_INSTALL_PREFIX=$(pwd)/../palace-install \
@@ -186,9 +186,31 @@ jobs:
             -DPALACE_WITH_MUMPS=$WITH_MUMPS \
             -DPALACE_WITH_SLEPC=$WITH_SLEPC \
             -DPALACE_WITH_ARPACK=$WITH_ARPACK
-          make -j$NUM_PROC_BUILD
+          make -j$NUM_PROC_BUILD palace-tests
 
-      - name: Run tests
+      - name: Run unit tests
+        env:
+          NUM_PROC_TEST_MAX: '2'
+        run: |
+          # Configure environment
+          if [[ "${{ matrix.compiler }}" == 'intel' ]] || \
+             [[ "${{ matrix.mpi }}" == 'intelmpi' ]] || \
+             [[ "${{ matrix.math-libs }}" == 'intelmkl' ]]; then
+            source /opt/intel/oneapi/setvars.sh  # Sets PATH, MKLROOT
+          fi
+          if [[ "${{ matrix.with-openmp }}" == 'true' ]]; then
+            export OMP_NUM_THREADS=2
+          else
+            export OMP_NUM_THREADS=1
+          fi
+          export LD_LIBRARY_PATH=$(pwd)/palace-install/lib:$LD_LIBRARY_PATH
+          export LD_LIBRARY_PATH=$(pwd)/palace-install/lib64:$LD_LIBRARY_PATH
+          cd $(pwd)/palace-build/palace-build/test/unit
+
+          # Run tests
+          mpirun -np $NUM_PROC_TEST_MAX ./unit-tests --skip-benchmarks
+
+      - name: Run regression tests for examples/
         env:
           NUM_PROC_TEST_MAX: '8'
         run: |
@@ -198,6 +220,10 @@ jobs:
              [[ "${{ matrix.math-libs }}" == 'intelmkl' ]]; then
             source /opt/intel/oneapi/setvars.sh  # Sets PATH, MKLROOT
           fi
+          if [[ "${{ matrix.math-libs }}" == 'aocl' ]]; then
+            export AOCLROOT=/opt/AMD/aocl/aocl-linux-gcc-4.1.0/gcc
+            export LD_LIBRARY_PATH=$AOCLROOT/lib:$LD_LIBRARY_PATH
+          fi
           export NUM_PROC_TEST=$(nproc 2> /dev/null || sysctl -n hw.ncpu)
           if [[ "$NUM_PROC_TEST" -gt "$NUM_PROC_TEST_MAX" ]]; then
             NUM_PROC_TEST=$NUM_PROC_TEST_MAX
@@ -208,10 +234,6 @@ jobs:
           else
             export OMP_NUM_THREADS=1
           fi
-          if [[ "${{ matrix.math-libs }}" == 'aocl' ]]; then
-            export AOCLROOT=/opt/AMD/aocl/aocl-linux-gcc-4.1.0/gcc
-            export LD_LIBRARY_PATH=$AOCLROOT/lib:$LD_LIBRARY_PATH
-          fi
           export PATH=$(pwd)/palace-install/bin:$PATH
 
           # Run tests
diff --git a/.github/workflows/build-and-test-macos.yml b/.github/workflows/build-and-test-macos.yml
index f152e205d..b637e8a7c 100644
--- a/.github/workflows/build-and-test-macos.yml
+++ b/.github/workflows/build-and-test-macos.yml
@@ -95,7 +95,7 @@ jobs:
           [[ "${{ matrix.with-eigensolver }}" == 'slepc' ]] && WITH_SLEPC='ON' || WITH_SLEPC='OFF'
           [[ "${{ matrix.with-eigensolver }}" == 'arpack' ]] && WITH_ARPACK='ON' || WITH_ARPACK='OFF'
 
-          # Build and install
+          # Build and install (with unit tests)
           mkdir palace-build && cd palace-build
           cmake .. \
             -DCMAKE_INSTALL_PREFIX=$(pwd)/../palace-install \
@@ -107,9 +107,25 @@ jobs:
             -DPALACE_WITH_MUMPS=$WITH_MUMPS \
             -DPALACE_WITH_SLEPC=$WITH_SLEPC \
             -DPALACE_WITH_ARPACK=$WITH_ARPACK
-          make -j$NUM_PROC_BUILD
+          make -j$NUM_PROC_BUILD palace-tests
 
-      - name: Run tests
+      - name: Run unit tests
+        env:
+          NUM_PROC_TEST_MAX: '2'
+        run: |
+          # Configure environment
+          if [[ "${{ matrix.with-openmp }}" == 'true' ]]; then
+            export OMP_NUM_THREADS=2
+          else
+            export OMP_NUM_THREADS=1
+          fi
+          export DYLD_LIBRARY_PATH=$(pwd)/palace-install/lib:$DYLD_LIBRARY_PATH
+          cd $(pwd)/palace-build/palace-build/test/unit
+
+          # Run tests
+          mpirun -np $NUM_PROC_TEST_MAX ./unit-tests --skip-benchmarks
+
+      - name: Run regression tests for examples/
         env:
           NUM_PROC_TEST_MAX: '8'
         run: |