Fix intermittent CI failures due to corrupt XML (#1431)

* Filter downloaded artifacts to only pull test results * Add test option to write XML file through argument * Replace environment variable with command line arg
celeritas-project · Sep 28, 2024 · bec9f4c · bec9f4c
1 parent 64f248d
commit bec9f4c
Show file tree

Hide file tree

Showing 9 changed files with 28 additions and 29 deletions.
diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
@@ -88,8 +88,6 @@ jobs:
       - name: Test Celeritas
         id: test
         working-directory: build
-        env:
-          GTEST_OUTPUT: "xml:${{github.workspace}}/test-output/google/"
         run: |
           ctest --parallel $(nproc) --timeout 180 --output-on-failure \
             --test-output-size-passed=65536 --test-output-size-failed=1048576

diff --git a/.github/workflows/build-fast.yml b/.github/workflows/build-fast.yml
@@ -83,15 +83,11 @@ jobs:
           ninja
       - name: Run unit tests
         id: unittest
-        env:
-          GTEST_OUTPUT: "xml:${{github.workspace}}/test-output/google/"
         run: |
           ctest -LE app --preset=${CMAKE_PRESET}-unit
       - name: Run app tests
         id: apptest
         if: ${{!cancelled() && steps.build.outcome == 'success'}}
-        env:
-          CTEST_OUTPUT: "${{github.workspace}}/test-output/ctest/all.xml"
         run: |
           ctest -L app --preset=${CMAKE_PRESET}-app
       - name: Install
@@ -160,15 +156,11 @@ jobs:
       - name: Run unit tests
         id: unittest
         continue-on-error: true
-        env:
-          GTEST_OUTPUT: "xml:${{github.workspace}}\\test-output\\google\\"
         run: |
           ctest --preset=$Env:CMAKE_PRESET-unit
       - name: Run app tests
         id: apptest
         if: ${{!cancelled() && steps.build.outcome == 'success'}}
-        env:
-          CTEST_OUTPUT: "${{github.workspace}}\\test-output\\ctest\\all.xml"
         run: |
           ctest --preset=$Env:CMAKE_PRESET-app
       - name: Install

diff --git a/.github/workflows/build-spack.yml b/.github/workflows/build-spack.yml
@@ -146,8 +146,6 @@ jobs:
         id: unittest
         if: ${{matrix.special != 'clang-tidy'}}
         continue-on-error: ${{fromJSON(matrix.geant || '0') < 11}} # TODO: fix failing tests
-        env:
-          GTEST_OUTPUT: "xml:${{github.workspace}}/test-output/google/"
         run: |
           if [ "${{matrix.geant}}" == "11.0" ]; then
             # Note this is ignored for geant4, float, clhep
@@ -160,8 +158,6 @@ jobs:
             && matrix.special != 'clang-tidy'
             && steps.build.outcome == 'success'}}
         continue-on-error: ${{matrix.geant == '10.6'}} # TODO: rogue output from G4DeexPrecoParameters
-        env:
-          CTEST_OUTPUT: "${{github.workspace}}/test-output/ctest/all.xml"
         run: |
           ctest -L app --preset=spack-app
       - name: Upload test results

diff --git a/.github/workflows/build-ultralite.yml b/.github/workflows/build-ultralite.yml
@@ -48,8 +48,6 @@ jobs:
           ninja
       - name: Run tests
         id: test
-        env:
-          CTEST_OUTPUT: "${{github.workspace}}/test-output/ctest/all.xml"
         run: |
           ctest --preset=${CMAKE_PRESET} 
       - name: Install
@@ -123,8 +121,6 @@ jobs:
           cmake --build --preset=$Env:CMAKE_PRESET 
       - name: Test all
         id: test
-        env:
-          CTEST_OUTPUT: "${{github.workspace}}\\test-output\\ctest\\all.xml"
         run: |
           ctest --preset=$Env:CMAKE_PRESET
       - name: Show ccache stats

diff --git a/.github/workflows/pull_request_completed.yml b/.github/workflows/pull_request_completed.yml
@@ -20,6 +20,8 @@ jobs:
     - name: Download and Extract Artifacts
       uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d # v6
       with:
+        name: "(event-file|test-results-.*)"
+        name_is_regexp: true
         run_id: ${{github.event.workflow_run.id}}
         path: artifacts
     - name: Publish PR comment

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -79,6 +79,14 @@ cmake_dependent_option(CELERITAS_TEST_VERBOSE
   "Increase logging level for tests" "${CELERITAS_DEBUG}"
   "CELERITAS_BUILD_TESTS" OFF
 )
+if(CELERITAS_BUILD_TESTS)
+  # NOTE: CMake "normalizes" this path by stripping trailing directory
+  # separators, so this *must* be a directory.
+  set(CELERITAS_TEST_XML "" CACHE PATH
+    "If non-empty, write JUnit output from google tests to this directory"
+  )
+  mark_as_advanced(CELERITAS_TEST_XML)
+endif()
 
 #----------------------------------------------------------------------------#
 # CELERITAS CORE IMPLEMENTATION OPTIONS

diff --git a/cmake/CeleritasAddTest.cmake b/cmake/CeleritasAddTest.cmake
@@ -159,7 +159,7 @@ set(CELERITASTEST_NP_DEFAULT "${_procs}" CACHE INTERNAL
 set(_procs)
 
 if(NOT CELERITAS_USE_MPI)
-  # Construct test name with MPI enabled, or empty if not applicable
+  # Skip tests that require greater than 1 process
   function(_celeritasaddtest_test_name outvar test_name np suffix)
     set(_name "${test_name}${suffix}")
     if(np GREATER 1)
@@ -168,13 +168,13 @@ if(NOT CELERITAS_USE_MPI)
     set(${outvar} "${_name}" PARENT_SCOPE)
   endfunction()
 
-  # Construct MPI command, or empty if not applicable
+  # Construct execution command
   function(_celeritasaddtest_mpi_cmd outvar np test_exe)
     set(_cmd "${test_exe}" ${ARGN})
     set(${outvar} "${_cmd}" PARENT_SCOPE)
   endfunction()
 else()
-  # Construct test name with MPI enabled but not tribits
+  # Construct test name with number of processors
   function(_celeritasaddtest_test_name outvar test_name np suffix)
     if(np GREATER CELERITAS_MAX_NUMPROCS)
       set(_name)
@@ -186,6 +186,7 @@ else()
     set(${outvar} "${_name}" PARENT_SCOPE)
   endfunction()
 
+  # Construct MPI command
   function(_celeritasaddtest_mpi_cmd outvar np test_exe)
     if(np GREATER 1)
       set(_cmd "${MPIEXEC_EXECUTABLE}" ${MPIEXEC_NUMPROC_FLAG} "${np}"
@@ -407,10 +408,17 @@ function(celeritas_add_test SOURCE_FILE)
       # Launch with MPI directly
       _celeritasaddtest_mpi_cmd(_test_cmd "${_np}" "${_EXE_NAME}")
 
-      set(_test_args "${_EXE_ARGS}")
+      set(_test_args)
       if(_filter)
         list(APPEND _test_args "--gtest_filter=${_filter}")
       endif()
+      if(CELERITAS_TEST_XML)
+        string(REGEX REPLACE "\\*" "ALL" _xml_name "${_TEST_NAME}")
+        string(REGEX REPLACE "[^a-zA-Z0-9_.-]+" "_" _xml_name "${_xml_name}")
+        list(APPEND _test_args
+          "--gtest_output=xml:${CELERITAS_TEST_XML}/${_xml_name}.xml"
+        )
+      endif()
 
       add_test(NAME "${_TEST_NAME}" COMMAND ${_test_cmd} ${_test_args})
       list(APPEND _ADDED_TESTS "${_TEST_NAME}")

diff --git a/scripts/cmake-presets/ci-ubuntu-github.json b/scripts/cmake-presets/ci-ubuntu-github.json
@@ -26,7 +26,8 @@
         "CELERITAS_USE_ROOT":    {"type": "BOOL", "value": "OFF"},
         "CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"},
         "CELERITAS_HOSTNAME": "ubuntu-github",
-        "CMAKE_BUILD_TYPE":      "Release",
+        "CELERITAS_TEST_XML": "$env{GITHUB_WORKSPACE}/test-output/google/",
+        "CMAKE_BUILD_TYPE": "Release",
         "CMAKE_CXX_COMPILER_LAUNCHER": "ccache",
         "CMAKE_CXX_EXTENSIONS": {"type": "BOOL", "value": "OFF"},
         "CMAKE_CXX_FLAGS": "-Wall -Wextra -pedantic -Werror -Wno-error=deprecated-declarations",
@@ -185,7 +186,7 @@
       "configurePreset": "spack",
       "inherits": "base",
       "output": {
-        "outputJUnitFile": "$env{CTEST_OUTPUT}"
+        "outputJUnitFile": "$env{GITHUB_WORKSPACE}/test-output/ctest/all.xml"
       }
     },
     {
@@ -196,10 +197,7 @@
     {
       "name": "fast-app",
       "configurePreset": "fast",
-      "inherits": "base",
-      "output": {
-        "outputJUnitFile": "$env{CTEST_OUTPUT}"
-      }
+      "inherits": "spack-app"
     },
     {
       "name": "ultralite",

diff --git a/scripts/cmake-presets/ci-windows-github.json b/scripts/cmake-presets/ci-windows-github.json
@@ -25,6 +25,7 @@
         "CELERITAS_USE_ROOT":    {"type": "BOOL", "value": "OFF"},
         "CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"},
         "CELERITAS_HOSTNAME": "windows-github",
+        "CELERITAS_TEST_XML": "$env{GITHUB_WORKSPACE}/test-output/google/",
         "CMAKE_BUILD_TYPE":      "Release",
         "CMAKE_CXX_COMPILER_LAUNCHER": "ccache",
         "CMAKE_CXX_EXTENSIONS": {"type": "BOOL", "value": "OFF"},
@@ -89,7 +90,7 @@
       "configurePreset": "fast",
       "inherits": "base",
       "output": {
-        "outputJUnitFile": "$env{CTEST_OUTPUT}"
+        "outputJUnitFile": "$env{GITHUB_WORKSPACE}/test-output/ctest/all.xml"
       }
     },
     {"name": "ultralite", "configurePreset": "ultralite", "inherits": "base"}