Merge branch 'ci-sort-logs' into 'master'

Sort ctest logs in CI jobs

See merge request ogs/ogs!5041

scripts/ci/extends/test-artifacts.yml

@@ -2,7 +2,7 @@
   artifacts:
     when: always
     paths:
-      - build/*/logs/*.txt
+      - build/*/logs
       - build/*/Tests/ctest-junit.xml
       - build/*/Tests/testrunner.xml
       - build/*/make.txt

scripts/cmake/PythonSetup.cmake

@@ -210,9 +210,7 @@ function(setup_venv_dependent_ctests)
                 EXECUTABLE ogs
                 EXECUTABLE_ARGS ${ht_invalid_prj_file}
                 RUNTIME 1
-            )
-            set_tests_properties(
-                ogs-HT_${ht_invalid_prj_file_short} PROPERTIES WILL_FAIL TRUE
+                PROPERTIES WILL_FAIL TRUE
             )
         endforeach()
     endif()

scripts/cmake/test/AddTest.cmake

@@ -346,6 +346,9 @@ macro(_add_test TEST_NAME)
         )
     endif()
 
+    isTestCommandExpectedToSucceed(${TEST_NAME} ${AddTest_PROPERTIES})
+    message(DEBUG "Is test '${TEST_NAME}' expected to succeed? → ${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}")
+
     add_test(
         NAME ${TEST_NAME}
         COMMAND
@@ -355,8 +358,10 @@ macro(_add_test TEST_NAME)
             -DBINARY_PATH=${_binary_path} -DWRAPPER_COMMAND=${WRAPPER_COMMAND}
             "-DWRAPPER_ARGS=${AddTest_WRAPPER_ARGS}"
             -DWORKING_DIRECTORY=${AddTest_WORKING_DIRECTORY}
-            -DLOG_FILE=${PROJECT_BINARY_DIR}/logs/${TEST_NAME}.txt -P
-            ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestWrapper.cmake
+            "-DLOG_ROOT=${PROJECT_BINARY_DIR}/logs"
+            "-DLOG_FILE_BASENAME=${TEST_NAME}.txt"
+            "-DTEST_COMMAND_IS_EXPECTED_TO_SUCCEED=${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}"
+            -P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestWrapper.cmake
     )
 
     if(DEFINED AddTest_DEPENDS)
@@ -607,3 +612,47 @@ Use six arguments version of AddTest with absolute and relative tolerances"
                                   ${AddTest_DISABLED} LABELS "tester;${labels}"
     )
 endmacro()
+
+# Checks if a test is expected to succeed based on the properties WILL_FAIL,
+# PASS_REGULAR_EXPRESSION and FAIL_REGULAR_EXPRESSION.
+# The function expects the test name (used only for debugging purposes) and the
+# test properties as arguments.
+# The test does not need to exist, yet. This function does not query any test
+# case, but only uses the passed list of properties
+function(isTestCommandExpectedToSucceed TEST_NAME)
+    set(options WILL_FAIL)
+    set(oneValueArgs PASS_REGULAR_EXPRESSION FAIL_REGULAR_EXPRESSION)
+    set(multiValueArgs)
+    cmake_parse_arguments(TEST_FAILURE "${options}" "${oneValueArgs}"
+        "${multiValueArgs}" ${ARGN})
+
+    message(DEBUG "failure properties for test ${TEST_NAME}:")
+    list(APPEND CMAKE_MESSAGE_INDENT "  ")
+    message(DEBUG "WILL_FAIL: ${TEST_FAILURE_WILL_FAIL}")
+    message(DEBUG "PASS_RE: ${TEST_FAILURE_PASS_REGULAR_EXPRESSION}")
+    message(DEBUG "FAIL_RE: ${TEST_FAILURE_FAIL_REGULAR_EXPRESSION}")
+    list(POP_BACK CMAKE_MESSAGE_INDENT)
+
+    if (${TEST_FAILURE_WILL_FAIL})
+        if (DEFINED TEST_FAILURE_PASS_REGULAR_EXPRESSION)
+            # Note: if the test property PASS_REGULAR_EXPRESSION is set, the
+            # process return code will be ignored, see https://cmake.org/cmake/help/latest/prop_test/PASS_REGULAR_EXPRESSION.html
+            message(SEND_ERROR "Error in test '${TEST_NAME}': Please do not use both WILL_FAIL and PASS_REGULAR_EXPRESSION in the same test. The logic will be unclear, then.")
+        endif()
+        if (DEFINED TEST_FAILURE_FAIL_REGULAR_EXPRESSION)
+            message(SEND_ERROR "Error in test '${TEST_NAME}': Please do not use both WILL_FAIL and FAIL_REGULAR_EXPRESSION in the same test. The logic will be unclear, then.")
+        endif()
+
+        set(TEST_COMMAND_IS_EXPECTED_TO_SUCCEED false)
+    elseif(DEFINED TEST_FAILURE_PASS_REGULAR_EXPRESSION)
+        if (DEFINED TEST_FAILURE_FAIL_REGULAR_EXPRESSION)
+            message(SEND_ERROR "Error in test '${TEST_NAME}': Please do not use both PASS_REGULAR_EXPRESSION and FAIL_REGULAR_EXPRESSION in the same test. The logic will be unclear, then.")
+        endif()
+
+        set(TEST_COMMAND_IS_EXPECTED_TO_SUCCEED false)
+    else()
+        set(TEST_COMMAND_IS_EXPECTED_TO_SUCCEED true)
+    endif()
+
+    set(TEST_COMMAND_IS_EXPECTED_TO_SUCCEED "${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}" PARENT_SCOPE)
+endfunction()

scripts/cmake/test/AddTestWrapper.cmake

@@ -1,4 +1,4 @@
-# IMPORTANT: multiple arguments in one variables have to be in list notation (;)
+# IMPORTANT: multiple arguments in a single variable have to be in list notation (;)
 # and have to be quoted when passed
 # "-DEXECUTABLE_ARGS=${AddTest_EXECUTABLE_ARGS}"
 execute_process(
@@ -12,9 +12,43 @@ execute_process(
     COMMAND_ECHO STDOUT
 )
 
-if(EXIT_CODE STREQUAL "0" AND NOT DEFINED ENV{CI})
-    file(WRITE ${LOG_FILE} "${LOG}")
-elseif(NOT EXIT_CODE STREQUAL "0")
-    file(WRITE ${LOG_FILE} "${LOG}")
+set(SAVE_LOG true)
+set(TEST_LOG_DIR "${LOG_ROOT}")
+
+if (TEST_COMMAND_IS_EXPECTED_TO_SUCCEED)
+    if (EXIT_CODE STREQUAL "0")
+        # expected: success, actual: success
+        if (DEFINED ENV{CI})
+            set(SAVE_LOG false)
+        endif()
+    else()
+        # expected: success, actual: failure
+        # use default settings
+    endif()
+else()
+    if (EXIT_CODE STREQUAL "0")
+        # expected: failure, actual: success
+        if (DEFINED ENV{CI})
+            set(TEST_LOG_DIR "${LOG_ROOT}/command_succeeded_but_was_expected_to_fail")
+        endif()
+    else()
+        # expected: failure, actual: failure
+        if (DEFINED ENV{CI})
+            set(TEST_LOG_DIR "${LOG_ROOT}/command_failed_as_expected")
+        endif()
+    endif()
+endif()
+
+set(LOG_FILE "${TEST_LOG_DIR}/${LOG_FILE_BASENAME}")
+
+if (SAVE_LOG)
+    if(NOT EXISTS "${TEST_LOG_DIR}")
+        file(MAKE_DIRECTORY "${TEST_LOG_DIR}")
+    endif()
+
+    file(WRITE "${LOG_FILE}" "${LOG}")
+endif()
+
+if(NOT EXIT_CODE STREQUAL "0")
     message(FATAL_ERROR "Exit code: ${EXIT_CODE}; log file: ${LOG_FILE}")
 endif()

scripts/cmake/test/NotebookTest.cmake

@@ -98,14 +98,19 @@ function(NotebookTest)
         endif()
     endif()
 
+    isTestCommandExpectedToSucceed(${TEST_NAME} ${NotebookTest_PROPERTIES})
+    message(DEBUG "Is test '${TEST_NAME}' expected to succeed? → ${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}")
+
     add_test(
         NAME ${TEST_NAME}
         COMMAND
             ${CMAKE_COMMAND} ${CMAKE_COMMAND}
             # TODO: only works if notebook is in a leaf directory
             -DEXECUTABLE=${Python_EXECUTABLE} "-DEXECUTABLE_ARGS=${_exe_args}"
             -DWORKING_DIRECTORY=${Data_SOURCE_DIR}
-            -DLOG_FILE=${PROJECT_BINARY_DIR}/logs/${NotebookTest_NAME_WE}.txt
+            "-DLOG_ROOT=${PROJECT_BINARY_DIR}/logs"
+            "-DLOG_FILE_BASENAME=${NotebookTest_NAME_WE}.txt"
+            "-DTEST_COMMAND_IS_EXPECTED_TO_SUCCEED=${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}"
             -P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestWrapper.cmake
     )
 

scripts/cmake/test/OgsTest.cmake

@@ -90,7 +90,7 @@ function(OgsTest)
     endif()
 endfunction()
 
-# Add a ctest and sets properties
+# Adds a ctest and sets properties
 macro(_ogs_add_test TEST_NAME)
     if("${TEST_NAME}" MATCHES "-omp")
         set(OgsTest_BINARY_DIR "${Data_BINARY_DIR}/${OgsTest_DIR}-omp")
@@ -100,14 +100,20 @@ macro(_ogs_add_test TEST_NAME)
     file(MAKE_DIRECTORY ${OgsTest_BINARY_DIR})
     file(TO_NATIVE_PATH "${OgsTest_BINARY_DIR}" OgsTest_BINARY_DIR_NATIVE)
     string(REPLACE "/" "_" TEST_NAME_UNDERSCORE ${TEST_NAME})
+
+    isTestCommandExpectedToSucceed(${TEST_NAME} ${OgsTest_PROPERTIES})
+    message(DEBUG "Is test '${TEST_NAME}' expected to succeed? → ${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}")
+
     add_test(
         NAME ${TEST_NAME}
         COMMAND
             ${CMAKE_COMMAND} -DEXECUTABLE=$<TARGET_FILE:ogs>
             "-DEXECUTABLE_ARGS=${_exe_args}"
             "-DWRAPPER_COMMAND=${OgsTest_WRAPPER}"
             -DWORKING_DIRECTORY=${OgsTest_BINARY_DIR}
-            -DLOG_FILE=${PROJECT_BINARY_DIR}/logs/${TEST_NAME_UNDERSCORE}.txt
+            "-DLOG_FILE_BASENAME=${TEST_NAME_UNDERSCORE}.txt"
+            "-DLOG_ROOT=${PROJECT_BINARY_DIR}/logs"
+            "-DTEST_COMMAND_IS_EXPECTED_TO_SUCCEED=${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}"
             -P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestWrapper.cmake
     )