[INFRA] Simplify coverage

seqan · Jul 17, 2024 · 3a16f5a · 3a16f5a
1 parent 4ce9e00
commit 3a16f5a
Show file tree

Hide file tree

Showing 6 changed files with 56 additions and 169 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -4,6 +4,7 @@
 
 # See https://docs.codecov.io/docs/codecovyml-reference
 codecov:
+  token: a75b1e95-134c-4ada-adac-5846045f188
   require_ci_to_pass: no  # codecov reports its results independent of whether CI passed
   notify:
     wait_for_ci: no       # codecov has not to wait until the CI is finished to post its results
@@ -19,3 +20,7 @@ coverage:
         if_ci_failed: success # per default, codecov would fail if any CI fails
         informational: true   # the codecov/patch status is never "fail"
         only_pulls: true      # only post codecov/patch status on PRs
+
+parsers:
+  cobertura:
+    partials_as_hits: true
diff --git a/.github/workflows/ci_coverage.yml b/.github/workflows/ci_coverage.yml
@@ -61,8 +61,7 @@ jobs:
       - name: Configure tests
         run: |
           mkdir build && cd build
-          cmake ../test/coverage -DCMAKE_BUILD_TYPE=Coverage \
-                                 -DSEQAN3_COVERAGE_PARALLEL_LEVEL=$(nproc)
+          cmake ../test/coverage -DCMAKE_BUILD_TYPE=Coverage
           make gtest_build
 
       - name: Build tests
@@ -72,6 +71,27 @@ jobs:
           make -k
           ccache -sv
 
+      - name: Run tests
+        working-directory: build
+        run: ctest . -j --output-on-failure
+
+      - name: Generate coverage report
+        run: |
+          gcovr --root ${GITHUB_WORKSPACE}/test/coverage \
+                ${GITHUB_WORKSPACE}/build \
+                --filter ${GITHUB_WORKSPACE}/include/seqan3 \
+                --filter ${GITHUB_WORKSPACE}/test/include/seqan3/test \
+                --exclude ${GITHUB_WORKSPACE}/include/seqan3/contrib \
+                --exclude ${GITHUB_WORKSPACE}/include/seqan3/std \
+                --exclude-lines-by-pattern '^\s*$' \
+                --exclude-lines-by-pattern '^\s*};$' \
+                --exclude-unreachable-branches \
+                --exclude-throw-branches \
+                --exclude-noncode-lines \
+                -j \
+                --cobertura \
+                --output ${GITHUB_WORKSPACE}/build/coverage_report.xml
+
       - name: Submit coverage build
         uses: codecov/[email protected]
         with:

diff --git a/.github/workflows/scripts/README.md b/.github/workflows/scripts/README.md
@@ -11,12 +11,6 @@ nothing additionial. This mode might be necessary for CMake to properly detect Z
 When `DO_TIME` is `1`, every call will generate a `ram_usage.*` file with an unique extension containing the output
 of `time -v`.
 
-## gcov.sh
-
-We use `gcovr` to create our coverage reports. We do not want to report branch coverage, but only line coverage.
-However, when `gcovr` calls `gcov`, the arguments enabling branch coverage reporting are always added.
-`gcov.sh` will remove those arguments and pass the remaining arguments to `gcov`.
-
 ## process_compiler_error_log.py
 
 Processes the output of compiling tests (`make`) and extracts errors.

diff --git a/.github/workflows/scripts/gcov.sh b/.github/workflows/scripts/gcov.sh
diff --git a/test/coverage/CMakeLists.txt b/test/coverage/CMakeLists.txt
@@ -5,135 +5,32 @@
 cmake_minimum_required (VERSION 3.10...3.22)
 project (seqan3_test_coverage CXX)
 
-include (../seqan3-test.cmake)
-
-if (CMAKE_BUILD_TYPE AND NOT ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug"))
-    message (WARNING "Coverage test must be build in debug mode [build type = ${CMAKE_BUILD_TYPE}]")
-endif ()
-
-find_program (GCOVR_COMMAND NAMES gcovr)
-
-if (NOT GCOVR_COMMAND)
-    message (FATAL_ERROR "gcovr not found! Aborting...")
-endif ()
-
-# Holds all target's defined by seqan3_test
-set_property (GLOBAL PROPERTY GLOBAL_TEST_COVERAGE_ALL_TESTS "")
-
-set (SEQAN3_COVERAGE_PARALLEL_LEVEL
-     "1"
-     CACHE STRING "Number of threads to use for coverage report generation.")
-
-set (SEQAN3_GCOVR_ARGUMENTS "")
-
-macro (seqan3_append_gcovr_args)
-    set (key ${ARGV0})
-    if (${ARGC} EQUAL 1)
-        list (APPEND SEQAN3_GCOVR_ARGUMENTS ${key})
-    else ()
-        string (REPLACE "'^" "\\''^" value "${ARGV1}")
-        string (REPLACE "$'" "$$'\\'" value "${value}")
-        string (REPLACE ";" "\\\\\\;" value "${value}")
-        list (APPEND SEQAN3_GCOVR_ARGUMENTS ${key} ${value})
-    endif ()
-endmacro ()
-
-# Arguments for gcovr.
-# Certain chracters will be escaped by seqan3_append_gcovr_args.
-# Special CMake characters such as "\" must be escaped manually with "\\".
-# We can prevent one level of CMake's string evaluation by using bracket arguments, which are similar to raw strings:
-# (https://cmake.org/cmake/help/latest/manual/cmake-language.7.html#bracket-argument)
-# gcovr uses python regex expressions.
-# See https://gcovr.com/en/5.0/guide.html#the-gcovr-command for an overview of parameters.
-# See https://gcovr.com/en/5.0/faq.html#why-does-c-code-have-so-many-uncovered-branches for an explanation on branches.
-
-# The directory of the CMakeLists.txt used for invoking cmake.
-seqan3_append_gcovr_args ("--root" "${CMAKE_CURRENT_LIST_DIR}")
-# The build directory.
-seqan3_append_gcovr_args ("${PROJECT_BINARY_DIR}")
-# Include all files whose path match '${SEQAN3_CLONE_DIR}/include/seqan3/.*'.
-seqan3_append_gcovr_args ("--filter" "${SEQAN3_CLONE_DIR}/include/seqan3")
-# Include all files whose path match '${SEQAN3_CLONE_DIR}/test/include/seqan3/test/.*'.
-seqan3_append_gcovr_args ("--filter" "${SEQAN3_CLONE_DIR}/test/include/seqan3/test")
-# Remove all files whose path match '${SEQAN3_CLONE_DIR}/include/seqan3/contrib/.*'.
-seqan3_append_gcovr_args ("--exclude" "${SEQAN3_CLONE_DIR}/include/seqan3/contrib")
-# Remove all files whose path match '${SEQAN3_CLONE_DIR}/include/seqan3/std/.*'.
-seqan3_append_gcovr_args ("--exclude" "${SEQAN3_CLONE_DIR}/include/seqan3/std")
-# Remove line coverage for all lines that match '^\s*$', i.e. empty lines.
-seqan3_append_gcovr_args ("--exclude-lines-by-pattern" [['^\\s*$']])
-# Remove line coverage for all lines that match "^\s*[{}]{0,2}\s*;*\s*(//.*)?$", i.e. lines with a single '{' or '}'
-# or a combination of those (max 2) which might be followed by a semicolon or a comment '//'.
-seqan3_append_gcovr_args ("--exclude-lines-by-pattern" [['^\\s*[{}]{0,2}\\s*;*\\s*(//.*)?$']])
-# Will exclude branches that are unreachable.
-seqan3_append_gcovr_args ("--exclude-unreachable-branches")
-# Will exclude branches that are only generated for exception handling.
-seqan3_append_gcovr_args ("--exclude-throw-branches")
-# Will exclude non-code lines, e.g. lines with closing braces.
-seqan3_append_gcovr_args ("--exclude-noncode-lines")
-# Run up to this many gcov instances in parallel.
-seqan3_append_gcovr_args ("-j" "${SEQAN3_COVERAGE_PARALLEL_LEVEL}")
-
-add_custom_command (OUTPUT ${PROJECT_BINARY_DIR}/seqan3_coverage.xml
-                    # Run tests.
-                    COMMAND ${CMAKE_CTEST_COMMAND} -j ${SEQAN3_COVERAGE_PARALLEL_LEVEL} --output-on-failure
-                    # Run gcovr and create XML report.
-                    COMMAND ${GCOVR_COMMAND} ${SEQAN3_GCOVR_ARGUMENTS} --xml --output
-                            ${PROJECT_BINARY_DIR}/seqan3_coverage.xml
-                    WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
-                    COMMENT "Processing code coverage counters and generating XML report."
-                    COMMAND_EXPAND_LISTS)
-
-add_custom_target (coverage ALL DEPENDS ${PROJECT_BINARY_DIR}/seqan3_coverage.xml)
-
-add_custom_command (OUTPUT ${PROJECT_BINARY_DIR}/html/index.html
-                    # Run tests.
-                    COMMAND ${CMAKE_CTEST_COMMAND} -j ${SEQAN3_COVERAGE_PARALLEL_LEVEL} --output-on-failure
-                    # Create output directory.
-                    COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/html/
-                    # Run gcovr and create HTML report.
-                    COMMAND ${GCOVR_COMMAND} ${SEQAN3_GCOVR_ARGUMENTS} --html-details --output
-                            ${PROJECT_BINARY_DIR}/html/index.html
-                    WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
-                    COMMENT "Processing code coverage counters and generating HTML report."
-                    COMMAND_EXPAND_LISTS)
-
-add_custom_target (coverage_html
-                   DEPENDS ${PROJECT_BINARY_DIR}/html/index.html
-                   COMMENT "Generate coverage report.")
-
-add_custom_command (TARGET coverage_html
-                    POST_BUILD
-                    COMMAND ;
-                    COMMENT "Open ${PROJECT_BINARY_DIR}/html/index.html in your browser to view the coverage report.")
-
-macro (seqan3_test unit_test_cpp)
-    file (RELATIVE_PATH unit_test "${CMAKE_SOURCE_DIR}/../unit" "${CMAKE_CURRENT_LIST_DIR}/${unit_test_cpp}")
-    seqan3_test_component (target "${unit_test}" TARGET_NAME)
-    seqan3_test_component (test_name "${unit_test}" TEST_NAME)
-
-    add_executable (${target} ${unit_test_cpp})
-    target_link_libraries (${target} seqan3::test::coverage)
-    add_test (NAME "${test_name}" COMMAND ${target})
-
-    # any change of a target will invalidate the coverage result;
-    # NOTE that this is a GLOBAL variable, because a normal
-    # `set(GLOBAL_TEST_COVERAGE_ALL_TESTS)` would not propagate the result when
-    # CMakeLists.txt goes out of scope due to a `add_subdirectory`
-    set_property (GLOBAL APPEND PROPERTY GLOBAL_TEST_COVERAGE_ALL_TESTS ${target})
-
-    unset (unit_test)
-    unset (target)
-    unset (test_name)
-endmacro ()
-
-seqan3_require_ccache ()
-seqan3_require_test ()
-
-# add all unit tests
-add_subdirectories_of ("${CMAKE_CURRENT_SOURCE_DIR}/../unit")
-
-# add collected test cases as dependency
-get_property (TEST_COVERAGE_ALL_TESTS GLOBAL PROPERTY GLOBAL_TEST_COVERAGE_ALL_TESTS)
-add_custom_command (OUTPUT ${PROJECT_BINARY_DIR}/seqan3_coverage.xml
-                    DEPENDS ${TEST_COVERAGE_ALL_TESTS}
-                    APPEND)
+# Add a custom build type: Coverage
+#
+# `--coverage` is equivalent to `-fprofile-arcs -ftest-coverage`
+# https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html#index-gcov
+#
+# `-fprofile-update=atomic` prevents profile corruption in multi-threaded tests
+# https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html#index-fprofile-update
+#
+# `-fprofile-abs-path` converts relative source file names to absolute paths in the coverage files
+# https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html#index-fprofile-abs-path
+
+set (CMAKE_CXX_FLAGS_COVERAGE
+     "${CMAKE_CXX_FLAGS_DEBUG} --coverage -g -O0 -fprofile-abs-path -fprofile-update=atomic"
+     CACHE STRING "Flags used by the C++ compiler during coverage builds." FORCE)
+set (CMAKE_C_FLAGS_COVERAGE
+     "${CMAKE_C_FLAGS_DEBUG} --coverage -g -O0 -fprofile-abs-path -fprofile-update=atomic"
+     CACHE STRING "Flags used by the C compiler during coverage builds." FORCE)
+set (CMAKE_EXE_LINKER_FLAGS_COVERAGE
+     "${CMAKE_EXE_LINKER_FLAGS_DEBUG} -Wl,-lgcov"
+     CACHE STRING "Flags used for linking binaries during coverage builds." FORCE)
+set (CMAKE_SHARED_LINKER_FLAGS_COVERAGE
+     "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -Wl,-lgcov"
+     CACHE STRING "Flags used by the shared libraries linker during coverage builds." FORCE)
+
+mark_as_advanced (CMAKE_CXX_FLAGS_COVERAGE CMAKE_C_FLAGS_COVERAGE CMAKE_EXE_LINKER_FLAGS_COVERAGE
+                  CMAKE_SHARED_LINKER_FLAGS_COVERAGE)
+
+enable_testing ()
+add_subdirectory ("../unit" "${CMAKE_CURRENT_BINARY_DIR}/unit")
diff --git a/test/seqan3-test.cmake b/test/seqan3-test.cmake
@@ -97,24 +97,6 @@ if (NOT TARGET seqan3::test::unit)
     add_library (seqan3::test::unit ALIAS seqan3_test_unit)
 endif ()
 
-# seqan3::test::coverage specifies required flags, includes and libraries
-# needed for coverage test cases in seqan3/test/coverage
-if (NOT TARGET seqan3::test::coverage)
-    add_library (seqan3_test_coverage INTERFACE)
-    target_compile_options (seqan3_test_coverage INTERFACE "--coverage" "-fprofile-arcs" "-ftest-coverage" "-fprofile-update=atomic")
-    # -fprofile-abs-path requires at least gcc8, it forces gcov to report absolute instead of relative paths.
-    # gcovr has trouble detecting the headers otherwise.
-    # ccache is not aware of this option, so it needs to be skipped with `--ccache-skip`.
-    find_program (CCACHE_PROGRAM ccache)
-    if (CCACHE_PROGRAM)
-        target_compile_options (seqan3_test_coverage INTERFACE "--ccache-skip" "-fprofile-abs-path")
-    else ()
-        target_compile_options (seqan3_test_coverage INTERFACE "-fprofile-abs-path")
-    endif ()
-    target_link_libraries (seqan3_test_coverage INTERFACE "seqan3::test::unit" "gcov")
-    add_library (seqan3::test::coverage ALIAS seqan3_test_coverage)
-endif ()
-
 # seqan3::test::header specifies required flags, includes and libraries
 # needed for header test cases in seqan3/test/header
 if (NOT TARGET seqan3::test::header)