diff --git a/.codecov.yml b/.codecov.yml index 2b56c5c224..3c4f6a9d82 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -4,6 +4,7 @@ # See https://docs.codecov.io/docs/codecovyml-reference codecov: + token: a75b1e95-134c-4ada-adac-5846045f188 require_ci_to_pass: no # codecov reports its results independent of whether CI passed notify: wait_for_ci: no # codecov has not to wait until the CI is finished to post its results @@ -19,3 +20,7 @@ coverage: if_ci_failed: success # per default, codecov would fail if any CI fails informational: true # the codecov/patch status is never "fail" only_pulls: true # only post codecov/patch status on PRs + +parsers: + cobertura: + partials_as_hits: true diff --git a/.github/workflows/ci_coverage.yml b/.github/workflows/ci_coverage.yml index 498dbbfa25..490c10e018 100644 --- a/.github/workflows/ci_coverage.yml +++ b/.github/workflows/ci_coverage.yml @@ -61,8 +61,7 @@ jobs: - name: Configure tests run: | mkdir build && cd build - cmake ../test/coverage -DCMAKE_BUILD_TYPE=Coverage \ - -DSEQAN3_COVERAGE_PARALLEL_LEVEL=$(nproc) + cmake ../test/coverage -DCMAKE_BUILD_TYPE=Coverage make gtest_build - name: Build tests @@ -72,6 +71,27 @@ jobs: make -k ccache -sv + - name: Run tests + working-directory: build + run: ctest . -j --output-on-failure + + - name: Generate coverage report + run: | + gcovr --root ${GITHUB_WORKSPACE}/test/coverage \ + ${GITHUB_WORKSPACE}/build \ + --filter ${GITHUB_WORKSPACE}/include/seqan3 \ + --filter ${GITHUB_WORKSPACE}/test/include/seqan3/test \ + --exclude ${GITHUB_WORKSPACE}/include/seqan3/contrib \ + --exclude ${GITHUB_WORKSPACE}/include/seqan3/std \ + --exclude-lines-by-pattern '^\s*$' \ + --exclude-lines-by-pattern '^\s*};$' \ + --exclude-unreachable-branches \ + --exclude-throw-branches \ + --exclude-noncode-lines \ + -j \ + --cobertura \ + --output ${GITHUB_WORKSPACE}/build/coverage_report.xml + - name: Submit coverage build uses: codecov/codecov-action@v3.1.5 with: diff --git a/.github/workflows/scripts/README.md b/.github/workflows/scripts/README.md index 94c116e620..ee7337d6cf 100644 --- a/.github/workflows/scripts/README.md +++ b/.github/workflows/scripts/README.md @@ -11,12 +11,6 @@ nothing additionial. This mode might be necessary for CMake to properly detect Z When `DO_TIME` is `1`, every call will generate a `ram_usage.*` file with an unique extension containing the output of `time -v`. -## gcov.sh - -We use `gcovr` to create our coverage reports. We do not want to report branch coverage, but only line coverage. -However, when `gcovr` calls `gcov`, the arguments enabling branch coverage reporting are always added. -`gcov.sh` will remove those arguments and pass the remaining arguments to `gcov`. - ## process_compiler_error_log.py Processes the output of compiling tests (`make`) and extracts errors. diff --git a/.github/workflows/scripts/gcov.sh b/.github/workflows/scripts/gcov.sh deleted file mode 100755 index f28434d435..0000000000 --- a/.github/workflows/scripts/gcov.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -# SPDX-FileCopyrightText: 2006-2024 Knut Reinert & Freie Universität Berlin -# SPDX-FileCopyrightText: 2016-2024 Knut Reinert & MPI für molekulare Genetik -# SPDX-License-Identifier: CC0-1.0 - - -args=${@/--branch-counts/""} -args=${args/--branch-probabilities/""} - -exec gcov $args diff --git a/test/coverage/CMakeLists.txt b/test/coverage/CMakeLists.txt index eed8508cdf..10046684d1 100644 --- a/test/coverage/CMakeLists.txt +++ b/test/coverage/CMakeLists.txt @@ -5,135 +5,32 @@ cmake_minimum_required (VERSION 3.10...3.22) project (seqan3_test_coverage CXX) -include (../seqan3-test.cmake) - -if (CMAKE_BUILD_TYPE AND NOT ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")) - message (WARNING "Coverage test must be build in debug mode [build type = ${CMAKE_BUILD_TYPE}]") -endif () - -find_program (GCOVR_COMMAND NAMES gcovr) - -if (NOT GCOVR_COMMAND) - message (FATAL_ERROR "gcovr not found! Aborting...") -endif () - -# Holds all target's defined by seqan3_test -set_property (GLOBAL PROPERTY GLOBAL_TEST_COVERAGE_ALL_TESTS "") - -set (SEQAN3_COVERAGE_PARALLEL_LEVEL - "1" - CACHE STRING "Number of threads to use for coverage report generation.") - -set (SEQAN3_GCOVR_ARGUMENTS "") - -macro (seqan3_append_gcovr_args) - set (key ${ARGV0}) - if (${ARGC} EQUAL 1) - list (APPEND SEQAN3_GCOVR_ARGUMENTS ${key}) - else () - string (REPLACE "'^" "\\''^" value "${ARGV1}") - string (REPLACE "$'" "$$'\\'" value "${value}") - string (REPLACE ";" "\\\\\\;" value "${value}") - list (APPEND SEQAN3_GCOVR_ARGUMENTS ${key} ${value}) - endif () -endmacro () - -# Arguments for gcovr. -# Certain chracters will be escaped by seqan3_append_gcovr_args. -# Special CMake characters such as "\" must be escaped manually with "\\". -# We can prevent one level of CMake's string evaluation by using bracket arguments, which are similar to raw strings: -# (https://cmake.org/cmake/help/latest/manual/cmake-language.7.html#bracket-argument) -# gcovr uses python regex expressions. -# See https://gcovr.com/en/5.0/guide.html#the-gcovr-command for an overview of parameters. -# See https://gcovr.com/en/5.0/faq.html#why-does-c-code-have-so-many-uncovered-branches for an explanation on branches. - -# The directory of the CMakeLists.txt used for invoking cmake. -seqan3_append_gcovr_args ("--root" "${CMAKE_CURRENT_LIST_DIR}") -# The build directory. -seqan3_append_gcovr_args ("${PROJECT_BINARY_DIR}") -# Include all files whose path match '${SEQAN3_CLONE_DIR}/include/seqan3/.*'. -seqan3_append_gcovr_args ("--filter" "${SEQAN3_CLONE_DIR}/include/seqan3") -# Include all files whose path match '${SEQAN3_CLONE_DIR}/test/include/seqan3/test/.*'. -seqan3_append_gcovr_args ("--filter" "${SEQAN3_CLONE_DIR}/test/include/seqan3/test") -# Remove all files whose path match '${SEQAN3_CLONE_DIR}/include/seqan3/contrib/.*'. -seqan3_append_gcovr_args ("--exclude" "${SEQAN3_CLONE_DIR}/include/seqan3/contrib") -# Remove all files whose path match '${SEQAN3_CLONE_DIR}/include/seqan3/std/.*'. -seqan3_append_gcovr_args ("--exclude" "${SEQAN3_CLONE_DIR}/include/seqan3/std") -# Remove line coverage for all lines that match '^\s*$', i.e. empty lines. -seqan3_append_gcovr_args ("--exclude-lines-by-pattern" [['^\\s*$']]) -# Remove line coverage for all lines that match "^\s*[{}]{0,2}\s*;*\s*(//.*)?$", i.e. lines with a single '{' or '}' -# or a combination of those (max 2) which might be followed by a semicolon or a comment '//'. -seqan3_append_gcovr_args ("--exclude-lines-by-pattern" [['^\\s*[{}]{0,2}\\s*;*\\s*(//.*)?$']]) -# Will exclude branches that are unreachable. -seqan3_append_gcovr_args ("--exclude-unreachable-branches") -# Will exclude branches that are only generated for exception handling. -seqan3_append_gcovr_args ("--exclude-throw-branches") -# Will exclude non-code lines, e.g. lines with closing braces. -seqan3_append_gcovr_args ("--exclude-noncode-lines") -# Run up to this many gcov instances in parallel. -seqan3_append_gcovr_args ("-j" "${SEQAN3_COVERAGE_PARALLEL_LEVEL}") - -add_custom_command (OUTPUT ${PROJECT_BINARY_DIR}/seqan3_coverage.xml - # Run tests. - COMMAND ${CMAKE_CTEST_COMMAND} -j ${SEQAN3_COVERAGE_PARALLEL_LEVEL} --output-on-failure - # Run gcovr and create XML report. - COMMAND ${GCOVR_COMMAND} ${SEQAN3_GCOVR_ARGUMENTS} --xml --output - ${PROJECT_BINARY_DIR}/seqan3_coverage.xml - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - COMMENT "Processing code coverage counters and generating XML report." - COMMAND_EXPAND_LISTS) - -add_custom_target (coverage ALL DEPENDS ${PROJECT_BINARY_DIR}/seqan3_coverage.xml) - -add_custom_command (OUTPUT ${PROJECT_BINARY_DIR}/html/index.html - # Run tests. - COMMAND ${CMAKE_CTEST_COMMAND} -j ${SEQAN3_COVERAGE_PARALLEL_LEVEL} --output-on-failure - # Create output directory. - COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/html/ - # Run gcovr and create HTML report. - COMMAND ${GCOVR_COMMAND} ${SEQAN3_GCOVR_ARGUMENTS} --html-details --output - ${PROJECT_BINARY_DIR}/html/index.html - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - COMMENT "Processing code coverage counters and generating HTML report." - COMMAND_EXPAND_LISTS) - -add_custom_target (coverage_html - DEPENDS ${PROJECT_BINARY_DIR}/html/index.html - COMMENT "Generate coverage report.") - -add_custom_command (TARGET coverage_html - POST_BUILD - COMMAND ; - COMMENT "Open ${PROJECT_BINARY_DIR}/html/index.html in your browser to view the coverage report.") - -macro (seqan3_test unit_test_cpp) - file (RELATIVE_PATH unit_test "${CMAKE_SOURCE_DIR}/../unit" "${CMAKE_CURRENT_LIST_DIR}/${unit_test_cpp}") - seqan3_test_component (target "${unit_test}" TARGET_NAME) - seqan3_test_component (test_name "${unit_test}" TEST_NAME) - - add_executable (${target} ${unit_test_cpp}) - target_link_libraries (${target} seqan3::test::coverage) - add_test (NAME "${test_name}" COMMAND ${target}) - - # any change of a target will invalidate the coverage result; - # NOTE that this is a GLOBAL variable, because a normal - # `set(GLOBAL_TEST_COVERAGE_ALL_TESTS)` would not propagate the result when - # CMakeLists.txt goes out of scope due to a `add_subdirectory` - set_property (GLOBAL APPEND PROPERTY GLOBAL_TEST_COVERAGE_ALL_TESTS ${target}) - - unset (unit_test) - unset (target) - unset (test_name) -endmacro () - -seqan3_require_ccache () -seqan3_require_test () - -# add all unit tests -add_subdirectories_of ("${CMAKE_CURRENT_SOURCE_DIR}/../unit") - -# add collected test cases as dependency -get_property (TEST_COVERAGE_ALL_TESTS GLOBAL PROPERTY GLOBAL_TEST_COVERAGE_ALL_TESTS) -add_custom_command (OUTPUT ${PROJECT_BINARY_DIR}/seqan3_coverage.xml - DEPENDS ${TEST_COVERAGE_ALL_TESTS} - APPEND) +# Add a custom build type: Coverage +# +# `--coverage` is equivalent to `-fprofile-arcs -ftest-coverage` +# https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html#index-gcov +# +# `-fprofile-update=atomic` prevents profile corruption in multi-threaded tests +# https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html#index-fprofile-update +# +# `-fprofile-abs-path` converts relative source file names to absolute paths in the coverage files +# https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html#index-fprofile-abs-path + +set (CMAKE_CXX_FLAGS_COVERAGE + "${CMAKE_CXX_FLAGS_DEBUG} --coverage -g -O0 -fprofile-abs-path -fprofile-update=atomic" + CACHE STRING "Flags used by the C++ compiler during coverage builds." FORCE) +set (CMAKE_C_FLAGS_COVERAGE + "${CMAKE_C_FLAGS_DEBUG} --coverage -g -O0 -fprofile-abs-path -fprofile-update=atomic" + CACHE STRING "Flags used by the C compiler during coverage builds." FORCE) +set (CMAKE_EXE_LINKER_FLAGS_COVERAGE + "${CMAKE_EXE_LINKER_FLAGS_DEBUG} -Wl,-lgcov" + CACHE STRING "Flags used for linking binaries during coverage builds." FORCE) +set (CMAKE_SHARED_LINKER_FLAGS_COVERAGE + "${CMAKE_SHARED_LINKER_FLAGS_DEBUG} -Wl,-lgcov" + CACHE STRING "Flags used by the shared libraries linker during coverage builds." FORCE) + +mark_as_advanced (CMAKE_CXX_FLAGS_COVERAGE CMAKE_C_FLAGS_COVERAGE CMAKE_EXE_LINKER_FLAGS_COVERAGE + CMAKE_SHARED_LINKER_FLAGS_COVERAGE) + +enable_testing () +add_subdirectory ("../unit" "${CMAKE_CURRENT_BINARY_DIR}/unit") diff --git a/test/seqan3-test.cmake b/test/seqan3-test.cmake index 5632235bce..71a4d69268 100644 --- a/test/seqan3-test.cmake +++ b/test/seqan3-test.cmake @@ -97,24 +97,6 @@ if (NOT TARGET seqan3::test::unit) add_library (seqan3::test::unit ALIAS seqan3_test_unit) endif () -# seqan3::test::coverage specifies required flags, includes and libraries -# needed for coverage test cases in seqan3/test/coverage -if (NOT TARGET seqan3::test::coverage) - add_library (seqan3_test_coverage INTERFACE) - target_compile_options (seqan3_test_coverage INTERFACE "--coverage" "-fprofile-arcs" "-ftest-coverage" "-fprofile-update=atomic") - # -fprofile-abs-path requires at least gcc8, it forces gcov to report absolute instead of relative paths. - # gcovr has trouble detecting the headers otherwise. - # ccache is not aware of this option, so it needs to be skipped with `--ccache-skip`. - find_program (CCACHE_PROGRAM ccache) - if (CCACHE_PROGRAM) - target_compile_options (seqan3_test_coverage INTERFACE "--ccache-skip" "-fprofile-abs-path") - else () - target_compile_options (seqan3_test_coverage INTERFACE "-fprofile-abs-path") - endif () - target_link_libraries (seqan3_test_coverage INTERFACE "seqan3::test::unit" "gcov") - add_library (seqan3::test::coverage ALIAS seqan3_test_coverage) -endif () - # seqan3::test::header specifies required flags, includes and libraries # needed for header test cases in seqan3/test/header if (NOT TARGET seqan3::test::header)