diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5570409..6f1bedf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,7 @@ jobs: needs: checks name: ${{ matrix.config.name }} runs-on: ${{ matrix.config.os }} + continue-on-error: false strategy: fail-fast: false matrix: @@ -101,23 +102,23 @@ jobs: cmake_args: "-DCMAKE_OSX_ARCHITECTURES=arm64", } - # # AppleClang - # - { - # name: "Clang 18 / LLVM 18 @ macOS Coverage & Debug", - # os: macos-14, - # build_type: Release, - # cxx: "clang++", - # llvm_version: "18.1.0", - # llvm_config: "llvm-config", - # coverage: "Yes", - # static: "No", - # debug: "Yes", - # tidy: "No", - # run_tests: "No", - # bin_name: "insights", - # archive_name: "insights-macos", - # upload: "No", - # } + # AppleClang + - { + name: "Clang 18 / LLVM 18 @ macOS Coverage & Debug", + os: macos-14, + build_type: Release, + cxx: "clang++", + llvm_version: "18.1.0", + llvm_config: "llvm-config", + coverage: "Yes", + static: "No", + debug: "Yes", + tidy: "No", + run_tests: "No", + bin_name: "insights", + archive_name: "insights-macos", + upload: "No", + } # # MSVC 2019 # - { @@ -264,7 +265,7 @@ jobs: key: ${{ runner.os }}-clang-${{ matrix.config.llvm_version }}-${{ hashFiles('${{ github.workspace }}/current') }} - name: Install Clang - if: "(startsWith(matrix.config.os, 'macos') || startsWith(matrix.config.os, 'Window'))" + if: "(startsWith(matrix.config.os, 'macos') || startsWith(matrix.config.os, 'Window')) && steps.cache-clang-binary.outputs.cache-hit != 'true'" shell: cmake -P {0} run: | set(llvm_version ${{ matrix.config.llvm_version }}) @@ -304,6 +305,7 @@ jobs: - name: Cache grcov id: cache-grcov-binary + if: "(startsWith(matrix.config.os, 'Window') && (matrix.config.coverage == 'Yes'))" uses: actions/cache@v4 with: path: | @@ -319,14 +321,6 @@ jobs: execute_process(COMMAND ${CMAKE_COMMAND} -E tar xf ./grcov.tar.bz2) execute_process(COMMAND ${CMAKE_COMMAND} -E remove ./grcov.tar.bz2) - - name: Install lcov for macOS - if: "(startsWith(matrix.config.os, 'macos') && (matrix.config.coverage == 'Yes'))" - run: | - export HOMEBREW_NO_AUTO_UPDATE=1 - brew update > /dev/null - brew install lcov || brew upgrade lcov - brew install gcc@10 || brew upgrade gcc@10 # for Clang 18 and newer gcov - - name: Setup MSVC Dev if: "startsWith(matrix.config.os, 'Windows')" uses: ilammy/msvc-dev-cmd@v1 @@ -375,7 +369,6 @@ jobs: - name: Build shell: cmake -P {0} - continue-on-error: false run: | execute_process( COMMAND ${{ steps.cmake_and_ninja_setup.outputs.cmake_dir }}/cmake --build build @@ -389,7 +382,6 @@ jobs: id: run_tests if: matrix.config.run_tests == 'Yes' shell: cmake -P {0} - continue-on-error: false run: | if ("${{ runner.os }}" STREQUAL "macOS") set(ENV{CPLUS_INCLUDE_PATH} "$ENV{GITHUB_WORKSPACE}/current/include/c++/v1:/Library/Developer/CommandLineTools/SDKs/MacOSX11.sdk/usr/include") @@ -407,7 +399,6 @@ jobs: - name: Run coverage if: matrix.config.coverage == 'Yes' shell: cmake -P {0} - continue-on-error: false run: | if ("${{ runner.os }}" STREQUAL "macOS") set(ENV{CPLUS_INCLUDE_PATH} "$ENV{GITHUB_WORKSPACE}/current/include/c++/v1:/Library/Developer/CommandLineTools/SDKs/MacOSX11.sdk/usr/include") @@ -427,15 +418,16 @@ jobs: uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} - file: ./build/filtered.info # don't use a absolute path on Windows with gitBash. + files: ./build/filtered.info # don't use a absolute path on Windows with gitBash. + exclude: build flags: ${{ matrix.config.archive_name }} + disable_search: true fail_ci_if_error: true - name: Create archive if: matrix.config.upload == 'Yes' shell: cmake -P {0} working-directory: ${{ github.workspace }}/build - continue-on-error: false run: | file(MAKE_DIRECTORY "$ENV{GITHUB_WORKSPACE}/build/archive") execute_process(COMMAND ${CMAKE_COMMAND} -E tar cvz "$ENV{GITHUB_WORKSPACE}/build/archive/${{ matrix.config.archive_name }}.tar.gz" -- "${{ matrix.config.bin_name }}") @@ -457,6 +449,7 @@ jobs: docker: runs-on: ubuntu-22.04 name: ${{ matrix.config.name }} @${{ matrix.arch }} + continue-on-error: false strategy: fail-fast: false matrix: @@ -525,7 +518,6 @@ jobs: - name: Create docker shell shell: bash - continue-on-error: false run: | cat > docker-shell << "EOF" #! /bin/bash @@ -565,14 +557,12 @@ jobs: - name: Build shell: docker-shell {0} - continue-on-error: false run: | cmake --build build - name: Simple test if: "((matrix.config.coverage != 'Yes') && (matrix.config.run_tests != 'Yes'))" shell: docker-shell {0} - continue-on-error: false run: | ./build/insights cppinsights/tests/SimpleCICompileTest.cpp ./build/insights --use-libc++ cppinsights/tests/SimpleCICompileTest.cpp @@ -581,7 +571,6 @@ jobs: if: matrix.config.run_tests == 'Yes' id: run_tests shell: docker-shell {0} - continue-on-error: false run: | cmake --build build --target tests @@ -608,7 +597,6 @@ jobs: if: matrix.config.coverage == 'Yes' id: run_coverage shell: docker-shell {0} - continue-on-error: false run: | cmake --build build --target coverage rm -f build/coverage.info @@ -619,15 +607,16 @@ jobs: uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} - file: ./build/filtered.info # don't use a absolute path on Windows with gitBash. + files: ./build/filtered.info # don't use a absolute path on Windows with gitBash. + exclude: build flags: ${{ matrix.config.archive_name }}-${{ matrix.arch }}-libcxx-${{ matrix.config.libcxx }} + disable_search: true fail_ci_if_error: true - name: Create archive if: matrix.config.upload == 'Yes' working-directory: ${{ github.workspace }}/build shell: bash - continue-on-error: false run: | mkdir -p ${GITHUB_WORKSPACE}/build/archive @@ -648,6 +637,7 @@ jobs: needs: [build, docker] if: github.ref == 'refs/heads/main' name: Final Deploy + continue-on-error: false runs-on: ubuntu-22.04 steps: diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b7430d..aee8b0b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -347,18 +347,22 @@ if (BUILD_INSIGHTS_OUTSIDE_LLVM) set(COVERAGE_LINK_FLAGS "-lgcov") endif() - if(WIN32) + if(WIN32) add_definitions(-fprofile-instr-generate) set(COVERAGE_LINK_FLAGS "${COVERAGE_LINK_FLAGS} ${LLVM_LIBDIR}/clang/${LLVM_PACKAGE_VERSION_PLAIN}/lib/windows/clang_rt.profile-x86_64.lib /FORCE:MULTIPLE") - else() + else() add_definitions(-g) add_definitions(-O0) -# add_definitions(-fprofile-arcs) +# add_definitions(-fprofile-arcs) add_definitions(-ftest-coverage) set(COVERAGE_LINK_FLAGS "${COVERAGE_LINK_FLAGS} --coverage") - endif() + endif() - add_definitions(--coverage) + if(IS_CLANG) + add_definitions(-fprofile-instr-generate -fcoverage-mapping) + else() + add_definitions(--coverage) + endif() set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${COVERAGE_LINK_FLAGS}") endif() @@ -425,12 +429,12 @@ if(WIN32) set( CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} ) set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} ) # Second, for multi-config builds (e.g. msvc) - foreach( OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES} ) - string( TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG ) - set( CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_BINARY_DIR} ) - set( CMAKE_LIBRARY_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_BINARY_DIR} ) - set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_BINARY_DIR} ) - endforeach( OUTPUTCONFIG CMAKE_CONFIGURATION_TYPES ) + foreach( OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES} ) + string( TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG ) + set( CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_BINARY_DIR} ) + set( CMAKE_LIBRARY_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_BINARY_DIR} ) + set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_BINARY_DIR} ) + endforeach( OUTPUTCONFIG CMAKE_CONFIGURATION_TYPES ) endif() @@ -608,6 +612,11 @@ if(INSIGHTS_COVERAGE OR WIN32) set(TEST_FAILURE_IS_OK "--failure-is-ok") endif() +set(LLVM_PROF_DIR "") +if(INSIGHTS_COVERAGE AND IS_CLANG) + set(LLVM_PROF_DIR "--llvm-prof-dir=${CMAKE_CURRENT_BINARY_DIR}/llvmprof") +endif() + set(TEST_USE_LIBCPP "") if(INSIGHTS_USE_LIBCPP) set(TEST_USE_LIBCPP "--use-libcpp") @@ -628,7 +637,9 @@ else() # add a target to generate run tests add_custom_target(tests - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/runTest.py --insights ${CMAKE_CURRENT_BINARY_DIR}/$ --cxx ${CMAKE_CXX_COMPILER} ${TEST_FAILURE_IS_OK} ${TEST_USE_LIBCPP} + COMMAND cmake -E rm -rf ${CMAKE_CURRENT_BINARY_DIR}/llvmprof/ + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/runTest.py --insights + ${CMAKE_CURRENT_BINARY_DIR}/$ --cxx ${CMAKE_CXX_COMPILER} ${TEST_FAILURE_IS_OK} ${TEST_USE_LIBCPP} ${LLVM_PROF_DIR} COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/testSTDIN.sh ${CMAKE_CURRENT_BINARY_DIR}/$ COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/testInvalidOption.sh ${CMAKE_CURRENT_BINARY_DIR}/$ DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/$ ${CMAKE_CURRENT_SOURCE_DIR}/tests/runTest.py @@ -718,14 +729,13 @@ if(INSIGHTS_COVERAGE) COMMENT "Generating code coverage HTML" VERBATIM ) endif() - else() + elseif(IS_GNU) find_program(LCOV_BIN lcov) find_program(GENHTML_BIN genhtml) find_package_handle_standard_args(lcov REQUIRED_VARS LCOV_BIN GENHTML_BIN ) - if (NOT LCOV_FOUND) message(FATAL_ERROR "Lcov not found") else() @@ -764,8 +774,38 @@ if(INSIGHTS_COVERAGE) COMMAND genhtml ${CMAKE_CURRENT_BINARY_DIR}/filtered.info --demangle-cpp --output-directory ${CMAKE_CURRENT_BINARY_DIR}/out COMMENT "Generating code coverage HTML" VERBATIM ) - endif() + + else() # Clang + find_program(LCOV_BIN llvm-cov) + find_program(PROFDATA_BIN llvm-profdata) + # find_package_handle_standard_args(llvm-lcov + # REQUIRED_VARS LCOV_BIN PROFDATA_BIN + # ) + + + # if (NOT LCOV_FOUND) + # message(FATAL_ERROR "llvm-cov not found") + # elseif (NOT PROFDATA_FOUND) + # message(FATAL_ERROR "llvm-profdata not found") + # else() + message(STATUS "Target coverage available") + message(STATUS "lcov : ${LCOV_BIN}") + message(STATUS "llvm-profdata : ${PROFDATA_BIN}") + + # https://clang.llvm.org/docs/SourceBasedCodeCoverage.html + add_custom_target(coverage + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/llvm-coverage.py --insights ${CMAKE_CURRENT_BINARY_DIR}/insights --llvm-prof-dir=${CMAKE_CURRENT_BINARY_DIR}/llvmprof/ --format=text --output=${CMAKE_CURRENT_BINARY_DIR}/filtered.info + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/insights ${CMAKE_CURRENT_SOURCE_DIR}/tests/runTest.py + COMMENT "Running code coverage analysis" VERBATIM + ) + + add_custom_target(coverage-html + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/llvm-coverage.py --insights ${CMAKE_CURRENT_BINARY_DIR}/insights --llvm-prof-dir=${CMAKE_CURRENT_BINARY_DIR}/llvmprof/ --format=html --output=${CMAKE_CURRENT_BINARY_DIR}/filtered.html + COMMENT "Generating code coverage HTML" VERBATIM + ) + + # endif() endif() add_dependencies(coverage tests) diff --git a/codecov.yml b/codecov.yml index 0e00de6..5326743 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,3 +1,6 @@ fixes: - "cppinsights/::" + - "/home/runner/work/cppinsights/cppinsights/::" + - "/Users/runner/work/cppinsights/cppinsights/::" + - "github/workspace/::" diff --git a/scripts/llvm-coverage.py b/scripts/llvm-coverage.py new file mode 100755 index 0000000..9a64f50 --- /dev/null +++ b/scripts/llvm-coverage.py @@ -0,0 +1,54 @@ +#! /usr/bin/env python3 +#------------------------------------------------------------------------------ + +import os +import sys +import argparse +import glob +import subprocess +#------------------------------------------------------------------------------ + +def runCmd(cmd, data=None): + if input is None: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + else: + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate(input=data) + + return stdout.decode('utf-8'), stderr.decode('utf-8'), p.returncode +#------------------------------------------------------------------------------ + +def main(): + parser = argparse.ArgumentParser(description='llvm-coverage') + parser.add_argument('--insights', help='C++ Insights binary', required=True) + parser.add_argument('--llvm-prof-dir', help='LLVM profiles data dir', default='') + parser.add_argument('--format', help='Output format: html/text', default='text') + parser.add_argument('--output', help='Output filename', required=True) + parser.add_argument('args', nargs=argparse.REMAINDER) + args = vars(parser.parse_args()) + + insightsPath = args['insights'] + rawProfiles = glob.glob(os.path.join(args['llvm_prof_dir'], "*.profraw")) + profilesManifest = os.path.join(args['llvm_prof_dir'], 'profiles.manifest') + profilesData = os.path.join(args['llvm_prof_dir'], 'insights.profdata') + + with open(profilesManifest, "w") as manifest: + manifest.write("\n".join(rawProfiles)) + + cmd = ['llvm-profdata', 'merge', '-sparse', '-f', profilesManifest, '-o', profilesData] + stdout, stderr, returncode = runCmd(cmd) + print(stderr) + + cmd = ['llvm-cov', 'show', insightsPath, f'-instr-profile={profilesData}', f'--format={args["format"]}', '-ignore-filename-regex=build/'] + stdout, stderr, returncode = runCmd(cmd) + print(stderr) + + open(args['output'], 'w').write(stdout) +#------------------------------------------------------------------------------ + + +sys.exit(main()) +#------------------------------------------------------------------------------ + + diff --git a/tests/runTest.py b/tests/runTest.py index cd5fac9..af6b4d4 100755 --- a/tests/runTest.py +++ b/tests/runTest.py @@ -122,6 +122,7 @@ def main(): parser.add_argument('--update-tests', help='Update failing tests', default=False, action='store_true') parser.add_argument('--std', help='C++ Standard to used', default='c++17') parser.add_argument('--use-libcpp', help='Use libst++', default=False, action='store_true') + parser.add_argument('--llvm-prof-dir', help='LLVM profiles data dir', default='') parser.add_argument('args', nargs=argparse.REMAINDER) args = vars(parser.parse_args()) @@ -131,6 +132,9 @@ def main(): bUpdateTests = args['update_tests'] defaultCppStd = '-std=%s'% (args['std']) + if args['llvm_prof_dir'] != '': + os.environ['LLVM_PROFILE_FILE'] = os.path.join(args['llvm_prof_dir'], 'prof%p.profraw') + if 0 == len(remainingArgs): cppFiles = [f for f in os.listdir(mypath) if (os.path.isfile(os.path.join(mypath, f)) and f.endswith('.cpp'))] else: