Skip to content

Commit

Permalink
Re-enable code coverage for macOS using llvm infrastructure.
Browse files Browse the repository at this point in the history
  • Loading branch information
andreasfertig committed Apr 6, 2024
1 parent f80b7c3 commit 292723c
Show file tree
Hide file tree
Showing 5 changed files with 144 additions and 53 deletions.
66 changes: 28 additions & 38 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ jobs:
needs: checks
name: ${{ matrix.config.name }}
runs-on: ${{ matrix.config.os }}
continue-on-error: false
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -101,23 +102,23 @@ jobs:
cmake_args: "-DCMAKE_OSX_ARCHITECTURES=arm64",
}

# # AppleClang
# - {
# name: "Clang 18 / LLVM 18 @ macOS Coverage & Debug",
# os: macos-14,
# build_type: Release,
# cxx: "clang++",
# llvm_version: "18.1.0",
# llvm_config: "llvm-config",
# coverage: "Yes",
# static: "No",
# debug: "Yes",
# tidy: "No",
# run_tests: "No",
# bin_name: "insights",
# archive_name: "insights-macos",
# upload: "No",
# }
# AppleClang
- {
name: "Clang 18 / LLVM 18 @ macOS Coverage & Debug",
os: macos-14,
build_type: Release,
cxx: "clang++",
llvm_version: "18.1.0",
llvm_config: "llvm-config",
coverage: "Yes",
static: "No",
debug: "Yes",
tidy: "No",
run_tests: "No",
bin_name: "insights",
archive_name: "insights-macos",
upload: "No",
}

# # MSVC 2019
# - {
Expand Down Expand Up @@ -264,7 +265,7 @@ jobs:
key: ${{ runner.os }}-clang-${{ matrix.config.llvm_version }}-${{ hashFiles('${{ github.workspace }}/current') }}

- name: Install Clang
if: "(startsWith(matrix.config.os, 'macos') || startsWith(matrix.config.os, 'Window'))"
if: "(startsWith(matrix.config.os, 'macos') || startsWith(matrix.config.os, 'Window')) && steps.cache-clang-binary.outputs.cache-hit != 'true'"
shell: cmake -P {0}
run: |
set(llvm_version ${{ matrix.config.llvm_version }})
Expand Down Expand Up @@ -304,6 +305,7 @@ jobs:
- name: Cache grcov
id: cache-grcov-binary
if: "(startsWith(matrix.config.os, 'Window') && (matrix.config.coverage == 'Yes'))"
uses: actions/cache@v4
with:
path: |
Expand All @@ -319,14 +321,6 @@ jobs:
execute_process(COMMAND ${CMAKE_COMMAND} -E tar xf ./grcov.tar.bz2)
execute_process(COMMAND ${CMAKE_COMMAND} -E remove ./grcov.tar.bz2)
- name: Install lcov for macOS
if: "(startsWith(matrix.config.os, 'macos') && (matrix.config.coverage == 'Yes'))"
run: |
export HOMEBREW_NO_AUTO_UPDATE=1
brew update > /dev/null
brew install lcov || brew upgrade lcov
brew install gcc@10 || brew upgrade gcc@10 # for Clang 18 and newer gcov
- name: Setup MSVC Dev
if: "startsWith(matrix.config.os, 'Windows')"
uses: ilammy/msvc-dev-cmd@v1
Expand Down Expand Up @@ -375,7 +369,6 @@ jobs:
- name: Build
shell: cmake -P {0}
continue-on-error: false
run: |
execute_process(
COMMAND ${{ steps.cmake_and_ninja_setup.outputs.cmake_dir }}/cmake --build build
Expand All @@ -389,7 +382,6 @@ jobs:
id: run_tests
if: matrix.config.run_tests == 'Yes'
shell: cmake -P {0}
continue-on-error: false
run: |
if ("${{ runner.os }}" STREQUAL "macOS")
set(ENV{CPLUS_INCLUDE_PATH} "$ENV{GITHUB_WORKSPACE}/current/include/c++/v1:/Library/Developer/CommandLineTools/SDKs/MacOSX11.sdk/usr/include")
Expand All @@ -407,7 +399,6 @@ jobs:
- name: Run coverage
if: matrix.config.coverage == 'Yes'
shell: cmake -P {0}
continue-on-error: false
run: |
if ("${{ runner.os }}" STREQUAL "macOS")
set(ENV{CPLUS_INCLUDE_PATH} "$ENV{GITHUB_WORKSPACE}/current/include/c++/v1:/Library/Developer/CommandLineTools/SDKs/MacOSX11.sdk/usr/include")
Expand All @@ -427,15 +418,16 @@ jobs:
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./build/filtered.info # don't use a absolute path on Windows with gitBash.
files: ./build/filtered.info # don't use a absolute path on Windows with gitBash.
exclude: build
flags: ${{ matrix.config.archive_name }}
disable_search: true
fail_ci_if_error: true

- name: Create archive
if: matrix.config.upload == 'Yes'
shell: cmake -P {0}
working-directory: ${{ github.workspace }}/build
continue-on-error: false
run: |
file(MAKE_DIRECTORY "$ENV{GITHUB_WORKSPACE}/build/archive")
execute_process(COMMAND ${CMAKE_COMMAND} -E tar cvz "$ENV{GITHUB_WORKSPACE}/build/archive/${{ matrix.config.archive_name }}.tar.gz" -- "${{ matrix.config.bin_name }}")
Expand All @@ -457,6 +449,7 @@ jobs:
docker:
runs-on: ubuntu-22.04
name: ${{ matrix.config.name }} @${{ matrix.arch }}
continue-on-error: false
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -525,7 +518,6 @@ jobs:

- name: Create docker shell
shell: bash
continue-on-error: false
run: |
cat > docker-shell << "EOF"
#! /bin/bash
Expand Down Expand Up @@ -565,14 +557,12 @@ jobs:
- name: Build
shell: docker-shell {0}
continue-on-error: false
run: |
cmake --build build
- name: Simple test
if: "((matrix.config.coverage != 'Yes') && (matrix.config.run_tests != 'Yes'))"
shell: docker-shell {0}
continue-on-error: false
run: |
./build/insights cppinsights/tests/SimpleCICompileTest.cpp
./build/insights --use-libc++ cppinsights/tests/SimpleCICompileTest.cpp
Expand All @@ -581,7 +571,6 @@ jobs:
if: matrix.config.run_tests == 'Yes'
id: run_tests
shell: docker-shell {0}
continue-on-error: false
run: |
cmake --build build --target tests
Expand All @@ -608,7 +597,6 @@ jobs:
if: matrix.config.coverage == 'Yes'
id: run_coverage
shell: docker-shell {0}
continue-on-error: false
run: |
cmake --build build --target coverage
rm -f build/coverage.info
Expand All @@ -619,15 +607,16 @@ jobs:
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./build/filtered.info # don't use a absolute path on Windows with gitBash.
files: ./build/filtered.info # don't use a absolute path on Windows with gitBash.
exclude: build
flags: ${{ matrix.config.archive_name }}-${{ matrix.arch }}-libcxx-${{ matrix.config.libcxx }}
disable_search: true
fail_ci_if_error: true

- name: Create archive
if: matrix.config.upload == 'Yes'
working-directory: ${{ github.workspace }}/build
shell: bash
continue-on-error: false
run: |
mkdir -p ${GITHUB_WORKSPACE}/build/archive
Expand All @@ -648,6 +637,7 @@ jobs:
needs: [build, docker]
if: github.ref == 'refs/heads/main'
name: Final Deploy
continue-on-error: false
runs-on: ubuntu-22.04

steps:
Expand Down
70 changes: 55 additions & 15 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -347,18 +347,22 @@ if (BUILD_INSIGHTS_OUTSIDE_LLVM)
set(COVERAGE_LINK_FLAGS "-lgcov")
endif()

if(WIN32)
if(WIN32)
add_definitions(-fprofile-instr-generate)
set(COVERAGE_LINK_FLAGS "${COVERAGE_LINK_FLAGS} ${LLVM_LIBDIR}/clang/${LLVM_PACKAGE_VERSION_PLAIN}/lib/windows/clang_rt.profile-x86_64.lib /FORCE:MULTIPLE")
else()
else()
add_definitions(-g)
add_definitions(-O0)
# add_definitions(-fprofile-arcs)
# add_definitions(-fprofile-arcs)
add_definitions(-ftest-coverage)
set(COVERAGE_LINK_FLAGS "${COVERAGE_LINK_FLAGS} --coverage")
endif()
endif()

add_definitions(--coverage)
if(IS_CLANG)
add_definitions(-fprofile-instr-generate -fcoverage-mapping)
else()
add_definitions(--coverage)
endif()

set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${COVERAGE_LINK_FLAGS}")
endif()
Expand Down Expand Up @@ -425,12 +429,12 @@ if(WIN32)
set( CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} )
set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} )
# Second, for multi-config builds (e.g. msvc)
foreach( OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES} )
string( TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG )
set( CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_BINARY_DIR} )
set( CMAKE_LIBRARY_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_BINARY_DIR} )
set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_BINARY_DIR} )
endforeach( OUTPUTCONFIG CMAKE_CONFIGURATION_TYPES )
foreach( OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES} )
string( TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG )
set( CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_BINARY_DIR} )
set( CMAKE_LIBRARY_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_BINARY_DIR} )
set( CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${OUTPUTCONFIG} ${CMAKE_BINARY_DIR} )
endforeach( OUTPUTCONFIG CMAKE_CONFIGURATION_TYPES )
endif()


Expand Down Expand Up @@ -608,6 +612,11 @@ if(INSIGHTS_COVERAGE OR WIN32)
set(TEST_FAILURE_IS_OK "--failure-is-ok")
endif()

set(LLVM_PROF_DIR "")
if(INSIGHTS_COVERAGE AND IS_CLANG)
set(LLVM_PROF_DIR "--llvm-prof-dir=${CMAKE_CURRENT_BINARY_DIR}/llvmprof")
endif()

set(TEST_USE_LIBCPP "")
if(INSIGHTS_USE_LIBCPP)
set(TEST_USE_LIBCPP "--use-libcpp")
Expand All @@ -628,7 +637,9 @@ else()

# add a target to generate run tests
add_custom_target(tests
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/runTest.py --insights ${CMAKE_CURRENT_BINARY_DIR}/$<TARGET_FILE_NAME:insights> --cxx ${CMAKE_CXX_COMPILER} ${TEST_FAILURE_IS_OK} ${TEST_USE_LIBCPP}
COMMAND cmake -E rm -rf ${CMAKE_CURRENT_BINARY_DIR}/llvmprof/
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/runTest.py --insights
${CMAKE_CURRENT_BINARY_DIR}/$<TARGET_FILE_NAME:insights> --cxx ${CMAKE_CXX_COMPILER} ${TEST_FAILURE_IS_OK} ${TEST_USE_LIBCPP} ${LLVM_PROF_DIR}
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/testSTDIN.sh ${CMAKE_CURRENT_BINARY_DIR}/$<TARGET_FILE_NAME:insights>
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/testInvalidOption.sh ${CMAKE_CURRENT_BINARY_DIR}/$<TARGET_FILE_NAME:insights>
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/$<TARGET_FILE_NAME:insights> ${CMAKE_CURRENT_SOURCE_DIR}/tests/runTest.py
Expand Down Expand Up @@ -718,14 +729,13 @@ if(INSIGHTS_COVERAGE)
COMMENT "Generating code coverage HTML" VERBATIM
)
endif()
else()
elseif(IS_GNU)
find_program(LCOV_BIN lcov)
find_program(GENHTML_BIN genhtml)
find_package_handle_standard_args(lcov
REQUIRED_VARS LCOV_BIN GENHTML_BIN
)


if (NOT LCOV_FOUND)
message(FATAL_ERROR "Lcov not found")
else()
Expand Down Expand Up @@ -764,8 +774,38 @@ if(INSIGHTS_COVERAGE)
COMMAND genhtml ${CMAKE_CURRENT_BINARY_DIR}/filtered.info --demangle-cpp --output-directory ${CMAKE_CURRENT_BINARY_DIR}/out
COMMENT "Generating code coverage HTML" VERBATIM
)

endif()

else() # Clang
find_program(LCOV_BIN llvm-cov)
find_program(PROFDATA_BIN llvm-profdata)
# find_package_handle_standard_args(llvm-lcov
# REQUIRED_VARS LCOV_BIN PROFDATA_BIN
# )


# if (NOT LCOV_FOUND)
# message(FATAL_ERROR "llvm-cov not found")
# elseif (NOT PROFDATA_FOUND)
# message(FATAL_ERROR "llvm-profdata not found")
# else()
message(STATUS "Target coverage available")
message(STATUS "lcov : ${LCOV_BIN}")
message(STATUS "llvm-profdata : ${PROFDATA_BIN}")

# https://clang.llvm.org/docs/SourceBasedCodeCoverage.html
add_custom_target(coverage
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/llvm-coverage.py --insights ${CMAKE_CURRENT_BINARY_DIR}/insights --llvm-prof-dir=${CMAKE_CURRENT_BINARY_DIR}/llvmprof/ --format=text --output=${CMAKE_CURRENT_BINARY_DIR}/filtered.info
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/insights ${CMAKE_CURRENT_SOURCE_DIR}/tests/runTest.py
COMMENT "Running code coverage analysis" VERBATIM
)

add_custom_target(coverage-html
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/llvm-coverage.py --insights ${CMAKE_CURRENT_BINARY_DIR}/insights --llvm-prof-dir=${CMAKE_CURRENT_BINARY_DIR}/llvmprof/ --format=html --output=${CMAKE_CURRENT_BINARY_DIR}/filtered.html
COMMENT "Generating code coverage HTML" VERBATIM
)

# endif()
endif()

add_dependencies(coverage tests)
Expand Down
3 changes: 3 additions & 0 deletions codecov.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
fixes:
- "cppinsights/::"
- "/home/runner/work/cppinsights/cppinsights/::"
- "/Users/runner/work/cppinsights/cppinsights/::"
- "github/workspace/::"

54 changes: 54 additions & 0 deletions scripts/llvm-coverage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#! /usr/bin/env python3
#------------------------------------------------------------------------------

import os
import sys
import argparse
import glob
import subprocess
#------------------------------------------------------------------------------

def runCmd(cmd, data=None):
if input is None:
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
else:
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate(input=data)

return stdout.decode('utf-8'), stderr.decode('utf-8'), p.returncode
#------------------------------------------------------------------------------

def main():
parser = argparse.ArgumentParser(description='llvm-coverage')
parser.add_argument('--insights', help='C++ Insights binary', required=True)
parser.add_argument('--llvm-prof-dir', help='LLVM profiles data dir', default='')
parser.add_argument('--format', help='Output format: html/text', default='text')
parser.add_argument('--output', help='Output filename', required=True)
parser.add_argument('args', nargs=argparse.REMAINDER)
args = vars(parser.parse_args())

insightsPath = args['insights']
rawProfiles = glob.glob(os.path.join(args['llvm_prof_dir'], "*.profraw"))
profilesManifest = os.path.join(args['llvm_prof_dir'], 'profiles.manifest')
profilesData = os.path.join(args['llvm_prof_dir'], 'insights.profdata')

with open(profilesManifest, "w") as manifest:
manifest.write("\n".join(rawProfiles))

cmd = ['llvm-profdata', 'merge', '-sparse', '-f', profilesManifest, '-o', profilesData]
stdout, stderr, returncode = runCmd(cmd)
print(stderr)

cmd = ['llvm-cov', 'show', insightsPath, f'-instr-profile={profilesData}', f'--format={args["format"]}', '-ignore-filename-regex=build/']
stdout, stderr, returncode = runCmd(cmd)
print(stderr)

open(args['output'], 'w').write(stdout)
#------------------------------------------------------------------------------


sys.exit(main())
#------------------------------------------------------------------------------


Loading

0 comments on commit 292723c

Please sign in to comment.