From d640851b29fc001fdd22307d7a366af057341437 Mon Sep 17 00:00:00 2001 From: mcbarton <150042563+mcbarton@users.noreply.github.com> Date: Fri, 3 May 2024 21:48:44 +0100 Subject: [PATCH] Update ci to match other repos --- .github/workflows/ci.yml | 699 +++++++++++++++++++++++++++++++++------ 1 file changed, 595 insertions(+), 104 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3d8873a..6571c831 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,20 +21,163 @@ jobs: fail-fast: false matrix: include: - - name: ubu22-gcc9-clang13 + - name: ubu22-x86-gcc12-clang-repl-18-cppyy + os: ubuntu-22.04 + compiler: gcc-12 + clang-runtime: '18' + cling: Off + cppyy: On + - name: ubu22-x86-gcc12-clang-repl-17-cppyy + os: ubuntu-22.04 + compiler: gcc-12 + clang-runtime: '17' + cling: Off + cppyy: On + - name: ubu22-x86-gcc9-clang-repl-16-cppyy os: ubuntu-22.04 compiler: gcc-9 - clang-runtime: '13' - cling: On -# - name: ubu22-gcc9-clang14 -# os: ubuntu-22.04 -# compiler: gcc-9 -# clang-runtime: '14' - - name: ubu22-gcc9-clang16 + clang-runtime: '16' + cling: Off + cppyy: On + - name: ubu22-x86-gcc9-clang13-cling-cppyy os: ubuntu-22.04 compiler: gcc-9 + clang-runtime: '13' + cling: On + cling-version: '1.0' + cppyy: On + #Commented out until Ubuntu on arm Github runner becomes + #available os key to be replaced once known + #- name: ubu22-arm-gcc12-clang-repl-17 + # os: ubuntu-22.04-arm + # compiler: gcc-12 + # clang-runtime: '17' + # cling: Off + # cppyy: Off + #- name: ubu22-arm-gcc12-clang-repl-17-cppyy + # os: ubuntu-22.04-arm + # compiler: gcc-12 + # clang-runtime: '17' + # cling: Off + # cppyy: On + #- name: ubu22-arm-gcc9-clang-repl-16 + # os: ubuntu-22.04-arm + # compiler: gcc-9 + # clang-runtime: '16' + # cling: Off + # cppyy: Off + #- name: ubu22-arm-gcc9-clang-repl-16-cppyy + # os: ubuntu-22.04-arm + # compiler: gcc-9 + # clang-runtime: '16' + # cling: Off + # cppyy: On + #- name: ubu22-arm-gcc9-clang13-cling + # os: ubuntu-22.04-arm + # compiler: gcc-9 + # clang-runtime: '13' + # cling: On + # cling-version: '1.0' + # cppyy: Off + #- name: ubu22-arm-gcc9-clang13-cling-cppyy + # os: ubuntu-22.04-arm + # compiler: gcc-9 + # clang-runtime: '13' + # cling: On + # cling-version: '1.0' + # cppyy: On + # - name: win2022-msvc-clang-repl-17 + # os: windows-2022 + # compiler: msvc + # clang-runtime: '17' + # cling: Off + # cppyy: Off + #Commented out until rest of ci for Windows tested + #- name: win2022-msvc-clang-repl-17-cppyy + # os: windows-2022 + # compiler: msvc + # clang-runtime: '17' + # cling: Off + # cppyy: On + # - name: win2022-msvc-clang-repl-16 + # os: windows-2022 + # compiler: msvc + # clang-runtime: '16' + # cling: Off + # cppyy: Off + #Commented out until rest of ci for Windows tested + #- name: win2022-msvc-clang-repl-16-cppyy + # os: windows-2022 + # compiler: msvc + # clang-runtime: '16' + # cling: Off + # cppyy: On + # - name: win2022-msvc-cling + # os: windows-2022 + # compiler: msvc + # clang-runtime: '13' + # cling: On + # cling-version: '1.0' + # cppyy: Off + #Commented out until rest of ci for Windows tested + #- name: win2022-msvc-cling-cppyy + # os: windows-2022 + # compiler: msvc + # clang-runtime: '13' + # cling: On + # cling-version: '1.0' + # cppyy: On + - name: osx13-x86-clang-clang-repl-18-cppyy + os: macos-13 + compiler: clang + clang-runtime: '18' + cling: Off + cppyy: On + - name: osx13-x86-clang-clang-repl-17-cppyy + os: macos-13 + compiler: clang + clang-runtime: '17' + cling: Off + cppyy: On + - name: osx13-x86-clang-clang-repl-16-cppyy + os: macos-13 + compiler: clang clang-runtime: '16' cling: Off + cppyy: On + - name: osx13-x86-clang-clang13-cling-cppyy + os: macos-13 + compiler: clang + clang-runtime: '13' + cling: On + cling-version: '1.0' + cppyy: On + - name: osx14-arm-clang-clang-repl-18-cppyy + os: macos-14 + compiler: clang + clang-runtime: '18' + cling: Off + cppyy: On + - name: osx14-arm-clang-clang-repl-17-cppyy + os: macos-14 + compiler: clang + clang-runtime: '17' + cling: Off + cppyy: On + - name: osx14-arm-clang-clang-repl-16-cppyy + os: macos-14 + compiler: clang + clang-runtime: '16' + cling: Off + cppyy: On + - name: osx14-arm-clang-clang13-cling-cppyy + os: macos-14 + compiler: clang + clang-runtime: '13' + cling: On + cling-version: '1.0' + cppyy: On + steps: - uses: actions/checkout@v3 with: @@ -43,31 +186,84 @@ jobs: uses: actions/setup-python@v4 with: python-version: '3.10' - - name: Save PR Info + + - name: Save PR Info on Unix systems + if: ${{ runner.os != 'windows' }} run: | mkdir -p ./pr echo ${{ github.event.number }} > ./pr/NR echo ${{ github.repository }} > ./pr/REPO - cling_on="${{ matrix.cling }}" - if [[ "${cling_on^^}" == "ON" ]]; then - export CLING_HASH=$(git ls-remote https://github.com/root-project/cling.git HEAD| tr '\t' '-') + cling_on=$(echo "${{ matrix.cling }}" | tr '[:lower:]' '[:upper:]') + if [[ "$cling_on" == "ON" ]]; then + export CLING_HASH=$(git ls-remote https://github.com/root-project/cling.git refs/tags/v${{ matrix.cling-version }} | tr '\t' '-') + export LLVM_HASH=$(git ls-remote https://github.com/root-project/llvm-project.git cling-llvm${{ matrix.clang-runtime}} | tr '\t' '-') else export CLING_HASH="Repl" + # May need to revert back to both having same llvm_hash, as below cause llvm to be rebuilt everytime commit is made to llvm/llvm-project for release a.x + # which could be quite often for new releases + export LLVM_HASH=$(git ls-remote https://github.com/llvm/llvm-project.git refs/heads/release/${{ matrix.clang-runtime}}.x | tr '\t' '-') fi - # FIXME: We need something like cling-llvm to bump automatically... - export LLVM_HASH=$(git ls-remote https://github.com/root-project/llvm-project.git cling-llvm${{ matrix.clang-runtime}} | tr '\t' '-') - #patch in LLVM_HASH + echo "CLING_HASH=$CLING_HASH" >> $GITHUB_ENV echo "LLVM_HASH=$LLVM_HASH" >> $GITHUB_ENV + + - name: Save PR Info on Windows systems + if: ${{ runner.os == 'windows' }} + run: | + mkdir ./pr + echo "${{ github.event.number }}" > ./pr/NR + echo ${{ github.repository }} > ./pr/REPO + + if ( "${{ matrix.cling }}" -imatch "On" ) + { + $env:CLING_HASH_TEMP = ( git ls-remote https://github.com/root-project/cling.git refs/tags/v${{ matrix.cling-version }} ) + $env:CLING_HASH = $env:CLING_HASH_TEMP -replace "\t","-" + } + else + { + $env:CLING_HASH="Repl" + # May need to revert back to both having same llvm_hash, as below cause llvm to be rebuilt everytime commit is made to llvm/llvm-project for release a.x + # which could be quite often for new releases + $env:LLVM_HASH_TEMP = (git ls-remote https://github.com/llvm/llvm-project.git refs/heads/release/${{ matrix.clang-runtime}}.x ) + $env:LLVM_HASH = $env:LLVM_HASH_TEMP -replace "\t","-" + } + + echo "CLING_HASH=$env:CLING_HASH" + echo "LLVM_HASH=$env:LLVM_HASH" + + echo "CLING_HASH=$env:CLING_HASH" >> $GITHUB_ENV + echo "LLVM_HASH=$env:LLVM_HASH" >> $GITHUB_ENV + - uses: nelonoel/branch-name@v1.0.1 + + - name: Setup default Build Type on *nux + if: runner.os != 'windows' + run: | + echo "BUILD_TYPE=Release" >> $GITHUB_ENV + echo "CACHE=OFF" >> $GITHUB_ENV + os="${{ matrix.os }}" + if [[ "${os}" == "macos"* ]]; then + echo "ncpus=$(sysctl -n hw.ncpu)" >> $GITHUB_ENV + else + echo "ncpus=$(nproc --all)" >> $GITHUB_ENV + fi + + - name: Setup default Build Type on Windows + if: runner.os == 'windows' + run: | + echo "BUILD_TYPE=Release" >> $env:GITHUB_ENV + echo "CACHE=OFF" >> $env:GITHUB_ENV + echo "ncpus=%NUMBER_OF_PROCESSORS%" + echo "ncpus=%NUMBER_OF_PROCESSORS%" >> $GITHUB_ENV + - name: Setup compiler on Linux if: runner.os == 'Linux' run: | # https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html vers="${compiler#*-}" os_codename="`cat /etc/os-release | grep UBUNTU_CODENAME | cut -d = -f 2`" - ###sudo apt update + ##sudo apt update if [[ "${{ matrix.compiler }}" == *"gcc"* ]]; then sudo apt install -y gcc-${vers} g++-${vers} lld echo "CC=gcc-${vers}" >> $GITHUB_ENV @@ -82,17 +278,82 @@ jobs: echo "CC=clang-${vers}" >> $GITHUB_ENV echo "CXX=clang++-${vers}" >> $GITHUB_ENV fi + + env: + compiler: ${{ matrix.compiler }} + + - name: Setup compiler on macOS + if: runner.os == 'macOS' + run: | + vers="${compiler#*-}" + if [[ "${{ matrix.compiler }}" == *"gcc"* ]]; then + brew install "gcc@$vers" + echo "CC=gcc-${vers}" >> $GITHUB_ENV + echo "CXX=g++-${vers}" >> $GITHUB_ENV + else + echo "SDKROOT=$(xcrun --sdk macosx --show-sdk-path)" >> $GITHUB_ENV + #Use clang-15 and clang++-15 compiler that is installed on runner, instead of one + #provided by MacOS (could brew install clang-16/17 to use consistent version + #of clang) + echo "CC=$(brew --prefix llvm@15)/bin/clang" >> $GITHUB_ENV + echo "CXX=$(brew --prefix llvm@15)/bin/clang++" >> $GITHUB_ENV + fi env: compiler: ${{ matrix.compiler }} + + #Section slightly modified version of + #https://github.com/vgvassilev/clad/blob/40d8bec11bde47b14a281078183a4f6147abeac5/.github/workflows/ci.yml#L510C1-L534C10 + - name: Setup compiler on Windows + if: runner.os == 'windows' + run: | + if ( "${{ matrix.compiler }}" -imatch "clang" ) + { + $ver="${{ matrix.compiler }}".split("-")[1] + choco install llvm --version=$ver --no-progress -my + clang --version + # + $env:CC="clang" + $env:CXX="clang++" + echo "CC=clang" >> $env:GITHUB_ENV + echo "CXX=clang++" >> $env:GITHUB_ENV + } + elseif ( "${{ matrix.compiler }}" -imatch "msvc" ) + { + # MSVC is builtin in container image + } + else + { + echo "Unsupported compiler - fix YAML file" + } + + - name: Install deps on Windows + if: runner.os == 'windows' + run: | + choco install findutils + $env:PATH="C:\Program Files (x86)\GnuWin32\bin;$env:PATH" + - name: Install deps on Linux if: runner.os == 'Linux' run: | # Install deps - ###sudo apt update - sudo apt install git g++ debhelper devscripts gnupg python3 valgrind + sudo apt-get update + sudo apt-get install git g++ debhelper devscripts gnupg python3 valgrind conda install -y -q -c conda-forge \ distro \ pytest + + - name: Install deps on MacOS + if: runner.os == 'macOS' + run: | + brew update + # workaround for https://github.com/actions/setup-python/issues/577 + for pkg in $(brew list | grep '^python@'); do + brew unlink "$pkg" + brew link --overwrite "$pkg" + done + brew upgrade + pip install distro pytest + - name: Restore Cache LLVM/Clang runtime build directory uses: actions/cache/restore@v3 id: cache @@ -100,57 +361,162 @@ jobs: path: | llvm-project ${{ matrix.cling=='On' && 'cling' || '' }} - #key: ...-.x-patch-${{ hashFiles('patches/llvm/*') }} - #key: ${{ env.CLING_HASH }}-${{ env.LLVM_HASH }}-${{ runner.os }}-${{ matrix.os }}-${{ matrix.compiler }}-clang-${{ matrix.clang-runtime }}.x key: ${{ env.CLING_HASH }}-${{ runner.os }}-${{ matrix.os }}-${{ matrix.compiler }}-clang-${{ matrix.clang-runtime }}.x-patch-${{ hashFiles(format('patches/llvm/clang{0}-*.patch', matrix.clang-runtime)) || 'none' }} - - name: Build LLVM/Cling on Linux if the cache is invalid - if: ${{ runner.os == 'Linux' && steps.cache.outputs.cache-hit != 'true' }} + #FIXME: Must be a way to use absolute path in restore to avoid this + - name: Move llvm-project to top level directory Unix + if: ${{ runner.os != 'windows' && steps.cache.outputs.cache-hit == 'true' }} + run: | + mv ./llvm-project/ ../ + echo "CACHE=ON" >> $GITHUB_ENV + + + - name: Move llvm-project to top level directory + if: ${{ runner.os == 'windows' && steps.cache.outputs.cache-hit == 'true' }} + run: | + mv .\llvm-project\ ..\ + echo "CACHE=ON" >> $env:GITHUB_ENV + + - name: Build LLVM/Cling on Unix systems if the cache is invalid + if: ${{ runner.os != 'windows' && steps.cache.outputs.cache-hit != 'true' }} run: | - cling_on="${{ matrix.cling }}" - if [[ "${cling_on^^}" == "ON" ]]; then - git clone --depth=1 https://github.com/root-project/cling.git + cd .. + cling_on=$(echo "${{ matrix.cling }}" | tr '[:lower:]' '[:upper:]') + if [[ "${cling_on}" == "ON" ]]; then + git clone https://github.com/root-project/cling.git + cd ./cling + git checkout tags/v${{ matrix.cling-version }} + cd .. git clone --depth=1 -b cling-llvm${{ matrix.clang-runtime }} https://github.com/root-project/llvm-project.git else # repl - git clone --depth=1 -b release/${{ matrix.clang-runtime }}.x https://github.com/llvm/llvm-project.git + git clone --depth=1 -b release/${{ matrix.clang-runtime }}.x https://github.com/llvm/llvm-project.git fi cd llvm-project - # Apply patches - echo "Apply clang${{ matrix.clang-runtime }}-*.patch patches:" - compgen -G "../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch" > /dev/null && find ../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch -printf "%f\n" && git apply ../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch # Build mkdir build - cd build - if [[ "${cling_on^^}" == "ON" ]]; then + if [[ "${cling_on}" == "ON" ]]; then + cd build cmake -DLLVM_ENABLE_PROJECTS=clang \ -DLLVM_EXTERNAL_PROJECTS=cling \ -DLLVM_EXTERNAL_CLING_SOURCE_DIR=../../cling \ -DLLVM_TARGETS_TO_BUILD="host;NVPTX" \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_USE_LINKER=lld \ + -DLLVM_ENABLE_LLD=ON \ -DCLANG_ENABLE_STATIC_ANALYZER=OFF \ -DCLANG_ENABLE_ARCMT=OFF \ -DCLANG_ENABLE_FORMAT=OFF \ -DCLANG_ENABLE_BOOTSTRAP=OFF \ ../llvm - cmake --build . --target clang --parallel $(nproc --all) - cmake --build . --target cling --parallel $(nproc --all) + cmake --build . --target clang --parallel ${{ env.ncpus }} + cmake --build . --target cling --parallel ${{ env.ncpus }} # Now build gtest.a and gtest_main for CppInterOp to run its tests. - cmake --build . --target gtest_main --parallel $(nproc --all) + cmake --build . --target gtest_main --parallel ${{ env.ncpus }} else + # Apply patches + llvm_vers=$(echo "${{ matrix.clang-runtime }}" | tr '[:lower:]' '[:upper:]') + if [[ "${llvm_vers}" != "18" ]]; then + git apply -v ../CPyCppyy/patches/llvm/clang${{ matrix.clang-runtime }}-*.patch + echo "Apply clang${{ matrix.clang-runtime }}-*.patch patches:" + fi + cd build cmake -DLLVM_ENABLE_PROJECTS=clang \ -DLLVM_TARGETS_TO_BUILD="host;NVPTX" \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_USE_LINKER=lld \ + -DLLVM_ENABLE_LLD=ON \ -DCLANG_ENABLE_STATIC_ANALYZER=OFF \ -DCLANG_ENABLE_ARCMT=OFF \ -DCLANG_ENABLE_FORMAT=OFF \ -DCLANG_ENABLE_BOOTSTRAP=OFF \ ../llvm - cmake --build . --target clang clang-repl --parallel $(nproc --all) + cmake --build . --target clang clang-repl --parallel ${{ env.ncpus }} fi - cd ../../ + cd ../ + rm -rf $(find . -maxdepth 1 ! -name "build" ! -name "llvm" ! -name "clang" ! -name ".") + cd ../ + mkdir ./CPyCppyy/llvm-project/ + #FIXME: Avoid copy command by caching based on absolute path + cp -r -v ./llvm-project/* ./CPyCppyy/llvm-project/ + echo "Copy llvm-project folder to CPyCppyy for caching" + + - name: Build LLVM/Cling on Windows systems if the cache is invalid + if: ${{ runner.os == 'windows' && steps.cache.outputs.cache-hit != 'true' }} + run: | + cd .. + if ( "${{ matrix.cling }}" -imatch "On" ) + { + git clone https://github.com/root-project/cling.git + cd ./cling + git checkout tags/v${{ matrix.cling-version }} + cd .. + git clone --depth=1 -b cling-llvm${{ matrix.clang-runtime }} https://github.com/root-project/llvm-project.git + $env:PWD_DIR= $PWD.Path + $env:CLING_DIR="$env:PWD_DIR\cling" + echo "CLING_DIR=$env:CLING_DIR" + } + else + { + git clone --depth=1 -b release/${{ matrix.clang-runtime }}.x https://github.com/llvm/llvm-project.git + } + + cd llvm-project + # Build + mkdir build + if ( "${{ matrix.cling }}" -imatch "On" ) + { + cd build + cmake -DLLVM_ENABLE_PROJECTS=clang ` + -DLLVM_EXTERNAL_PROJECTS=cling ` + -DLLVM_EXTERNAL_CLING_SOURCE_DIR="$env:CLING_DIR" ` + -DLLVM_TARGETS_TO_BUILD="host;NVPTX" ` + -DCMAKE_BUILD_TYPE=Release ` + -DLLVM_ENABLE_ASSERTIONS=ON ` + -DLLVM_ENABLE_LLD=ON ` + -DCLANG_ENABLE_STATIC_ANALYZER=OFF ` + -DCLANG_ENABLE_ARCMT=OFF ` + -DCLANG_ENABLE_FORMAT=OFF ` + -DCLANG_ENABLE_BOOTSTRAP=OFF ` + ../llvm + cmake --build . --config Release --target clang --parallel ${{ env.ncpus }} + cmake --build . --config Release --target cling --parallel ${{ env.ncpus }} + # Now build gtest.a and gtest_main for CppInterOp to run its tests. + cmake --build . --config Release --target gtest_main --parallel ${{ env.ncpus }} + } + else + { + cp -r ..\CPyCppyy\patches\llvm\clang${{ matrix.clang-runtime }}* + #FIXME: Apply patches without hardcoding + if ( "${{ matrix.clang-runtime }}" -imatch "16" ) + { + git apply -v clang16-1-Value.patch + git apply -v clang16-2-CUDA.patch + git apply -v clang16-3-WeakRef.patch + } + elseif ( "${{ matrix.clang-runtime }}" -imatch "16" ) + { + git apply -v clang17-1-NewOperator.patch + } + cd build + echo "Apply clang${{ matrix.clang-runtime }}-*.patch patches:" + cmake -DLLVM_ENABLE_PROJECTS=clang ` + -DLLVM_TARGETS_TO_BUILD="host;NVPTX" ` + -DCMAKE_BUILD_TYPE=Release ` + -DLLVM_ENABLE_ASSERTIONS=ON ` + -DLLVM_ENABLE_LLD=ON ` + -DCLANG_ENABLE_STATIC_ANALYZER=OFF ` + -DCLANG_ENABLE_ARCMT=OFF ` + -DCLANG_ENABLE_FORMAT=OFF ` + -DCLANG_ENABLE_BOOTSTRAP=OFF ` + ../llvm + cmake --build . --config Release --target clang clang-repl --parallel ${{ env.ncpus }} + } + cd ..\ + rm -r -force $(find.exe . -maxdepth 1 ! -name "build" ! -name "llvm" ! -name "clang" ! -name ".") + cd ..\ + mkdir .\CPyCppyy\llvm-project\ + cp -r -v .\llvm-project\ .\CPyCppyy\ + echo "Copy llvm-project folder to CPyCppyy for caching" + - name: Save Cache LLVM/Clang runtime build directory uses: actions/cache/save@v3 if: ${{ steps.cache.outputs.cache-hit != 'true' }} @@ -159,79 +525,177 @@ jobs: llvm-project ${{ matrix.cling=='On' && 'cling' || '' }} key: ${{ steps.cache.outputs.cache-primary-key }} - - name: Build and Install CppInterOp on Linux - if: runner.os == 'Linux' + + - name: Build and Test/Install CppInterOp on Unix systems + if: ${{ runner.os != 'windows' }} run: | - # Build CppInterOp next to cling and llvm-project. - LLVM_DIR="$(realpath llvm-project)" - LLVM_BUILD_DIR="$(realpath llvm-project/build)" - cling_on="${{ matrix.cling }}" - if [[ "${cling_on^^}" == "ON" ]]; then - CLING_DIR="$(realpath cling)" - CLING_BUILD_DIR="$(realpath cling/build)" - CPLUS_INCLUDE_PATH="${CLING_DIR}/tools/cling/include:${CLING_BUILD_DIR}/include:${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include" - else # repl - CPLUS_INCLUDE_PATH="${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include" + cd .. + git clone --depth=1 https://github.com/compiler-research/CppInterOp.git + LLVM_DIR="$(pwd)/llvm-project" + LLVM_BUILD_DIR="$(pwd)/llvm-project/build" + cling_on=$(echo "${{ matrix.cling }}" | tr '[:lower:]' '[:upper:]') + if [[ "${cling_on}" == "ON" ]]; then + + #FIXME: Currently Cling directory not found if using cache, so cloning again, until reason why is found. + if [[ $CACHE == "ON" ]]; then + echo "Cling folder not found. Cloning again." + git clone https://github.com/root-project/cling.git + cd ./cling/ + git checkout tags/v1.0 + cd .. + fi + + CLING_DIR="$(pwd)/cling" + CLING_BUILD_DIR="$(pwd)/cling/build" + CPLUS_INCLUDE_PATH="${CLING_DIR}/tools/cling/include:${CLING_BUILD_DIR}/include:${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include:$PWD/include" + else + CPLUS_INCLUDE_PATH="${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include:$PWD/include" fi - git clone https://github.com/compiler-research/CppInterOp.git - export CPPINTEROP_DIR=$PWD/cppyy-backend/python/cppyy_backend/ - cd CppInterOp - mkdir build - cd build + + export CB_PYTHON_DIR="$PWD/cppyy-backend/python" + export CPPINTEROP_DIR="$CB_PYTHON_DIR/cppyy_backend" + + # Build CppInterOp next to cling and llvm-project. + mkdir CppInterOp/build + cd CppInterOp/build export CPPINTEROP_BUILD_DIR=$PWD - cling_on="${{ matrix.cling }}" - if [[ "${cling_on^^}" == "ON" ]]; then - cmake -DCMAKE_BUILD_TYPE=Release \ - -DUSE_CLING=ON \ - -DUSE_REPL=OFF \ - -DCling_DIR=$LLVM_BUILD_DIR/tools/cling \ - -DLLVM_DIR=$LLVM_BUILD_DIR \ - -DLLVM_USE_LINKER=lld \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR \ + cling_on=$(echo "${{ matrix.cling }}" | tr '[:lower:]' '[:upper:]') + if [[ "${cling_on}" == "ON" ]]; then + cmake -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DUSE_CLING=ON \ + -DUSE_REPL=OFF \ + -DCling_DIR=$LLVM_BUILD_DIR/tools/cling \ + -DLLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm \ + -DClang_DIR=$LLVM_BUILD_DIR/lib/cmake/clang \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR \ ../ else - cmake -DCMAKE_BUILD_TYPE=Release \ - -DUSE_CLING=OFF \ - -DUSE_REPL=ON \ - -DLLVM_DIR=$LLVM_BUILD_DIR \ - -DLLVM_USE_LINKER=lld \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR \ + cmake -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DUSE_CLING=OFF \ + -DUSE_REPL=ON \ + -DLLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm \ + -DClang_DIR=$LLVM_BUILD_DIR/lib/cmake/clang \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR \ ../ fi - cmake --build . --parallel $(nproc --all) + os="${{ matrix.os }}" + cmake --build . --target check-cppinterop --parallel ${{ env.ncpus }} + cppyy_on=$(echo "${{ matrix.cppyy }}" | tr '[:lower:]' '[:upper:]') + if [[ ("${cppyy_on}" != "ON") && ("${os}" == "ubuntu"*) ]]; then + # TODO: Remove "|| true" when fix memory issues in LLVM/Clang 17 + valgrind --track-origins=yes --error-exitcode=1 unittests/CppInterOp/CppInterOpTests 2>&1 >/dev/null || true + fi cd ../.. + # We need CB_PYTHON_DIR later + echo "CB_PYTHON_DIR=$CB_PYTHON_DIR" >> $GITHUB_ENV # We need CPPINTEROP_DIR, LLVM_BUILD_DIR and CPLUS_INCLUDE_PATH later - echo "CPPINTEROP_DIR=$CPPINTEROP_DIR" >> $GITHUB_ENV echo "CPPINTEROP_BUILD_DIR=$CPPINTEROP_BUILD_DIR" >> $GITHUB_ENV + echo "CPPINTEROP_DIR=$CPPINTEROP_DIR" >> $GITHUB_ENV echo "LLVM_BUILD_DIR=$LLVM_BUILD_DIR" >> $GITHUB_ENV echo "CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH" >> $GITHUB_ENV - - name: Build and Install cppyy-backend on Linux - if: runner.os == 'Linux' + + - name: Build and Test/Install CppInterOp on Windows systems + if: ${{ runner.os == 'windows' }} + run: | + cd .. + git clone --depth=1 https://github.com/compiler-research/CppInterOp.git + $env:PWD_DIR= $PWD.Path + + $env:LLVM_DIR="$env:PWD_DIR\llvm-project" + echo "LLVM_DIR=$env:LLVM_DIR" + echo "LLVM_DIR=$env:LLVM_DIR" >> $env:GITHUB_ENV + + $env:LLVM_BUILD_DIR="$env:PWD_DIR\llvm-project\build" + echo "LLVM_BUILD_DIR=$env:LLVM_BUILD_DIR" + echo "LLVM_BUILD_DIR=$env:LLVM_BUILD_DIR" >> $env:GITHUB_ENV + + if ( "${{ matrix.cling }}" -imatch "On" ) + { + $env:CLING_DIR="$env:PWD_DIR\cling" + echo "CLING_DIR=$env:CLING_DIR" + echo "CLING_DIR=$env:CLING_DIR" >> $env:GITHUB_ENV + + $env:CLING_BUILD_DIR="$env:PWD_DIR\cling\build" + echo "CLING_BUILD_DIR=$env:CLING_BUILD_DIR" + echo "CLING_BUILD_DIR=$env:CLING_BUILD_DIR" >> $env:GITHUB_ENV + + $env:CPLUS_INCLUDE_PATH="$env:CLING_DIR\tools\cling\include;$env:CLING_BUILD_DIR\include;$env:LLVM_DIR\llvm\include;$env:LLVM_DIR\clang\include;$env:LLVM_BUILD_DIR\include;$env:LLVM_BUILD_DIR\tools\clang\include;$env:PWD_DIR\include;" + echo "CPLUS_INCLUDE_PATH=$env:CPLUS_INCLUDE_PATH" + echo "CPLUS_INCLUDE_PATH=$env:CPLUS_INCLUDE_PATH" >> $env:GITHUB_ENV + } + else + { + $env:CPLUS_INCLUDE_PATH="$env:LLVM_DIR\llvm\include;$env:LLVM_DIR\clang\include;$env:LLVM_BUILD_DIR\include;$env:LLVM_BUILD_DIR\tools\clang\include;$env:PWD_DIR\include;" + echo "CPLUS_INCLUDE_PATH=$env:CPLUS_INCLUDE_PATH" + echo "CPLUS_INCLUDE_PATH=$env:CPLUS_INCLUDE_PATH" >> $env:GITHUB_ENV + } + + $env:CB_PYTHON_DIR="$env:PWD_DIR\cppyy-backend\python" + echo "CB_PYTHON_DIR=$env:CB_PYTHON_DIR" + echo "CB_PYTHON_DIR=$env:CB_PYTHON_DIR" >> $env:GITHUB_ENV + + $env:CPPINTEROP_DIR="$env:CB_PYTHON_DIR\cppyy-backend" + echo "CPPINTEROP_DIR=$env:CPPINTEROP_DIR" + echo "CPPINTEROP_DIR=$env:CPPINTEROP_DIR" >> $env:GITHUB_ENV + + # Build CppInterOp next to cling and llvm-project. + mkdir CppInterOp\build + cd CppInterOp\build + $env:CPPINTEROP_BUILD_DIR="$env:PWD_DIR" + echo "CPPINTEROP_BUILD_DIR=$env:CPPINTEROP_BUILD_DIR" + echo "CPPINTEROP_BUILD_DIR=$env:CPPINTEROP_BUILD_DIR" >> $env:GITHUB_ENV + if ( "${{ matrix.cling }}" -imatch "On" ) + { + cmake -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} ` + -DUSE_CLING=ON ` + -DUSE_REPL=OFF ` + -DCling_DIR="$env:LLVM_BUILD_DIR\tools\cling" ` + -DLLVM_DIR="$env:LLVM_BUILD_DIR" ` + -DClang_DIR="$env:LLVM_BUILD_DIR" -DCMAKE_INSTALL_PREFIX="$env:CPPINTEROP_DIR" ..\ + } + else + { + cmake -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} ` + -DUSE_CLING=OFF ` + -DUSE_REPL=ON ` + -DLLVM_DIR="$env:LLVM_BUILD_DIR\lib\cmake\llvm" ` + -DClang_DIR="$env:LLVM_BUILD_DIR\lib\cmake\clang" -DCMAKE_INSTALL_PREFIX="$env:CPPINTEROP_DIR" ..\ + } + cmake --build . --config ${{ env.BUILD_TYPE }} --target check-cppinterop --parallel ${{ env.ncpus }} + cd ..\.. + + - name: Build and Install cppyy-backend on Unix Systems + if: ${{ (runner.os != 'windows') && (matrix.cppyy == 'On') }} run: | - # Install cppyy-backend - git clone https://github.com/compiler-research/cppyy-backend.git - cd cppyy-backend - mkdir -p python/cppyy_backend/lib build && cd build + cd .. + git clone --depth=1 https://github.com/compiler-research/cppyy-backend.git + mkdir -p $CPPINTEROP_DIR/lib cppyy-backend/build + cd cppyy-backend/build + # Install CppInterOp (cd $CPPINTEROP_BUILD_DIR && cmake --build . --target install --parallel $(nproc --all)) + # Build and Install cppyy-backend cmake -DCppInterOp_DIR=$CPPINTEROP_DIR .. cmake --build . --parallel $(nproc --all) - cp libcppyy-backend.so ../python/cppyy_backend/lib/ - # - cd ../python - export CB_PYTHON_DIR=$PWD - cd ../.. - # We need CB_PYTHON_DIR later - echo "CB_PYTHON_DIR=$CB_PYTHON_DIR" >> $GITHUB_ENV - - name: Install CPyCppyy on Linux - if: runner.os == 'Linux' + OS=$(uname -s) + if [[ "$OS" == "Darwin" ]]; then + cp libcppyy-backend.dylib $CPPINTEROP_DIR/lib/ + else + cp libcppyy-backend.so $CPPINTEROP_DIR/lib/ + fi + cd .. + + - name: Install CPyCppyy on Unix Systems + if: ${{ (runner.os != 'windows') && (matrix.cppyy == 'On') }} run: | + cd .. # Setup virtual environment python3 -m venv .venv source .venv/bin/activate # Install CPyCppyy - mkdir build && cd build + mkdir CPyCppyy/build + cd CPyCppyy/build cmake .. cmake --build . --parallel $(nproc --all) # @@ -239,28 +703,31 @@ jobs: cd ../.. # We need CPYCPPYY_DIR later echo "CPYCPPYY_DIR=$CPYCPPYY_DIR" >> $GITHUB_ENV - - name: Install cppyy on Linux - if: runner.os == 'Linux' + - name: Install cppyy on Unix Systems + if: ${{ (runner.os != 'windows') && (matrix.cppyy == 'On') }} run: | # source virtual environment + cd .. + git clone --depth=1 https://github.com/compiler-research/cppyy.git source .venv/bin/activate - # Install cppyy - git clone https://github.com/compiler-research/cppyy.git cd cppyy + # Install cppyy python -m pip install --upgrade . --no-deps - cd .. - - name: Run cppyy on Linux - if: runner.os == 'Linux' + - name: Run cppyy on Unix Systems + if: ${{ (runner.os != 'windows') && (matrix.cppyy == 'On') }} run: | + cd .. # Run cppyy source .venv/bin/activate + cd cppyy export PYTHONPATH=$PYTHONPATH:$CPYCPPYY_DIR:$CB_PYTHON_DIR python -c "import cppyy" # We need PYTHONPATH later echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV - - name: Run the tests on Linux - if: runner.os == 'Linux' + - name: Run the tests on Unix Systems + if: ${{ (runner.os != 'windows') && (matrix.cppyy == 'On') }} run: | + cd .. # Run the tests source .venv/bin/activate cd cppyy/test @@ -269,12 +736,17 @@ jobs: python -m pip install --upgrade pip python -m pip install pytest python -m pip install pytest-xdist + python -m pip install numba echo ::endgroup:: + echo ::group::Run complete test suite + set -o pipefail + python -m pytest -sv | tee complete_testrun.log 2>&1 + set +o pipefail echo ::group::Crashing Test Logs # See if we don't have a crash that went away # Comment out all xfails but the ones that have a run=False condition. find . -name "*.py" -exec sed -i '/run=False/!s/^ *@mark.xfail\(.*\)/#&/' {} \; - python -m pytest -n 1 -m "xfail" --runxfail -sv --max-worker-restart 512 | tee test_crashed.log 2>&1 + python -m pytest -n 1 -m "xfail" --runxfail -sv --max-worker-restart 512 | tee test_crashed.log 2>&1 || true git checkout . echo ::endgroup:: echo ::group::XFAIL Test Logs @@ -282,7 +754,7 @@ jobs: # avoid conditionally crashing xfails find . -name "*.py" -exec sed -i -E 's/(^ *)@mark.xfail\(run=(.*)/\1@mark.skipif(condition=not \2/g' {} \; # See if we don't have an xfail that went away - python -m pytest --runxfail -sv | tee test_xfailed.log 2>&1 + python -m pytest --runxfail -sv | tee test_xfailed.log 2>&1 || true git checkout . echo ::endgroup:: echo ::group::Passing Test Logs @@ -291,20 +763,39 @@ jobs: declare -i RETCODE=0 set -o pipefail - valgrind --error-exitcode=1 --suppressions=../etc/valgrind-cppyy-cling.supp python -m pytest -m "not xfail" -v + if [[ "${{ matrix.os }}" == macos-* ]]; then + echo "Skipping Valgrind checks on macOS" + else + if [[ "${{ matrix.clang-runtime }}" == "17" || "${{ matrix.clang-runtime }}" == "18" ]]; then + echo "Valgrind reports true for clang-runtime 17 or 18, due to memory leaks with LLVM" + valgrind --error-exitcode=1 --suppressions=../etc/valgrind-cppyy-cling.supp python -m pytest -m "not xfail" -v || true + else + echo "Running valgrind on passing tests" + valgrind --error-exitcode=1 --suppressions=../etc/valgrind-cppyy-cling.supp python -m pytest -m "not xfail" -v + fi + fi export RETCODE=+$? echo ::endgroup:: RETCODE=+$? + echo "Complete Test Suite Summary: \n" + tail -n1 complete_testrun.log echo "Crashing Summary: \n" tail -n1 test_crashed.log echo "XFAIL Summary:" tail -n1 test_xfailed.log echo "Return Code: ${RETCODE}" exit $RETCODE + + - name: Show debug info + if: ${{ failure() && (runner.os != 'windows') }} + run: | + export + echo $GITHUB_ENV + - name: Setup tmate session - if: ${{ failure() }} + if: ${{ failure() && runner.debug }} uses: mxschmitt/action-tmate@v3 # When debugging increase to a suitable value! - timeout-minutes: ${{ github.event.pull_request && 1 || 20 }} + timeout-minutes: 30