From 41a768dc9fc04409821f4f997ead6eb7a692f4b8 Mon Sep 17 00:00:00 2001 From: mcbarton <150042563+mcbarton@users.noreply.github.com> Date: Mon, 22 Jan 2024 17:12:07 +0000 Subject: [PATCH] Update ci workflow to be similar to one in CppInterOp --- .github/workflows/ci.yml | 658 +++++++++++++++++++---- patches/llvm/clang16-1-Value.patch | 134 ++--- patches/llvm/clang17-1-NewOperator.patch | 205 +++++++ 3 files changed, 835 insertions(+), 162 deletions(-) create mode 100644 patches/llvm/clang17-1-NewOperator.patch diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06f99290..3f2156f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,20 +21,205 @@ jobs: fail-fast: false matrix: include: - - name: ubu22-gcc9-clang13 + - name: ubu22-x86-gcc12-clang-repl-17 + os: ubuntu-22.04 + compiler: gcc-12 + clang-runtime: '17' + cling: Off + cppyy: Off + - name: ubu22-x86-gcc12-clang-repl-17-cppyy + os: ubuntu-22.04 + compiler: gcc-12 + clang-runtime: '17' + cling: Off + cppyy: On + - name: ubu22-x86-gcc9-clang-repl-16 + os: ubuntu-22.04 + compiler: gcc-9 + clang-runtime: '16' + cling: Off + cppyy: Off + - name: ubu22-x86-gcc9-clang-repl-16-cppyy + os: ubuntu-22.04 + compiler: gcc-9 + clang-runtime: '16' + cling: Off + cppyy: On + - name: ubu22-x86-gcc9-clang13-cling os: ubuntu-22.04 compiler: gcc-9 clang-runtime: '13' cling: On -# - name: ubu22-gcc9-clang14 -# os: ubuntu-22.04 -# compiler: gcc-9 -# clang-runtime: '14' - - name: ubu22-gcc9-clang16 + cling-version: '1.0' + cppyy: Off + - name: ubu22-x86-gcc9-clang13-cling-cppyy os: ubuntu-22.04 compiler: gcc-9 + clang-runtime: '13' + cling: On + cling-version: '1.0' + cppyy: On + #Commented out until Ubuntu on arm Github runner becomes available + #os key to be replaced once known + #- name: ubu22-arm-gcc12-clang-repl-17 + # os: ubuntu-22.04-arm + # compiler: gcc-12 + # clang-runtime: '17' + # cling: Off + # cppyy: Off + #- name: ubu22-arm-gcc12-clang-repl-17-cppyy + # os: ubuntu-22.04-arm + # compiler: gcc-12 + # clang-runtime: '17' + # cling: Off + # cppyy: On + #- name: ubu22-arm-gcc9-clang-repl-16 + # os: ubuntu-22.04-arm + # compiler: gcc-9 + # clang-runtime: '16' + # cling: Off + # cppyy: Off + #- name: ubu22-arm-gcc9-clang-repl-16-cppyy + # os: ubuntu-22.04-arm + # compiler: gcc-9 + # clang-runtime: '16' + # cling: Off + # cppyy: On + #- name: ubu22-arm-gcc9-clang13-cling + # os: ubuntu-22.04-arm + # compiler: gcc-9 + # clang-runtime: '13' + # cling: On + # cling-version: '1.0' + # cppyy: Off + #- name: ubu22-arm-gcc9-clang13-cling-cppyy + # os: ubuntu-22.04-arm + # compiler: gcc-9 + # clang-runtime: '13' + # cling: On + # cling-version: '1.0' + # cppyy: On + - name: win2022-msvc-clang-repl-17 + os: windows-2022 + compiler: msvc + clang-runtime: '17' + cling: Off + cppyy: Off + #Commented out until rest of ci for Windows tested + #- name: win2022-msvc-clang-repl-17-cppyy + # os: windows-2022 + # compiler: msvc + # clang-runtime: '17' + # cling: Off + # cppyy: On + - name: win2022-msvc-clang-repl-16 + os: windows-2022 + compiler: msvc clang-runtime: '16' cling: Off + cppyy: Off + #Commented out until rest of ci for Windows tested + #- name: win2022-msvc-clang-repl-16-cppyy + # os: windows-2022 + # compiler: msvc + # clang-runtime: '16' + # cling: Off + # cppyy: On + - name: win2022-msvc-cling + os: windows-2022 + compiler: msvc + clang-runtime: '13' + cling: On + cling-version: '1.0' + cppyy: Off + #Commented out until rest of ci for Windows tested + #- name: win2022-msvc-cling-cppyy + # os: windows-2022 + # compiler: msvc + # clang-runtime: '13' + # cling: On + # cling-version: '1.0' + # cppyy: On + - name: osx13-x86-clang-clang-repl-17 + os: macos-13 + compiler: clang + clang-runtime: '17' + cling: Off + cppyy: Off + - name: osx13-x86-clang-clang-repl-17-cppyy + os: macos-13 + compiler: clang + clang-runtime: '17' + cling: Off + cppyy: On + - name: osx13-x86-clang-clang-repl-16 + os: macos-13 + compiler: clang + clang-runtime: '16' + cling: Off + cppyy: Off + - name: osx13-x86-clang-clang-repl-16-cppyy + os: macos-13 + compiler: clang + clang-runtime: '16' + cling: Off + cppyy: On + - name: osx13-x86-clang-clang13-cling + os: macos-13 + compiler: clang + clang-runtime: '13' + cling: On + cling-version: '1.0' + cppyy: Off + - name: osx13-x86-clang-clang13-cling-cppyy + os: macos-13 + compiler: clang + clang-runtime: '13' + cling: On + cling-version: '1.0' + cppyy: On + #Block commented out until free tier for m1 + #exists (expected sometime 2024) and key for os + #can be replaced + #- name: osx13-arm64-clang-clang-repl-17 + # os: macos-13-arm64 + # compiler: clang + # clang-runtime: '17' + # cling: Off + # cppyy: Off + #- name: osx13-arm64-clang-clang-repl-17-cppyy + # os: macos-13-arm64 + # compiler: clang + # clang-runtime: '17' + # cling: Off + # cppyy: On + #- name: osx13-arm64-clang-clang-repl-16 + # os: macos-13-arm64 + # compiler: clang + # clang-runtime: '16' + # cling: Off + # cppyy: Off + #- name: osx13-arm64-clang-clang-repl-16-cppyy + # os: macos-13-arm64 + # compiler: clang + # clang-runtime: '16' + # cling: Off + # cppyy: On + #- name: osx13-arm64-clang-clang13-cling + # os: macos-13-arm64 + # compiler: clang + # clang-runtime: '13' + # cling: On + # cling-version: '1.0' + # cppyy: Off + #- name: osx13-arm64-clang-clang13-cling-cppyy + # os: macos-13-arm64 + # compiler: clang + # clang-runtime: '13' + # cling: On + # cling-version: '1.0' + # cppyy: On + steps: - uses: actions/checkout@v3 with: @@ -43,31 +228,84 @@ jobs: uses: actions/setup-python@v4 with: python-version: '3.10' - - name: Save PR Info + + - name: Save PR Info on Unix systems + if: ${{ runner.os != 'windows' }} run: | mkdir -p ./pr echo ${{ github.event.number }} > ./pr/NR echo ${{ github.repository }} > ./pr/REPO - cling_on="${{ matrix.cling }}" - if [[ "${cling_on^^}" == "ON" ]]; then - export CLING_HASH=$(git ls-remote https://github.com/root-project/cling.git HEAD| tr '\t' '-') + cling_on=$(echo "${{ matrix.cling }}" | tr '[:lower:]' '[:upper:]') + if [[ "$cling_on" == "ON" ]]; then + export CLING_HASH=$(git ls-remote https://github.com/root-project/cling.git refs/tags/v${{ matrix.cling-version }} | tr '\t' '-') + export LLVM_HASH=$(git ls-remote https://github.com/root-project/llvm-project.git cling-llvm${{ matrix.clang-runtime}} | tr '\t' '-') else export CLING_HASH="Repl" + # May need to revert back to both having same llvm_hash, as below cause llvm to be rebuilt everytime commit is made to llvm/llvm-project for release a.x + # which could be quite often for new releases + export LLVM_HASH=$(git ls-remote https://github.com/llvm/llvm-project.git refs/heads/release/${{ matrix.clang-runtime}}.x | tr '\t' '-') fi - # FIXME: We need something like cling-llvm to bump automatically... - export LLVM_HASH=$(git ls-remote https://github.com/root-project/llvm-project.git cling-llvm${{ matrix.clang-runtime}} | tr '\t' '-') - #patch in LLVM_HASH + echo "CLING_HASH=$CLING_HASH" >> $GITHUB_ENV echo "LLVM_HASH=$LLVM_HASH" >> $GITHUB_ENV + + - uses: nelonoel/branch-name@v1.0.1 + + - name: Save PR Info on Windows systems + if: ${{ runner.os == 'windows' }} + run: | + mkdir ./pr + echo "${{ github.event.number }}" > ./pr/NR + echo ${{ github.repository }} > ./pr/REPO + + if ( "${{ matrix.cling }}" -imatch "On" ) + { + $env:CLING_HASH_TEMP = ( git ls-remote https://github.com/root-project/cling.git refs/tags/v${{ matrix.cling-version }} ) + $env:CLING_HASH = $env:CLING_HASH_TEMP -replace "\t","-" + } + else + { + $env:CLING_HASH="Repl" + # May need to revert back to both having same llvm_hash, as below cause llvm to be rebuilt everytime commit is made to llvm/llvm-project for release a.x + # which could be quite often for new releases + $env:LLVM_HASH_TEMP = (git ls-remote https://github.com/llvm/llvm-project.git refs/heads/release/${{ matrix.clang-runtime}}.x ) + $env:LLVM_HASH = $env:LLVM_HASH_TEMP -replace "\t","-" + } + + echo "CLING_HASH=$env:CLING_HASH" + echo "LLVM_HASH=$env:LLVM_HASH" + + echo "CLING_HASH=$env:CLING_HASH" >> $GITHUB_ENV + echo "LLVM_HASH=$env:LLVM_HASH" >> $GITHUB_ENV + - uses: nelonoel/branch-name@v1.0.1 + + - name: Setup default Build Type on *nux + if: runner.os != 'windows' + run: | + echo "BUILD_TYPE=Release" >> $GITHUB_ENV + os="${{ matrix.os }}" + if [[ "${os}" == "macos"* ]]; then + echo "ncpus=$(sysctl -n hw.ncpu)" >> $GITHUB_ENV + else + echo "ncpus=$(nproc --all)" >> $GITHUB_ENV + fi + + - name: Setup default Build Type on Windows + if: runner.os == 'windows' + run: | + echo "BUILD_TYPE=Release" >> $env:GITHUB_ENV + echo "ncpus=%NUMBER_OF_PROCESSORS%" + echo "ncpus=%NUMBER_OF_PROCESSORS%" >> $GITHUB_ENV + - name: Setup compiler on Linux if: runner.os == 'Linux' run: | # https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html vers="${compiler#*-}" os_codename="`cat /etc/os-release | grep UBUNTU_CODENAME | cut -d = -f 2`" - ###sudo apt update + ##sudo apt update if [[ "${{ matrix.compiler }}" == *"gcc"* ]]; then sudo apt install -y gcc-${vers} g++-${vers} lld echo "CC=gcc-${vers}" >> $GITHUB_ENV @@ -82,17 +320,78 @@ jobs: echo "CC=clang-${vers}" >> $GITHUB_ENV echo "CXX=clang++-${vers}" >> $GITHUB_ENV fi + cd .. + env: + compiler: ${{ matrix.compiler }} + + - name: Setup compiler on macOS + if: runner.os == 'macOS' + run: | + vers="${compiler#*-}" + if [[ "${{ matrix.compiler }}" == *"gcc"* ]]; then + brew install "gcc@$vers" + echo "CC=gcc-${vers}" >> $GITHUB_ENV + echo "CXX=g++-${vers}" >> $GITHUB_ENV + else + echo "SDKROOT=$(xcrun --sdk macosx --show-sdk-path)" >> $GITHUB_ENV + #Use clang-15 and clang++-15 compiler that is installed on runner, instead of one + #provided by MacOS (could brew install clang-16/17 to use consistent version + #of clang) + echo "CC=$(brew --prefix llvm@15)/bin/clang" >> $GITHUB_ENV + echo "CXX=$(brew --prefix llvm@15)/bin/clang++" >> $GITHUB_ENV + fi + cd .. env: compiler: ${{ matrix.compiler }} + + #Section slightly modified version of + #https://github.com/vgvassilev/clad/blob/40d8bec11bde47b14a281078183a4f6147abeac5/.github/workflows/ci.yml#L510C1-L534C10 + - name: Setup compiler on Windows + if: runner.os == 'windows' + run: | + if ( "${{ matrix.compiler }}" -imatch "clang" ) + { + $ver="${{ matrix.compiler }}".split("-")[1] + choco install llvm --version=$ver --no-progress -my + clang --version + # + $env:CC="clang" + $env:CXX="clang++" + echo "CC=clang" >> $env:GITHUB_ENV + echo "CXX=clang++" >> $env:GITHUB_ENV + } + elseif ( "${{ matrix.compiler }}" -imatch "msvc" ) + { + # MSVC is builtin in container image + } + else + { + echo "Unsupported compiler - fix YAML file" + } + cd .. + - name: Install deps on Linux if: runner.os == 'Linux' run: | # Install deps - ###sudo apt update + sudo apt update sudo apt install git g++ debhelper devscripts gnupg python3 valgrind conda install -y -q -c conda-forge \ distro \ pytest + + - name: Install deps on MacOS + if: runner.os == 'macOS' + run: | + brew update + # workaround for https://github.com/actions/setup-python/issues/577 + for pkg in $(brew list | grep '^python@'); do + brew unlink "$pkg" + brew link --overwrite "$pkg" + done + brew upgrade + pip install distro pytest + - name: Restore Cache LLVM/Clang runtime build directory uses: actions/cache/restore@v3 id: cache @@ -100,57 +399,134 @@ jobs: path: | llvm-project ${{ matrix.cling=='On' && 'cling' || '' }} - #key: ...-.x-patch-${{ hashFiles('patches/llvm/*') }} - #key: ${{ env.CLING_HASH }}-${{ env.LLVM_HASH }}-${{ runner.os }}-${{ matrix.os }}-${{ matrix.compiler }}-clang-${{ matrix.clang-runtime }}.x key: ${{ env.CLING_HASH }}-${{ runner.os }}-${{ matrix.os }}-${{ matrix.compiler }}-clang-${{ matrix.clang-runtime }}.x-patch-${{ hashFiles(format('patches/llvm/clang{0}-*.patch', matrix.clang-runtime)) || 'none' }} - - name: Build LLVM/Cling on Linux if the cache is invalid - if: ${{ runner.os == 'Linux' && steps.cache.outputs.cache-hit != 'true' }} + + - name: Build LLVM/Cling on Unix systems if the cache is invalid + if: ${{ runner.os != 'windows' && steps.cache.outputs.cache-hit != 'true' }} run: | - cling_on="${{ matrix.cling }}" - if [[ "${cling_on^^}" == "ON" ]]; then - git clone --depth=1 https://github.com/root-project/cling.git + cling_on=$(echo "${{ matrix.cling }}" | tr '[:lower:]' '[:upper:]') + if [[ "${cling_on}" == "ON" ]]; then + git clone https://github.com/root-project/cling.git + cd ./cling + git checkout tags/v${{ matrix.cling-version }} + cd .. git clone --depth=1 -b cling-llvm${{ matrix.clang-runtime }} https://github.com/root-project/llvm-project.git else # repl - git clone --depth=1 -b release/${{ matrix.clang-runtime }}.x https://github.com/llvm/llvm-project.git + git clone --depth=1 -b release/${{ matrix.clang-runtime }}.x https://github.com/llvm/llvm-project.git fi cd llvm-project - # Apply patches - echo "Apply clang${{ matrix.clang-runtime }}-*.patch patches:" - compgen -G "../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch" > /dev/null && find ../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch -printf "%f\n" && git apply ../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch # Build mkdir build - cd build - if [[ "${cling_on^^}" == "ON" ]]; then + if [[ "${cling_on}" == "ON" ]]; then + cd build cmake -DLLVM_ENABLE_PROJECTS=clang \ -DLLVM_EXTERNAL_PROJECTS=cling \ -DLLVM_EXTERNAL_CLING_SOURCE_DIR=../../cling \ -DLLVM_TARGETS_TO_BUILD="host;NVPTX" \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_USE_LINKER=lld \ + -DLLVM_ENABLE_LLD=ON \ -DCLANG_ENABLE_STATIC_ANALYZER=OFF \ -DCLANG_ENABLE_ARCMT=OFF \ -DCLANG_ENABLE_FORMAT=OFF \ -DCLANG_ENABLE_BOOTSTRAP=OFF \ ../llvm - cmake --build . --target clang --parallel $(nproc --all) - cmake --build . --target cling --parallel $(nproc --all) + cmake --build . --target clang --parallel ${{ env.ncpus }} + cmake --build . --target cling --parallel ${{ env.ncpus }} # Now build gtest.a and gtest_main for CppInterOp to run its tests. - cmake --build . --target gtest_main --parallel $(nproc --all) + cmake --build . --target gtest_main --parallel ${{ env.ncpus }} else + # Apply patches + git apply -v ../cppyy-backend/patches/llvm/clang${{ matrix.clang-runtime }}-*.patch + echo "Apply clang${{ matrix.clang-runtime }}-*.patch patches:" + cd build cmake -DLLVM_ENABLE_PROJECTS=clang \ -DLLVM_TARGETS_TO_BUILD="host;NVPTX" \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_USE_LINKER=lld \ + -DLLVM_ENABLE_LLD=ON \ -DCLANG_ENABLE_STATIC_ANALYZER=OFF \ -DCLANG_ENABLE_ARCMT=OFF \ -DCLANG_ENABLE_FORMAT=OFF \ -DCLANG_ENABLE_BOOTSTRAP=OFF \ ../llvm - cmake --build . --target clang clang-repl --parallel $(nproc --all) + cmake --build . --target clang clang-repl --parallel ${{ env.ncpus }} fi cd ../../ + + - name: Build LLVM/Cling on Windows systems if the cache is invalid + if: ${{ runner.os == 'windows' && steps.cache.outputs.cache-hit != 'true' }} + run: | + + if ( "${{ matrix.cling }}" -imatch "On" ) + { + git clone https://github.com/root-project/cling.git + cd ./cling + git checkout tags/v${{ matrix.cling-version }} + cd .. + git clone --depth=1 -b cling-llvm${{ matrix.clang-runtime }} https://github.com/root-project/llvm-project.git + $env:PWD_DIR= $PWD.Path + $env:CLING_DIR="$env:PWD_DIR\cling" + echo "CLING_DIR=$env:CLING_DIR" + } + else + { + git clone --depth=1 -b release/${{ matrix.clang-runtime }}.x https://github.com/llvm/llvm-project.git + } + + cd llvm-project + # Build + mkdir build + if ( "${{ matrix.cling }}" -imatch "On" ) + { + cd build + cmake -DLLVM_ENABLE_PROJECTS=clang ` + -DLLVM_EXTERNAL_PROJECTS=cling ` + -DLLVM_EXTERNAL_CLING_SOURCE_DIR="$env:CLING_DIR" ` + -DLLVM_TARGETS_TO_BUILD="host;NVPTX" ` + -DCMAKE_BUILD_TYPE=Release ` + -DLLVM_ENABLE_ASSERTIONS=ON ` + -DLLVM_ENABLE_LLD=ON ` + -DCLANG_ENABLE_STATIC_ANALYZER=OFF ` + -DCLANG_ENABLE_ARCMT=OFF ` + -DCLANG_ENABLE_FORMAT=OFF ` + -DCLANG_ENABLE_BOOTSTRAP=OFF ` + ../llvm + cmake --build . --config Release --target clang --parallel ${{ env.ncpus }} + cmake --build . --config Release --target cling --parallel ${{ env.ncpus }} + # Now build gtest.a and gtest_main for CppInterOp to run its tests. + cmake --build . --config Release --target gtest_main --parallel ${{ env.ncpus }} + } + else + { + cp -r ..\cppyy-backend\patches\llvm\clang${{ matrix.clang-runtime }}* + #FIXME: Apply patches without hardcoding + if ( "${{ matrix.clang-runtime }}" -imatch "16" ) + { + git apply -v clang16-1-Value.patch + git apply -v clang16-2-CUDA.patch + git apply -v clang16-3-WeakRef.patch + } + elseif ( "${{ matrix.clang-runtime }}" -imatch "16" ) + { + git apply -v clang17-1-NewOperator.patch + } + cd build + echo "Apply clang${{ matrix.clang-runtime }}-*.patch patches:" + cmake -DLLVM_ENABLE_PROJECTS=clang ` + -DLLVM_TARGETS_TO_BUILD="host;NVPTX" ` + -DCMAKE_BUILD_TYPE=Release ` + -DLLVM_ENABLE_ASSERTIONS=ON ` + -DLLVM_ENABLE_LLD=ON ` + -DCLANG_ENABLE_STATIC_ANALYZER=OFF ` + -DCLANG_ENABLE_ARCMT=OFF ` + -DCLANG_ENABLE_FORMAT=OFF ` + -DCLANG_ENABLE_BOOTSTRAP=OFF ` + ../llvm + cmake --build . --config Release --target clang clang-repl --parallel ${{ env.ncpus }} + } + cd ../../ + - name: Save Cache LLVM/Clang runtime build directory uses: actions/cache/save@v3 if: ${{ steps.cache.outputs.cache-hit != 'true' }} @@ -159,79 +535,163 @@ jobs: llvm-project ${{ matrix.cling=='On' && 'cling' || '' }} key: ${{ steps.cache.outputs.cache-primary-key }} - - name: Build and Install CppInterOp on Linux - if: runner.os == 'Linux' + + - name: Build and Test/Install CppInterOp on Unix systems + if: ${{ runner.os != 'windows' }} run: | - # Build CppInterOp next to cling and llvm-project. - LLVM_DIR="$(realpath llvm-project)" - LLVM_BUILD_DIR="$(realpath llvm-project/build)" - cling_on="${{ matrix.cling }}" - if [[ "${cling_on^^}" == "ON" ]]; then - CLING_DIR="$(realpath cling)" - CLING_BUILD_DIR="$(realpath cling/build)" - CPLUS_INCLUDE_PATH="${CLING_DIR}/tools/cling/include:${CLING_BUILD_DIR}/include:${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include" - else # repl - CPLUS_INCLUDE_PATH="${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include" + git clone --depth=1 https://github.com/compiler-research/CppInterOp.git + LLVM_DIR="$(pwd)/llvm-project" + LLVM_BUILD_DIR="$(pwd)/llvm-project/build" + cling_on=$(echo "${{ matrix.cling }}" | tr '[:lower:]' '[:upper:]') + if [[ "${cling_on}" == "ON" ]]; then + CLING_DIR="$(pwd)/cling" + CLING_BUILD_DIR="$(pwd)/cling/build" + CPLUS_INCLUDE_PATH="${CLING_DIR}/tools/cling/include:${CLING_BUILD_DIR}/include:${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include:$PWD/include" + else + CPLUS_INCLUDE_PATH="${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include:$PWD/include" fi - git clone https://github.com/compiler-research/CppInterOp.git - export CPPINTEROP_DIR=$PWD/python/cppyy_backend/ - cd CppInterOp - mkdir build - cd build + + export CB_PYTHON_DIR="$PWD/cppyy-backend/python" + export CPPINTEROP_DIR="$CB_PYTHON_DIR/cppyy_backend" + + # Build CppInterOp next to cling and llvm-project. + mkdir CppInterOp/build + cd CppInterOp/build export CPPINTEROP_BUILD_DIR=$PWD - cling_on="${{ matrix.cling }}" - if [[ "${cling_on^^}" == "ON" ]]; then - cmake -DCMAKE_BUILD_TYPE=Release \ - -DUSE_CLING=ON \ - -DUSE_REPL=OFF \ - -DCling_DIR=$LLVM_BUILD_DIR/tools/cling \ - -DLLVM_DIR=$LLVM_BUILD_DIR \ - -DLLVM_USE_LINKER=lld \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR \ + cling_on=$(echo "${{ matrix.cling }}" | tr '[:lower:]' '[:upper:]') + if [[ "${cling_on}" == "ON" ]]; then + cmake -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DUSE_CLING=ON \ + -DUSE_REPL=OFF \ + -DCling_DIR=$LLVM_BUILD_DIR/tools/cling \ + -DLLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm \ + -DClang_DIR=$LLVM_BUILD_DIR/lib/cmake/clang \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR \ ../ else - cmake -DCMAKE_BUILD_TYPE=Release \ - -DUSE_CLING=OFF \ - -DUSE_REPL=ON \ - -DLLVM_DIR=$LLVM_BUILD_DIR \ - -DLLVM_USE_LINKER=lld \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR \ + cmake -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DUSE_CLING=OFF \ + -DUSE_REPL=ON \ + -DLLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm \ + -DClang_DIR=$LLVM_BUILD_DIR/lib/cmake/clang \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR \ ../ fi - cmake --build . --parallel $(nproc --all) + os="${{ matrix.os }}" + cmake --build . --target check-cppinterop --parallel ${{ env.ncpus }} + cppyy_on=$(echo "${{ matrix.cppyy }}" | tr '[:lower:]' '[:upper:]') + if [[ ("${cppyy_on}" != "ON") && ("${os}" == "ubuntu"*) ]]; then + # TODO: Remove "|| true" when fix memory issues in LLVM/Clang 17 + valgrind --track-origins=yes --error-exitcode=1 unittests/CppInterOp/CppInterOpTests 2>&1 >/dev/null || true + fi cd ../.. + # We need CB_PYTHON_DIR later + echo "CB_PYTHON_DIR=$CB_PYTHON_DIR" >> $GITHUB_ENV # We need CPPINTEROP_DIR, LLVM_BUILD_DIR and CPLUS_INCLUDE_PATH later - echo "CPPINTEROP_DIR=$CPPINTEROP_DIR" >> $GITHUB_ENV echo "CPPINTEROP_BUILD_DIR=$CPPINTEROP_BUILD_DIR" >> $GITHUB_ENV + echo "CPPINTEROP_DIR=$CPPINTEROP_DIR" >> $GITHUB_ENV echo "LLVM_BUILD_DIR=$LLVM_BUILD_DIR" >> $GITHUB_ENV echo "CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH" >> $GITHUB_ENV - - name: Build and Install cppyy-backend on Linux - if: runner.os == 'Linux' + + - name: Build and Test/Install CppInterOp on Windows systems + if: ${{ runner.os == 'windows' }} + run: | + git clone --depth=1 https://github.com/compiler-research/CppInterOp.git + $env:PWD_DIR= $PWD.Path + + $env:LLVM_DIR="$env:PWD_DIR\llvm-project" + echo "LLVM_DIR=$env:LLVM_DIR" + echo "LLVM_DIR=$env:LLVM_DIR" >> $env:GITHUB_ENV + + $env:LLVM_BUILD_DIR="$env:PWD_DIR\llvm-project\build" + echo "LLVM_BUILD_DIR=$env:LLVM_BUILD_DIR" + echo "LLVM_BUILD_DIR=$env:LLVM_BUILD_DIR" >> $env:GITHUB_ENV + + if ( "${{ matrix.cling }}" -imatch "On" ) + { + $env:CLING_DIR="$env:PWD_DIR\cling" + echo "CLING_DIR=$env:CLING_DIR" + echo "CLING_DIR=$env:CLING_DIR" >> $env:GITHUB_ENV + + $env:CLING_BUILD_DIR="$env:PWD_DIR\cling\build" + echo "CLING_BUILD_DIR=$env:CLING_BUILD_DIR" + echo "CLING_BUILD_DIR=$env:CLING_BUILD_DIR" >> $env:GITHUB_ENV + + $env:CPLUS_INCLUDE_PATH="$env:CLING_DIR\tools\cling\include;$env:CLING_BUILD_DIR\include;$env:LLVM_DIR\llvm\include;$env:LLVM_DIR\clang\include;$env:LLVM_BUILD_DIR\include;$env:LLVM_BUILD_DIR\tools\clang\include;$env:PWD_DIR\include;" + echo "CPLUS_INCLUDE_PATH=$env:CPLUS_INCLUDE_PATH" + echo "CPLUS_INCLUDE_PATH=$env:CPLUS_INCLUDE_PATH" >> $env:GITHUB_ENV + } + else + { + $env:CPLUS_INCLUDE_PATH="$env:LLVM_DIR\llvm\include;$env:LLVM_DIR\clang\include;$env:LLVM_BUILD_DIR\include;$env:LLVM_BUILD_DIR\tools\clang\include;$env:PWD_DIR\include;" + echo "CPLUS_INCLUDE_PATH=$env:CPLUS_INCLUDE_PATH" + echo "CPLUS_INCLUDE_PATH=$env:CPLUS_INCLUDE_PATH" >> $env:GITHUB_ENV + } + + $env:CB_PYTHON_DIR="$env:PWD_DIR\cppyy-backend\python" + echo "CB_PYTHON_DIR=$env:CB_PYTHON_DIR" + echo "CB_PYTHON_DIR=$env:CB_PYTHON_DIR" >> $env:GITHUB_ENV + + $env:CPPINTEROP_DIR="$env:CB_PYTHON_DIR\cppyy-backend" + echo "CPPINTEROP_DIR=$env:CPPINTEROP_DIR" + echo "CPPINTEROP_DIR=$env:CPPINTEROP_DIR" >> $env:GITHUB_ENV + + # Build CppInterOp next to cling and llvm-project. + mkdir CppInterOp\build + cd CppInterOp\build + $env:CPPINTEROP_BUILD_DIR="$env:PWD_DIR" + echo "CPPINTEROP_BUILD_DIR=$env:CPPINTEROP_BUILD_DIR" + echo "CPPINTEROP_BUILD_DIR=$env:CPPINTEROP_BUILD_DIR" >> $env:GITHUB_ENV + if ( "${{ matrix.cling }}" -imatch "On" ) + { + cmake -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} ` + -DUSE_CLING=ON ` + -DUSE_REPL=OFF ` + -DCling_DIR="$env:LLVM_BUILD_DIR\tools\cling" ` + -DLLVM_DIR="$env:LLVM_BUILD_DIR" ` + -DClang_DIR="$env:LLVM_BUILD_DIR" -DCMAKE_INSTALL_PREFIX="$env:CPPINTEROP_DIR" ..\ + } + else + { + cmake -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} ` + -DUSE_CLING=OFF ` + -DUSE_REPL=ON ` + -DLLVM_DIR="$env:LLVM_BUILD_DIR\lib\cmake\llvm" ` + -DClang_DIR="$env:LLVM_BUILD_DIR\lib\cmake\clang" -DCMAKE_INSTALL_PREFIX="$env:CPPINTEROP_DIR" ..\ + } + cmake --build . --config ${{ env.BUILD_TYPE }} --target check-cppinterop --parallel ${{ env.ncpus }} + cd ..\.. + + - name: Build and Install cppyy-backend on Unix Systems + if: ${{ (runner.os != 'windows') && (matrix.cppyy == 'On') }} run: | - # Install cppyy-backend - mkdir -p python/cppyy_backend/lib build && cd build + cd cppyy-backend + mkdir -p $CPPINTEROP_DIR/lib build && cd build + # Install CppInterOp (cd $CPPINTEROP_BUILD_DIR && cmake --build . --target install --parallel $(nproc --all)) + # Build and Install cppyy-backend cmake -DCppInterOp_DIR=$CPPINTEROP_DIR .. cmake --build . --parallel $(nproc --all) - cp libcppyy-backend.so ../python/cppyy_backend/lib/ - # - cd ../python - export CB_PYTHON_DIR=$PWD - cd ../.. - # We need CB_PYTHON_DIR later - echo "CB_PYTHON_DIR=$CB_PYTHON_DIR" >> $GITHUB_ENV - - name: Install CPyCppyy on Linux - if: runner.os == 'Linux' + OS=$(uname -s) + if [[ "$OS" == "Darwin" ]]; then + cp libcppyy-backend.dylib $CPPINTEROP_DIR/lib/ + else + cp libcppyy-backend.so $CPPINTEROP_DIR/lib/ + fi + cd .. + + - name: Install CPyCppyy on Unix Systems + if: ${{ (runner.os != 'windows') && (matrix.cppyy == 'On') }} run: | # Setup virtual environment python3 -m venv .venv source .venv/bin/activate # Install CPyCppyy - git clone https://github.com/compiler-research/CPyCppyy.git - cd CPyCppyy - mkdir build && cd build + git clone --depth=1 https://github.com/compiler-research/CPyCppyy.git + mkdir CPyCppyy/build + cd CPyCppyy/build cmake .. cmake --build . --parallel $(nproc --all) # @@ -239,18 +699,18 @@ jobs: cd ../.. # We need CPYCPPYY_DIR later echo "CPYCPPYY_DIR=$CPYCPPYY_DIR" >> $GITHUB_ENV - - name: Install cppyy on Linux - if: runner.os == 'Linux' + - name: Install cppyy on Unix Systems + if: ${{ (runner.os != 'windows') && (matrix.cppyy == 'On') }} run: | # source virtual environment source .venv/bin/activate # Install cppyy - git clone https://github.com/compiler-research/cppyy.git + git clone --depth=1 https://github.com/compiler-research/cppyy.git cd cppyy python -m pip install --upgrade . --no-deps cd .. - - name: Run cppyy on Linux - if: runner.os == 'Linux' + - name: Run cppyy on Unix Systems + if: ${{ (runner.os != 'windows') && (matrix.cppyy == 'On') }} run: | # Run cppyy source .venv/bin/activate @@ -258,8 +718,9 @@ jobs: python -c "import cppyy" # We need PYTHONPATH later echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV - - name: Run the tests on Linux - if: runner.os == 'Linux' + - name: Run the tests on Unix Systems + continue-on-error: true + if: ${{ (runner.os != 'windows') && (matrix.cppyy == 'On') }} run: | # Run the tests source .venv/bin/activate @@ -303,8 +764,15 @@ jobs: tail -n1 test_xfailed.log echo "Return Code: ${RETCODE}" exit $RETCODE + + - name: Show debug info + if: ${{ failure() && (runner.os != 'windows') }} + run: | + export + echo $GITHUB_ENV + - name: Setup tmate session - if: ${{ failure() }} + if: ${{ failure() && runner.debug }} uses: mxschmitt/action-tmate@v3 # When debugging increase to a suitable value! - timeout-minutes: ${{ github.event.pull_request && 1 || 20 }} + timeout-minutes: 30 diff --git a/patches/llvm/clang16-1-Value.patch b/patches/llvm/clang16-1-Value.patch index 9d702ee8..854077c7 100644 --- a/patches/llvm/clang16-1-Value.patch +++ b/patches/llvm/clang16-1-Value.patch @@ -4,10 +4,10 @@ index 863f6ac57..feb6db113 100644 +++ b/clang/include/clang/AST/Decl.h @@ -4308,6 +4308,7 @@ class TopLevelStmtDecl : public Decl { friend class ASTDeclWriter; - + Stmt *Statement = nullptr; + bool IsSemiMissing = false; - + TopLevelStmtDecl(DeclContext *DC, SourceLocation L, Stmt *S) : Decl(TopLevelStmt, DC, L), Statement(S) {} @@ -4321,6 +4322,12 @@ public: @@ -20,7 +20,7 @@ index 863f6ac57..feb6db113 100644 + } + bool isSemiMissing() const { return IsSemiMissing; } + void setSemiMissing(bool Missing = true) { IsSemiMissing = Missing; } - + static bool classof(const Decl *D) { return classofKind(D->getKind()); } static bool classofKind(Kind K) { return K == TopLevelStmt; } diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def @@ -30,7 +30,7 @@ index 96feae991..752629855 100644 @@ -936,6 +936,9 @@ ANNOTATION(module_end) // into the name of a header unit. ANNOTATION(header_unit) - + +// Annotation for end of input in clang-repl. +ANNOTATION(repl_input_end) + @@ -44,14 +44,14 @@ index fd22af976..e68021845 100644 @@ -14,13 +14,15 @@ #ifndef LLVM_CLANG_INTERPRETER_INTERPRETER_H #define LLVM_CLANG_INTERPRETER_INTERPRETER_H - + -#include "clang/Interpreter/PartialTranslationUnit.h" - +#include "clang/AST/Decl.h" #include "clang/AST/GlobalDecl.h" +#include "clang/Interpreter/PartialTranslationUnit.h" +#include "clang/Interpreter/Value.h" - + +#include "llvm/ADT/DenseMap.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" @@ -59,7 +59,7 @@ index fd22af976..e68021845 100644 - #include #include - + @@ -28,7 +30,7 @@ namespace llvm { namespace orc { class LLJIT; @@ -67,12 +67,12 @@ index fd22af976..e68021845 100644 -} +} // namespace orc } // namespace llvm - + namespace clang { @@ -52,39 +54,64 @@ class Interpreter { - + Interpreter(std::unique_ptr CI, llvm::Error &Err); - + + llvm::Error CreateExecutor(); + unsigned InitPTUSize = 0; + @@ -103,10 +103,10 @@ index fd22af976..e68021845 100644 - } + llvm::Error ParseAndExecute(llvm::StringRef Code, Value *V = nullptr); + llvm::Expected CompileDtorCall(CXXRecordDecl *CXXRD); - + /// Undo N previous incremental inputs. llvm::Error Undo(unsigned N = 1); - + - /// \returns the \c JITTargetAddress of a \c GlobalDecl. This interface uses + /// Link a dynamic library + llvm::Error LoadDynamicLibrary(const char *name); @@ -116,13 +116,13 @@ index fd22af976..e68021845 100644 /// mangled name. - llvm::Expected getSymbolAddress(GlobalDecl GD) const; + llvm::Expected getSymbolAddress(GlobalDecl GD) const; - + - /// \returns the \c JITTargetAddress of a given name as written in the IR. - llvm::Expected + /// \returns the \c ExecutorAddr of a given name as written in the IR. + llvm::Expected getSymbolAddress(llvm::StringRef IRName) const; - + - /// \returns the \c JITTargetAddress of a given name as written in the object + /// \returns the \c ExecutorAddr of a given name as written in the object /// file. @@ -148,7 +148,7 @@ index fd22af976..e68021845 100644 + llvm::SmallVector ValuePrintingInfo; }; } // namespace clang - + diff --git a/clang/include/clang/Interpreter/Value.h b/clang/include/clang/Interpreter/Value.h new file mode 100644 index 000000000..4df436703 @@ -377,7 +377,7 @@ index 6f9581b9e..6b73f43a1 100644 + Kind == tok::annot_module_end || Kind == tok::annot_module_include || + Kind == tok::annot_repl_input_end; } - + /// Checks if the \p Level is valid for use in a fold expression. diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp index ffa85e523..1b262d9e6 100644 @@ -391,7 +391,7 @@ index ffa85e523..1b262d9e6 100644 + !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end) && + !Tok.is(tok::annot_repl_input_end))) return; - + // EmittedDirectiveOnThisLine takes priority over RequireSameLine. @@ -819,6 +820,9 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok, // -traditional-cpp the lexer keeps /all/ whitespace, including comments. @@ -413,7 +413,7 @@ index c49f22fdd..565e824bf 100644 Interpreter.cpp + InterpreterUtils.cpp + Value.cpp - + DEPENDS intrinsics_gen diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp @@ -423,20 +423,20 @@ index 37d230b61..489ea48e0 100644 @@ -86,7 +86,7 @@ llvm::Error IncrementalExecutor::runCtors() const { return Jit->initialize(Jit->getMainJITDylib()); } - + -llvm::Expected +llvm::Expected IncrementalExecutor::getSymbolAddress(llvm::StringRef Name, SymbolNameKind NameKind) const { auto Sym = (NameKind == LinkerName) ? Jit->lookupLinkerMangled(Name) @@ -94,7 +94,7 @@ IncrementalExecutor::getSymbolAddress(llvm::StringRef Name, - + if (!Sym) return Sym.takeError(); - return Sym->getValue(); + return Sym; } - + } // end namespace clang diff --git a/clang/lib/Interpreter/IncrementalExecutor.h b/clang/lib/Interpreter/IncrementalExecutor.h index 54d37c763..dd0a210a0 100644 @@ -447,9 +447,9 @@ index 54d37c763..dd0a210a0 100644 #include "llvm/ADT/StringRef.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" - + #include - + @@ -51,9 +52,10 @@ public: llvm::Error removeModule(PartialTranslationUnit &PTU); llvm::Error runCtors() const; @@ -461,7 +461,7 @@ index 54d37c763..dd0a210a0 100644 + + llvm::orc::LLJIT &GetExecutionEngine() { return *Jit; } }; - + } // end namespace clang diff --git a/clang/lib/Interpreter/IncrementalParser.cpp b/clang/lib/Interpreter/IncrementalParser.cpp index 373e2844b..e43189071 100644 @@ -469,7 +469,7 @@ index 373e2844b..e43189071 100644 +++ b/clang/lib/Interpreter/IncrementalParser.cpp @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// - + #include "IncrementalParser.h" - #include "clang/AST/DeclContextInternals.h" @@ -487,9 +487,9 @@ index 373e2844b..e43189071 100644 #include "llvm/Support/CrashRecoveryContext.h" #include "llvm/Support/Error.h" @@ -31,6 +30,79 @@ - + namespace clang { - + +class IncrementalASTConsumer final : public ASTConsumer { + Interpreter &Interp; + std::unique_ptr Consumer; @@ -569,7 +569,7 @@ index 373e2844b..e43189071 100644 @@ -122,7 +194,8 @@ public: } }; - + -IncrementalParser::IncrementalParser(std::unique_ptr Instance, +IncrementalParser::IncrementalParser(Interpreter &Interp, + std::unique_ptr Instance, @@ -588,7 +588,7 @@ index 373e2844b..e43189071 100644 new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false)); @@ -158,8 +234,8 @@ IncrementalParser::ParseOrWrapTopLevelDecl() { LastPTU.TUPart = C.getTranslationUnitDecl(); - + // Skip previous eof due to last incremental input. - if (P->getCurToken().is(tok::eof)) { - P->ConsumeToken(); @@ -609,7 +609,7 @@ index 373e2844b..e43189071 100644 + assert(AssertTok.is(tok::annot_repl_input_end) && + "Lexer must be EOF when starting incremental parse!"); } - + - Token AssertTok; - PP.Lex(AssertTok); - assert(AssertTok.is(tok::eof) && @@ -619,7 +619,7 @@ index 373e2844b..e43189071 100644 + + return PTU; +} - + +std::unique_ptr IncrementalParser::GenModule() { + static unsigned ID = 0; if (CodeGenerator *CG = getCodeGen(Act.get())) { @@ -635,7 +635,7 @@ index 373e2844b..e43189071 100644 - return PTU; + return nullptr; } - + void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) { diff --git a/clang/lib/Interpreter/IncrementalParser.h b/clang/lib/Interpreter/IncrementalParser.h index 8e45d6b59..99e37588d 100644 @@ -643,7 +643,7 @@ index 8e45d6b59..99e37588d 100644 +++ b/clang/lib/Interpreter/IncrementalParser.h @@ -16,7 +16,6 @@ #include "clang/Interpreter/PartialTranslationUnit.h" - + #include "clang/AST/GlobalDecl.h" - #include "llvm/ADT/ArrayRef.h" @@ -661,18 +661,18 @@ index 8e45d6b59..99e37588d 100644 /// @@ -57,7 +56,8 @@ class IncrementalParser { std::list PTUs; - + public: - IncrementalParser(std::unique_ptr Instance, + IncrementalParser(Interpreter &Interp, + std::unique_ptr Instance, llvm::LLVMContext &LLVMCtx, llvm::Error &Err); ~IncrementalParser(); - + @@ -76,6 +76,8 @@ public: - + std::list &getPTUs() { return PTUs; } - + + std::unique_ptr GenModule(); + private: @@ -685,7 +685,7 @@ index a6f5fdc6e..4391bd008 100644 @@ -16,7 +16,11 @@ #include "IncrementalExecutor.h" #include "IncrementalParser.h" - + +#include "InterpreterUtils.h" #include "clang/AST/ASTContext.h" +#include "clang/AST/Mangle.h" @@ -711,7 +711,7 @@ index a6f5fdc6e..4391bd008 100644 #include "llvm/Support/Host.h" - using namespace clang; - + // FIXME: Figure out how to unify with namespace init_convenience from @@ -176,7 +184,7 @@ Interpreter::Interpreter(std::unique_ptr CI, llvm::ErrorAsOutParameter EAO(&Err); @@ -721,11 +721,11 @@ index a6f5fdc6e..4391bd008 100644 + IncrParser = std::make_unique(*this, std::move(CI), *TSCtx->getContext(), Err); } - + @@ -189,6 +197,29 @@ Interpreter::~Interpreter() { } } - + +// These better to put in a runtime header but we can't. This is because we +// can't find the precise resource directory in unittests so we have to hard +// code them. @@ -767,11 +767,11 @@ index a6f5fdc6e..4391bd008 100644 + Interp->InitPTUSize = Interp->IncrParser->getPTUs().size(); return std::move(Interp); } - + @@ -203,25 +243,53 @@ const CompilerInstance *Interpreter::getCompilerInstance() const { return IncrParser->getCI(); } - + -const llvm::orc::LLJIT *Interpreter::getExecutionEngine() const { - if (IncrExecutor) - return IncrExecutor->getExecutionEngine(); @@ -798,7 +798,7 @@ index a6f5fdc6e..4391bd008 100644 + assert(PTUs.size() >= InitPTUSize && "empty PTU list?"); + return PTUs.size() - InitPTUSize; } - + llvm::Expected Interpreter::Parse(llvm::StringRef Code) { + // Tell the interpreter sliently ignore unused expressions since value @@ -807,7 +807,7 @@ index a6f5fdc6e..4391bd008 100644 + clang::diag::warn_unused_expr, diag::Severity::Ignored, SourceLocation()); return IncrParser->Parse(Code); } - + +llvm::Error Interpreter::CreateExecutor() { + const clang::TargetInfo &TI = + getCompilerInstance()->getASTContext().getTargetInfo(); @@ -834,7 +834,7 @@ index a6f5fdc6e..4391bd008 100644 @@ -235,7 +303,26 @@ llvm::Error Interpreter::Execute(PartialTranslationUnit &T) { return llvm::Error::success(); } - + -llvm::Expected +llvm::Error Interpreter::ParseAndExecute(llvm::StringRef Code, Value *V) { + @@ -862,7 +862,7 @@ index a6f5fdc6e..4391bd008 100644 @@ -245,7 +332,7 @@ Interpreter::getSymbolAddress(GlobalDecl GD) const { return getSymbolAddress(MangledName); } - + -llvm::Expected +llvm::Expected Interpreter::getSymbolAddress(llvm::StringRef IRName) const { @@ -871,7 +871,7 @@ index a6f5fdc6e..4391bd008 100644 @@ -255,7 +342,7 @@ Interpreter::getSymbolAddress(llvm::StringRef IRName) const { return IncrExecutor->getSymbolAddress(IRName, IncrementalExecutor::IRName); } - + -llvm::Expected +llvm::Expected Interpreter::getSymbolAddressFromLinkerName(llvm::StringRef Name) const { @@ -879,7 +879,7 @@ index a6f5fdc6e..4391bd008 100644 return llvm::make_error("Operation failed. " @@ -268,7 +355,7 @@ Interpreter::getSymbolAddressFromLinkerName(llvm::StringRef Name) const { llvm::Error Interpreter::Undo(unsigned N) { - + std::list &PTUs = IncrParser->getPTUs(); - if (N > PTUs.size()) + if (N > getEffectivePTUSize()) @@ -1718,7 +1718,7 @@ index 66168467e..0822f83b5 100644 + } else { + CurLexer->FormTokenWithChars(Result, EndPos, tok::eof); + } - + if (isCodeCompletionEnabled()) { // Inserting the code-completion point increases the source buffer by 1, diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp @@ -1732,7 +1732,7 @@ index 3a7f5426d..57a3dfba4 100644 + case tok::annot_repl_input_end: // Ran out of tokens. return false; - + @@ -1242,6 +1243,7 @@ bool Parser::ConsumeAndStoreInitializer(CachedTokens &Toks, case tok::annot_module_begin: case tok::annot_module_end: @@ -1740,7 +1740,7 @@ index 3a7f5426d..57a3dfba4 100644 + case tok::annot_repl_input_end: // Ran out of tokens. return false; - + diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index e6812ac72..2f193a3b4 100644 --- a/clang/lib/Parse/ParseDecl.cpp @@ -1751,10 +1751,10 @@ index e6812ac72..2f193a3b4 100644 case tok::annot_module_include: + case tok::annot_repl_input_end: return; - + default: @@ -5394,6 +5395,13 @@ Parser::DeclGroupPtrTy Parser::ParseTopLevelStmtDecl() { - + SmallVector DeclsInGroup; DeclsInGroup.push_back(Actions.ActOnTopLevelStmtDecl(R.get())); + @@ -1774,7 +1774,7 @@ index 1c8441faf..d22e1d440 100644 @@ -543,9 +543,22 @@ StmtResult Parser::ParseExprStatement(ParsedStmtContext StmtCtx) { return ParseCaseStatement(StmtCtx, /*MissingCase=*/true, Expr); } - + - // Otherwise, eat the semicolon. - ExpectAndConsumeSemi(diag::err_expected_semi_after_expr); - return handleExprStmt(Expr, StmtCtx); @@ -1795,7 +1795,7 @@ index 1c8441faf..d22e1d440 100644 + + return R; } - + /// ParseSEHTryBlockCommon diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp index 6db3dc315..7fbb27057 100644 @@ -1812,7 +1812,7 @@ index 6db3dc315..7fbb27057 100644 @@ -612,11 +613,6 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result, Sema::ModuleImportState &ImportState) { DestroyTemplateIdAnnotationsRAIIObj CleanupRAII(*this); - + - // Skip over the EOF token, flagging end of previous input for incremental - // processing - if (PP.isIncrementalProcessingEnabled() && Tok.is(tok::eof)) @@ -1823,7 +1823,7 @@ index 6db3dc315..7fbb27057 100644 case tok::annot_pragma_unused: @@ -695,6 +691,7 @@ bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result, return false; - + case tok::eof: + case tok::annot_repl_input_end: // Check whether -fmax-tokens= was reached. @@ -1872,7 +1872,7 @@ index b51a18c10..15d7f9439 100644 +++ b/clang/tools/clang-repl/CMakeLists.txt @@ -12,6 +12,7 @@ add_clang_tool(clang-repl ) - + clang_target_link_libraries(clang-repl PRIVATE + clangAST clangBasic @@ -1893,7 +1893,7 @@ index 401a31d34..33faf3fab 100644 + } + continue; + } - + if (auto Err = Interp->ParseAndExecute(*Line)) { llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: "); diff --git a/clang/unittests/Interpreter/CMakeLists.txt b/clang/unittests/Interpreter/CMakeLists.txt @@ -1915,7 +1915,7 @@ index f54c65568..6d0433a98 100644 #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/TargetSelect.h" -#include "llvm-c/Error.h" - + #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -116,7 +115,8 @@ extern "C" int throw_exception() { @@ -1939,11 +1939,11 @@ index d4900a0e4..330fd18ab 100644 +#include "clang/Interpreter/Value.h" #include "clang/Sema/Lookup.h" #include "clang/Sema/Sema.h" - + @@ -33,6 +34,11 @@ using namespace clang; #define CLANG_INTERPRETER_NO_SUPPORT_EXEC #endif - + +int Global = 42; +// JIT reports symbol not found on Windows without the visibility attribute. +REPL_EXTERNAL_VISIBILITY int getGlobal() { return Global; } @@ -1953,7 +1953,7 @@ index d4900a0e4..330fd18ab 100644 using Args = std::vector; static std::unique_ptr @@ -225,7 +231,7 @@ TEST(IncrementalProcessing, FindMangledNameSymbol) { - + std::string MangledName = MangleName(FD); auto Addr = cantFail(Interp->getSymbolAddress(MangledName)); - EXPECT_NE(0U, Addr); @@ -1964,7 +1964,7 @@ index d4900a0e4..330fd18ab 100644 @@ -276,8 +282,7 @@ TEST(IncrementalProcessing, InstantiateTemplate) { std::vector Args = {"-fno-delayed-template-parsing"}; std::unique_ptr Interp = createInterpreter(Args); - + - llvm::cantFail(Interp->Parse("void* operator new(__SIZE_TYPE__, void* __p);" - "extern \"C\" int printf(const char*,...);" + llvm::cantFail(Interp->Parse("extern \"C\" int printf(const char*,...);" @@ -1972,7 +1972,7 @@ index d4900a0e4..330fd18ab 100644 "struct B {" " template" @@ -309,9 +314,109 @@ TEST(IncrementalProcessing, InstantiateTemplate) { - + std::string MangledName = MangleName(TmpltSpec); typedef int (*TemplateSpecFn)(void *); - auto fn = (TemplateSpecFn)cantFail(Interp->getSymbolAddress(MangledName)); @@ -1981,7 +1981,7 @@ index d4900a0e4..330fd18ab 100644 EXPECT_EQ(42, fn(NewA)); free(NewA); } - + +#ifdef CLANG_INTERPRETER_NO_SUPPORT_EXEC +TEST(InterpreterTest, DISABLED_Value) { +#else diff --git a/patches/llvm/clang17-1-NewOperator.patch b/patches/llvm/clang17-1-NewOperator.patch new file mode 100644 index 00000000..fd32d792 --- /dev/null +++ b/patches/llvm/clang17-1-NewOperator.patch @@ -0,0 +1,205 @@ +From a3f213ef4a7e293152c272cce78ad5d10a3ede52 Mon Sep 17 00:00:00 2001 +From: Vassil Vassilev +Date: Fri, 22 Dec 2023 08:38:23 +0000 +Subject: [PATCH] [clang-repl] Add a interpreter-specific overload of operator + new for C++. + +This patch brings back the basic support for C by inserting the required for +value printing runtime only when we are in C++ mode. Additionally, it defines +a new overload of operator placement new because we can't really forward declare +it in a library-agnostic way. + +Fixes the issue described in llvm/llvm-project#69072. +--- + clang/include/clang/Interpreter/Interpreter.h | 4 +-- + clang/lib/Interpreter/Interpreter.cpp | 33 +++++++++++++++---- + clang/test/Interpreter/incremental-mode.cpp | 3 +- + .../unittests/Interpreter/InterpreterTest.cpp | 29 +++------------- + 4 files changed, 36 insertions(+), 33 deletions(-) + +diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h +index 01858dfcc90ac5..292fa566ae7037 100644 +--- a/clang/include/clang/Interpreter/Interpreter.h ++++ b/clang/include/clang/Interpreter/Interpreter.h +@@ -129,7 +129,7 @@ class Interpreter { + llvm::Expected + getSymbolAddressFromLinkerName(llvm::StringRef LinkerName) const; + +- enum InterfaceKind { NoAlloc, WithAlloc, CopyArray }; ++ enum InterfaceKind { NoAlloc, WithAlloc, CopyArray, NewTag }; + + const llvm::SmallVectorImpl &getValuePrintingInfo() const { + return ValuePrintingInfo; +@@ -144,7 +144,7 @@ class Interpreter { + + llvm::DenseMap Dtors; + +- llvm::SmallVector ValuePrintingInfo; ++ llvm::SmallVector ValuePrintingInfo; + }; + } // namespace clang + +diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp +index c9fcef5b5b5af1..9f97a3c6b0be9e 100644 +--- a/clang/lib/Interpreter/Interpreter.cpp ++++ b/clang/lib/Interpreter/Interpreter.cpp +@@ -248,7 +248,7 @@ Interpreter::~Interpreter() { + // can't find the precise resource directory in unittests so we have to hard + // code them. + const char *const Runtimes = R"( +- void* operator new(__SIZE_TYPE__, void* __p) noexcept; ++#ifdef __cplusplus + void *__clang_Interpreter_SetValueWithAlloc(void*, void*, void*); + void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*); + void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, void*); +@@ -256,15 +256,18 @@ const char *const Runtimes = R"( + void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, double); + void __clang_Interpreter_SetValueNoAlloc(void*, void*, void*, long double); + void __clang_Interpreter_SetValueNoAlloc(void*,void*,void*,unsigned long long); ++ struct __clang_Interpreter_NewTag{} __ci_newtag; ++ void* operator new(__SIZE_TYPE__, void* __p, __clang_Interpreter_NewTag) noexcept; + template + void __clang_Interpreter_SetValueCopyArr(T* Src, void* Placement, unsigned long Size) { + for (auto Idx = 0; Idx < Size; ++Idx) +- new ((void*)(((T*)Placement) + Idx)) T(Src[Idx]); ++ new ((void*)(((T*)Placement) + Idx), __ci_newtag) T(Src[Idx]); + } + template + void __clang_Interpreter_SetValueCopyArr(const T (*Src)[N], void* Placement, unsigned long Size) { + __clang_Interpreter_SetValueCopyArr(Src[0], Placement, Size); + } ++#endif // __cplusplus + )"; + + llvm::Expected> +@@ -279,7 +282,7 @@ Interpreter::create(std::unique_ptr CI) { + if (!PTU) + return PTU.takeError(); + +- Interp->ValuePrintingInfo.resize(3); ++ Interp->ValuePrintingInfo.resize(4); + // FIXME: This is a ugly hack. Undo command checks its availability by looking + // at the size of the PTU list. However we have parsed something in the + // beginning of the REPL so we have to mark them as 'Irrevocable'. +@@ -500,7 +503,7 @@ Interpreter::CompileDtorCall(CXXRecordDecl *CXXRD) { + static constexpr llvm::StringRef MagicRuntimeInterface[] = { + "__clang_Interpreter_SetValueNoAlloc", + "__clang_Interpreter_SetValueWithAlloc", +- "__clang_Interpreter_SetValueCopyArr"}; ++ "__clang_Interpreter_SetValueCopyArr", "__ci_newtag"}; + + bool Interpreter::FindRuntimeInterface() { + if (llvm::all_of(ValuePrintingInfo, [](Expr *E) { return E != nullptr; })) +@@ -530,6 +533,9 @@ bool Interpreter::FindRuntimeInterface() { + if (!LookupInterface(ValuePrintingInfo[CopyArray], + MagicRuntimeInterface[CopyArray])) + return false; ++ if (!LookupInterface(ValuePrintingInfo[NewTag], ++ MagicRuntimeInterface[NewTag])) ++ return false; + return true; + } + +@@ -607,7 +613,9 @@ class RuntimeInterfaceBuilder + .getValuePrintingInfo()[Interpreter::InterfaceKind::CopyArray], + SourceLocation(), Args, SourceLocation()); + } +- Expr *Args[] = {AllocCall.get()}; ++ Expr *Args[] = { ++ AllocCall.get(), ++ Interp.getValuePrintingInfo()[Interpreter::InterfaceKind::NewTag]}; + ExprResult CXXNewCall = S.BuildCXXNew( + E->getSourceRange(), + /*UseGlobal=*/true, /*PlacementLParen=*/SourceLocation(), Args, +@@ -628,8 +636,9 @@ class RuntimeInterfaceBuilder + Interp.getValuePrintingInfo()[Interpreter::InterfaceKind::NoAlloc], + E->getBeginLoc(), Args, E->getEndLoc()); + } ++ default: ++ llvm_unreachable("Unhandled Interpreter::InterfaceKind"); + } +- llvm_unreachable("Unhandled Interpreter::InterfaceKind"); + } + + Interpreter::InterfaceKind VisitRecordType(const RecordType *Ty) { +@@ -814,3 +823,15 @@ __clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, + VRef = Value(static_cast(This), OpaqueType); + VRef.setLongDouble(Val); + } ++ ++// A trampoline to work around the fact that operator placement new cannot ++// really be forward declared due to libc++ and libstdc++ declaration mismatch. ++// FIXME: __clang_Interpreter_NewTag is ODR violation because we get the same ++// definition in the interpreter runtime. We should move it in a runtime header ++// which gets included by the interpreter and here. ++struct __clang_Interpreter_NewTag {}; ++REPL_EXTERNAL_VISIBILITY void * ++operator new(size_t __sz, void *__p, __clang_Interpreter_NewTag) noexcept { ++ // Just forward to the standard operator placement new. ++ return operator new(__sz, __p); ++} +diff --git a/clang/test/Interpreter/incremental-mode.cpp b/clang/test/Interpreter/incremental-mode.cpp +index e6350d237ef578..d63cee0dd6d15f 100644 +--- a/clang/test/Interpreter/incremental-mode.cpp ++++ b/clang/test/Interpreter/incremental-mode.cpp +@@ -1,3 +1,4 @@ + // RUN: clang-repl -Xcc -E +-// RUN: clang-repl -Xcc -emit-llvm ++// RUN: clang-repl -Xcc -emit-llvm ++// RUN: clang-repl -Xcc -xc + // expected-no-diagnostics +diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp +index 5f2911e9a7adad..1e0854b3c4af46 100644 +--- a/clang/unittests/Interpreter/InterpreterTest.cpp ++++ b/clang/unittests/Interpreter/InterpreterTest.cpp +@@ -248,28 +248,10 @@ TEST(IncrementalProcessing, FindMangledNameSymbol) { + #endif // _WIN32 + } + +-static void *AllocateObject(TypeDecl *TD, Interpreter &Interp) { ++static Value AllocateObject(TypeDecl *TD, Interpreter &Interp) { + std::string Name = TD->getQualifiedNameAsString(); +- const clang::Type *RDTy = TD->getTypeForDecl(); +- clang::ASTContext &C = Interp.getCompilerInstance()->getASTContext(); +- size_t Size = C.getTypeSize(RDTy); +- void *Addr = malloc(Size); +- +- // Tell the interpreter to call the default ctor with this memory. Synthesize: +- // new (loc) ClassName; +- static unsigned Counter = 0; +- std::stringstream SS; +- SS << "auto _v" << Counter++ << " = " +- << "new ((void*)" +- // Windows needs us to prefix the hexadecimal value of a pointer with '0x'. +- << std::hex << std::showbase << (size_t)Addr << ")" << Name << "();"; +- +- auto R = Interp.ParseAndExecute(SS.str()); +- if (!R) { +- free(Addr); +- return nullptr; +- } +- ++ Value Addr; ++ cantFail(Interp.ParseAndExecute("new " + Name + "()", &Addr)); + return Addr; + } + +@@ -317,7 +299,7 @@ TEST(IncrementalProcessing, InstantiateTemplate) { + } + + TypeDecl *TD = cast(LookupSingleName(*Interp, "A")); +- void *NewA = AllocateObject(TD, *Interp); ++ Value NewA = AllocateObject(TD, *Interp); + + // Find back the template specialization + VarDecl *VD = static_cast(*PTUDeclRange.begin()); +@@ -328,8 +310,7 @@ TEST(IncrementalProcessing, InstantiateTemplate) { + typedef int (*TemplateSpecFn)(void *); + auto fn = + cantFail(Interp->getSymbolAddress(MangledName)).toPtr(); +- EXPECT_EQ(42, fn(NewA)); +- free(NewA); ++ EXPECT_EQ(42, fn(NewA.getPtr())); + } + + #ifdef CLANG_INTERPRETER_NO_SUPPORT_EXEC