diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bf507bb..b0b0a0e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,10 +136,12 @@ jobs: # conda update --all conda install -y -q -c conda-forge \ - 'xeus>=2.0,<3.0' \ + 'xeus>=2.0' \ + xeus-zmq \ 'nlohmann_json>=3.9.1,<3.10' \ 'cppzmq>=4.6.0,<5' \ 'xtl>=0.7,<0.8' \ + 'openssl<4' \ pugixml \ 'cxxopts>=2.2.1,<2.3' \ libuuid \ diff --git a/CMakeLists.txt b/CMakeLists.txt index f22292f1..7e9e23d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,6 +89,7 @@ set(xeus_REQUIRED_VERSION 1.0.0) set(cppzmq_REQUIRED_VERSION 4.3.0) find_package(xeus ${xeus_REQUIRED_VERSION} REQUIRED) +find_package(xeus-zmq REQUIRED) find_package(cppzmq REQUIRED CONFIG) find_package(pugixml REQUIRED) @@ -103,6 +104,10 @@ if(POLICY CMP0077) endif() find_package(cxxopts REQUIRED) + +# Added as system unable to locate cxxopts include folder on MacOS without +include_directories(${cxxopts_DIR}/../include/) + set(Python_FIND_VIRTUALENV ONLY) find_package(Python COMPONENTS Interpreter Development) @@ -251,7 +256,7 @@ target_include_directories(xeus-clang-repl SYSTEM PUBLIC $ $) -target_link_libraries(xeus-clang-repl PUBLIC clangCppInterOp xeus pugixml cxxopts::cxxopts) +target_link_libraries(xeus-clang-repl PUBLIC clangCppInterOp xeus xeus-zmq pugixml) if(APPLE) target_link_libraries(xeus-clang-repl PUBLIC -Wl,-w -Wl,-bind_at_load -Wl,-undefined,dynamic_lookup) diff --git a/Dockerfile b/Dockerfile index bc7c7606..a18aad26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,9 @@ RUN \ ###libomp-dev \ # Other "our" apt installs (development and testing) build-essential \ + #FIXME: Update to clang-17 once available in apt repository (expected with next LTS version of Ubuntu April 2024) + # to be use a consistent version of clang throughout + clang-15 \ git \ nano-tiny \ less \ @@ -54,7 +57,9 @@ RUN \ #cuda \ $([ -n "$_CUDA_" ] && echo nvidia-cuda-toolkit) \ && \ - apt-get clean && rm -rf /var/lib/apt/lists/* && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* && \ echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ locale-gen @@ -130,13 +135,16 @@ RUN \ # Other "our" conda installs cmake \ #"clangdev=$LLVM_REQUIRED_VERSION" \ - 'xeus>=2.0,<3.0' \ + 'xeus>=2.0' \ + xeus-zmq \ 'nlohmann_json>=3.9.1,<3.10' \ 'cppzmq>=4.6.0,<5' \ 'xtl>=0.7,<0.8' \ - 'openssl<2' \ + 'openssl<4' \ ipykernel \ pugixml \ + zlib \ + libxml2 \ 'cxxopts>=2.2.1,<2.3' \ libuuid \ # Test dependencies @@ -159,6 +167,9 @@ RUN \ set -x && \ source /home/jovyan/.conda.init && \ conda activate .venv && \ + export ARCHITECHURE=$(uname -m) && \ + if [ "$ARCHITECHURE" == "x86_64" ] ; \ + then \ # artifact_name="clang-dev" && \ # @@ -167,9 +178,10 @@ RUN \ echo $PWD && git_remote_origin_url=$(git config --get remote.origin.url) && \ echo "Debug: Remote origin url: $git_remote_origin_url" && \ arr=(${git_remote_origin_url//\// }) && \ - gh_repo_owner=${arr[2]} && \ + arr2=(${arr[0]//:/ }) && \ + gh_repo_owner=${arr2[1]} && \ gh_f_repo_owner="compiler-research" && \ - arr=(${arr[3]//./ }) && \ + arr=(${arr[1]//./ }) && \ gh_repo_name=${arr[0]} && \ gh_repo="${gh_repo_owner}/${gh_repo_name}" && \ gh_f_repo_name=${gh_repo_name} && \ @@ -193,7 +205,6 @@ RUN \ pushd /home/runner/work/xeus-clang-repl/xeus-clang-repl && \ # repo echo "Debug: Repo owner/name: ${gh_repo_owner} / ${gh_repo_name}" && \ - #RUN \ # set -x && \ source /home/jovyan/.conda.init && \ @@ -231,26 +242,48 @@ RUN \ popd && \ # echo "Debug clang path: $PATH_TO_CLANG_DEV" && \ - export PATH_TO_LLVM_BUILD=$PATH_TO_CLANG_DEV/inst && \ + export PATH_TO_LLVM_BUILD=$PATH_TO_CLANG_DEV/inst/ && \ + export PATH_TO_LLVM_CONFIG=$PATH_TO_CLANG_DEV/inst/lib/cmake/llvm/ && \ + export PATH_TO_CLANG_CONFIG=$PATH_TO_CLANG_DEV/inst/lib/cmake/clang/ ; \ + fi && \ export VENV=${CONDA_DIR}/envs/.venv && \ - export PATH=${VENV}/bin:${CONDA_DIR}/bin:$PATH_TO_LLVM_BUILD/bin:$PATH && \ - export LD_LIBRARY_PATH=$PATH_TO_LLVM_BUILD/lib:$LD_LIBRARY_PATH && \ echo "export VENV=$VENV" >> ~/.profile && \ - echo "export PATH=$PATH" >> ~/.profile && \ echo "export EDITOR=emacs" >> ~/.profile && \ # - # Build CppInterOp - # + # Build LLVM (if on arm) & CppInterOp + # sys_incs=$(LC_ALL=C c++ -xc++ -E -v /dev/null 2>&1 | LC_ALL=C sed -ne '/starts here/,/End of/p' | LC_ALL=C sed '/^ /!d' | cut -c2- | tr '\n' ':') && \ export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/include/llvm:${PATH_TO_LLVM_BUILD}/include/clange:$CPLUS_INCLUDE_PATH:${sys_incs%:}" && \ git clone https://github.com/compiler-research/CppInterOp.git && \ + if [ "$ARCHITECHURE" == "aarch64" ] ; \ + then \ + #FIXME: Update to clang-17 once clang-17 is available in apt repository (expected with next LTS version of Ubuntu + # around April 2024) + export CC=/usr/bin/clang-15 && \ + export CXX=/usr/bin/clang++-15 && \ + #FIXME: Currently build with version 16.x on arm due to issue mentioned here https://github.com/compiler-research/xeus-clang-repl/issues/78 + git clone --depth=1 --branch release/16.x https://github.com/llvm/llvm-project.git && \ + cd ./llvm-project/ && \ + mkdir build && \ + export LLVM_DIR=$(pwd) && \ + compgen -G "../CppInterOp/patches/llvm/clang16-*.patch" > /dev/null && find ../CppInterOp/patches/llvm/clang16-*.patch -printf "%f\n" && git apply ../CppInterOp/patches/llvm/clang16-*.patch && \ + cd build && \ + cmake -DLLVM_ENABLE_PROJECTS=clang -DLLVM_TARGETS_TO_BUILD="host;NVPTX" -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_USE_LINKER=gold -DCLANG_ENABLE_STATIC_ANALYZER=OFF -DCLANG_ENABLE_ARCMT=OFF -DCLANG_ENABLE_FORMAT=OFF -DCLANG_ENABLE_BOOTSTRAP=OFF ../llvm && \ + cmake --build . --target clang clang-repl --parallel $(nproc --all) && \ + export PATH_TO_LLVM_CONFIG=$LLVM_DIR/build/lib/cmake/llvm/ && \ + export PATH_TO_CLANG_CONFIG=$LLVM_DIR/build/lib/cmake/clang/ && \ + export PATH_TO_LLVM_BUILD=$LLVM_DIR/build && \ + cd ../../ ; \ + fi && \ + export PATH=${VENV}/bin:${CONDA_DIR}/bin:$PATH_TO_LLVM_BUILD/bin:$PATH && \ + export LD_LIBRARY_PATH=$PATH_TO_LLVM_BUILD/lib:$LD_LIBRARY_PATH && \ + echo "export PATH=$PATH" >> ~/.profile && \ export CB_PYTHON_DIR="$PWD/cppyy-backend/python" && \ export CPPINTEROP_DIR="$CB_PYTHON_DIR/cppyy_backend" && \ - cd CppInterOp && \ - mkdir build && \ - cd build && \ + mkdir CppInterOp/build && \ + cd CppInterOp/build && \ export CPPINTEROP_BUILD_DIR=$PWD && \ - cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DUSE_CLING=OFF -DUSE_REPL=ON -DLLVM_DIR=$PATH_TO_LLVM_BUILD -DLLVM_USE_LINKER=gold -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR .. && \ + cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DUSE_CLING=OFF -DUSE_REPL=ON -DLLVM_DIR=$PATH_TO_LLVM_CONFIG -DClang_DIR=$PATH_TO_CLANG_CONFIG -DLLVM_USE_LINKER=gold -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR .. && \ cmake --build . --parallel $(nproc --all) && \ #make install -j$(nproc --all) export CPLUS_INCLUDE_PATH="$CPPINTEROP_DIR/include:$CPLUS_INCLUDE_PATH" && \ @@ -260,9 +293,10 @@ RUN \ # # Build and Install cppyy-backend # - git clone https://github.com/compiler-research/cppyy-backend.git && \ + git clone --depth=1 https://github.com/compiler-research/cppyy-backend.git && \ cd cppyy-backend && \ - mkdir -p $CPPINTEROP_DIR/lib build && cd build && \ + mkdir -p $CPPINTEROP_DIR/lib build && \ + cd build && \ # Install CppInterOp (cd $CPPINTEROP_BUILD_DIR && cmake --build . --target install --parallel $(nproc --all)) && \ # Build and Install cppyy-backend @@ -274,9 +308,9 @@ RUN \ # Build and Install CPyCppyy # # Install CPyCppyy - git clone https://github.com/compiler-research/CPyCppyy.git && \ - cd CPyCppyy && \ - mkdir build && cd build && \ + git clone --depth=1 https://github.com/compiler-research/CPyCppyy.git && \ + mkdir CPyCppyy/build && \ + cd CPyCppyy/build && \ cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. && \ cmake --build . --parallel $(nproc --all) && \ export CPYCPPYY_DIR=$PWD && \ @@ -285,7 +319,7 @@ RUN \ # Build and Install cppyy # # Install cppyy - git clone https://github.com/compiler-research/cppyy.git && \ + git clone --depth=1 https://github.com/compiler-research/cppyy.git && \ cd cppyy && \ python -m pip install --upgrade . --no-deps && \ cd .. && \ @@ -294,9 +328,9 @@ RUN \ export PYTHONPATH=$PYTHONPATH:$CPYCPPYY_DIR:$CB_PYTHON_DIR:/home/jovyan && \ echo "export PYTHONPATH=$PYTHONPATH" >> ~/.profile && \ export CPLUS_INCLUDE_PATH="/home/jovyan/CPyCppyy/include/:$CPLUS_INCLUDE_PATH" && \ - # FIXME: Remove the hardcoded version of python here. + export PYTHON_VERSION=$(python --version | cut -c8- | cut -f1,2 -d'.') && \ export CPLUS_INCLUDE_PATH="/home/jovyan/clad/include:$CPLUS_INCLUDE_PATH" && \ - export CPLUS_INCLUDE_PATH="${VENV}/include:${VENV}/include/python3.10:$CPLUS_INCLUDE_PATH" && \ + export CPLUS_INCLUDE_PATH="${VENV}/include:${VENV}/include/python${PYTHON_VERSION}:$CPLUS_INCLUDE_PATH" && \ python -c "import cppyy" && \ # # Build and Install xeus-clang-repl @@ -312,13 +346,11 @@ RUN \ # Build and Install Clad # git clone --depth=1 https://github.com/vgvassilev/clad.git && \ - cd clad && \ - mkdir build && \ - cd build && \ - cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. -DClang_DIR=${PATH_TO_LLVM_BUILD}/lib/cmake/clang/ -DLLVM_DIR=${PATH_TO_LLVM_BUILD}/lib/cmake/llvm/ -DCMAKE_INSTALL_PREFIX=${CONDA_DIR} -DLLVM_EXTERNAL_LIT="$(which lit)" && \ - #make -j$(nproc --all) && \ - make && \ - make install && \ + mkdir clad/build && \ + cd clad/build && \ + cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. -DClang_DIR=$PATH_TO_CLANG_CONFIG -DLLVM_DIR=$PATH_TO_LLVM_CONFIG -DCMAKE_INSTALL_PREFIX=${CONDA_DIR} -DLLVM_EXTERNAL_LIT="$(which lit)" && \ + make -j$(nproc --all) && \ + make install -j$(nproc --all) && \ ### install clad in all exist kernels ##for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/clad-xcpp.* ]]; then jq '.argv += ["-fplugin=$KERNEL_PYTHON_PREFIX/lib/clad.so"] | .display_name += " (with clad)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done && \ ### @@ -339,8 +371,8 @@ RUN \ k="/opt/conda/share/jupyter/kernels/python3/kernel.json" && \ jq ".argv[0] = \"${VENV}/bin/python\"" $k > $k.$$.tmp && mv $k.$$.tmp $k && \ # xtensor - git clone https://github.com/xtensor-stack/xtensor.git && \ - cd xtensor && \ - mkdir build && cd build && \ + git clone --depth=1 https://github.com/xtensor-stack/xtensor.git && \ + mkdir xtensor/build && \ + cd xtensor/build && \ cmake -DCMAKE_INSTALL_PREFIX=$KERNEL_PYTHON_PREFIX .. && \ - make install + make install -j$(nproc --all) diff --git a/README.md b/README.md index b3cf7574..afa1c30c 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,6 @@ link: ### Xeus-Clang-REPL -Clone the repository locally and execute ./run-docker.sh - ## Description `xeus-clang-repl` integrates clang-repl with the xeus protocol and is a platform for C++ usage in Jupyter Notebooks. The demo developed in this repository shows a Python - CPP integraton in Jupyter Notebooks, where variables can be transfered between Python and CPP. @@ -26,7 +24,7 @@ fresh environment. It is also needed to use a You will first need to install dependencies ```bash -mamba install cmake cxx-compiler nlohmann_json cppzmq xtl jupyterlab clangdev=14 cxxopts pugixml -c conda-forge +mamba install xeus xeus-zmq cmake cxx-compiler nlohmann_json cppzmq xtl jupyterlab clangdev=14 cxxopts pugixml -c conda-forge ``` **Note:** Use a mamba environment with python version >= 3.11 for fetching clang-versions @@ -47,9 +45,9 @@ mamba install`xeus-clang-repl` notebook -c conda-forge ``` --> ```bash -git clone https://github.com/llvm/llvm-project +git clone --depth=1 --branch release/15.0x https://github.com/llvm/llvm-project -git checkout -b release/15.0x +cd llvm-project git apply patches/llvm/clang15-D127284.patch @@ -75,6 +73,18 @@ cmake ../ -DClang_DIR=/usr/lib/llvm-15/build/lib/cmake/clang\ make -j n ``` +## Docker +It is possible to build xeus-clang-repl within a Docker container. On an arm based Apple machine with docker, docker-compose and colima installed, you can try out xeus-clang-repl using the following commands (cpu represents number of cores, and memory is in GiB, so adjust according to your machine) +```bash +git clone https://github.com/compiler-research/xeus-clang-repl.git +cd xeus-clang-repl +colima start --cpu 8 --memory 8 --arch aarch64 --vm-type=vz --vz-rosetta +chmod a+x ./build-and-run-docker.sh +./build-and-run-docker.sh +``` +Once the bash script finishes running, you can experiment with xeus-clang-repl by opening up your browser and visiting http://127.0.0.1:8888/lab + + ## Try it online (for developers) To try out xeus-clang-repl interactively in your web browser, just click on the binder diff --git a/build-and-run-docker.sh b/build-and-run-docker.sh new file mode 100644 index 00000000..75fe8315 --- /dev/null +++ b/build-and-run-docker.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +docker build . -t xeus-clang-repl + +export OS=$(uname -s) + +if [ "$OS" == "Darwin" ] +then + docker run --publish 8888:8888 --name xeus-clang-repl-c -i -t xeus-clang-repl "start-notebook.sh" +else + docker container run --rm -i hadolint/hadolint hadolint - < Dockerfile + +jupyter-repo2docker \ + --no-run \ + --user-name=jovyan \ + --image-name xeus-clang-repl \ + . + +docker run --gpus all --publish 8888:8888 --name xeus-clang-repl-c -i -t xeus-clang-repl "start-notebook.sh" +#docker run --rm --runtime=nvidia --gpus all --publish 8888:8888 --name xeus-clang-repl-c -i -t xeus-clang-repl "start-notebook.sh" + +# --editable \ +# --ref InterOpIntegration \ +# https://github.com/alexander-penev/xeus-clang-repl.git \ + +fi diff --git a/include/xeus-clang-repl/xmanager.hpp b/include/xeus-clang-repl/xmanager.hpp index 6ba412d1..8c686d36 100644 --- a/include/xeus-clang-repl/xmanager.hpp +++ b/include/xeus-clang-repl/xmanager.hpp @@ -20,6 +20,7 @@ #include "nlohmann/json.hpp" +#include #include #include #include diff --git a/run-docker.sh b/run-docker.sh deleted file mode 100755 index 7afe4708..00000000 --- a/run-docker.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -docker container run --rm -i hadolint/hadolint hadolint - < Dockerfile - -jupyter-repo2docker \ - --no-run \ - --user-name=jovyan \ - --image-name xeus-clang-repl \ - . - -#docker run --gpus all --publish 8888:8888 --name xeus-clang-repl-c -i -t xeus-clang-repl "start-notebook.sh" -docker run --rm --runtime=nvidia --gpus all --publish 8888:8888 --name xeus-clang-repl-c -i -t xeus-clang-repl "start-notebook.sh" - -# --editable \ -# --ref InterOpIntegration \ -# https://github.com/alexander-penev/xeus-clang-repl.git \ diff --git a/src/main.cpp b/src/main.cpp index d3a72b87..3ee134bc 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -14,7 +14,7 @@ #include "xeus/xkernel.hpp" #include "xeus/xkernel_configuration.hpp" -#include "xeus/xserver_zmq.hpp" +#include "xeus-zmq/xserver_zmq.hpp" #include "xeus-clang-repl/xeus_clang-repl_config.hpp" #include "xeus-clang-repl/xinterpreter.hpp"