From 705092802947c79e72d5f082ac9da1b87864bd1d Mon Sep 17 00:00:00 2001 From: zhaoguochun1995 <109069909+zhaoguochun1995@users.noreply.github.com> Date: Sat, 7 Oct 2023 17:55:01 +0800 Subject: [PATCH] supports automatic search for Python.h files (#320) * supports automatic search for Pyhton.h files * supports automatically finding the pytorch path --- .github/workflows/main.yml | 6 ++---- dipu/CMakeLists.txt | 8 ++++++++ dipu/QuickStart.md | 6 +----- dipu/scripts/ci/ascend/ci_ascend_script.sh | 5 +---- dipu/scripts/ci/camb/ci_camb_env.sh | 1 - dipu/scripts/ci/camb/ci_camb_script.sh | 8 ++------ dipu/scripts/ci/nv/ci_nv_env.sh | 1 - dipu/scripts/ci/nv/ci_nv_script.sh | 9 ++------- dipu/scripts/ci/topsrider/ci_topsrider_env.sh | 1 - dipu/scripts/ci/topsrider/ci_topsrider_script.sh | 6 ++---- dipu/setup.py | 1 - dipu/template_build_sh | 8 ++------ 12 files changed, 20 insertions(+), 40 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 07ea58c13..a61917a98 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -97,7 +97,6 @@ jobs: set -ex export USE_COVERAGE=${USE_COVERAGE} cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu - export PYTORCH_DIR=${CAMB_TORCH_BASE_DIR}/${PYTORCH_COMMIT} source scripts/ci/camb/ci_camb_env.sh rsync -a /mnt/lustre/share_data/PAT/datasets/huggingface mmlab_pack/ srun --job-name=${GITHUB_JOB} --partition=${CAMB_PARTATION} --time=40 \ @@ -190,7 +189,6 @@ jobs: ssh ${CAMB_CLUSTER} """ set -ex cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu - export PYTORCH_DIR=${CAMB_TORCH_BASE_DIR}/${PYTORCH_COMMIT} source scripts/ci/camb/ci_camb_env.sh srun --job-name=${GITHUB_JOB} --partition=${CAMB_PARTATION} --time=40 \ --gres=mlu:${MLU_REQUESTS} bash scripts/ci/camb/ci_camb_script.sh build_dipu \ @@ -365,7 +363,7 @@ jobs: source scripts/ci/ascend/ci_ascend_env.sh bash scripts/ci/ascend/ci_ascend_script.sh build_dipu \ || ( cd ${ASCEND_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 ) - + Test-Ascend: name: Test-dipu-ascend needs: [Build-Ascend] @@ -428,7 +426,7 @@ jobs: source scripts/ci/ascend/ci_ascend_env.sh bash scripts/ci/ascend/ci_ascend_script.sh build_dipu \ || ( cd ${ASCEND_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 ) - + Test-Ascend-Latest-Target: name: Test-dipu-ascend-latest-target needs: [Build-Ascend-Latest-Target] diff --git a/dipu/CMakeLists.txt b/dipu/CMakeLists.txt index 907398b4c..8178ca74c 100644 --- a/dipu/CMakeLists.txt +++ b/dipu/CMakeLists.txt @@ -27,6 +27,14 @@ endif() message(STATUS "DIPU_GIT_HASH: " ${DIPU_GIT_HASH}) add_compile_options(-DDIPU_GIT_HASH="${DIPU_GIT_HASH}") +execute_process(COMMAND sh -c "dirname $(find $(dirname $(which python))/../ -name Python.h)" OUTPUT_VARIABLE PYTHON_INCLUDE_DIR) +message(STATUS "PYTHON_INCLUDE_DIR: " ${PYTHON_INCLUDE_DIR}) + +execute_process(COMMAND sh -c "dirname $(python -c 'import torch;print(torch.__path__[0])')" OUTPUT_VARIABLE PYTORCH_DIR) +string (REPLACE "\n" "" PYTORCH_DIR "${PYTORCH_DIR}") +string (REPLACE "\r" "" PYTORCH_DIR "${PYTORCH_DIR}") +message(STATUS "PYTORCH_DIR: " ${PYTORCH_DIR}) + # config include(cmake/BaseFuncions.cmake) _set_cpp_flags() diff --git a/dipu/QuickStart.md b/dipu/QuickStart.md index d27a8b9f6..8a5a18ced 100644 --- a/dipu/QuickStart.md +++ b/dipu/QuickStart.md @@ -59,11 +59,7 @@ git clone git@github.com:DeepLink-org/dipu.git cd dipu git submodule update --init --recursive -# 修改 template_build _sh 中 PYTORCH_DIR、PYTHON_INCLUDE_DIR # 示例 -# PYTORCH_DIR="/home/$USER/code/pytorch" -# PYTHON_INCLUDE_DIR="/home/$USER/env/py3.8/include/python3.8" - # DIPU_DEVICE设置成厂商在dipu的设备名，即 https://github.com/DeepLink-org/dipu/blob/main/CMakeLists.txt 中的DEVICE_CAMB、DEVICE_ASCEND对应的字符串 # DIOPI_IMPL设置成厂商在DIOPI/impl的实现代号，即 https://github.com/DeepLink-org/DIOPI/blob/main/impl/CMakeLists.txt 中的IMPL_CAMB、IMPL_ASCEND等对应的字符串 # 示例 @@ -78,7 +74,7 @@ sh template_build_sh builddp $DIPU_DEVICE $DIOPI_IMPL ``` bash export DIOPI_ROOT=/home/$USER/code/dipu/third_party/DIOPI/impl/lib export DIPU_ROOT=/home/$USER/code/dipu/torch_dipu -export LIBRARY_PATH=$DIPU_ROOT:$DIOPI_ROOT:$LIBRARY_PATH; +export LIBRARY_PATH=$DIPU_ROOT:$DIOPI_ROOT:$LIBRARY_PATH; export LD_LIBRARY_PATH=$DIPU_ROOT:$DIOPI_ROOT:$LD_LIBRARY_PATH export PYTHONPATH=/home/$USER/code/dipu:$PYTHONPATH diff --git a/dipu/scripts/ci/ascend/ci_ascend_script.sh b/dipu/scripts/ci/ascend/ci_ascend_script.sh index 936576749..b74954d7a 100644 --- a/dipu/scripts/ci/ascend/ci_ascend_script.sh +++ b/dipu/scripts/ci/ascend/ci_ascend_script.sh @@ -20,11 +20,8 @@ function build_dipu_py() { function config_dipu_ascend_cmake() { mkdir -p build && cd ./build && rm -rf ./* - echo "PYTORCH_DIR: ${PYTORCH_DIR}" - echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}" cmake ../ -DCMAKE_BUILD_TYPE=Debug \ - -DDEVICE=ascend -DPYTORCH_DIR=${PYTORCH_DIR} \ - -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} + -DDEVICE=ascend cd ../ } diff --git a/dipu/scripts/ci/camb/ci_camb_env.sh b/dipu/scripts/ci/camb/ci_camb_env.sh index a1f35b63f..b76151d8b 100644 --- a/dipu/scripts/ci/camb/ci_camb_env.sh +++ b/dipu/scripts/ci/camb/ci_camb_env.sh @@ -16,7 +16,6 @@ export LIBRARY_PATH=$DIPU_ROOT:${DIOPI_ROOT}:${LIBRARY_PATH}; LD_LIBRARY_PATH=$D export PYTHONPATH=${PYTORCH_DIR}/install_path/lib/python3.8/site-packages:${PYTHONPATH} export PATH=${GCC_ROOT}/bin:${PYTORCH_DIR}/install_path/bin:${CONDA_ROOT}/envs/dipu_poc/bin:${CONDA_ROOT}/bin:${PATH} export LD_PRELOAD=${GCC_ROOT}/lib64/libstdc++.so.6 -export PYTHON_INCLUDE_DIR="/mnt/lustre/share/platform/env/miniconda3.8/envs/dipu_poc/include/python3.8" export NEUWARE_ROOT_DIR=${NEUWARE_HOME} diff --git a/dipu/scripts/ci/camb/ci_camb_script.sh b/dipu/scripts/ci/camb/ci_camb_script.sh index a0d5ab826..740cf6fd2 100644 --- a/dipu/scripts/ci/camb/ci_camb_script.sh +++ b/dipu/scripts/ci/camb/ci_camb_script.sh @@ -13,11 +13,9 @@ function build_dipu_py() { function config_dipu_camb_cmake() { mkdir -p build && cd ./build && rm -rf ./* - echo "PYTORCH_DIR: ${PYTORCH_DIR}" - echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}" cmake ../ -DCMAKE_BUILD_TYPE=Release \ - -DDEVICE=camb -DPYTORCH_DIR=${PYTORCH_DIR} \ - -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} -DENABLE_COVERAGE=${USE_COVERAGE} + -DDEVICE=camb \ + -DENABLE_COVERAGE=${USE_COVERAGE} cd ../ } @@ -50,8 +48,6 @@ function build_diopi_lib() { function build_dipu_lib() { echo "building dipu_lib:$(pwd)" echo "DIOPI_ROOT:${DIOPI_ROOT}" - echo "PYTORCH_DIR:${PYTORCH_DIR}" - echo "PYTHON_INCLUDE_DIR:${PYTHON_INCLUDE_DIR}" export LIBRARY_PATH=$DIOPI_ROOT:$LIBRARY_PATH; config_dipu_camb_cmake 2>&1 | tee ./cmake_camb.log cd build && make -j8 2>&1 | tee ./build.log && cd .. diff --git a/dipu/scripts/ci/nv/ci_nv_env.sh b/dipu/scripts/ci/nv/ci_nv_env.sh index 64c584b9f..2c0000f18 100644 --- a/dipu/scripts/ci/nv/ci_nv_env.sh +++ b/dipu/scripts/ci/nv/ci_nv_env.sh @@ -29,7 +29,6 @@ export LIBRARY_PATH=$DIPU_ROOT:${DIOPI_ROOT}:${LIBRARY_PATH}; LD_LIBRARY_PATH=$D export PYTHONPATH=${PYTORCH_DIR}:${PYTHONPATH} export PATH=${GCC_ROOT}/bin:${CONDA_ROOT}/envs/dipu_poc/bin:${CONDA_ROOT}/bin:${PLATFORM}/dep/binutils-2.27/bin:${PATH} export LD_PRELOAD=${GCC_ROOT}/lib64/libstdc++.so.6 -export PYTHON_INCLUDE_DIR=${PLATFORM}/env/miniconda3.8/envs/pt2.0_diopi/include/python3.8 export PYTORCH_TEST_DIR=${PLATFORM}/env/miniconda3.8/envs/pt2.0_diopi/pytorch2.0 export CUBLAS_WORKSPACE_CONFIG=:4096:8 diff --git a/dipu/scripts/ci/nv/ci_nv_script.sh b/dipu/scripts/ci/nv/ci_nv_script.sh index 5b2c6e107..606745baa 100644 --- a/dipu/scripts/ci/nv/ci_nv_script.sh +++ b/dipu/scripts/ci/nv/ci_nv_script.sh @@ -4,7 +4,6 @@ echo "pwd: $(pwd)" function build_dipu_py() { echo "building dipu_py:$(pwd)" - echo "building dipu_py PYTORCH_DIR: ${PYTORCH_DIR}" export CMAKE_BUILD_TYPE=Release export _GLIBCXX_USE_CXX11_ABI=1 export MAX_JOBS=12 @@ -16,11 +15,9 @@ function config_dipu_nv_cmake() { # export NCCL_ROOT="you nccl path should exist" mkdir -p build && cd ./build && rm -rf ./* - echo "config_dipu_nv_cmake PYTORCH_DIR: ${PYTORCH_DIR}" - echo "config_dipu_nv_cmake PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}" cmake ../ -DCMAKE_BUILD_TYPE=Release \ - -DDEVICE=cuda -DPYTORCH_DIR=${PYTORCH_DIR} \ - -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} -DENABLE_COVERAGE=${USE_COVERAGE} + -DDEVICE=cuda \ + -DENABLE_COVERAGE=${USE_COVERAGE} cd ../ } @@ -49,8 +46,6 @@ function build_diopi_lib() { function build_dipu_lib() { echo "building dipu_lib:$(pwd)" echo "DIOPI_ROOT:${DIOPI_ROOT}" - echo "PYTORCH_DIR:${PYTORCH_DIR}" - echo "PYTHON_INCLUDE_DIR:${PYTHON_INCLUDE_DIR}" export DIOPI_BUILD_TESTRT=1 export LIBRARY_PATH=$DIOPI_ROOT:$LIBRARY_PATH; config_dipu_nv_cmake 2>&1 | tee ./cmake_nv.log diff --git a/dipu/scripts/ci/topsrider/ci_topsrider_env.sh b/dipu/scripts/ci/topsrider/ci_topsrider_env.sh index 81c0ca7cf..2537daf6a 100644 --- a/dipu/scripts/ci/topsrider/ci_topsrider_env.sh +++ b/dipu/scripts/ci/topsrider/ci_topsrider_env.sh @@ -10,7 +10,6 @@ export DIPU_ROOT=${DIPU_LOCAL_DIR}/torch_dipu export LIBRARY_PATH=$DIPU_ROOT:${DIOPI_ROOT}:${LIBRARY_PATH}; LD_LIBRARY_PATH=$DIPU_ROOT:$DIOPI_ROOT:$LD_LIBRARY_PATH export PYTHONPATH=${CONDA_ROOT}/envs/dipu/lib/python3.8:${DIPU_LOCAL_DIR}:${PYTHONPATH} export PATH=${PYTORCH_DIR}/build/bin:${CONDA_ROOT}/envs/dipu/bin:${CONDA_ROOT}/bin:${PATH} -export PYTHON_INCLUDE_DIR="${CONDA_ROOT}/envs/dipu/include/python3.8" # this is for mmcv export VENDOR_INCLUDE_DIRS=/usr/include/tops diff --git a/dipu/scripts/ci/topsrider/ci_topsrider_script.sh b/dipu/scripts/ci/topsrider/ci_topsrider_script.sh index 307462253..8d2f0dc5b 100644 --- a/dipu/scripts/ci/topsrider/ci_topsrider_script.sh +++ b/dipu/scripts/ci/topsrider/ci_topsrider_script.sh @@ -12,13 +12,11 @@ function build_dipu_py() { function config_dipu_cmake() { mkdir -p build && cd ./build && rm -rf ./* - # PYTORCH_DIR="/you_pytorch/torch20/pytorch" - # PYTHON_INCLUDE_DIR="/you_conda/envs/torch20/include/python3.8" cmake ../ -DCMAKE_BUILD_TYPE=Debug \ - -DDEVICE=tops -DPYTORCH_DIR=${PYTORCH_DIR} \ - -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} + -DDEVICE=tops \ # -DCMAKE_C_FLAGS_DEBUG="-g -O0" \ # -DCMAKE_CXX_FLAGS_DEBUG="-g -O0" + cd ../ } diff --git a/dipu/setup.py b/dipu/setup.py index c3e29c38b..fe223780a 100644 --- a/dipu/setup.py +++ b/dipu/setup.py @@ -90,7 +90,6 @@ def run(self): # '-DCMAKE_INSTALL_PREFIX=' + os.path.abspath(output_lib_path), # '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + os.path.abspath(output_lib_path), # '-DTORCHDIPU_INSTALL_LIBDIR=' + os.path.abspath(output_lib_path), - # '-DPYTHON_INCLUDE_DIR=' + get_paths()['include'], # '-DPYTORCH_INSTALL_DIR=' + get_pytorch_dir() ] build_args = ['-j', 12] diff --git a/dipu/template_build_sh b/dipu/template_build_sh index 444ed3dcf..84b141fc5 100755 --- a/dipu/template_build_sh +++ b/dipu/template_build_sh @@ -10,21 +10,17 @@ function build_dipu_py() { # PYTORCH_INSTALL_DIR is /you_pytorch/torch20/pytorch/torch # python setup.py build_clib 2>&1 | tee ./build1.log python setup.py build_ext 2>&1 | tee ./build1.log - cp build/python_ext/torch_dipu/_C.cpython-38-$(uname -m)-linux-gnu.so torch_dipu + cp build/python_ext/torch_dipu/_C.cpython-*.so torch_dipu } function config_dipu_cmake() { cd ./build && rm -rf ./* - PYTORCH_DIR="/you_pytorch/torch20/pytorch" - PYTHON_INCLUDE_DIR="/you_conda/envs/torch20/include/python3.8" - # export NEUWARE_ROOT="/usr/local/neuware-2.4.1/" # export NCCL_ROOT="you nccl path when use cuda device" cmake ../ -DCMAKE_BUILD_TYPE=Debug \ - -DDEVICE=${DIPU_DEVICE} -DPYTORCH_DIR=${PYTORCH_DIR} \ - -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} + -DDEVICE=${DIPU_DEVICE} # -DCMAKE_C_FLAGS_DEBUG="-g -O0" \ # -DCMAKE_CXX_FLAGS_DEBUG="-g -O0" cd ../