supports automatic search for Python.h files (#320)

* supports automatic search for Pyhton.h files

* supports automatically finding the pytorch path

.github/workflows/main.yml

@@ -97,7 +97,6 @@ jobs:
           set -ex
           export USE_COVERAGE=${USE_COVERAGE}
           cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB}  && cd ${GITHUB_JOB}/dipu
-          export PYTORCH_DIR=${CAMB_TORCH_BASE_DIR}/${PYTORCH_COMMIT}
           source scripts/ci/camb/ci_camb_env.sh
           rsync -a /mnt/lustre/share_data/PAT/datasets/huggingface mmlab_pack/
           srun --job-name=${GITHUB_JOB} --partition=${CAMB_PARTATION} --time=40 \
@@ -190,7 +189,6 @@ jobs:
           ssh ${CAMB_CLUSTER} """
           set -ex
           cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB}  && cd ${GITHUB_JOB}/dipu
-          export PYTORCH_DIR=${CAMB_TORCH_BASE_DIR}/${PYTORCH_COMMIT}
           source scripts/ci/camb/ci_camb_env.sh
           srun --job-name=${GITHUB_JOB} --partition=${CAMB_PARTATION} --time=40 \
           --gres=mlu:${MLU_REQUESTS} bash scripts/ci/camb/ci_camb_script.sh build_dipu \
@@ -365,7 +363,7 @@ jobs:
           source scripts/ci/ascend/ci_ascend_env.sh
           bash scripts/ci/ascend/ci_ascend_script.sh build_dipu \
           || ( cd ${ASCEND_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
-  
+
   Test-Ascend:
     name: Test-dipu-ascend
     needs: [Build-Ascend]
@@ -428,7 +426,7 @@ jobs:
           source scripts/ci/ascend/ci_ascend_env.sh
           bash scripts/ci/ascend/ci_ascend_script.sh build_dipu \
           || ( cd ${ASCEND_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 )
- 
+
   Test-Ascend-Latest-Target:
     name: Test-dipu-ascend-latest-target
     needs: [Build-Ascend-Latest-Target]

dipu/CMakeLists.txt

@@ -27,6 +27,14 @@ endif()
 message(STATUS "DIPU_GIT_HASH: " ${DIPU_GIT_HASH})
 add_compile_options(-DDIPU_GIT_HASH="${DIPU_GIT_HASH}")
 
+execute_process(COMMAND  sh -c "dirname $(find  $(dirname $(which python))/../ -name Python.h)" OUTPUT_VARIABLE PYTHON_INCLUDE_DIR)
+message(STATUS "PYTHON_INCLUDE_DIR: " ${PYTHON_INCLUDE_DIR})
+
+execute_process(COMMAND  sh -c "dirname $(python -c 'import torch;print(torch.__path__[0])')" OUTPUT_VARIABLE PYTORCH_DIR)
+string (REPLACE "\n" "" PYTORCH_DIR "${PYTORCH_DIR}")
+string (REPLACE "\r" "" PYTORCH_DIR "${PYTORCH_DIR}")
+message(STATUS "PYTORCH_DIR: " ${PYTORCH_DIR})
+
 # config
 include(cmake/BaseFuncions.cmake)
 _set_cpp_flags()

dipu/QuickStart.md

@@ -59,11 +59,7 @@ git clone [email protected]:DeepLink-org/dipu.git
 cd dipu
 git submodule update --init --recursive
 
-# 修改 template_build _sh 中 PYTORCH_DIR、PYTHON_INCLUDE_DIR
 # 示例
-# PYTORCH_DIR="/home/$USER/code/pytorch"
-# PYTHON_INCLUDE_DIR="/home/$USER/env/py3.8/include/python3.8"
-
 # DIPU_DEVICE设置成厂商在dipu的设备名，即 https://github.com/DeepLink-org/dipu/blob/main/CMakeLists.txt 中的DEVICE_CAMB、DEVICE_ASCEND对应的字符串
 # DIOPI_IMPL设置成厂商在DIOPI/impl的实现代号，即 https://github.com/DeepLink-org/DIOPI/blob/main/impl/CMakeLists.txt 中的IMPL_CAMB、IMPL_ASCEND等对应的字符串
 # 示例
@@ -78,7 +74,7 @@ sh template_build_sh builddp $DIPU_DEVICE $DIOPI_IMPL
 ``` bash
 export DIOPI_ROOT=/home/$USER/code/dipu/third_party/DIOPI/impl/lib
 export DIPU_ROOT=/home/$USER/code/dipu/torch_dipu
-export LIBRARY_PATH=$DIPU_ROOT:$DIOPI_ROOT:$LIBRARY_PATH; 
+export LIBRARY_PATH=$DIPU_ROOT:$DIOPI_ROOT:$LIBRARY_PATH;
 export LD_LIBRARY_PATH=$DIPU_ROOT:$DIOPI_ROOT:$LD_LIBRARY_PATH
 export PYTHONPATH=/home/$USER/code/dipu:$PYTHONPATH
 

dipu/scripts/ci/ascend/ci_ascend_script.sh

@@ -20,11 +20,8 @@ function build_dipu_py() {
 
 function config_dipu_ascend_cmake() {
     mkdir -p build && cd ./build && rm -rf ./*
-    echo "PYTORCH_DIR: ${PYTORCH_DIR}"
-    echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
     cmake ../  -DCMAKE_BUILD_TYPE=Debug \
-        -DDEVICE=ascend -DPYTORCH_DIR=${PYTORCH_DIR} \
-        -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR}
+        -DDEVICE=ascend
     cd ../
 }
 

dipu/scripts/ci/camb/ci_camb_env.sh

@@ -16,7 +16,6 @@ export LIBRARY_PATH=$DIPU_ROOT:${DIOPI_ROOT}:${LIBRARY_PATH}; LD_LIBRARY_PATH=$D
 export PYTHONPATH=${PYTORCH_DIR}/install_path/lib/python3.8/site-packages:${PYTHONPATH}
 export PATH=${GCC_ROOT}/bin:${PYTORCH_DIR}/install_path/bin:${CONDA_ROOT}/envs/dipu_poc/bin:${CONDA_ROOT}/bin:${PATH}
 export LD_PRELOAD=${GCC_ROOT}/lib64/libstdc++.so.6
-export PYTHON_INCLUDE_DIR="/mnt/lustre/share/platform/env/miniconda3.8/envs/dipu_poc/include/python3.8"
 
 
 export NEUWARE_ROOT_DIR=${NEUWARE_HOME}

dipu/scripts/ci/camb/ci_camb_script.sh

@@ -13,11 +13,9 @@ function build_dipu_py() {
 
 function config_dipu_camb_cmake() {
     mkdir -p build && cd ./build && rm -rf ./*
-    echo "PYTORCH_DIR: ${PYTORCH_DIR}"
-    echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
     cmake ../  -DCMAKE_BUILD_TYPE=Release \
-        -DDEVICE=camb -DPYTORCH_DIR=${PYTORCH_DIR} \
-        -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} -DENABLE_COVERAGE=${USE_COVERAGE}
+        -DDEVICE=camb  \
+        -DENABLE_COVERAGE=${USE_COVERAGE}
     cd ../
 }
 
@@ -50,8 +48,6 @@ function build_diopi_lib() {
 function build_dipu_lib() {
     echo "building dipu_lib:$(pwd)"
     echo  "DIOPI_ROOT:${DIOPI_ROOT}"
-    echo  "PYTORCH_DIR:${PYTORCH_DIR}"
-    echo  "PYTHON_INCLUDE_DIR:${PYTHON_INCLUDE_DIR}"
     export LIBRARY_PATH=$DIOPI_ROOT:$LIBRARY_PATH;
     config_dipu_camb_cmake 2>&1 | tee ./cmake_camb.log
     cd build && make -j8  2>&1 | tee ./build.log &&  cd ..

dipu/scripts/ci/nv/ci_nv_env.sh

@@ -29,7 +29,6 @@ export LIBRARY_PATH=$DIPU_ROOT:${DIOPI_ROOT}:${LIBRARY_PATH}; LD_LIBRARY_PATH=$D
 export PYTHONPATH=${PYTORCH_DIR}:${PYTHONPATH}
 export PATH=${GCC_ROOT}/bin:${CONDA_ROOT}/envs/dipu_poc/bin:${CONDA_ROOT}/bin:${PLATFORM}/dep/binutils-2.27/bin:${PATH}
 export LD_PRELOAD=${GCC_ROOT}/lib64/libstdc++.so.6
-export PYTHON_INCLUDE_DIR=${PLATFORM}/env/miniconda3.8/envs/pt2.0_diopi/include/python3.8
 export PYTORCH_TEST_DIR=${PLATFORM}/env/miniconda3.8/envs/pt2.0_diopi/pytorch2.0
 export CUBLAS_WORKSPACE_CONFIG=:4096:8
 

dipu/scripts/ci/nv/ci_nv_script.sh

@@ -4,7 +4,6 @@ echo "pwd: $(pwd)"
 
 function build_dipu_py() {
     echo "building dipu_py:$(pwd)"
-    echo "building dipu_py PYTORCH_DIR: ${PYTORCH_DIR}"
     export CMAKE_BUILD_TYPE=Release
     export _GLIBCXX_USE_CXX11_ABI=1
     export MAX_JOBS=12
@@ -16,11 +15,9 @@ function config_dipu_nv_cmake() {
     # export NCCL_ROOT="you nccl path should exist"
 
     mkdir -p build && cd ./build && rm -rf ./*
-    echo "config_dipu_nv_cmake PYTORCH_DIR: ${PYTORCH_DIR}"
-    echo "config_dipu_nv_cmake PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
     cmake ../  -DCMAKE_BUILD_TYPE=Release \
-        -DDEVICE=cuda -DPYTORCH_DIR=${PYTORCH_DIR} \
-        -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} -DENABLE_COVERAGE=${USE_COVERAGE}
+        -DDEVICE=cuda \
+        -DENABLE_COVERAGE=${USE_COVERAGE}
     cd ../
 }
 
@@ -49,8 +46,6 @@ function build_diopi_lib() {
 function build_dipu_lib() {
     echo "building dipu_lib:$(pwd)"
     echo  "DIOPI_ROOT:${DIOPI_ROOT}"
-    echo  "PYTORCH_DIR:${PYTORCH_DIR}"
-    echo  "PYTHON_INCLUDE_DIR:${PYTHON_INCLUDE_DIR}"
     export DIOPI_BUILD_TESTRT=1
     export LIBRARY_PATH=$DIOPI_ROOT:$LIBRARY_PATH;
     config_dipu_nv_cmake 2>&1 | tee ./cmake_nv.log

dipu/scripts/ci/topsrider/ci_topsrider_env.sh

@@ -10,7 +10,6 @@ export DIPU_ROOT=${DIPU_LOCAL_DIR}/torch_dipu
 export LIBRARY_PATH=$DIPU_ROOT:${DIOPI_ROOT}:${LIBRARY_PATH}; LD_LIBRARY_PATH=$DIPU_ROOT:$DIOPI_ROOT:$LD_LIBRARY_PATH
 export PYTHONPATH=${CONDA_ROOT}/envs/dipu/lib/python3.8:${DIPU_LOCAL_DIR}:${PYTHONPATH}
 export PATH=${PYTORCH_DIR}/build/bin:${CONDA_ROOT}/envs/dipu/bin:${CONDA_ROOT}/bin:${PATH}
-export PYTHON_INCLUDE_DIR="${CONDA_ROOT}/envs/dipu/include/python3.8"
 
 # this is for mmcv
 export VENDOR_INCLUDE_DIRS=/usr/include/tops

dipu/scripts/ci/topsrider/ci_topsrider_script.sh

@@ -12,13 +12,11 @@ function build_dipu_py() {
 
 function config_dipu_cmake() {
     mkdir -p build && cd ./build && rm -rf ./*
-    # PYTORCH_DIR="/you_pytorch/torch20/pytorch"
-    # PYTHON_INCLUDE_DIR="/you_conda/envs/torch20/include/python3.8"
     cmake ../  -DCMAKE_BUILD_TYPE=Debug \
-     -DDEVICE=tops -DPYTORCH_DIR=${PYTORCH_DIR} \
-     -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR}
+     -DDEVICE=tops \
       # -DCMAKE_C_FLAGS_DEBUG="-g -O0" \
       # -DCMAKE_CXX_FLAGS_DEBUG="-g -O0"
+
     cd ../
 }
 

dipu/setup.py

@@ -90,7 +90,6 @@ def run(self):
             # '-DCMAKE_INSTALL_PREFIX=' + os.path.abspath(output_lib_path),
             # '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + os.path.abspath(output_lib_path),
             # '-DTORCHDIPU_INSTALL_LIBDIR=' + os.path.abspath(output_lib_path),
-            # '-DPYTHON_INCLUDE_DIR=' + get_paths()['include'],
             # '-DPYTORCH_INSTALL_DIR=' + get_pytorch_dir()
         ]
         build_args = ['-j', 12]

dipu/template_build_sh

@@ -10,21 +10,17 @@ function build_dipu_py() {
     # PYTORCH_INSTALL_DIR is /you_pytorch/torch20/pytorch/torch
     # python  setup.py build_clib 2>&1 | tee ./build1.log
     python setup.py build_ext 2>&1 | tee ./build1.log
-    cp build/python_ext/torch_dipu/_C.cpython-38-$(uname -m)-linux-gnu.so torch_dipu
+    cp build/python_ext/torch_dipu/_C.cpython-*.so torch_dipu
 }
 
 function config_dipu_cmake() {
     cd ./build && rm -rf ./*
-    PYTORCH_DIR="/you_pytorch/torch20/pytorch"
-    PYTHON_INCLUDE_DIR="/you_conda/envs/torch20/include/python3.8"
-
     # export NEUWARE_ROOT="/usr/local/neuware-2.4.1/"
 
     # export NCCL_ROOT="you nccl path when use cuda device"
 
     cmake ../  -DCMAKE_BUILD_TYPE=Debug \
-     -DDEVICE=${DIPU_DEVICE} -DPYTORCH_DIR=${PYTORCH_DIR} \
-     -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR}
+     -DDEVICE=${DIPU_DEVICE}
       # -DCMAKE_C_FLAGS_DEBUG="-g -O0" \
       # -DCMAKE_CXX_FLAGS_DEBUG="-g -O0"
     cd ../