Gqw/s2 pccl compile #6170
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: dipu ci | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| - dev_v0.25 | |
| tags: | |
| - 'v*' | |
| pull_request: | |
| paths-ignore: | |
| - "dicp/**" | |
| env: | |
| CAMB_CI_PATH: '/mnt/lustre/share/parrotsci/github/cibuild/${{ github.repository }}' | |
| CAMB_PARTATION: ${{ vars.CAMB_SLURM_PAR != '' && vars.CAMB_SLURM_PAR || 'camb_mlu370_m8' }} | |
| CAMB_CLUSTER: CAMB | |
| CUDA_CI_PATH: '/mnt/cache/share/parrotsci/github/cibuild/${{ github.repository }}' | |
| CUDA_PARTATION: ${{ vars.SH1988_SLURM_PAR != '' && vars.SH1988_SLURM_PAR || 'pat_rd' }} | |
| CUDA_CLUSTER: SCO | |
| CUDA_ENV_PATH: '/mnt/cache/share/deeplinkci/github' | |
| DEEPLINK_PATH: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}' | |
| ASCEND_CLUSTER: ASCEND | |
| CLUSTER_ASCEND_910B: ASCEND-910B | |
| ASCEND_TORCH_DIR: '/mnt/cache/share/platform/cienv/pytorch' | |
| CI_BUILD_FLAG: "ci_build_flag" | |
| # need deprecated but ascend still use it, PYTORCH_COMMIT not support multi-version torch | |
| PYTORCH_COMMIT: ${{ vars.PYTORCH_COMMIT != '' && vars.PYTORCH_COMMIT || 'c263bd43e8e8502d4726643bc6fd046f0130ac0e' }} # pytorch tag 2.0 | |
| ALL_COVERAGE: ${{ (contains( github.ref, 'main') || startsWith(github.ref, 'refs/heads/v') || startsWith(github.ref, 'refs/heads/dev')) && 'ON' || 'OFF' }} | |
| REQUIRE_COVERAGE: ${{ vars.REQUIRE_COVERAGE != '' && vars.REQUIRE_COVERAGE || '0' }} | |
| REPO: ${{ github.event.repository.name }} | |
| concurrency: | |
| group: dipu-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| Rsync: | |
| name: Rsync code | |
| runs-on: github-poc-ci | |
| outputs: | |
| getrunner: ${{ steps.set-run-vars.outputs.GETRUNNER }} | |
| steps: | |
| - name: clone repo | |
| run: | | |
| set -ex | |
| cd ${GITHUB_WORKSPACE} && rm -rf DIPU ${REPO}_DIOPI ${REPO} ${REPO}.dev | |
| if [ -n "${{ github.event.pull_request.head.repo.full_name }}" ] && [[ ! "${{ github.event.pull_request.head.repo.full_name }}" == "${{ github.repository }}" ]]; then | |
| git clone ${{ github.event.pull_request.head.repo.ssh_url }} ${REPO} | |
| cd ${GITHUB_WORKSPACE}/${REPO} && git checkout ${{ github.event.pull_request.head.sha }} | |
| git remote add mainrepo [email protected]:${GITHUB_REPOSITORY}.git | |
| git fetch mainrepo && git merge --no-edit mainrepo/${{ github.base_ref }} | |
| else | |
| git clone https://github.com/DeepLink-org/${REPO}.git && cd ${REPO} | |
| if [ $GITHUB_EVENT_NAME == "pull_request" ]; then | |
| echo "${{ github.base_ref }} " | |
| git checkout ${{ github.event.pull_request.head.sha }} && git merge --no-edit origin/${{ github.base_ref }} | |
| else | |
| echo $GITHUB_EVENT_NAME | |
| git checkout ${{ github.sha }} | |
| fi | |
| fi | |
| cd ${GITHUB_WORKSPACE}/${REPO}/dipu && rm -rf third_party/kineto | |
| git clone --reference /home/autolink/rsync/sourcecode/DeepLink-org/kineto https://github.com/DeepLink-org/kineto.git third_party/kineto | |
| git submodule update --init && cd third_party/kineto && git submodule update --init | |
| cd ${GITHUB_WORKSPACE} && cp -R ${REPO} ${REPO}_DIOPI | |
| cd ${REPO}/dipu && bash /home/autolink/rsync/sourcecode/update_code.sh | |
| rsync -a /home/autolink/rsync/sourcecode/mmlab_pack . && cd mmlab_pack | |
| bash ../scripts/ci/ci_one_iter.sh clone | |
| # dipu_diopi depend on latest target diopi branch, not diopi in submodule. here assume diopi and dipu use same 'target branch' " github.base_ref " | |
| cd ${GITHUB_WORKSPACE}/${REPO}_DIOPI/dipu/third_party && rm -rf DIOPI && git clone https://github.com/DeepLink-org/DIOPI.git | |
| if [ $GITHUB_EVENT_NAME == "pull_request" ]; then | |
| cd ./DIOPI && git checkout ${{ github.base_ref }} | |
| fi | |
| - name: set output and Rsync to Server | |
| id: set-run-vars | |
| run: | | |
| GETRUNNER=$(cat ~/rsync/cuda_runner_dipu) | |
| if [[ ${GETRUNNER} == *diopi* ]]; then | |
| CUDA_CLUSTER="SH1424" | |
| fi | |
| echo "GETRUNNER=$GETRUNNER" >> "$GITHUB_OUTPUT" | |
| ssh ${CAMB_CLUSTER} "mkdir -p ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main" \ | |
| && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source/ \ | |
| && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CAMB_CLUSTER}:${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to camb" | |
| ssh ${CUDA_CLUSTER} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main" \ | |
| && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \ | |
| && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CUDA_CLUSTER}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to cuda" | |
| ssh ${CLUSTER_ASCEND_910B} "mkdir -p ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main" \ | |
| && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/ \ | |
| && rsync -a --delete ${GITHUB_WORKSPACE}/${REPO}_DIOPI/ ${CLUSTER_ASCEND_910B}:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/ || echo "failure to connect to ascend" | |
| Deploy_on_tag: | |
| runs-on: tps-sco-ci | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| steps: | |
| - name: Run script on tag | |
| run: | | |
| TAG=${GITHUB_REF#refs/tags/} # Extract the tag name | |
| echo $TAG | |
| bash /mnt/cache/share/platform/dep/Deploy_DIPU_trigger.sh "$TAG" | |
| Build-Camb: | |
| name: Build-dipu-camb | |
| needs: [Runs-On-Nv-Step1] | |
| runs-on: github-poc-ci | |
| env: | |
| MLU_REQUESTS: 1 | |
| steps: | |
| - name: Build dipu | |
| run: | | |
| ssh ${CAMB_CLUSTER} """ | |
| set -ex | |
| export USE_COVERAGE=ON | |
| cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu | |
| source scripts/ci/camb/ci_camb_env.sh | |
| rsync -a /mnt/lustre/share_data/PAT/datasets/huggingface mmlab_pack/ | |
| srun --job-name=${GITHUB_JOB} --partition=${CAMB_PARTATION} --time=40 \ | |
| --gres=mlu:${MLU_REQUESTS} bash scripts/ci/camb/ci_camb_script.sh build_dipu \ | |
| || ( cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 ) | |
| """ | |
| Test-Camb: | |
| name: Test-dipu-camb | |
| needs: [Build-Camb] | |
| runs-on: github-poc-ci | |
| env: | |
| MLU_REQUESTS: 4 | |
| steps: | |
| - name: Run-test | |
| run: | | |
| ssh ${CAMB_CLUSTER} """ | |
| set -ex | |
| export USE_COVERAGE=ON | |
| cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Camb/dipu | |
| source scripts/ci/camb/ci_camb_env.sh | |
| srun --job-name=${GITHUB_JOB} --partition=${CAMB_PARTATION} --time=40 --gres=mlu:${MLU_REQUESTS} sh tests/run_camb_tests.sh | |
| if [ "${ALL_COVERAGE}" = "ON" ]; then | |
| sh /mnt/lustre/share/platform/dep/sonar/coverage_DIPU_camb.sh ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb ${GITHUB_RUN_NUMBER} || echo "get coverage fail" | |
| fi | |
| """ | |
| - name: increment coverage check | |
| if: ${{ contains( github.event_name, 'pull_request' ) && contains( github.base_ref, 'main' ) }} | |
| run: | | |
| ssh ${CAMB_CLUSTER} """ | |
| set -e | |
| cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb | |
| rm -rf scripts | |
| ln -s ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/source-main/dipu/third_party/DIOPI/scripts scripts | |
| source /mnt/cache/share/platform/env/pt2.0_diopi | |
| bash scripts/increment_coverage.sh ${REQUIRE_COVERAGE} | |
| """ | |
| Build-Camb-Pt210: | |
| name: Build-dipu-camb-pt210 | |
| needs: [Runs-On-Nv-Step1] | |
| runs-on: github-poc-ci | |
| env: | |
| MLU_REQUESTS: 1 | |
| steps: | |
| - name: Build dipu | |
| run: | | |
| ssh ${CAMB_CLUSTER} """ | |
| set -ex | |
| cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu | |
| source scripts/ci/camb/ci_camb_env.sh 2.1 | |
| srun --job-name=${GITHUB_JOB} --partition=${CAMB_PARTATION} --time=40 \ | |
| --gres=mlu:${MLU_REQUESTS} bash scripts/ci/camb/ci_camb_script.sh build_dipu \ | |
| || ( cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 ) | |
| """ | |
| # torch2.1.0 in camb ci is not a complete one, no test dir. | |
| # Test-Camb-Pt210: | |
| # name: Test-dipu-camb-pt210 | |
| # needs: [Build-Camb-Pt210] | |
| # runs-on: github-poc-ci | |
| # env: | |
| # MLU_REQUESTS: 1 | |
| # steps: | |
| # - name: Run-test | |
| # run: | | |
| # ssh ${CAMB_CLUSTER} """ | |
| # set -ex | |
| # cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Camb-Pt210/dipu | |
| # source scripts/ci/camb/ci_camb_env.sh 2.1 | |
| # srun --job-name=${GITHUB_JOB} --partition=${CAMB_PARTATION} --time=40 --gres=mlu:${MLU_REQUESTS} sh tests/run_camb_tests.sh && \ | |
| # cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Camb-Pt210 | |
| # """ | |
| Build-Camb-Pt211: | |
| name: Build-dipu-camb-pt211 | |
| needs: [Runs-On-Nv-Step1] | |
| runs-on: github-poc-ci | |
| env: | |
| MLU_REQUESTS: 1 | |
| steps: | |
| - name: Build dipu | |
| run: | | |
| ssh ${CAMB_CLUSTER} """ | |
| set -ex | |
| cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu | |
| source scripts/ci/camb/ci_camb_env.sh 2.1.1 | |
| srun --job-name=${GITHUB_JOB} --partition=${CAMB_PARTATION} --time=40 \ | |
| --gres=mlu:${MLU_REQUESTS} bash scripts/ci/camb/ci_camb_script.sh build_dipu \ | |
| || ( cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 ) | |
| """ | |
| Test-Camb-Pt211: | |
| name: Test-dipu-camb-pt211 | |
| needs: [Build-Camb-Pt211] | |
| runs-on: github-poc-ci | |
| env: | |
| MLU_REQUESTS: 4 | |
| steps: | |
| - name: Run-test | |
| run: | | |
| ssh ${CAMB_CLUSTER} """ | |
| set -ex | |
| cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Camb-Pt211/dipu | |
| source scripts/ci/camb/ci_camb_env.sh 2.1.1 | |
| srun --job-name=${GITHUB_JOB} --partition=${CAMB_PARTATION} --time=40 --gres=mlu:${MLU_REQUESTS} sh tests/run_camb_tests.sh && \ | |
| cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Camb-Pt211 | |
| """ | |
| Test-One-Iter-Camb: | |
| name: Test-one-iter-camb | |
| needs: [Build-Camb] | |
| runs-on: github-poc-ci | |
| env: | |
| MLU_REQUESTS: 1 | |
| steps: | |
| - name: build-some-env | |
| run: | | |
| ssh ${CAMB_CLUSTER} """ | |
| set -ex | |
| cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb/dipu | |
| source scripts/ci/camb/ci_camb_env.sh | |
| basic_path=${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb/dipu/mmlab_pack | |
| export PYTHONPATH=\${basic_path}/mmengine:\$PYTHONPATH | |
| export PYTHONPATH=\${basic_path}/mmcv:\$PYTHONPATH | |
| export PYTHONPATH=\$(pwd):\$PYTHONPATH | |
| cd mmlab_pack | |
| srun --job-name=${GITHUB_RUN_NUMBER}_${GITHUB_JOB} --partition=${CAMB_PARTATION} --gres=mlu:${MLU_REQUESTS} sh ../scripts/ci/ci_one_iter.sh build_camb | |
| """ | |
| - name: run-one-iter-for-tradition | |
| run: | | |
| ssh ${CAMB_CLUSTER} """ | |
| set -ex | |
| cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb/dipu | |
| source scripts/ci/camb/ci_camb_env.sh | |
| basic_path=${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb/dipu/mmlab_pack | |
| source scripts/ci/ci_one_iter.sh export_pythonpath_camb \${basic_path} | |
| export PYTHONPATH=\$(pwd):\$PYTHONPATH | |
| cd mmlab_pack | |
| rm -rf one_iter_data | |
| python ../scripts/ci/ci_run_one_iter.py camb ${GITHUB_JOB} "mlu:${MLU_REQUESTS}" \"${CAMB_PARTATION}\" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1) | |
| """ | |
| - name: run-one-iter-for-llm | |
| run: | | |
| ssh ${CAMB_CLUSTER} """ | |
| set -ex | |
| cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb/dipu | |
| source scripts/ci/camb/ci_camb_env.sh | |
| basic_path=${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb/dipu/mmlab_pack | |
| source scripts/ci/ci_one_iter.sh export_pythonpath_camb \${basic_path} | |
| export PYTHONPATH=\$(pwd):\$PYTHONPATH | |
| cd mmlab_pack | |
| rm -rf one_iter_data | |
| python ../scripts/ci/ci_run_one_iter.py camb ${GITHUB_JOB} "mlu:${MLU_REQUESTS}" \"${CAMB_PARTATION}\" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1) | |
| """ | |
| - name: Perform cleanup one iter data | |
| if: always() | |
| run: | | |
| ssh ${CAMB_CLUSTER} """ | |
| set -ex | |
| echo "${GITHUB_RUN_NUMBER}_${GITHUB_JOB}" | |
| scancel -n "${GITHUB_RUN_NUMBER}_${GITHUB_JOB}" | |
| cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/Build-Camb/dipu/mmlab_pack | |
| rm -rf one_iter_data | |
| touch one_iter_data # 用于占位，防止创建新的 one_iter_data 文件夹 | |
| """ | |
| - name: Check for failure | |
| if: ${{ failure() }} | |
| run: exit 1 | |
| Build-Camb-Latest-Target: | |
| name: Build-dipu-camb-latest-target | |
| needs: [Runs-On-Nv-Step1] | |
| runs-on: github-poc-ci | |
| env: | |
| MLU_REQUESTS: 1 | |
| steps: | |
| - name: Build dipu diopi-latest-target | |
| run: | | |
| ssh ${CAMB_CLUSTER} """ | |
| set -ex | |
| cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu | |
| source scripts/ci/camb/ci_camb_env.sh | |
| srun --job-name=${GITHUB_JOB} --partition=${CAMB_PARTATION} --time=40 \ | |
| --gres=mlu:${MLU_REQUESTS} bash scripts/ci/camb/ci_camb_script.sh build_dipu \ | |
| || ( cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 ) | |
| """ | |
| Test-Camb-Latest-Target: | |
| name: Test-dipu-camb-latest-target | |
| needs: [Build-Camb-Latest-Target] | |
| runs-on: github-poc-ci | |
| env: | |
| MLU_REQUESTS: 4 | |
| steps: | |
| - name: Run-test | |
| run: | | |
| ssh ${CAMB_CLUSTER} """ | |
| set -ex | |
| cd ${CAMB_CI_PATH}/${GITHUB_RUN_NUMBER}/ && cd Build-Camb-Latest-Target/dipu | |
| source scripts/ci/camb/ci_camb_env.sh | |
| srun --job-name=${GITHUB_JOB} --partition=${CAMB_PARTATION} --time=40 --gres=mlu:${MLU_REQUESTS} sh tests/run_camb_tests.sh | |
| """ | |
| Runs-On-Nv-Step1: | |
| name: runs on nv step1 | |
| needs: [Rsync] | |
| uses: ./.github/workflows/_runs-on-nv-step1.yml | |
| with: | |
| runner: ${{ needs.Rsync.outputs.getrunner }} | |
| Runs-On-Nv-Step2: | |
| name: runs on nv step2 | |
| needs: [Rsync,Runs-On-Nv-Step1] | |
| uses: ./.github/workflows/_runs-on-nv-step2.yml | |
| with: | |
| runner: ${{ needs.Rsync.outputs.getrunner }} | |
| all_coverage: ${{ (contains( github.ref, 'main') || startsWith(github.ref, 'refs/heads/v') || startsWith(github.ref, 'refs/heads/dev')) && 'ON' || 'OFF' }} | |
| require_coverage: ${{ vars.REQUIRE_COVERAGE != '' && vars.REQUIRE_COVERAGE || '0' }} | |
| Build-PyTorch-For-Ascend-910b: | |
| name: Build-dipu-pytorch-for-ascend-910b | |
| needs: [Runs-On-Nv-Step1] | |
| runs-on: tps-ascend-ci-910b | |
| steps: | |
| - name: Build PyTorch | |
| run: | | |
| set -ex | |
| cd ${ASCEND_TORCH_DIR} | |
| if [ "$(git rev-parse HEAD)" != "${PYTORCH_COMMIT}" ]; then | |
| rm -rf build | |
| fi | |
| if [ ! -d "build" ]; then | |
| source ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source/dipu/scripts/ci/ascend/ci_ascend_env.sh | |
| export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} | |
| BUILD_BINARY=0 USE_PRECOMPILED_HEADERS=1 BUILD_TEST=0 USE_CUDA=0 python setup.py build_ext -i || (rm -rf build && exit 1) | |
| fi | |
| Build-Ascend-910b: | |
| name: Build-dipu-ascend-910b | |
| needs: [Build-PyTorch-For-Ascend-910b] | |
| runs-on: tps-ascend-ci-910b | |
| steps: | |
| - name: Build dipu | |
| run: | | |
| set -ex | |
| export USE_COVERAGE=ON | |
| cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu | |
| source scripts/ci/ascend/ci_ascend_env.sh | |
| bash scripts/ci/ascend/ci_ascend_script.sh build_dipu \ | |
| || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 ) | |
| Test-Ascend-910b: | |
| name: Test-dipu-ascend-910b | |
| needs: [Build-Ascend-910b] | |
| runs-on: tps-ascend-ci-910b | |
| steps: | |
| - name: Run a test | |
| run: | | |
| set -ex | |
| cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Ascend-910b/dipu | |
| source scripts/ci/ascend/ci_ascend_env.sh | |
| bash tests/run_ascend_tests.sh | |
| if [ "${ALL_COVERAGE}" = "ON" ]; then | |
| bash /mnt/cache/share/platform/dep/sonar/coverage_DIPU.sh ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Ascend-910b/dipu || echo "get coverage fail" | |
| fi | |
| - name: increment coverage check | |
| if: ${{ contains( github.event_name, 'pull_request' ) && contains( github.base_ref, 'main' ) }} | |
| run: | | |
| set -ex | |
| cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Ascend-910b/ | |
| rm -rf scripts | |
| ln -s ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/source-main/dipu/third_party/DIOPI/scripts scripts | |
| source dipu/scripts/ci/ascend/ci_ascend_env.sh | |
| bash scripts/increment_coverage.sh ${REQUIRE_COVERAGE} | |
| Build-Dicp-On-Dipu-Ascend: | |
| name: Build-dicp-on-dipu-ascend | |
| needs: [Build-PyTorch-For-Ascend-910b] | |
| runs-on: tps-ascend-ci-910b | |
| steps: | |
| - name: Build dipu | |
| run: | | |
| set -ex | |
| export PYTHONUSERBASE=${HOME}/.local.dicp | |
| cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && cp -R source ${GITHUB_JOB} && cd ${GITHUB_JOB}/ | |
| source dicp/scripts/ci/ascend/ci_runs_on_dipu_env.sh | |
| cd dipu && bash scripts/ci/ascend/ci_ascend_script.sh build_dipu_without_diopi \ | |
| || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 ) | |
| Test-Dicp-On-Dipu-Ascend: | |
| name: Test-dicp-on-dipu-ascend | |
| needs: [Build-Dicp-On-Dipu-Ascend] | |
| runs-on: tps-ascend-ci-910b | |
| steps: | |
| - name: Run dicp test | |
| run: | | |
| set -ex | |
| export PYTHONUSERBASE=${HOME}/.local.dicp | |
| rm -rf /tmp/dicp_ascend/* | |
| cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Dicp-On-Dipu-Ascend | |
| source dicp/scripts/ci/ascend/ci_runs_on_dipu_env.sh | |
| export PYTHONPATH=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Dicp-On-Dipu-Ascend/dipu:${PYTHONPATH} | |
| pip uninstall dicp -y | |
| cd dicp && python setup.py clean && python setup.py install --user | |
| export TEST_DIR=$(pwd)/test | |
| source scripts/ci/ascend/test_env.sh /mnt/cache/share/deeplinkci/dicp_env/llama_models | |
| bash ${TEST_DIR}/ascend_scripts/ops/run_test_ops.sh false && \ | |
| pytest test/model/test_llama.py --backend ascendgraph --dynamic false \ | |
| || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Dicp-On-Dipu-Ascend && exit 1 ) | |
| Test-One-Iter-Ascend-910b: | |
| name: Test-one-iter-ascend-910b | |
| needs: [Build-Ascend-910b] | |
| runs-on: tps-ascend-ci-910b | |
| steps: | |
| - name: Build some env | |
| run: | | |
| set -ex | |
| cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Ascend-910b/dipu | |
| source scripts/ci/ascend/ci_ascend_env.sh | |
| basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Ascend-910b/dipu/mmlab_pack | |
| source scripts/ci/ci_one_iter.sh export_pythonpath_ascend ${basic_path} | |
| cd mmlab_pack | |
| bash ../scripts/ci/ci_one_iter.sh build_ascend910b | |
| - name: run-one-iter-for-tradition | |
| run: | | |
| set -ex | |
| cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Ascend-910b/dipu | |
| source scripts/ci/ascend/ci_ascend_env.sh | |
| basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Ascend-910b/dipu/mmlab_pack | |
| source scripts/ci/ci_one_iter.sh export_pythonpath_ascend ${basic_path} | |
| cd mmlab_pack | |
| rm -rf one_iter_data | |
| python ../scripts/ci/ci_run_one_iter.py ascend "" "" "" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1) | |
| - name: run-one-iter-for-llm | |
| run: | | |
| set -ex | |
| cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Ascend-910b/dipu | |
| source scripts/ci/ascend/ci_ascend_env.sh | |
| basic_path=${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Ascend-910b/dipu/mmlab_pack | |
| source scripts/ci/ci_one_iter.sh export_pythonpath_ascend ${basic_path} | |
| cd mmlab_pack | |
| rm -rf one_iter_data | |
| python ../scripts/ci/ci_run_one_iter.py ascend "" "" "" "llm" && rm -rf one_iter_data || (rm -rf one_iter_data && exit 1) | |
| - name: Perform cleanup one iter data | |
| if: always() | |
| run: | | |
| set -ex | |
| cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Ascend-910b/dipu/mmlab_pack | |
| rm -rf one_iter_data | |
| touch one_iter_data # 用于占位，防止创建新的 one_iter_data 文件夹 | |
| - name: Check for failure | |
| if: ${{ failure() }} | |
| run: exit 1 | |
| Build-Ascend-Latest-Target-910b: | |
| name: Build-dipu-ascend-latest-target-910b | |
| needs: [Build-PyTorch-For-Ascend-910b] | |
| runs-on: tps-ascend-ci-910b | |
| steps: | |
| - name: Build dipu diopi-latest-target | |
| run: | | |
| set -ex | |
| cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && cp -R source-main ${GITHUB_JOB} && cd ${GITHUB_JOB}/dipu | |
| source scripts/ci/ascend/ci_ascend_env.sh | |
| bash scripts/ci/ascend/ci_ascend_script.sh build_dipu \ | |
| || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${GITHUB_JOB} && exit 1 ) | |
| Test-Ascend-Latest-Target-910b: | |
| name: Test-dipu-ascend-latest-target-910b | |
| needs: [Build-Ascend-Latest-Target-910b] | |
| runs-on: tps-ascend-ci-910b | |
| steps: | |
| - name: Run a test | |
| run: | | |
| set -ex | |
| cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/Build-Ascend-Latest-Target-910b/dipu | |
| source scripts/ci/ascend/ci_ascend_env.sh | |
| bash tests/run_ascend_tests.sh && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Ascend-Latest-Target \ | |
| || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf Build-Ascend-Latest-Target && exit 1 ) |