azure-pipelines.yml

trigger:
  branches:
    include:
    - '*'

pr:
- master
- v*.*

jobs:
- job: CI
  timeoutInMinutes: 120
  cancelTimeoutInMinutes: 2
  pool:
    vmImage: 'ubuntu-latest'
  strategy:
    matrix:
      DataFrame:
        mars.test.module: 'dataframe'
      Learn:
        mars.test.module: 'learn'
      Tensor:
        mars.test.module: 'tensor'

  variables:
    PYTHON: '3.8'

  steps:
    - powershell: |
        $header = @{ Authorization = "Bearer $env:SYSTEM_ACCESSTOKEN" }
        $buildsUrl = "$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_apis/build/builds?api-version=6.0"
        $builds = Invoke-RestMethod -Uri $buildsUrl -Method Get -Header $header
        $buildsToStop = $builds.value.Where({ ($_.status -eq 'inProgress') -and ($_.sourceBranch -eq '$(Build.SourceBranch)') -and ($_.definition.name -eq '$(Build.DefinitionName)') -and ($_.id -ne '$(Build.BuildId)') })
        ForEach($build in $buildsToStop)
        {
            $build.status = "cancelling"
            $body = $build | ConvertTo-Json -Depth 10
            $urlToCancel = "$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_apis/build/builds/" + $build.id + "?api-version=6.0"
            Write-Output "Cancelling $urlToCancel"
            Invoke-RestMethod -Uri $urlToCancel -Method Patch -ContentType application/json -Header $header -Body $body -SkipHttpErrorCheck
        }
      env:
        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
      displayName: 'Cancel previous jobs'

    - bash: |
        set -e
        source ci/install-conda.sh
      displayName: 'Install conda'

    - bash: |
        set -e
        source ci/reload-env.sh
        export DEFAULT_VENV=$VIRTUAL_ENV

        if [[ ! "$PYTHON" =~ "3.9" ]]; then
          conda install -n test --quiet --yes -c conda-forge python=$PYTHON numba
        fi

        # todo remove this when fastparquet release new version
        if [[ "$PYTHON" =~ "3.6" ]]; then
          pip install numpy\<1.20.0
        fi

        source ./ci/rewrite-cov-config.sh

        if [[ "$(mars.test.module)" == "learn" ]]; then
          pip install numpy\<1.20.0 scipy cython
        else
          pip install numpy scipy cython
        fi
        pip install -e ".[dev,extra]"
        pip install virtualenv flaky

        if [ -z "$NO_COMMON_TESTS" ]; then
          if [[ ! "$PYTHON" =~ "3.6" ]] && [[ ! "$PYTHON" =~ "3.9" ]]; then
            pip install h5py zarr matplotlib fastparquet\<0.7.0
            conda install -n test --quiet --yes -c conda-forge python=$PYTHON \
              "tiledb-py>=0.4.3,<0.6.0" "tiledb<2.0.0" || true
          fi

          conda install -n test --quiet --yes -c pkgs/main python=$PYTHON certifi

          if [[ "$(mars.test.module)" == "learn" ]]; then
            # remove version limit when blue-yonder/tsfresh#897 is fixed.
            # remove keras version after https://github.com/tensorflow/tensorflow/issues/52922 is fixed.
            pip install xgboost lightgbm keras==2.6.0 tensorflow faiss-cpu torch torchvision \
              statsmodels\<0.13.0 tsfresh
          fi
        fi
        conda list -n test
      displayName: 'Install dependencies'

    - bash: |
        set -e
        source ci/reload-env.sh
        mkdir -p build
        pytest $PYTEST_CONFIG mars/$(mars.test.module)
        mv .coverage build/.coverage.main.file
        
        # do compatibility test for earliest supported pandas release
        if [[ "$(mars.test.module)" == "dataframe" ]]; then
          pip install pandas==1.0.5
          pytest $PYTEST_CONFIG -m pd_compat mars/dataframe
          mv .coverage build/.coverage.pd_compat.file
        fi
        
        coverage combine build/ && coverage report
        coverage xml
      displayName: 'Run tests'

    - bash: |
        GIT_ORIGIN="$(git config --get remote.origin.url)"
        if [[ "$GIT_ORIGIN" == *"mars-project/mars"* ]]; then
          export CODECOV_TOKEN="f578555e-b4f4-49a8-8d33-b21a305299ee"
        fi
        bash <(curl -s https://codecov.io/bash)
      displayName: 'Upload coverage'

- job: Checks
  timeoutInMinutes: 120
  cancelTimeoutInMinutes: 2

  pool:
    vmImage: 'ubuntu-latest'

  variables:
    PYTHON: '3.8'

  steps:
    - bash: |
        set -e
        source ci/install-conda.sh
      displayName: 'Install conda'

    - bash: |
        set -e
        source ./ci/reload-env.sh
        export DEFAULT_VENV=$VIRTUAL_ENV

        pip install numpy scipy cython
        pip install -e ".[dev,extra]"
        pip install virtualenv flake8 codespell sphinx sphinx-intl black
        conda list -n test
      displayName: 'Install dependencies'

    - bash: |
        set -e
        source ./ci/reload-env.sh

        # stop the build if there are Python syntax errors or undefined names
        flake8 mars --count --show-source --statistics

        # special check for __init__.py
        grep -A 10000 '\[flake8\]' setup.cfg | awk '!/(F401|F811|__init__\.py)/' > flake8_init.ini
        flake8 --config=flake8_init.ini

        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 mars --config="default" --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
      displayName: 'Lint with flake8'

    - bash: |
        set -e
        source ./ci/reload-env.sh

        black --check --diff --verbose mars
      displayName: 'Check code style with black'

    - bash: |
        set -e
        source ./ci/reload-env.sh
        codespell
      displayName: 'Check spellings'

    - bash: |
        set -e
        source ./ci/reload-env.sh
        git fetch origin master
        bash ci/modecheck.sh
      displayName: 'Check file mode changes'

    - bash: |
        set -e
        source ./ci/reload-env.sh
        python ci/importcheck.py
      displayName: 'Check imports'

    - bash: |
        set -e
        source ./ci/reload-env.sh
        python ci/copycheck.py
      displayName: 'Check copyright headers'

    - bash: |
        set -e
        source ./ci/reload-env.sh
        pushd mars/services/web/ui
        npm install
        npm run lint
        popd
      displayName: 'Check JS with ESLint'

    - bash: |
        set -e
        source ./ci/reload-env.sh
        pushd docs
        pip install -r requirements-doc.txt
        make html
        popd
      displayName: 'Check documentation build'