KEP-2170: Add Torch Distributed Runtime #1719
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: integration test | |
| on: | |
| - pull_request | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| integration-test: | |
| runs-on: ubuntu-latest | |
| # Almost similar to the following: | |
| # | |
| # ```yaml | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # kubernetes-version: ["v1.27.11", "v1.28.7", "v1.29.2"] | |
| # gang-scheduler-name: ["none", "scheduler-plugins", "volcano"] | |
| # ``` | |
| # The difference is that each combination is randomly assigned various Python versions | |
| # to verify Python SDK operations. | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # TODO (tenzen-y): Add volcano. | |
| include: | |
| - kubernetes-version: v1.29.2 | |
| gang-scheduler-name: "none" | |
| python-version: "3.10" | |
| - kubernetes-version: v1.27.11 | |
| gang-scheduler-name: "none" | |
| python-version: "3.11" | |
| - kubernetes-version: v1.28.7 | |
| gang-scheduler-name: "none" | |
| python-version: "3.8" | |
| - kubernetes-version: v1.29.2 | |
| gang-scheduler-name: "scheduler-plugins" | |
| python-version: "3.9" | |
| - kubernetes-version: v1.27.11 | |
| gang-scheduler-name: "scheduler-plugins" | |
| python-version: "3.10" | |
| - kubernetes-version: v1.28.7 | |
| gang-scheduler-name: "scheduler-plugins" | |
| python-version: "3.11" | |
| - kubernetes-version: v1.29.2 | |
| gang-scheduler-name: "volcano" | |
| python-version: "3.8" | |
| - kubernetes-version: v1.27.11 | |
| gang-scheduler-name: "volcano" | |
| python-version: "3.9" | |
| - kubernetes-version: v1.28.7 | |
| gang-scheduler-name: "volcano" | |
| python-version: "3.10" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Free-Up Disk Space | |
| uses: ./.github/workflows/free-up-disk-space | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Setup Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: go.mod | |
| - name: Create k8s Kind Cluster | |
| uses: helm/[email protected] | |
| with: | |
| node_image: kindest/node:${{ matrix.kubernetes-version }} | |
| cluster_name: training-operator-cluster | |
| kubectl_version: ${{ matrix.kubernetes-version }} | |
| - name: Build training-operator | |
| run: | | |
| ./scripts/gha/build-image.sh | |
| env: | |
| TRAINING_CI_IMAGE: kubeflowtraining/training-operator:test | |
| - name: Deploy training operator | |
| run: | | |
| ./scripts/gha/setup-training-operator.sh | |
| env: | |
| KIND_CLUSTER: training-operator-cluster | |
| TRAINING_CI_IMAGE: kubeflowtraining/training-operator:test | |
| GANG_SCHEDULER_NAME: ${{ matrix.gang-scheduler-name }} | |
| KUBERNETES_VERSION: ${{ matrix.kubernetes-version }} | |
| - name: Run tests | |
| run: | | |
| pip install pytest | |
| python3 -m pip install -e sdk/python; pytest -s sdk/python/test --log-cli-level=debug --namespace=default | |
| env: | |
| GANG_SCHEDULER_NAME: ${{ matrix.gang-scheduler-name }} | |
| - name: Collect volcano logs | |
| if: ${{ failure() && matrix.gang-scheduler-name == 'volcano' }} | |
| run: | | |
| echo "dump volcano-scheduler logs..." | |
| kubectl logs -n volcano-system -l app=volcano-scheduler --tail=-1 | |
| echo "dump volcano-admission logs..." | |
| kubectl logs -n volcano-system -l app=volcano-admission --tail=-1 | |
| echo "dump volcano-controllers logs..." | |
| kubectl logs -n volcano-system -l app=volcano-controller --tail=-1 | |
| echo "dump podgroups description..." | |
| kubectl describe podgroups.scheduling.volcano.sh -A |