python.yml

name: python
on:
  pull_request:
  push:
    branches:
      - main
jobs:
  path-filter:
    outputs:
      common: ${{steps.changes.outputs.common}}
      pytorch: ${{steps.changes.outputs.pytorch}}
      ray: ${{steps.changes.outputs.ray}}
      river: ${{steps.changes.outputs.river}}
      tensorflow: ${{steps.changes.outputs.tensorflow}}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        id: changes
        with:
          filters: |
            common:
              - .python-version
              - .tool-versions
              - poetry.lock
              - pyproject.toml
              - .github/workflows/python.yml
            pytorch:
              - 'pytorch/**'
              - '!pytorch/README.md'
              - '!pytorch/**/README.md'
            ray:
              - 'ray/**'
              - '!ray/README.md'
              - '!ray/**/README.md'
            river:
              - 'river/**'
              - '!river/README.md'
              - '!river/**/README.md'
            tensorflow:
              - 'tensorflow/**'
              - '!tensorflow/README.md'
              - '!tensorflow/**/README.md'

  status-check-python:
    # This job is used for the branch's branch protection rule's status check.
    # If all dependent jobs succeed or are skipped this job succeeds.
    runs-on: ubuntu-latest
    needs:
      - pytorch
      - ray
      - river
      - tensorflow
    permissions: {}
    if: failure()
    steps:
      - run: exit 1

  pytorch:
    if: needs.path-filter.outputs.common == 'true' || needs.path-filter.outputs.pytorch == 'true'
    needs: path-filter
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python & Poetry
        uses: nakamasato/github-actions/setup-poetry@1.6.3
        with:
          install-dependencies: false

      - name: run
        working-directory: pytorch
        run: |
          poetry install --only pytorch --no-root
          poetry run python distributed_example.py

  ray:
    if: needs.path-filter.outputs.common == 'true' || needs.path-filter.outputs.ray == 'true'
    needs: path-filter
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python & Poetry
        uses: nakamasato/github-actions/setup-poetry@1.6.3
        with:
          install-dependencies: false

      - name: Install dependencies
        run: poetry install --only ray,tensorflow,pytorch --no-root

      - name: 02-task-pattern
        run: poetry run python ray/02-task-pattern/task_pattern_map_and_reduce.py

      - name: 05-train simple with torch
        run: poetry run python ray/05-train/simple_example.py -b torch

      - name: 05-train simple with tensorflow
        run: poetry run python ray/05-train/simple_example.py -b tensorflow

      # == Status ==
      # Current time: 2023-09-23 10:32:09 (running for 00:07:35.39)
      # Memory usage on this node: 1.6/6.8 GiB
      # Using FIFO scheduling algorithm.
      # Resources requested: 0/2 CPUs, 0/0 GPUs, 0.0/3.48 GiB heap, 0.0/1.74 GiB objects
      # Result logdir: /home/runner/ray_results/TensorflowTrainer_2023-09-23_10-24-33
      # Number of trials: 1/1 (1 PENDING)
      # +-------------------------------+----------+-------+
      # | Trial name                    | status   | loc   |
      # |-------------------------------+----------+-------|
      # | TensorflowTrainer_6383c_00000 | PENDING  |       |
      # +-------------------------------+----------+-------+

      # - name: 05-train tensorflow example
      #   run: python 05-train/tensorflow_example.py --smoke-test
      # TODO: https://github.com/nakamasato/ml-training/issues/298

  river:
    if: needs.path-filter.outputs.common == 'true' || needs.path-filter.outputs.river == 'true'
    needs: path-filter
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python & Poetry
        uses: nakamasato/github-actions/setup-poetry@1.6.3
        with:
          install-dependencies: false

      - name: Install dependencies
        run: poetry install --only river --no-root

      - name: run river/quickstart/main.py
        run: poetry run python river/quickstart/main.py

      - name: run river/examples/fm-for-datalens/main.py
        run: poetry run python river/examples/fm-for-datalens/main.py --native-prediction --linear-regression --funk-mf --biased-mf --mimic-biased-mf --mf-with-improved-feature

  tensorflow:
    if: needs.path-filter.outputs.common == 'true' || needs.path-filter.outputs.tensorflow == 'true'
    needs: path-filter
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python & Poetry
        uses: nakamasato/github-actions/setup-poetry@1.6.3
        with:
          install-dependencies: false

      - name: Install dependencies
        run: poetry install --only tensorflow --no-root

      - name: parameter-server-training
        run: poetry run python tensorflow/parameter-server-training/train_in_process_cluster.py

      - name: dataset
        run: poetry run python tensorflow/dataset/practice.py

      - name: movielens - retrieve
        env:
          TF_USE_LEGACY_KERAS: 1
        run: |
          poetry run python tensorflow/examples/movielens/retrieve.py

      - name: movielens - rank
        env:
          WRAPT_DISABLE_EXTENSIONS: 1
        run: |
          poetry run python tensorflow/examples/movielens/rank.py