update_pytorch.yml

# name: Run updater for pytorch that will check for pytorch packages

on:
    schedule:
      # run the job every 30 minutes
      - cron:  '*/30 * * * *'
    workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}

jobs:
  generate_hash_letters:
    runs-on: ubuntu-latest
    outputs:
        string_list: ${{ steps.generate.outputs.string_list }}
    steps:
        - uses: actions/checkout@v4

        - name: Set up Python
          uses: actions/setup-python@v5
          with:
            python-version: '3.9'  # Specify your Python version

        - uses: prefix-dev/setup-pixi@v0.5.1
          with:
            pixi-version: "latest"
            manifest-path: pyproject.toml


        - name: pixi install
          run: |
           pixi install

        - name: Generate combination of missing subdirs and letters
          id: generate
          run: |
            # Run your script here to produce a list of missing subdirs and letters
            # example: linux-64@p linux-64@d
            message=$(pixi run parselmouth updater-producer --channel pytorch)
            if [ "$message" = "[]" ]; then
              echo "Skipping job as no new packages found"
              exit 0
            fi
            echo $message
            echo "string_list=$message" >> $GITHUB_OUTPUT
          env:
            R2_PREFIX_ACCOUNT_ID: ${{ secrets.R2_PREFIX_ACCOUNT_ID }}
            R2_PREFIX_ACCESS_KEY_ID: ${{ secrets.R2_PREFIX_ACCESS_KEY_ID }}
            R2_PREFIX_SECRET_ACCESS_KEY: ${{ secrets.R2_PREFIX_SECRET_ACCESS_KEY }}
            R2_PREFIX_BUCKET: ${{ secrets.R2_PREFIX_BUCKET }}

        - name: Upload requested index.json
          uses: actions/upload-artifact@v4
          with:
            name: index
            path: output_index


  updater_of_records:
    runs-on: ubuntu-latest
    needs: generate_hash_letters
    strategy:
        matrix:
            subdirs: ${{fromJson(needs.generate_hash_letters.outputs.string_list)}}

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'

      - uses: prefix-dev/setup-pixi@v0.5.1
        with:
          pixi-version: "latest"
          manifest-path: pyproject.toml

      - name: pixi install
        run: |
          pixi install

      - name: Get partial index artifacts
        uses: actions/download-artifact@v4
        with:
            name: index
            path: output_index

      - name: Get artifact info for subdir@letter and save into partial index
        run: pixi run parselmouth updater ${{ matrix.subdirs }} --channel pytorch --upload
        env:
            R2_PREFIX_ACCOUNT_ID: ${{ secrets.R2_PREFIX_ACCOUNT_ID }}
            R2_PREFIX_ACCESS_KEY_ID: ${{ secrets.R2_PREFIX_ACCESS_KEY_ID }}
            R2_PREFIX_SECRET_ACCESS_KEY: ${{ secrets.R2_PREFIX_SECRET_ACCESS_KEY }}
            R2_PREFIX_BUCKET: ${{ secrets.R2_PREFIX_BUCKET }}

      - name: Upload partial index
        uses: actions/upload-artifact@v4
        with:
            name: partial_index_${{ matrix.subdirs }}
            path: output

  updater_of_index:
    runs-on: ubuntu-latest
    needs: updater_of_records
    steps:
        - uses: actions/checkout@v4

        - name: Set up Python
          uses: actions/setup-python@v5
          with:
            python-version: '3.9'  # Specify your Python version
        - uses: prefix-dev/setup-pixi@v0.5.1
          with:
            pixi-version: "latest"
            manifest-path: pyproject.toml

        - name: pixi install
          run: |
            pixi install

        - name: Get partial index artifacts
          uses: actions/download-artifact@v4
          with:
            merge-multiple: true
            path: output

        - name: Combine all partial index and save into S3 index
          run: pixi run parselmouth updater-merger --upload --channel pytorch
          env:
            R2_PREFIX_ACCOUNT_ID: ${{ secrets.R2_PREFIX_ACCOUNT_ID }}
            R2_PREFIX_ACCESS_KEY_ID: ${{ secrets.R2_PREFIX_ACCESS_KEY_ID }}
            R2_PREFIX_SECRET_ACCESS_KEY: ${{ secrets.R2_PREFIX_SECRET_ACCESS_KEY }}
            R2_PREFIX_BUCKET: ${{ secrets.R2_PREFIX_BUCKET }}

  update_file:
    runs-on: ubuntu-latest
    permissions:
      contents: write  # To push on main branch
    needs: updater_of_index

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'

      - uses: prefix-dev/setup-pixi@v0.5.1
        with:
          pixi-version: "latest"
          manifest-path: pyproject.toml

      - name: Update compressed mapping
        run: |
          # for pytorch we dont need to use legacy mapping anymore
          pixi run parselmouth update-mapping --channel pytorch
        env:
          GH_TOKEN: ${{ secrets.GH_TOKEN }}
          R2_PREFIX_ACCOUNT_ID: ${{ secrets.R2_PREFIX_ACCOUNT_ID }}
          R2_PREFIX_ACCESS_KEY_ID: ${{ secrets.R2_PREFIX_ACCESS_KEY_ID }}
          R2_PREFIX_SECRET_ACCESS_KEY: ${{ secrets.R2_PREFIX_SECRET_ACCESS_KEY }}
          R2_PREFIX_BUCKET: ${{ secrets.R2_PREFIX_BUCKET }}

      - name: Commit and push changes
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"
          git pull
          git add files/v0/pytorch
          git commit -m "Update file via GitHub Action" || true
          git push