fetch_all_tools.yaml

name: Fetch all tools

on:
  workflow_dispatch:
  schedule:
    #Every Sunday at 8:00 am
    - cron: "0 8 * * 0" 

# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
concurrency:
  group: "tools"
  cancel-in-progress: false
  
permissions:
  contents: write

jobs:
  fetch-all-tools-stepwise:
    runs-on: ubuntu-20.04
    environment: fetch-tools
    name: Fetch all tool stepwise
    strategy:
      matrix:
        python-version: [3.8]
        subset:
         - repositories01.list
         - repositories02.list
         - repositories03.list
         - repositories04.list
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install requirement
        run: python -m pip install -r requirements.txt
      - name: Run script
        run: | 
          export GITHUB_API_KEY=${{ secrets.GH_API_TOKEN }}
          bash ./bin/extract_all_tools_stepwise.sh "${{ matrix.subset }}"
      - name: Commit all tools 
        # add or commit any changes in results if there was a change, merge with main and push as bot
        run: |
          git config user.name github-actions
          git config user.email github-actions@github.com
          git pull --no-rebase -s recursive -X ours
          git add results
          git status
          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step fetch") 
          git push

  fetch-all-tools-merge:
    runs-on: ubuntu-20.04
    needs: fetch-all-tools-stepwise
    name: Fetch all tools merge
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
      - name: Install requirement
        run: python -m pip install -r requirements.txt
      - name: Run script
        run: | 
          cat results/repositories*.list_tools.tsv > results/all_tools.tsv
          bash ./bin/extract_all_tools_downstream.sh
      - name: Commit all tools
        # add or commit any changes in results if there was a change, merge with main and push as bot
        run: |
          git config user.name github-actions
          git config user.email github-actions@github.com
          git pull --no-rebase -s recursive -X ours
          git add results
          git status
          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step merge") 
          git push