Skip to content
Fetch all tools

Fetch all tools #53

Sign in to view logs

Fetch all tools

Fetch all tools #53

Workflow file for this run

.github/workflows/fetch_all_tools.yaml at 72e2d3c
name: Fetch all tools
on:
workflow_dispatch:
schedule:
#Every Sunday at 8:00 am
- cron: "0 8 * * 0"
# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
concurrency:
group: "tools"
cancel-in-progress: false
permissions:
contents: write
jobs:
fetch-available-servers:
runs-on: ubuntu-20.04
name: Fetch list of all available servers
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
- name: Install requirement
run: python -m pip install -r requirements.txt
- name: Run script
run: |
python ./bin/get_public_galaxy_servers.py -o data/available_public_servers.csv
- name: Commit servers
# add or commit any changes in results if there was a change, merge with main, and push as bot
run: |
git config user.name github-actions
git config user.email [email protected]
git pull --no-rebase -s recursive -X ours
git add data
git status
git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot - step get available servers")
git push
fetch-all-tools-stepwise:
runs-on: ubuntu-20.04
environment: fetch-tools
name: Fetch all tool stepwise
strategy:
matrix:
python-version: [3.8]
subset:
- repositories01.list
- repositories02.list
- repositories03.list
- repositories04.list
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install requirement
run: python -m pip install -r requirements.txt
- name: Run script #needs PAT to access other repos
run: |
bash ./bin/extract_all_tools_stepwise.sh "${{ matrix.subset }}"
env:
GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }}
- name: Commit all tools
# add or commit any changes in results if there was a change, merge with main and push as bot
run: |
git config user.name github-actions
git config user.email [email protected]
git pull --no-rebase -s recursive -X ours
git add results
git status
git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot - step fetch")
git push
fetch-all-tools-merge:
runs-on: ubuntu-20.04
needs: fetch-all-tools-stepwise
name: Fetch all tools merge
steps:
- uses: actions/checkout@v4
with:
ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
- uses: actions/setup-python@v5
- name: Install requirement
run: |
python -m pip install -r requirements.txt
sudo apt-get install jq
- name: Merge all tools
run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry)
awk 'FNR==1 && NR!=1{next;}{print}' results/repositories*.list_tools.tsv > results/all_tools.tsv
jq -s 'map(.[])' results/repositories*.list_tools.json > results/all_tools.json
- name: Wordcloud and interactive table
run: |
bash ./bin/extract_all_tools_downstream.sh
- name: Commit all tools
# add or commit any changes in results if there was a change, merge with main and push as bot
run: |
git config user.name github-actions
git config user.email [email protected]
git pull --no-rebase -s recursive -X ours
git add results
git status
git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot - step merge")
git push