Merge branch 'main' of https://github.com/galaxyproject/galaxy_codex …

…into sc-lab-updates

.github/workflows/fetch_filter_resources.yaml

@@ -0,0 +1,124 @@
+name: Weekly resource fetching and community filtering
+
+on:
+  workflow_dispatch:
+  schedule:
+    #Every Sunday at 8:00 am
+    - cron: "0 8 * * 0" 
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "tools"
+  cancel-in-progress: false
+
+jobs:
+  fetch-servers:
+    runs-on: ubuntu-20.04
+    name: Fetch servers
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Install requirement
+        run: |
+          python -m pip install -r requirements.txt
+          sudo apt-get install jq
+      - name: Fetch list of all available servers
+        run: | 
+          python sources/bin/get_public_galaxy_servers.py -o sources/data/available_public_servers.csv
+      - name: Archive available servers
+        uses: actions/upload-artifact@v4
+        with:
+          name: available-servers
+          path: sources/data/available_public_servers.csv
+  fetch-tools-stepwise:
+    runs-on: ubuntu-20.04
+    name: Fetch tool stepwise
+    environment: fetch-tools
+    needs: fetch-servers
+    strategy:
+      #max-parallel: 1 #need to run one after another, since otherwise there is a chance, that mulitple jobs want to push to the results branch at the same time (which fails due to merge)
+      matrix:
+        subset:
+         - repositories01.list
+         - repositories02.list
+         - repositories03.list
+         - repositories04.list
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Download available servers
+        uses: actions/download-artifact@v4
+        with:
+          name: available-servers
+          path: sources/data/
+      - name: Fetch all tool stepwise
+        run: | 
+          bash sources/bin/extract_all_tools.sh "${{ matrix.subset }}"
+        env: 
+          GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }}
+      - name: Archive tool sublists production artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: tools-${{ matrix.subset }}
+          path: communities/all/resources/${{ matrix.subset }}_tools.*
+  merge-fetch-filter:
+    runs-on: ubuntu-20.04
+    needs: fetch-tools-stepwise
+    name: Merge tools, fetch tutorials and filter the resources for communities
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Install requirement
+        run: |
+          python -m pip install -r requirements.txt
+          sudo apt-get install jq
+      - name: Download stepwise tool lists
+        uses: actions/download-artifact@v4
+        with:
+          pattern: tools-*
+          merge-multiple: true
+          path: communities/all/resources/
+      - name: Display structure of downloaded files
+        run: ls -R communities/all/resources/
+      - name: Merge all tools
+        run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry)
+          awk 'FNR==1 && NR!=1{next;}{print}' communities/all/resources/repositories*.list_tools.tsv > communities/all/resources/tools.tsv
+          jq -s 'map(.[])' communities/all/resources/repositories*.list_tools.json > communities/all/resources/all_tools.json 
+      - name: Generate wordcloud and interactive table
+        run: |
+          bash sources/bin/format_tools.sh
+      - name: Fetch all tutorials
+        run: | 
+          bash sources/bin/extract_all_tutorials.sh
+        env: 
+          PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }}
+      - name: Filter tutorials for communities
+        run: | 
+          bash sources/bin/get_community_tutorials.sh
+      - name: Update tool to keep and exclude for communities
+        run: | 
+          bash sources/bin/update_tools_to_keep_exclude.sh
+      - name: Filter tools for communities
+        run: | 
+          bash sources/bin/get_community_tools.sh
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v4
+        with:
+          commit-message: Update resources
+          title: Automatic resources update
+          body: Automatic resource update done via GitHub Action once a week
+          base: main
+          branch: resource-update
+          delete-branch: true