Move Central Page and Tools into CoDex

.github/workflows/fetch_all_tools.yaml

@@ -0,0 +1,105 @@
+name: Fetch all tools
+
+on:
+  workflow_dispatch:
+  schedule:
+    #Every Sunday at 8:00 am
+    - cron: "0 8 * * 0" 
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "tools"
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+
+jobs:
+  fetch-available-servers:
+    runs-on: ubuntu-20.04
+    name: Fetch list of all available servers
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Run script 
+        run: | 
+          python ./bin/get_public_galaxy_servers.py -o data/available_public_servers.csv
+      - name: Commit servers 
+        # add or commit any changes in results if there was a change, merge with main, and push as bot
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git pull --no-rebase -s recursive -X ours
+          git add data
+          git status
+          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step get available servers") 
+          git push
+
+  fetch-all-tools-stepwise:
+    runs-on: ubuntu-20.04
+    environment: fetch-tools
+    name: Fetch all tool stepwise
+    strategy:
+      matrix:
+        python-version: [3.8]
+        subset:
+         - repositories01.list
+         - repositories02.list
+         - repositories03.list
+         - repositories04.list
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Run script #needs PAT to access other repos
+        run: | 
+          bash ./bin/extract_all_tools_stepwise.sh "${{ matrix.subset }}"
+        env: 
+          GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }}
+      - name: Commit all tools 
+        # add or commit any changes in results if there was a change, merge with main and push as bot
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git pull --no-rebase -s recursive -X ours
+          git add results
+          git status
+          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step fetch") 
+          git push
+
+  fetch-all-tools-merge:
+    runs-on: ubuntu-20.04
+    needs: fetch-all-tools-stepwise
+    name: Fetch all tools merge
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
+      - uses: actions/setup-python@v5
+      - name: Install requirement
+        run: |
+          python -m pip install -r requirements.txt
+          sudo apt-get install jq
+      - name: Merge all tools
+        run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry)
+          awk 'FNR==1 && NR!=1{next;}{print}' results/repositories*.list_tools.tsv > results/all_tools.tsv
+          jq -s 'map(.[])' results/repositories*.list_tools.json > results/all_tools.json 
+      - name: Wordcloud and interactive table
+        run: |
+          bash ./bin/extract_all_tools_downstream.sh
+      - name: Commit all tools
+        # add or commit any changes in results if there was a change, merge with main and push as bot
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git pull --no-rebase -s recursive -X ours
+          git add results
+          git status
+          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step merge") 
+          git push

.github/workflows/fetch_all_tutorials.yaml

@@ -0,0 +1,44 @@
+name: Fetch all tutorials
+
+on:
+  workflow_dispatch:
+  schedule:
+    #Every Sunday at 8:00 am
+    - cron: "0 8 * * 0" 
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "tutorials"
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+
+jobs:
+  fetch-all-tutorials:
+    runs-on: ubuntu-20.04
+    environment: fetch-tutorials
+    name: Fetch all tutorials
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Run script #needs PAT to access other repos
+        run: | 
+          bash ./bin/extract_all_tutorials.sh
+        env: 
+          PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }}
+      - name: Commit all tools 
+        # add or commit any changes in results if there was a change, merge with main and push as bot
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git pull --no-rebase -s recursive -X ours
+          git add results
+          git status
+          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tutorials bot  - step fetch") 
+          git push

.github/workflows/filter_communities.yaml

@@ -0,0 +1,98 @@
+name: Filter community tools
+
+on:
+  workflow_dispatch:
+
+  # the workflow it triggered when all tools are fetched
+  workflow_run:
+      workflows: ["Fetch all tools"]
+      types:
+        - completed
+
+  # the workflow it also triggered when the community definitions are changed
+  push:
+    paths:
+    - 'data/communities**'
+    branches: ["main"]
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "filter"
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+
+jobs:
+  filter-all-tutorials:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.8' 
+    - name: Install requirement
+      run: python -m pip install -r requirements.txt
+    - name: Run script
+      run: | 
+        bash ./bin/get_community_tutorials.sh
+    - name: Commit results
+      # commit the new filtered data, only if stuff was changed
+      run: |
+        git config user.name github-actions
+        git config user.email [email protected]
+        git pull --no-rebase -s recursive -X ours
+        git add results
+        git status
+        git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tutorials / tools bot  - step tutorial filter") 
+        git push
+
+  update-tools-to-keep-exclude:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.8' 
+    - name: Install requirement
+      run: python -m pip install -r requirements.txt
+    - name: Run script
+      run: | 
+        bash ./bin/update_tools_to_keep_exclude.sh
+    - name: Commit results
+      # commit the new filtered data, only if stuff was changed
+      run: |
+        git config user.name github-actions
+        git config user.email [email protected]
+        git pull --no-rebase -s recursive -X ours
+        git add results
+        git status
+        git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tutorials / tools bot  - step exluded/kept tool list update") 
+        git push
+
+  filter-all-tools:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.8' 
+    - name: Install requirement
+      run: python -m pip install -r requirements.txt
+    - name: Run script
+      run: | 
+        bash ./bin/get_community_tools.sh
+    - name: Commit results
+      # commit the new filtered data, only if stuff was changed
+      run: |
+        git config user.name github-actions
+        git config user.email [email protected]
+        git pull --no-rebase -s recursive -X ours
+        git add results
+        git status
+        git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step tool filter") 
+        git push
+
+
+

.github/workflows/lint.yaml

@@ -0,0 +1,20 @@
+name: Lint
+on: [push, pull_request]
+concurrency:
+  group: lint-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.11']
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install tox
+      run: python -m pip install 'tox>=1.8.0'
+    - name: Lint
+      run: tox -e lint

.github/workflows/run_tests.yaml

@@ -0,0 +1,74 @@
+name: Run tests
+on: [push, pull_request]
+
+jobs:
+  test-tools:
+    runs-on: ubuntu-20.04
+    environment: fetch-tools
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Tool extraction
+        # run: bash bin/extract_all_tools.sh
+        run: | 
+          python bin/extract_galaxy_tools.py \
+            extractools \
+            --api $GITHUB_API_KEY \
+            --all-tools "results/test_tools.tsv" \
+            --all-tools-json "results/test_tools.json" \
+            --planemo-repository-list "test.list" \
+            --test
+        env: 
+          GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }}
+      - name: Tool filter
+        run: | 
+          python bin/extract_galaxy_tools.py \
+            filtertools \
+            --tools "results/all_tools.json" \
+            --ts-filtered-tools "results/microgalaxy/tools_filtered_by_ts_categories.tsv" \
+            --filtered-tools "results/microgalaxy/tools.tsv" \
+            --categories "data/communities/microgalaxy/categories" \
+            --status "data/communities/microgalaxy/tool_status.tsv"
+      - name: Create interactive table
+        run: | 
+          python bin/create_interactive_table.py \
+            --table "results/microgalaxy/tools.tsv" \
+            --template "data/interactive_table_template.html" \
+            --output "results/microgalaxy/index.html"
+      - name: Create wordcloud
+        run: | 
+          python bin/create_wordcloud.py \
+            --table  "results/microgalaxy/tools.tsv" \
+            --wordcloud_mask "data/usage_stats/wordcloud_mask.png" \
+            --output "results/microgalaxy/tools_wordcloud.png" \
+            --stats_column "No. of tool users (2022-2023) (usegalaxy.eu)"
+  test-tutorials:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Tutorial extraction
+        run: | 
+          python bin/extract_gtn_tutorials.py \
+            extracttutorials \
+            --all_tutorials "results/test_tutorials.json" \
+            --tools "results/all_tools.json" \
+            --api $PLAUSIBLE_API_KEY \
+            --test
+        env: 
+          PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }}
+      - name: Tutorial filtering
+        run: | 
+          python bin/extract_gtn_tutorials.py \
+            filtertutorials \
+            --all_tutorials "results/test_tutorials.json" \
+            --filtered_tutorials "results/microgalaxy/test_tutorials.tsv" \
+            --tags "data/communities/microgalaxy/tutorial_tags"

.github/workflows/static.yml

@@ -0,0 +1,47 @@
+# Simple workflow for deploying static content to GitHub Pages
+name: Deploy static content to Pages
+
+on:
+
+  # the workflow it triggered when the tools where filtered
+  workflow_run:
+    workflows: ["Filter community tools"]
+    types:
+      - completed
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Single deploy job since we're just deploying
+  deploy:
+    if: github.repository_owner == 'galaxyproject'
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Pages
+        uses: actions/configure-pages@v4
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+        with:
+          # Upload results index.html repository
+          path: './results'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v3

.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+__pycache__