Ci prod (galaxyproject#35)

* action in main to manually dispatch * update * update * update * update * All tools bot * full wf * update * filter community tool bot * filter communities bot * test flter * filter communities bot * trigger pages only when new results are created * Update .github/workflows/static.yml Co-authored-by: Nicola Soranzo <[email protected]> * fetch tools stepwise with CI * update CI * updat CI * update CI * update CI * update CI and get more verbose tool logs * add a test case to check what up with the CI * allow to run the test ever * try test with other api key * next try * as secrets * fetch all tools bot * merge test * cat in CI * CI all tools * CI test * force push * fetch all tools bot - step merge * test git merge * more tests * cover staged change as well * fetch all tools bot - step fetch * CI test * fetch all tools bot - step fetch * update fetch and filter CI * removed results from test runs * fetch all tools bot - step fetch * try pull merge with rebase * merge then push * test * fetch all tools bot - step fetch * fetch all tools bot - step fetch * fetch all tools bot - step fetch * fetch all tools bot - step fetch * fetch all tools bot - step fetch * CI update * fetch all tools bot - step fetch * fetch all tools bot - step merge * fetch all tools bot - step fetch * fetch all tools bot - step fetch * fetch all tools bot - step merge * fetch all tools bot - step fetch * fetch all tools bot - step fetch * fetch all tools bot - step fetch * fetch all tools bot - step fetch * fetch all tools bot - step merge * fetch all tools bot - step fetch * fetch all tools bot - step fetch * fetch all tools bot - step fetch * fetch all tools bot - step fetch * fetch all tools bot - step merge * fetch all tools bot - step fetch * fetch all tools bot - step merge * fetch all tools bot - step fetch * fetch all tools bot - step fetch * fetch all tools bot - step fetch * fetch all tools bot - step fetch * fetch all tools bot - step merge * Add nsoranzo review --------- Co-authored-by: github-actions <[email protected]> Co-authored-by: Nicola Soranzo <[email protected]>
nomadscientist · Jan 10, 2024 · b28b3a0 · b28b3a0
1 parent e85bfc4
commit b28b3a0
Show file tree

Hide file tree

Showing 19 changed files with 2,883 additions and 828 deletions.
diff --git a/.github/workflows/fetch_all_tools.yaml b/.github/workflows/fetch_all_tools.yaml
@@ -0,0 +1,75 @@
+name: Fetch all tools
+
+on:
+  workflow_dispatch:
+  schedule:
+    #Every Sunday at 8:00 am
+    - cron: "0 8 * * 0" 
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "tools"
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+
+jobs:
+  fetch-all-tools-stepwise:
+    runs-on: ubuntu-20.04
+    environment: fetch-tools
+    name: Fetch all tool stepwise
+    strategy:
+      matrix:
+        python-version: [3.8]
+        subset:
+         - repositories01.list
+         - repositories02.list
+         - repositories03.list
+         - repositories04.list
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Run script
+        run: | 
+          export GITHUB_API_KEY=${{ secrets.GH_API_TOKEN }}
+          bash ./bin/extract_all_tools_stepwise.sh "${{ matrix.subset }}"
+      - name: Commit all tools 
+        # add or commit any changes in results if there was a change, merge with main and push as bot
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git pull --no-rebase -s recursive -X ours
+          git add results
+          git status
+          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step fetch") 
+          git push
+
+  fetch-all-tools-merge:
+    runs-on: ubuntu-20.04
+    needs: fetch-all-tools-stepwise
+    name: Fetch all tools merge
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Run script
+        run: | 
+          cat results/repositories*.list_tools.tsv > results/all_tools.tsv
+          bash ./bin/extract_all_tools_downstream.sh
+      - name: Commit all tools
+        # add or commit any changes in results if there was a change, merge with main and push as bot
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git pull --no-rebase -s recursive -X ours
+          git add results
+          git status
+          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step merge") 
+          git push
diff --git a/.github/workflows/filter_communities.yaml b/.github/workflows/filter_communities.yaml
@@ -0,0 +1,38 @@
+name: Filter community tools
+
+on:
+  workflow_dispatch:
+
+  # the workflow it triggered when all_tools_tsv is changed
+  push:
+    paths:
+    - 'results/all_tools_tsv'
+    branches: ["main"]
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "filter"
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+
+jobs:
+  filter-all-tools:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+    - name: Install requirement
+      run: python -m pip install -r requirements.txt
+    - name: Run script
+      run: | 
+        bash ./bin/get_community_tools.sh
+    - name: Commit results
+      # commit the new filtered data, only if stuff was changed
+      run: |
+        git config user.name github-actions
+        git config user.email [email protected]
+        git diff --quiet || (git add results && git commit -m "filter communities bot") 
+        git push
diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
@@ -0,0 +1,66 @@
+name: Run tests
+
+on:
+  workflow_dispatch:
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+# concurrency:
+#   group: "tools"
+#   cancel-in-progress: false
+
+permissions:
+  contents: write
+
+jobs:
+  fetch-all-tools-stepwise:
+    runs-on: ubuntu-20.04
+    environment: fetch-tools
+    name: Fetch all tool stepwise
+    strategy:
+      matrix:
+        python-version: [3.8]
+        subset:
+         - test.list
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Run script
+        # run: bash bin/extract_all_tools.sh
+        run: | 
+          export GITHUB_API_KEY=${{ secrets.GH_API_TOKEN }}
+          bash ./bin/extract_all_tools_test.sh "${{ matrix.subset }}"
+      - name: Commit all tools 
+        # add or commit any changes in results if there was a change, merge with main and push as bot
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git pull --no-rebase -s recursive -X ours
+          git add results
+          git status
+          git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step fetch") 
+          git push
+
+  # fetch-all-tools-merge:
+  #   runs-on: ubuntu-20.04
+  #   needs: fetch-all-tools-stepwise
+  #   name: Fetch all tools merge
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: actions/setup-python@v5
+  #     - name: Install requirement
+  #       run: python -m pip install -r requirements.txt
+  #     - name: Run script
+  #       run: | 
+  #         cat results/repositories*.list_tools.tsv > results/all_tools.tsv
+  #         bash ./bin/extract_all_tools_downstream.sh
+  #     - name: Commit all tools
+  #       run: |
+  #         git config user.name github-actions
+  #         git config user.email [email protected]
+  #         git diff --quiet || (git add results && git commit -m "fetch all tools bot  - step merge") 
+  #         git push
diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml
@@ -2,10 +2,13 @@
 name: Deploy static content to Pages
 
 on:
-  # Runs on pushes targeting the default branch
+  # the workflow is triggered only when results are changed
   push:
+    paths:
+    - 'results'
     branches: ["main"]
 
+
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 

diff --git a/bin/extract_all_tools.sh b/bin/extract_all_tools.sh
diff --git a/bin/extract_all_tools_downstream.sh b/bin/extract_all_tools_downstream.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+mkdir -p 'results/'
+
+python bin/create_interactive_table.py \
+        --table "results/all_tools.tsv" \
+        --template "data/interactive_table_template.html" \
+        --output "results/index.html"
diff --git a/bin/extract_all_tools_stepwise.sh b/bin/extract_all_tools_stepwise.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+mkdir -p 'results/'
+
+output="results/${1}_tools.tsv"
+
+python bin/extract_galaxy_tools.py \
+        extractools \
+        --api $GITHUB_API_KEY \
+        --all_tools $output \
+        --planemorepository $1
+
diff --git a/bin/extract_all_tools_test.sh b/bin/extract_all_tools_test.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+mkdir -p 'results/'
+
+output="results/${1}_tools.tsv"
+
+python bin/extract_galaxy_tools.py \
+        extractools \
+        --api $GITHUB_API_KEY \
+        --all_tools $output \
+        --planemorepository $1 \
+        --test
+
diff --git a/bin/extract_galaxy_tools.py b/bin/extract_galaxy_tools.py
@@ -57,18 +57,36 @@ def get_string_content(cf: ContentFile) -> str:
     return base64.b64decode(cf.content).decode("utf-8")
 
 
-def get_tool_github_repositories(g: Github) -> List[str]:
+def get_tool_github_repositories(g: Github, RepoSelection: Optional[str], run_test: bool) -> List[str]:
     """
     Get list of tool GitHub repositories to parse
 
     :param g: GitHub instance
+    :param RepoSelection: The selection to use from the repository (needed to split the process for CI jobs)
+    :run_test: for CI testing only use one repository
     """
+
+    if run_test:
+        return ["https://github.com/TGAC/earlham-galaxytools"]
+
     repo = g.get_user("galaxyproject").get_repo("planemo-monitor")
     repo_list: List[str] = []
     for i in range(1, 5):
-        repo_f = repo.get_contents(f"repositories0{i}.list")
-        repo_l = get_string_content(repo_f).rstrip()
-        repo_list.extend(repo_l.split("\n"))
+        repo_selection = f"repositories0{i}.list"
+        if RepoSelection:  # only get these repositories
+            if RepoSelection == repo_selection:
+                repo_f = repo.get_contents(repo_selection)
+                repo_l = get_string_content(repo_f).rstrip()
+                repo_list.extend(repo_l.split("\n"))
+        else:
+            repo_f = repo.get_contents(repo_selection)
+            repo_l = get_string_content(repo_f).rstrip()
+            repo_list.extend(repo_l.split("\n"))
+
+    print("Parsing repositories from:")
+    for repo in repo_list:
+        print("\t", repo)
+
     return repo_list
 
 
@@ -475,11 +493,18 @@ def filter_tools(
     # Extract tools
     extractools = subparser.add_parser("extractools", help="Extract tools")
     extractools.add_argument("--api", "-a", required=True, help="GitHub access token")
+    extractools.add_argument("--all_tools", "-o", required=True, help="Filepath to TSV with all extracted tools")
     extractools.add_argument(
-        "--all_tools",
-        "-o",
-        required=True,
-        help="Filepath to TSV with all extracted tools",
+        "--planemorepository", "-pr", required=False, help="Repository list to use from the planemo-monitor repository"
+    )
+
+    extractools.add_argument(
+        "--test",
+        "-t",
+        action="store_true",
+        default=False,
+        required=False,
+        help="Run a small test case using only the repository: https://github.com/TGAC/earlham-galaxytools",
     )
 
     # Filter tools
@@ -517,12 +542,11 @@ def filter_tools(
         # connect to GitHub
         g = Github(args.api)
         # get list of GitHub repositories to parse
-        repo_list = get_tool_github_repositories(g)
-
+        repo_list = get_tool_github_repositories(g, args.planemorepository, args.test)
         # parse tools in GitHub repositories to extract metada, filter by TS categories and export to output file
         tools: List[Dict] = []
         for r in repo_list:
-            print(r)
+            print("Parsing tools from:", (r))
             if "github" not in r:
                 continue
             try:

diff --git a/bin/get_community_tools.sh b/bin/get_community_tools.sh