add a test case to check what up with the CI

.github/workflows/run_tests.yaml

@@ -0,0 +1,65 @@
+name: Run tests
+
+on:
+  workflow_dispatch:
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "tools"
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+
+jobs:
+  fetch-all-tools-stepwise:
+    runs-on: ubuntu-20.04
+    name: Fetch all tool stepwise
+    strategy:
+      matrix:
+        python-version: [3.8]
+        subset:
+         - repositories01.list
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Run script
+        # run: bash bin/extract_all_tools.sh
+        run: | 
+          bash ./bin/extract_all_tools_test.sh "${{ matrix.subset }}"
+        env:
+          GITHUB_API_KEY:  ${{ secrets.GITHUB_TOKEN }}
+      - name: Commit all tools 
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git add results
+          git commit -m "fetch all tools bot"
+          git push
+  fetch-all-tools-merge:
+    runs-on: ubuntu-20.04
+    needs: fetch-all-tools-stepwise
+    name: Fetch all tools merge
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Run script
+        run: | 
+          bash ./bin/extract_all_tools_merge.sh
+          bash ./bin/extract_all_tools_downstream.sh
+        env:
+          GITHUB_API_KEY:  ${{ secrets.GITHUB_TOKEN }}
+      - name: Commit all tools
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git add results
+          git commit -m "fetch all tools bot"
+          git push

bin/extract_all_tools_test.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+mkdir -p 'results/'
+
+output="results/${1}_tools.tsv"
+
+python bin/extract_galaxy_tools.py \
+        extractools \
+        --api $GITHUB_API_KEY \
+        --all_tools $output \
+        --planemorepository $1 \
+        --test
+

bin/extract_galaxy_tools.py

@@ -50,14 +50,18 @@ def get_string_content(cf: ContentFile) -> str:
     return base64.b64decode(cf.content).decode("utf-8")
 
 
-def get_tool_github_repositories(g: Github, RepoSelection: Optional[str]) -> List[str]:
+def get_tool_github_repositories(g: Github, RepoSelection: Optional[str], run_test: bool) -> List[str]:
     """
     Get list of tool GitHub repositories to parse
 
     :param g: GitHub instance
     :param RepoSelection: The selection to use from the repository (needed to split the process for CI jobs)
+    :run_test: for CI testing only use one repository
     """
 
+    if run_test:
+        return ["https://github.com/TGAC/earlham-galaxytools"]
+
     repo = g.get_user("galaxyproject").get_repo("planemo-monitor")
     repo_list: List[str] = []
     for i in range(1, 5):
@@ -408,6 +412,15 @@ def filter_tools(tools: List[Dict], ts_cat: List[str], excluded_tools: List[str]
         "--planemorepository", "-pr", required=False, help="Repository list to use from the planemo-monitor repository"
     )
 
+    extractools.add_argument(
+        "--test",
+        "-t",
+        action="store_true",
+        default=False,
+        required=False,
+        help="Run a small test case using only the repository: https://github.com/TGAC/earlham-galaxytools",
+    )
+
     # Filter tools
     filtertools = subparser.add_parser("filtertools", help="Filter tools")
     filtertools.add_argument(
@@ -430,7 +443,7 @@ def filter_tools(tools: List[Dict], ts_cat: List[str], excluded_tools: List[str]
         # connect to GitHub
         g = Github(args.api)
         # get list of GitHub repositories to parse
-        repo_list = get_tool_github_repositories(g, args.planemorepository)
+        repo_list = get_tool_github_repositories(g, args.planemorepository, args.test)
         # parse tools in GitHub repositories to extract metada, filter by TS categories and export to output file
         tools: List[Dict] = []
         for r in repo_list: