diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml new file mode 100644 index 00000000..58fd2b2f --- /dev/null +++ b/.github/workflows/run_tests.yaml @@ -0,0 +1,65 @@ +name: Run tests + +on: + workflow_dispatch: + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "tools" + cancel-in-progress: false + +permissions: + contents: write + +jobs: + fetch-all-tools-stepwise: + runs-on: ubuntu-20.04 + name: Fetch all tool stepwise + strategy: + matrix: + python-version: [3.8] + subset: + - repositories01.list + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install requirement + run: python -m pip install -r requirements.txt + - name: Run script + # run: bash bin/extract_all_tools.sh + run: | + bash ./bin/extract_all_tools_test.sh "${{ matrix.subset }}" + env: + GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }} + - name: Commit all tools + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git add results + git commit -m "fetch all tools bot" + git push + fetch-all-tools-merge: + runs-on: ubuntu-20.04 + needs: fetch-all-tools-stepwise + name: Fetch all tools merge + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + - name: Install requirement + run: python -m pip install -r requirements.txt + - name: Run script + run: | + bash ./bin/extract_all_tools_merge.sh + bash ./bin/extract_all_tools_downstream.sh + env: + GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }} + - name: Commit all tools + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git add results + git commit -m "fetch all tools bot" + git push \ No newline at end of file diff --git a/bin/extract_all_tools_test.sh b/bin/extract_all_tools_test.sh new file mode 100755 index 00000000..9960b9ea --- /dev/null +++ b/bin/extract_all_tools_test.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +mkdir -p 'results/' + +output="results/${1}_tools.tsv" + +python bin/extract_galaxy_tools.py \ + extractools \ + --api $GITHUB_API_KEY \ + --all_tools $output \ + --planemorepository $1 \ + --test + diff --git a/bin/extract_galaxy_tools.py b/bin/extract_galaxy_tools.py index 736fc37c..00303a32 100644 --- a/bin/extract_galaxy_tools.py +++ b/bin/extract_galaxy_tools.py @@ -50,14 +50,18 @@ def get_string_content(cf: ContentFile) -> str: return base64.b64decode(cf.content).decode("utf-8") -def get_tool_github_repositories(g: Github, RepoSelection: Optional[str]) -> List[str]: +def get_tool_github_repositories(g: Github, RepoSelection: Optional[str], run_test: bool) -> List[str]: """ Get list of tool GitHub repositories to parse :param g: GitHub instance :param RepoSelection: The selection to use from the repository (needed to split the process for CI jobs) + :run_test: for CI testing only use one repository """ + if run_test: + return ["https://github.com/TGAC/earlham-galaxytools"] + repo = g.get_user("galaxyproject").get_repo("planemo-monitor") repo_list: List[str] = [] for i in range(1, 5): @@ -408,6 +412,15 @@ def filter_tools(tools: List[Dict], ts_cat: List[str], excluded_tools: List[str] "--planemorepository", "-pr", required=False, help="Repository list to use from the planemo-monitor repository" ) + extractools.add_argument( + "--test", + "-t", + action="store_true", + default=False, + required=False, + help="Run a small test case using only the repository: https://github.com/TGAC/earlham-galaxytools", + ) + # Filter tools filtertools = subparser.add_parser("filtertools", help="Filter tools") filtertools.add_argument( @@ -430,7 +443,7 @@ def filter_tools(tools: List[Dict], ts_cat: List[str], excluded_tools: List[str] # connect to GitHub g = Github(args.api) # get list of GitHub repositories to parse - repo_list = get_tool_github_repositories(g, args.planemorepository) + repo_list = get_tool_github_repositories(g, args.planemorepository, args.test) # parse tools in GitHub repositories to extract metada, filter by TS categories and export to output file tools: List[Dict] = [] for r in repo_list: