Rename script argument, merge bash scripts

galaxyproject · Jul 8, 2024 · b504192 · b504192
1 parent 94f78b2
commit b504192
Show file tree

Hide file tree

Showing 13 changed files with 269 additions and 232 deletions.
diff --git a/.github/workflows/fetch_all_tools.yaml b/.github/workflows/fetch_all_tools.yaml
@@ -26,7 +26,7 @@ jobs:
         run: python -m pip install -r requirements.txt
       - name: Run script 
         run: | 
-          python ./bin/get_public_galaxy_servers.py -o data/available_public_servers.csv
+          python bin/get_public_galaxy_servers.py -o data/available_public_servers.csv
       - name: Commit servers 
         # add or commit any changes in results if there was a change, merge with main, and push as bot
         run: |
@@ -59,7 +59,7 @@ jobs:
         run: python -m pip install -r requirements.txt
       - name: Run script #needs PAT to access other repos
         run: | 
-          bash ./bin/extract_all_tools_stepwise.sh "${{ matrix.subset }}"
+          bash bin/extract_all_tools.sh "${{ matrix.subset }}"
         env: 
           GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }}
       - name: Commit all tools 
@@ -92,7 +92,7 @@ jobs:
           jq -s 'map(.[])' results/repositories*.list_tools.json > results/all_tools.json 
       - name: Wordcloud and interactive table
         run: |
-          bash ./bin/extract_all_tools_downstream.sh
+          bash bin/format_tools.sh
       - name: Commit all tools
         # add or commit any changes in results if there was a change, merge with main and push as bot
         run: |

diff --git a/.github/workflows/fetch_all_tutorials.yaml b/.github/workflows/fetch_all_tutorials.yaml
@@ -29,7 +29,7 @@ jobs:
         run: python -m pip install -r requirements.txt
       - name: Run script #needs PAT to access other repos
         run: | 
-          bash ./bin/extract_all_tutorials.sh
+          bash bin/extract_all_tutorials.sh
         env: 
           PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }}
       - name: Commit all tools 

diff --git a/.github/workflows/filter_communities.yaml b/.github/workflows/filter_communities.yaml
@@ -36,7 +36,7 @@ jobs:
       run: python -m pip install -r requirements.txt
     - name: Run script
       run: | 
-        bash ./bin/get_community_tutorials.sh
+        bash bin/get_community_tutorials.sh
     - name: Commit results
       # commit the new filtered data, only if stuff was changed
       run: |
@@ -59,7 +59,7 @@ jobs:
       run: python -m pip install -r requirements.txt
     - name: Run script
       run: | 
-        bash ./bin/update_tools_to_keep_exclude.sh
+        bash bin/update_tools_to_keep_exclude.sh
     - name: Commit results
       # commit the new filtered data, only if stuff was changed
       run: |
@@ -82,7 +82,7 @@ jobs:
       run: python -m pip install -r requirements.txt
     - name: Run script
       run: | 
-        bash ./bin/get_community_tools.sh
+        bash bin/get_community_tools.sh
     - name: Commit results
       # commit the new filtered data, only if stuff was changed
       run: |

diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
@@ -13,39 +13,16 @@ jobs:
       - name: Install requirement
         run: python -m pip install -r requirements.txt
       - name: Tool extraction
-        # run: bash bin/extract_all_tools.sh
         run: | 
-          python bin/extract_galaxy_tools.py \
-            extractools \
-            --api $GITHUB_API_KEY \
-            --all-tools "results/test_tools.tsv" \
-            --all-tools-json "results/test_tools.json" \
-            --planemo-repository-list "test.list" \
-            --test
+          bash bin/extract_all_tools.sh test 
         env: 
           GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }}
       - name: Tool filter
         run: | 
-          python bin/extract_galaxy_tools.py \
-            filtertools \
-            --tools "results/all_tools.json" \
-            --ts-filtered-tools "results/microgalaxy/tools_filtered_by_ts_categories.tsv" \
-            --filtered-tools "results/microgalaxy/tools.tsv" \
-            --categories "data/communities/microgalaxy/categories" \
-            --status "data/communities/microgalaxy/tool_status.tsv"
-      - name: Create interactive table
+          bash bin/get_community_tools.sh test
+      - name: Create interactive table and wordcloud
         run: | 
-          python bin/create_interactive_table.py \
-            --table "results/microgalaxy/tools.tsv" \
-            --template "data/interactive_table_template.html" \
-            --output "results/microgalaxy/index.html"
-      - name: Create wordcloud
-        run: | 
-          python bin/create_wordcloud.py \
-            --table  "results/microgalaxy/tools.tsv" \
-            --wordcloud_mask "data/usage_stats/wordcloud_mask.png" \
-            --output "results/microgalaxy/tools_wordcloud.png" \
-            --stats_column "No. of tool users (2022-2023) (usegalaxy.eu)"
+          bash bin/format_tools.sh
   test-tutorials:
     runs-on: ubuntu-20.04
     steps:
@@ -57,18 +34,9 @@ jobs:
         run: python -m pip install -r requirements.txt
       - name: Tutorial extraction
         run: | 
-          python bin/extract_gtn_tutorials.py \
-            extracttutorials \
-            --all_tutorials "results/test_tutorials.json" \
-            --tools "results/all_tools.json" \
-            --api $PLAUSIBLE_API_KEY \
-            --test
+          bash bin/extract_all_tutorials.sh test 
         env: 
           PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }}
       - name: Tutorial filtering
         run: | 
-          python bin/extract_gtn_tutorials.py \
-            filtertutorials \
-            --all_tutorials "results/test_tutorials.json" \
-            --filtered_tutorials "results/microgalaxy/test_tutorials.tsv" \
-            --tags "data/communities/microgalaxy/tutorial_tags"
+          bash bin/get_community_tutorials.sh test
diff --git a/README.md b/README.md
@@ -113,9 +113,10 @@ The script will generate a TSV file with each tool found in the list of GitHub r
 
     ```
     $ python bin/extract_galaxy_tools.py \
-        --tools <Path to JSON file with all extracted tools> \
-        --ts-filtered-tools <Path to output TSV with tools filtered based on ToolShed category>
-        --filtered-tools <Path to output TSV with filtered tools based on ToolShed category and manual curation> \
+        filter \
+        --all <Path to JSON file with all extracted tools> \
+        --ts-filtered <Path to output TSV with tools filtered based on ToolShed category>
+        --filtered <Path to output TSV with filtered tools based on ToolShed category and manual curation> \
         [--categories <Path to ToolShed category file>] \
         [--status <Path to a TSV file with tool status - 3 columns: ToolShed ids of tool suites, Boolean with True to keep and False to exclude, Boolean with True if deprecated and False if not>]
     ```
@@ -145,18 +146,49 @@ The script will generate a TSV file with each tool found in the list of GitHub r
 
     ```
     $ python bin/extract_gtn_tutorials.py \
-        filtertutorials \
-        --all_tutorials "results/all_tutorials.json" \
-        --filtered_tutorials "results/<your community>/tutorials.tsv" \
+        filter\
+        --all "results/all_tutorials.json" \
+        --filtered "results/<your community>/tutorials.tsv" \
         --tags "data/communities/<your community>/tutorial_tags"
     ```
 
 ## Development
 
+### Tools
+
 To make a test run of the tool to check its functionalities follow [Usage](#Usage) to set-up the environnement and the API key, then run
 
-```bash
-bash ./bin/extract_all_tools_test.sh test.list
-```
+1. Tool extraction
+
+    ```bash
+    $ bash bin/extract_all_tools.sh test
+    ```
+
+    This runs the tool, but only parses the test repository [Galaxy-Tool-Metadata-Extractor-Test-Wrapper](https://github.com/paulzierep/Galaxy-Tool-Metadata-Extractor-Test-Wrapper)
+
+2. Tool filter
+
+    ```bash
+    $ bash bin/get_community_tools.sh test
+    ```
+
+3. Create interactive table and wordcloud
+        
+    ```bash
+    $ bash bin/format_tools.sh
+    ```
+
+### Tutorials
+
+1. Tutorial extraction
+
+    ```bash
+    $ bash bin/extract_all_tutorials.sh test 
+    ```
+
+2. Tutorial filtering
+
+    ```bash
+    $ bash bin/get_community_tutorials.sh test
+    ```
 
-This runs the tool, but only parses the test repository [Galaxy-Tool-Metadata-Extractor-Test-Wrapper](https://github.com/paulzierep/Galaxy-Tool-Metadata-Extractor-Test-Wrapper)
diff --git a/bin/extract_all_tools.sh b/bin/extract_all_tools.sh
@@ -2,13 +2,45 @@
 
 mkdir -p 'results/'
 
-python bin/extract_galaxy_tools.py \
-        extractools \
-        --api $GITHUB_API_KEY \
-        --all-tools 'results/all_tools.tsv' \
-        --all-tools-json 'results/all_tools.json'
-
-python bin/create_interactive_table.py \
-        --table "results/all_tools.tsv" \
-        --template "data/interactive_table_template.html" \
-        --output "results/index.html"
+if [ ! -z $1 ] 
+then 
+        if [ $1=="test" ]
+        then 
+                echo "Test tool extraction"
+                python bin/extract_galaxy_tools.py \
+                        extract \
+                        --api $GITHUB_API_KEY \
+                        --all-tsv "results/test_tools.tsv" \
+                        --all "results/test_tools.json" \
+                        --planemo-repository-list "test.list" \
+                        --test
+        else
+                tsv_output="results/${1}_tools.tsv"
+                json_output="results/${1}_tools.json"
+
+                if [[ $1 =~ "01" ]]; then
+                python bin/extract_galaxy_tools.py \
+                        extract \
+                        --api $GITHUB_API_KEY \
+                        --all-tsv $tsv_output \
+                        --all $json_output \
+                        --planemo-repository-list $1
+                else
+                python bin/extract_galaxy_tools.py \
+                        extract \
+                        --api $GITHUB_API_KEY \
+                        --all-tsv $tsv_output \
+                        --all $json_output \
+                        --planemo-repository-list $1 \
+                        --avoid-extra-repositories
+                fi
+        fi
+else
+        echo "Tool extraction";
+        python bin/extract_galaxy_tools.py \
+                extract \
+                --api $GITHUB_API_KEY \
+                --all-tsv 'results/all_tools.tsv' \
+                --all 'results/all_tools.json'
+fi
+
diff --git a/bin/extract_all_tools_stepwise.sh b/bin/extract_all_tools_stepwise.sh
diff --git a/bin/extract_all_tutorials.sh b/bin/extract_all_tutorials.sh
@@ -1,7 +1,17 @@
 #!/usr/bin/env bash
 
-python bin/extract_gtn_tutorials.py \
-        extracttutorials \
-        --all_tutorials "results/all_tutorials.json" \
-        --tools "results/all_tools.json" \
-        --api $PLAUSIBLE_API_KEY
+if [ ! -z $1 ] 
+then
+        python bin/extract_gtn_tutorials.py \
+                extract \
+                --all "results/test_tutorials.json" \
+                --tools "results/all_tools.json" \
+                --api $PLAUSIBLE_API_KEY \
+                --test
+else
+        python bin/extract_gtn_tutorials.py \
+                extract \
+                --all "results/all_tutorials.json" \
+                --tools "results/all_tools.json" \
+                --api $PLAUSIBLE_API_KEY
+fi