Merge pull request #878 from mvdbeek/add_script_for_workflow_repos

Add script to add iwc workflow tools

Makefile

@@ -6,18 +6,18 @@ help:
 	@egrep '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[94m%-16s\033[0m %s\n", $$1, $$2}'
 
 lint: ## Lint all yaml files
-	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py
+	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix_lockfile.py
 	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 -I{} pykwalify -d '{}' -s .schema.yml
 
 fix: ## Fix all lockfiles and add any missing revisions
 	@# Generates the lockfile or updates it if it is missing tools
-	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py
+	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix_lockfile.py
 	@# --without says only add those hashes for those missing hashes (zB new tools)
-	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py --without
+	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update_tool.py --without
 
 fix-no-deps:
-	find ./$(TOOLSET) -name '*.yml'! -path .//.schema.yml  | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py --no-install-repository-dependencies --no-install-resolver-dependencies
-	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py --without
+	find ./$(TOOLSET) -name '*.yml'! -path .//.schema.yml  | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix_lockfile.py --no-install-repository-dependencies --no-install-resolver-dependencies
+	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update_tool.py --without
 
 #install:
 	#@echo "Installing any updated versions of $<"
@@ -26,9 +26,9 @@ fix-no-deps:
 
 update-trusted: ## Run the update script for a subset of repos
 	@# Missing --without, so this updates all tools in the file.
-	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py --owner $(OWNER)
+	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update_tool.py --owner $(OWNER)
 
 update-all: ## Run the update script for all repos
-	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py
+	find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update_tool.py
 
 .PHONY: lint update-trusted update-all help fix

README.md

@@ -76,7 +76,7 @@ Use `make TOOLSET=<toolset_dir> <target>` to limit a make action to a specific t
 
 ```console
 $ make TOOLSET=usegalaxy.org lint
-find ./usegalaxy.org -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py
+find ./usegalaxy.org -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix_lockfile.py
 find ./usegalaxy.org -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 -I{} pykwalify -d '{}' -s .schema.yml
  INFO - validation.valid
  INFO - validation.valid

scripts/get_iwc_tools.py

@@ -0,0 +1,80 @@
+import argparse
+import glob
+import json
+import os
+from collections import defaultdict
+
+import yaml
+
+from ephemeris.generate_tool_list_from_ga_workflow_files import (
+    generate_repo_list_from_workflow,
+)
+from steal_sections import steal_section
+from fix_lockfile import update_file as fix_lockfile
+from update_tool import update_file
+
+GALAXY_URL = "https://usegalaxy.eu"
+
+
+def find_workflows(workflow_path):
+    workflow_files = []
+    for dirpath, _, filenames in os.walk(workflow_path):
+        workflow_files.extend(
+            (
+                os.path.join(dirpath, filename)
+                for filename in filenames
+                if filename.endswith(".ga")
+            )
+        )
+    return workflow_files
+
+
+def add_repos(workflow_path, toolset, uncategorized_file):
+    workflow_paths = find_workflows(workflow_path)
+    repo_list = generate_repo_list_from_workflow(workflow_paths, "Uncategorized")
+    steal_section(
+        {"tools": repo_list},
+        toolset,
+        leftovers_file=os.path.join(toolset, uncategorized_file),
+        galaxy_url=GALAXY_URL,
+        verbose=True,
+    )
+    section_files = glob.glob(f"{toolset}/*.yml")
+    for section_file in section_files:
+        fix_lockfile(
+            section_file,
+            install_repository_dependencies=True,
+            install_resolver_dependencies=False,
+        )
+        update_file(section_file, without=True)
+    lock_files = glob.glob(f"{toolset}/*.yml.lock")
+    lock_file_contents = {}
+    repo_name_owner_entries = defaultdict(lambda: defaultdict(dict))
+    for lock_file in lock_files:
+        with open(lock_file) as lock_file_fh:
+            lock_contents = yaml.safe_load(lock_file_fh)
+            lock_file_contents[lock_file] = lock_contents
+            for repo in lock_contents["tools"]:
+                repo_name_owner_entries[repo["owner"]][repo["name"]] = repo
+    for workflow_repo in repo_list:
+        lock_file_entry = repo_name_owner_entries[workflow_repo["owner"]][
+            workflow_repo["name"]
+        ]
+        lock_file_entry["revisions"] = sorted(
+            list(set(lock_file_entry["revisions"] + workflow_repo["revisions"]))
+        )
+    for lock_file, entries in lock_file_contents.items():
+        with open(lock_file, "w") as lock_file_fh:
+            yaml.safe_dump(json.loads(json.dumps(entries)), stream=lock_file_fh)
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(description="")
+    parser.add_argument("-w", "--workflow-path", help="Path to directory with workflows")
+    parser.add_argument("-s", "--toolset", default="usegalaxy.org", help="The toolset dir to add versions to")
+    parser.add_argument("-u", "--uncategorized-file", default="leftovers.yaml", help="The file to store leftover (uninstalled) repos in.")
+
+    args = parser.parse_args()
+
+    add_repos(workflow_path=args.workflow_path, toolset=args.toolset, uncategorized_file=args.uncategorized_file)

scripts/steal_sections.py

@@ -4,41 +4,20 @@
 
 import yaml
 import glob
-import re
 import os
 import string
-import sys
 import argparse
 import requests
 
-def main():
-
-    VERSION = 0.1
-
-    parser = argparse.ArgumentParser(description="")
-    parser.add_argument("-t", "--tools", default="tools.yaml", help="Input tools.yaml")
-    parser.add_argument("-s", "--toolset", default="usegalaxy.org", help="The toolset dir to add versions to")
-    parser.add_argument("-l", "--leftovers-file", default="leftovers.yaml", help="The file to store leftover (unmatched) repos in.")
-    parser.add_argument("-g", "--galaxy-url", default="https://usegalaxy.eu", help="The Galaxy server to steal from")
-    parser.add_argument("--version", action='store_true')
-    parser.add_argument("--verbose", action='store_true')
-
-    args = parser.parse_args()
-
-    if args.version:
-        print("merge_versions.py version: %.1f" % VERSION)
-        return
 
-    tools_yaml = args.tools
-    toolset = args.toolset
-
-    section_files = glob.glob(os.path.join(args.toolset, "*.yml"))
+def steal_section(repo_dict, toolset: str, leftovers_file: str, galaxy_url: str, verbose: bool = False):
+    section_files = glob.glob(os.path.join(toolset, "*.yml"))
 
     other_tools = {}
     other_labels = {}
 
-    url = f"{args.galaxy_url}/api/tools?in_panel=false"
-    if args.verbose:
+    url = f"{galaxy_url}/api/tools?in_panel=false"
+    if verbose:
         print(f"Loading tools from: {url}")
     for tool in requests.get(url).json():
         if 'tool_shed_repository' not in tool:
@@ -55,22 +34,19 @@ def main():
     new = {}
 
     for section_file in section_files:
-        if args.verbose:
+        if verbose:
             print(f"Reading section file: {section_file}")
         a = yaml.safe_load(open(section_file, 'r'))
         tools = a['tools']
         for tool in tools:
             tool_key = (tool['name'], tool['owner'])
             existing[tool_key] = section_file
 
-    if args.verbose:
-        print(f"Processing input tools.yaml: {tools_yaml}")
-    a = yaml.safe_load(open(tools_yaml, 'r'))
-    tools = a['tools']
+    tools = repo_dict['tools']
     for tool in tools:
         tool_key = (tool['name'], tool['owner'])
         if tool_key in existing:
-            if args.verbose:
+            if verbose:
                 print(f"Skipping existing tool: {tool['owner']}/{tool['name']}")
             continue
         elif tool_key in other_tools:
@@ -87,20 +63,45 @@ def main():
         section_file = os.path.join(toolset, section + ".yml")
         if not os.path.exists(section_file):
             a = {'tool_panel_section_label': other_labels[section], 'tools': []}
-            if args.verbose:
+            if verbose:
                 print(f"Adding to new section file: {section_file}")
         else:
             a = yaml.safe_load(open(section_file, 'r'))
-            if args.verbose:
+            if verbose:
                 print(f"Adding to existing section file: {section_file}")
         tools = a['tools']
         tools.extend({"name": t[0], "owner": t[1]} for t in repos)
 
         with open(section_file, 'w') as out:
             yaml.dump(a, out, default_flow_style=False)
 
-    with open(args.leftovers_file, 'w') as out:
-        yaml.dump({'tools': leftover_tools}, out, default_flow_style=False)
+    if leftover_tools:
+        with open(leftovers_file, 'w') as out:
+            yaml.dump({'tools': leftover_tools}, out, default_flow_style=False)
+
+def main():
+
+    VERSION = 0.1
+
+    parser = argparse.ArgumentParser(description="")
+    parser.add_argument("-t", "--tools", default="tools.yaml", help="Input tools.yaml")
+    parser.add_argument("-s", "--toolset", default="usegalaxy.org", help="The toolset dir to add versions to")
+    parser.add_argument("-l", "--leftovers-file", default="leftovers.yaml", help="The file to store leftover (unmatched) repos in.")
+    parser.add_argument("-g", "--galaxy-url", default="https://usegalaxy.eu", help="The Galaxy server to steal from")
+    parser.add_argument("--version", action='store_true')
+    parser.add_argument("--verbose", action='store_true')
+
+    args = parser.parse_args()
+
+    if args.version:
+        print("merge_versions.py version: %.1f" % VERSION)
+        return
+
+    with open(args.tools) as fh:
+        repo_dict = yaml.safe_load(fh)
+    toolset = args.toolset
+    steal_section(repo_dict=repo_dict, toolset=toolset, leftovers_file=args.leftovers_file, galaxy_url=args.galaxy_url, verbose=args.verbose)
+
 
 if __name__ == "__main__":
     main()

scripts/to_sections.py

@@ -0,0 +1,35 @@
+import yaml
+import glob
+import os
+from collections import defaultdict
+from ruamel.yaml import YAML
+_yaml = YAML()
+
+ROOT_DIR = '../'
+IGNORE_REPOS = ('package_', 'suite_')
+toolset = 'usegalaxy-eu-tools'
+toolset_dir = os.path.join(ROOT_DIR, toolset)
+repos = defaultdict(set)
+repos = []
+for subset in glob.glob("{toolset_dir}/*.y*ml".format(toolset_dir=toolset_dir)):
+    with open(subset) as s:
+        loaded_repos = yaml.safe_load(s.read())['tools']
+        repos.extend(loaded_repos)
+repos
+missing_repos = yaml.safe_load(open('../missing_repos.yaml'))
+missing_repo_list_with_section = []
+for m_repo in missing_repos:
+    for r in repos:
+        if m_repo['name'] == r['name'] and m_repo['owner'] == r['owner']:
+            missing_repo_list_with_section.append(r)
+            break
+missing_repo_list_with_section
+missing_repo_list_with_section.sort(key=lambda x: x['tool_panel_section_label'])
+per_section = defaultdict(list)
+for repo in missing_repo_list_with_section:
+    label = repo['tool_panel_section_label']
+    del repo['tool_panel_section_label']
+    per_section[label].append(repo)
+d = {k: v for k, v in per_section.items()}
+_yaml.safe_dump(d)
+_yaml.dump(d, open('sections.yaml', 'w'))