diff --git a/Makefile b/Makefile index a6d96f11..6a2fc08f 100644 --- a/Makefile +++ b/Makefile @@ -6,18 +6,18 @@ help: @egrep '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[94m%-16s\033[0m %s\n", $$1, $$2}' lint: ## Lint all yaml files - find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py + find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix_lockfile.py find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 -I{} pykwalify -d '{}' -s .schema.yml fix: ## Fix all lockfiles and add any missing revisions @# Generates the lockfile or updates it if it is missing tools - find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py + find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix_lockfile.py @# --without says only add those hashes for those missing hashes (zB new tools) - find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py --without + find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update_tool.py --without fix-no-deps: - find ./$(TOOLSET) -name '*.yml'! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py --no-install-repository-dependencies --no-install-resolver-dependencies - find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py --without + find ./$(TOOLSET) -name '*.yml'! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix_lockfile.py --no-install-repository-dependencies --no-install-resolver-dependencies + find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update_tool.py --without #install: #@echo "Installing any updated versions of $<" @@ -26,9 +26,9 @@ fix-no-deps: update-trusted: ## Run the update script for a subset of repos @# Missing --without, so this updates all tools in the file. - find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py --owner $(OWNER) + find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update_tool.py --owner $(OWNER) update-all: ## Run the update script for all repos - find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update-tool.py + find ./$(TOOLSET) -name '*.yml' ! -path .//.schema.yml | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/update_tool.py .PHONY: lint update-trusted update-all help fix diff --git a/README.md b/README.md index 80a4d7a9..d76b7438 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Use `make TOOLSET= ` to limit a make action to a specific t ```console $ make TOOLSET=usegalaxy.org lint -find ./usegalaxy.org -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix-lockfile.py +find ./usegalaxy.org -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 python scripts/fix_lockfile.py find ./usegalaxy.org -name '*.yml' | grep '^\./[^/]*/' | xargs -n 1 -P 8 -I{} pykwalify -d '{}' -s .schema.yml INFO - validation.valid INFO - validation.valid diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/fix-lockfile.py b/scripts/fix_lockfile.py similarity index 100% rename from scripts/fix-lockfile.py rename to scripts/fix_lockfile.py diff --git a/scripts/get_iwc_tools.py b/scripts/get_iwc_tools.py new file mode 100644 index 00000000..1ed9f919 --- /dev/null +++ b/scripts/get_iwc_tools.py @@ -0,0 +1,80 @@ +import argparse +import glob +import json +import os +from collections import defaultdict + +import yaml + +from ephemeris.generate_tool_list_from_ga_workflow_files import ( + generate_repo_list_from_workflow, +) +from steal_sections import steal_section +from fix_lockfile import update_file as fix_lockfile +from update_tool import update_file + +GALAXY_URL = "https://usegalaxy.eu" + + +def find_workflows(workflow_path): + workflow_files = [] + for dirpath, _, filenames in os.walk(workflow_path): + workflow_files.extend( + ( + os.path.join(dirpath, filename) + for filename in filenames + if filename.endswith(".ga") + ) + ) + return workflow_files + + +def add_repos(workflow_path, toolset, uncategorized_file): + workflow_paths = find_workflows(workflow_path) + repo_list = generate_repo_list_from_workflow(workflow_paths, "Uncategorized") + steal_section( + {"tools": repo_list}, + toolset, + leftovers_file=os.path.join(toolset, uncategorized_file), + galaxy_url=GALAXY_URL, + verbose=True, + ) + section_files = glob.glob(f"{toolset}/*.yml") + for section_file in section_files: + fix_lockfile( + section_file, + install_repository_dependencies=True, + install_resolver_dependencies=False, + ) + update_file(section_file, without=True) + lock_files = glob.glob(f"{toolset}/*.yml.lock") + lock_file_contents = {} + repo_name_owner_entries = defaultdict(lambda: defaultdict(dict)) + for lock_file in lock_files: + with open(lock_file) as lock_file_fh: + lock_contents = yaml.safe_load(lock_file_fh) + lock_file_contents[lock_file] = lock_contents + for repo in lock_contents["tools"]: + repo_name_owner_entries[repo["owner"]][repo["name"]] = repo + for workflow_repo in repo_list: + lock_file_entry = repo_name_owner_entries[workflow_repo["owner"]][ + workflow_repo["name"] + ] + lock_file_entry["revisions"] = sorted( + list(set(lock_file_entry["revisions"] + workflow_repo["revisions"])) + ) + for lock_file, entries in lock_file_contents.items(): + with open(lock_file, "w") as lock_file_fh: + yaml.safe_dump(json.loads(json.dumps(entries)), stream=lock_file_fh) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="") + parser.add_argument("-w", "--workflow-path", help="Path to directory with workflows") + parser.add_argument("-s", "--toolset", default="usegalaxy.org", help="The toolset dir to add versions to") + parser.add_argument("-u", "--uncategorized-file", default="leftovers.yaml", help="The file to store leftover (uninstalled) repos in.") + + args = parser.parse_args() + + add_repos(workflow_path=args.workflow_path, toolset=args.toolset, uncategorized_file=args.uncategorized_file) diff --git a/scripts/steal_sections.py b/scripts/steal_sections.py index 331c9757..507a5689 100755 --- a/scripts/steal_sections.py +++ b/scripts/steal_sections.py @@ -4,41 +4,20 @@ import yaml import glob -import re import os import string -import sys import argparse import requests -def main(): - - VERSION = 0.1 - - parser = argparse.ArgumentParser(description="") - parser.add_argument("-t", "--tools", default="tools.yaml", help="Input tools.yaml") - parser.add_argument("-s", "--toolset", default="usegalaxy.org", help="The toolset dir to add versions to") - parser.add_argument("-l", "--leftovers-file", default="leftovers.yaml", help="The file to store leftover (unmatched) repos in.") - parser.add_argument("-g", "--galaxy-url", default="https://usegalaxy.eu", help="The Galaxy server to steal from") - parser.add_argument("--version", action='store_true') - parser.add_argument("--verbose", action='store_true') - - args = parser.parse_args() - - if args.version: - print("merge_versions.py version: %.1f" % VERSION) - return - tools_yaml = args.tools - toolset = args.toolset - - section_files = glob.glob(os.path.join(args.toolset, "*.yml")) +def steal_section(repo_dict, toolset: str, leftovers_file: str, galaxy_url: str, verbose: bool = False): + section_files = glob.glob(os.path.join(toolset, "*.yml")) other_tools = {} other_labels = {} - url = f"{args.galaxy_url}/api/tools?in_panel=false" - if args.verbose: + url = f"{galaxy_url}/api/tools?in_panel=false" + if verbose: print(f"Loading tools from: {url}") for tool in requests.get(url).json(): if 'tool_shed_repository' not in tool: @@ -55,7 +34,7 @@ def main(): new = {} for section_file in section_files: - if args.verbose: + if verbose: print(f"Reading section file: {section_file}") a = yaml.safe_load(open(section_file, 'r')) tools = a['tools'] @@ -63,14 +42,11 @@ def main(): tool_key = (tool['name'], tool['owner']) existing[tool_key] = section_file - if args.verbose: - print(f"Processing input tools.yaml: {tools_yaml}") - a = yaml.safe_load(open(tools_yaml, 'r')) - tools = a['tools'] + tools = repo_dict['tools'] for tool in tools: tool_key = (tool['name'], tool['owner']) if tool_key in existing: - if args.verbose: + if verbose: print(f"Skipping existing tool: {tool['owner']}/{tool['name']}") continue elif tool_key in other_tools: @@ -87,11 +63,11 @@ def main(): section_file = os.path.join(toolset, section + ".yml") if not os.path.exists(section_file): a = {'tool_panel_section_label': other_labels[section], 'tools': []} - if args.verbose: + if verbose: print(f"Adding to new section file: {section_file}") else: a = yaml.safe_load(open(section_file, 'r')) - if args.verbose: + if verbose: print(f"Adding to existing section file: {section_file}") tools = a['tools'] tools.extend({"name": t[0], "owner": t[1]} for t in repos) @@ -99,8 +75,33 @@ def main(): with open(section_file, 'w') as out: yaml.dump(a, out, default_flow_style=False) - with open(args.leftovers_file, 'w') as out: - yaml.dump({'tools': leftover_tools}, out, default_flow_style=False) + if leftover_tools: + with open(leftovers_file, 'w') as out: + yaml.dump({'tools': leftover_tools}, out, default_flow_style=False) + +def main(): + + VERSION = 0.1 + + parser = argparse.ArgumentParser(description="") + parser.add_argument("-t", "--tools", default="tools.yaml", help="Input tools.yaml") + parser.add_argument("-s", "--toolset", default="usegalaxy.org", help="The toolset dir to add versions to") + parser.add_argument("-l", "--leftovers-file", default="leftovers.yaml", help="The file to store leftover (unmatched) repos in.") + parser.add_argument("-g", "--galaxy-url", default="https://usegalaxy.eu", help="The Galaxy server to steal from") + parser.add_argument("--version", action='store_true') + parser.add_argument("--verbose", action='store_true') + + args = parser.parse_args() + + if args.version: + print("merge_versions.py version: %.1f" % VERSION) + return + + with open(args.tools) as fh: + repo_dict = yaml.safe_load(fh) + toolset = args.toolset + steal_section(repo_dict=repo_dict, toolset=toolset, leftovers_file=args.leftovers_file, galaxy_url=args.galaxy_url, verbose=args.verbose) + if __name__ == "__main__": main() diff --git a/scripts/to_sections.py b/scripts/to_sections.py new file mode 100644 index 00000000..e4596949 --- /dev/null +++ b/scripts/to_sections.py @@ -0,0 +1,35 @@ +import yaml +import glob +import os +from collections import defaultdict +from ruamel.yaml import YAML +_yaml = YAML() + +ROOT_DIR = '../' +IGNORE_REPOS = ('package_', 'suite_') +toolset = 'usegalaxy-eu-tools' +toolset_dir = os.path.join(ROOT_DIR, toolset) +repos = defaultdict(set) +repos = [] +for subset in glob.glob("{toolset_dir}/*.y*ml".format(toolset_dir=toolset_dir)): + with open(subset) as s: + loaded_repos = yaml.safe_load(s.read())['tools'] + repos.extend(loaded_repos) +repos +missing_repos = yaml.safe_load(open('../missing_repos.yaml')) +missing_repo_list_with_section = [] +for m_repo in missing_repos: + for r in repos: + if m_repo['name'] == r['name'] and m_repo['owner'] == r['owner']: + missing_repo_list_with_section.append(r) + break +missing_repo_list_with_section +missing_repo_list_with_section.sort(key=lambda x: x['tool_panel_section_label']) +per_section = defaultdict(list) +for repo in missing_repo_list_with_section: + label = repo['tool_panel_section_label'] + del repo['tool_panel_section_label'] + per_section[label].append(repo) +d = {k: v for k, v in per_section.items()} +_yaml.safe_dump(d) +_yaml.dump(d, open('sections.yaml', 'w')) diff --git a/scripts/update-tool.py b/scripts/update_tool.py similarity index 100% rename from scripts/update-tool.py rename to scripts/update_tool.py