From 4e92fcfa122ba8cdc1f29104a5b1faffad0f16d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Cavalcante?= Date: Fri, 29 Nov 2024 15:41:13 -0300 Subject: [PATCH] Update bug two (#19) * wip: Add bug two * refact: Try defaulting to a dict * fix: Dont allow remote to be unbound * refact: Check if not equal to component type * refact: Re-add isinstance dict * fix: Remove mention of component_name * refact: Add special case for strings following dicts * test: Try removing branch from test case * refact: Remove isinstance for str * refact: Use dict only with get_comps out * Revert "refact: Use dict only with get_comps out" This reverts commit 7ebc6dced3b69563a54761a559e24854db7dd1b4. * refact: Check if content has subwfs * refact: Always set current remote * refact: Check sha before resetting current_repo * refact: Do the negative test * refact: Use another variable for remote * refact: Simplify check * refact: Expand check once again * refact: Roll back to previous check * fix: self.sha must not be none to reset to none * Revert "fix: self.sha must not be none to reset to none" This reverts commit 65aa1c4a2e4b96a1548a3131acc2b0eb525e9259. * Try removing the section entirely * refact: Try moving check to reset * refact: Remove unnecessary current_sha * refact: Change check in get_comps * refact: Set git remote beforehand * refact: Change indent so previous check is the same * refact: Remove current repo check * Revert "refact: Remove current repo check" This reverts commit a956e0432cf1f5581c785727129050e0d81a1f80. * refact: Try using name list * fix: Remove break in loop * refactor: Always set current_repo * fix: Check if name is in updated * refactor: Remove unused sections and logs --- nf_core/components/install.py | 11 +++--- nf_core/components/update.py | 60 ++++++++++++++++++++++++------- tests/subworkflows/test_update.py | 4 +-- 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index a7e21395c7..f40b0712bf 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -58,10 +58,13 @@ def install(self, component: Union[str, Dict[str, str]], silent: bool = False) - self.modules_repo = ModulesRepo(remote_url, branch) component = component["name"] - if self.current_remote == self.modules_repo.remote_url and self.sha is not None: - self.current_sha = self.sha - else: - self.current_sha = None + if self.current_remote is None: + self.current_remote = self.modules_repo.remote_url + + if self.current_remote == self.modules_repo.remote_url and self.sha is not None: + self.current_sha = self.sha + else: + self.current_sha = None if self.repo_type == "modules": log.error(f"You cannot install a {component} in a clone of nf-core/modules") diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 3e4694adc8..b3c6dfc986 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -1,5 +1,6 @@ import logging import os +import re import shutil import tempfile from pathlib import Path @@ -41,6 +42,8 @@ def __init__( limit_output=False, ): super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) + self.current_remote = remote_url + self.branch = branch self.force = force self.prompt = prompt self.sha = sha @@ -92,6 +95,13 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr Returns: (bool): True if the update was successful, False otherwise. """ + if isinstance(component, dict): + # Override modules_repo when the component to install is a dependency from a subworkflow. + remote_url = component.get("git_remote", self.current_remote) + branch = component.get("branch", self.branch) + self.modules_repo = ModulesRepo(remote_url, branch) + component = component["name"] + self.component = component if updated is None: updated = [] @@ -868,7 +878,17 @@ def get_components_to_update(self, component): if self.component_type == "modules": # All subworkflow names in the installed_by section of a module are subworkflows using this module # We need to update them too - subworkflows_to_update = [subworkflow for subworkflow in installed_by if subworkflow != self.component_type] + git_remote = self.current_remote + for subworkflow in installed_by: + if subworkflow != component: + for remote_url, content in mods_json["repos"].items(): + if (all_subworkflows := content.get("subworkflows")) is not None: + for _, details in all_subworkflows.items(): + if subworkflow in details: + git_remote = remote_url + if subworkflow != self.component_type: + subworkflows_to_update.append({"name": subworkflow, "git_remote": git_remote}) + elif self.component_type == "subworkflows": for repo, repo_content in mods_json["repos"].items(): for component_type, dir_content in repo_content.items(): @@ -879,9 +899,9 @@ def get_components_to_update(self, component): # We need to update it too if component in comp_content["installed_by"]: if component_type == "modules": - modules_to_update.append(comp) + modules_to_update.append({"name": comp, "git_remote": repo, "org_path": dir}) elif component_type == "subworkflows": - subworkflows_to_update.append(comp) + subworkflows_to_update.append({"name": comp, "git_remote": repo, "org_path": dir}) return modules_to_update, subworkflows_to_update @@ -896,7 +916,7 @@ def update_linked_components( Update modules and subworkflows linked to the component being updated. """ for s_update in subworkflows_to_update: - if s_update in updated: + if s_update["name"] in updated: continue original_component_type, original_update_all = self._change_component_type("subworkflows") self.update( @@ -908,7 +928,7 @@ def update_linked_components( self._reset_component_type(original_component_type, original_update_all) for m_update in modules_to_update: - if m_update in updated: + if m_update["name"] in updated: continue original_component_type, original_update_all = self._change_component_type("modules") try: @@ -931,28 +951,42 @@ def update_linked_components( def manage_changes_in_linked_components(self, component, modules_to_update, subworkflows_to_update): """Check for linked components added or removed in the new subworkflow version""" if self.component_type == "subworkflows": - subworkflow_directory = Path(self.directory, self.component_type, self.modules_repo.repo_path, component) + org_path_match = re.search(r"(?:https://|git@)[\w\.]+[:/](.*?)/", self.current_remote) + if org_path_match: + org_path = org_path_match.group(1) + + subworkflow_directory = Path(self.directory, self.component_type, org_path, component) included_modules, included_subworkflows = get_components_to_install(subworkflow_directory) # If a module/subworkflow has been removed from the subworkflow for module in modules_to_update: - if module not in included_modules: + module = module["name"] + included_modules_names = [m["name"] for m in included_modules] + if module not in included_modules_names: log.info(f"Removing module '{module}' which is not included in '{component}' anymore.") remove_module_object = ComponentRemove("modules", self.directory) remove_module_object.remove(module, removed_by=component) for subworkflow in subworkflows_to_update: - if subworkflow not in included_subworkflows: + subworkflow = subworkflow["name"] + included_subworkflow_names = [m["name"] for m in included_subworkflows] + if subworkflow not in included_subworkflow_names: log.info(f"Removing subworkflow '{subworkflow}' which is not included in '{component}' anymore.") remove_subworkflow_object = ComponentRemove("subworkflows", self.directory) remove_subworkflow_object.remove(subworkflow, removed_by=component) # If a new module/subworkflow is included in the subworklfow and wasn't included before for module in included_modules: - if module not in modules_to_update: - log.info(f"Installing newly included module '{module}' for '{component}'") + module_name = module["name"] + module["git_remote"] = module.get("git_remote", self.current_remote) + module["branch"] = module.get("branch", self.branch) + if module_name not in modules_to_update: + log.info(f"Installing newly included module '{module_name}' for '{component}'") install_module_object = ComponentInstall(self.directory, "modules", installed_by=component) install_module_object.install(module, silent=True) for subworkflow in included_subworkflows: - if subworkflow not in subworkflows_to_update: - log.info(f"Installing newly included subworkflow '{subworkflow}' for '{component}'") + subworkflow_name = subworkflow["name"] + subworkflow["git_remote"] = subworkflow.get("git_remote", self.current_remote) + subworkflow["branch"] = subworkflow.get("branch", self.branch) + if subworkflow_name not in subworkflows_to_update: + log.info(f"Installing newly included subworkflow '{subworkflow_name}' for '{component}'") install_subworkflow_object = ComponentInstall( self.directory, "subworkflows", installed_by=component ) @@ -971,3 +1005,5 @@ def _reset_component_type(self, original_component_type, original_update_all): self.component_type = original_component_type self.modules_json.pipeline_components = None self.update_all = original_update_all + if self.current_remote is None: + self.current_remote = self.modules_repo.remote_url diff --git a/tests/subworkflows/test_update.py b/tests/subworkflows/test_update.py index 374acc71bf..7842900cdc 100644 --- a/tests/subworkflows/test_update.py +++ b/tests/subworkflows/test_update.py @@ -15,7 +15,7 @@ from nf_core.subworkflows.update import SubworkflowUpdate from ..test_subworkflows import TestSubworkflows -from ..utils import CROSS_ORGANIZATION_BRANCH, CROSS_ORGANIZATION_URL, OLD_SUBWORKFLOWS_SHA, cmp_component +from ..utils import CROSS_ORGANIZATION_URL, OLD_SUBWORKFLOWS_SHA, cmp_component class TestSubworkflowsUpdate(TestSubworkflows): @@ -379,7 +379,6 @@ def test_update_subworkflow_across_orgs(self): install_obj = SubworkflowInstall( self.pipeline_dir, remote_url=CROSS_ORGANIZATION_URL, - branch=CROSS_ORGANIZATION_BRANCH, sha="b7dc6a4fcfdf780c9228b3abf6bd821b466c2f81", ) # The fastq_trim_fastp_fastqc subworkflow contains the cross-org fastqc module, not the nf-core one @@ -388,7 +387,6 @@ def test_update_subworkflow_across_orgs(self): update_obj = SubworkflowUpdate( self.pipeline_dir, remote_url=CROSS_ORGANIZATION_URL, - branch=CROSS_ORGANIZATION_BRANCH, update_all=False, update_deps=True, show_diff=False,