diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 445680bdd..672eb7181 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -33,6 +33,7 @@ jobs: tox_env: [py37] galaxy_version: - dev + - release_23.1 - release_23.0 - release_22.05 - release_22.01 diff --git a/bioblend/_tests/TestGalaxyToolContainerResolution.py b/bioblend/_tests/TestGalaxyToolContainerResolution.py new file mode 100644 index 000000000..6f2c435de --- /dev/null +++ b/bioblend/_tests/TestGalaxyToolContainerResolution.py @@ -0,0 +1,96 @@ +""" +Test functions in bioblend.galaxy.tool_dependencies +""" +from . import ( + GalaxyTestBase, + test_util, +) + + +class TestGalaxyContainerResolution(GalaxyTestBase.GalaxyTestBase): + @test_util.skip_unless_galaxy("release_22.05") + def test_get_container_resolvers(self): + container_resolvers = self.gi.container_resolution.get_container_resolvers() + assert isinstance(container_resolvers, list) + assert len(container_resolvers) > 0 + assert isinstance(container_resolvers[0], dict) + assert container_resolvers[0]["model_class"] == "ExplicitContainerResolver" + assert container_resolvers[0]["resolver_type"] == "explicit" + assert container_resolvers[0]["can_uninstall_dependencies"] is False + assert container_resolvers[0]["builds_on_resolution"] is False + + @test_util.skip_unless_galaxy("release_22.05") + def test_show_container_resolver(self): + container_resolver = self.gi.container_resolution.show_container_resolver(0) + print(container_resolver) + assert isinstance(container_resolver, dict) + assert container_resolver["model_class"] == "ExplicitContainerResolver" + assert container_resolver["resolver_type"] == "explicit" + assert container_resolver["can_uninstall_dependencies"] is False + assert container_resolver["builds_on_resolution"] is False + + @test_util.skip_unless_galaxy("release_22.05") + def test_resolve(self): + tool = self.gi.container_resolution.resolve(tool_id="CONVERTER_parquet_to_csv") + print(tool) + assert isinstance(tool, dict) + + tool_requirements_only = self.gi.container_resolution.resolve( + tool_id="CONVERTER_parquet_to_csv", requirements_only=True + ) + assert isinstance(tool_requirements_only, dict) + + @test_util.skip_unless_galaxy("release_22.05") + def test_resolve_toolbox(self): + toolbox = self.gi.container_resolution.resolve_toolbox() + assert isinstance(toolbox, list) + assert len(toolbox) > 0 + assert isinstance(toolbox[0], dict) + + toolbox_by_tool_ids = self.gi.container_resolution.resolve_toolbox(tool_ids=[toolbox[0]["tool_id"]]) + assert isinstance(toolbox_by_tool_ids, list) + assert len(toolbox_by_tool_ids) == 1 + assert isinstance(toolbox_by_tool_ids[0], dict) + + toolbox_by_resolver_type = self.gi.container_resolution.resolve_toolbox(resolver_type="mulled") + assert isinstance(toolbox_by_resolver_type, list) + assert len(toolbox_by_resolver_type) > 0 + assert isinstance(toolbox_by_resolver_type[0], dict) + assert len(toolbox) == len(toolbox_by_resolver_type) + for tool in toolbox_by_resolver_type: + print(tool) + assert ( + tool["status"]["dependency_type"] is None + or tool["status"]["container_resolver"]["resolver_type"] == "mulled" + ) + + toolbox_by_container_type = self.gi.container_resolution.resolve_toolbox(container_type="docker") + assert isinstance(toolbox_by_container_type, list) + assert len(toolbox_by_container_type) > 0 + assert isinstance(toolbox_by_container_type[0], dict) + assert len(toolbox) == len(toolbox_by_container_type) + for tool in toolbox_by_container_type: + assert tool["status"]["dependency_type"] is None or tool["status"]["dependency_type"] == "docker" + assert ( + tool["status"]["dependency_type"] is None or tool["status"]["container_description"]["type"] == "docker" + ) + + toolbox_requirements_only = self.gi.container_resolution.resolve_toolbox(requirements_only=True) + assert isinstance(toolbox_requirements_only, list) + assert len(toolbox_requirements_only) > 0 + assert isinstance(toolbox_requirements_only[0], dict) + assert len(toolbox) == len(toolbox_requirements_only) + + # TODO unless containers are available this may fallback to conda by default? + # depending on Galaxy's config + # toolbox_by_index = self.gi.container_resolution.resolve_toolbox(tool_ids=[toolbox[0]['tool_id']], index=0, install=True) + # assert isinstance(toolbox_by_index, list) + # assert len(toolbox_by_index) > 0 + # assert isinstance(toolbox_by_index[0], dict) + + # TODO unless containers are available this may fallback to conda by default? + # depending on Galaxy's config + # def test_resolve_toolbox_with_install(self): + # toolbox = self.gi.container_resolution.resolve_toolbox_with_install(tool_ids=[]) + # assert isinstance(toolbox, list) + # assert len(toolbox) == 0 diff --git a/bioblend/galaxy/__init__.py b/bioblend/galaxy/__init__.py index 35d3db48c..7fb780e02 100644 --- a/bioblend/galaxy/__init__.py +++ b/bioblend/galaxy/__init__.py @@ -5,6 +5,7 @@ from bioblend.galaxy import ( config, + container_resolution, dataset_collections, datasets, datatypes, @@ -94,6 +95,7 @@ def __init__( self.toolshed = toolshed.ToolShedClient(self) self.toolShed = self.toolshed # historical alias self.config = config.ConfigClient(self) + self.container_resolution = container_resolution.ContainerResolutionClient(self) self.visual = visual.VisualClient(self) self.quotas = quotas.QuotaClient(self) self.groups = groups.GroupsClient(self) diff --git a/bioblend/galaxy/container_resolution/__init__.py b/bioblend/galaxy/container_resolution/__init__.py new file mode 100644 index 000000000..c39d81eba --- /dev/null +++ b/bioblend/galaxy/container_resolution/__init__.py @@ -0,0 +1,238 @@ +""" +Contains interactions dealing with Galaxy container resolvers. +Works only with Galaxy > 22.01 +""" +from typing import ( + List, + Optional, +) + +from bioblend.galaxy.client import Client + + +class ContainerResolutionClient(Client): + module = "container_resolvers" + + def get_container_resolvers(self) -> list: + """ + List container resolvers + + :rtype: list + return: List of container resolvers + + For example: + [{'builds_on_resolution': False, + 'can_uninstall_dependencies': False, + 'model_class': 'CachedExplicitSingularityContainerResolver', + 'resolver_type': 'cached_explicit_singularity'}, + {'builds_on_resolution': False, + 'can_uninstall_dependencies': False, + 'model_class': 'CachedMulledSingularityContainerResolver', + 'resolver_type': 'cached_mulled_singularity'}, + {'builds_on_resolution': False, + 'can_uninstall_dependencies': False, + 'model_class': 'MulledSingularityContainerResolver', + 'resolver_type': 'mulled_singularity'}] {'builds_on_resolution': False, + """ + url = self._make_url() + return self._get(url=url) + + def show_container_resolver(self, index: int) -> dict: + """ + Show container resolver + + :type index: int + :param index: index of the dependency resolver with respect to + the dependency resolvers config file + + :rtype: dict + return: Dict of properties of a given container resolver + + {'builds_on_resolution': False, + 'can_uninstall_dependencies': False, + 'model_class': 'CachedMulledSingularityContainerResolver', + 'resolver_type': 'cached_mulled_singularity'} + """ + url = f"{self._make_url()}/{index}" + return self._get(url=url) + + def resolve( + self, + tool_id: str, + index: Optional[int] = None, + resolver_type: Optional[str] = None, + container_type: Optional[str] = None, + requirements_only: bool = False, + install: bool = False, + ) -> dict: + """ + Resolve described requirement against specified container resolvers. + + :type index: int + :param index: index of the dependency resolver with respect to + the dependency resolvers config file + + :type tool_id: str + :param tool_id: tool id to resolve against containers + + :type resolver_type: str + :param resolver_type: restrict to specified resolver type + + :type container_type: str + :param container_type: restrict to specified container type + + :type requirements_only: bool + :param requirements_only: ignore tool containers, properties - just search based on tool requirements set to True to mimic default behavior of tool dependency API. + + :type install: bool + :param install: allow installation of new containers (for build_mulled containers) the way job resolution will operate, defaults to False + + :rtype: dict + + For example: + { + 'requirements': [{'name': 'pyarrow', 'specs': [], 'type': 'package', 'version': '4.0.1'}], + 'status': { + 'cacheable': False, + 'container_description': {'identifier': 'quay.io/biocontainers/pyarrow:4.0.1', 'resolve_dependencies': False, 'shell': '/bin/bash', 'type': 'docker'}, + 'container_resolver': {'builds_on_resolution': False, 'can_uninstall_dependencies': False, 'model_class': 'MulledDockerContainerResolver', 'resolver_type': 'mulled'}, + 'dependency_type': 'docker', + ... + }, + 'tool_id': 'CONVERTER_parquet_to_csv' + } + """ + params = {} + if tool_id: + params["tool_id"] = tool_id + if resolver_type: + params["resolver_type"] = resolver_type + if container_type: + params["container_type"] = container_type + params["requirements_only"] = str(requirements_only) + params["install"] = str(install) + if index is not None: + url = "/".join((self._make_url(), str(index), "resolve")) + else: + url = "/".join((self._make_url(), "resolve")) + return self._get(url=url, params=params) + + def resolve_toolbox( + self, + index: Optional[int] = None, + tool_ids: Optional[List[str]] = None, + resolver_type: Optional[str] = None, + container_type: Optional[str] = None, + requirements_only: bool = False, + install: bool = False, + ) -> list: + """ + Apply resolve() to each tool in the toolbox and return the results as a list. + See documentation for resolve() for a description of parameters that can be + consumed and a description of the resulting items. + + :type index: int + :param index: index of the dependency resolver with respect to + the dependency resolvers config file + + :type tool_ids: list + :param tool_ids: tool_ids to filter toolbox on + + :type resolver_type: str + :param resolver_type: restrict to specified resolver type + + :type container_type: str + :param container_type: restrict to specified container type + + :type requirements_only: bool + :param requirements_only: ignore tool containers, properties - just search based on tool requirements set to True to mimic default behavior of tool dependency API. + + :type install: bool + :param install: allow installation of new containers (for build_mulled containers) the way job resolution will operate, defaults to False + + :rtype: list + For example:: + [{'tool_id': 'upload1', 'status': {'model_class': 'NullDependency', 'dependency_type': None, 'exact': True, 'name': None, 'version': None, 'cacheable': False}, 'requirements': []}, ...] + """ + params = {} + if tool_ids: + params["tool_ids"] = ",".join(tool_ids) + if resolver_type: + params["resolver_type"] = resolver_type + if container_type: + params["container_type"] = container_type + params["requirements_only"] = str(requirements_only) + params["install"] = str(install) + if index is not None: + url = "/".join((self._make_url(), str(index), "toolbox")) + else: + url = "/".join((self._make_url(), "toolbox")) + return self._get(url=url, params=params) + + def resolve_toolbox_with_install( + self, + index: Optional[int] = None, + tool_ids: Optional[List[str]] = None, + resolver_type: Optional[str] = None, + container_type: Optional[str] = None, + requirements_only: bool = False, + ) -> list: + """ + Do the resolution of dependencies like resolve_toolbox(), but allow building and installing new containers. + + :type index: int + :param index: index of the dependency resolver with respect to + the dependency resolvers config file + + :type tool_ids: list + :param tool_ids: tool_ids to filter toolbox on + + :type resolver_type: str + :param resolver_type: restrict to specified resolver type + + :type container_type: str + :param container_type: restrict to specified container type + + :type requirements_only: bool + :param requirements_only: ignore tool containers, properties - just search based on tool requirements set to True to mimic default behavior of tool dependency API. + + + :rtype: list of dicts + :returns: dictified descriptions of the dependencies, with attribute + `dependency_type: None` if no match was found. + For example:: + + [{'requirements': [{'name': 'canu', + 'specs': [], + 'type': 'package', + 'version': '2.2'}], + 'status': {'cacheable': False, + 'container_description': {'identifier': 'docker://quay.io/biocontainers/canu:2.2--ha47f30e_0', + 'resolve_dependencies': False, + 'shell': '/bin/bash', + 'type': 'singularity'}, + 'container_resolver': {'builds_on_resolution': False, + 'can_uninstall_dependencies': False, + 'model_class': 'MulledSingularityContainerResolver', + 'resolver_type': 'mulled_singularity'}, + 'dependency_type': 'singularity', + 'environment_path': 'docker://quay.io/biocontainers/canu:2.2--ha47f30e_0', + 'exact': True, + 'model_class': 'ContainerDependency', + 'name': None, + 'version': None}, + 'tool_id': 'toolshed.g2.bx.psu.edu/repos/bgruening/canu/canu/2.2+galaxy0'}] + """ + params = {} + if tool_ids: + params["tool_ids"] = ",".join(tool_ids) + if resolver_type: + params["resolver_type"] = resolver_type + if container_type: + params["container_type"] = container_type + params["requirements_only"] = str(requirements_only) + if index is not None: + url = "/".join((self._make_url(), str(index), "toolbox", "install")) + else: + url = "/".join((self._make_url(), "toolbox", "install")) + return self._post(url=url, payload=params) diff --git a/bioblend/galaxy/tools/__init__.py b/bioblend/galaxy/tools/__init__.py index cdafb4f64..b2abf83a7 100644 --- a/bioblend/galaxy/tools/__init__.py +++ b/bioblend/galaxy/tools/__init__.py @@ -333,6 +333,7 @@ def run_tool( tool_id: str, tool_inputs: Union[InputsBuilder, dict], input_format: Literal["21.01", "legacy"] = "legacy", + data_manager_mode: Optional[Literal["populate", "dry_run", "bundle"]] = None, ) -> Dict[str, Any]: """ Runs tool specified by ``tool_id`` in history indicated @@ -344,6 +345,16 @@ def run_tool( :type tool_id: str :param tool_id: ID of the tool to be run + :type data_manager_mode: str + :param data_manager_mode: Possible values are 'populate', 'dry_run' and 'bundle'. + + 'populate' is the default behavior for data manager tools and results in tool data table + files being updated after the data manager job completes. + + 'dry_run' will skip any processing after the data manager job completes + + 'bundle' will create a data manager bundle that can be imported on other Galaxy servers. + :type tool_inputs: dict :param tool_inputs: dictionary of input datasets and parameters for the tool (see below) @@ -416,6 +427,10 @@ def run_tool( payload["inputs"] = tool_inputs.to_dict() else: payload["inputs"] = tool_inputs + + if data_manager_mode: + payload["data_manager_mode"] = data_manager_mode + return self._post(payload) def upload_file(