From a0b27e49a318d660a79b69353aca4a932d85f9ec Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Tue, 20 Feb 2024 18:04:14 +0100 Subject: [PATCH 1/8] Always disable check_values for data_source tools --- lib/galaxy/tools/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 311987a399ec..a8e926bd5f75 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2904,6 +2904,8 @@ def parse_inputs(self, tool_source): super().parse_inputs(tool_source) # Open all data_source tools in _top. self.target = "_top" + # data_source tools cannot check param values + self.check_values = False if "GALAXY_URL" not in self.inputs: self.inputs["GALAXY_URL"] = self._build_GALAXY_URL_parameter() self.inputs_by_page[0]["GALAXY_URL"] = self.inputs["GALAXY_URL"] From 55409ad50b10e93fe7ae728e080b68ea9b6c0420 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Wed, 21 Feb 2024 12:07:10 +0100 Subject: [PATCH 2/8] Update linting of data source tools --- lib/galaxy/tool_util/linters/_util.py | 2 +- lib/galaxy/tool_util/linters/xml_order.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/linters/_util.py b/lib/galaxy/tool_util/linters/_util.py index 025926259ae0..84ee89d1d981 100644 --- a/lib/galaxy/tool_util/linters/_util.py +++ b/lib/galaxy/tool_util/linters/_util.py @@ -3,7 +3,7 @@ def is_datasource(tool_xml): """Returns true if the tool is a datasource tool""" - return tool_xml.getroot().attrib.get("tool_type", "") == "data_source" + return tool_xml.getroot().attrib.get("tool_type", "") in ["data_source", "data_source_async"] def is_valid_cheetah_placeholder(name): diff --git a/lib/galaxy/tool_util/linters/xml_order.py b/lib/galaxy/tool_util/linters/xml_order.py index af3f1f2cdd66..6ead8908c2b2 100644 --- a/lib/galaxy/tool_util/linters/xml_order.py +++ b/lib/galaxy/tool_util/linters/xml_order.py @@ -42,6 +42,7 @@ DATASOURCE_TAG_ORDER = [ "description", "macros", + "requirements", "command", "configfiles", "inputs", @@ -62,7 +63,7 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): return tool_root = tool_xml.getroot() - if tool_root.attrib.get("tool_type", "") == "data_source": + if tool_root.attrib.get("tool_type", "") in ["data_source", "data_source_async"]: tag_ordering = DATASOURCE_TAG_ORDER else: tag_ordering = TAG_ORDER From a5eeb951c555f73e4eb3a792451f3ef26741e342 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Thu, 22 Feb 2024 14:40:45 +0100 Subject: [PATCH 3/8] Eliminate duplicate params translation and make params check more stringent --- .../webapps/galaxy/controllers/tool_runner.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py index c4ccf28b2a14..c40de6dedcc8 100644 --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -78,20 +78,30 @@ def __tool_404__(): # execute tool without displaying form # (used for datasource tools, but note that data_source_async tools # are handled separately by the async controller) - params = galaxy.util.Params(kwd, sanitize=False) + params = galaxy.util.Params(kwd, sanitize=False).__dict__ + if tool.input_translator: + # perform test translation of the incoming params without affecting originals + # the actual translation will happen later + # this is only for checking if we end up with required parameters + test_params = params.copy() + tool.input_translator.translate(test_params) + else: + test_params = params if tool.tool_type == "data_source": + if "URL" not in test_params: + error("Execution of `data_source` tools requires a `URL` parameter") # preserve original params sent by the remote server as extra dict - params.update({"incoming_request_params": params.__dict__.copy()}) - # do param translation here, used by datasource tools - if tool.input_translator: - tool.input_translator.translate(params) - if "runtool_btn" not in params.__dict__ and "URL" not in params.__dict__: - error("Tool execution through the `tool_runner` requires a `runtool_btn` flag or `URL` parameter.") + # before in-place translation happens + params.update({"incoming_request_params": params.copy()}) + else: + if "runtool_btn" not in test_params: + error("Tool execution through the `tool_runner` requires a `runtool_btn` flag") + # We may be visiting Galaxy for the first time ( e.g., sending data from UCSC ), # so make sure to create a new history if we've never had one before. history = tool.get_default_history_by_trans(trans, create=True) try: - vars = tool.handle_input(trans, params.__dict__, history=history) + vars = tool.handle_input(trans, params, history=history) except Exception as e: error(galaxy.util.unicodify(e)) if len(params) > 0: From fb07082bc1083df7ab0f57311df85cdd64c4c9c1 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Thu, 22 Feb 2024 15:48:55 +0100 Subject: [PATCH 4/8] Drop unexpected incoming params in data_source tools like in data_source_async tools --- lib/galaxy/tools/parameters/input_translation.py | 4 ++++ lib/galaxy/webapps/galaxy/controllers/tool_runner.py | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/input_translation.py b/lib/galaxy/tools/parameters/input_translation.py index 035a2c30ceb4..267d91dd72ab 100644 --- a/lib/galaxy/tools/parameters/input_translation.py +++ b/lib/galaxy/tools/parameters/input_translation.py @@ -62,6 +62,8 @@ def from_element(cls, elem): value_trans = {} append_param = None + rval.vocabulary.add(remote_name) + value_trans_elem = req_param.find("value_translation") if value_trans_elem is not None: for value_elem in value_trans_elem.findall("value"): @@ -81,6 +83,7 @@ def from_element(cls, elem): value_missing = value_elem.get("missing") if None not in [value_name, value_missing]: append_dict[value_name] = value_missing + rval.vocabulary.add(value_name) append_param = Bunch( separator=separator, first_separator=first_separator, join_str=join_str, append_dict=append_dict ) @@ -93,6 +96,7 @@ def from_element(cls, elem): def __init__(self): self.param_trans_dict = {} + self.vocabulary = set() def translate(self, params): """ diff --git a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py index c40de6dedcc8..23fa8f44fa36 100644 --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -91,8 +91,14 @@ def __tool_404__(): if "URL" not in test_params: error("Execution of `data_source` tools requires a `URL` parameter") # preserve original params sent by the remote server as extra dict - # before in-place translation happens + # before in-place translation happens, then clean the incoming params params.update({"incoming_request_params": params.copy()}) + if tool.input_translator: + for k in list(params.keys()): + if k not in tool.input_translator.vocabulary and k not in ("URL", "incoming_request_params"): + # the remote server has sent a param + # that the tool is not expecting -> drop it + del params[k] else: if "runtool_btn" not in test_params: error("Tool execution through the `tool_runner` requires a `runtool_btn` flag") From 76968a68c77008504b13b8d9fde78605870f26e9 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Thu, 22 Feb 2024 17:29:33 +0100 Subject: [PATCH 5/8] Adjust UCSC Table browser tools to code changes Since the code now drops undeclared incoming parameters it enforces cleaner wrapper style. --- tools/data_source/ucsc_tablebrowser.xml | 3 ++- tools/data_source/ucsc_tablebrowser_archaea.xml | 3 ++- tools/data_source/ucsc_tablebrowser_test.xml | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/data_source/ucsc_tablebrowser.xml b/tools/data_source/ucsc_tablebrowser.xml index 19852ecf6fcc..5e7f7744196f 100644 --- a/tools/data_source/ucsc_tablebrowser.xml +++ b/tools/data_source/ucsc_tablebrowser.xml @@ -27,6 +27,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz + @@ -41,7 +42,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz - + diff --git a/tools/data_source/ucsc_tablebrowser_archaea.xml b/tools/data_source/ucsc_tablebrowser_archaea.xml index a1707e441a1e..e67a30fc953b 100644 --- a/tools/data_source/ucsc_tablebrowser_archaea.xml +++ b/tools/data_source/ucsc_tablebrowser_archaea.xml @@ -27,6 +27,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz + @@ -41,7 +42,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz - + diff --git a/tools/data_source/ucsc_tablebrowser_test.xml b/tools/data_source/ucsc_tablebrowser_test.xml index 9782f8d2b41b..0e04e039bead 100644 --- a/tools/data_source/ucsc_tablebrowser_test.xml +++ b/tools/data_source/ucsc_tablebrowser_test.xml @@ -27,6 +27,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz + @@ -41,7 +42,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz - + From 82ca92ac536259df5cafc9a6a56ff5d6e2f10be7 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Thu, 22 Feb 2024 17:56:03 +0100 Subject: [PATCH 6/8] Fix tool validation --- lib/galaxy/tool_util/xsd/galaxy.xsd | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 79dec3a85fc5..3ad702ce7492 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -6688,6 +6688,7 @@ Examples are included in the test tools directory including: + From 2554a1b6b95870ac169b451e4a37bc0d7863c4e6 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Fri, 23 Feb 2024 10:59:07 +0100 Subject: [PATCH 7/8] Reuse is_datasource instead of duplicating code --- lib/galaxy/tool_util/linters/xml_order.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tool_util/linters/xml_order.py b/lib/galaxy/tool_util/linters/xml_order.py index 6ead8908c2b2..13132e909c45 100644 --- a/lib/galaxy/tool_util/linters/xml_order.py +++ b/lib/galaxy/tool_util/linters/xml_order.py @@ -4,16 +4,18 @@ https://github.com/galaxy-iuc/standards. """ -from typing import TYPE_CHECKING +from typing import ( + Optional, + TYPE_CHECKING, +) from galaxy.tool_util.lint import Linter +from ._util import is_datasource if TYPE_CHECKING: from galaxy.tool_util.lint import LintContext from galaxy.tool_util.parser.interface import ToolSource -from typing import Optional - # https://github.com/galaxy-iuc/standards # https://github.com/galaxy-iuc/standards/pull/7/files TAG_ORDER = [ @@ -63,7 +65,7 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"): return tool_root = tool_xml.getroot() - if tool_root.attrib.get("tool_type", "") in ["data_source", "data_source_async"]: + if is_datasource(tool_xml): tag_ordering = DATASOURCE_TAG_ORDER else: tag_ordering = TAG_ORDER From 41defa80ac33ba37cde28401a99470029607374f Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Sat, 24 Feb 2024 09:48:33 +0100 Subject: [PATCH 8/8] Make cleaning of request params profile-dependent as suggested by @martenson --- lib/galaxy/tool_util/xsd/galaxy.xsd | 1 + lib/galaxy/tools/__init__.py | 7 +++++++ lib/galaxy/webapps/galaxy/controllers/async.py | 2 +- lib/galaxy/webapps/galaxy/controllers/tool_runner.py | 2 +- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 3ad702ce7492..812e80198194 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -68,6 +68,7 @@ List of behavior changes associated with profile versions: ### 24.0 - Do not use Galaxy python environment for `data_source_async` tools. +- Drop request parameters received by data source tools that are not declared in `` section. ### Examples diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index a8e926bd5f75..d75aed1b09f6 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2895,6 +2895,13 @@ class DataSourceTool(OutputParameterJSONTool): tool_type = "data_source" default_tool_action = DataSourceToolAction + @property + def wants_params_cleaned(self): + """Indicates whether received, but undeclared request params should be cleaned.""" + if self.profile < 24.0: + return False + return True + def _build_GALAXY_URL_parameter(self): return ToolParameter.build( self, XML(f'') diff --git a/lib/galaxy/webapps/galaxy/controllers/async.py b/lib/galaxy/webapps/galaxy/controllers/async.py index d9ecae7f2305..4a9615fc700d 100644 --- a/lib/galaxy/webapps/galaxy/controllers/async.py +++ b/lib/galaxy/webapps/galaxy/controllers/async.py @@ -86,7 +86,7 @@ def index(self, trans, tool_id=None, data_secret=None, **kwd): translator.galaxy_name for translator in tool.input_translator.param_trans_dict.values() } for param in params: - if param in tool_declared_params: + if param in tool_declared_params or not tool.wants_params_cleaned: params_dict[param] = params.get(param, None) params = params_dict diff --git a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py index 23fa8f44fa36..f85e9b2d9a30 100644 --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -93,7 +93,7 @@ def __tool_404__(): # preserve original params sent by the remote server as extra dict # before in-place translation happens, then clean the incoming params params.update({"incoming_request_params": params.copy()}) - if tool.input_translator: + if tool.input_translator and tool.wants_params_cleaned: for k in list(params.keys()): if k not in tool.input_translator.vocabulary and k not in ("URL", "incoming_request_params"): # the remote server has sent a param