diff --git a/lib/galaxy/tool_util/linters/inputs.py b/lib/galaxy/tool_util/linters/inputs.py
index e4fd04694dec..815c1a6d0142 100644
--- a/lib/galaxy/tool_util/linters/inputs.py
+++ b/lib/galaxy/tool_util/linters/inputs.py
@@ -3,6 +3,7 @@
import ast
import re
import warnings
+from copy import deepcopy
from typing import (
Iterator,
Optional,
@@ -130,7 +131,6 @@
]
# TODO lint for valid param type - attribute combinations
-# TODO check if dataset is available for filters referring other datasets
# TODO check if ref input param is present for from_dataset
@@ -490,6 +490,190 @@ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
)
+FILTER_REQUIRED_ATTRIBUTES = {
+ "data_meta": ["type", "ref", "key"], # column needs special treatment
+ "param_value": ["type", "ref", "column"],
+ "static_value": ["type", "column", "value"],
+ "regexp": ["type", "column", "value"],
+ "unique_value": ["type", "column"],
+ "multiple_splitter": ["type", "column"],
+ "attribute_value_splitter": ["type", "column"],
+ "add_value": ["type", "value"],
+ "remove_value": ["type"], # this is handled separately in InputsOptionsRemoveValueFilterRequiredAttributes
+ "sort_by": ["type", "column"],
+ "data_table": ["type", "column", "table_name", "data_table_column"],
+}
+
+
+class InputsOptionsFiltersRequiredAttributes(Linter):
+ """
+ check required attributes of filters
+ """
+
+ @classmethod
+ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
+ tool_xml = getattr(tool_source, "xml_tree", None)
+ if not tool_xml:
+ return
+ for param, param_name in _iter_param(tool_xml):
+ options = param.find("./options")
+ if options is None:
+ continue
+ for filter in param.findall("./options/filter"):
+ filter_type = filter.get("type", None)
+ if filter_type is None or filter_type not in FILTER_ALLOWED_ATTRIBUTES:
+ continue
+ for attrib in FILTER_REQUIRED_ATTRIBUTES[filter_type]:
+ if attrib not in filter.attrib:
+ lint_ctx.error(
+ f"Select parameter [{param_name}] '{filter_type}' filter misses required attribute '{attrib}'",
+ node=filter,
+ )
+
+
+class InputsOptionsRemoveValueFilterRequiredAttributes(Linter):
+ """
+ check required attributes of remove_value filter
+ """
+
+ @classmethod
+ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
+ tool_xml = getattr(tool_source, "xml_tree", None)
+ if not tool_xml:
+ return
+ for param, param_name in _iter_param(tool_xml):
+ options = param.find("./options")
+ if options is None:
+ continue
+ for filter in param.findall("./options/filter"):
+ filter_type = filter.get("type", None)
+ # check for required attributes for filter (remove_value needs a bit more logic here)
+ if filter_type != "remove_value":
+ continue
+ if not (
+ (
+ "value" in filter.attrib
+ and "ref" not in filter.attrib
+ and "meta_ref" not in filter.attrib
+ and "key" not in filter.attrib
+ )
+ or (
+ "value" not in filter.attrib
+ and "ref" in filter.attrib
+ and "meta_ref" not in filter.attrib
+ and "key" not in filter.attrib
+ )
+ or (
+ "value" not in filter.attrib
+ and "ref" not in filter.attrib
+ and "meta_ref" in filter.attrib
+ and "key" in filter.attrib
+ )
+ ):
+ lint_ctx.error(
+ f"Select parameter [{param_name}] '{filter_type}'' filter needs either the 'value'; 'ref'; or 'meta' and 'key' attribute(s)",
+ node=filter,
+ )
+
+
+FILTER_ALLOWED_ATTRIBUTES = deepcopy(FILTER_REQUIRED_ATTRIBUTES)
+FILTER_ALLOWED_ATTRIBUTES["static_value"].append("keep")
+FILTER_ALLOWED_ATTRIBUTES["regexp"].append("keep")
+FILTER_ALLOWED_ATTRIBUTES["data_meta"].extend(["column", "multiple", "separator"])
+FILTER_ALLOWED_ATTRIBUTES["param_value"].extend(["keep", "ref_attribute"])
+FILTER_ALLOWED_ATTRIBUTES["multiple_splitter"].append("separator")
+FILTER_ALLOWED_ATTRIBUTES["attribute_value_splitter"].extend(["pair_separator", "name_val_separator"])
+FILTER_ALLOWED_ATTRIBUTES["add_value"].extend(["name", "index"])
+FILTER_ALLOWED_ATTRIBUTES["remove_value"].extend(["value", "ref", "meta_ref", "key"])
+FILTER_ALLOWED_ATTRIBUTES["data_table"].append("keep")
+
+
+class InputsOptionsFiltersAllowedAttributes(Linter):
+ """
+ check allowed attributes of filters
+ """
+
+ @classmethod
+ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
+ tool_xml = getattr(tool_source, "xml_tree", None)
+ if not tool_xml:
+ return
+
+ for param, param_name in _iter_param(tool_xml):
+ options = param.find("./options")
+ if options is None:
+ continue
+ for filter in param.findall("./options/filter"):
+ filter_type = filter.get("type", None)
+ if filter_type is None or filter_type not in FILTER_ALLOWED_ATTRIBUTES:
+ continue
+ for attrib in filter.attrib:
+ if attrib not in FILTER_ALLOWED_ATTRIBUTES[filter_type]:
+ lint_ctx.warn(
+ f"Select parameter [{param_name}] '{filter_type}' filter specifies unnecessary attribute '{attrib}'",
+ node=filter,
+ )
+
+
+class InputsOptionsRegexFilterExpression(Linter):
+ """
+ Check the regular expression of regexp filters
+ """
+
+ @classmethod
+ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
+ tool_xml = getattr(tool_source, "xml_tree", None)
+ if not tool_xml:
+ return
+
+ for param, param_name in _iter_param(tool_xml):
+ options = param.find("./options")
+ if options is None:
+ continue
+ for filter in param.findall("./options/filter"):
+ filter_type = filter.get("type", None)
+ if filter_type == "regexp" and "value" in filter.attrib:
+ try:
+ re.compile(filter.attrib["value"])
+ except re.error as re_error:
+ lint_ctx.error(
+ f"Select parameter [{param_name}] '{filter_type}'' filter 'value' is not a valid regular expression ({re_error})'",
+ node=filter,
+ )
+
+
+class InputsOptionsFiltersCheckReferences(Linter):
+ """
+ Check the references used in filters
+ """
+
+ @classmethod
+ def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
+ tool_xml = getattr(tool_source, "xml_tree", None)
+ if not tool_xml:
+ return
+
+ # get the set of param names
+ param_names = {param_name for _, param_name in _iter_param(tool_xml)}
+
+ for param, param_name in _iter_param(tool_xml):
+ options = param.find("./options")
+ if options is None:
+ continue
+ for filter in param.findall("./options/filter"):
+ filter_type = filter.get("type", None)
+ if filter_type is not None:
+ # check for references to other inputs
+ # TODO: currently ref and metaref seem only to work for top level params,
+ # once this is fixed the linter needs to be extended, e.g. `f.attrib[ref_attrib].split('|')[-1]`
+ for ref_attrib in ["meta_ref", "ref"]:
+ if ref_attrib in filter.attrib and filter.attrib[ref_attrib] not in param_names:
+ lint_ctx.error(
+ f"Select parameter [{param_name}] '{filter_type}'' filter attribute '{ref_attrib}' refers to non existing parameter '{filter.attrib[ref_attrib]}'",
+ node=filter,
+ )
+
+
class InputsDataOptionsFiltersRef(Linter):
"""
Lint for set ref for filters of data parameters
diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index 8975b36408ec..f30684b582fc 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -5646,8 +5646,9 @@ Currently the following filters are defined:
* ``data_meta`` populate or filter options based on the metadata of another input parameter specified by ``ref``. If a ``column`` is given options are filtered for which the entry in this column ``column`` is equal to metadata of the input parameter specified by ``ref``.
If no ``column`` is given the metadata value of the referenced input is added to the options list (in this case the corresponding ``options`` tag must not have the ``from_data_table`` or ``from_dataset`` attributes).
In both cases the desired metadata is selected by ``key``.
+* ``data_table`` remove values according to the entries of a data table. Remove options where the value in ``column`` appears in the data table ``table_name`` in column ``table_column``. Setting ``keep`` will to ``true`` will keep only entries also appearing in the data table.
-The ``static_value`` and ``regexp`` filters can be inverted by setting ``keep`` to true.
+The ``static_value``, ``regexp``, and ``data_table`` filters can be inverted by setting ``keep`` to true.
* ``add_value``: add an option with a given ``name`` and ``value`` to the options. By default, the new option is appended, with ``index`` the insertion position can be specified.
* ``remove_value``: remove a value from the options. Either specified explicitly with ``value``, the value of another input specified with ``ref``, or the metadata ``key`` of another input ``meta_ref``.
@@ -5821,7 +5822,7 @@ only used if ``multiple`` is set to ``true``.]]>
If ``true``, keep columns matching the
value, if ``false`` discard columns matching the value. Used when ``type`` is
-either ``static_value``, ``regexp`` or ``param_value``. Default: true
+either ``static_value``, ``regexp``, ``param_value`` or ``data_table``. Default: true.
@@ -5870,6 +5871,18 @@ from the list.
Only used if ``type`` is ``attribute_value_splitter``. This is used to separate attributes and values from each other within an attribute-value pair, i.e. ``=`` if the target content is ``A=V; B=W; C=Y``. Defaults to whitespace.
+
+
+ Only used when ``type`` is
+``data_table``. The name of the data table to use.
+
+
+
+
+ Only used when ``type`` is
+``data_table``. The column of the data table to use (0 based index or column name).
+
+
@@ -7922,6 +7935,7 @@ and ``bibtex`` are the only supported options.
+
diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py
index 8d9488128a03..5931fee08bed 100644
--- a/lib/galaxy/tools/parameters/dynamic_options.py
+++ b/lib/galaxy/tools/parameters/dynamic_options.py
@@ -537,6 +537,61 @@ def filter_options(self, options, trans, other_values):
return sorted(options, key=lambda x: x[self.column], reverse=self.reverse)
+class DataTableFilter(Filter):
+ """
+ Filters a list of options by entries present in a data table, i.e.
+ option[column] needs to be in the specified data table column
+
+ Type: data_table
+
+ Required Attributes:
+
+ - column: column in options to compare with
+ - table_name: data table to use
+ - data_table_column: data table column to use
+
+ Optional Attributes:
+
+ - keep: Keep options where option[column] is in the data table column (True)
+ Discard columns matching value (False)
+
+ """
+
+ def __init__(self, d_option, elem):
+ Filter.__init__(self, d_option, elem)
+ self.table_name = elem.get("table_name", None)
+ assert self.table_name is not None, "Required 'table_name' attribute missing from filter"
+ column = elem.get("column", None)
+ assert column is not None, "Required 'column' attribute missing from filter"
+ self.column = d_option.column_spec_to_index(column)
+ self.data_table_column = elem.get("data_table_column", None)
+ assert self.data_table_column is not None, "Required 'data_table_column' attribute missing from filter"
+ self.keep = string_as_bool(elem.get("keep", "True"))
+
+ def filter_options(self, options, trans, other_values):
+ # get column from data table, by index or column name
+ entries = None
+ try:
+ entries = {f[int(self.data_table_column)] for f in trans.app.tool_data_tables[self.table_name].get_fields()}
+ except ValueError:
+ pass
+ try:
+ entries = {
+ f[self.data_table_column] for f in trans.app.tool_data_tables[self.table_name].get_named_fields_list()
+ }
+ except KeyError:
+ pass
+ if entries is None:
+ log.error(f"could not get data from column {self.data_table_column} from data_table {self.table_name}")
+ return options
+
+ rval = []
+ for o in options:
+ if self.keep == (o[self.column] in entries):
+ rval.append(o)
+ return rval
+
+
filter_types = dict(
data_meta=DataMetaFilter,
param_value=ParamValueFilter,
@@ -548,6 +603,7 @@ def filter_options(self, options, trans, other_values):
add_value=AdditionalValueFilter,
remove_value=RemoveValueFilter,
sort_by=SortByColumnFilter,
+ data_table=DataTableFilter,
)
diff --git a/test/functional/tools/filter_data_table.xml b/test/functional/tools/filter_data_table.xml
new file mode 100644
index 000000000000..b30c5a201bd2
--- /dev/null
+++ b/test/functional/tools/filter_data_table.xml
@@ -0,0 +1,42 @@
+
+ Filter on datatable entries
+ '$output'
+ ]]>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/test/functional/tools/sample_tool_conf.xml b/test/functional/tools/sample_tool_conf.xml
index bb9d7568f600..0918d7dc92b4 100644
--- a/test/functional/tools/sample_tool_conf.xml
+++ b/test/functional/tools/sample_tool_conf.xml
@@ -66,6 +66,7 @@
+
diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py
index fc99e74ed963..939545c8a078 100644
--- a/test/unit/tool_util/test_tool_linters.py
+++ b/test/unit/tool_util/test_tool_linters.py
@@ -230,6 +230,7 @@
INPUTS_DATA_PARAM_OPTIONS = """
+
@@ -242,6 +243,7 @@
INPUTS_DATA_PARAM_OPTIONS_FILTER_ATTRIBUTE = """
+
@@ -257,7 +259,7 @@
-
+
@@ -548,6 +550,48 @@
"""
+INPUTS_FILTER_INCORRECT = """
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+INPUTS_FILTER_CORRECT = """
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
# test tool xml for outputs linter
OUTPUTS_MISSING = """
@@ -1345,7 +1389,7 @@ def test_inputs_data_param_options(lint_ctx):
tool_source = get_xml_tool_source(INPUTS_DATA_PARAM_OPTIONS)
run_lint_module(lint_ctx, inputs, tool_source)
assert not lint_ctx.valid_messages
- assert "Found 1 input parameters." in lint_ctx.info_messages
+ assert "Found 2 input parameters." in lint_ctx.info_messages
assert len(lint_ctx.info_messages) == 1
assert not lint_ctx.warn_messages
assert not lint_ctx.error_messages
@@ -1355,7 +1399,7 @@ def test_inputs_data_param_options_filter_attribute(lint_ctx):
tool_source = get_xml_tool_source(INPUTS_DATA_PARAM_OPTIONS_FILTER_ATTRIBUTE)
run_lint_module(lint_ctx, inputs, tool_source)
assert not lint_ctx.valid_messages
- assert "Found 1 input parameters." in lint_ctx.info_messages
+ assert "Found 2 input parameters." in lint_ctx.info_messages
assert len(lint_ctx.info_messages) == 1
assert not lint_ctx.warn_messages
assert not lint_ctx.error_messages
@@ -1507,7 +1551,7 @@ def test_inputs_select_filter(lint_ctx):
assert "Found 1 input parameters." in lint_ctx.info_messages
assert "Invalid XML: Element 'filter': The attribute 'type' is required but missing." in lint_ctx.error_messages
assert (
- "Invalid XML: Element 'filter', attribute 'type': [facet 'enumeration'] The value 'unknown_filter_type' is not an element of the set {'data_meta', 'param_value', 'static_value', 'regexp', 'unique_value', 'multiple_splitter', 'attribute_value_splitter', 'add_value', 'remove_value', 'sort_by'}."
+ "Invalid XML: Element 'filter', attribute 'type': [facet 'enumeration'] The value 'unknown_filter_type' is not an element of the set {'data_meta', 'param_value', 'static_value', 'regexp', 'unique_value', 'multiple_splitter', 'attribute_value_splitter', 'add_value', 'remove_value', 'sort_by', 'data_table'}."
in lint_ctx.error_messages
)
assert len(lint_ctx.info_messages) == 1
@@ -1610,6 +1654,49 @@ def test_inputs_duplicate_names(lint_ctx):
assert len(lint_ctx.error_messages) == 2
+def test_inputs_filter_correct(lint_ctx):
+ tool_source = get_xml_tool_source(INPUTS_FILTER_CORRECT)
+ run_lint_module(lint_ctx, inputs, tool_source)
+ assert len(lint_ctx.info_messages) == 1
+ assert not lint_ctx.valid_messages
+ assert not lint_ctx.warn_messages
+ assert not lint_ctx.error_messages
+
+
+def test_inputs_filter_incorrect(lint_ctx):
+ tool_source = get_xml_tool_source(INPUTS_FILTER_INCORRECT)
+ run_lint_module(lint_ctx, inputs, tool_source)
+ assert len(lint_ctx.info_messages) == 1
+ assert not lint_ctx.valid_messages
+ assert (
+ "Select parameter [select_param] 'regexp' filter specifies unnecessary attribute 'multiple'"
+ in lint_ctx.warn_messages
+ )
+ assert len(lint_ctx.warn_messages) == 1
+ assert "Invalid XML: Element 'filter': The attribute 'type' is required but missing." in lint_ctx.error_messages
+ assert (
+ "Invalid XML: Element 'filter', attribute 'type': [facet 'enumeration'] The value 'typo' is not an element of the set {'data_meta', 'param_value', 'static_value', 'regexp', 'unique_value', 'multiple_splitter', 'attribute_value_splitter', 'add_value', 'remove_value', 'sort_by', 'data_table'}."
+ in lint_ctx.error_messages
+ )
+ assert (
+ "Select parameter [select_param] 'static_value' filter misses required attribute 'column'"
+ in lint_ctx.error_messages
+ )
+ assert (
+ "Select parameter [select_param] 'remove_value'' filter needs either the 'value'; 'ref'; or 'meta' and 'key' attribute(s)"
+ in lint_ctx.error_messages
+ )
+ assert (
+ "Select parameter [select_param] 'data_meta'' filter attribute 'ref' refers to non existing parameter 'input'"
+ in lint_ctx.error_messages
+ )
+ assert (
+ "Select parameter [select_param] 'regexp'' filter 'value' is not a valid regular expression (nothing to repeat at position 1)'"
+ in lint_ctx.error_messages
+ )
+ assert len(lint_ctx.error_messages) == 6
+
+
def test_inputs_repeats(lint_ctx):
"""
this was previously covered by a linter in inputs
@@ -2237,7 +2324,7 @@ def test_skip_by_module(lint_ctx):
def test_list_linters():
linter_names = Linter.list_listers()
# make sure to add/remove a test for new/removed linters if this number changes
- assert len(linter_names) == 135
+ assert len(linter_names) == 140
assert "Linter" not in linter_names
# make sure that linters from all modules are available
for prefix in [