From 27ecea0bfc9f6e2813ac658e86bff6f4f67328e7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 6 Aug 2024 16:39:58 -0400 Subject: [PATCH 01/14] Address PR comment in gx_select.xml, improve comment. --- test/functional/tools/parameters/gx_select.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functional/tools/parameters/gx_select.xml b/test/functional/tools/parameters/gx_select.xml index 6b325da2550c..90833398af3c 100644 --- a/test/functional/tools/parameters/gx_select.xml +++ b/test/functional/tools/parameters/gx_select.xml @@ -24,8 +24,8 @@ echo '$parameter' >> '$output' - From e530227ac6e1897e576183a9722ce535322c2ff2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 6 Aug 2024 11:56:33 -0400 Subject: [PATCH 02/14] Implement group tag testing in tools. --- lib/galaxy/tool_util/verify/interactor.py | 5 ++++ lib/galaxy/tools/actions/upload_common.py | 8 +++++- .../tools/parameters/gx_group_tag.xml | 27 +++++++++++++++++++ tools/data_source/upload.xml | 1 + 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 test/functional/tools/parameters/gx_group_tag.xml diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py index e8971a0ec4f5..59acc5195bdb 100644 --- a/lib/galaxy/tool_util/verify/interactor.py +++ b/lib/galaxy/tool_util/verify/interactor.py @@ -505,10 +505,14 @@ def stage_data_async( tool_version: Optional[str] = None, ) -> Callable[[], None]: fname = test_data["fname"] + tags = test_data.get("tags") tool_input = { "file_type": test_data["ftype"], "dbkey": test_data["dbkey"], } + if tags: + tool_input["tags"] = tags + metadata = test_data.get("metadata", {}) if not hasattr(metadata, "items"): raise Exception(f"Invalid metadata description found for input [{fname}] - [{metadata}]") @@ -1865,6 +1869,7 @@ def test_data_iter(required_files): ftype=extra.get("ftype", DEFAULT_FTYPE), dbkey=extra.get("dbkey", DEFAULT_DBKEY), location=extra.get("location", None), + tags=extra.get("tags", []), ) edit_attributes = extra.get("edit_attributes", []) diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 8101af78b324..d4b8bee24ff6 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -276,7 +276,13 @@ def new_upload( ) else: upload_target_dataset_instance = __new_history_upload(trans, uploaded_dataset, history=history, state=state) - + tags_raw = getattr(uploaded_dataset, "tags", None) + if tags_raw: + new_tags = tag_handler.parse_tags_list(tags_raw.split(",")) + for tag in new_tags: + tag_handler.apply_item_tag( + user=trans.user, item=upload_target_dataset_instance, name=tag[0], value=tag[1], flush=True + ) if tag_list: tag_handler.add_tags_from_list(trans.user, upload_target_dataset_instance, tag_list, flush=False) diff --git a/test/functional/tools/parameters/gx_group_tag.xml b/test/functional/tools/parameters/gx_group_tag.xml new file mode 100644 index 000000000000..90dd50e6ae03 --- /dev/null +++ b/test/functional/tools/parameters/gx_group_tag.xml @@ -0,0 +1,27 @@ + + + macros.xml + + '$output' + ]]> + + + + + + + + + + + + + + + + + + + + diff --git a/tools/data_source/upload.xml b/tools/data_source/upload.xml index 548b63163334..d8c7ace6b2ab 100644 --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -59,6 +59,7 @@ + From d41544db70fa09eb9a8e7082cc73339f498befba Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 7 Aug 2024 10:25:09 -0400 Subject: [PATCH 03/14] Group tag parameter models. --- lib/galaxy/tool_util/parameters/factory.py | 5 +++++ lib/galaxy/tool_util/parameters/models.py | 16 ++++++++++++++++ test/unit/tool_util/parameter_specification.yml | 14 ++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/lib/galaxy/tool_util/parameters/factory.py b/lib/galaxy/tool_util/parameters/factory.py index cd872e43a974..0983a0fe8b17 100644 --- a/lib/galaxy/tool_util/parameters/factory.py +++ b/lib/galaxy/tool_util/parameters/factory.py @@ -33,6 +33,7 @@ DataParameterModel, DrillDownParameterModel, FloatParameterModel, + GroupTagParameterModel, HiddenParameterModel, IntegerParameterModel, LabelValue, @@ -164,6 +165,10 @@ def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: return DataColumnParameterModel( name=input_source.parse_name(), ) + elif param_type == "group_tag": + return GroupTagParameterModel( + name=input_source.parse_name(), + ) else: raise Exception(f"Unknown Galaxy parameter type {param_type}") elif input_type == "conditional": diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py index 27efc38feee2..5f7af47c5ed3 100644 --- a/lib/galaxy/tool_util/parameters/models.py +++ b/lib/galaxy/tool_util/parameters/models.py @@ -587,6 +587,21 @@ def request_requires_value(self) -> bool: return False +class GroupTagParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_group_tag"] = "gx_group_tag" + + @property + def py_type(self) -> Type: + return StrictStr + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return True + + DiscriminatorType = Union[bool, str] @@ -920,6 +935,7 @@ def request_requires_value(self) -> bool: DirectoryUriParameterModel, RulesParameterModel, DrillDownParameterModel, + GroupTagParameterModel, ColorParameterModel, ConditionalParameterModel, RepeatParameterModel, diff --git a/test/unit/tool_util/parameter_specification.yml b/test/unit/tool_util/parameter_specification.yml index 639f711a8825..bf80617f7192 100644 --- a/test/unit/tool_util/parameter_specification.yml +++ b/test/unit/tool_util/parameter_specification.yml @@ -657,6 +657,20 @@ gx_data_column: request_internal_invalid: - { ref_parameter: {src: hda, id: 123}, parameter: "0" } +gx_group_tag: + request_valid: + - { ref_parameter: { src: hdca, id: abcdabcd}, parameter: 'type:matched' } + - { ref_parameter: { src: hdca, id: abcdabcd}, parameter: 'matched' } + request_invalid: + - { ref_parameter: { src: hdca, id: abcdabcd}, parameter: null } + - { ref_parameter: { src: hdca, id: abcdabcd}, parameter: 7 } + - { ref_parameter: { src: hdca, id: abcdabcd}, parameter: true } + - { ref_parameter: { src: hdca, id: abcdabcd} } + request_internal_valid: + - { ref_parameter: {src: hdca, id: 123}, parameter: 'type: matched' } + request_internal_invalid: + - { ref_parameter: {src: hdca, id: 123}, parameter: null } + cwl_int: request_valid: - parameter: 5 From 01292d96e3dac398a824d01eface20a21a2b1f36 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 7 Aug 2024 10:29:54 -0400 Subject: [PATCH 04/14] Fix more test case errors caught by parameter validation. --- test/functional/tools/discover_sort_by.xml | 1 - test/functional/tools/discover_sort_by_legacy_test.xml | 1 - test/functional/tools/galaxy_json_discover_and_sleep.xml | 1 - test/functional/tools/multiple_versions_v01.xml | 2 +- test/functional/tools/multiple_versions_v02.xml | 2 +- 5 files changed, 2 insertions(+), 5 deletions(-) diff --git a/test/functional/tools/discover_sort_by.xml b/test/functional/tools/discover_sort_by.xml index 4f29087f3118..f3ac7dcca92b 100644 --- a/test/functional/tools/discover_sort_by.xml +++ b/test/functional/tools/discover_sort_by.xml @@ -71,7 +71,6 @@ done - diff --git a/test/functional/tools/discover_sort_by_legacy_test.xml b/test/functional/tools/discover_sort_by_legacy_test.xml index 2b62e12ea49e..275e1966189d 100644 --- a/test/functional/tools/discover_sort_by_legacy_test.xml +++ b/test/functional/tools/discover_sort_by_legacy_test.xml @@ -25,7 +25,6 @@ done - diff --git a/test/functional/tools/galaxy_json_discover_and_sleep.xml b/test/functional/tools/galaxy_json_discover_and_sleep.xml index 8b598bb72fa4..36f5ce5268c0 100644 --- a/test/functional/tools/galaxy_json_discover_and_sleep.xml +++ b/test/functional/tools/galaxy_json_discover_and_sleep.xml @@ -20,7 +20,6 @@ - diff --git a/test/functional/tools/multiple_versions_v01.xml b/test/functional/tools/multiple_versions_v01.xml index fa72fd5f65e7..6b656cf7a5b6 100644 --- a/test/functional/tools/multiple_versions_v01.xml +++ b/test/functional/tools/multiple_versions_v01.xml @@ -10,7 +10,7 @@ - + diff --git a/test/functional/tools/multiple_versions_v02.xml b/test/functional/tools/multiple_versions_v02.xml index 5eb00f8718d3..a32268dc0e98 100644 --- a/test/functional/tools/multiple_versions_v02.xml +++ b/test/functional/tools/multiple_versions_v02.xml @@ -10,7 +10,7 @@ - + From 5e57d007206f7041f92725737c3af58a4f23a963 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Aug 2024 09:30:10 -0400 Subject: [PATCH 05/14] Implement parameter models for baseurl parameters. --- lib/galaxy/tool_util/parameters/factory.py | 9 +++++++++ lib/galaxy/tool_util/parameters/models.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/lib/galaxy/tool_util/parameters/factory.py b/lib/galaxy/tool_util/parameters/factory.py index 0983a0fe8b17..68e1f7666c7a 100644 --- a/lib/galaxy/tool_util/parameters/factory.py +++ b/lib/galaxy/tool_util/parameters/factory.py @@ -16,6 +16,7 @@ ) from galaxy.util import string_as_bool from .models import ( + BaseUrlParameterModel, BooleanParameterModel, ColorParameterModel, ConditionalParameterModel, @@ -169,6 +170,10 @@ def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: return GroupTagParameterModel( name=input_source.parse_name(), ) + elif param_type == "baseurl": + return BaseUrlParameterModel( + name=input_source.parse_name(), + ) else: raise Exception(f"Unknown Galaxy parameter type {param_type}") elif input_type == "conditional": @@ -313,6 +318,10 @@ def input_models_for_pages(pages: PagesSource) -> List[ToolParameterT]: def input_models_for_page(page_source: PageSource) -> List[ToolParameterT]: input_models = [] for input_source in page_source.parse_input_sources(): + input_type = input_source.parse_input_type() + if input_type == "display": + # not a real input... just skip this. Should this be handled in the parser layer better? + continue tool_parameter_model = from_input_source(input_source) input_models.append(tool_parameter_model) return input_models diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py index 5f7af47c5ed3..04ace31ba9e7 100644 --- a/lib/galaxy/tool_util/parameters/models.py +++ b/lib/galaxy/tool_util/parameters/models.py @@ -22,6 +22,7 @@ Discriminator, Field, field_validator, + HttpUrl, RootModel, StrictBool, StrictFloat, @@ -602,6 +603,21 @@ def request_requires_value(self) -> bool: return True +class BaseUrlParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_baseurl"] = "gx_baseurl" + + @property + def py_type(self) -> Type: + return HttpUrl + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return True + + DiscriminatorType = Union[bool, str] @@ -936,6 +952,7 @@ def request_requires_value(self) -> bool: RulesParameterModel, DrillDownParameterModel, GroupTagParameterModel, + BaseUrlParameterModel, ColorParameterModel, ConditionalParameterModel, RepeatParameterModel, From 1f2cfcd835503bcd392bb36c01e14e0f14f61417 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Aug 2024 09:46:18 -0400 Subject: [PATCH 06/14] Drop older comments. --- lib/galaxy/tool_util/parameters/models.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py index 04ace31ba9e7..b58ddea802bd 100644 --- a/lib/galaxy/tool_util/parameters/models.py +++ b/lib/galaxy/tool_util/parameters/models.py @@ -49,10 +49,7 @@ # TODO: # - implement job vs request... -# - drill down # - implement data_ref on rules and implement some cross model validation -# - Optional conditionals... work through that? -# - Sections - fight that battle again... # + request: Return info needed to build request pydantic model at runtime. # + request_internal: This is a pydantic model to validate what Galaxy expects to find in the database, From 53a1ee520f64f297d6dcc7ed392e99c1665d3a02 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Aug 2024 09:55:40 -0400 Subject: [PATCH 07/14] Update parameter models for required integers. --- lib/galaxy/tool_util/parameters/factory.py | 3 +++ lib/galaxy/tool_util/parameters/models.py | 2 +- .../tools/parameters/gx_int_required.xml | 11 +++++++++++ .../gx_int_required_via_empty_string.xml | 12 ++++++++++++ test/unit/tool_util/parameter_specification.yml | 15 +++++++++++++++ 5 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 test/functional/tools/parameters/gx_int_required.xml create mode 100644 test/functional/tools/parameters/gx_int_required_via_empty_string.xml diff --git a/lib/galaxy/tool_util/parameters/factory.py b/lib/galaxy/tool_util/parameters/factory.py index 68e1f7666c7a..48af69565c9d 100644 --- a/lib/galaxy/tool_util/parameters/factory.py +++ b/lib/galaxy/tool_util/parameters/factory.py @@ -69,6 +69,9 @@ def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: int_value = int(value) elif optional: int_value = None + elif value == "" or value is None: + # A truly required parameter: https://github.com/galaxyproject/galaxy/pull/16966/files + int_value = None else: raise ParameterDefinitionError() return IntegerParameterModel(name=input_source.parse_name(), optional=optional, value=int_value) diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py index b58ddea802bd..d1e1f0fbebf0 100644 --- a/lib/galaxy/tool_util/parameters/models.py +++ b/lib/galaxy/tool_util/parameters/models.py @@ -183,7 +183,7 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam @property def request_requires_value(self) -> bool: - return False + return not self.optional and self.value is None class FloatParameterModel(BaseGalaxyToolParameterModelDefinition): diff --git a/test/functional/tools/parameters/gx_int_required.xml b/test/functional/tools/parameters/gx_int_required.xml new file mode 100644 index 000000000000..05a22fc2f3db --- /dev/null +++ b/test/functional/tools/parameters/gx_int_required.xml @@ -0,0 +1,11 @@ + + > '$output' + ]]> + + + + + + + diff --git a/test/functional/tools/parameters/gx_int_required_via_empty_string.xml b/test/functional/tools/parameters/gx_int_required_via_empty_string.xml new file mode 100644 index 000000000000..26e76cb46e91 --- /dev/null +++ b/test/functional/tools/parameters/gx_int_required_via_empty_string.xml @@ -0,0 +1,12 @@ + + > '$output' + ]]> + + + + + + + + diff --git a/test/unit/tool_util/parameter_specification.yml b/test/unit/tool_util/parameter_specification.yml index bf80617f7192..5ba52f525e7a 100644 --- a/test/unit/tool_util/parameter_specification.yml +++ b/test/unit/tool_util/parameter_specification.yml @@ -56,6 +56,21 @@ gx_int_optional: - parameter: "null" - parameter: [5] +gx_int_required: &gx_int_required + request_valid: + - parameter: 5 + - parameter: 6 + request_invalid: + - {} + - parameter: null + - parameter: "null" + - parameter: "None" + - parameter: { 5 } + - parameter: {__class__: 'ConnectedValue'} + +gx_int_required_via_empty_string: + <<: *gx_int_required + gx_text: request_valid: - parameter: moocow From 38b1a07f0c176a6b9ee9cb54b1648204e45693b0 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Aug 2024 11:50:46 -0400 Subject: [PATCH 08/14] More clarification of current advanced select behavior. --- lib/galaxy_test/api/test_tools.py | 32 +++++++++++++++++++ .../tool_util/parameter_specification.yml | 1 + 2 files changed, 33 insertions(+) diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index 031939e7064a..a8f0cb1ebc34 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -898,6 +898,38 @@ def test_select_first_by_default(self): self._assert_status_code_is(response, 400) assert "an invalid option" in response.text + @skip_without_tool("gx_select_multiple") + @skip_without_tool("gx_select_multiple_optional") + def test_select_multiple_null_handling(self): + with self.dataset_populator.test_history(require_new=False) as history_id: + inputs: Dict[str, Any] = {} + response = self._run("gx_select_multiple", history_id, inputs, assert_ok=True) + output = response["outputs"][0] + output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output) + assert output1_content.strip() == "None" + + inputs = {} + response = self._run("gx_select_multiple_optional", history_id, inputs, assert_ok=True) + output = response["outputs"][0] + output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output) + assert output1_content.strip() == "None" + + inputs = { + "parameter": None, + } + response = self._run("gx_select_multiple", history_id, inputs, assert_ok=True) + output = response["outputs"][0] + output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output) + assert output1_content.strip() == "None" + + inputs = { + "parameter": None, + } + response = self._run("gx_select_multiple_optional", history_id, inputs, assert_ok=True) + output = response["outputs"][0] + output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output) + assert output1_content.strip() == "None" + @skip_without_tool("gx_drill_down_exact") @skip_without_tool("gx_drill_down_exact_multiple") @skip_without_tool("gx_drill_down_recurse") diff --git a/test/unit/tool_util/parameter_specification.yml b/test/unit/tool_util/parameter_specification.yml index 5ba52f525e7a..4264e1af39ca 100644 --- a/test/unit/tool_util/parameter_specification.yml +++ b/test/unit/tool_util/parameter_specification.yml @@ -143,6 +143,7 @@ gx_select_multiple: - parameter: ["ex2"] request_invalid: - parameter: ["Ex1"] + # This doesn't refelct the actual behavior of pre 24.2 Galaxy I don't think - parameter: null - parameter: {} - parameter: 5 From aa569ae71b774b587822eb1db19d8daa2a329887 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Aug 2024 12:01:44 -0400 Subject: [PATCH 09/14] Parameter models for genomebuild parameters --- lib/galaxy/tool_util/parameters/factory.py | 9 ++++++ lib/galaxy/tool_util/parameters/models.py | 21 ++++++++++++ .../tools/parameters/gx_genomebuild.xml | 29 +++++++++++++++++ .../parameters/gx_genomebuild_multiple.xml | 29 +++++++++++++++++ .../parameters/gx_genomebuild_optional.xml | 29 +++++++++++++++++ .../tool_util/parameter_specification.yml | 32 +++++++++++++++++-- 6 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 test/functional/tools/parameters/gx_genomebuild.xml create mode 100644 test/functional/tools/parameters/gx_genomebuild_multiple.xml create mode 100644 test/functional/tools/parameters/gx_genomebuild_optional.xml diff --git a/lib/galaxy/tool_util/parameters/factory.py b/lib/galaxy/tool_util/parameters/factory.py index 48af69565c9d..c22bdb4ef90a 100644 --- a/lib/galaxy/tool_util/parameters/factory.py +++ b/lib/galaxy/tool_util/parameters/factory.py @@ -34,6 +34,7 @@ DataParameterModel, DrillDownParameterModel, FloatParameterModel, + GenomeBuildParameterModel, GroupTagParameterModel, HiddenParameterModel, IntegerParameterModel, @@ -177,6 +178,14 @@ def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: return BaseUrlParameterModel( name=input_source.parse_name(), ) + elif param_type == "genomebuild": + optional = input_source.parse_optional() + multiple = input_source.get_bool("multiple", False) + return GenomeBuildParameterModel( + name=input_source.parse_name(), + optional=optional, + multiple=multiple, + ) else: raise Exception(f"Unknown Galaxy parameter type {param_type}") elif input_type == "conditional": diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py index d1e1f0fbebf0..4543fddf9680 100644 --- a/lib/galaxy/tool_util/parameters/models.py +++ b/lib/galaxy/tool_util/parameters/models.py @@ -497,6 +497,26 @@ def request_requires_value(self) -> bool: return self.multiple and not self.optional +class GenomeBuildParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_genomebuild"] = "gx_genomebuild" + multiple: bool + + @property + def py_type(self) -> Type: + py_type: Type = StrictStr + if self.multiple: + py_type = list_type(py_type) + return optional_if_needed(py_type, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + # assumes it uses behavior of select parameters - an API test to reference for this would be nice + return self.multiple and not self.optional + + DrillDownHierarchyT = Literal["recurse", "exact"] @@ -950,6 +970,7 @@ def request_requires_value(self) -> bool: DrillDownParameterModel, GroupTagParameterModel, BaseUrlParameterModel, + GenomeBuildParameterModel, ColorParameterModel, ConditionalParameterModel, RepeatParameterModel, diff --git a/test/functional/tools/parameters/gx_genomebuild.xml b/test/functional/tools/parameters/gx_genomebuild.xml new file mode 100644 index 000000000000..516eb6efeca9 --- /dev/null +++ b/test/functional/tools/parameters/gx_genomebuild.xml @@ -0,0 +1,29 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_genomebuild_multiple.xml b/test/functional/tools/parameters/gx_genomebuild_multiple.xml new file mode 100644 index 000000000000..c70e712ee2db --- /dev/null +++ b/test/functional/tools/parameters/gx_genomebuild_multiple.xml @@ -0,0 +1,29 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_genomebuild_optional.xml b/test/functional/tools/parameters/gx_genomebuild_optional.xml new file mode 100644 index 000000000000..bffe9d9ff363 --- /dev/null +++ b/test/functional/tools/parameters/gx_genomebuild_optional.xml @@ -0,0 +1,29 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/unit/tool_util/parameter_specification.yml b/test/unit/tool_util/parameter_specification.yml index 4264e1af39ca..807dabe90201 100644 --- a/test/unit/tool_util/parameter_specification.yml +++ b/test/unit/tool_util/parameter_specification.yml @@ -136,14 +136,13 @@ gx_select_optional: - parameter: {} - parameter: 5 -# TODO: confirm null should vaguely not be allowed here gx_select_multiple: request_valid: - parameter: ["--ex1"] - parameter: ["ex2"] request_invalid: - parameter: ["Ex1"] - # This doesn't refelct the actual behavior of pre 24.2 Galaxy I don't think + # DIVERGES_FROM_CURRENT_API - parameter: null - parameter: {} - parameter: 5 @@ -160,6 +159,35 @@ gx_select_multiple_optional: - parameter: {} - parameter: 5 +gx_genomebuild: + request_valid: + - parameter: hg19 + - parameter: hg18 + # implicitly selects top option "?" + - {} + request_invalid: + - parameter: null + - parameter: 9 + +gx_genomebuild_optional: + request_valid: + - parameter: hg19 + - parameter: hg18 + # these yield actual null parameters instead of ? + - {} + - parameter: null + request_invalid: + - parameter: 8 + - parameter: true + +gx_genomebuild_multiple: + request_valid: + - parameter: ["hg18", hg19"] + request_invalid: + - parameter: 6 + # DIVERGES_FROM_CURRENT_API + - parameter: null + gx_hidden: request_valid: - parameter: moocow From ed720332b905501801d6675e4a23f908361684fe Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Aug 2024 13:04:35 -0400 Subject: [PATCH 10/14] Parameter models for directory_uri parameters. --- lib/galaxy/tool_util/parameters/factory.py | 5 +++++ lib/galaxy/tool_util/parameters/models.py | 11 +++++++++-- .../functional/tools/parameters/gx_directory_uri.xml | 12 ++++++++++++ test/unit/tool_util/parameter_specification.yml | 9 +++++++++ 4 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 test/functional/tools/parameters/gx_directory_uri.xml diff --git a/lib/galaxy/tool_util/parameters/factory.py b/lib/galaxy/tool_util/parameters/factory.py index c22bdb4ef90a..e21eb4794631 100644 --- a/lib/galaxy/tool_util/parameters/factory.py +++ b/lib/galaxy/tool_util/parameters/factory.py @@ -32,6 +32,7 @@ DataCollectionParameterModel, DataColumnParameterModel, DataParameterModel, + DirectoryUriParameterModel, DrillDownParameterModel, FloatParameterModel, GenomeBuildParameterModel, @@ -186,6 +187,10 @@ def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: optional=optional, multiple=multiple, ) + elif param_type == "directory_uri": + return DirectoryUriParameterModel( + name=input_source.parse_name(), + ) else: raise Exception(f"Unknown Galaxy parameter type {param_type}") elif input_type == "conditional": diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py index 4543fddf9680..131ee4d4c732 100644 --- a/lib/galaxy/tool_util/parameters/models.py +++ b/lib/galaxy/tool_util/parameters/models.py @@ -16,6 +16,7 @@ ) from pydantic import ( + AnyUrl, BaseModel, ConfigDict, create_model, @@ -421,8 +422,14 @@ def request_requires_value(self) -> bool: class DirectoryUriParameterModel(BaseGalaxyToolParameterModelDefinition): - parameter_type: Literal["gx_directory_uri"] - value: Optional[str] + parameter_type: Literal["gx_directory_uri"] = "gx_directory_uri" + + @property + def py_type(self) -> Type: + return AnyUrl + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) @property def request_requires_value(self) -> bool: diff --git a/test/functional/tools/parameters/gx_directory_uri.xml b/test/functional/tools/parameters/gx_directory_uri.xml new file mode 100644 index 000000000000..49ed4f98af85 --- /dev/null +++ b/test/functional/tools/parameters/gx_directory_uri.xml @@ -0,0 +1,12 @@ + + + macros.xml + + $output + ]]> + + + + + diff --git a/test/unit/tool_util/parameter_specification.yml b/test/unit/tool_util/parameter_specification.yml index 807dabe90201..699d49ec29c7 100644 --- a/test/unit/tool_util/parameter_specification.yml +++ b/test/unit/tool_util/parameter_specification.yml @@ -188,6 +188,15 @@ gx_genomebuild_multiple: # DIVERGES_FROM_CURRENT_API - parameter: null +gx_directory_uri: + request_valid: + - parameter: "gxfiles://foobar/" + - parameter: "gxfiles://foobar" + request_invalid: + - parameter: "justsomestring" + - parameter: true + - parameter: null + gx_hidden: request_valid: - parameter: moocow From 483df0c53ff861e563deb179c7d79fc247f51b12 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Aug 2024 14:59:08 -0400 Subject: [PATCH 11/14] Cleanup a test tool copy-pasta --- test/functional/tools/empty_select.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/functional/tools/empty_select.xml b/test/functional/tools/empty_select.xml index 76b3c9a03729..4c1128fb08a9 100644 --- a/test/functional/tools/empty_select.xml +++ b/test/functional/tools/empty_select.xml @@ -1,5 +1,4 @@ - multi_select Date: Thu, 8 Aug 2024 16:37:33 -0400 Subject: [PATCH 12/14] Validate wild static select with no options. --- lib/galaxy/tool_util/parameters/models.py | 7 +++-- .../tool_util/unittest_utils/parameters.py | 7 +++++ test/unit/tool_util/framework_tool_checks.yml | 8 +++++ .../tool_util/test_parameter_specification.py | 31 +++++++++++++++++-- 4 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 test/unit/tool_util/framework_tool_checks.yml diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py index 131ee4d4c732..2571ff5c59f5 100644 --- a/lib/galaxy/tool_util/parameters/models.py +++ b/lib/galaxy/tool_util/parameters/models.py @@ -469,8 +469,11 @@ class SelectParameterModel(BaseGalaxyToolParameterModelDefinition): @property def py_type(self) -> Type: if self.options is not None: - literal_options: List[Type] = [cast_as_type(Literal[o.value]) for o in self.options] - py_type = union_type(literal_options) + if len(self.options) > 0: + literal_options: List[Type] = [cast_as_type(Literal[o.value]) for o in self.options] + py_type = union_type(literal_options) + else: + py_type = type(None) else: py_type = StrictStr if self.multiple: diff --git a/lib/galaxy/tool_util/unittest_utils/parameters.py b/lib/galaxy/tool_util/unittest_utils/parameters.py index 704a2ba15091..07ec28ffcb70 100644 --- a/lib/galaxy/tool_util/unittest_utils/parameters.py +++ b/lib/galaxy/tool_util/unittest_utils/parameters.py @@ -11,6 +11,7 @@ ToolSource, ) from galaxy.util import galaxy_directory +from . import functional_test_tool_path class ParameterBundle(ToolParameterBundle): @@ -23,6 +24,12 @@ def parameter_bundle(parameter: ToolParameterT) -> ParameterBundle: return ParameterBundle(parameter) +def parameter_bundle_for_framework_tool(filename: str) -> ToolParameterBundleModel: + path = functional_test_tool_path(filename) + tool_source = get_tool_source(path, macro_paths=[]) + return input_models_for_tool_source(tool_source) + + def parameter_bundle_for_file(filename: str) -> ToolParameterBundleModel: tool_source = parameter_tool_source(filename) return input_models_for_tool_source(tool_source) diff --git a/test/unit/tool_util/framework_tool_checks.yml b/test/unit/tool_util/framework_tool_checks.yml new file mode 100644 index 000000000000..c9a6cbf661df --- /dev/null +++ b/test/unit/tool_util/framework_tool_checks.yml @@ -0,0 +1,8 @@ +# file format mirrors parameters_specification.yml but applies to any tool in the older +# (and more general) test/functional/tools directory. + +empty_select: + request_valid: + - {} + request_invalid: + - select_optional: anyoption diff --git a/test/unit/tool_util/test_parameter_specification.py b/test/unit/tool_util/test_parameter_specification.py index 52fd285a9b8a..556a6f89d76e 100644 --- a/test/unit/tool_util/test_parameter_specification.py +++ b/test/unit/tool_util/test_parameter_specification.py @@ -4,6 +4,7 @@ Callable, Dict, List, + Optional, ) import yaml @@ -21,7 +22,10 @@ validate_test_case, ) from galaxy.tool_util.parameters.json import to_json_schema_string -from galaxy.tool_util.unittest_utils.parameters import parameter_bundle_for_file +from galaxy.tool_util.unittest_utils.parameters import ( + parameter_bundle_for_file, + parameter_bundle_for_framework_tool, +) from galaxy.util.resources import resource_string RawStateDict = Dict[str, Any] @@ -37,12 +41,31 @@ def specification_object(): return yaml.safe_load(yaml_str) +def framework_tool_checks(): + # Extend parameter_specification with some extra tests against existing framework tools. + # There is something beautiful about a targeted tool for every parameter feature but realistically + # we've been doing a version of the for a decade with tool tests and we can leverage those also. + try: + yaml_str = resource_string(__package__, "framework_tool_checks.yml") + except AttributeError: + # hack for the main() function below where this file is interpreted as part of the + # Galaxy tree. + yaml_str = open("test/unit/tool_util/framework_tool_checks.yml").read() + return yaml.safe_load(yaml_str) + + def test_specification(): parameter_spec = specification_object() for file in parameter_spec.keys(): _test_file(file, parameter_spec) +def test_framework_tool_checks(): + parameter_spec = framework_tool_checks() + for file in parameter_spec.keys(): + _test_file(file, parameter_spec, parameter_bundle_for_framework_tool(f"{file}.xml")) + + def test_single(): # _test_file("gx_int") # _test_file("gx_float") @@ -54,10 +77,12 @@ def test_single(): _test_file("gx_conditional_boolean_checked") -def _test_file(file: str, specification=None): +def _test_file(file: str, specification=None, parameter_bundle: Optional[ToolParameterBundleModel] = None): spec = specification or specification_object() combos = spec[file] - parameter_bundle: ToolParameterBundleModel = parameter_bundle_for_file(file) + if parameter_bundle is None: + parameter_bundle = parameter_bundle_for_file(file) + assert parameter_bundle assertion_functions = { "request_valid": _assert_requests_validate, From 7dc12dfe273a88f68ff8b50f4af259a32efc1fac Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Aug 2024 14:56:32 -0400 Subject: [PATCH 13/14] Clarify behavior around unspecified hidden parameters. --- lib/galaxy/tool_util/parameters/factory.py | 2 ++ lib/galaxy/tool_util/parameters/models.py | 3 ++- test/unit/tool_util/parameter_specification.yml | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/parameters/factory.py b/lib/galaxy/tool_util/parameters/factory.py index e21eb4794631..ea735e7e7026 100644 --- a/lib/galaxy/tool_util/parameters/factory.py +++ b/lib/galaxy/tool_util/parameters/factory.py @@ -108,9 +108,11 @@ def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: ) elif param_type == "hidden": optional = input_source.parse_optional() + value = input_source.get("value") return HiddenParameterModel( name=input_source.parse_name(), optional=optional, + value=value, ) elif param_type == "color": optional = input_source.parse_optional() diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py index 2571ff5c59f5..0deffb02ea71 100644 --- a/lib/galaxy/tool_util/parameters/models.py +++ b/lib/galaxy/tool_util/parameters/models.py @@ -346,6 +346,7 @@ def request_requires_value(self) -> bool: class HiddenParameterModel(BaseGalaxyToolParameterModelDefinition): parameter_type: Literal["gx_hidden"] = "gx_hidden" + value: Optional[str] @property def py_type(self) -> Type: @@ -356,7 +357,7 @@ def pydantic_template(self, state_representation: StateRepresentationT) -> Dynam @property def request_requires_value(self) -> bool: - return not self.optional + return not self.optional and self.value is None def ensure_color_valid(value: Optional[Any]): diff --git a/test/unit/tool_util/parameter_specification.yml b/test/unit/tool_util/parameter_specification.yml index 699d49ec29c7..79cfeb9d1572 100644 --- a/test/unit/tool_util/parameter_specification.yml +++ b/test/unit/tool_util/parameter_specification.yml @@ -202,6 +202,8 @@ gx_hidden: - parameter: moocow - parameter: 'some spaces' - parameter: '' + # it will use a default if value is set in definition and no value supplied (see hidden_param.xml) + - {} request_invalid: - parameter: null - parameter: 5 From 39cb7f2ed5d5ce75786914fc0f7af3177d9a8584 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Aug 2024 12:24:06 -0400 Subject: [PATCH 14/14] Rebuild schema for parameter model changes... --- .../webapp/frontend/src/schema/schema.ts | 120 +++++++++++++++++- 1 file changed, 117 insertions(+), 3 deletions(-) diff --git a/lib/tool_shed/webapp/frontend/src/schema/schema.ts b/lib/tool_shed/webapp/frontend/src/schema/schema.ts index 8122e6ae0647..e12d45fc9996 100644 --- a/lib/tool_shed/webapp/frontend/src/schema/schema.ts +++ b/lib/tool_shed/webapp/frontend/src/schema/schema.ts @@ -263,6 +263,39 @@ export interface components { /** Api Key */ api_key: string } + /** BaseUrlParameterModel */ + BaseUrlParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_baseurl + * @constant + * @enum {string} + */ + parameter_type?: "gx_baseurl" + } /** Body_repositories__create_revision */ Body_repositories__create_revision: { /** Commit Message */ @@ -457,6 +490,9 @@ export interface components { | components["schemas"]["DirectoryUriParameterModel"] | components["schemas"]["RulesParameterModel"] | components["schemas"]["DrillDownParameterModel"] + | components["schemas"]["GroupTagParameterModel"] + | components["schemas"]["BaseUrlParameterModel"] + | components["schemas"]["GenomeBuildParameterModel"] | components["schemas"]["ColorParameterModel"] | components["schemas"]["ConditionalParameterModel"] | components["schemas"]["RepeatParameterModel"] @@ -843,12 +879,11 @@ export interface components { optional?: boolean /** * Parameter Type + * @default gx_directory_uri * @constant * @enum {string} */ - parameter_type: "gx_directory_uri" - /** Value */ - value: string | null + parameter_type?: "gx_directory_uri" } /** DrillDownOptionsDict */ DrillDownOptionsDict: { @@ -980,6 +1015,74 @@ export interface components { /** Value */ value?: number | null } + /** GenomeBuildParameterModel */ + GenomeBuildParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Multiple */ + multiple: boolean + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_genomebuild + * @constant + * @enum {string} + */ + parameter_type?: "gx_genomebuild" + } + /** GroupTagParameterModel */ + GroupTagParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_group_tag + * @constant + * @enum {string} + */ + parameter_type?: "gx_group_tag" + } /** HiddenParameterModel */ HiddenParameterModel: { /** Argument */ @@ -1012,6 +1115,8 @@ export interface components { * @enum {string} */ parameter_type?: "gx_hidden" + /** Value */ + value: string | null } /** ImageData */ ImageData: { @@ -1154,6 +1259,9 @@ export interface components { | components["schemas"]["DirectoryUriParameterModel"] | components["schemas"]["RulesParameterModel"] | components["schemas"]["DrillDownParameterModel"] + | components["schemas"]["GroupTagParameterModel"] + | components["schemas"]["BaseUrlParameterModel"] + | components["schemas"]["GenomeBuildParameterModel"] | components["schemas"]["ColorParameterModel"] | components["schemas"]["ConditionalParameterModel"] | components["schemas"]["RepeatParameterModel"] @@ -1237,6 +1345,9 @@ export interface components { | components["schemas"]["DirectoryUriParameterModel"] | components["schemas"]["RulesParameterModel"] | components["schemas"]["DrillDownParameterModel"] + | components["schemas"]["GroupTagParameterModel"] + | components["schemas"]["BaseUrlParameterModel"] + | components["schemas"]["GenomeBuildParameterModel"] | components["schemas"]["ColorParameterModel"] | components["schemas"]["ConditionalParameterModel"] | components["schemas"]["RepeatParameterModel"] @@ -1585,6 +1696,9 @@ export interface components { | components["schemas"]["DirectoryUriParameterModel"] | components["schemas"]["RulesParameterModel"] | components["schemas"]["DrillDownParameterModel"] + | components["schemas"]["GroupTagParameterModel"] + | components["schemas"]["BaseUrlParameterModel"] + | components["schemas"]["GenomeBuildParameterModel"] | components["schemas"]["ColorParameterModel"] | components["schemas"]["ConditionalParameterModel"] | components["schemas"]["RepeatParameterModel"]