From 9e2a3ccfab370ab85c899a222b3defbb60e39911 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 16 Dec 2024 15:00:52 +0000 Subject: [PATCH 1/7] Replace deprecated ``example`` parameter of ``fastapi.Path()`` See https://fastapi.tiangolo.com/reference/parameters/#fastapi.Path --- client/src/api/schema/schema.ts | 1 - lib/galaxy/webapps/galaxy/services/library_contents.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index d1ce7e7aaa2f..3a7c79e4de31 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -28958,7 +28958,6 @@ export interface operations { }; path: { library_id: string; - /** @example F0123456789ABCDEF */ id: string; }; cookie?: never; diff --git a/lib/galaxy/webapps/galaxy/services/library_contents.py b/lib/galaxy/webapps/galaxy/services/library_contents.py index 971c356f56d3..ee94caa1fba2 100644 --- a/lib/galaxy/webapps/galaxy/services/library_contents.py +++ b/lib/galaxy/webapps/galaxy/services/library_contents.py @@ -58,7 +58,7 @@ str, Path( title="The encoded ID of a library folder or dataset.", - example="F0123456789ABCDEF", + examples=["F0123456789ABCDEF"], min_length=16, pattern="F?[0-9a-fA-F]+", ), From b1cf76c3ea084086d43d25d3e67233a078c44839 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 17 Dec 2024 15:45:06 +0000 Subject: [PATCH 2/7] Fail earlier if archive download failed Simplify debugging this traceback (which was just an intermittent network error): https://github.com/galaxyproject/galaxy/actions/runs/12374234876/job/34536295999 ``` ______________________ ERROR at setup of test_build_index ______________________ cls = , name = None, mode = 'r' fileobj = , compresslevel = 9, kwargs = {} GzipFile = @classmethod def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): """Open gzip compressed tar archive name for reading or writing. Appending is not allowed. """ if mode not in ("r", "w", "x"): raise ValueError("mode must be 'r', 'w' or 'x'") try: from gzip import GzipFile except ImportError: raise CompressionError("gzip module is not available") try: fileobj = GzipFile(name, mode + "b", compresslevel, fileobj) except OSError: if fileobj is not None and mode == 'r': raise ReadError("not a gzip file") raise try: > t = cls.taropen(name, mode, fileobj, **kwargs) /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/tarfile.py:1854: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/tarfile.py:1831: in taropen return cls(name, mode, fileobj, **kwargs) /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/tarfile.py:1694: in __init__ self.firstmember = self.next() /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/tarfile.py:2578: in next tarinfo = self.tarinfo.fromtarfile(self) /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/tarfile.py:1282: in fromtarfile buf = tarfile.fileobj.read(BLOCKSIZE) /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/gzip.py:292: in read return self._buffer.read(size) /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/_compression.py:68: in readinto data = self.read(len(byte_view)) /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/gzip.py:479: in read if not self._read_gzip_header(): _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = def _read_gzip_header(self): magic = self._fp.read(2) if magic == b'': return False if magic != b'\037\213': > raise BadGzipFile('Not a gzipped file (%r)' % magic) E gzip.BadGzipFile: Not a gzipped file (b' tarfile.open(fileobj=b, mode="r:gz").extractall(extracted_archive_dir) test/unit/tool_shed/test_shed_index.py:30: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/tarfile.py:1801: in open return func(name, filemode, fileobj, **kwargs) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ cls = , name = None, mode = 'r' fileobj = , compresslevel = 9, kwargs = {} GzipFile = @classmethod def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs): """Open gzip compressed tar archive name for reading or writing. Appending is not allowed. """ if mode not in ("r", "w", "x"): raise ValueError("mode must be 'r', 'w' or 'x'") try: from gzip import GzipFile except ImportError: raise CompressionError("gzip module is not available") try: fileobj = GzipFile(name, mode + "b", compresslevel, fileobj) except OSError: if fileobj is not None and mode == 'r': raise ReadError("not a gzip file") raise try: t = cls.taropen(name, mode, fileobj, **kwargs) except OSError: fileobj.close() if mode == 'r': > raise ReadError("not a gzip file") E tarfile.ReadError: not a gzip file /opt/hostedtoolcache/Python/3.8.18/x64/lib/python3.8/tarfile.py:1858: ReadError ------------------------------ Captured log setup ------------------------------ DEBUG urllib3.connectionpool:connectionpool.py:1022 Starting new HTTPS connection (1): github.com:443 DEBUG urllib3.connectionpool:connectionpool.py:475 https://github.com:443 "GET /mvdbeek/toolshed-test-data/blob/master/toolshed_community_files.tgz?raw=true HTTP/1.1" 503 54894 ``` --- test/unit/tool_shed/test_shed_index.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/unit/tool_shed/test_shed_index.py b/test/unit/tool_shed/test_shed_index.py index bc929dcddfb1..880489151e2d 100644 --- a/test/unit/tool_shed/test_shed_index.py +++ b/test/unit/tool_shed/test_shed_index.py @@ -26,7 +26,9 @@ def whoosh_index_dir(): @pytest.fixture(scope="module") def community_file_dir(): extracted_archive_dir = tempfile.mkdtemp() - b = BytesIO(requests.get(URL).content) + response = requests.get(URL) + response.raise_for_status() + b = BytesIO(response.content) tarfile.open(fileobj=b, mode="r:gz").extractall(extracted_archive_dir) try: yield extracted_archive_dir From 2c3c416b1242fecc0a299c1623be99550d813004 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 17 Dec 2024 16:04:34 +0000 Subject: [PATCH 3/7] Small refactorings --- lib/galaxy/tools/parameters/basic.py | 11 +++++------ lib/galaxy/tools/parameters/grouping.py | 16 ++++++++-------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 2a150806d60b..6b06711c9c1f 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -258,13 +258,12 @@ def value_from_basic(self, value, app, ignore_errors=False): elif isinstance(value, MutableMapping) and value.get("__class__") == "UnvalidatedValue": return value["value"] # Delegate to the 'to_python' method - if ignore_errors: - try: - return self.to_python(value, app) - except Exception: - return value - else: + try: return self.to_python(value, app) + except Exception: + if not ignore_errors: + raise + return value def value_to_display_text(self, value) -> str: if is_runtime_value(value): diff --git a/lib/galaxy/tools/parameters/grouping.py b/lib/galaxy/tools/parameters/grouping.py index 26b4e171c0de..5a0e256fcbf2 100644 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -161,9 +161,9 @@ def value_from_basic(self, value, app, ignore_errors=False): else: rval_dict[input.name] = input.value_from_basic(d[input.name], app, ignore_errors) rval.append(rval_dict) - except Exception as e: + except Exception: if not ignore_errors: - raise e + raise return rval def get_initial_value(self, trans, context): @@ -226,9 +226,9 @@ def value_from_basic(self, value, app, ignore_errors=False): for input in self.inputs.values(): if not ignore_errors or input.name in value: rval[input.name] = input.value_from_basic(value[input.name], app, ignore_errors) - except Exception as e: + except Exception: if not ignore_errors: - raise e + raise return rval def get_initial_value(self, trans, context): @@ -389,9 +389,9 @@ def value_from_basic(self, value, app, ignore_errors=False): else: rval_dict[input.name] = input.value_from_basic(d[input.name], app, ignore_errors) rval.append(rval_dict) - except Exception as e: + except Exception: if not ignore_errors: - raise e + raise return rval def get_file_count(self, trans, context): @@ -794,9 +794,9 @@ def value_from_basic(self, value, app, ignore_errors=False): # conditional's values dictionary. if not ignore_errors or input.name in value: rval[input.name] = input.value_from_basic(value[input.name], app, ignore_errors) - except Exception as e: + except Exception: if not ignore_errors: - raise e + raise return rval def get_initial_value(self, trans, context): From 0f5df5f89643c0cbaffb2cd7f84c0848559ecb9c Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 17 Dec 2024 16:58:40 +0000 Subject: [PATCH 4/7] Fix validation of config files --- lib/galaxy/config/config_manage.py | 11 ++++++----- lib/galaxy/config/schemas/config_schema.yml | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/config/config_manage.py b/lib/galaxy/config/config_manage.py index a1ae2de306a9..12b0806347f2 100644 --- a/lib/galaxy/config/config_manage.py +++ b/lib/galaxy/config/config_manage.py @@ -57,7 +57,6 @@ APP_DESCRIPTION = """Application to target for operation (i.e. galaxy, tool_shed, or reports))""" DRY_RUN_DESCRIPTION = """If this action modifies files, just print what would be the result and continue.""" UNKNOWN_OPTION_MESSAGE = "Option [%s] not found in schema - either it is invalid or the Galaxy team hasn't documented it. If invalid, you should manually remove it. If the option is valid but undocumented, please file an issue with the Galaxy team." -USING_SAMPLE_MESSAGE = "Config file not found, using sample." NO_APP_MAIN_MESSAGE = "No app:main section found, using application defaults throughout." YAML_COMMENT_WRAPPER = TextWrapper( initial_indent="# ", subsequent_indent="# ", break_long_words=False, break_on_hyphens=False @@ -306,9 +305,11 @@ def _find_config(args: Namespace, app_desc: App) -> str: if os.path.exists(possible_ini_config): path = possible_ini_config - if not path: - _warn(USING_SAMPLE_MESSAGE) + if path: + print(f"Found config file {path}") + else: path = os.path.join(args.galaxy_root, app_desc.sample_destination) + _warn(f"Config file not found, using sample {path}") return path @@ -354,8 +355,8 @@ def _validate(args: Namespace, app_desc: App) -> None: path = _find_config(args, app_desc) # Allow empty mapping (not allowed by pykwalify) raw_config = _order_load_path(path) - if raw_config.get(app_desc.app_name) is None: - raw_config[app_desc.app_name] = {} + # Drop top-level keys (e.g. "gravity") except for app_desc.app_name + raw_config = {app_desc.app_name: raw_config.get(app_desc.app_name) or {}} # Rewrite the file any way to merge any duplicate keys with tempfile.NamedTemporaryFile("w", delete=False, suffix=".yml") as config_p: ordered_dump(raw_config, config_p) diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index d1e55b6a6eac..a5609884a53e 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -1717,7 +1717,7 @@ mapping: helpsite_url: type: str - required: true + required: false default: https://help.galaxyproject.org/ per_host: true desc: | From 6546205c345b3bde2e7eafb3cf9443f5f848bf55 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 17 Jul 2024 17:22:26 +0100 Subject: [PATCH 5/7] Add ``entry_points`` to the schema template --- doc/schema_template.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/schema_template.md b/doc/schema_template.md index d6577843ba2f..d3c2ac63bc77 100644 --- a/doc/schema_template.md +++ b/doc/schema_template.md @@ -21,6 +21,8 @@ number of tutorials on building Galaxy tools that would better serve that purpos $tag:tool://element[@name='tool'] $tag:tool|description://element[@name='tool']//element[@name='description'] $tag:tool|macros://complexType[@name='Macros'] +$tag:tool|entry_points://complexType[@name='EntryPoints'] +$tag:tool|entry_points|entry_point://complexType[@name='EntryPoint'] $tag:tool|edam_topics://complexType[@name='EdamTopics'] $tag:tool|edam_operations://complexType[@name='EdamOperations'] $tag:tool|xrefs://complexType[@name='xrefs'] From b100c1e23a47c3c5adcba24319cdf196f6f22bf9 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 17 Jul 2024 17:07:48 +0100 Subject: [PATCH 6/7] Reuse ``Tabular.displayable()`` in the inherited method Follow up on https://github.com/galaxyproject/galaxy/pull/18547 . --- lib/galaxy/datatypes/interval.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index 9bd5b323b8b7..e3b3cf102050 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -193,12 +193,7 @@ def set_meta( def displayable(self, dataset: DatasetProtocol) -> bool: try: return ( - not dataset.deleted - and not dataset.dataset.purged - and dataset.has_data() - and dataset.state == dataset.states.OK - and dataset.metadata.columns > 0 - and dataset.metadata.data_lines != 0 + super().displayable(dataset) and dataset.metadata.chromCol and dataset.metadata.startCol and dataset.metadata.endCol From 4e650b4e9ae8aef39309a3d149134fec2b600cf3 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 16 Dec 2024 15:02:48 +0000 Subject: [PATCH 7/7] Fix test_singularity_container_test test (which started failing since ubuntu-latest was moved to 24.04) by upgrading apptainer to >=1.3.3 to include https://github.com/apptainer/apptainer/pull/2262 . Fix: ``` > assert "samtools:1.0--1" in results["passed"], results E AssertionError: E {'failed': [{'commands': ['python -c "import pyBigWig; assert(pyBigWig.numpy ' E '== 1); assert(pyBigWig.remote == 1)"'], E 'container': 'pybigwig:0.3.22--py36h54a71a5_0', E 'errors': [{'command': 'python -c "import pyBigWig; ' E 'assert(pyBigWig.numpy == 1); ' E 'assert(pyBigWig.remote == 1)"', E 'output': '\x1b[91mERROR : Could not write info to ' E 'setgroups: Permission denied\n' E '\x1b[0m\x1b[91mERROR : Error while ' E 'waiting event for user namespace mappings: ' E 'no event received\n' E '\x1b[0m'}, E {'import': 'pyBigWig', E 'output': '\x1b[91mERROR : Could not write info to ' E 'setgroups: Permission denied\n' E '\x1b[0m\x1b[91mERROR : Error while ' E 'waiting event for user namespace mappings: ' E 'no event received\n' E '\x1b[0m'}], E 'import_lang': 'python -c', E 'imports': ['pyBigWig']}, E {'commands': ['samtools view --help 2>&1 | grep Notes > /dev/null'], E 'container': 'samtools:1.0--1', E 'errors': [{'command': 'samtools view --help 2>&1 | grep Notes > ' E '/dev/null', E 'output': '\x1b[91mERROR : Could not write info to ' E 'setgroups: Permission denied\n' E '\x1b[0m\x1b[91mERROR : Error while ' E 'waiting event for user namespace mappings: ' E 'no event received\n' E '\x1b[0m'}], E 'import_lang': 'python -c'}], E 'notest': ['yasm:1.3.0--0'], E 'passed': []} E assert 'samtools:1.0--1' in [] ``` Also: - Improve assertion messages to facilitate debugging --- .github/workflows/mulled.yaml | 2 ++ pytest.ini | 2 +- .../mulled/test_mulled_update_singularity_containers.py | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/mulled.yaml b/.github/workflows/mulled.yaml index 4d654af6ddbb..a5702df1b6a7 100644 --- a/.github/workflows/mulled.yaml +++ b/.github/workflows/mulled.yaml @@ -41,6 +41,8 @@ jobs: key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-mulled - name: Install Apptainer's singularity uses: eWaterCycle/setup-apptainer@v2 + with: + apptainer-version: 1.3.6 # https://github.com/eWaterCycle/setup-apptainer/pull/68 - name: Install tox run: pip install tox - name: Run tests diff --git a/pytest.ini b/pytest.ini index 31665649c393..706dc325bb03 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = --doctest-continue-on-failure --verbosity=1 +addopts = --doctest-continue-on-failure --verbosity=1 --showlocals asyncio_mode = auto log_level = DEBUG # Install pytest-memray and set memray to true here to enable memory profiling of tests diff --git a/test/unit/tool_util/mulled/test_mulled_update_singularity_containers.py b/test/unit/tool_util/mulled/test_mulled_update_singularity_containers.py index 728757b52f31..9ec4e47b44f8 100644 --- a/test/unit/tool_util/mulled/test_mulled_update_singularity_containers.py +++ b/test/unit/tool_util/mulled/test_mulled_update_singularity_containers.py @@ -46,6 +46,6 @@ def test_singularity_container_test(tmp_path) -> None: "singularity", tmp_path, ) - assert "samtools:1.0--1" in results["passed"] - assert "pybigwig:0.3.22--py36h54a71a5_0" in results["passed"] - assert "yasm:1.3.0--0" in results["notest"] + assert "samtools:1.0--1" in results["passed"], results + assert "pybigwig:0.3.22--py36h54a71a5_0" in results["passed"], results + assert "yasm:1.3.0--0" in results["notest"], results