From f0420d70a3242fd3303b446e63cd988376f7afa3 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 6 Sep 2024 11:20:20 +0100 Subject: [PATCH 1/4] Pin minimum package versions for Python 3.13 support Also: - Cheetah3 package on PyPI has been renamed to CT3 since 3.3.0, see https://cheetahtemplate.org/news.html --- .../dependencies/conditional-requirements.txt | 2 +- .../dependencies/pinned-requirements.txt | 2 +- packages/app/setup.cfg | 4 ++-- packages/data/setup.cfg | 3 +-- packages/files/setup.cfg | 1 - packages/test_base/setup.cfg | 2 +- packages/tool_shed/setup.cfg | 2 +- packages/util/setup.cfg | 2 +- packages/web_apps/setup.cfg | 6 +++--- pyproject.toml | 18 +++++++++--------- 10 files changed, 20 insertions(+), 22 deletions(-) diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index d59bd0916e69..168fea8adcd0 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -1,5 +1,5 @@ # These dependencies are only required when certain config options are set -psycopg2-binary==2.9.9 +psycopg2-binary==2.9.10 mysqlclient fluent-logger sentry-sdk[fastapi] diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index c911f9b8b305..827d67b8354d 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -39,7 +39,6 @@ celery==5.4.0 certifi==2024.8.30 cffi==1.17.1 charset-normalizer==3.4.0 -cheetah3==3.2.6.post1 circus==0.18.0 click==8.1.7 click-didyoumean==0.3.1 @@ -51,6 +50,7 @@ colorama==0.4.6 ; platform_system == 'Windows' coloredlogs==15.0.1 conda-package-streaming==0.11.0 cryptography==43.0.3 +ct3==3.3.3.post1 cwl-upgrader==1.2.11 cwl-utils==0.35 cwltool==3.1.20241007082533 diff --git a/packages/app/setup.cfg b/packages/app/setup.cfg index 8864f0c693d7..fe0dfe430c91 100644 --- a/packages/app/setup.cfg +++ b/packages/app/setup.cfg @@ -57,7 +57,7 @@ install_requires = Mako Markdown MarkupSafe - mercurial + mercurial>=6.8.2 packaging paramiko!=2.9.0,!=2.9.1 pebble @@ -74,7 +74,7 @@ install_requires = starlette svgwrite typing-extensions - WebOb + WebOb>=1.8.9 Whoosh packages = find: python_requires = >=3.8 diff --git a/packages/data/setup.cfg b/packages/data/setup.cfg index b8916e8ce2de..01f623c7d199 100644 --- a/packages/data/setup.cfg +++ b/packages/data/setup.cfg @@ -46,7 +46,7 @@ install_requires = h5grove>=1.2.1 h5py isa-rwval @ git+https://github.com/nsoranzo/isa-rwval.git@master - isal + isal>=1.7.0 MarkupSafe msal mrcfile @@ -62,7 +62,6 @@ install_requires = SQLAlchemy>=2.0,<2.1,!=2.0.36 tifffile typing-extensions - WebOb packages = find: python_requires = >=3.8 diff --git a/packages/files/setup.cfg b/packages/files/setup.cfg index 9bf5eec1d63f..81e727080f7a 100644 --- a/packages/files/setup.cfg +++ b/packages/files/setup.cfg @@ -34,7 +34,6 @@ include_package_data = True install_requires = galaxy-util[config_template] fs - isal typing-extensions packages = find: python_requires = >=3.8 diff --git a/packages/test_base/setup.cfg b/packages/test_base/setup.cfg index 2fb287bad14c..62247a0ae264 100644 --- a/packages/test_base/setup.cfg +++ b/packages/test_base/setup.cfg @@ -35,7 +35,7 @@ install_requires = galaxy-tool-util galaxy-util bioblend - cwltest + cwltest>=2.5.20240906231108 gxformat2 pytest PyYAML diff --git a/packages/tool_shed/setup.cfg b/packages/tool_shed/setup.cfg index 217d4d4d2f1e..7cd328385896 100644 --- a/packages/tool_shed/setup.cfg +++ b/packages/tool_shed/setup.cfg @@ -48,7 +48,7 @@ install_requires = graphql-core Mako MarkupSafe - mercurial + mercurial>=6.8.2 Paste pydantic>=2.7.4 Routes diff --git a/packages/util/setup.cfg b/packages/util/setup.cfg index 064f331f621c..bb6a232cc1e2 100644 --- a/packages/util/setup.cfg +++ b/packages/util/setup.cfg @@ -51,7 +51,7 @@ python_requires = >=3.7 jstree = dictobj template = - Cheetah3 + CT3>=3.3.3 future>=1.0.0 config_template = Jinja2 diff --git a/packages/web_apps/setup.cfg b/packages/web_apps/setup.cfg index bcf88a282c43..19ddb35ae0b3 100644 --- a/packages/web_apps/setup.cfg +++ b/packages/web_apps/setup.cfg @@ -42,7 +42,7 @@ install_requires = a2wsgi apispec Babel - Cheetah3 + CT3>=3.3.3 fastapi>=0.101.0 gunicorn gxformat2 @@ -63,8 +63,8 @@ install_requires = tuswsgi typing-extensions uvicorn - uvloop - WebOb + uvloop>=0.21.0 + WebOb>=1.8.9 packages = find: python_requires = >=3.8 diff --git a/pyproject.toml b/pyproject.toml index 131444f7c10b..145ddce07bf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,11 +25,11 @@ dependencies = [ "boto", "bx-python", "celery", - "Cheetah3", "cloudauthz==0.6.0", "cloudbridge", "circus", "conda-package-streaming", + "CT3>=3.3.3", # Python 3.13 support "cwltool>=3.1.20230624081518", # save time, minimum needed by cwl-1.0 branch "dictobj", "dnspython", @@ -50,14 +50,14 @@ dependencies = [ "h5py>=3.12 ; python_version>='3.9'", # Python 3.13 support "importlib-resources ; python_version<'3.9'", # for importlib.{abc.Traversable,resources.{files, Package}} "isa-rwval @ git+https://github.com/nsoranzo/isa-rwval.git@master", # https://github.com/ISA-tools/isa-rwval/pull/17 - "isal", + "isal>=1.7.0", # Python 3.13 support "jinja2", "kombu>=5.3", # for importlib-metadata fix "lagom", "Mako", "Markdown", "MarkupSafe", - "mercurial", + "mercurial>=6.8.2", # Python 3.13 support "mrcfile", "msal", "nodeenv", @@ -89,7 +89,7 @@ dependencies = [ "rocrate", "Routes", "s3fs>=2023.1.0", - "schema-salad!=8.3.20220721194857", # https://github.com/common-workflow-language/schema_salad/issues/575 + "schema-salad>=8.7.20240905150001", # Python 3.13 support "social-auth-core>=4.5.0", # to drop dependency on abandoned python-jose "sortedcontainers", "SQLAlchemy>=2.0,<2.1,!=2.0.36", # https://github.com/sqlalchemy/sqlalchemy/issues/12019, @@ -103,8 +103,8 @@ dependencies = [ "tuswsgi", "typing-extensions", "uvicorn!=0.28.0", # https://github.com/galaxyproject/galaxy/issues/17669 - "uvloop", - "WebOb", + "uvloop>=0.21.0", # Python 3.13 support + "WebOb>=1.8.9", # Python 3.13 support "Whoosh", "zipstream-new", ] @@ -122,7 +122,7 @@ dev = [ "black", "boto3", "codespell", - "cwltest", + "cwltest>=2.5.20240906231108", # Python 3.13 support "darker", "fluent-logger", "galaxy-release-util", @@ -133,7 +133,7 @@ dev = [ "myst-parser", "onedatafilerestclient==21.2.5.2", "pkce", - "playwright", + "playwright>=1.48.0", # Python 3.13 support "pytest<8", # https://github.com/galaxyproject/galaxy/issues/17561 "pytest-asyncio", "pytest-cov", @@ -154,7 +154,7 @@ dev = [ "statsd", "testfixtures", "tuspy", - "twill>=3.2.1", # https://github.com/twill-tools/twill/issues/18 + "twill>=3.2.5", # Python 3.13 support "watchdog", "Werkzeug", ] From 695e86eaa7e131a10532fad81531e2e312d453d4 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 28 Jun 2024 09:07:49 +0100 Subject: [PATCH 2/4] Test under Python 3.13 --- .github/workflows/first_startup.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/lint_openapi_schema.yml | 2 +- .github/workflows/osx_startup.yaml | 2 +- .github/workflows/reports_startup.yaml | 2 +- .github/workflows/test_galaxy_packages.yaml | 2 +- .github/workflows/toolshed.yaml | 2 +- .github/workflows/unit.yaml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/first_startup.yaml b/.github/workflows/first_startup.yaml index e35170d063a0..05efc45367b9 100644 --- a/.github/workflows/first_startup.yaml +++ b/.github/workflows/first_startup.yaml @@ -23,7 +23,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.12'] + python-version: ['3.8', '3.13'] defaults: run: shell: bash -l {0} diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 130be9861afb..37d337fb95fe 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.12'] + python-version: ['3.8', '3.13'] env: LINT_PATH: 'lib/galaxy/dependencies/pinned-lint-requirements.txt' TYPE_PATH: 'lib/galaxy/dependencies/pinned-typecheck-requirements.txt' diff --git a/.github/workflows/lint_openapi_schema.yml b/.github/workflows/lint_openapi_schema.yml index 2a94508ce6f2..4dc692fc2739 100644 --- a/.github/workflows/lint_openapi_schema.yml +++ b/.github/workflows/lint_openapi_schema.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.12'] + python-version: ['3.8', '3.13'] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/osx_startup.yaml b/.github/workflows/osx_startup.yaml index 22ffc4d7e5b6..13960eb8e85f 100644 --- a/.github/workflows/osx_startup.yaml +++ b/.github/workflows/osx_startup.yaml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.12'] + python-version: ['3.8', '3.13'] defaults: run: shell: bash -l {0} diff --git a/.github/workflows/reports_startup.yaml b/.github/workflows/reports_startup.yaml index 137c1a5e1204..b7449dc5979c 100644 --- a/.github/workflows/reports_startup.yaml +++ b/.github/workflows/reports_startup.yaml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.12'] + python-version: ['3.8', '3.13'] defaults: run: shell: bash -l {0} diff --git a/.github/workflows/test_galaxy_packages.yaml b/.github/workflows/test_galaxy_packages.yaml index 293e98e658bd..f38f05846e74 100644 --- a/.github/workflows/test_galaxy_packages.yaml +++ b/.github/workflows/test_galaxy_packages.yaml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.12'] + python-version: ['3.8', '3.13'] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/toolshed.yaml b/.github/workflows/toolshed.yaml index fae33b4e1225..14be278e16bc 100644 --- a/.github/workflows/toolshed.yaml +++ b/.github/workflows/toolshed.yaml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.12'] + python-version: ['3.8', '3.13'] shed-api: ['v1', 'v2'] test-install-client: ['galaxy_api', 'standalone'] services: diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml index 80f54e87a704..671dcac0162d 100644 --- a/.github/workflows/unit.yaml +++ b/.github/workflows/unit.yaml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.12'] + python-version: ['3.8', '3.13'] steps: - uses: actions/checkout@v4 with: From ac480e7df51ecc438f9f64ea906ab1ee825408b1 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 5 Sep 2024 15:40:22 +0100 Subject: [PATCH 3/4] Drop use of deprecated imghdr standard library module which has been removed in the upcoming Python 3.13 . --- lib/galaxy/util/image_util.py | 5 ----- packages/app/setup.cfg | 2 +- packages/data/setup.cfg | 2 +- packages/test.sh | 2 +- packages/tool_shed/setup.cfg | 2 +- packages/tool_util/setup.cfg | 2 +- packages/util/setup.cfg | 2 ++ test/unit/util/test_checkers.py | 13 ++++++++++++- 8 files changed, 19 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/util/image_util.py b/lib/galaxy/util/image_util.py index 3b11a50d2fda..b8c26cf0ff54 100644 --- a/lib/galaxy/util/image_util.py +++ b/lib/galaxy/util/image_util.py @@ -1,6 +1,5 @@ """Provides utilities for working with image files.""" -import imghdr import logging from typing import ( List, @@ -22,11 +21,7 @@ def image_type(filename: str) -> Optional[str]: with Image.open(filename) as im: fmt = im.format except Exception: - # We continue to try with imghdr, so this is a rare case of an - # exception we expect to happen frequently, so we're not logging pass - if not fmt: - fmt = imghdr.what(filename) if fmt: return fmt.upper() else: diff --git a/packages/app/setup.cfg b/packages/app/setup.cfg index fe0dfe430c91..7f3e9775529e 100644 --- a/packages/app/setup.cfg +++ b/packages/app/setup.cfg @@ -41,7 +41,7 @@ install_requires = galaxy-objectstore galaxy-tool-util[cwl,edam] galaxy-tours - galaxy-util + galaxy-util[image_util] galaxy-web-framework galaxy-web-stack Beaker diff --git a/packages/data/setup.cfg b/packages/data/setup.cfg index 01f623c7d199..2dd4e7e16b07 100644 --- a/packages/data/setup.cfg +++ b/packages/data/setup.cfg @@ -36,7 +36,7 @@ install_requires = galaxy-objectstore galaxy-schema galaxy-tool-util - galaxy-util[template] + galaxy-util[image_util,template] alembic alembic-utils bdbag>=1.6.3 diff --git a/packages/test.sh b/packages/test.sh index d2ed377c352b..2120ac2b922a 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -49,7 +49,7 @@ while read -r package_dir || [ -n "$package_dir" ]; do # https://stackoverflow. # Install extras (if needed) if [ "$package_dir" = "util" ]; then - pip install '.[template,jstree,config_template]' + pip install '.[image_util,template,jstree,config_template]' elif [ "$package_dir" = "tool_util" ]; then pip install '.[cwl,mulled,edam,extended-assertions]' else diff --git a/packages/tool_shed/setup.cfg b/packages/tool_shed/setup.cfg index 7cd328385896..0e9de75e1a4e 100644 --- a/packages/tool_shed/setup.cfg +++ b/packages/tool_shed/setup.cfg @@ -36,7 +36,7 @@ install_requires = galaxy-auth galaxy-config galaxy-data - galaxy-util + galaxy-util[image_util] galaxy-web-framework galaxy-web-stack galaxy-web-apps diff --git a/packages/tool_util/setup.cfg b/packages/tool_util/setup.cfg index eabca1a6e1b9..c0cdb4935917 100644 --- a/packages/tool_util/setup.cfg +++ b/packages/tool_util/setup.cfg @@ -33,7 +33,7 @@ version = 24.2.dev0 [options] include_package_data = True install_requires = - galaxy-util>=22.1 + galaxy-util[image_util]>=22.1 conda-package-streaming lxml!=4.2.2 MarkupSafe diff --git a/packages/util/setup.cfg b/packages/util/setup.cfg index bb6a232cc1e2..09380cd65be1 100644 --- a/packages/util/setup.cfg +++ b/packages/util/setup.cfg @@ -48,6 +48,8 @@ packages = find: python_requires = >=3.7 [options.extras_require] +image_util = + pillow jstree = dictobj template = diff --git a/test/unit/util/test_checkers.py b/test/unit/util/test_checkers.py index 1e30d25656cd..b474defc965a 100644 --- a/test/unit/util/test_checkers.py +++ b/test/unit/util/test_checkers.py @@ -1,6 +1,10 @@ +import os.path import tempfile -from galaxy.util.checkers import check_html +from galaxy.util.checkers import ( + check_html, + check_image, +) def test_check_html(): @@ -17,3 +21,10 @@ def test_check_html(): tmpb.write(b"\x1f\x8b") tmpb.flush() assert not check_html(tmpb.name) + + +def test_check_image(): + for filename, expected in (("1.tiff", True), ("454Score.png", True), ("1.bam", False)): + path = f"test-data/{filename}" + assert os.path.exists(path) + assert check_image(path) is expected From febd6f9413756f2c3e7144a65eb7417c22fd7a11 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 23 Oct 2024 11:57:55 +0100 Subject: [PATCH 4/4] Use fissix to replace lib2to3 under Python >=3.13 Add entries to `sys.modules` so that `future`/`past` can find lib2to3 modules. --- .../dependencies/pinned-requirements.txt | 1 + lib/galaxy/util/template.py | 20 +++++++++++++++++-- packages/util/setup.cfg | 1 + pyproject.toml | 1 + 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 827d67b8354d..6227984653a6 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -66,6 +66,7 @@ email-validator==2.2.0 exceptiongroup==1.2.2 ; python_full_version < '3.11' fastapi-slim==0.115.4 filelock==3.16.1 +fissix==24.4.24 ; python_full_version >= '3.13' frozenlist==1.5.0 fs==2.4.16 fsspec==2024.10.0 diff --git a/lib/galaxy/util/template.py b/lib/galaxy/util/template.py index 9b7f7965592e..01a2cd68c840 100644 --- a/lib/galaxy/util/template.py +++ b/lib/galaxy/util/template.py @@ -1,18 +1,34 @@ """Entry point for the usage of Cheetah templating within Galaxy.""" +import sys import traceback -from lib2to3.refactor import RefactoringTool from Cheetah.Compiler import Compiler from Cheetah.NameMapper import NotFound from Cheetah.Parser import ParseError from Cheetah.Template import Template from packaging.version import Version -from past.translation import myfixes from galaxy.util.tree_dict import TreeDict from . import unicodify +if sys.version_info >= (3, 13): + import fissix + from fissix import ( + fixes as fissix_fixes, + pgen2 as fissix_pgen2, + refactor as fissix_refactor, + ) + + sys.modules["lib2to3"] = fissix + sys.modules["lib2to3.fixes"] = fissix_fixes + sys.modules["lib2to3.pgen2"] = fissix_pgen2 + sys.modules["lib2to3.refactor"] = fissix_refactor + +from lib2to3.refactor import RefactoringTool + +from past.translation import myfixes + # Skip libpasteurize fixers, which make sure code is py2 and py3 compatible. # This is not needed, we only translate code on py3. myfixes = [f for f in myfixes if not f.startswith("libpasteurize")] diff --git a/packages/util/setup.cfg b/packages/util/setup.cfg index 09380cd65be1..808606b77042 100644 --- a/packages/util/setup.cfg +++ b/packages/util/setup.cfg @@ -54,6 +54,7 @@ jstree = dictobj template = CT3>=3.3.3 + fissix;python_version>='3.13' future>=1.0.0 config_template = Jinja2 diff --git a/pyproject.toml b/pyproject.toml index 145ddce07bf9..b50664156ef4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "dparse", "edam-ontology", "fastapi-slim>=0.111.0", + "fissix ; python_version>='3.13'", "fs", "future>=1.0.0", # Python 3.12 support "graphene",