From 3650f433aaa4a0420ec9a9d242e9c78522d97224 Mon Sep 17 00:00:00 2001 From: Michael Puehringer Date: Fri, 6 Dec 2024 00:02:13 +0100 Subject: [PATCH] feat: Pydantic v2 --- requirements.txt | 64 ++++++++++--------------------- requirements_dev.txt | 8 ++-- src/app/login/UserStoreUIMap.tsx | 1 - visyn_core/dbview.py | 4 +- visyn_core/id_mapping/manager.py | 2 +- visyn_core/plugin/model.py | 2 +- visyn_core/plugin/parser.py | 2 +- visyn_core/plugin/registry.py | 3 +- visyn_core/rdkit/img_api.py | 2 +- visyn_core/rdkit/models.py | 39 ++++++++++++------- visyn_core/security/manager.py | 2 +- visyn_core/server/utils.py | 2 +- visyn_core/server/visyn_server.py | 4 +- visyn_core/settings/model.py | 26 ++++++------- visyn_core/telemetry/__init__.py | 12 +++--- visyn_core/tests/fixtures/app.py | 8 ++-- 16 files changed, 85 insertions(+), 96 deletions(-) diff --git a/requirements.txt b/requirements.txt index 720889b6c..65ee4dc3e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,50 +1,26 @@ # a2wsgi==1.6.0 # This WSIGMiddleware is not compatible with starlette_context -alembic==1.13.2 -cachetools==5.3.3 -fastapi==0.112.2 +alembic==1.14.0 +cachetools==5.5.0 +fastapi[all]==0.115.6 json-cfg==0.4.2 openpyxl==3.1.5 -opentelemetry-api==1.27.0 -opentelemetry-exporter-otlp==1.27.0 -opentelemetry-exporter-prometheus==0.48b0 -opentelemetry-instrumentation-fastapi==0.48b0 -opentelemetry-instrumentation-httpx==0.48b0 -opentelemetry-instrumentation-logging==0.48b0 -opentelemetry-instrumentation-requests==0.48b0 -opentelemetry-instrumentation-sqlalchemy==0.48b0 -opentelemetry-instrumentation-system-metrics==0.48b0 -opentelemetry-sdk==1.27.0 -psycopg==3.2.1 -psycopg2==2.9.9 -pydantic==1.10.17 -pyjwt[crypto]==2.9.0 -pytest-postgresql==6.0.1 +opentelemetry-api==1.28.2 +opentelemetry-exporter-otlp==1.28.2 +opentelemetry-exporter-prometheus==0.49b2 +opentelemetry-instrumentation-fastapi==0.49b2 +opentelemetry-instrumentation-httpx==0.49b2 +opentelemetry-instrumentation-logging==0.49b2 +opentelemetry-instrumentation-requests==0.49b2 +opentelemetry-instrumentation-sqlalchemy==0.49b2 +opentelemetry-instrumentation-system-metrics==0.49b2 +opentelemetry-sdk==1.28.2 +psycopg==3.2.3 +psycopg2==2.9.10 +pydantic==2.10.3 +pyjwt[crypto]==2.10.1 +pytest-postgresql==6.1.1 python-dateutil==2.9.0.post0 requests==2.32.3 -sentry-sdk~=2.13.0 -SQLAlchemy>=1.4.40,<=1.4.53 +sentry-sdk~=2.19.1 +SQLAlchemy>=1.4.50,<2.0.0 starlette-context==0.3.6 -# Extras from fastapi[all], which we can't install because it requires pydantic v2: https://github.com/tiangolo/fastapi/blob/master/pyproject.toml#L79-L103 -fastapi-cli[standard]>=0.0.5 -# For the test client -httpx>=0.23.0 -# For templates -jinja2>=2.11.2 -# For forms and file uploads -python-multipart>=0.0.7 -# For Starlette's SessionMiddleware, not commonly used with FastAPI -itsdangerous>=1.1.0 -# For Starlette's schema generation, would not be used with FastAPI -pyyaml>=5.3.1 -# For UJSONResponse -ujson>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0 -# For ORJSONResponse -orjson>=3.2.1 -# To validate email fields -email-validator>=2.0.0 -# Uvicorn with uvloop -uvicorn[standard]>=0.30.6 -# Disabled as we are still on Pydantic 1: Settings management -# pydantic-settings>=2.0.0 -# Disabled as we are still on Pydantic 1: Extra Pydantic data types -# pydantic-extra-types>=2.0.0 diff --git a/requirements_dev.txt b/requirements_dev.txt index 141a1451f..ea562a5ee 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,5 @@ -black~=24.4.2 -pyright==1.1.366 +black~=24.10.0 +pyright==1.1.390 pytest-runner~=6.0.1 -pytest~=8.2.2 -ruff==0.4.10 +pytest~=8.3.4 +ruff==0.8.2 diff --git a/src/app/login/UserStoreUIMap.tsx b/src/app/login/UserStoreUIMap.tsx index 7c5150aba..5a2c5e80f 100644 --- a/src/app/login/UserStoreUIMap.tsx +++ b/src/app/login/UserStoreUIMap.tsx @@ -122,7 +122,6 @@ export function AutoLoginForm({ setError, store }: IUserStoreRenderProps) { e.stopPropagation(); login(); }} - size="sm" > here {' '} diff --git a/visyn_core/dbview.py b/visyn_core/dbview.py index 5da167201..765a29b66 100644 --- a/visyn_core/dbview.py +++ b/visyn_core/dbview.py @@ -118,13 +118,13 @@ def is_valid_replacement(self, key, value): v = self.valid_replacements[key] if isinstance(v, list): return value in v - if v == int: + if v is int: try: int(value) # try to cast value to int return True # successful type cast except ValueError: return False - if v == float: + if v is float: try: float(value) # try to cast value to float return True # successful type cast diff --git a/visyn_core/id_mapping/manager.py b/visyn_core/id_mapping/manager.py index 9b79c84e1..00b619137 100644 --- a/visyn_core/id_mapping/manager.py +++ b/visyn_core/id_mapping/manager.py @@ -203,6 +203,6 @@ def create_id_mapping_manager() -> MappingManager: # Load mapping providers providers = [] for plugin in manager.registry.list("mapping_provider"): - providers = providers + list(plugin.load().factory()) + providers = providers + list(plugin.load().factory()) # type: ignore _log.info(f"Initializing MappingManager with {len(providers)} provider(s)") return MappingManager(providers) diff --git a/visyn_core/plugin/model.py b/visyn_core/plugin/model.py index 952f4c525..62ea79239 100644 --- a/visyn_core/plugin/model.py +++ b/visyn_core/plugin/model.py @@ -2,7 +2,7 @@ from typing import Any from fastapi import FastAPI -from pydantic import BaseModel +from pydantic.v1 import BaseModel class RegHelper: diff --git a/visyn_core/plugin/parser.py b/visyn_core/plugin/parser.py index 27175d187..aebeeb8c6 100644 --- a/visyn_core/plugin/parser.py +++ b/visyn_core/plugin/parser.py @@ -3,7 +3,7 @@ from functools import cached_property, lru_cache from importlib.metadata import EntryPoint, entry_points -from pydantic import BaseModel +from pydantic.v1 import BaseModel from .. import manager from .model import AVisynPlugin, RegHelper diff --git a/visyn_core/plugin/registry.py b/visyn_core/plugin/registry.py index edb64da91..89adc2239 100644 --- a/visyn_core/plugin/registry.py +++ b/visyn_core/plugin/registry.py @@ -1,4 +1,5 @@ import logging +from typing import Any from fastapi import FastAPI @@ -30,7 +31,7 @@ def __call__(self, *args, **kwargs): self._cache = v return v - def factory(self, *args, **kwargs): + def factory(self, *args, **kwargs) -> Any: return self(*args, **kwargs) diff --git a/visyn_core/rdkit/img_api.py b/visyn_core/rdkit/img_api.py index aa18cd836..ffb7c340a 100644 --- a/visyn_core/rdkit/img_api.py +++ b/visyn_core/rdkit/img_api.py @@ -20,7 +20,7 @@ def draw_smiles( structure: SmilesMolecule, substructure: SmilesMolecule | None = None, align: SmilesMolecule | None = None, size: int = 300 ): - return draw(structure.mol, size=size, substructure=aligned(structure.mol, align and align.mol) or substructure and substructure.mol) + return draw(structure.mol, size=size, substructure=aligned(structure.mol, align and align.mol) or (substructure and substructure.mol)) @app.post("/") diff --git a/visyn_core/rdkit/models.py b/visyn_core/rdkit/models.py index 88354dbce..900947512 100644 --- a/visyn_core/rdkit/models.py +++ b/visyn_core/rdkit/models.py @@ -1,6 +1,9 @@ -from typing import ClassVar +from typing import Any, ClassVar -from pydantic import BaseModel +from pydantic import BaseModel, GetCoreSchemaHandler +from pydantic.annotated_handlers import GetJsonSchemaHandler +from pydantic.json_schema import JsonSchemaValue +from pydantic_core import core_schema from rdkit.Chem import Mol, MolFromSmarts, MolFromSmiles # type: ignore from starlette.responses import Response @@ -8,28 +11,38 @@ class SmilesMolecule(str): """We can't directly extend mol, as this would break swagger""" - parsers: ClassVar = [MolFromSmiles] - _mol: Mol - - @property - def mol(self): - return self._mol + @classmethod + def __get_pydantic_core_schema__( + cls, + _source: type[Any], + _handler: GetCoreSchemaHandler, + ) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function(cls._validate, core_schema.str_schema()) @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_json_schema__(cls, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler) -> JsonSchemaValue: + field_schema = handler(core_schema) + field_schema.update(type="string", format="email") + return field_schema @classmethod - def validate(cls, value: str | None) -> "SmilesMolecule": + def _validate(cls, input_value: str, /) -> "SmilesMolecule": for parser in cls.parsers: - mol = parser(value) + mol = parser(input_value) if mol: - sm = SmilesMolecule(value) + sm = SmilesMolecule(input_value) sm._mol = mol return sm else: raise ValueError("Unparsable smiles") + parsers: ClassVar = [MolFromSmiles] + _mol: Mol + + @property + def mol(self): + return self._mol + class SmilesSmartsMolecule(SmilesMolecule): """Try parings smiles first, then smarts""" diff --git a/visyn_core/security/manager.py b/visyn_core/security/manager.py index 9ac392f21..b9343d896 100644 --- a/visyn_core/security/manager.py +++ b/visyn_core/security/manager.py @@ -111,7 +111,7 @@ def _delegate_stores_until_not_none(self, store_method_name: str, *args): method = getattr(store, store_method_name, None) if callable(method): try: - value = method(*args) + value: User | None = method(*args) # type: ignore except Exception: _log.exception(f"Error executing {store_method_name} in {store}") else: diff --git a/visyn_core/server/utils.py b/visyn_core/server/utils.py index eff3efc34..1627a7ad5 100644 --- a/visyn_core/server/utils.py +++ b/visyn_core/server/utils.py @@ -39,7 +39,7 @@ def _init_legacy_app(app: Flask): if manager.settings.visyn_core: app.config["SECRET_KEY"] = manager.settings.secret_key - @app.errorhandler(FlaskHTTPException) + @app.errorhandler(FlaskHTTPException) # type: ignore @app.errorhandler(Exception) # type: ignore async def handle_exception(e): """Handles Flask exceptions by returning the same JSON response as FastAPI#HTTPException would.""" diff --git a/visyn_core/server/visyn_server.py b/visyn_core/server/visyn_server.py index 593370edc..03b172986 100644 --- a/visyn_core/server/visyn_server.py +++ b/visyn_core/server/visyn_server.py @@ -6,8 +6,8 @@ from fastapi import FastAPI from fastapi.middleware.wsgi import WSGIMiddleware -from pydantic import create_model -from pydantic.utils import deep_update +from pydantic.v1 import create_model +from pydantic.v1.utils import deep_update from starlette_context.middleware import RawContextMiddleware from ..settings.constants import default_logging_dict diff --git a/visyn_core/settings/model.py b/visyn_core/settings/model.py index 1fee6ef36..4699c9226 100644 --- a/visyn_core/settings/model.py +++ b/visyn_core/settings/model.py @@ -1,8 +1,8 @@ import contextlib from typing import Any, Literal -from pydantic import AnyHttpUrl, BaseConfig, BaseModel, BaseSettings, Extra, Field -from pydantic.env_settings import EnvSettingsSource, SettingsSourceCallable +from pydantic.v1 import AnyHttpUrl, BaseConfig, BaseModel, BaseSettings, Extra, Field +from pydantic.v1.env_settings import EnvSettingsSource, SettingsSourceCallable from .constants import default_logging_dict @@ -12,13 +12,13 @@ class DBMigrationSettings(BaseModel): class DisableSettings(BaseModel): - plugins: list[str] = [] - extensions: list[str] = [] + plugins: list[str] = [] # NOQA RUF012 + extensions: list[str] = [] # NOQA RUF012 class DummyStoreSettings(BaseModel): enable: bool = False - users: list[dict[str, Any]] = [ + users: list[dict[str, Any]] = [ # NOQA RUF012 { "name": "admin", "salt": "dcf46ce914154a44b1557eba91c1f50d", @@ -44,7 +44,7 @@ class AlbSecurityStoreSettings(BaseModel): enable: bool = False cookie_name: str | None = None signout_url: str | None = None - email_token_field: str | list[str] = ["email"] + email_token_field: str | list[str] = ["email"] # NOQA RUF012 """ Field in the JWT token that contains the email address of the user. """ @@ -65,7 +65,7 @@ class AlbSecurityStoreSettings(BaseModel): """ The region of the ALB to fetch the public key from. """ - decode_algorithms: list[str] = ["ES256"] + decode_algorithms: list[str] = ["ES256"] # NOQA RUF012 """ The algorithm used to sign the JWT token. See https://pyjwt.readthedocs.io/en/stable/algorithms.html for details. """ @@ -76,13 +76,13 @@ class OAuth2SecurityStoreSettings(BaseModel): cookie_name: str | None = None signout_url: str | None = None access_token_header_name: str = "X-Forwarded-Access-Token" - email_token_field: str | list[str] = ["email"] + email_token_field: str | list[str] = ["email"] # NOQA RUF012 class NoSecurityStoreSettings(BaseModel): enable: bool = False user: str = "admin" - roles: list[str] = [] + roles: list[str] = [] # NOQA RUF012 class SecurityStoreSettings(BaseModel): @@ -108,7 +108,7 @@ class BaseExporterTelemetrySettings(BaseModel): endpoint: AnyHttpUrl # could be "http://localhost:4318" headers: dict[str, str] | None = None timeout: int | None = None - kwargs: dict[str, Any] = {} + kwargs: dict[str, Any] = {} # NOQA RUF012 class MetricsExporterTelemetrySettings(BaseExporterTelemetrySettings): @@ -163,7 +163,7 @@ class SentrySettings(BaseModel): """ Proxy Sentry envelopes to this URL. Used if an internal Sentry server is used, otherwise the original DSN is used. """ - server_init_options: dict[str, Any] = {} + server_init_options: dict[str, Any] = {} # NOQA RUF012 """ Options to be passed to the Sentry SDK during initialization. """ @@ -198,7 +198,7 @@ class VisynCoreSettings(BaseModel): """ disable: DisableSettings = DisableSettings() - enabled_plugins: list[str] = [] + enabled_plugins: list[str] = [] # NOQA RUF012 # TODO: Proper typing. This is 1:1 passed to the logging.config.dictConfig(...). logging: dict = Field(default_logging_dict) @@ -231,7 +231,7 @@ class GlobalSettings(BaseSettings): secret_key: str = "VERY_SECRET_STUFF_T0IB84wlQrdMH8RVT28w" # JWT options mostly inspired by flask-jwt-extended: https://flask-jwt-extended.readthedocs.io/en/stable/options/#general-options - jwt_token_location: list[str] = ["headers", "cookies"] + jwt_token_location: list[str] = ["headers", "cookies"] # NOQA RUF012 jwt_expire_in_seconds: int = 24 * 60 * 60 jwt_refresh_if_expiring_in_seconds: int = 30 * 60 jwt_algorithm: str = "HS256" diff --git a/visyn_core/telemetry/__init__.py b/visyn_core/telemetry/__init__.py index 208c552c3..eff438cd4 100644 --- a/visyn_core/telemetry/__init__.py +++ b/visyn_core/telemetry/__init__.py @@ -57,9 +57,9 @@ def filter(self, record: logging.LogRecord) -> bool: exporter=OTLPMetricExporter( # If we are using the global exporter settings, append the metrics path endpoint=( - _append_metrics_path(exporter_settings.endpoint) + _append_metrics_path(str(exporter_settings.endpoint)) if exporter_settings == global_exporter_settings - else exporter_settings.endpoint + else str(exporter_settings.endpoint) ), headers=exporter_settings.headers, timeout=exporter_settings.timeout, @@ -123,9 +123,9 @@ def deprecated_prometheus_metrics(): OTLPSpanExporter( # If we are using the global exporter settings, append the traces path endpoint=( - _append_trace_path(exporter_settings.endpoint) + _append_trace_path(str(exporter_settings.endpoint)) if exporter_settings == global_exporter_settings - else exporter_settings.endpoint + else str(exporter_settings.endpoint) ), headers=exporter_settings.headers, timeout=exporter_settings.timeout, @@ -168,9 +168,9 @@ def shutdown_tracer_event(): OTLPLogExporter( # If we are using the global exporter settings, append the logs path endpoint=( - _append_logs_path(exporter_settings.endpoint) + _append_logs_path(str(exporter_settings.endpoint)) if exporter_settings == global_exporter_settings - else exporter_settings.endpoint + else str(exporter_settings.endpoint) ), headers=exporter_settings.headers, timeout=exporter_settings.timeout, diff --git a/visyn_core/tests/fixtures/app.py b/visyn_core/tests/fixtures/app.py index 4ff11e186..afcbd47f3 100644 --- a/visyn_core/tests/fixtures/app.py +++ b/visyn_core/tests/fixtures/app.py @@ -12,7 +12,7 @@ from ...settings import client_config -@pytest.fixture() +@pytest.fixture def _mock_plugins(monkeypatch): def mock_current_user_in_manager(self): return permissions.User(id="admin") @@ -20,14 +20,14 @@ def mock_current_user_in_manager(self): monkeypatch.setattr(SecurityManager, "current_user", property(mock_current_user_in_manager)) -@pytest.fixture() +@pytest.fixture def workspace_config() -> dict: return { "visyn_core": {"enabled_plugins": ["visyn_core"]}, } -@pytest.fixture() +@pytest.fixture def app(workspace_config) -> FastAPI: # Reset the client config globals client_config._has_been_initialized = False @@ -48,7 +48,7 @@ class MyClassAppConfigModel(BaseModel): return create_visyn_server(workspace_config=workspace_config) -@pytest.fixture() +@pytest.fixture def client(app: FastAPI) -> Generator[TestClient, Any, None]: with TestClient(app) as client: yield client