Release 14.4.0

Makefile

@@ -9,6 +9,10 @@ ruff = ruff check $(pkg_src) setup.py --line-length 140 --select E,W,F,N,I,C,B,U
 start:
 	python $(pkg_src)
 
+.PHONY: celery  ## Start the celery worker
+celery:
+	celery -A $(pkg_src).dev_celery worker --loglevel=INFO --concurrency=8 -O fair -P prefork
+
 .PHONY: all  ## Perform the most common development-time rules
 all: format lint test
 
@@ -40,17 +44,17 @@ documentation:
 .PHONY: install  ## Install the requirements
 install:
 	@if [ ! -z "${CI}" ]; then \
-		uv pip install -e . --system; \
+		uv pip install -e . --system --upgrade; \
 	else \
-		uv pip install -e .; \
+		uv pip install -e . --upgrade; \
 	fi
 
 .PHONY: develop  ## Set up the development environment
 develop:
 	@if [ ! -z "${CI}" ]; then \
-		uv pip install -e ".[develop]" --system; \
+		uv pip install -e ".[develop]" --system --upgrade; \
 	else \
-		uv pip install -e ".[develop]"; \
+		uv pip install -e ".[develop]" --upgrade; \
 	fi
 
 .PHONY: env_encrypt ## Encrypts the current ./<app>/.env

docker-compose.yml

@@ -0,0 +1,8 @@
+version: '3.0'
+
+services:
+  redis:
+    image: redis:7.4-alpine
+    ports:
+      - '6379:6379'
+    command: redis-server --requirepass admin

package.json

@@ -1,7 +1,7 @@
 {
   "name": "visyn_core",
   "description": "Core repository for datavisyn applications.",
-  "version": "14.3.0",
+  "version": "14.4.0",
   "author": {
     "name": "datavisyn GmbH",
     "email": "[email protected]",

requirements.txt

@@ -1,6 +1,8 @@
 # a2wsgi==1.6.0 # This WSIGMiddleware is not compatible with starlette_context
 alembic==1.13.2
 cachetools==5.3.3
+celery>=5.0.0,<6.0.0
+celery-types~=0.22.0
 fastapi==0.112.2
 json-cfg==0.4.2
 openpyxl==3.1.5
@@ -18,9 +20,10 @@ psycopg==3.2.1
 psycopg2==2.9.9
 pydantic==1.10.17
 pyjwt[crypto]==2.9.0
-pytest-postgresql==6.0.1
+pytest-postgresql==6.1.1
 python-dateutil==2.9.0.post0
 requests==2.32.3
+redis>=5.0.0,<6.0.0
 sentry-sdk~=2.13.0
 SQLAlchemy>=1.4.40,<=1.4.53
 starlette-context==0.3.6

src/icons/Icons.tsx

@@ -436,3 +436,21 @@ export const dvProteinBackbone: IconDefinition = {
     `M3.47472 11.5486C2.62747 10.7018 2.41571 10.0262 2.44306 9.54479C2.47063 9.05963 2.75193 8.57865 3.29892 8.14683C3.84567 7.7152 4.60047 7.38394 5.40483 7.21366C6.21113 7.04297 7.0041 7.04727 7.62078 7.22213L7.65794 7.23267L7.69597 7.23933C8.24462 7.33544 8.74122 7.38558 9.19858 7.29628C9.69694 7.19899 10.0717 6.9537 10.4099 6.61702L10.525 6.50246L10.5825 6.35054C11.3769 4.24813 11.9875 3.29079 12.4131 2.92133C12.5994 2.75959 12.69 2.76332 12.7078 2.76406C12.7452 2.76561 12.8379 2.78536 13.0066 2.91768C13.1757 3.05024 13.3594 3.24646 13.5761 3.50371C13.6473 3.58832 13.728 3.68688 13.8127 3.79046C13.9562 3.96582 14.1115 4.15576 14.253 4.31661C14.4894 4.58535 14.7715 4.87491 15.097 5.08606C15.4326 5.30372 15.8615 5.46768 16.3586 5.39711C17.0488 5.29912 17.6507 4.97154 18.1571 4.64755C18.3641 4.51516 18.5711 4.3725 18.7659 4.23821L18.8988 4.14675C19.1377 3.98274 19.3608 3.83328 19.5827 3.70369C20.0289 3.44305 20.4137 3.29738 20.7927 3.28763C21.1532 3.27835 21.5998 3.39018 22.175 3.82289C23.3411 4.70013 23.8911 5.48283 24.0885 6.11246C24.2756 6.70958 24.1706 7.2447 23.8392 7.74065C23.1271 8.80659 21.3568 9.67002 19.3561 9.86394L19.2984 9.86954L19.2422 9.88394C18.9717 9.95328 18.6096 10.0819 18.3846 10.4208C18.1748 10.7368 18.1938 11.087 18.2017 11.2321L18.2025 11.2474C18.2368 11.9069 18.4727 12.5978 18.6982 13.203C18.753 13.35 18.8075 13.4929 18.8607 13.6324C19.0417 14.1068 19.2077 14.5421 19.3195 14.9695C19.4632 15.519 19.4753 15.9164 19.3697 16.2109C19.2775 16.4678 19.049 16.7732 18.3989 17.0408C18.2555 17.0998 18.1139 17.1589 17.9739 17.2173C17.5033 17.4137 17.0515 17.6022 16.6169 17.7533C16.0424 17.953 15.6061 18.0463 15.3047 18.0298C15.166 18.0222 15.0912 17.9926 15.0514 17.9683C15.0187 17.9483 14.9746 17.9113 14.9304 17.8142C14.8246 17.5818 14.7493 17.07 14.9172 16.063C14.9853 15.6545 14.7093 15.268 14.3007 15.1999C13.8921 15.1318 13.5057 15.4078 13.4376 15.8164C13.2568 16.9011 13.2686 17.7838 13.5651 18.4355C13.7221 18.7803 13.958 19.0578 14.2688 19.248C14.5727 19.4338 14.9053 19.5102 15.2226 19.5276C15.8332 19.5611 16.5035 19.3808 17.1095 19.1701C17.5915 19.0025 18.1011 18.7898 18.5788 18.5904C18.712 18.5348 18.8429 18.4801 18.9699 18.4279C19.9008 18.0446 20.51 17.4743 20.7816 16.7175C21.0397 15.9981 20.9417 15.2436 20.7707 14.5899C20.6361 14.0753 20.4303 13.5367 20.2446 13.0507C20.1957 12.9228 20.1482 12.7985 20.1038 12.6792C19.9052 12.1462 19.7633 11.7043 19.7156 11.3338C21.8682 11.0785 24.0545 10.1186 25.0865 8.57393C25.6444 7.73888 25.8559 6.73628 25.5198 5.66386C25.1939 4.62394 24.3808 3.60527 23.0767 2.6242C22.2856 2.02908 21.5156 1.76852 20.7541 1.78812C20.011 1.80726 19.3683 2.09177 18.8261 2.40847C18.5538 2.56755 18.2925 2.74354 18.05 2.91002L17.9056 3.00933C17.7125 3.14244 17.5319 3.26685 17.3488 3.38398C16.8899 3.67754 16.5104 3.86052 16.1478 3.912C16.1185 3.91616 16.0527 3.91801 15.9133 3.8276C15.7639 3.73068 15.5907 3.56623 15.3793 3.32591C15.2485 3.17716 15.1322 3.03467 15.0073 2.88151C14.9197 2.77404 14.8277 2.66125 14.7234 2.53744C14.4952 2.26646 14.2294 1.97031 13.9322 1.73727C13.6347 1.50401 13.2443 1.28494 12.7697 1.26534C12.2756 1.24493 11.8253 1.44523 11.4298 1.78858C10.7112 2.41238 10.0122 3.64886 9.24229 5.65488C9.11183 5.76461 9.01491 5.80382 8.91115 5.82408C8.74333 5.85684 8.48463 5.8527 7.99039 5.768C7.09318 5.52254 6.06121 5.54146 5.09417 5.74618C4.11094 5.95432 3.13167 6.36778 2.36947 6.96949C1.60752 7.57101 1.00501 8.41198 0.945478 9.4597C0.885737 10.5112 1.38361 11.5794 2.41432 12.6096C2.73824 12.9333 3.24866 13.2865 3.73219 13.621C3.85865 13.7085 3.98327 13.7947 4.10224 13.8788C4.73391 14.3253 5.30217 14.7701 5.66196 15.2379C6.00378 15.6823 6.09269 16.0547 5.96 16.4468C5.80425 16.907 5.28789 17.5739 3.94155 18.4154C3.5903 18.6349 3.48352 19.0976 3.70306 19.4489C3.9226 19.8001 4.38531 19.9069 4.73656 19.6873C6.17912 18.7857 7.05722 17.8838 7.38083 16.9276C7.72748 15.9034 7.38063 15.012 6.85091 14.3233C6.33917 13.6581 5.60012 13.1007 4.96804 12.6539C4.80139 12.5361 4.64497 12.4276 4.49948 12.3266C4.04264 12.0096 3.69365 11.7675 3.47472 11.5486Z`,
   ],
 };
+
+/*
+ * Aevidence app icons
+ */
+export const dvJunction: IconDefinition = {
+  prefix: 'dv' as IconPrefix,
+  iconName: 'junction' as IconName,
+  icon: [
+    24,
+    24,
+    [],
+    null,
+    `M9.90909 19.7647H0V24H9.90909V19.7647Z
+    M24 19.7647H14.0909V24H24V19.7647Z
+    M16.8868 4.85812C16.8868 6.69237 15.8643 8.2892 14.3549 9.11596C15.2396 10.7537 16.5039 12.1457 17.1972 12.7664C17.9371 13.3696 18.364 14.3022 18.6149 15.0594C18.8801 15.86 19.0126 16.6656 19.0552 17.1858C19.1038 17.78 18.6587 18.301 18.0609 18.3493C17.4631 18.3977 16.9391 17.9551 16.8904 17.3609C16.8599 16.9876 16.7565 16.3524 16.5519 15.7348C16.3358 15.0824 16.066 14.632 15.8128 14.4302L15.79 14.4121L15.7682 14.3928C14.926 13.6444 13.2884 11.871 12.2215 9.71133C12.1481 9.71459 12.0742 9.71623 12 9.71623C11.9258 9.71623 11.8519 9.71459 11.7785 9.71133C10.7116 11.871 9.074 13.6444 8.23175 14.3928L8.21 14.4121L8.18723 14.4302C7.93401 14.632 7.66419 15.0824 7.44808 15.7348C7.24346 16.3524 7.14014 16.9876 7.10959 17.3609C7.06094 17.9551 6.53691 18.3977 5.93913 18.3493C5.34134 18.301 4.89617 17.78 4.94481 17.1858C4.98739 16.6656 5.11991 15.86 5.38514 15.0594C5.63596 14.3022 6.06293 13.3696 6.80283 12.7664C7.49615 12.1457 8.76044 10.7537 9.64505 9.11596C8.13572 8.2892 7.11316 6.69237 7.11316 4.85812C7.11316 2.17505 9.30107 0 12 0C14.6989 0 16.8868 2.17505 16.8868 4.85812Z
+    `,
+  ],
+};

src/vis/bar/BarChart.tsx

@@ -417,6 +417,7 @@ export function BarChart({
                 scrollHideDelay={0}
                 offsetScrollbars
               >
+                {/* @ts-ignore */}
                 <VariableSizeList
                   height={containerHeight - CHART_HEIGHT_MARGIN / 2}
                   itemCount={filteredUniqueFacetVals.length}

visyn_core/celery/app.py

@@ -0,0 +1,44 @@
+import logging
+import logging.config
+
+from celery import Celery
+
+from ..plugin.parser import EntryPointPlugin
+
+
+def init_celery_manager(*, plugins: list[EntryPointPlugin]):
+    """
+    Create a new Celery app and initialize it with the given plugins.
+    """
+    from .. import manager
+
+    _log = logging.getLogger(__name__)
+
+    if not manager.settings.visyn_core.celery:
+        _log.warning("No Celery settings found in configuration, skipping Celery initialization")
+        return None
+
+    manager.celery = Celery("visyn", **manager.settings.visyn_core.celery)
+
+    _log.info("Initializing celery app")
+
+    # Discover tasks from all plugins, i.e. visyn_core.tasks, visyn_plugin.tasks
+    manager.celery.autodiscover_tasks([p.id for p in plugins])
+
+    return manager.celery
+
+
+def create_celery_worker_app(*, workspace_config: dict | None = None):
+    """
+    Create a new Celery app in standalone mode, i.e. without a FastAPI instance.
+    """
+    from .. import manager
+    from ..server.visyn_server import create_visyn_server
+
+    # Create the whole FastAPI instance to ensure that the configuration and plugins are loaded, extension points are registered, database migrations are executed, ...
+    create_visyn_server(workspace_config=workspace_config)
+
+    _log = logging.getLogger(__name__)
+    _log.info("Starting celery worker")
+
+    return init_celery_manager(plugins=manager.registry.plugins)

visyn_core/celery/main.py

@@ -0,0 +1,3 @@
+from .app import create_celery_worker_app
+
+celery_app = create_celery_worker_app()

visyn_core/dev_app.py

@@ -1,10 +1,8 @@
-import os
+import pathlib
 import sys
 
 from .server.visyn_server import create_visyn_server
 
 # This app is either started via the uvicorn runner in __main__.py,
 # or as module to execute commands via `python -m <app>.dev_app db-migration exec ...`
-app = create_visyn_server(
-    start_cmd=" ".join(sys.argv[1:]), workspace_config={"_env_file": os.path.join(os.path.dirname(os.path.realpath(__file__)), ".env")}
-)
+app = create_visyn_server(start_cmd=" ".join(sys.argv[1:]), workspace_config={"_env_file": pathlib.Path(__file__).parent / ".env"})

visyn_core/dev_celery.py

@@ -0,0 +1,5 @@
+import pathlib
+
+from .celery.app import create_celery_worker_app
+
+celery_app = create_celery_worker_app(workspace_config={"_env_file": pathlib.Path(__file__).parent / ".env"})

visyn_core/manager.py

@@ -5,17 +5,24 @@
 # Additionally, we have to wrap our types/classes in '', such that they are evaluated lazily.
 # See https://peps.python.org/pep-0563/#runtime-annotation-resolution-and-type-checking for more information.
 if TYPE_CHECKING:
+    # Monkey-patch Celery to support proper type hints: https://pypi.org/project/celery-types/
+    from celery.app.task import Task
+
+    from .celery.app import Celery
     from .dbmanager import DBManager
     from .dbmigration.manager import DBMigrationManager
     from .id_mapping.manager import MappingManager
     from .plugin.registry import Registry
     from .security.manager import SecurityManager
     from .settings.model import GlobalSettings
 
+    Task.__class_getitem__ = classmethod(lambda cls, *args, **kwargs: cls)  # type: ignore[attr-defined]
+
 
 db: "DBManager" = None  # type: ignore
 db_migration: "DBMigrationManager" = None  # type: ignore
 id_mapping: "MappingManager" = None  # type: ignore
 security: "SecurityManager" = None  # type: ignore
 registry: "Registry" = None  # type: ignore
 settings: "GlobalSettings" = None  # type: ignore
+celery: "Celery" = None  # type: ignore

visyn_core/server/utils.py

@@ -1,9 +1,12 @@
 import http
 import logging
+import logging.config
 import time
 import traceback
 
 from fastapi import HTTPException
+from pydantic import create_model
+from pydantic.utils import deep_update
 
 from .. import manager
 
@@ -17,6 +20,44 @@
 _log = logging.getLogger(__name__)
 
 
+def init_settings_manager(workspace_config: dict | None = None):
+    from ..settings.model import GlobalSettings
+    from ..settings.utils import load_workspace_config
+
+    # Load the workspace config.json and initialize the global settings
+    workspace_config = workspace_config if isinstance(workspace_config, dict) else load_workspace_config()
+    # Temporary backwards compatibility: if no visyn_core config entry is found, copy the one from tdp_core.
+    if "visyn_core" not in workspace_config and "tdp_core" in workspace_config:
+        logging.warn('You are still using "tdp_core" config entries instead of "visyn_core" entries. Please migrate as soon as possible!')
+        workspace_config["visyn_core"] = workspace_config["tdp_core"]
+
+    manager.settings = GlobalSettings(**workspace_config)
+
+    # Initialize the logging
+    logging_config = manager.settings.visyn_core.logging
+
+    if manager.settings.visyn_core.log_level:
+        try:
+            logging_config["root"]["level"] = manager.settings.visyn_core.log_level
+        except KeyError:
+            logging.warn("You have set visyn_core.log_level, but no root logger is defined in visyn_core.logging")
+
+    logging.config.dictConfig(logging_config)
+
+    # Load the initial plugins
+    from ..plugin.parser import get_config_from_plugins, load_all_plugins
+
+    plugins = load_all_plugins()
+
+    # With all the plugins, load the corresponding configuration files and create a new model based on the global settings, with all plugin models as sub-models
+    [plugin_config_files, plugin_settings_models] = get_config_from_plugins(plugins)
+    visyn_server_settings = create_model("VisynServerSettings", __base__=GlobalSettings, **plugin_settings_models)  # type: ignore
+    # Patch the global settings by instantiating the new settings model with the global config, all config.json(s), and pydantic models
+    manager.settings: GlobalSettings = visyn_server_settings(**deep_update(*plugin_config_files, workspace_config))  # type: ignore
+
+    return plugins
+
+
 init_legacy_app = None
 try:
     # Flask is an optional dependency and must be added to the requirements for legacy apps.

visyn_core/server/visyn_server.py

@@ -4,21 +4,13 @@
 import threading
 from typing import Any
 
-from fastapi import FastAPI
-from fastapi.middleware.wsgi import WSGIMiddleware
-from pydantic import create_model
-from pydantic.utils import deep_update
-from starlette_context.middleware import RawContextMiddleware
-
 from ..settings.constants import default_logging_dict
 
 # Initialize the logging very early as otherwise the already created loggers receive a default loglevel WARN, leading to logs not being shown.
 logging.config.dictConfig(default_logging_dict)
 
 
-def create_visyn_server(
-    *, fast_api_args: dict[str, Any] | None = None, start_cmd: str | None = None, workspace_config: dict | None = None
-) -> FastAPI:
+def create_visyn_server(*, fast_api_args: dict[str, Any] | None = None, start_cmd: str | None = None, workspace_config: dict | None = None):
     """
     Create a new FastAPI instance while ensuring that the configuration and plugins are loaded, extension points are registered, database migrations are executed, ...
 
@@ -30,48 +22,14 @@ def create_visyn_server(
     if fast_api_args is None:
         fast_api_args = {}
     from .. import manager
-    from ..settings.model import GlobalSettings
-    from ..settings.utils import load_workspace_config
-
-    # Load the workspace config.json and initialize the global settings
-    workspace_config = workspace_config if isinstance(workspace_config, dict) else load_workspace_config()
-    # Temporary backwards compatibility: if no visyn_core config entry is found, copy the one from tdp_core.
-    if "visyn_core" not in workspace_config and "tdp_core" in workspace_config:
-        logging.warn('You are still using "tdp_core" config entries instead of "visyn_core" entries. Please migrate as soon as possible!')
-        workspace_config["visyn_core"] = workspace_config["tdp_core"]
-
-    manager.settings = GlobalSettings(**workspace_config)
-
-    # Initialize the logging
-    logging_config = manager.settings.visyn_core.logging
-
-    if manager.settings.visyn_core.log_level:
-        try:
-            logging_config["root"]["level"] = manager.settings.visyn_core.log_level
-        except KeyError:
-            logging.warn("You have set visyn_core.log_level, but no root logger is defined in visyn_core.logging")
-
-    logging.config.dictConfig(logging_config)
+    from .utils import init_settings_manager
 
-    # Filter out the metrics endpoint from the access log
-    class EndpointFilter(logging.Filter):
-        def filter(self, record: logging.LogRecord) -> bool:
-            return "GET /api/metrics" and "GET /api/health" and "GET /metrics" and "GET /health" not in record.getMessage()
-
-    logging.getLogger("uvicorn.access").addFilter(EndpointFilter())
+    plugins = init_settings_manager(workspace_config=workspace_config)
 
     _log = logging.getLogger(__name__)
-    _log.info(f"Starting visyn_server in {manager.settings.env} mode")
+    _log.info(f"Starting in {manager.settings.env} mode")
 
-    # Load the initial plugins
-    from ..plugin.parser import get_config_from_plugins, load_all_plugins
-
-    plugins = load_all_plugins()
-    # With all the plugins, load the corresponding configuration files and create a new model based on the global settings, with all plugin models as sub-models
-    [plugin_config_files, plugin_settings_models] = get_config_from_plugins(plugins)
-    visyn_server_settings = create_model("VisynServerSettings", __base__=GlobalSettings, **plugin_settings_models)  # type: ignore
-    # Patch the global settings by instantiating the new settings model with the global config, all config.json(s), and pydantic models
-    manager.settings = visyn_server_settings(**deep_update(*plugin_config_files, workspace_config))
+    from fastapi import FastAPI
 
     app = FastAPI(
         debug=manager.settings.is_development_mode,
@@ -84,6 +42,13 @@ def filter(self, record: logging.LogRecord) -> bool:
         **fast_api_args,
     )
 
+    # Filter out the metrics endpoint from the access log
+    class EndpointFilter(logging.Filter):
+        def filter(self, record: logging.LogRecord) -> bool:
+            return "GET /api/metrics" and "GET /api/health" and "GET /metrics" and "GET /health" not in record.getMessage()
+
+    logging.getLogger("uvicorn.access").addFilter(EndpointFilter())
+
     from ..middleware.exception_handler_middleware import ExceptionHandlerMiddleware
 
     # TODO: For some reason, a @app.exception_handler(Exception) is not called here. We use a middleware instead.
@@ -113,6 +78,10 @@ def filter(self, record: logging.LogRecord) -> bool:
         )
 
     # Initialize global managers.
+    from ..celery.app import init_celery_manager
+
+    app.state.celery = init_celery_manager(plugins=plugins)
+
     from ..plugin.registry import Registry
 
     app.state.registry = manager.registry = Registry()
@@ -149,6 +118,8 @@ def filter(self, record: logging.LogRecord) -> bool:
         sys.exit(0)
 
     # Load all namespace plugins as WSGIMiddleware plugins
+    from fastapi.middleware.wsgi import WSGIMiddleware
+
     from .utils import init_legacy_app, load_after_server_started_hooks
 
     namespace_plugins = manager.registry.list("namespace")
@@ -203,6 +174,8 @@ async def change_anyio_total_tokens():
 
     init_client_config(app)
 
+    from starlette_context.middleware import RawContextMiddleware
+
     from ..middleware.request_context_plugin import RequestContextPlugin
 
     # Use starlette-context to store the current request globally, i.e. accessible via context['request']

visyn_core/settings/model.py

@@ -180,6 +180,10 @@ class VisynCoreSettings(BaseModel):
     """
     sentry: SentrySettings = SentrySettings()
     """
+    Settings for celery. If not set, celery will not be initialized.
+    """
+    celery: dict[str, Any] | None = None
+    """
     Settings for Sentry. DSN will be shared via the client config.
     """
     cypress: bool = False

visyn_core/tests/fixtures/app.py

@@ -23,7 +23,13 @@ def mock_current_user_in_manager(self):
 @pytest.fixture()
 def workspace_config() -> dict:
     return {
-        "visyn_core": {"enabled_plugins": ["visyn_core"]},
+        "visyn_core": {
+            "enabled_plugins": ["visyn_core"],
+            "celery": {
+                "broker": "memory://localhost/",
+                "task_always_eager": True,
+            },
+        },
     }