From 40b95bea45e7c39953786cdc3bd6437ebf6465db Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 15 Oct 2024 08:18:45 -0300 Subject: [PATCH 01/15] Process pubsub messages in foreground by default --- app/settings/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/settings/base.py b/app/settings/base.py index 400ff90..440a737 100644 --- a/app/settings/base.py +++ b/app/settings/base.py @@ -66,5 +66,5 @@ REGISTER_ON_START = env.bool("REGISTER_ON_START", False) INTEGRATION_TYPE_SLUG = env.str("INTEGRATION_TYPE_SLUG", None) # Define a string id here e.g. "my_tracker" INTEGRATION_SERVICE_URL = env.str("INTEGRATION_SERVICE_URL", None) # Define a string id here e.g. "my_tracker" -PROCESS_PUBSUB_MESSAGES_IN_BACKGROUND = env.bool("PROCESS_PUBSUB_MESSAGES_IN_BACKGROUND", True) +PROCESS_PUBSUB_MESSAGES_IN_BACKGROUND = env.bool("PROCESS_PUBSUB_MESSAGES_IN_BACKGROUND", False) PROCESS_WEBHOOKS_IN_BACKGROUND = env.bool("PROCESS_WEBHOOKS_IN_BACKGROUND", True) From 53d707c85016846f90dedb0e24fa2b34368ac9e1 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 09:45:42 -0300 Subject: [PATCH 02/15] Support ui schema in action config models --- app/actions/core.py | 9 +- app/services/self_registration.py | 4 +- app/services/utils.py | 214 +++++++++++++++++++++++++++++- 3 files changed, 220 insertions(+), 7 deletions(-) diff --git a/app/actions/core.py b/app/actions/core.py index f646b4d..7a9d115 100644 --- a/app/actions/core.py +++ b/app/actions/core.py @@ -1,9 +1,16 @@ import importlib import inspect +from typing import Optional + from pydantic import BaseModel +from app.services.utils import UISchemaModelMixin + + +class ActionConfiguration(UISchemaModelMixin, BaseModel): + pass -class ActionConfiguration(BaseModel): +class ExecutableActionMixin: pass diff --git a/app/services/self_registration.py b/app/services/self_registration.py index 2ddce42..8596c2e 100644 --- a/app/services/self_registration.py +++ b/app/services/self_registration.py @@ -14,8 +14,6 @@ async def register_integration_in_gundi(gundi_client, type_slug=None, service_url=None): - #from ..webhooks.configurations import LiquidTechPayload - #print(GenericJsonTransformConfig.schema_json()) # Prepare the integration name and value integration_type_slug = type_slug or INTEGRATION_TYPE_SLUG if not integration_type_slug: @@ -38,6 +36,7 @@ async def register_integration_in_gundi(gundi_client, type_slug=None, service_ur _, config_model = handler action_name = action_id.replace("_", " ").title() action_schema = json.loads(config_model.schema_json()) + action_ui_schema = config_model.ui_schema() if issubclass(config_model, AuthActionConfiguration): action_type = ActionTypeEnum.AUTHENTICATION.value elif issubclass(config_model, PullActionConfiguration): @@ -53,6 +52,7 @@ async def register_integration_in_gundi(gundi_client, type_slug=None, service_ur "value": action_id, "description": f"{integration_type_name} {action_name} action", "schema": action_schema, + "ui_schema": action_ui_schema, "is_periodic_action": True if issubclass(config_model, PullActionConfiguration) else False, } ) diff --git a/app/services/utils.py b/app/services/utils.py index 673cd66..d83b901 100644 --- a/app/services/utils.py +++ b/app/services/utils.py @@ -1,9 +1,9 @@ import struct -from typing import Annotated, Union +from typing import Annotated import typing -from pydantic import create_model -from pydantic.fields import Field - +from pydantic import create_model, BaseModel +from pydantic.fields import Field, FieldInfo, Undefined, NoArgAnyCallable +from typing import Any, Dict, Optional, Union, List def find_config_for_action(configurations, action_id): @@ -167,3 +167,209 @@ def _make_field(self, factory, field, alias) -> None: ... ) + +class GlobalUISchemaOptions(BaseModel): + order: Optional[List[str]] + addable: Optional[bool] = Field(default=True) + copyable: Optional[bool] = Field(default=False) + orderable: Optional[bool] = Field(default=True) + removable: Optional[bool] = Field(default=True) + label: Optional[bool] = Field(default=True) + duplicateKeySuffixSeparator: Optional[str] = Field(default='-') + + +class UIOptions(GlobalUISchemaOptions): + classNames: Optional[str] + style: Optional[Dict[str, Any]] # Assuming style is a dictionary of CSS properties + title: Optional[str] + description: Optional[str] + placeholder: Optional[str] + help: Optional[str] + autofocus: Optional[bool] + autocomplete: Optional[str] # Type of HTMLInputElement['autocomplete'] + disabled: Optional[bool] + emptyValue: Optional[Any] + enumDisabled: Optional[Union[List[Union[str, int, bool]], None]] # List of disabled enum options + hideError: Optional[bool] + readonly: Optional[bool] + filePreview: Optional[bool] + inline: Optional[bool] + inputType: Optional[str] + rows: Optional[int] + submitButtonOptions: Optional[Dict[str, Any]] # Assuming UISchemaSubmitButtonOptions is a dict + widget: Optional[Union[str, Any]] # Either a widget implementation or its name + enumNames: Optional[List[str]] # List of labels for enum values + + +class FieldInfoWithUIOptions(FieldInfo): + + def __init__(self, *args, **kwargs): + """ + Extends the Pydantic Field class to support ui:schema generation + :param kwargs: ui_options: UIOptions + """ + self.ui_options = kwargs.pop("ui_options", None) + super().__init__(*args, **kwargs) + + def ui_schema(self, *args, **kwargs): + """Generates a UI schema from model field ui_schema""" + if not self.ui_options: + return {} + ui_schema = {} + ui_options = self.ui_options.__fields__ + for field_name, model_field in ui_options.items(): + if value := getattr(self.ui_options, field_name, model_field.default): + ui_schema[f"ui:{field_name}"] = value + return ui_schema + + +def FieldWithUIOptions( + default: Any = Undefined, + *, + default_factory: Optional[NoArgAnyCallable] = None, + alias: Optional[str] = None, + title: Optional[str] = None, + description: Optional[str] = None, + exclude: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny', Any]] = None, + include: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny', Any]] = None, + const: Optional[bool] = None, + gt: Optional[float] = None, + ge: Optional[float] = None, + lt: Optional[float] = None, + le: Optional[float] = None, + multiple_of: Optional[float] = None, + allow_inf_nan: Optional[bool] = None, + max_digits: Optional[int] = None, + decimal_places: Optional[int] = None, + min_items: Optional[int] = None, + max_items: Optional[int] = None, + unique_items: Optional[bool] = None, + min_length: Optional[int] = None, + max_length: Optional[int] = None, + allow_mutation: bool = True, + regex: Optional[str] = None, + discriminator: Optional[str] = None, + repr: bool = True, + ui_options: UIOptions = None, + **extra: Any, +) -> FieldInfoWithUIOptions: + """ + Used to provide extra information about a field, either for the model schema or complex validation. Some arguments + apply only to number fields (``int``, ``float``, ``Decimal``) and some apply only to ``str``. + + :param default: since this is replacing the field’s default, its first argument is used + to set the default, use ellipsis (``...``) to indicate the field is required + :param default_factory: callable that will be called when a default value is needed for this field + If both `default` and `default_factory` are set, an error is raised. + :param alias: the public name of the field + :param title: can be any string, used in the schema + :param description: can be any string, used in the schema + :param exclude: exclude this field while dumping. + Takes same values as the ``include`` and ``exclude`` arguments on the ``.dict`` method. + :param include: include this field while dumping. + Takes same values as the ``include`` and ``exclude`` arguments on the ``.dict`` method. + :param const: this field is required and *must* take it's default value + :param gt: only applies to numbers, requires the field to be "greater than". The schema + will have an ``exclusiveMinimum`` validation keyword + :param ge: only applies to numbers, requires the field to be "greater than or equal to". The + schema will have a ``minimum`` validation keyword + :param lt: only applies to numbers, requires the field to be "less than". The schema + will have an ``exclusiveMaximum`` validation keyword + :param le: only applies to numbers, requires the field to be "less than or equal to". The + schema will have a ``maximum`` validation keyword + :param multiple_of: only applies to numbers, requires the field to be "a multiple of". The + schema will have a ``multipleOf`` validation keyword + :param allow_inf_nan: only applies to numbers, allows the field to be NaN or infinity (+inf or -inf), + which is a valid Python float. Default True, set to False for compatibility with JSON. + :param max_digits: only applies to Decimals, requires the field to have a maximum number + of digits within the decimal. It does not include a zero before the decimal point or trailing decimal zeroes. + :param decimal_places: only applies to Decimals, requires the field to have at most a number of decimal places + allowed. It does not include trailing decimal zeroes. + :param min_items: only applies to lists, requires the field to have a minimum number of + elements. The schema will have a ``minItems`` validation keyword + :param max_items: only applies to lists, requires the field to have a maximum number of + elements. The schema will have a ``maxItems`` validation keyword + :param unique_items: only applies to lists, requires the field not to have duplicated + elements. The schema will have a ``uniqueItems`` validation keyword + :param min_length: only applies to strings, requires the field to have a minimum length. The + schema will have a ``minLength`` validation keyword + :param max_length: only applies to strings, requires the field to have a maximum length. The + schema will have a ``maxLength`` validation keyword + :param allow_mutation: a boolean which defaults to True. When False, the field raises a TypeError if the field is + assigned on an instance. The BaseModel Config must set validate_assignment to True + :param regex: only applies to strings, requires the field match against a regular expression + pattern string. The schema will have a ``pattern`` validation keyword + :param discriminator: only useful with a (discriminated a.k.a. tagged) `Union` of sub models with a common field. + The `discriminator` is the name of this common field to shorten validation and improve generated schema + :param repr: show this field in the representation + :param ui_options: UIOptions instance used to set ui properties for the ui schema + :param **extra: any additional keyword arguments will be added as is to the schema + """ + field_info = FieldInfoWithUIOptions( + default, + default_factory=default_factory, + alias=alias, + title=title, + description=description, + exclude=exclude, + include=include, + const=const, + gt=gt, + ge=ge, + lt=lt, + le=le, + multiple_of=multiple_of, + allow_inf_nan=allow_inf_nan, + max_digits=max_digits, + decimal_places=decimal_places, + min_items=min_items, + max_items=max_items, + unique_items=unique_items, + min_length=min_length, + max_length=max_length, + allow_mutation=allow_mutation, + regex=regex, + discriminator=discriminator, + repr=repr, + ui_options=ui_options, + **extra, + ) + field_info._validate() + return field_info + + +class UISchemaModelMixin: + + @classmethod + def ui_schema(cls, *args, **kwargs): + """Generates a UI schema from model""" + ui_schema = {} + # Iterate through the fields and generate UI schema + for field_name, model_field in cls.__fields__.items(): + if getattr(model_field.field_info, "ui_options", None): + ui_schema[field_name] = model_field.field_info.ui_schema() + # Include global options + if global_options := cls.__fields__.get('ui_global_options'): + if getattr(global_options, "type_", None) == GlobalUISchemaOptions: + model = global_options.default + for field_name, model_field in model.__fields__.items(): + if value := getattr(model, field_name, model_field.default): + ui_schema[f"ui:{field_name}"] = value + return ui_schema + + + @classmethod + def schema(cls, **kwargs): + # Call the parent schema method to get the original schema + json_schema_dict = super().schema(**kwargs) + + # Remove ui schema fields from the properties and definitions + properties = json_schema_dict.get('properties', {}) + for field in ["ui_options", "ui_global_options"]: + properties.pop(field, None) + json_schema_dict['properties'] = properties + definitions = json_schema_dict.get('definitions', {}) + for field in ["UIOptions", "GlobalUISchemaOptions"]: + definitions.pop(field, None) + json_schema_dict['definitions'] = definitions + return json_schema_dict From 6375455b73e286616d457494299e0a6f14755a50 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 09:57:04 -0300 Subject: [PATCH 03/15] Minor improvements --- app/actions/core.py | 4 ---- app/services/utils.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/app/actions/core.py b/app/actions/core.py index 7a9d115..a7dbe09 100644 --- a/app/actions/core.py +++ b/app/actions/core.py @@ -10,10 +10,6 @@ class ActionConfiguration(UISchemaModelMixin, BaseModel): pass -class ExecutableActionMixin: - pass - - class PullActionConfiguration(ActionConfiguration): pass diff --git a/app/services/utils.py b/app/services/utils.py index d83b901..c6241c6 100644 --- a/app/services/utils.py +++ b/app/services/utils.py @@ -212,7 +212,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def ui_schema(self, *args, **kwargs): - """Generates a UI schema from model field ui_schema""" + """Generates a UI schema from model field ui_options""" if not self.ui_options: return {} ui_schema = {} From 2a4c9a0951a1b76926717f0ff827e71250b993c2 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 11:44:38 -0300 Subject: [PATCH 04/15] Fix unit tests for self-registration --- app/services/tests/test_self_registration.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/services/tests/test_self_registration.py b/app/services/tests/test_self_registration.py index 2e18579..1ed7325 100644 --- a/app/services/tests/test_self_registration.py +++ b/app/services/tests/test_self_registration.py @@ -33,9 +33,11 @@ async def test_register_integration_with_slug_setting( 'type': 'integer' } }, + 'definitions': {}, 'title': 'MockPullActionConfiguration', 'type': 'object' }, + "ui_schema": {}, 'type': 'pull', 'value': 'pull_observations' } @@ -95,9 +97,11 @@ async def test_register_integration_with_slug_arg( 'type': 'integer' } }, + 'definitions': {}, 'title': 'MockPullActionConfiguration', 'type': 'object' }, + "ui_schema": {}, 'type': 'pull', 'value': 'pull_observations' } @@ -162,9 +166,11 @@ async def test_register_integration_with_service_url_arg( 'type': 'integer' } }, + 'definitions': {}, 'title': 'MockPullActionConfiguration', 'type': 'object' }, + "ui_schema": {}, 'type': 'pull', 'value': 'pull_observations' } @@ -229,9 +235,11 @@ async def test_register_integration_with_service_url_setting( 'type': 'integer' } }, + 'definitions': {}, 'title': 'MockPullActionConfiguration', 'type': 'object' }, + "ui_schema": {}, 'type': 'pull', 'value': 'pull_observations' } From 6bc24d6acfa93bf091a664e2cee87a28cc55a9e7 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 12:08:58 -0300 Subject: [PATCH 05/15] Test coverage for ui schemas --- app/services/tests/test_self_registration.py | 246 ++++++++++++++----- 1 file changed, 185 insertions(+), 61 deletions(-) diff --git a/app/services/tests/test_self_registration.py b/app/services/tests/test_self_registration.py index 1ed7325..9f3a241 100644 --- a/app/services/tests/test_self_registration.py +++ b/app/services/tests/test_self_registration.py @@ -22,24 +22,55 @@ async def test_register_integration_with_slug_setting( "description": f"Default type for integrations with X Tracker", "actions": [ { - 'description': 'X Tracker Pull Observations action', - 'is_periodic_action': True, - 'name': 'Pull Observations', - 'schema': { - 'properties': { - 'lookback_days': { - 'default': 10, - 'title': 'Lookback Days', - 'type': 'integer' + "type": "pull", + "name": "Pull Observations", + "value": "pull_observations", + "description": "X Tracker Pull Observations action", + "schema": { + "title": "MockPullActionConfiguration", + "type": "object", + "properties": { + "lookback_days": { + "title": "Data lookback days", + "description": "Number of days to look back for data.", + "default": 30, + "minimum": 1, "maximum": 30, + "type": "integer" + }, + "force_fetch": { + "title": "Force fetch", + "description": "Force fetch even if in a quiet period.", + "default": False, + "type": "boolean" } }, - 'definitions': {}, - 'title': 'MockPullActionConfiguration', - 'type': 'object' + "definitions": {} }, - "ui_schema": {}, - 'type': 'pull', - 'value': 'pull_observations' + "ui_schema": { + "lookback_days": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "range" + }, + "force_fetch": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "select" + }, + "ui:order": ["lookback_days", "force_fetch"], + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-" + }, + "is_periodic_action": True } ], "webhook": { @@ -86,24 +117,55 @@ async def test_register_integration_with_slug_arg( "description": f"Default type for integrations with X Tracker", "actions": [ { - 'description': 'X Tracker Pull Observations action', - 'is_periodic_action': True, - 'name': 'Pull Observations', - 'schema': { - 'properties': { - 'lookback_days': { - 'default': 10, - 'title': 'Lookback Days', - 'type': 'integer' + "type": "pull", + "name": "Pull Observations", + "value": "pull_observations", + "description": "X Tracker Pull Observations action", + "schema": { + "title": "MockPullActionConfiguration", + "type": "object", + "properties": { + "lookback_days": { + "title": "Data lookback days", + "description": "Number of days to look back for data.", + "default": 30, + "minimum": 1, "maximum": 30, + "type": "integer" + }, + "force_fetch": { + "title": "Force fetch", + "description": "Force fetch even if in a quiet period.", + "default": False, + "type": "boolean" } }, - 'definitions': {}, - 'title': 'MockPullActionConfiguration', - 'type': 'object' + "definitions": {} + }, + "ui_schema": { + "lookback_days": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "range" + }, + "force_fetch": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "select" + }, + "ui:order": ["lookback_days", "force_fetch"], + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-" }, - "ui_schema": {}, - 'type': 'pull', - 'value': 'pull_observations' + "is_periodic_action": True } ], "webhook": { @@ -155,24 +217,55 @@ async def test_register_integration_with_service_url_arg( 'service_url': service_url, "actions": [ { - 'description': 'X Tracker Pull Observations action', - 'is_periodic_action': True, - 'name': 'Pull Observations', - 'schema': { - 'properties': { - 'lookback_days': { - 'default': 10, - 'title': 'Lookback Days', - 'type': 'integer' + "type": "pull", + "name": "Pull Observations", + "value": "pull_observations", + "description": "X Tracker Pull Observations action", + "schema": { + "title": "MockPullActionConfiguration", + "type": "object", + "properties": { + "lookback_days": { + "title": "Data lookback days", + "description": "Number of days to look back for data.", + "default": 30, + "minimum": 1, "maximum": 30, + "type": "integer" + }, + "force_fetch": { + "title": "Force fetch", + "description": "Force fetch even if in a quiet period.", + "default": False, + "type": "boolean" } }, - 'definitions': {}, - 'title': 'MockPullActionConfiguration', - 'type': 'object' + "definitions": {} }, - "ui_schema": {}, - 'type': 'pull', - 'value': 'pull_observations' + "ui_schema": { + "lookback_days": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "range" + }, + "force_fetch": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "select" + }, + "ui:order": ["lookback_days", "force_fetch"], + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-" + }, + "is_periodic_action": True } ], "webhook": { @@ -213,7 +306,7 @@ async def test_register_integration_with_service_url_setting( mocker.patch("app.services.self_registration.action_handlers", mock_action_handlers) mocker.patch("app.services.self_registration.get_webhook_handler", mock_get_webhook_handler_for_fixed_json_payload) - await register_integration_in_gundi(gundi_client=mock_gundi_client_v2,) + await register_integration_in_gundi(gundi_client=mock_gundi_client_v2, ) assert mock_gundi_client_v2.register_integration_type.called mock_gundi_client_v2.register_integration_type.assert_called_with( @@ -224,24 +317,55 @@ async def test_register_integration_with_service_url_setting( 'service_url': service_url, "actions": [ { - 'description': 'X Tracker Pull Observations action', - 'is_periodic_action': True, - 'name': 'Pull Observations', - 'schema': { - 'properties': { - 'lookback_days': { - 'default': 10, - 'title': 'Lookback Days', - 'type': 'integer' + "type": "pull", + "name": "Pull Observations", + "value": "pull_observations", + "description": "X Tracker Pull Observations action", + "schema": { + "title": "MockPullActionConfiguration", + "type": "object", + "properties": { + "lookback_days": { + "title": "Data lookback days", + "description": "Number of days to look back for data.", + "default": 30, + "minimum": 1, "maximum": 30, + "type": "integer" + }, + "force_fetch": { + "title": "Force fetch", + "description": "Force fetch even if in a quiet period.", + "default": False, + "type": "boolean" } }, - 'definitions': {}, - 'title': 'MockPullActionConfiguration', - 'type': 'object' + "definitions": {} + }, + "ui_schema": { + "lookback_days": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "range" + }, + "force_fetch": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "select" + }, + "ui:order": ["lookback_days", "force_fetch"], + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-" }, - "ui_schema": {}, - 'type': 'pull', - 'value': 'pull_observations' + "is_periodic_action": True } ], "webhook": { From ee2d0d19affa3ab364463c201f64f5fcf06013df Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 12:35:02 -0300 Subject: [PATCH 06/15] Test coverage for ui schemas --- app/conftest.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/app/conftest.py b/app/conftest.py index 8f62237..2e69aec 100644 --- a/app/conftest.py +++ b/app/conftest.py @@ -1,15 +1,13 @@ import asyncio import datetime import json - import pydantic import pytest from unittest.mock import MagicMock from app import settings from gcloud.aio import pubsub -from gundi_core.schemas.v2 import Integration, IntegrationActionConfiguration, IntegrationActionSummary +from gundi_core.schemas.v2 import Integration from gundi_core.events import ( - SystemEventBaseModel, IntegrationActionCustomLog, CustomActivityLog, IntegrationActionStarted, @@ -28,8 +26,8 @@ CustomWebhookLog, LogLevel ) - from app.actions import PullActionConfiguration +from app.services.utils import GlobalUISchemaOptions, FieldWithUIOptions, UIOptions from app.webhooks import GenericJsonTransformConfig, GenericJsonPayload, WebhookPayload @@ -898,7 +896,30 @@ def mock_publish_event(gcp_pubsub_publish_response): class MockPullActionConfiguration(PullActionConfiguration): - lookback_days: int = 10 + lookback_days: int = FieldWithUIOptions( + 30, + le=30, + ge=1, + title="Data lookback days", + description="Number of days to look back for data.", + ui_options=UIOptions( + widget="range", + ) + ) + force_fetch: bool = FieldWithUIOptions( + False, + title="Force fetch", + description="Force fetch even if in a quiet period.", + ui_options=UIOptions( + widget="select", + ) + ) + ui_global_options = GlobalUISchemaOptions( + order=[ + "lookback_days", + "force_fetch", + ], + ) @pytest.fixture From 913760f82471794ed5e6f5ffcbfef6760693d1a3 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 15:08:10 -0300 Subject: [PATCH 07/15] Remove unecessary default properties from ui schema --- app/services/tests/test_self_registration.py | 61 +------------------- app/services/utils.py | 12 ++-- 2 files changed, 7 insertions(+), 66 deletions(-) diff --git a/app/services/tests/test_self_registration.py b/app/services/tests/test_self_registration.py index 9f3a241..a090865 100644 --- a/app/services/tests/test_self_registration.py +++ b/app/services/tests/test_self_registration.py @@ -48,27 +48,12 @@ async def test_register_integration_with_slug_setting( }, "ui_schema": { "lookback_days": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "range" }, "force_fetch": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "select" }, "ui:order": ["lookback_days", "force_fetch"], - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-" }, "is_periodic_action": True } @@ -143,27 +128,13 @@ async def test_register_integration_with_slug_arg( }, "ui_schema": { "lookback_days": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "range" }, "force_fetch": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "select" }, "ui:order": ["lookback_days", "force_fetch"], - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-" }, "is_periodic_action": True } @@ -243,27 +214,12 @@ async def test_register_integration_with_service_url_arg( }, "ui_schema": { "lookback_days": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "range" }, "force_fetch": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "select" }, "ui:order": ["lookback_days", "force_fetch"], - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-" }, "is_periodic_action": True } @@ -343,27 +299,12 @@ async def test_register_integration_with_service_url_setting( }, "ui_schema": { "lookback_days": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "range" }, "force_fetch": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "select" }, "ui:order": ["lookback_days", "force_fetch"], - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-" }, "is_periodic_action": True } diff --git a/app/services/utils.py b/app/services/utils.py index c6241c6..da4a1b5 100644 --- a/app/services/utils.py +++ b/app/services/utils.py @@ -170,12 +170,12 @@ def _make_field(self, factory, field, alias) -> None: class GlobalUISchemaOptions(BaseModel): order: Optional[List[str]] - addable: Optional[bool] = Field(default=True) - copyable: Optional[bool] = Field(default=False) - orderable: Optional[bool] = Field(default=True) - removable: Optional[bool] = Field(default=True) - label: Optional[bool] = Field(default=True) - duplicateKeySuffixSeparator: Optional[str] = Field(default='-') + addable: Optional[bool] + copyable: Optional[bool] + orderable: Optional[bool] + removable: Optional[bool] + label: Optional[bool] + duplicateKeySuffixSeparator: Optional[str] class UIOptions(GlobalUISchemaOptions): From bd2761ce0dac132cae9a4fa34fc9d4c1786debf6 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 15:31:47 -0300 Subject: [PATCH 08/15] Support ui schemas in webhook configs --- app/services/self_registration.py | 1 + app/webhooks/core.py | 36 +++++++++++++++++++++++-------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/app/services/self_registration.py b/app/services/self_registration.py index 8596c2e..88e68a9 100644 --- a/app/services/self_registration.py +++ b/app/services/self_registration.py @@ -70,6 +70,7 @@ async def register_integration_in_gundi(gundi_client, type_slug=None, service_ur "value": f"{integration_type_slug}_webhook", "description": f"Webhook Integration with {integration_type_name}", "schema": json.loads(config_model.schema_json()), + "ui_schema": config_model.ui_schema(), } logger.info(f"Registering '{integration_type_slug}' with actions: '{actions}'") diff --git a/app/webhooks/core.py b/app/webhooks/core.py index 855761c..25653c0 100644 --- a/app/webhooks/core.py +++ b/app/webhooks/core.py @@ -3,12 +3,11 @@ import json from typing import Optional, Union from pydantic import BaseModel -from pydantic.fields import Field from fastapi.encoders import jsonable_encoder -from app.services.utils import StructHexString +from app.services.utils import StructHexString, UISchemaModelMixin, FieldWithUIOptions, UIOptions -class WebhookConfiguration(BaseModel): +class WebhookConfiguration(UISchemaModelMixin, BaseModel): class Config: extra = "allow" @@ -19,26 +18,45 @@ class HexStringConfig(WebhookConfiguration): class DynamicSchemaConfig(WebhookConfiguration): - json_schema: dict + json_schema: dict = FieldWithUIOptions( + default={}, + description="JSON Schema to validate the data.", + ui_options=UIOptions( + widget="textarea", # ToDo: Use a better (custom) widget to render the JSON schema + ) + ) -class JQTransformConfig(BaseModel): - jq_filter: str = Field( +class JQTransformConfig(UISchemaModelMixin, BaseModel): + jq_filter: str = FieldWithUIOptions( default=".", description="JQ filter to transform JSON data.", - example=". | map(select(.isActive))" + example=". | map(select(.isActive))", + ui_options=UIOptions( + widget="textarea", # ToDo: Use a better (custom) widget to render the JQ filter + ) ) class GenericJsonTransformConfig(JQTransformConfig, DynamicSchemaConfig): - output_type: str = Field(..., description="Output type for the transformed data: 'obv' or 'event'") + output_type: str = FieldWithUIOptions( + ..., + description="Output type for the transformed data: 'obv' or 'event'", + ui_options=UIOptions( + widget="select", + options=[ + {"label": "Observations", "value": "obv"}, + {"label": "Events", "value": "event"}, + ] + ) + ) class GenericJsonTransformWithHexStrConfig(HexStringConfig, GenericJsonTransformConfig): pass -class WebhookPayload(BaseModel): +class WebhookPayload(UISchemaModelMixin, BaseModel): class Config: extra = "allow" From 1a4d90cd4ee3b53909e31564474dc7dd7523b264 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 16:07:26 -0300 Subject: [PATCH 09/15] Change default widget for output type used in generic integrations --- app/webhooks/core.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/webhooks/core.py b/app/webhooks/core.py index 25653c0..8b1288d 100644 --- a/app/webhooks/core.py +++ b/app/webhooks/core.py @@ -43,11 +43,7 @@ class GenericJsonTransformConfig(JQTransformConfig, DynamicSchemaConfig): ..., description="Output type for the transformed data: 'obv' or 'event'", ui_options=UIOptions( - widget="select", - options=[ - {"label": "Observations", "value": "obv"}, - {"label": "Events", "value": "event"}, - ] + widget="text", # ToDo: Use a select or a better widget to render the output type ) ) From 1ea4aba09d12e5d41a3c565945bc35d07beda189 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 16:56:37 -0300 Subject: [PATCH 10/15] Add test coverage for webhooks with ui schema --- app/conftest.py | 39 ++++++++++++++++++-- app/services/tests/test_self_registration.py | 28 ++++++++++++-- app/webhooks/core.py | 2 +- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/app/conftest.py b/app/conftest.py index 2e69aec..07d7733 100644 --- a/app/conftest.py +++ b/app/conftest.py @@ -28,7 +28,7 @@ ) from app.actions import PullActionConfiguration from app.services.utils import GlobalUISchemaOptions, FieldWithUIOptions, UIOptions -from app.webhooks import GenericJsonTransformConfig, GenericJsonPayload, WebhookPayload +from app.webhooks import GenericJsonTransformConfig, GenericJsonPayload, WebhookPayload, WebhookConfiguration class AsyncMock(MagicMock): @@ -137,6 +137,14 @@ def integration_v2_with_webhook(): "allowed_devices_list": {"title": "Allowed Devices List", "type": "array", "items": {}}, "deduplication_enabled": {"title": "Deduplication Enabled", "type": "boolean"}}, "required": ["allowed_devices_list", "deduplication_enabled"] + }, + "ui_schema": { + "allowed_devices_list": { + "ui:widget": "select" + }, + "deduplication_enabled": { + "ui:widget": "radio" + } } } }, @@ -216,6 +224,17 @@ def integration_v2_with_webhook_generic(): "description": "Output type for the transformed data: 'obv' or 'event'" } } + }, + "ui_schema": { + "jq_filter": { + "ui:widget": "textarea" + }, + "json_schema": { + "ui:widget": "textarea" + }, + "output_type": { + "ui:widget": "text" + } } } }, @@ -1193,9 +1212,21 @@ class MockWebhookPayloadModel(WebhookPayload): lon: float -class MockWebhookConfigModel(pydantic.BaseModel): - allowed_devices_list: list - deduplication_enabled: bool +class MockWebhookConfigModel(WebhookConfiguration): + allowed_devices_list: list = FieldWithUIOptions( + ..., + title="Allowed Devices List", + ui_options=UIOptions( + widget="list", + ) + ) + deduplication_enabled: bool = FieldWithUIOptions( + ..., + title="Deduplication Enabled", + ui_options=UIOptions( + widget="radio", + ) + ) @pytest.fixture diff --git a/app/services/tests/test_self_registration.py b/app/services/tests/test_self_registration.py index a090865..302d2a9 100644 --- a/app/services/tests/test_self_registration.py +++ b/app/services/tests/test_self_registration.py @@ -76,11 +76,16 @@ async def test_register_integration_with_slug_setting( "type": "boolean" } }, + "definitions": {}, "required": [ "allowed_devices_list", "deduplication_enabled" ] - } + }, + "ui_schema": { + "allowed_devices_list": {"ui:widget": "list"}, + "deduplication_enabled": {"ui:widget": "radio"} + }, } } ) @@ -157,11 +162,16 @@ async def test_register_integration_with_slug_arg( "type": "boolean" } }, + "definitions": {}, "required": [ "allowed_devices_list", "deduplication_enabled" ] - } + }, + "ui_schema": { + "allowed_devices_list": {"ui:widget": "list"}, + "deduplication_enabled": {"ui:widget": "radio"} + }, } } ) @@ -242,11 +252,16 @@ async def test_register_integration_with_service_url_arg( "type": "boolean" } }, + "definitions": {}, "required": [ "allowed_devices_list", "deduplication_enabled" ] - } + }, + "ui_schema": { + "allowed_devices_list": {"ui:widget": "list"}, + "deduplication_enabled": {"ui:widget": "radio"} + }, } } ) @@ -327,11 +342,16 @@ async def test_register_integration_with_service_url_setting( "type": "boolean" } }, + "definitions": {}, "required": [ "allowed_devices_list", "deduplication_enabled" ] - } + }, + "ui_schema": { + "allowed_devices_list": {"ui:widget": "list"}, + "deduplication_enabled": {"ui:widget": "radio"} + }, } } ) diff --git a/app/webhooks/core.py b/app/webhooks/core.py index 8b1288d..da32b79 100644 --- a/app/webhooks/core.py +++ b/app/webhooks/core.py @@ -52,7 +52,7 @@ class GenericJsonTransformWithHexStrConfig(HexStringConfig, GenericJsonTransform pass -class WebhookPayload(UISchemaModelMixin, BaseModel): +class WebhookPayload(BaseModel): class Config: extra = "allow" From b6fda511e42f8a35c73bdd984364993a34a809c0 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 17:54:31 -0300 Subject: [PATCH 11/15] Add usage examples for ui schemas in the readme --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/README.md b/README.md index 555d834..8f95e02 100644 --- a/README.md +++ b/README.md @@ -403,3 +403,57 @@ Sample configuration in Gundi: """ ``` Notice: This can also be combined with Dynamic Schema and JSON Transformations. In that case the hex string will be parsed first, adn then the JQ filter can be applied to the extracted data. + +### Custom UI for configurations (ui schema) +It's possible to customize how the forms for configurations are displayed in the Gundi portal. +To do that, use `FieldWithUIOptions` in your models. The `UIOptions` and `GlobalUISchemaOptions` will allow you to customize the appearance of the fields in the portal by setting any of the ["ui schema"](https://rjsf-team.github.io/react-jsonschema-form/docs/api-reference/uiSchema) supported options. + +```python +# Example +import pydantic +from app.services.utils import FieldWithUIOptions, GlobalUISchemaOptions, UIOptions +from .core import AuthActionConfiguration, PullActionConfiguration + + +class AuthenticateConfig(AuthActionConfiguration): + email: str # This will be rendered with default widget and settings + password: pydantic.SecretStr = FieldWithUIOptions( + ..., + format="password", + title="Password", + description="Password for the Global Forest Watch account.", + ui_options=UIOptions( + widget="password", # This will be rendered as a password input hiding the input + ) + ) + ui_global_options = GlobalUISchemaOptions( + order=["email", "password"], # This will set the order of the fields in the form + ) + + +class MyPullActionConfiguration(PullActionConfiguration): + lookback_days: int = FieldWithUIOptions( + 10, + le=30, + ge=1, + title="Data lookback days", + description="Number of days to look back for data.", + ui_options=UIOptions( + widget="range", # This will be rendered ad a range slider + ) + ) + force_fetch: bool = FieldWithUIOptions( + False, + title="Force fetch", + description="Force fetch even if in a quiet period.", + ui_options=UIOptions( + widget="radio", # This will be rendered as a radio button + ) + ) + ui_global_options = GlobalUISchemaOptions( + order=[ + "lookback_days", + "force_fetch", + ], + ) +``` From 3d68ea087b106b5b9a06544329906306bc94b060 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 29 Oct 2024 13:40:45 -0300 Subject: [PATCH 12/15] Update dependencies --- requirements-base.in | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements-base.in b/requirements-base.in index 9db68bd..e22173e 100644 --- a/requirements-base.in +++ b/requirements-base.in @@ -4,10 +4,11 @@ environs~=9.5.0 pydantic~=1.10.15 fastapi~=0.103.2 uvicorn~=0.23.2 -gundi-core~=1.5.0 -gundi-client-v2~=2.3.2 +gundi-core~=1.7.0 +gundi-client-v2~=2.3.8 stamina~=23.2.0 redis~=5.0.1 gcloud-aio-pubsub~=6.0.0 click~=8.1.7 -pyjq~=2.6.0 +#pyjq~=2.6.0 +python-json-logger~=2.0.7 From 633621e7029640b69c84185708cc2c2280df0645 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 29 Oct 2024 13:42:57 -0300 Subject: [PATCH 13/15] Update dependencies --- requirements-base.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-base.in b/requirements-base.in index e22173e..cb9441e 100644 --- a/requirements-base.in +++ b/requirements-base.in @@ -10,5 +10,5 @@ stamina~=23.2.0 redis~=5.0.1 gcloud-aio-pubsub~=6.0.0 click~=8.1.7 -#pyjq~=2.6.0 +pyjq~=2.6.0 python-json-logger~=2.0.7 From 8d9dca833fdeb1ad7e575b39bb082e17a4e7f304 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 5 Nov 2024 16:05:59 -0300 Subject: [PATCH 14/15] Update api mocks for integration status --- app/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/conftest.py b/app/conftest.py index 07d7733..c48044c 100644 --- a/app/conftest.py +++ b/app/conftest.py @@ -105,9 +105,9 @@ def integration_v2(): 'value': 'auth'}, 'data': {'token': 'testtoken2a97022f21732461ee103a08fac8a35'}}], 'additional': {}, 'default_route': {'id': '5abf3845-7c9f-478a-bc0f-b24d87038c4b', 'name': 'Gundi X Provider - Default Route'}, - 'status': {'id': 'mockid-b16a-4dbd-ad32-197c58aeef59', 'is_healthy': True, - 'details': 'Last observation has been delivered with success.', - 'observation_delivered_24hrs': 50231, 'last_observation_delivered_at': '2023-03-31T11:20:00+0200'}} + 'status': 'healthy', + 'status_details': '', + } ) From a33d42f6dd6118698fe43e82cee363008432d706 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 5 Nov 2024 16:09:12 -0300 Subject: [PATCH 15/15] Update api mocks for integration status --- app/conftest.py | 18 ++++-------------- requirements-base.in | 2 +- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/app/conftest.py b/app/conftest.py index c48044c..ae20973 100644 --- a/app/conftest.py +++ b/app/conftest.py @@ -169,13 +169,8 @@ def integration_v2_with_webhook(): }, "additional": {}, "default_route": None, - "status": { - "id": "mockid-b16a-4dbd-ad32-197c58aeef59", - "is_healthy": True, - "details": "Last observation has been delivered with success.", - "observation_delivered_24hrs": 50231, - "last_observation_delivered_at": "2023-03-31T11:20:00+0200" - } + "status": "healthy", + "status_details": "", } ) @@ -472,13 +467,8 @@ def integration_v2_with_webhook_generic(): }, "additional": {}, "default_route": None, - "status": { - "id": "mockid-b16a-4dbd-ad32-197c58aeef59", - "is_healthy": True, - "details": "Last observation has been delivered with success.", - "observation_delivered_24hrs": 50231, - "last_observation_delivered_at": "2023-03-31T11:20:00+0200" - } + "status": "healthy", + "status_details": "", } ) diff --git a/requirements-base.in b/requirements-base.in index cb9441e..7c4e316 100644 --- a/requirements-base.in +++ b/requirements-base.in @@ -4,7 +4,7 @@ environs~=9.5.0 pydantic~=1.10.15 fastapi~=0.103.2 uvicorn~=0.23.2 -gundi-core~=1.7.0 +gundi-core~=1.7.1 gundi-client-v2~=2.3.8 stamina~=23.2.0 redis~=5.0.1