From 3de2917f1357d013a89fdbb0f5691ae0813ce72c Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 25 Jun 2024 17:41:14 -0300 Subject: [PATCH 1/5] Consume activity events from webhook integrations --- .../api/v2/tests/test_integrations_api.py | 2 +- .../integration_events_consumer.py | 98 ++++++++++++++++++- 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/cdip_admin/api/v2/tests/test_integrations_api.py b/cdip_admin/api/v2/tests/test_integrations_api.py index 2a2126ca0..f9e812777 100644 --- a/cdip_admin/api/v2/tests/test_integrations_api.py +++ b/cdip_admin/api/v2/tests/test_integrations_api.py @@ -27,7 +27,7 @@ def _test_list_integrations(api_client, user, organization): if user.is_superuser: # The superuser can see all the integrations integrations_qs = Integration.objects.all() - else: # Only see integrations owned by the organization(s) where the user is a member + else: # Only see integrations owned by the organization(s) where the user is a member integrations_qs = Integration.objects.filter(owner=organization) expected_integrations_ids = [str(uid) for uid in integrations_qs.values_list("id", flat=True)] assert len(integrations) == len(expected_integrations_ids) diff --git a/cdip_admin/event_consumers/integration_events_consumer.py b/cdip_admin/event_consumers/integration_events_consumer.py index 303429cd0..44b5d90c7 100644 --- a/cdip_admin/event_consumers/integration_events_consumer.py +++ b/cdip_admin/event_consumers/integration_events_consumer.py @@ -15,6 +15,11 @@ logger = logging.getLogger(__name__) +def _clean_event_title(title: str) -> str: + title = title.strip() + return title if len(title) < 200 else f"{title[:197]}..." + + def handle_integration_action_started_event(event_dict: dict): event = system_events.IntegrationActionStarted.parse_obj(event_dict) event_data = event.payload @@ -29,7 +34,7 @@ def handle_integration_action_started_event(event_dict: dict): origin=ActivityLog.Origin.INTEGRATION, integration=integration, value="integration_action_started", - title=message, + title=_clean_event_title(message), details=event_dict["payload"], is_reversible=False, ) @@ -49,7 +54,7 @@ def handle_integration_action_complete_event(event_dict: dict): origin=ActivityLog.Origin.INTEGRATION, integration=integration, value="integration_action_complete", - title=message, + title=_clean_event_title(message), details=event_dict["payload"], is_reversible=False, ) @@ -70,7 +75,7 @@ def handle_integration_action_failed_event(event_dict: dict): origin=ActivityLog.Origin.INTEGRATION, integration=integration, value="integration_action_failed", - title=message, + title=_clean_event_title(message), details=event_dict["payload"], is_reversible=False, ) @@ -89,17 +94,102 @@ def handle_integration_action_custom_log_event(event_dict: dict): origin=ActivityLog.Origin.INTEGRATION, integration=integration, value="integration_custom_log", - title=custom_log.title, + title=_clean_event_title(custom_log.title), details=event_dict["payload"], is_reversible=False, ) +def handle_integration_webhook_started_event(event_dict: dict): + event = system_events.IntegrationWebhookStarted.parse_obj(event_dict) + event_data = event.payload + integration_id = event_data.integration_id + integration = Integration.objects.get(id=integration_id) + message = f"Webhook request received." + logger.info(message, extra={"event": event_dict}) + ActivityLog.objects.create( + log_level=ActivityLog.LogLevels.INFO, + log_type=ActivityLog.LogTypes.EVENT, + origin=ActivityLog.Origin.INTEGRATION, + integration=integration, + value="integration_webhook_started", + title=_clean_event_title(message), + details=event_dict["payload"], + is_reversible=False, + ) + + +def handle_integration_webhook_complete_event(event_dict: dict): + event = system_events.IntegrationWebhookComplete.parse_obj(event_dict) + event_data = event.payload + integration_id = event_data.integration_id + integration = Integration.objects.get(id=integration_id) + message = f"Webhook request processing complete." + logger.info(message, extra={"event": event_dict}) + ActivityLog.objects.create( + log_level=ActivityLog.LogLevels.INFO, + log_type=ActivityLog.LogTypes.EVENT, + origin=ActivityLog.Origin.INTEGRATION, + integration=integration, + value="integration_webhook_complete", + title=_clean_event_title(message), + details=event_dict["payload"], + is_reversible=False, + ) + + +def handle_integration_webhook_failed_event(event_dict: dict): + event = system_events.IntegrationWebhookFailed.parse_obj(event_dict) + event_data = event.payload + integration_id = event_data.integration_id + integration = Integration.objects.get(id=integration_id) if integration_id else None + error = event_dict["payload"].get("error", "No details.") + logger.info(error, extra={"event": event_dict}) + ActivityLog.objects.create( + log_level=ActivityLog.LogLevels.ERROR, + log_type=ActivityLog.LogTypes.EVENT, + origin=ActivityLog.Origin.INTEGRATION, + integration=integration, + value="integration_webhook_failed", + title=_clean_event_title(error), + details=event_dict["payload"], + is_reversible=False, + ) + + +def handle_integration_webhook_custom_log_event(event_dict: dict): + event = system_events.IntegrationWebhookCustomLog.parse_obj(event_dict) + custom_log = event.payload + integration_id = custom_log.integration_id + integration = Integration.objects.get(id=integration_id) if integration_id else None + message = f"Webhook Custom Log: {custom_log.title}." + logger.info(message, extra={"event": event_dict}) + ActivityLog.objects.create( + log_level=custom_log.level, + log_type=ActivityLog.LogTypes.EVENT, + origin=ActivityLog.Origin.INTEGRATION, + integration=integration, + value="integration_webhook_custom_log", + title=_clean_event_title(custom_log.title), + details=event_dict["payload"], + is_reversible=False, + ) + + +# IntegrationWebhookStarted {"asctime": "2024-06-24 14:22:01,459", "levelname": "DEBUG", "processName": "MainProcess", "thread": 132979787536128, "name": "__main__", "message": "Event Details", "event": {"event_id": "7782da01-f59b-425c-aea8-d46065b9d149", "timestamp": "2024-06-24 14:21:58.205638+00:00", "schema_version": "v1", "payload": {"integration_id": "ed8ed116-efb4-4fb1-9d68-0ecc4b0996a1", "webhook_id": "onyesha_wh_webhook", "config_data": {"json_schema": {"type": "object", "properties": {"received_at": {"type": "string", "format": "date-time"}, "end_device_ids": {"type": "object", "properties": {"dev_eui": {"type": "string"}, "dev_addr": {"type": "string"}, "device_id": {"type": "string"}, "application_ids": {"type": "object", "properties": {"application_id": {"type": "string"}}, "additionalProperties": false}}, "additionalProperties": false}, "uplink_message": {"type": "object", "properties": {"f_cnt": {"type": "integer"}, "f_port": {"type": "integer"}, "settings": {"type": "object", "properties": {"time": {"type": "string", "format": "date-time"}, "data_rate": {"type": "object", "properties": {"lora": {"type": "object", "properties": {"bandwidth": {"type": "integer"}, "coding_rate": {"type": "string"}, "spreading_factor": {"type": "integer"}}, "additionalProperties": false}}, "additionalProperties": false}, "frequency": {"type": "string"}, "timestamp": {"type": "integer"}}, "additionalProperties": false}, "locations": {"type": "object", "properties": {"frm-payload": {"type": "object", "properties": {"source": {"type": "string"}, "latitude": {"type": "number"}, "longitude": {"type": "number"}}, "additionalProperties": false}}, "additionalProperties": false}, "frm_payload": {"type": "string"}, "network_ids": {"type": "object", "properties": {"ns_id": {"type": "string"}, "net_id": {"type": "string"}, "tenant_id": {"type": "string"}, "cluster_id": {"type": "string"}, "tenant_address": {"type": "string"}, "cluster_address": {"type": "string"}}, "additionalProperties": false}, "received_at": {"type": "string", "format": "date-time"}, "rx_metadata": {"type": "array", "items": {"type": "object", "properties": {"snr": {"type": "number"}, "rssi": {"type": "integer"}, "time": {"type": "string", "format": "date-time"}, "gps_time": {"type": "string", "format": "date-time"}, "timestamp": {"type": "integer"}, "gateway_ids": {"type": "object", "properties": {"eui": {"type": "string"}, "gateway_id": {"type": "string"}}, "additionalProperties": false}, "received_at": {"type": "string", "format": "date-time"}, "channel_rssi": {"type": "integer"}, "uplink_token": {"type": "string"}, "channel_index": {"type": "integer"}}, "additionalProperties": false}}, "decoded_payload": {"type": "object", "properties": {"gps": {"type": "string"}, "latitude": {"type": "number"}, "longitude": {"type": "number"}, "batterypercent": {"type": "integer"}}, "additionalProperties": false}, "consumed_airtime": {"type": "string"}}, "additionalProperties": false}, "correlation_ids": {"type": "array", "items": {"type": "string"}}}, "additionalProperties": false}, "jq_filter": "{ \"source\": .end_device_ids.device_id, \"source_name\": .end_device_ids.device_id, \"type\": .uplink_message.locations.\"frm-payload\".source, \"recorded_at\": .uplink_message.settings.time, \"location\": { \"lat\": .uplink_message.locations.\"frm-payload\".latitude, \"lon\": .uplink_message.locations.\"frm-payload\".longitude }, \"additional\": { \"application_id\": .end_device_ids.application_ids.application_id, \"dev_eui\": .end_device_ids.dev_eui, \"dev_addr\": .end_device_ids.dev_addr, \"batterypercent\": .uplink_message.decoded_payload.batterypercent, \"gps\": .uplink_message.decoded_payload.gps } }", "output_type": "obv"}}, "event_type": "IntegrationWebhookStarted"}} +# IntegrationWebhookComplete {"asctime": "2024-06-24 14:34:05,949", "levelname": "DEBUG", "processName": "MainProcess", "thread": 132979292628736, "name": "__main__", "message": "Event Details", "event": {"event_id": "53d3ba77-dd8f-428a-b202-f7609ff71f68", "timestamp": "2024-06-24 14:34:02.890406+00:00", "schema_version": "v1", "payload": {"integration_id": "ed8ed116-efb4-4fb1-9d68-0ecc4b0996a1", "webhook_id": "onyesha_wh_webhook", "config_data": {"json_schema": {"type": "object", "properties": {"received_at": {"type": "string", "format": "date-time"}, "end_device_ids": {"type": "object", "properties": {"dev_eui": {"type": "string"}, "dev_addr": {"type": "string"}, "device_id": {"type": "string"}, "application_ids": {"type": "object", "properties": {"application_id": {"type": "string"}}, "additionalProperties": false}}, "additionalProperties": false}, "uplink_message": {"type": "object", "properties": {"f_cnt": {"type": "integer"}, "f_port": {"type": "integer"}, "settings": {"type": "object", "properties": {"time": {"type": "string", "format": "date-time"}, "data_rate": {"type": "object", "properties": {"lora": {"type": "object", "properties": {"bandwidth": {"type": "integer"}, "coding_rate": {"type": "string"}, "spreading_factor": {"type": "integer"}}, "additionalProperties": false}}, "additionalProperties": false}, "frequency": {"type": "string"}, "timestamp": {"type": "integer"}}, "additionalProperties": false}, "locations": {"type": "object", "properties": {"frm-payload": {"type": "object", "properties": {"source": {"type": "string"}, "latitude": {"type": "number"}, "longitude": {"type": "number"}}, "additionalProperties": false}}, "additionalProperties": false}, "frm_payload": {"type": "string"}, "network_ids": {"type": "object", "properties": {"ns_id": {"type": "string"}, "net_id": {"type": "string"}, "tenant_id": {"type": "string"}, "cluster_id": {"type": "string"}, "tenant_address": {"type": "string"}, "cluster_address": {"type": "string"}}, "additionalProperties": false}, "received_at": {"type": "string", "format": "date-time"}, "rx_metadata": {"type": "array", "items": {"type": "object", "properties": {"snr": {"type": "number"}, "rssi": {"type": "integer"}, "time": {"type": "string", "format": "date-time"}, "gps_time": {"type": "string", "format": "date-time"}, "timestamp": {"type": "integer"}, "gateway_ids": {"type": "object", "properties": {"eui": {"type": "string"}, "gateway_id": {"type": "string"}}, "additionalProperties": false}, "received_at": {"type": "string", "format": "date-time"}, "channel_rssi": {"type": "integer"}, "uplink_token": {"type": "string"}, "channel_index": {"type": "integer"}}, "additionalProperties": false}}, "decoded_payload": {"type": "object", "properties": {"gps": {"type": "string"}, "latitude": {"type": "number"}, "longitude": {"type": "number"}, "batterypercent": {"type": "integer"}}, "additionalProperties": false}, "consumed_airtime": {"type": "string"}}, "additionalProperties": false}, "correlation_ids": {"type": "array", "items": {"type": "string"}}}, "additionalProperties": false}, "jq_filter": "{ \"source\": .end_device_ids.device_id, \"source_name\": .end_device_ids.device_id, \"type\": .uplink_message.locations.\"frm-payload\".source, \"recorded_at\": .uplink_message.settings.time, \"location\": { \"lat\": .uplink_message.locations.\"frm-payload\".latitude, \"lon\": .uplink_message.locations.\"frm-payload\".longitude }, \"additional\": { \"application_id\": .end_device_ids.application_ids.application_id, \"dev_eui\": .end_device_ids.dev_eui, \"dev_addr\": .end_device_ids.dev_addr, \"batterypercent\": .uplink_message.decoded_payload.batterypercent, \"gps\": .uplink_message.decoded_payload.gps } }", "output_type": "obv"}, "result": {"data_points_qty": 1}}, "event_type": "IntegrationWebhookComplete"}} +# IntegrationWebhookFailed {"asctime": "2024-06-24 14:22:27,594", "levelname": "DEBUG", "processName": "MainProcess", "thread": 132979787536128, "name": "__main__", "message": "Event Details", "event": {"event_id": "272a44d2-9867-43a0-8950-de31df36fa26", "timestamp": "2024-06-24 14:22:24.399021+00:00", "schema_version": "v1", "payload": {"integration_id": "ed8ed116-efb4-4fb1-9d68-0ecc4b0996a1", "webhook_id": "onyesha_wh_webhook", "config_data": {"json_schema": {"type": "object", "properties": {"received_at": {"type": "string", "format": "date-time"}, "end_device_ids": {"type": "object", "properties": {"dev_eui": {"type": "string"}, "dev_addr": {"type": "string"}, "device_id": {"type": "string"}, "application_ids": {"type": "object", "properties": {"application_id": {"type": "string"}}, "additionalProperties": false}}, "additionalProperties": false}, "uplink_message": {"type": "object", "properties": {"f_cnt": {"type": "integer"}, "f_port": {"type": "integer"}, "settings": {"type": "object", "properties": {"time": {"type": "string", "format": "date-time"}, "data_rate": {"type": "object", "properties": {"lora": {"type": "object", "properties": {"bandwidth": {"type": "integer"}, "coding_rate": {"type": "string"}, "spreading_factor": {"type": "integer"}}, "additionalProperties": false}}, "additionalProperties": false}, "frequency": {"type": "string"}, "timestamp": {"type": "integer"}}, "additionalProperties": false}, "locations": {"type": "object", "properties": {"frm-payload": {"type": "object", "properties": {"source": {"type": "string"}, "latitude": {"type": "number"}, "longitude": {"type": "number"}}, "additionalProperties": false}}, "additionalProperties": false}, "frm_payload": {"type": "string"}, "network_ids": {"type": "object", "properties": {"ns_id": {"type": "string"}, "net_id": {"type": "string"}, "tenant_id": {"type": "string"}, "cluster_id": {"type": "string"}, "tenant_address": {"type": "string"}, "cluster_address": {"type": "string"}}, "additionalProperties": false}, "received_at": {"type": "string", "format": "date-time"}, "rx_metadata": {"type": "array", "items": {"type": "object", "properties": {"snr": {"type": "number"}, "rssi": {"type": "integer"}, "time": {"type": "string", "format": "date-time"}, "gps_time": {"type": "string", "format": "date-time"}, "timestamp": {"type": "integer"}, "gateway_ids": {"type": "object", "properties": {"eui": {"type": "string"}, "gateway_id": {"type": "string"}}, "additionalProperties": false}, "received_at": {"type": "string", "format": "date-time"}, "channel_rssi": {"type": "integer"}, "uplink_token": {"type": "string"}, "channel_index": {"type": "integer"}}, "additionalProperties": false}}, "decoded_payload": {"type": "object", "properties": {"gps": {"type": "string"}, "latitude": {"type": "number"}, "longitude": {"type": "number"}, "batterypercent": {"type": "integer"}}, "additionalProperties": false}, "consumed_airtime": {"type": "string"}}, "additionalProperties": false}, "correlation_ids": {"type": "array", "items": {"type": "string"}}}, "additionalProperties": false}, "jq_filter": "{ \"source\": .end_device_ids.device_id, \"source_name\": .end_device_ids.device_id, \"type\": .uplink_message.locations.\"frm-payload\".source, \"recorded_at\": .uplink_message.settings.time, \"location\": { \"lat\": .uplink_message.locations.\"frm-payload\".latitude, \"lon\": .uplink_message.locations.\"frm-payload\".longitude }, \"additional\": { \"application_id\": .end_device_ids.application_ids.application_id, \"dev_eui\": .end_device_ids.dev_eui, \"dev_addr\": .end_device_ids.dev_addr, \"batterypercent\": .uplink_message.decoded_payload.batterypercent, \"gps\": .uplink_message.decoded_payload.gps } }", "output_type": "obv"}, "error": "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate (_ssl.c:1007)"}, "event_type": "IntegrationWebhookFailed"}} +# IntegrationWebhookCustomLog {"asctime": "2024-06-24 15:11:47,772", "levelname": "DEBUG", "processName": "MainProcess", "thread": 132979275843328, "name": "__main__", "message": "Event Details", "event": {"event_id": "a0b73b33-0e2e-457d-b650-49adaa7f593d", "timestamp": "2024-06-24 15:11:46.366931+00:00", "schema_version": "v1", "payload": {"integration_id": "ed8ed116-efb4-4fb1-9d68-0ecc4b0996a1", "webhook_id": "webhook", "config_data": {}, "title": "Webhook data transformed successfully", "level": 10, "data": {"transformed_data": [{"source": "test-webhooks-mm", "source_name": "test-webhooks-mm", "type": "SOURCE_GPS", "recorded_at": "2024-06-07T15:08:19.841Z", "location": {"lat": -2.3828796, "lon": 37.338060999999996}, "additional": {"application_id": "lt10-globalsat", "dev_eui": "70B3D57ED80027EF", "dev_addr": "27027D02", "batterypercent": 100, "gps": "3D fix"}}]}}, "event_type": "IntegrationWebhookCustomLog"}} + event_handlers = { "IntegrationActionStarted": handle_integration_action_started_event, "IntegrationActionComplete": handle_integration_action_complete_event, "IntegrationActionFailed": handle_integration_action_failed_event, "IntegrationActionCustomLog": handle_integration_action_custom_log_event, + "IntegrationWebhookStarted": handle_integration_webhook_started_event, + "IntegrationWebhookComplete": handle_integration_webhook_complete_event, + "IntegrationWebhookFailed": handle_integration_webhook_failed_event, + "IntegrationWebhookCustomLog": handle_integration_webhook_custom_log_event, } From 4258e574a3b4a9297ad1abda35393580f9a3cb63 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 25 Jun 2024 17:43:54 -0300 Subject: [PATCH 2/5] Remove commented code --- cdip_admin/event_consumers/integration_events_consumer.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cdip_admin/event_consumers/integration_events_consumer.py b/cdip_admin/event_consumers/integration_events_consumer.py index 44b5d90c7..fadc49de2 100644 --- a/cdip_admin/event_consumers/integration_events_consumer.py +++ b/cdip_admin/event_consumers/integration_events_consumer.py @@ -176,11 +176,6 @@ def handle_integration_webhook_custom_log_event(event_dict: dict): ) -# IntegrationWebhookStarted {"asctime": "2024-06-24 14:22:01,459", "levelname": "DEBUG", "processName": "MainProcess", "thread": 132979787536128, "name": "__main__", "message": "Event Details", "event": {"event_id": "7782da01-f59b-425c-aea8-d46065b9d149", "timestamp": "2024-06-24 14:21:58.205638+00:00", "schema_version": "v1", "payload": {"integration_id": "ed8ed116-efb4-4fb1-9d68-0ecc4b0996a1", "webhook_id": "onyesha_wh_webhook", "config_data": {"json_schema": {"type": "object", "properties": {"received_at": {"type": "string", "format": "date-time"}, "end_device_ids": {"type": "object", "properties": {"dev_eui": {"type": "string"}, "dev_addr": {"type": "string"}, "device_id": {"type": "string"}, "application_ids": {"type": "object", "properties": {"application_id": {"type": "string"}}, "additionalProperties": false}}, "additionalProperties": false}, "uplink_message": {"type": "object", "properties": {"f_cnt": {"type": "integer"}, "f_port": {"type": "integer"}, "settings": {"type": "object", "properties": {"time": {"type": "string", "format": "date-time"}, "data_rate": {"type": "object", "properties": {"lora": {"type": "object", "properties": {"bandwidth": {"type": "integer"}, "coding_rate": {"type": "string"}, "spreading_factor": {"type": "integer"}}, "additionalProperties": false}}, "additionalProperties": false}, "frequency": {"type": "string"}, "timestamp": {"type": "integer"}}, "additionalProperties": false}, "locations": {"type": "object", "properties": {"frm-payload": {"type": "object", "properties": {"source": {"type": "string"}, "latitude": {"type": "number"}, "longitude": {"type": "number"}}, "additionalProperties": false}}, "additionalProperties": false}, "frm_payload": {"type": "string"}, "network_ids": {"type": "object", "properties": {"ns_id": {"type": "string"}, "net_id": {"type": "string"}, "tenant_id": {"type": "string"}, "cluster_id": {"type": "string"}, "tenant_address": {"type": "string"}, "cluster_address": {"type": "string"}}, "additionalProperties": false}, "received_at": {"type": "string", "format": "date-time"}, "rx_metadata": {"type": "array", "items": {"type": "object", "properties": {"snr": {"type": "number"}, "rssi": {"type": "integer"}, "time": {"type": "string", "format": "date-time"}, "gps_time": {"type": "string", "format": "date-time"}, "timestamp": {"type": "integer"}, "gateway_ids": {"type": "object", "properties": {"eui": {"type": "string"}, "gateway_id": {"type": "string"}}, "additionalProperties": false}, "received_at": {"type": "string", "format": "date-time"}, "channel_rssi": {"type": "integer"}, "uplink_token": {"type": "string"}, "channel_index": {"type": "integer"}}, "additionalProperties": false}}, "decoded_payload": {"type": "object", "properties": {"gps": {"type": "string"}, "latitude": {"type": "number"}, "longitude": {"type": "number"}, "batterypercent": {"type": "integer"}}, "additionalProperties": false}, "consumed_airtime": {"type": "string"}}, "additionalProperties": false}, "correlation_ids": {"type": "array", "items": {"type": "string"}}}, "additionalProperties": false}, "jq_filter": "{ \"source\": .end_device_ids.device_id, \"source_name\": .end_device_ids.device_id, \"type\": .uplink_message.locations.\"frm-payload\".source, \"recorded_at\": .uplink_message.settings.time, \"location\": { \"lat\": .uplink_message.locations.\"frm-payload\".latitude, \"lon\": .uplink_message.locations.\"frm-payload\".longitude }, \"additional\": { \"application_id\": .end_device_ids.application_ids.application_id, \"dev_eui\": .end_device_ids.dev_eui, \"dev_addr\": .end_device_ids.dev_addr, \"batterypercent\": .uplink_message.decoded_payload.batterypercent, \"gps\": .uplink_message.decoded_payload.gps } }", "output_type": "obv"}}, "event_type": "IntegrationWebhookStarted"}} -# IntegrationWebhookComplete {"asctime": "2024-06-24 14:34:05,949", "levelname": "DEBUG", "processName": "MainProcess", "thread": 132979292628736, "name": "__main__", "message": "Event Details", "event": {"event_id": "53d3ba77-dd8f-428a-b202-f7609ff71f68", "timestamp": "2024-06-24 14:34:02.890406+00:00", "schema_version": "v1", "payload": {"integration_id": "ed8ed116-efb4-4fb1-9d68-0ecc4b0996a1", "webhook_id": "onyesha_wh_webhook", "config_data": {"json_schema": {"type": "object", "properties": {"received_at": {"type": "string", "format": "date-time"}, "end_device_ids": {"type": "object", "properties": {"dev_eui": {"type": "string"}, "dev_addr": {"type": "string"}, "device_id": {"type": "string"}, "application_ids": {"type": "object", "properties": {"application_id": {"type": "string"}}, "additionalProperties": false}}, "additionalProperties": false}, "uplink_message": {"type": "object", "properties": {"f_cnt": {"type": "integer"}, "f_port": {"type": "integer"}, "settings": {"type": "object", "properties": {"time": {"type": "string", "format": "date-time"}, "data_rate": {"type": "object", "properties": {"lora": {"type": "object", "properties": {"bandwidth": {"type": "integer"}, "coding_rate": {"type": "string"}, "spreading_factor": {"type": "integer"}}, "additionalProperties": false}}, "additionalProperties": false}, "frequency": {"type": "string"}, "timestamp": {"type": "integer"}}, "additionalProperties": false}, "locations": {"type": "object", "properties": {"frm-payload": {"type": "object", "properties": {"source": {"type": "string"}, "latitude": {"type": "number"}, "longitude": {"type": "number"}}, "additionalProperties": false}}, "additionalProperties": false}, "frm_payload": {"type": "string"}, "network_ids": {"type": "object", "properties": {"ns_id": {"type": "string"}, "net_id": {"type": "string"}, "tenant_id": {"type": "string"}, "cluster_id": {"type": "string"}, "tenant_address": {"type": "string"}, "cluster_address": {"type": "string"}}, "additionalProperties": false}, "received_at": {"type": "string", "format": "date-time"}, "rx_metadata": {"type": "array", "items": {"type": "object", "properties": {"snr": {"type": "number"}, "rssi": {"type": "integer"}, "time": {"type": "string", "format": "date-time"}, "gps_time": {"type": "string", "format": "date-time"}, "timestamp": {"type": "integer"}, "gateway_ids": {"type": "object", "properties": {"eui": {"type": "string"}, "gateway_id": {"type": "string"}}, "additionalProperties": false}, "received_at": {"type": "string", "format": "date-time"}, "channel_rssi": {"type": "integer"}, "uplink_token": {"type": "string"}, "channel_index": {"type": "integer"}}, "additionalProperties": false}}, "decoded_payload": {"type": "object", "properties": {"gps": {"type": "string"}, "latitude": {"type": "number"}, "longitude": {"type": "number"}, "batterypercent": {"type": "integer"}}, "additionalProperties": false}, "consumed_airtime": {"type": "string"}}, "additionalProperties": false}, "correlation_ids": {"type": "array", "items": {"type": "string"}}}, "additionalProperties": false}, "jq_filter": "{ \"source\": .end_device_ids.device_id, \"source_name\": .end_device_ids.device_id, \"type\": .uplink_message.locations.\"frm-payload\".source, \"recorded_at\": .uplink_message.settings.time, \"location\": { \"lat\": .uplink_message.locations.\"frm-payload\".latitude, \"lon\": .uplink_message.locations.\"frm-payload\".longitude }, \"additional\": { \"application_id\": .end_device_ids.application_ids.application_id, \"dev_eui\": .end_device_ids.dev_eui, \"dev_addr\": .end_device_ids.dev_addr, \"batterypercent\": .uplink_message.decoded_payload.batterypercent, \"gps\": .uplink_message.decoded_payload.gps } }", "output_type": "obv"}, "result": {"data_points_qty": 1}}, "event_type": "IntegrationWebhookComplete"}} -# IntegrationWebhookFailed {"asctime": "2024-06-24 14:22:27,594", "levelname": "DEBUG", "processName": "MainProcess", "thread": 132979787536128, "name": "__main__", "message": "Event Details", "event": {"event_id": "272a44d2-9867-43a0-8950-de31df36fa26", "timestamp": "2024-06-24 14:22:24.399021+00:00", "schema_version": "v1", "payload": {"integration_id": "ed8ed116-efb4-4fb1-9d68-0ecc4b0996a1", "webhook_id": "onyesha_wh_webhook", "config_data": {"json_schema": {"type": "object", "properties": {"received_at": {"type": "string", "format": "date-time"}, "end_device_ids": {"type": "object", "properties": {"dev_eui": {"type": "string"}, "dev_addr": {"type": "string"}, "device_id": {"type": "string"}, "application_ids": {"type": "object", "properties": {"application_id": {"type": "string"}}, "additionalProperties": false}}, "additionalProperties": false}, "uplink_message": {"type": "object", "properties": {"f_cnt": {"type": "integer"}, "f_port": {"type": "integer"}, "settings": {"type": "object", "properties": {"time": {"type": "string", "format": "date-time"}, "data_rate": {"type": "object", "properties": {"lora": {"type": "object", "properties": {"bandwidth": {"type": "integer"}, "coding_rate": {"type": "string"}, "spreading_factor": {"type": "integer"}}, "additionalProperties": false}}, "additionalProperties": false}, "frequency": {"type": "string"}, "timestamp": {"type": "integer"}}, "additionalProperties": false}, "locations": {"type": "object", "properties": {"frm-payload": {"type": "object", "properties": {"source": {"type": "string"}, "latitude": {"type": "number"}, "longitude": {"type": "number"}}, "additionalProperties": false}}, "additionalProperties": false}, "frm_payload": {"type": "string"}, "network_ids": {"type": "object", "properties": {"ns_id": {"type": "string"}, "net_id": {"type": "string"}, "tenant_id": {"type": "string"}, "cluster_id": {"type": "string"}, "tenant_address": {"type": "string"}, "cluster_address": {"type": "string"}}, "additionalProperties": false}, "received_at": {"type": "string", "format": "date-time"}, "rx_metadata": {"type": "array", "items": {"type": "object", "properties": {"snr": {"type": "number"}, "rssi": {"type": "integer"}, "time": {"type": "string", "format": "date-time"}, "gps_time": {"type": "string", "format": "date-time"}, "timestamp": {"type": "integer"}, "gateway_ids": {"type": "object", "properties": {"eui": {"type": "string"}, "gateway_id": {"type": "string"}}, "additionalProperties": false}, "received_at": {"type": "string", "format": "date-time"}, "channel_rssi": {"type": "integer"}, "uplink_token": {"type": "string"}, "channel_index": {"type": "integer"}}, "additionalProperties": false}}, "decoded_payload": {"type": "object", "properties": {"gps": {"type": "string"}, "latitude": {"type": "number"}, "longitude": {"type": "number"}, "batterypercent": {"type": "integer"}}, "additionalProperties": false}, "consumed_airtime": {"type": "string"}}, "additionalProperties": false}, "correlation_ids": {"type": "array", "items": {"type": "string"}}}, "additionalProperties": false}, "jq_filter": "{ \"source\": .end_device_ids.device_id, \"source_name\": .end_device_ids.device_id, \"type\": .uplink_message.locations.\"frm-payload\".source, \"recorded_at\": .uplink_message.settings.time, \"location\": { \"lat\": .uplink_message.locations.\"frm-payload\".latitude, \"lon\": .uplink_message.locations.\"frm-payload\".longitude }, \"additional\": { \"application_id\": .end_device_ids.application_ids.application_id, \"dev_eui\": .end_device_ids.dev_eui, \"dev_addr\": .end_device_ids.dev_addr, \"batterypercent\": .uplink_message.decoded_payload.batterypercent, \"gps\": .uplink_message.decoded_payload.gps } }", "output_type": "obv"}, "error": "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate (_ssl.c:1007)"}, "event_type": "IntegrationWebhookFailed"}} -# IntegrationWebhookCustomLog {"asctime": "2024-06-24 15:11:47,772", "levelname": "DEBUG", "processName": "MainProcess", "thread": 132979275843328, "name": "__main__", "message": "Event Details", "event": {"event_id": "a0b73b33-0e2e-457d-b650-49adaa7f593d", "timestamp": "2024-06-24 15:11:46.366931+00:00", "schema_version": "v1", "payload": {"integration_id": "ed8ed116-efb4-4fb1-9d68-0ecc4b0996a1", "webhook_id": "webhook", "config_data": {}, "title": "Webhook data transformed successfully", "level": 10, "data": {"transformed_data": [{"source": "test-webhooks-mm", "source_name": "test-webhooks-mm", "type": "SOURCE_GPS", "recorded_at": "2024-06-07T15:08:19.841Z", "location": {"lat": -2.3828796, "lon": 37.338060999999996}, "additional": {"application_id": "lt10-globalsat", "dev_eui": "70B3D57ED80027EF", "dev_addr": "27027D02", "batterypercent": 100, "gps": "3D fix"}}]}}, "event_type": "IntegrationWebhookCustomLog"}} - event_handlers = { "IntegrationActionStarted": handle_integration_action_started_event, "IntegrationActionComplete": handle_integration_action_complete_event, From 453279c7863dcad0843556a22550dda3ddbea758 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 25 Jun 2024 18:58:43 -0300 Subject: [PATCH 3/5] Guard against events with no payload --- .../integration_events_consumer.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cdip_admin/event_consumers/integration_events_consumer.py b/cdip_admin/event_consumers/integration_events_consumer.py index fadc49de2..0f1bcc32e 100644 --- a/cdip_admin/event_consumers/integration_events_consumer.py +++ b/cdip_admin/event_consumers/integration_events_consumer.py @@ -35,7 +35,7 @@ def handle_integration_action_started_event(event_dict: dict): integration=integration, value="integration_action_started", title=_clean_event_title(message), - details=event_dict["payload"], + details=event_dict.get("payload", {}), is_reversible=False, ) @@ -55,7 +55,7 @@ def handle_integration_action_complete_event(event_dict: dict): integration=integration, value="integration_action_complete", title=_clean_event_title(message), - details=event_dict["payload"], + details=event_dict.get("payload", {}), is_reversible=False, ) @@ -76,7 +76,7 @@ def handle_integration_action_failed_event(event_dict: dict): integration=integration, value="integration_action_failed", title=_clean_event_title(message), - details=event_dict["payload"], + details=event_dict.get("payload", {}), is_reversible=False, ) @@ -95,7 +95,7 @@ def handle_integration_action_custom_log_event(event_dict: dict): integration=integration, value="integration_custom_log", title=_clean_event_title(custom_log.title), - details=event_dict["payload"], + details=event_dict.get("payload", {}), is_reversible=False, ) @@ -114,7 +114,7 @@ def handle_integration_webhook_started_event(event_dict: dict): integration=integration, value="integration_webhook_started", title=_clean_event_title(message), - details=event_dict["payload"], + details=event_dict.get("payload", {}), is_reversible=False, ) @@ -133,7 +133,7 @@ def handle_integration_webhook_complete_event(event_dict: dict): integration=integration, value="integration_webhook_complete", title=_clean_event_title(message), - details=event_dict["payload"], + details=event_dict.get("payload", {}), is_reversible=False, ) @@ -152,7 +152,7 @@ def handle_integration_webhook_failed_event(event_dict: dict): integration=integration, value="integration_webhook_failed", title=_clean_event_title(error), - details=event_dict["payload"], + details=event_dict.get("payload", {}), is_reversible=False, ) @@ -171,7 +171,7 @@ def handle_integration_webhook_custom_log_event(event_dict: dict): integration=integration, value="integration_webhook_custom_log", title=_clean_event_title(custom_log.title), - details=event_dict["payload"], + details=event_dict.get("payload", {}), is_reversible=False, ) From 33977aa8147136774996f6511c05edf4a8f04cc9 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 25 Jun 2024 19:13:38 -0300 Subject: [PATCH 4/5] Add test coverage for activity logs on webhook integrations --- .secrets.baseline | 4 +- cdip_admin/conftest.py | 100 +++++++++++++++++- .../tests/test_integration_events_consumer.py | 75 +++++++++++++ 3 files changed, 175 insertions(+), 4 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 2bd44d649..b5849a04d 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -258,7 +258,7 @@ "filename": "cdip_admin/conftest.py", "hashed_secret": "27d933b78d3ea4a51a3ff26aef5831c92d43925c", "is_verified": false, - "line_number": 2206 + "line_number": 2302 } ], "cdip_admin/integrations/migrations/0047_init_erinreach_config_schema.py": [ @@ -296,5 +296,5 @@ } ] }, - "generated_at": "2024-06-06T20:54:02Z" + "generated_at": "2024-06-25T22:13:22Z" } diff --git a/cdip_admin/conftest.py b/cdip_admin/conftest.py index 477a3b3c3..82381945e 100644 --- a/cdip_admin/conftest.py +++ b/cdip_admin/conftest.py @@ -2,7 +2,6 @@ import base64 from datetime import datetime from unittest.mock import PropertyMock - import pytest import random from typing import NamedTuple, Any @@ -40,7 +39,14 @@ ) from organizations.models import Organization from pathlib import Path -from google.cloud import pubsub_v1 +from gundi_core.events import ( + IntegrationWebhookStarted, + WebhookExecutionStarted, + IntegrationWebhookComplete, + WebhookExecutionComplete, + IntegrationWebhookFailed, + WebhookExecutionFailed, IntegrationWebhookCustomLog, CustomWebhookLog, LogLevel +) def async_return(result): @@ -2191,6 +2197,96 @@ def pull_observations_action_custom_log_event(mocker, provider_lotek_panthera): return message +@pytest.fixture +def webhook_started_event_pubsub(mocker, provider_liquidtech_with_webhook_config): + pubsub_message = mocker.MagicMock() + event = IntegrationWebhookStarted( + payload=WebhookExecutionStarted( + integration_id=str(provider_liquidtech_with_webhook_config.id), + webhook_id='liquidtech_webhook', + config_data={ + 'json_schema': {'type': 'object', 'properties': {'received_at': {'type': 'string', 'format': 'date-time'}, 'end_device_ids': {'type': 'object', 'properties': {'dev_eui': {'type': 'string'}, 'dev_addr': {'type': 'string'}, 'device_id': {'type': 'string'}, 'application_ids': {'type': 'object', 'properties': {'application_id': {'type': 'string'}}, 'additionalProperties': False}}, 'additionalProperties': False}, 'uplink_message': {'type': 'object', 'properties': {'f_cnt': {'type': 'integer'}, 'f_port': {'type': 'integer'}, 'settings': {'type': 'object', 'properties': {'time': {'type': 'string', 'format': 'date-time'}, 'data_rate': {'type': 'object', 'properties': {'lora': {'type': 'object', 'properties': {'bandwidth': {'type': 'integer'}, 'coding_rate': {'type': 'string'}, 'spreading_factor': {'type': 'integer'}}, 'additionalProperties': False}}, 'additionalProperties': False}, 'frequency': {'type': 'string'}, 'timestamp': {'type': 'integer'}}, 'additionalProperties': False}, 'locations': {'type': 'object', 'properties': {'frm-payload': {'type': 'object', 'properties': {'source': {'type': 'string'}, 'latitude': {'type': 'number'}, 'longitude': {'type': 'number'}}, 'additionalProperties': False}}, 'additionalProperties': False}, 'frm_payload': {'type': 'string'}, 'network_ids': {'type': 'object', 'properties': {'ns_id': {'type': 'string'}, 'net_id': {'type': 'string'}, 'tenant_id': {'type': 'string'}, 'cluster_id': {'type': 'string'}, 'tenant_address': {'type': 'string'}, 'cluster_address': {'type': 'string'}}, 'additionalProperties': False}, 'received_at': {'type': 'string', 'format': 'date-time'}, 'rx_metadata': {'type': 'array', 'items': {'type': 'object', 'properties': {'snr': {'type': 'number'}, 'rssi': {'type': 'integer'}, 'time': {'type': 'string', 'format': 'date-time'}, 'gps_time': {'type': 'string', 'format': 'date-time'}, 'timestamp': {'type': 'integer'}, 'gateway_ids': {'type': 'object', 'properties': {'eui': {'type': 'string'}, 'gateway_id': {'type': 'string'}}, 'additionalProperties': False}, 'received_at': {'type': 'string', 'format': 'date-time'}, 'channel_rssi': {'type': 'integer'}, 'uplink_token': {'type': 'string'}, 'channel_index': {'type': 'integer'}}, 'additionalProperties': False}}, 'decoded_payload': {'type': 'object', 'properties': {'gps': {'type': 'string'}, 'latitude': {'type': 'number'}, 'longitude': {'type': 'number'}, 'batterypercent': {'type': 'integer'}}, 'additionalProperties': False}, 'consumed_airtime': {'type': 'string'}}, 'additionalProperties': False}, 'correlation_ids': {'type': 'array', 'items': {'type': 'string'}}}, 'additionalProperties': False}, + 'jq_filter': '{"source": .end_device_ids.device_id, "source_name": .end_device_ids.device_id, "type": .uplink_message.locations."frm-payload".source, "recorded_at": .uplink_message.settings.time, "location": { "lat": .uplink_message.locations."frm-payload".latitude, "lon": .uplink_message.locations."frm-payload".longitude}, "additional": {"application_id": .end_device_ids.application_ids.application_id, "dev_eui": .end_device_ids.dev_eui, "dev_addr": .end_device_ids.dev_addr, "batterypercent": .uplink_message.decoded_payload.batterypercent, "gps": .uplink_message.decoded_payload.gps}}', + 'output_type': 'obv' + } + ) + ) + pubsub_message.data = json.dumps(event.dict(), default=str).encode("utf-8") + return pubsub_message + + +@pytest.fixture +def webhook_complete_event_pubsub(mocker, provider_liquidtech_with_webhook_config): + pubsub_message = mocker.MagicMock() + event = IntegrationWebhookComplete( + payload=WebhookExecutionComplete( + integration_id=str(provider_liquidtech_with_webhook_config.id), + webhook_id='liquidtech_webhook', + config_data={ + 'json_schema': {'type': 'object', 'properties': {'received_at': {'type': 'string', 'format': 'date-time'}, 'end_device_ids': {'type': 'object', 'properties': {'dev_eui': {'type': 'string'}, 'dev_addr': {'type': 'string'}, 'device_id': {'type': 'string'}, 'application_ids': {'type': 'object', 'properties': {'application_id': {'type': 'string'}}, 'additionalProperties': False}}, 'additionalProperties': False}, 'uplink_message': {'type': 'object', 'properties': {'f_cnt': {'type': 'integer'}, 'f_port': {'type': 'integer'}, 'settings': {'type': 'object', 'properties': {'time': {'type': 'string', 'format': 'date-time'}, 'data_rate': {'type': 'object', 'properties': {'lora': {'type': 'object', 'properties': {'bandwidth': {'type': 'integer'}, 'coding_rate': {'type': 'string'}, 'spreading_factor': {'type': 'integer'}}, 'additionalProperties': False}}, 'additionalProperties': False}, 'frequency': {'type': 'string'}, 'timestamp': {'type': 'integer'}}, 'additionalProperties': False}, 'locations': {'type': 'object', 'properties': {'frm-payload': {'type': 'object', 'properties': {'source': {'type': 'string'}, 'latitude': {'type': 'number'}, 'longitude': {'type': 'number'}}, 'additionalProperties': False}}, 'additionalProperties': False}, 'frm_payload': {'type': 'string'}, 'network_ids': {'type': 'object', 'properties': {'ns_id': {'type': 'string'}, 'net_id': {'type': 'string'}, 'tenant_id': {'type': 'string'}, 'cluster_id': {'type': 'string'}, 'tenant_address': {'type': 'string'}, 'cluster_address': {'type': 'string'}}, 'additionalProperties': False}, 'received_at': {'type': 'string', 'format': 'date-time'}, 'rx_metadata': {'type': 'array', 'items': {'type': 'object', 'properties': {'snr': {'type': 'number'}, 'rssi': {'type': 'integer'}, 'time': {'type': 'string', 'format': 'date-time'}, 'gps_time': {'type': 'string', 'format': 'date-time'}, 'timestamp': {'type': 'integer'}, 'gateway_ids': {'type': 'object', 'properties': {'eui': {'type': 'string'}, 'gateway_id': {'type': 'string'}}, 'additionalProperties': False}, 'received_at': {'type': 'string', 'format': 'date-time'}, 'channel_rssi': {'type': 'integer'}, 'uplink_token': {'type': 'string'}, 'channel_index': {'type': 'integer'}}, 'additionalProperties': False}}, 'decoded_payload': {'type': 'object', 'properties': {'gps': {'type': 'string'}, 'latitude': {'type': 'number'}, 'longitude': {'type': 'number'}, 'batterypercent': {'type': 'integer'}}, 'additionalProperties': False}, 'consumed_airtime': {'type': 'string'}}, 'additionalProperties': False}, 'correlation_ids': {'type': 'array', 'items': {'type': 'string'}}}, 'additionalProperties': False}, + 'jq_filter': '{"source": .end_device_ids.device_id, "source_name": .end_device_ids.device_id, "type": .uplink_message.locations."frm-payload".source, "recorded_at": .uplink_message.settings.time, "location": { "lat": .uplink_message.locations."frm-payload".latitude, "lon": .uplink_message.locations."frm-payload".longitude}, "additional": {"application_id": .end_device_ids.application_ids.application_id, "dev_eui": .end_device_ids.dev_eui, "dev_addr": .end_device_ids.dev_addr, "batterypercent": .uplink_message.decoded_payload.batterypercent, "gps": .uplink_message.decoded_payload.gps}}', + 'output_type': 'obv' + }, + result={'data_points_qty': 1} + ) + ) + pubsub_message.data = json.dumps(event.dict(), default=str).encode("utf-8") + return pubsub_message + + +@pytest.fixture +def webhook_failed_event_pubsub(mocker, provider_liquidtech_with_webhook_config): + pubsub_message = mocker.MagicMock() + event = IntegrationWebhookFailed( + payload=WebhookExecutionFailed( + integration_id=str(provider_liquidtech_with_webhook_config.id), + webhook_id='liquidtech_webhook', + config_data={ + 'json_schema': {'type': 'object', 'properties': {'received_at': {'type': 'string', 'format': 'date-time'}, 'end_device_ids': {'type': 'object', 'properties': {'dev_eui': {'type': 'string'}, 'dev_addr': {'type': 'string'}, 'device_id': {'type': 'string'}, 'application_ids': {'type': 'object', 'properties': {'application_id': {'type': 'string'}}, 'additionalProperties': False}}, 'additionalProperties': False}, 'uplink_message': {'type': 'object', 'properties': {'f_cnt': {'type': 'integer'}, 'f_port': {'type': 'integer'}, 'settings': {'type': 'object', 'properties': {'time': {'type': 'string', 'format': 'date-time'}, 'data_rate': {'type': 'object', 'properties': {'lora': {'type': 'object', 'properties': {'bandwidth': {'type': 'integer'}, 'coding_rate': {'type': 'string'}, 'spreading_factor': {'type': 'integer'}}, 'additionalProperties': False}}, 'additionalProperties': False}, 'frequency': {'type': 'string'}, 'timestamp': {'type': 'integer'}}, 'additionalProperties': False}, 'locations': {'type': 'object', 'properties': {'frm-payload': {'type': 'object', 'properties': {'source': {'type': 'string'}, 'latitude': {'type': 'number'}, 'longitude': {'type': 'number'}}, 'additionalProperties': False}}, 'additionalProperties': False}, 'frm_payload': {'type': 'string'}, 'network_ids': {'type': 'object', 'properties': {'ns_id': {'type': 'string'}, 'net_id': {'type': 'string'}, 'tenant_id': {'type': 'string'}, 'cluster_id': {'type': 'string'}, 'tenant_address': {'type': 'string'}, 'cluster_address': {'type': 'string'}}, 'additionalProperties': False}, 'received_at': {'type': 'string', 'format': 'date-time'}, 'rx_metadata': {'type': 'array', 'items': {'type': 'object', 'properties': {'snr': {'type': 'number'}, 'rssi': {'type': 'integer'}, 'time': {'type': 'string', 'format': 'date-time'}, 'gps_time': {'type': 'string', 'format': 'date-time'}, 'timestamp': {'type': 'integer'}, 'gateway_ids': {'type': 'object', 'properties': {'eui': {'type': 'string'}, 'gateway_id': {'type': 'string'}}, 'additionalProperties': False}, 'received_at': {'type': 'string', 'format': 'date-time'}, 'channel_rssi': {'type': 'integer'}, 'uplink_token': {'type': 'string'}, 'channel_index': {'type': 'integer'}}, 'additionalProperties': False}}, 'decoded_payload': {'type': 'object', 'properties': {'gps': {'type': 'string'}, 'latitude': {'type': 'number'}, 'longitude': {'type': 'number'}, 'batterypercent': {'type': 'integer'}}, 'additionalProperties': False}, 'consumed_airtime': {'type': 'string'}}, 'additionalProperties': False}, 'correlation_ids': {'type': 'array', 'items': {'type': 'string'}}}, 'additionalProperties': False}, + 'jq_filter': '{"source": .end_device_ids.device_id, "source_name": .end_device_ids.device_id, "type": .uplink_message.locations."frm-payload".source, "recorded_at": .uplink_message.settings.time, "location": { "lat": .uplink_message.locations."frm-payload".latitude, "lon": .uplink_message.locations."frm-payload".longitude}, "additional": {"application_id": .end_device_ids.application_ids.application_id, "dev_eui": .end_device_ids.dev_eui, "dev_addr": .end_device_ids.dev_addr, "batterypercent": .uplink_message.decoded_payload.batterypercent, "gps": .uplink_message.decoded_payload.gps}}', + 'output_type': 'patrol' + }, + error='Invalid output type: patrol. Please review the configuration.' + ) + ) + pubsub_message.data = json.dumps(event.dict(), default=str).encode("utf-8") + return pubsub_message + + +@pytest.fixture +def webhook_custom_activity_log_event(mocker, provider_liquidtech_with_webhook_config): + pubsub_message = mocker.MagicMock() + event = IntegrationWebhookCustomLog( + payload=CustomWebhookLog( + integration_id=str(provider_liquidtech_with_webhook_config.id), + webhook_id='liquidtech_webhook', + config_data={}, + title='Webhook data transformed successfully', + level=LogLevel.DEBUG, + data={ + 'transformed_data': [ + { + 'source': 'test-webhooks-mm', + 'source_name': 'test-webhooks-mm', + 'type': 'SOURCE_GPS', + 'recorded_at': '2024-06-07T15:08:19.841Z', + 'location': {'lat': -4.1234567, 'lon': 32.01234567890123}, + 'additional': { + 'application_id': 'lt10-globalsat', + 'dev_eui': '123456789ABCDEF0', # pragma: allowlist secret + 'dev_addr': '12345ABC', + 'batterypercent': 100, + 'gps': '3D fix' + } + } + ] + } + ) + ) + pubsub_message.data = json.dumps(event.dict(), default=str).encode("utf-8") + return pubsub_message + + @pytest.fixture def mock_dispatcher_secrets_er(dispatcher_source_release_1): return {'env_vars': {'REDIS_HOST': '127.0.0.1', 'BUCKET_NAME': 'cdip-files-dev', 'LOGGING_LEVEL': 'INFO', diff --git a/cdip_admin/event_consumers/tests/test_integration_events_consumer.py b/cdip_admin/event_consumers/tests/test_integration_events_consumer.py index d8d9b7513..fede756ff 100644 --- a/cdip_admin/event_consumers/tests/test_integration_events_consumer.py +++ b/cdip_admin/event_consumers/tests/test_integration_events_consumer.py @@ -74,3 +74,78 @@ def test_process_custom_activity_log_event( assert activity_log.title == event_payload["title"] assert activity_log.details == event_payload assert activity_log.is_reversible is False + + +def test_process_webhook_started_event( + provider_liquidtech_with_webhook_config, webhook_started_event_pubsub +): + process_event(webhook_started_event_pubsub) + + # Check that an activity logs is recorded + activity_log = ActivityLog.objects.filter( + log_type=ActivityLog.LogTypes.EVENT, integration=provider_liquidtech_with_webhook_config + ).first() + assert activity_log + assert activity_log.log_level == ActivityLog.LogLevels.INFO + assert activity_log.origin == ActivityLog.Origin.INTEGRATION + assert activity_log.value == "integration_webhook_started" + assert not activity_log.is_reversible + event_dict = json.loads(webhook_started_event_pubsub.data) + assert activity_log.details == event_dict.get("payload") + + +def test_process_webhook_complete_event( + provider_liquidtech_with_webhook_config, webhook_complete_event_pubsub +): + process_event(webhook_complete_event_pubsub) + + # Check that an activity logs is recorded + activity_log = ActivityLog.objects.filter( + log_type=ActivityLog.LogTypes.EVENT, integration=provider_liquidtech_with_webhook_config + ).first() + assert activity_log + assert activity_log.log_level == ActivityLog.LogLevels.INFO + assert activity_log.origin == ActivityLog.Origin.INTEGRATION + assert activity_log.value == "integration_webhook_complete" + assert not activity_log.is_reversible + event_dict = json.loads(webhook_complete_event_pubsub.data) + assert activity_log.details == event_dict.get("payload") + + +def test_process_webhook_failed_event( + provider_liquidtech_with_webhook_config, webhook_failed_event_pubsub +): + process_event(webhook_failed_event_pubsub) + + # Check that an activity logs is recorded + activity_log = ActivityLog.objects.filter( + log_type=ActivityLog.LogTypes.EVENT, integration=provider_liquidtech_with_webhook_config + ).first() + assert activity_log + assert activity_log.log_level == ActivityLog.LogLevels.ERROR + assert activity_log.origin == ActivityLog.Origin.INTEGRATION + assert activity_log.value == "integration_webhook_failed" + assert not activity_log.is_reversible + event_dict = json.loads(webhook_failed_event_pubsub.data) + assert activity_log.details == event_dict.get("payload") + + +def test_process_webhook_custom_activity_log_event( + provider_liquidtech_with_webhook_config, webhook_custom_activity_log_event +): + event_payload = json.loads(webhook_custom_activity_log_event.data)[ + "payload" + ] + process_event(webhook_custom_activity_log_event) + + # Check that an activity logs is recorded + activity_log = ActivityLog.objects.filter( + log_type=ActivityLog.LogTypes.EVENT, integration=provider_liquidtech_with_webhook_config + ).first() + assert activity_log + assert activity_log.origin == ActivityLog.Origin.INTEGRATION + assert activity_log.value == "integration_webhook_custom_log" + assert activity_log.log_level == event_payload["level"] + assert activity_log.title == event_payload["title"] + assert activity_log.details == event_payload + assert activity_log.is_reversible is False From 8730c7184e2ba65b5167780d1ab70728cb1077d8 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 25 Jun 2024 19:29:49 -0300 Subject: [PATCH 5/5] Update dependencies --- dependencies/requirements-dev.txt | 22 ++++++++++++++++------ dependencies/requirements.in | 6 +++--- dependencies/requirements.txt | 6 +++--- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/dependencies/requirements-dev.txt b/dependencies/requirements-dev.txt index 70ce824a4..9d8a073a8 100644 --- a/dependencies/requirements-dev.txt +++ b/dependencies/requirements-dev.txt @@ -16,6 +16,8 @@ attrs==23.1.0 # via # jsonschema # pytest +autopep8==2.3.1 + # via django-silk babel==2.8.0 # via # -r requirements.in @@ -125,6 +127,7 @@ django==3.2.25 # django-model-utils # django-phonenumber-field # django-rest-swagger + # django-silk # django-sslserver # django-storages # django-tables2 @@ -166,6 +169,8 @@ django-rest-swagger @ git+https://github.com/PADAS/django-rest-swagger.git@0d933 # via -r requirements.in django-scopes==1.2.0 # via -r requirements.in +django-silk==5.1.0 + # via -r requirements.in django-simple-history==3.1.1 # via -r requirements.in django-sslserver==0.22 @@ -248,6 +253,8 @@ googleapis-common-protos[grpc]==1.60.0 # grpc-google-iam-v1 # grpcio-status # opentelemetry-exporter-otlp-proto-http +gprof2dot==2024.6.6 + # via django-silk greenlet==1.1.3.post0 # via gevent grpc-google-iam-v1==0.12.6 @@ -268,13 +275,13 @@ grpcio-status==1.57.0 # via # google-api-core # google-cloud-pubsub -gundi-client==1.0.1 +gundi-client==1.0.2 # via # -r requirements.in # cdip-connector -gundi-client-v2==2.1.5 +gundi-client-v2==2.3.0 # via -r requirements.in -gundi-core==1.3.0 +gundi-core==1.4.1 # via # -r requirements.in # cdip-connector @@ -446,6 +453,8 @@ pyasn1==0.4.8 # rsa pyasn1-modules==0.3.0 # via google-auth +pycodestyle==2.12.0 + # via autopep8 pycparser==2.20 # via # -r requirements.in @@ -598,6 +607,7 @@ sqlparse==0.3.1 # via # -r requirements.in # django + # django-silk statsd==3.3.0 # via cdip-connector timezonefinder==5.2.0 @@ -605,7 +615,9 @@ timezonefinder==5.2.0 toml==0.10.2 # via pytest tomli==2.0.1 - # via coverage + # via + # autopep8 + # coverage typing-extensions==4.7.1 # via # asgiref @@ -619,8 +631,6 @@ tzdata==2023.3 # django-celery-beat tzlocal==5.0.1 # via dateparser -unidecode==1.3.8 - # via -r requirements.in untangle==1.2.1 # via smartconnect-client uritemplate==4.1.1 diff --git a/dependencies/requirements.in b/dependencies/requirements.in index 88805795d..72134b2e2 100644 --- a/dependencies/requirements.in +++ b/dependencies/requirements.in @@ -68,9 +68,9 @@ jsonschema==4.17.3 django-celery-beat==2.5.0 django-storages==1.13.2 walrus==0.8.1 -gundi-core==1.3.0 -gundi-client==1.0.1 -gundi-client-v2==2.1.5 +gundi-core==1.4.1 +gundi-client==1.0.2 +gundi-client-v2==2.3.0 google-cloud-functions==1.13.1 google-cloud-run==0.10.1 google-cloud-eventarc==1.10.0 diff --git a/dependencies/requirements.txt b/dependencies/requirements.txt index 57432761a..3c299ae15 100644 --- a/dependencies/requirements.txt +++ b/dependencies/requirements.txt @@ -260,13 +260,13 @@ grpcio-status==1.56.0 # via # google-api-core # google-cloud-pubsub -gundi-client==1.0.1 +gundi-client==1.0.2 # via # -r requirements.in # cdip-connector -gundi-client-v2==2.1.5 +gundi-client-v2==2.3.0 # via -r requirements.in -gundi-core==1.3.0 +gundi-core==1.4.1 # via # -r requirements.in # cdip-connector