diff --git a/.env.filip.EXAMPLE b/.env.filip.EXAMPLE index 3dbb2c9e..a1e70dad 100644 --- a/.env.filip.EXAMPLE +++ b/.env.filip.EXAMPLE @@ -3,6 +3,7 @@ # local file `.env.filip` in the root folder, i.e., ".../FiLiP/.env.filip" # Do not add the created `.env.filip` to the git CB_URL="http://localhost:1026" +LD_CB_URL="http://localhost:1027" IOTA_URL="http://localhost:4041/" QL_URL="http://localhost:8668" MQTT_BROKER_URL="mqtt://localhost:1883" \ No newline at end of file diff --git a/.gitignore b/.gitignore index f9e444a3..0aaa3742 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,7 @@ requirements.txt .idea .cache venv - +.venv # Unittest # ##################### .pytest_cache diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e71047b..5a9de6ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,21 @@ -### v0.6.X +### v0.6.0 - add: Tutorial for connecting with secured endpoints ([#319](https://github.com/RWTH-EBC/FiLiP/pull/319)) -- add: tests for clear functions ([#318](https://github.com/RWTH-EBC/FiLiP/pull/336)) -- fix: clear functions for context broker ([#318](https://github.com/RWTH-EBC/FiLiP/pull/336)) +- add: Example for notification based command ([#332](https://github.com/RWTH-EBC/FiLiP/pull/332)) +- add: tests for clear functions ([#336](https://github.com/RWTH-EBC/FiLiP/pull/336)) +- 🚀 **add: API client for NGSI-LD context broker** ([#338](https://github.com/RWTH-EBC/FiLiP/pull/338) +,[#356](https://github.com/RWTH-EBC/FiLiP/pull/356) +,[#327](https://github.com/RWTH-EBC/FiLiP/pull/327) +,[#300](https://github.com/RWTH-EBC/FiLiP/pull/300) +,[#301](https://github.com/RWTH-EBC/FiLiP/pull/301) +,[#212](https://github.com/RWTH-EBC/FiLiP/pull/212) +,[#222](https://github.com/RWTH-EBC/FiLiP/pull/222) +,[#221](https://github.com/RWTH-EBC/FiLiP/pull/221) + ) +- fix: clear functions for context broker ([#336](https://github.com/RWTH-EBC/FiLiP/pull/336)) +- fix: validation error of ``ngsipayloadattr`` when the attribute substitution is used([#351](https://github.com/RWTH-EBC/FiLiP/pull/351)) +- update: integrate the key-values endpoints with normalized endpoints ([#318](https://github.com/RWTH-EBC/FiLiP/pull/318)) +- remove: ``update_entity_attributes_key_values`` and ``update_entity_key_values`` are removed ([#318](https://github.com/RWTH-EBC/FiLiP/pull/318)) + ### v0.5.0 - update: allow duplicated name in device, check uniqueness of object_id ([#279](https://github.com/RWTH-EBC/FiLiP/pull/279)) diff --git a/README.md b/README.md index d9219479..239ec78e 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,11 @@ Therefore, FiLiP currently only covers the APIs of the following GEs: - [github](https://github.com/telefonicaid/fiware-orion) - [swagger](https://swagger.lab.fiware.org/) - [NGSI v2 specifications](https://github.com/FIWARE/specifications/tree/master/OpenAPI/ngsiv2) - + +- NGSI-LD Context Broker for managing context data with Linked Data concept. The functionalities that FiLiP supports are closely aligned with the specification **_NGSI-LD V1.3.1_**, which is according to the FIWARE [catalogue](https://github.com/FIWARE/catalogue#core-context-broker-components) the latest spec version that has been implemented by all three brokers (Orion-LD, Scorpio, and Stellio). We currently use Orion-LD for testing. + - [github](https://github.com/FIWARE/context.Orion-LD) + - [swagger](https://swagger.lab.fiware.org/?url=https://raw.githubusercontent.com/FIWARE/specifications/master/OpenAPI/ngsi-ld/full_api.json#/) + > **Note**: `-experimental` flag need to be set for Orion-LD Context Broker to enable the full functionality. Check this [issue](https://github.com/FIWARE/context.Orion-LD/issues/1648) for more information - IoT-Agents for managing IoT Devices. IoT agents are implemented using the FIWARE IoT Agent Node Lib as a common framework. diff --git a/filip/__init__.py b/filip/__init__.py index 3162f5eb..4d600ea4 100644 --- a/filip/__init__.py +++ b/filip/__init__.py @@ -4,4 +4,4 @@ from filip.config import settings from filip.clients.ngsi_v2 import HttpClient -__version__ = '0.5.0' +__version__ = '0.6.0' diff --git a/filip/clients/base_http_client.py b/filip/clients/base_http_client.py index f987eca9..b43337c6 100644 --- a/filip/clients/base_http_client.py +++ b/filip/clients/base_http_client.py @@ -5,9 +5,17 @@ from pydantic import AnyHttpUrl from typing import Dict, ByteString, List, IO, Tuple, Union import requests - -from filip.models.base import FiwareHeader +from filip.models.base import FiwareHeader, FiwareLDHeader from filip.utils import validate_http_url +from enum import Enum + + +class NgsiURLVersion(str, Enum): + """ + URL part that defines the NGSI version for the API. + """ + v2_url = "/v2" + ld_url = "/ngsi-ld/v1" class BaseHttpClient: @@ -26,7 +34,7 @@ def __init__(self, url: Union[AnyHttpUrl, str] = None, *, session: requests.Session = None, - fiware_header: Union[Dict, FiwareHeader] = None, + fiware_header: Union[Dict, FiwareHeader, FiwareLDHeader] = None, **kwargs): self.logger = logging.getLogger( @@ -92,10 +100,16 @@ def fiware_headers(self, headers: Union[Dict, FiwareHeader]) -> None: """ if isinstance(headers, FiwareHeader): self._fiware_headers = headers + elif isinstance(headers, FiwareLDHeader): + self._fiware_headers = headers elif isinstance(headers, dict): self._fiware_headers = FiwareHeader.model_validate(headers) elif isinstance(headers, str): self._fiware_headers = FiwareHeader.model_validate_json(headers) + elif isinstance(headers, dict): + self._fiware_headers = FiwareLDHeader.parse_obj(headers) + elif isinstance(headers, str): + self._fiware_headers = FiwareLDHeader.parse_raw(headers) else: raise TypeError(f'Invalid headers! {type(headers)}') self.headers.update(self.fiware_headers.model_dump(by_alias=True)) diff --git a/filip/clients/ngsi_ld/cb.py b/filip/clients/ngsi_ld/cb.py new file mode 100644 index 00000000..775e5a3a --- /dev/null +++ b/filip/clients/ngsi_ld/cb.py @@ -0,0 +1,863 @@ +""" +Context Broker Module for API Client +""" +import json +import os +from math import inf +from typing import Any, Dict, List, Union, Optional, Literal +from urllib.parse import urljoin +import requests +from pydantic import \ + TypeAdapter, \ + PositiveInt, \ + PositiveFloat +from filip.clients.base_http_client import BaseHttpClient, NgsiURLVersion +from filip.config import settings +from filip.models.base import FiwareLDHeader, PaginationMethod, core_context +from filip.models.ngsi_v2.base import AttrsFormat +from filip.models.ngsi_ld.subscriptions import SubscriptionLD +from filip.models.ngsi_ld.context import ContextLDEntity, ContextLDEntityKeyValues, \ + ContextProperty, ContextRelationship, NamedContextProperty, \ + NamedContextRelationship, ActionTypeLD, UpdateLD +from filip.models.ngsi_v2.context import Query + + +class ContextBrokerLDClient(BaseHttpClient): + """ + Implementation of NGSI-LD Context Broker functionalities, such as creating + entities and subscriptions; retrieving, updating and deleting data. + Further documentation: + https://fiware-orion.readthedocs.io/en/master/ + + Api specifications for LD are located here: + https://www.etsi.org/deliver/etsi_gs/CIM/001_099/009/01.04.01_60/gs_cim009v010401p.pdf + """ + + def __init__(self, + url: str = None, + *, + session: requests.Session = None, + fiware_header: FiwareLDHeader = None, + **kwargs): + """ + + Args: + url: Url of context broker server + session (requests.Session): + fiware_header (FiwareHeader): fiware service and fiware service path + **kwargs (Optional): Optional arguments that ``request`` takes. + """ + # set service url + url = url or settings.LD_CB_URL + #base_http_client overwrites empty header with FiwareHeader instead of FiwareLD + init_header = fiware_header if fiware_header else FiwareLDHeader() + if init_header.link_header is None: + init_header.set_context(core_context) + super().__init__(url=url, + session=session, + fiware_header=init_header, + **kwargs) + # set the version specific url-pattern + self._url_version = NgsiURLVersion.ld_url.value + # For uplink requests, the Content-Type header is essential, + # Accept will be ignored + # For downlink requests, the Accept header is essential, + # Content-Type will be ignored + + # default uplink content JSON + self.headers.update({'Content-Type': 'application/json'}) + # default downlink content JSON-LD + self.headers.update({'Accept': 'application/ld+json'}) + + if init_header.ngsild_tenant is not None: + self.__make_tenant() + + def __pagination(self, + *, + method: PaginationMethod = PaginationMethod.GET, + url: str, + headers: Dict, + limit: Union[PositiveInt, PositiveFloat] = None, + params: Dict = None, + data: str = None) -> List[Dict]: + """ + NGSIv2 implements a pagination mechanism in order to help clients to + retrieve large sets of resources. This mechanism works for all listing + operations in the API (e.g. GET /v2/entities, GET /v2/subscriptions, + POST /v2/op/query, etc.). This function helps getting datasets that are + larger than the limit for the different GET operations. + + https://fiware-orion.readthedocs.io/en/master/user/pagination/index.html + + Args: + url: Information about the url, obtained from the original function + headers: The headers from the original function + params: + limit: + + Returns: + object: + + """ + if limit is None: + limit = inf + if limit > 1000: + params['limit'] = 1000 # maximum items per request + else: + params['limit'] = limit + + if self.session: + session = self.session + else: + session = requests.Session() + with session: + res = session.request(method=method, + url=url, + params=params, + headers=headers, + data=data) + if res.ok: + items = res.json() + # do pagination + if self._url_version == NgsiURLVersion.v2_url.value: + count = int(res.headers['Fiware-Total-Count']) + elif self._url_version == NgsiURLVersion.ld_url.value: + count = int(res.headers['NGSILD-Results-Count']) + else: + count = 0 + + while len(items) < limit and len(items) < count: + # Establishing the offset from where entities are retrieved + params['offset'] = len(items) + params['limit'] = min(1000, (limit - len(items))) + res = session.request(method=method, + url=url, + params=params, + headers=headers, + data=data) + if res.ok: + items.extend(res.json()) + else: + res.raise_for_status() + self.logger.debug('Received: %s', items) + return items + res.raise_for_status() + + def get_version(self) -> Dict: + """ + Gets version of Orion-LD context broker + Returns: + Dictionary with response + """ + url = urljoin(self.base_url, '/version') + try: + res = self.get(url=url) + if res.ok: + return res.json() + res.raise_for_status() + except requests.RequestException as err: + self.logger.error(err) + raise + + def __make_tenant(self): + """ + Create tenant if tenant + is given in headers + """ + idhex = f"urn:ngsi-ld:{os.urandom(6).hex()}" + e = ContextLDEntity(id=idhex,type=f"urn:ngsi-ld:{os.urandom(6).hex()}") + try: + self.post_entity(entity=e) + self.delete_entity_by_id(idhex) + except Exception as err: + self.log_error(err=err,msg="Error while creating tenant") + raise + + def get_statistics(self) -> Dict: + """ + Gets statistics of context broker + Returns: + Dictionary with response + """ + url = urljoin(self.base_url, 'statistics') + try: + res = self.get(url=url) + if res.ok: + return res.json() + res.raise_for_status() + except requests.RequestException as err: + self.logger.error(err) + raise + + def post_entity(self, + entity: ContextLDEntity, + append: bool = False, + update: bool = False): + """ + Function registers an Object with the NGSI-LD Context Broker, + if it already exists it can be automatically updated + if the update flag bool is True. + First a post request with the entity is tried, if the response code + is 422 the entity is uncrossable, as it already exists there are two + options, either overwrite it, if the attribute have changed + (e.g. at least one new/new values) (update = True) or leave + it the way it is (update=False) + + """ + url = urljoin(self.base_url, f'{self._url_version}/entities') + headers = self.headers.copy() + if entity.model_dump().get('@context',None) is not None: + headers.update({'Content-Type':'application/ld+json'}) + headers.update({'Link':None}) + try: + res = self.post( + url=url, + headers=headers, + json=entity.model_dump(exclude_unset=True, + exclude_defaults=True, + exclude_none=True)) + if res.ok: + self.logger.info("Entity successfully posted!") + return res.headers.get('Location') + res.raise_for_status() + except requests.RequestException as err: + if err.response.status_code == 409: + if append: # 409 entity already exists + return self.append_entity_attributes(entity=entity) + elif update: + return self.override_entities(entities=[entity]) + msg = f"Could not post entity {entity.id}" + self.log_error(err=err, msg=msg) + raise + + def override_entities(self, entities: List[ContextLDEntity]): + """ + Function to create or override existing entites with the NGSI-LD Context Broker. + The batch operation with Upsert will be used. + """ + return self.entity_batch_operation(entities=entities, + action_type=ActionTypeLD.UPSERT, + options="replace") + + def get_entity(self, + entity_id: str, + entity_type: str = None, + attrs: List[str] = None, + options: Optional[str] = None, + geometryProperty: Optional[str] = None, + ) \ + -> Union[ContextLDEntity, ContextLDEntityKeyValues, Dict[str, Any]]: + """ + This operation must return one entity element only, but there may be + more than one entity with the same ID (e.g. entities with same ID but + different types). In such case, an error message is returned, with + the HTTP status code set to 409 Conflict. + + Args: + entity_id (String): Id of the entity to be retrieved + entity_type (String): Entity type, to avoid ambiguity in case + there are several entities with the same entity id. + attrs (List of Strings): List of attribute names whose data must be + included in the response. The attributes are retrieved in the + order specified by this parameter. + See "Filtering out attributes and metadata" section for more + detail. If this parameter is not included, the attributes are + retrieved in arbitrary order, and all the attributes of the + entity are included in the response. + Example: temperature, humidity. + options (String): keyValues (simplified representation of entity) + or sysAttrs (include generated attrs createdAt and modifiedAt) + geometryProperty (String): Name of a GeoProperty. In the case of GeoJSON + Entity representation, this parameter indicates which GeoProperty to + use for the "geometry" element. By default, it shall be 'location'. + Returns: + ContextEntity + """ + url = urljoin(self.base_url, f'{self._url_version}/entities/{entity_id}') + headers = self.headers.copy() + params = {} + if entity_type: + params.update({'type': entity_type}) + if attrs: + params.update({'attrs': ','.join(attrs)}) + if geometryProperty: + params.update({'geometryProperty': geometryProperty}) + if options: + if options != 'keyValues' and options != 'sysAttrs': + raise ValueError(f'Only available options are \'keyValues\' and \'sysAttrs\'') + params.update({'options': options}) + + try: + res = self.get(url=url, params=params, headers=headers) + if res.ok: + self.logger.info("Entity successfully retrieved!") + self.logger.debug("Received: %s", res.json()) + if options == "keyValues": + return ContextLDEntityKeyValues(**res.json()) + else: + return ContextLDEntity(**res.json()) + res.raise_for_status() + except requests.RequestException as err: + msg = f"Could not load entity {entity_id}" + self.log_error(err=err, msg=msg) + raise + + GeometryShape = Literal["Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon"] + + def get_entity_list(self, + entity_id: Optional[str] = None, + id_pattern: Optional[str] = ".*", + entity_type: Optional[str] = None, + attrs: Optional[List[str]] = None, + q: Optional[str] = None, + georel: Optional[str] = None, + geometry: Optional[GeometryShape] = None, + coordinates: Optional[str] = None, + geoproperty: Optional[str] = None, + # csf: Optional[str] = None, # Context Source Filter + limit: Optional[PositiveInt] = None, + options: Optional[str] = None, + ) -> List[Union[ContextLDEntity, ContextLDEntityKeyValues]]: + """ + This operation retrieves a list of entities based on different query options. + By default, the operation retrieves all the entities in the context broker. + Args: + entity_id: + Id of the entity to be retrieved + id_pattern: + Regular expression to match the entity id + entity_type: + Entity type, to avoid ambiguity in case there are several + entities with the same entity id. + attrs: + List of attribute names whose data must be included in the response. + q: + Query expression, composed of attribute names, operators and values. + georel: + Geospatial relationship between the query geometry and the entities. + geometry: + Type of geometry for the query. The possible values are Point, + MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon. + coordinates: + Coordinates of the query geometry. The coordinates must be + expressed as a string of comma-separated values. + geoproperty: + Name of a GeoProperty. In the case of GeoJSON Entity representation, + this parameter indicates which GeoProperty to use for the "geometry" element. + limit: + Maximum number of entities to retrieve. + options: + Further options for the query. The available options are: + - keyValues (simplified representation of entity) + - sysAttrs (including createdAt and modifiedAt, etc.) + - count (include number of all matched entities in response header) + """ + url = urljoin(self.base_url, f'{self._url_version}/entities/') + headers = self.headers.copy() + params = {} + if entity_id: + params.update({'id': entity_id}) + if id_pattern: + params.update({'idPattern': id_pattern}) + if entity_type: + params.update({'type': entity_type}) + if attrs: + params.update({'attrs': ','.join(attrs)}) + if q: + params.update({'q': q}) + if georel: + params.update({'georel': georel}) + if geometry: + params.update({'geometry': geometry}) + if coordinates: + params.update({'coordinates': coordinates}) + if geoproperty: + params.update({'geoproperty': geoproperty}) + # if csf: # ContextSourceRegistration not supported yet + # params.update({'csf': csf}) + if limit: + if limit > 1000: + raise ValueError("limit must be an integer value <= 1000") + params.update({'limit': limit}) + if options: + if options != 'keyValues' and options != 'sysAttrs': + raise ValueError(f'Only available options are \'keyValues\' and \'sysAttrs\'') + params.update({'options': options}) + # params.update({'local': 'true'}) + + try: + res = self.get(url=url, params=params, headers=headers) + if res.ok: + self.logger.info("Entity successfully retrieved!") + entity_list: List[Union[ContextLDEntity, ContextLDEntityKeyValues]] = [] + if options == "keyValues": + entity_list = [ContextLDEntityKeyValues(**item) for item in res.json()] + return entity_list + else: + entity_list = [ContextLDEntity(**item) for item in res.json()] + return entity_list + res.raise_for_status() + except requests.RequestException as err: + msg = f"Could not load entity matching{params}" + self.log_error(err=err, msg=msg) + raise + + def replace_existing_attributes_of_entity(self, entity: ContextLDEntity, append: bool = False): + """ + The attributes previously existing in the entity are removed and + replaced by the ones in the request. + + Args: + entity (ContextEntity): + append (bool): + options: + Returns: + + """ + url = urljoin(self.base_url, f'{self._url_version}/entities/{entity.id}/attrs') + headers = self.headers.copy() + if entity.model_dump().get('@context',None) is not None: + headers.update({'Content-Type':'application/ld+json'}) + headers.update({'Link':None}) + try: + res = self.patch(url=url, + headers=headers, + json=entity.model_dump(exclude={'id', 'type'}, + exclude_unset=True, + exclude_none=True)) + if res.ok: + self.logger.info(f"Entity {entity.id} successfully " + "updated!") + else: + res.raise_for_status() + except requests.RequestException as err: + if append and err.response.status_code == 207: + return self.append_entity_attributes(entity=entity) + msg = f"Could not replace attribute of entity {entity.id} !" + self.log_error(err=err, msg=msg) + raise + + def update_entity_attribute(self, + entity_id: str, + attr: Union[ContextProperty, ContextRelationship, + NamedContextProperty, NamedContextRelationship], + attr_name: str = None): + """ + Updates a specified attribute from an entity. + Args: + attr: context attribute to update + entity_id: Id of the entity. Example: Bcn_Welt + entity_type: Entity type, to avoid ambiguity in case there are + several entities with the same entity id. + """ + headers = self.headers.copy() + if not isinstance(attr, NamedContextProperty) or not isinstance(attr, NamedContextRelationship): + assert attr_name is not None, "Missing name for attribute. " \ + "attr_name must be present if" \ + "attr is of type ContextAttribute" + else: + assert attr_name is None, "Invalid argument attr_name. Do not set " \ + "attr_name if attr is of type " \ + "NamedContextAttribute or NamedContextRelationship" + + url = urljoin(self.base_url, + f'{self._url_version}/entities/{entity_id}/attrs/{attr_name}') + + jsonnn = {} + if isinstance(attr, list) or isinstance(attr, NamedContextProperty): + jsonnn = attr.model_dump(exclude={'name'}, + exclude_unset=True, + exclude_none=True) + else: + prop = attr.model_dump() + for key, value in prop.items(): + if value and value != 'Property': + jsonnn[key] = value + + try: + res = self.patch(url=url, + headers=headers, + json=jsonnn) + if res.ok: + self.logger.info(f"Attribute {attr_name} of {entity_id} successfully updated!") + else: + res.raise_for_status() + except requests.RequestException as err: + msg = f"Could not update attribute '{attr_name}' of entity {entity_id}" + self.log_error(err=err, msg=msg) + raise + + def append_entity_attributes(self, + entity: ContextLDEntity, + options: Optional[str] = None + ): + """ + Append new Entity attributes to an existing Entity within an NGSI-LD system + Args: + entity (ContextLDEntity): + Entity to append attributes to. + options (str): + Options for the request. The only available value is + 'noOverwrite'. If set, it will raise 400, if all attributes + exist already. + + """ + url = urljoin(self.base_url, f'{self._url_version}/entities/{entity.id}/attrs') + headers = self.headers.copy() + if entity.model_dump().get('@context',None) is not None: + headers.update({'Content-Type':'application/ld+json'}) + headers.update({'Link':None}) + params = {} + + if options: + if options != 'noOverwrite': + raise ValueError(f'The only available value is \'noOverwrite\'') + params.update({'options': options}) + + try: + res = self.post(url=url, + headers=headers, + params=params, + json=entity.model_dump(exclude={'id', 'type'}, + exclude_unset=True, + exclude_none=True)) + if res.ok: + self.logger.info(f"Entity {entity.id} successfully updated!") + else: + res.raise_for_status() + except requests.RequestException as err: + msg = f"Could not update entity {entity.id}!" + self.log_error(err=err, msg=msg) + raise + + # def update_existing_attribute_by_name(self, entity: ContextLDEntity + # ): + # pass + + def delete_entity_by_id(self, + entity_id: str, + entity_type: Optional[str] = None): + """ + Deletes an entity by its id. For deleting mulitple entities at once, + entity_batch_operation() is more efficient. + Args: + entity_id: + ID of entity to delete. + entity_type: + Type of entity to delete. + """ + url = urljoin(self.base_url, f'{self._url_version}/entities/{entity_id}') + headers = self.headers.copy() + params = {} + + if entity_type: + params.update({'type': entity_type}) + + try: + res = self.delete(url=url, headers=headers, params=params) + if res.ok: + self.logger.info(f"Entity {entity_id} successfully deleted") + else: + res.raise_for_status() + except requests.RequestException as err: + msg = f"Could not delete entity {entity_id}" + self.log_error(err=err, msg=msg) + raise + + def delete_attribute(self, + entity_id: str, + attribute_id: str): + """ + Deletes an attribute from an entity. + Args: + entity_id: + ID of the entity. + attribute_id: + Name of the attribute to delete. + Returns: + + """ + url = urljoin(self.base_url, f'{self._url_version}/entities/{entity_id}/attrs/{attribute_id}') + headers = self.headers.copy() + + try: + res = self.delete(url=url, headers=headers) + if res.ok: + self.logger.info(f"Attribute {attribute_id} of Entity {entity_id} successfully deleted") + else: + res.raise_for_status() + except requests.RequestException as err: + msg = f"Could not delete attribute {attribute_id} of entity {entity_id}" + self.log_error(err=err, msg=msg) + raise + + # SUBSCRIPTION API ENDPOINTS + def get_subscription_list(self, + limit: PositiveInt = inf) -> List[SubscriptionLD]: + """ + Returns a list of all the subscriptions present in the system. + Args: + limit: Limit the number of subscriptions to be retrieved + Returns: + list of subscriptions + """ + url = urljoin(self.base_url, f'{self._url_version}/subscriptions/') + headers = self.headers.copy() + params = {} + + # We always use the 'count' option to check weather pagination is + # required + params.update({'options': 'count'}) + try: + items = self.__pagination(limit=limit, + url=url, + params=params, + headers=headers) + adapter = TypeAdapter(List[SubscriptionLD]) + return adapter.validate_python(items) + except requests.RequestException as err: + msg = "Could not load subscriptions!" + self.log_error(err=err, msg=msg) + raise + + def post_subscription(self, subscription: SubscriptionLD, + update: bool = False) -> str: + """ + Creates a new subscription. The subscription is represented by a + Subscription object defined in filip.cb.models. + + If the subscription already exists, the adding is prevented and the id + of the existing subscription is returned. + + A subscription is deemed as already existing if there exists a + subscription with the exact same subject and notification fields. All + optional fields are not considered. + + Args: + subscription: Subscription + update: True - If the subscription already exists, update it + False- If the subscription already exists, throw warning + Returns: + str: Id of the (created) subscription + + """ + existing_subscriptions = self.get_subscription_list() + + sub_hash = subscription.model_dump_json(include={'subject', 'notification', 'type'}) + for ex_sub in existing_subscriptions: + if sub_hash == ex_sub.model_dump_json(include={'subject', 'notification', 'type'}): + self.logger.info("Subscription already exists") + if update: + self.logger.info("Updated subscription") + subscription.id = ex_sub.id + self.update_subscription(subscription) + else: + self.logger.warning(f"Subscription existed already with the id" + f" {ex_sub.id}") + return ex_sub.id + + url = urljoin(self.base_url, f'{self._url_version}/subscriptions') + headers = self.headers.copy() + if subscription.model_dump().get('@context',None) is not None: + headers.update({'Content-Type':'application/ld+json'}) + headers.update({'Link':None}) + try: + res = self.post( + url=url, + headers=headers, + data=subscription.model_dump_json(exclude_unset=False, + exclude_defaults=False, + exclude_none=True)) + if res.ok: + self.logger.info("Subscription successfully created!") + return res.headers['Location'].split('/')[-1] + res.raise_for_status() + except requests.RequestException as err: + msg = "Could not send subscription!" + self.log_error(err=err, msg=msg) + raise + + def get_subscription(self, subscription_id: str) -> SubscriptionLD: + """ + Retrieves a subscription from the context broker. + Args: + subscription_id: id of the subscription + + Returns: + + """ + url = urljoin(self.base_url, f'{self._url_version}/subscriptions/{subscription_id}') + headers = self.headers.copy() + try: + res = self.get(url=url, headers=headers) + if res.ok: + self.logger.debug('Received: %s', res.json()) + return SubscriptionLD(**res.json()) + res.raise_for_status() + except requests.RequestException as err: + msg = f"Could not load subscription {subscription_id}" + self.log_error(err=err, msg=msg) + raise + + def update_subscription(self, subscription: SubscriptionLD) -> None: + """ + Only the fields included in the request are updated in the subscription. + Args: + subscription: Subscription to update + Returns: + + """ + url = urljoin(self.base_url, f'{self._url_version}/subscriptions/{subscription.id}') + headers = self.headers.copy() + if subscription.model_dump().get('@context',None) is not None: + headers.update({'Content-Type':'application/ld+json'}) + headers.update({'Link':None}) + try: + res = self.patch( + url=url, + headers=headers, + data=subscription.model_dump_json(exclude={'id'}, + exclude_unset=True, + exclude_defaults=True, + exclude_none=True)) + if res.ok: + self.logger.info("Subscription successfully updated!") + else: + res.raise_for_status() + except requests.RequestException as err: + msg = f"Could not update subscription {subscription.id}" + self.log_error(err=err, msg=msg) + raise + + def delete_subscription(self, subscription_id: str) -> None: + """ + Deletes a subscription from a Context Broker + Args: + subscription_id: id of the subscription + """ + url = urljoin(self.base_url, + f'{self._url_version}/subscriptions/{subscription_id}') + headers = self.headers.copy() + try: + res = self.delete(url=url, headers=headers) + if res.ok: + self.logger.info(f"Subscription '{subscription_id}' " + f"successfully deleted!") + else: + res.raise_for_status() + except requests.RequestException as err: + msg = f"Could not delete subscription {subscription_id}" + self.log_error(err=err, msg=msg) + raise + + def log_multi_errors(self, errors: List[Dict]) -> None: + for error in errors: + entity_id = error['entityId'] + error_details: dict = error['error'] + error_title = error_details.get('title') + error_status = error_details.get('status') + # error_detail = error_details['detail'] + self.logger.error("Response status: %d, Entity: %s, Reason: %s", + error_status, entity_id, error_title) + + def handle_multi_status_response(self, res: requests.Response): + """ + Handles the response of a batch_operation. If the response contains + errors, they are logged. If the response contains only errors, a RuntimeError + is raised. + Args: + res: + + Returns: + + """ + try: + res.raise_for_status() + if res.text: + response_data = res.json() + if 'errors' in response_data: + errors = response_data['errors'] + self.log_multi_errors(errors) + if 'success' in response_data: + successList = response_data['success'] + if len(successList) == 0: + raise RuntimeError("Batch operation resulted in errors only, see logs") + else: + self.logger.info("Empty response received.") + except json.JSONDecodeError: + self.logger.info("Error decoding JSON. Response may not be in valid JSON format.") + + # Batch operation API + def entity_batch_operation(self, + *, + entities: List[ContextLDEntity], + action_type: Union[ActionTypeLD, str], + options: Literal['noOverwrite', 'replace', 'update'] = None) -> None: + """ + This operation allows to create, update and/or delete several entities + in a single batch operation. + + This operation is split in as many individual operations as entities + in the entities vector, so the actionType is executed for each one of + them. Depending on the actionType, a mapping with regular non-batch + operations can be done: + + append: maps to POST /v2/entities (if the entity does not already exist) + or POST /v2/entities//attrs (if the entity already exists). + + appendStrict: maps to POST /v2/entities (if the entity does not + already exist) or POST /v2/entities//attrs?options=append (if the + entity already exists). + + update: maps to PATCH /v2/entities//attrs. + + delete: maps to DELETE /v2/entities//attrs/ on every + attribute included in the entity or to DELETE /v2/entities/ if + no attribute were included in the entity. + + replace: maps to PUT /v2/entities//attrs. + + Args: + entities: "an array of entities, each entity specified using the " + "JSON entity representation format " + action_type (Update): "actionType, to specify the kind of update + action to do: either append, appendStrict, update, delete, + or replace. " + options (str): Optional 'noOverwrite' 'replace' 'update' + + Returns: + + """ + + url = urljoin(self.base_url, f'{self._url_version}/entityOperations/{action_type.value}') + headers = self.headers.copy() + headers.update({'Content-Type': 'application/json'}) + params = {} + if options: + params.update({'options': options}) + update = UpdateLD(entities=entities) + try: + if action_type == ActionTypeLD.DELETE: + id_list = [entity.id for entity in entities] + res = self.post( + url=url, + headers=headers, + params=params, + data=json.dumps(id_list)) + else: + res = self.post( + url=url, + headers=headers, + params=params, + data=json.dumps(update.model_dump(by_alias=True, + exclude_unset=True, + exclude_none=True, + ).get('entities')) + ) + self.handle_multi_status_response(res) + except RuntimeError as rerr: + raise rerr + except Exception as err: + raise err + else: + self.logger.info(f"Update operation {action_type} succeeded!") diff --git a/filip/clients/ngsi_v2/cb.py b/filip/clients/ngsi_v2/cb.py index 0da0f04a..c4f499f8 100644 --- a/filip/clients/ngsi_v2/cb.py +++ b/filip/clients/ngsi_v2/cb.py @@ -5,6 +5,7 @@ import copy from copy import deepcopy +from enum import Enum from math import inf from pkg_resources import parse_version from pydantic import PositiveInt, PositiveFloat, AnyHttpUrl @@ -14,7 +15,7 @@ import requests from urllib.parse import urljoin import warnings -from filip.clients.base_http_client import BaseHttpClient +from filip.clients.base_http_client import BaseHttpClient, NgsiURLVersion from filip.config import settings from filip.models.base import FiwareHeader, PaginationMethod from filip.utils.simple_ql import QueryString @@ -33,7 +34,6 @@ from filip.models.ngsi_v2.base import AttrsFormat from filip.models.ngsi_v2.subscriptions import Subscription, Message from filip.models.ngsi_v2.registrations import Registration - if TYPE_CHECKING: from filip.clients.ngsi_v2.iota import IoTAClient @@ -71,6 +71,7 @@ def __init__( """ # set service url url = url or settings.CB_URL + self._url_version = NgsiURLVersion.v2_url.value super().__init__( url=url, session=session, fiware_header=fiware_header, **kwargs ) @@ -122,7 +123,7 @@ def __pagination( if res.ok: items = res.json() # do pagination - count = int(res.headers["Fiware-Total-Count"]) + count = int(res.headers['Fiware-Total-Count']) while len(items) < limit and len(items) < count: # Establishing the offset from where entities are retrieved @@ -167,7 +168,7 @@ def get_resources(self) -> Dict: Returns: Dict """ - url = urljoin(self.base_url, "v2") + url = urljoin(self.base_url, self._url_version) try: res = self.get(url=url, headers=self.headers) if res.ok: @@ -235,7 +236,7 @@ def post_entity( the keyValues simplified entity representation, i.e. ContextEntityKeyValues. """ - url = urljoin(self.base_url, "v2/entities") + url = urljoin(self.base_url, f'{self._url_version}/entities') headers = self.headers.copy() params = {} options = [] @@ -345,7 +346,7 @@ def get_entity_list( Returns: """ - url = urljoin(self.base_url, "v2/entities/") + url = urljoin(self.base_url, f'{self._url_version}/entities/') headers = self.headers.copy() params = {} @@ -450,7 +451,7 @@ def get_entity( Returns: ContextEntity """ - url = urljoin(self.base_url, f"v2/entities/{entity_id}") + url = urljoin(self.base_url, f'{self._url_version}/entities/{entity_id}') headers = self.headers.copy() params = {} if entity_type: @@ -515,7 +516,7 @@ def get_entity_attributes( Returns: Dict """ - url = urljoin(self.base_url, f"v2/entities/{entity_id}/attrs") + url = urljoin(self.base_url, f'{self._url_version}/entities/{entity_id}/attrs') headers = self.headers.copy() params = {} if entity_type: @@ -681,7 +682,7 @@ def delete_entity( Returns: None """ - url = urljoin(self.base_url, f"v2/entities/{entity_id}") + url = urljoin(self.base_url, f'{self._url_version}/entities/{entity_id}') headers = self.headers.copy() if entity_type: params = {'type': entity_type} @@ -809,7 +810,7 @@ def update_or_append_entity_attributes( None """ - url = urljoin(self.base_url, f"v2/entities/{entity_id}/attrs") + url = urljoin(self.base_url, f'{self._url_version}/entities/{entity_id}/attrs') headers = self.headers.copy() params = {} if entity_type: @@ -931,7 +932,7 @@ def update_existing_entity_attributes( None """ - url = urljoin(self.base_url, f"v2/entities/{entity_id}/attrs") + url = urljoin(self.base_url, f'{self._url_version}/entities/{entity_id}/attrs') headers = self.headers.copy() if entity_type: params = {"type": entity_type} @@ -1029,7 +1030,7 @@ def replace_entity_attributes( Returns: None """ - url = urljoin(self.base_url, f"v2/entities/{entity_id}/attrs") + url = urljoin(self.base_url, f'{self._url_version}/entities/{entity_id}/attrs') headers = self.headers.copy() params = {} options = [] @@ -1097,7 +1098,8 @@ def get_attribute( Error """ - url = urljoin(self.base_url, f"v2/entities/{entity_id}/attrs/{attr_name}") + url = urljoin(self.base_url, + f'{self._url_version}/entities/{entity_id}/attrs/{attr_name}') headers = self.headers.copy() params = {} if entity_type: @@ -1166,7 +1168,8 @@ def update_entity_attribute(self, ) attr_name = attr.name - url = urljoin(self.base_url, f"v2/entities/{entity_id}/attrs/{attr_name}") + url = urljoin(self.base_url, + f'{self._url_version}/entities/{entity_id}/attrs/{attr_name}') params = {} if entity_type: params.update({"type": entity_type}) @@ -1218,7 +1221,8 @@ def delete_entity_attribute( Error """ - url = urljoin(self.base_url, f"v2/entities/{entity_id}/attrs/{attr_name}") + url = urljoin(self.base_url, + f'{self._url_version}/entities/{entity_id}/attrs/{attr_name}') headers = self.headers.copy() params = {} if entity_type: @@ -1258,7 +1262,8 @@ def get_attribute_value( Returns: """ - url = urljoin(self.base_url, f"v2/entities/{entity_id}/attrs/{attr_name}/value") + url = urljoin(self.base_url, + f'{self._url_version}/entities/{entity_id}/attrs/{attr_name}/value') headers = self.headers.copy() params = {} if entity_type: @@ -1301,7 +1306,8 @@ def update_attribute_value(self, *, Returns: """ - url = urljoin(self.base_url, f"v2/entities/{entity_id}/attrs/{attr_name}/value") + url = urljoin(self.base_url, + f'{self._url_version}/entities/{entity_id}/attrs/{attr_name}/value') headers = self.headers.copy() params = {} if entity_type: @@ -1349,7 +1355,7 @@ def get_entity_types( Returns: """ - url = urljoin(self.base_url, "v2/types") + url = urljoin(self.base_url, f'{self._url_version}/types') headers = self.headers.copy() params = {} if limit: @@ -1378,7 +1384,7 @@ def get_entity_type(self, entity_type: str) -> Dict[str, Any]: Returns: """ - url = urljoin(self.base_url, f"v2/types/{entity_type}") + url = urljoin(self.base_url, f'{self._url_version}/types/{entity_type}') headers = self.headers.copy() params = {} try: @@ -1401,7 +1407,7 @@ def get_subscription_list(self, limit: PositiveInt = inf) -> List[Subscription]: Returns: list of subscriptions """ - url = urljoin(self.base_url, "v2/subscriptions/") + url = urljoin(self.base_url, f'{self._url_version}/subscriptions/') headers = self.headers.copy() params = {} @@ -1513,7 +1519,7 @@ def get_subscription(self, subscription_id: str) -> Subscription: Returns: """ - url = urljoin(self.base_url, f"v2/subscriptions/{subscription_id}") + url = urljoin(self.base_url, f'{self._url_version}/subscriptions/{subscription_id}') headers = self.headers.copy() try: res = self.get(url=url, headers=headers) @@ -1559,7 +1565,7 @@ def update_subscription( DeprecationWarning, ) - url = urljoin(self.base_url, f"v2/subscriptions/{subscription.id}") + url = urljoin(self.base_url, f'{self._url_version}/subscriptions/{subscription.id}') headers = self.headers.copy() headers.update({"Content-Type": "application/json"}) try: @@ -1586,7 +1592,8 @@ def delete_subscription(self, subscription_id: str) -> None: Args: subscription_id: id of the subscription """ - url = urljoin(self.base_url, f"v2/subscriptions/{subscription_id}") + url = urljoin(self.base_url, + f'{self._url_version}/subscriptions/{subscription_id}') headers = self.headers.copy() try: res = self.delete(url=url, headers=headers) @@ -1611,7 +1618,7 @@ def get_registration_list(self, *, limit: PositiveInt = None) -> List[Registrati Returns: """ - url = urljoin(self.base_url, "v2/registrations/") + url = urljoin(self.base_url, f'{self._url_version}/registrations/') headers = self.headers.copy() params = {} @@ -1641,7 +1648,7 @@ def post_registration(self, registration: Registration): Returns: """ - url = urljoin(self.base_url, "v2/registrations") + url = urljoin(self.base_url, f'{self._url_version}/registrations') headers = self.headers.copy() headers.update({"Content-Type": "application/json"}) try: @@ -1669,7 +1676,7 @@ def get_registration(self, registration_id: str) -> Registration: Returns: Registration """ - url = urljoin(self.base_url, f"v2/registrations/{registration_id}") + url = urljoin(self.base_url, f'{self._url_version}/registrations/{registration_id}') headers = self.headers.copy() try: res = self.get(url=url, headers=headers) @@ -1691,7 +1698,7 @@ def update_registration(self, registration: Registration): Returns: """ - url = urljoin(self.base_url, f"v2/registrations/{registration.id}") + url = urljoin(self.base_url, f'{self._url_version}/registrations/{registration.id}') headers = self.headers.copy() headers.update({"Content-Type": "application/json"}) try: @@ -1718,7 +1725,8 @@ def delete_registration(self, registration_id: str) -> None: Args: registration_id: id of the subscription """ - url = urljoin(self.base_url, f"v2/registrations/{registration_id}") + url = urljoin(self.base_url, + f'{self._url_version}/registrations/{registration_id}') headers = self.headers.copy() try: res = self.delete(url=url, headers=headers) @@ -1783,7 +1791,7 @@ def update(self, """ - url = urljoin(self.base_url, "v2/op/update") + url = urljoin(self.base_url, f'{self._url_version}/op/update') headers = self.headers.copy() headers.update({"Content-Type": "application/json"}) params = {} @@ -1837,7 +1845,7 @@ def query( follow the JSON entity representation format (described in the section "JSON Entity Representation"). """ - url = urljoin(self.base_url, "v2/op/query") + url = urljoin(self.base_url, f'{self._url_version}/op/query') headers = self.headers.copy() headers.update({"Content-Type": "application/json"}) params = {"options": "count"} diff --git a/filip/config.py b/filip/config.py index e8d3f81b..075fe357 100644 --- a/filip/config.py +++ b/filip/config.py @@ -26,6 +26,12 @@ class Settings(BaseSettings): validation_alias=AliasChoices( 'ORION_URL', 'CB_URL', 'CB_HOST', 'CONTEXTBROKER_URL', 'OCB_URL')) + LD_CB_URL: AnyHttpUrl = Field(default="http://127.0.0.1:1027", + validation_alias=AliasChoices('LD_ORION_URL', + 'LD_CB_URL', + 'ORION_LD_URL', + 'SCORPIO_URL', + 'STELLIO_URL')) IOTA_URL: AnyHttpUrl = Field(default="http://127.0.0.1:4041", validation_alias='IOTA_URL') @@ -38,6 +44,11 @@ class Settings(BaseSettings): 'MQTT_BROKER_URL', 'MQTT_URL', 'MQTT_BROKER')) + LD_MQTT_BROKER_URL: AnyUrl = Field(default="mqtt://127.0.0.1:1884", + validation_alias=AliasChoices( + 'LD_MQTT_BROKER_URL', + 'LD_MQTT_URL', + 'LD_MQTT_BROKER')) # create settings object settings = Settings() diff --git a/filip/models/__init__.py b/filip/models/__init__.py index 620efb18..c3505733 100644 --- a/filip/models/__init__.py +++ b/filip/models/__init__.py @@ -1 +1,2 @@ -from .base import FiwareHeader \ No newline at end of file +from .base import FiwareHeader +from .base import FiwareLDHeader diff --git a/filip/models/base.py b/filip/models/base.py index ce953b9f..6bbcb70a 100644 --- a/filip/models/base.py +++ b/filip/models/base.py @@ -3,11 +3,13 @@ """ from aenum import Enum -from pydantic import ConfigDict, BaseModel, Field, field_validator +from pydantic import ConfigDict, BaseModel, Field, field_validator, computed_field from filip.utils.validators import (validate_fiware_service_path, validate_fiware_service) +core_context = "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.6.jsonld" + class NgsiVersion(str, Enum): """ @@ -153,3 +155,35 @@ def _missing_name_(cls, name): for member in cls: if member.value.casefold() == name.casefold(): return member + + +class FiwareLDHeader(BaseModel): + """ + Define entity service paths which are supported by the NGSI + Context Brokers to support hierarchical scopes: + https://fiware-orion.readthedocs.io/en/master/user/service_path/index.html + """ + model_config = ConfigDict(populate_by_name=True, validate_assignment=True) + ngsild_tenant: str = Field( + alias="NGSILD-Tenant", + default=None, + max_length=50, + description="Alias to the Fiware service to used for multitenancy", + pattern=r"\w*$" + ) + link_header: str = Field( + alias="Link", + default=f'<{core_context}>; ' + 'rel="http://www.w3.org/ns/json-ld#context"; ' + 'type="application/ld+json"', + description="Fiware service used for multi-tenancy", + pattern=r"\w*$") + # @computed_field + # def Link(self) -> str: + # link_header = f'<{self.context}>; ' \ + # 'rel="http://www.w3.org/ns/json-ld#context"; ' \ + # 'type="application/ld+json"' + # return link_header + + def set_context(self, context: str): + self.link_header = f'<{context}>; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json"' diff --git a/filip/models/ngsi_ld/base.py b/filip/models/ngsi_ld/base.py new file mode 100644 index 00000000..6e58e9f0 --- /dev/null +++ b/filip/models/ngsi_ld/base.py @@ -0,0 +1,35 @@ +from typing import Union, Optional +from pydantic import BaseModel, Field, ConfigDict + + +class GeoQuery(BaseModel): + """ + GeoQuery used for Subscriptions, as described in NGSI-LD Spec section 5.2.13 + """ + geometry: str = Field( + description="A valid GeoJSON [8] geometry, type excepting GeometryCollection" + ) + coordinates: Union[list, str] = Field( + description="A JSON Array coherent with the geometry type as per " + "IETF RFC 7946 [8]" + ) + georel: str = Field( + description="A valid geo-relationship as defined by clause 4.10 (near, " + "within, etc.)" + ) + geoproperty: Optional[str] = Field( + default=None, + description="Attribute Name as a short-hand string" + ) + model_config = ConfigDict(populate_by_name=True) + + +def validate_ngsi_ld_query(q: str) -> str: + """ + Valid query string as described in NGSI-LD Spec section 5.2.12 + Args: + q: query string + Returns: + + """ + return q diff --git a/filip/models/ngsi_ld/context.py b/filip/models/ngsi_ld/context.py new file mode 100644 index 00000000..13f7613a --- /dev/null +++ b/filip/models/ngsi_ld/context.py @@ -0,0 +1,808 @@ +""" +NGSI LD models for context broker interaction +""" +import logging +from typing import Any, List, Dict, Union, Optional +from geojson_pydantic import Point, MultiPoint, LineString, MultiLineString, Polygon, \ + MultiPolygon +from typing_extensions import Self +from aenum import Enum +from pydantic import field_validator, ConfigDict, BaseModel, Field, model_validator +from filip.models.ngsi_v2 import ContextEntity +from filip.utils.validators import FiwareRegex, \ + validate_fiware_datatype_string_protect, validate_fiware_standard_regex +from pydantic_core import ValidationError + + +class DataTypeLD(str, Enum): + """ + In NGSI-LD the data types on context entities are only divided into properties and relationships. + """ + _init_ = 'value __doc__' + GEOPROPERTY = "GeoProperty", "A property that represents a geometry value" + PROPERTY = "Property", "All attributes that do not represent a relationship" + RELATIONSHIP = "Relationship", "Reference to another context entity, which can be identified with a URN." + + +# NGSI-LD entity models +class ContextProperty(BaseModel): + """ + The model for a property is represented by a JSON object with the following syntax: + + The attribute value is specified by the value, whose value can be any data type. This does not need to be + specified further. + + The NGSI type of the attribute is fixed and does not need to be specified. + Example: + + >>> data = {"value": <...>} + + >>> attr = ContextProperty(**data) + + """ + model_config = ConfigDict(extra='allow') # In order to allow nested properties + type: Optional[str] = Field( + default="Property", + title="type", + frozen=True + ) + value: Optional[Union[Union[float, int, bool, str, List, Dict[str, Any]], + List[Union[float, int, bool, str, List, + Dict[str, Any]]]]] = Field( + default=None, + title="Property value", + description="the actual data" + ) + observedAt: Optional[str] = Field( + None, title="Timestamp", + description="Representing a timestamp for the " + "incoming value of the property.", + max_length=256, + min_length=1, + ) + field_validator("observedAt")(validate_fiware_datatype_string_protect) + + createdAt: Optional[str] = Field( + None, title="Timestamp", + description="Representing a timestamp for the " + "creation time of the property.", + max_length=256, + min_length=1, + ) + field_validator("createdAt")(validate_fiware_datatype_string_protect) + + modifiedAt: Optional[str] = Field( + None, title="Timestamp", + description="Representing a timestamp for the " + "last modification of the property.", + max_length=256, + min_length=1, + ) + field_validator("modifiedAt")(validate_fiware_datatype_string_protect) + + UnitCode: Optional[str] = Field( + None, title="Unit Code", + description="Representing the unit of the value. " + "Should be part of the defined units " + "by the UN/ECE Recommendation No. 21" + "https://unece.org/fileadmin/DAM/cefact/recommendations/rec20/rec20_rev3_Annex2e.pdf ", + max_length=256, + min_length=1, + ) + field_validator("UnitCode")(validate_fiware_datatype_string_protect) + + datasetId: Optional[str] = Field( + None, title="dataset Id", + description="It allows identifying a set or group of property values", + max_length=256, + min_length=1, + ) + field_validator("datasetId")(validate_fiware_datatype_string_protect) + + @classmethod + def get_model_fields_set(cls): + """ + Get all names and aliases of the model fields. + """ + return set([field.validation_alias + for (_, field) in cls.model_fields.items()] + + [field_name for field_name in cls.model_fields]) + + @field_validator("type") + @classmethod + def check_property_type(cls, value): + """ + Force property type to be "Property" + Args: + value: value field + Returns: + value + """ + valid_property_types = ["Property", "Relationship", "TemporalProperty"] + if value not in valid_property_types: + msg = f'NGSI_LD Properties must have type {valid_property_types}, ' \ + f'not "{value}"' + logging.warning(msg=msg) + raise ValueError(msg) + return value + + +class NamedContextProperty(ContextProperty): + """ + Context properties are properties of context entities. For example, the current speed of a car could be modeled + as the current_speed property of the car-104 entity. + + In the NGSI-LD data model, properties have a name, the type "property" and a value. + """ + name: str = Field( + title="Property name", + description="The property name describes what kind of property the " + "attribute value represents of the entity, for example " + "current_speed. Allowed characters " + "are the ones in the plain ASCII set, except the following " + "ones: control characters, whitespace, &, ?, / and #.", + max_length=256, + min_length=1, + ) + field_validator("name")(validate_fiware_datatype_string_protect) + + +class ContextGeoPropertyValue(BaseModel): + """ + The value for a Geo property is represented by a JSON object with the following syntax: + + A type with value "Point" and the + coordinates with a list containing the coordinates as value + + Example: + "value": { + "type": "Point", + "coordinates": [ + -3.80356167695194, + 43.46296641666926 + ] + } + } + + """ + type: Optional[str] = Field( + default=None, + title="type", + frozen=True + ) + model_config = ConfigDict(extra='allow') + + @model_validator(mode='after') + def check_geoproperty_value(self) -> Self: + """ + Check if the value is a valid GeoProperty + """ + if self.model_dump().get("type") == "Point": + return Point(**self.model_dump()) + elif self.model_dump().get("type") == "LineString": + return LineString(**self.model_dump()) + elif self.model_dump().get("type") == "Polygon": + return Polygon(**self.model_dump()) + elif self.model_dump().get("type") == "MultiPoint": + return MultiPoint(**self.model_dump()) + elif self.model_dump().get("type") == "MultiLineString": + return MultiLineString(**self.model_dump()) + elif self.model_dump().get("type") == "MultiPolygon": + return MultiPolygon(**self.model_dump()) + elif self.model_dump().get("type") == "GeometryCollection": + raise ValueError("GeometryCollection is not supported") + + +class ContextGeoProperty(BaseModel): + """ + The model for a Geo property is represented by a JSON object with the following syntax: + + The attribute value is a JSON object with two contents. + + Example: + + { + "type": "GeoProperty", + "value": { + "type": "Point", + "coordinates": [ + -3.80356167695194, + 43.46296641666926 + ] + } + + """ + model_config = ConfigDict(extra='allow') + type: Optional[str] = Field( + default="GeoProperty", + title="type", + frozen=True + ) + value: Optional[Union[ContextGeoPropertyValue, + Point, LineString, Polygon, + MultiPoint, MultiPolygon, + MultiLineString]] = Field( + default=None, + title="GeoProperty value", + description="the actual data" + ) + observedAt: Optional[str] = Field( + default=None, + title="Timestamp", + description="Representing a timestamp for the " + "incoming value of the property.", + max_length=256, + min_length=1, + ) + field_validator("observedAt")(validate_fiware_datatype_string_protect) + + datasetId: Optional[str] = Field( + None, title="dataset Id", + description="It allows identifying a set or group of property values", + max_length=256, + min_length=1, + ) + field_validator("datasetId")(validate_fiware_datatype_string_protect) + + +class NamedContextGeoProperty(ContextGeoProperty): + """ + Context GeoProperties are geo properties of context entities. For example, the coordinates of a building . + + In the NGSI-LD data model, properties have a name, the type "Geoproperty" and a value. + """ + name: str = Field( + title="Property name", + description="The property name describes what kind of property the " + "attribute value represents of the entity, for example " + "current_speed. Allowed characters " + "are the ones in the plain ASCII set, except the following " + "ones: control characters, whitespace, &, ?, / and #.", + max_length=256, + min_length=1, + ) + field_validator("name")(validate_fiware_datatype_string_protect) + + +class ContextRelationship(BaseModel): + """ + The model for a relationship is represented by a JSON object with the following syntax: + + The attribute value is specified by the object, whose value can be a reference to another context entity. This + should be specified as the URN. The existence of this entity is not assumed. + + The NGSI type of the attribute is fixed and does not need to be specified. + + Example: + + >>> data = {"object": <...>} + + >>> attr = ContextRelationship(**data) + + """ + model_config = ConfigDict(extra='allow') # In order to allow nested relationships + type: Optional[str] = Field( + default="Relationship", + title="type", + frozen=True + ) + object: Optional[Union[Union[float, int, bool, str, List, Dict[str, Any]], + List[Union[float, int, bool, str, List, + Dict[str, Any]]]]] = Field( + default=None, + title="Realtionship object", + description="the actual object id" + ) + + datasetId: Optional[str] = Field( + None, title="dataset Id", + description="It allows identifying a set or group of property values", + max_length=256, + min_length=1, + ) + field_validator("datasetId")(validate_fiware_datatype_string_protect) + + observedAt: Optional[str] = Field( + None, titel="Timestamp", + description="Representing a timestamp for the " + "incoming value of the property.", + max_length=256, + min_length=1, + ) + field_validator("observedAt")(validate_fiware_datatype_string_protect) + + @field_validator("type") + @classmethod + def check_relationship_type(cls, value): + """ + Force property type to be "Relationship" + Args: + value: value field + Returns: + value + """ + if not value == "Relationship": + logging.warning(msg='NGSI_LD relationships must have type "Relationship"') + value = "Relationship" + return value + + +class NamedContextRelationship(ContextRelationship): + """ + Context Relationship are relations of context entities to each other. + For example, the current_speed of the entity car-104 could be modeled. + The location could be modeled as located_in the entity Room-001. + + In the NGSI-LD data model, relationships have a name, the type "relationship" and an object. + """ + name: str = Field( + title="Attribute name", + description="The attribute name describes what kind of property the " + "attribute value represents of the entity, for example " + "current_speed. Allowed characters " + "are the ones in the plain ASCII set, except the following " + "ones: control characters, whitespace, &, ?, / and #.", + max_length=256, + min_length=1, + # pattern=FiwareRegex.string_protect.value, + # Make it FIWARE-Safe + ) + field_validator("name")(validate_fiware_datatype_string_protect) + + +class ContextLDEntityKeyValues(BaseModel): + """ + Base Model for an entity is represented by a JSON object with the following + syntax. + + The entity id is specified by the object's id property, whose value + is a string containing the entity id. + + The entity type is specified by the object's type property, whose value + is a string containing the entity's type name. + + """ + model_config = ConfigDict(extra='allow', validate_default=True, + validate_assignment=True) + id: str = Field( + ..., + title="Entity Id", + description="Id of an entity in an NGSI context broker. Allowed " + "characters are the ones in the plain ASCII set, except " + "the following ones: control characters, " + "whitespace, &, ?, / and #." + "the id should be structured according to the urn naming scheme.", + json_schema_extra={"example":"urn:ngsi-ld:Room:001"}, + max_length=256, + min_length=1, + # pattern=FiwareRegex.standard.value, # Make it FIWARE-Safe + frozen=True + ) + field_validator("id")(validate_fiware_standard_regex) + type: str = Field( + ..., + title="Entity Type", + description="Id of an entity in an NGSI context broker. " + "Allowed characters are the ones in the plain ASCII set, " + "except the following ones: control characters, " + "whitespace, &, ?, / and #.", + json_schema_extra={"example":"Room"}, + max_length=256, + min_length=1, + # pattern=FiwareRegex.standard.value, # Make it FIWARE-Safe + frozen=True + ) + field_validator("type")(validate_fiware_standard_regex) + + +class PropertyFormat(str, Enum): + """ + Format to decide if properties of ContextEntity class are returned as + List of NamedContextAttributes or as Dict of ContextAttributes. + """ + LIST = 'list' + DICT = 'dict' + + +class ContextLDEntity(ContextLDEntityKeyValues): + """ + Context LD entities, or simply entities, are the center of gravity in the + FIWARE NGSI-LD information model. An entity represents a thing, i.e., any + physical or logical object (e.g., a sensor, a person, a room, an issue in + a ticketing system, etc.). Each entity has an entity id. + Furthermore, the type system of FIWARE NGSI enables entities to have an + entity type. Entity types are semantic types; they are intended to describe + the type of thing represented by the entity. For example, a context + entity #with id sensor-365 could have the type temperatureSensor. + + Each entity is uniquely identified by its id. + + The entity id is specified by the object's id property, whose value + is a string containing the entity id. + + The entity type is specified by the object's type property, whose value + is a string containing the entity's type name. + + Entity attributes are specified by additional properties and relationships, whose names are + the name of the attribute and whose representation is described in the + "ContextProperty"/"ContextRelationship"-model. Obviously, id and type are + not allowed to be used as attribute names. + + Example: + + >>> data = {'id': 'MyId', + 'type': 'MyType', + 'my_attr': {'value': 20}} + + >>> entity = ContextLDEntity(**data) + + """ + model_config = ConfigDict(extra='allow', + validate_default=True, + validate_assignment=True, + populate_by_name=True) + + observationSpace: Optional[ContextGeoProperty] = Field( + default=None, + title="Observation Space", + description="The geospatial Property representing " + "the geographic location that is being " + "observed, e.g. by a sensor. " + "For example, in the case of a camera, " + "the location of the camera and the " + "observationspace are different and " + "can be disjoint. " + ) + context: Optional[Union[str, List[str], Dict]] = Field( + title="@context", + default=None, + description="The @context in JSON-LD is used to expand terms, provided as short " + "hand strings, to concepts, specified as URIs, and vice versa, " + "to compact URIs into terms " + "The main implication of NGSI-LD API is that if the @context is " + "a compound one, i.e. an @context which references multiple " + "individual @context, served by resources behind different URIs, " + "then a wrapper @context has to be created and hosted.", + examples=["https://n5geh.github.io/n5geh.test-context.io/context_saref.jsonld"], + alias="@context", + validation_alias="@context", + frozen=False + ) + + @field_validator("context") + @classmethod + def return_context(cls, context): + return context + + operationSpace: Optional[ContextGeoProperty] = Field( + default=None, + title="Operation Space", + description="The geospatial Property representing " + "the geographic location in which an " + "Entity,e.g. an actuator is active. " + "For example, a crane can have a " + "certain operation space." + ) + + createdAt: Optional[str] = Field( + None, title="Timestamp", + description="Representing a timestamp for the " + "creation time of the property.", + max_length=256, + min_length=1, + ) + field_validator("createdAt")(validate_fiware_datatype_string_protect) + + modifiedAt: Optional[str] = Field( + None, title="Timestamp", + description="Representing a timestamp for the " + "last modification of the property.", + max_length=256, + min_length=1, + ) + field_validator("modifiedAt")(validate_fiware_datatype_string_protect) + + def __init__(self, + **data): + # There is currently no validation for extra fields + data.update(self._validate_attributes(data)) + super().__init__(**data) + + @classmethod + def get_model_fields_set(cls): + """ + Get all names and aliases of the model fields. + """ + return set([field.validation_alias + for (_, field) in cls.model_fields.items()] + + [field_name for field_name in cls.model_fields]) + + @classmethod + def _validate_single_property(cls, attr) -> ContextProperty: + property_fields = ContextProperty.get_model_fields_set() + property_fields.remove(None) + # subattrs = {} + if attr.get("type") == "Relationship": + attr_instance = ContextRelationship.model_validate(attr) + elif attr.get("type") == "GeoProperty": + try: + attr_instance = ContextGeoProperty.model_validate(attr) + except Exception as e: + pass + elif attr.get("type") == "Property" or attr.get("type") is None: + attr_instance = ContextProperty.model_validate(attr) + else: + raise ValueError(f"Attribute {attr.get('type')} " + "is not a valid type") + for subkey, subattr in attr.items(): + if isinstance(subattr, dict) and subkey not in property_fields: + attr_instance.model_extra.update( + {subkey: cls._validate_single_property(attr=subattr)} + ) + return attr_instance + + @classmethod + def _validate_attributes(cls, data: Dict): + entity_fields = cls.get_model_fields_set() + entity_fields.remove(None) + # Initialize the attribute dictionary + attrs = {} + # Iterate through the data + for key, attr in data.items(): + # Check if the keyword is not already present in the fields + if key not in entity_fields: + attrs[key] = cls._validate_single_property(attr=attr) + return attrs + + def model_dump( + self, + *args, + by_alias: bool = True, + **kwargs + ): + return super().model_dump(*args, by_alias=by_alias, **kwargs) + + @field_validator("id") + @classmethod + def _validate_id(cls, id: str): + if not id.startswith("urn:ngsi-ld:"): + logging.warning(msg='It is recommended that the entity id to be a URN,' + 'starting with the namespace "urn:ngsi-ld:"') + return id + + def get_properties(self, + response_format: Union[str, PropertyFormat] = + PropertyFormat.LIST) -> \ + Union[List[NamedContextProperty], + Dict[str, ContextProperty]]: + """ + Get all properties of the entity. + Args: + response_format: + + Returns: + + """ + response_format = PropertyFormat(response_format) + # response format dict: + if response_format == PropertyFormat.DICT: + final_dict = {} + for key, value in self.model_dump(exclude_unset=True).items(): + if key not in ContextLDEntity.get_model_fields_set(): + if value.get('type') != DataTypeLD.RELATIONSHIP: + if value.get('type') == DataTypeLD.GEOPROPERTY: + final_dict[key] = ContextGeoProperty(**value) + elif value.get('type') == DataTypeLD.PROPERTY: + final_dict[key] = ContextProperty(**value) + else: # named context property by default + final_dict[key] = ContextProperty(**value) + return final_dict + # response format list: + final_list = [] + for key, value in self.model_dump(exclude_unset=True).items(): + if key not in ContextLDEntity.get_model_fields_set(): + if value.get('type') != DataTypeLD.RELATIONSHIP: + if value.get('type') == DataTypeLD.GEOPROPERTY: + final_list.append(NamedContextGeoProperty(name=key, **value)) + elif value.get('type') == DataTypeLD.PROPERTY: + final_list.append(NamedContextProperty(name=key, **value)) + else: # named context property by default + final_list.append(NamedContextProperty(name=key, **value)) + return final_list + + def add_attributes(self, **kwargs): + """ + Invalid in NGSI-LD + """ + raise NotImplementedError( + "This method should not be used in NGSI-LD") + + def get_attribute(self, **kwargs): + """ + Invalid in NGSI-LD + """ + raise NotImplementedError( + "This method should not be used in NGSI-LD") + + def get_attributes(self, **kwargs): + """ + Invalid in NGSI-LD + """ + raise NotImplementedError( + "This method should not be used in NGSI-LD") + + def delete_attributes(self, **kwargs): + """ + Invalid in NGSI-LD + """ + raise NotImplementedError( + "This method should not be used in NGSI-LD") + + def delete_relationships(self, relationships: List[str]): + """ + Delete the given relationships from the entity + + Args: + relationships: List of relationship names + + Returns: + + """ + all_relationships = self.get_relationships(response_format='dict') + for relationship in relationships: + # check they are relationships + if relationship not in all_relationships: + raise ValueError(f"Relationship {relationship} does not exist") + delattr(self, relationship) + + def delete_properties(self, props: Union[Dict[str, ContextProperty], + List[NamedContextProperty], + List[str]]): + """ + Delete the given properties from the entity + + Args: + props: can be given in multiple forms + 1) Dict: {"": ContextProperty, ...} + 2) List: [NamedContextProperty, ...] + 3) List: ["", ...] + + Returns: + + """ + names: List[str] = [] + if isinstance(props, list): + for entry in props: + if isinstance(entry, str): + names.append(entry) + elif isinstance(entry, NamedContextProperty): + names.append(entry.name) + else: + names.extend(list(props.keys())) + + # check there are no relationships + relationship_names = [rel.name for rel in self.get_relationships()] + for name in names: + if name in relationship_names: + raise TypeError(f"{name} is a relationship") + + for name in names: + delattr(self, name) + + def add_geo_properties(self, attrs: Union[Dict[str, ContextGeoProperty], + List[NamedContextGeoProperty]]) -> None: + """ + Add property to entity + Args: + attrs: + Returns: + None + """ + if isinstance(attrs, list): + attrs = {attr.name: ContextGeoProperty(**attr.model_dump(exclude={'name'}, + exclude_unset=True)) + for attr in attrs} + for key, attr in attrs.items(): + self.__setattr__(name=key, value=attr) + + def add_properties(self, attrs: Union[Dict[str, ContextProperty], + List[NamedContextProperty]]) -> None: + """ + Add property to entity + Args: + attrs: + Returns: + None + """ + if isinstance(attrs, list): + attrs = {attr.name: ContextProperty(**attr.model_dump(exclude={'name'}, + exclude_unset=True)) + for attr in attrs} + for key, attr in attrs.items(): + self.__setattr__(name=key, value=attr) + + def add_relationships(self, relationships: Union[Dict[str, ContextRelationship], + List[NamedContextRelationship]]) -> None: + """ + Add relationship to entity + Args: + relationships: + Returns: + None + """ + if isinstance(relationships, list): + relationships = {attr.name: ContextRelationship(**attr.dict(exclude={'name'})) + for attr in relationships} + for key, attr in relationships.items(): + self.__setattr__(name=key, value=attr) + + def get_relationships(self, + response_format: Union[str, PropertyFormat] = + PropertyFormat.LIST) \ + -> Union[List[NamedContextRelationship], + Dict[str, ContextRelationship]]: + """ + Get all relationships of the context entity + + Args: + response_format: + + Returns: + + """ + response_format = PropertyFormat(response_format) + # response format dict: + if response_format == PropertyFormat.DICT: + final_dict = {} + for key, value in self.model_dump(exclude_unset=True).items(): + if key not in ContextLDEntity.get_model_fields_set(): + try: + if value.get('type') == DataTypeLD.RELATIONSHIP: + final_dict[key] = ContextRelationship(**value) + except AttributeError: # if context attribute + if isinstance(value, list): + pass + return final_dict + # response format list: + final_list = [] + for key, value in self.model_dump(exclude_unset=True).items(): + if key not in ContextLDEntity.get_model_fields_set(): + if value.get('type') == DataTypeLD.RELATIONSHIP: + final_list.append(NamedContextRelationship(name=key, **value)) + return final_list + + def get_context(self): + """ + Args: + response_format: + + Returns: context of the entity as list + + """ + _, context = self.model_dump(include={"context"}).popitem() + if not context: + logging.warning("No context in entity") + return None + else: + return context + + +class ActionTypeLD(str, Enum): + """ + Options for queries + """ + + CREATE = "create" + UPSERT = "upsert" + UPDATE = "update" + DELETE = "delete" + + +class UpdateLD(BaseModel): + """ + Model for update action + """ + entities: List[Union[ContextLDEntity, ContextLDEntityKeyValues]] = Field( + description="an array of entities, each entity specified using the " + "JSON entity representation format " + ) diff --git a/filip/models/ngsi_ld/subscriptions.py b/filip/models/ngsi_ld/subscriptions.py new file mode 100644 index 00000000..73e6640a --- /dev/null +++ b/filip/models/ngsi_ld/subscriptions.py @@ -0,0 +1,297 @@ +from typing import List, Optional, Literal +from pydantic import ConfigDict, BaseModel, Field, HttpUrl, AnyUrl, \ + field_validator, model_validator +import dateutil.parser +from filip.models.ngsi_ld.base import GeoQuery, validate_ngsi_ld_query + + +class EntityInfo(BaseModel): + """ + In v1.3.1 it is specified as EntityInfo + In v1.6.1 it is specified in a new data type, namely EntitySelector + """ + id: Optional[HttpUrl] = Field( + default=None, + description="Entity identifier (valid URI)" + ) + idPattern: Optional[str] = Field( + default=None, + description="Regular expression as per IEEE POSIX 1003.2™ [11]" + ) + type: str = Field( + description="Fully Qualified Name of an Entity Type or the Entity Type Name as a " + "short-hand string. See clause 4.6.2" + ) + model_config = ConfigDict(populate_by_name=True) + + +class KeyValuePair(BaseModel): + key: str + value: str + + +class Endpoint(BaseModel): + """ + This datatype represents the parameters that are required in order to define + an endpoint for notifications. This can include the endpoint's URI, a + generic{key, value} array, named receiverInfo, which contains, in a + generalized form, whatever extra information the broker shall convey to the + receiver in order for the broker to successfully communicate with + receiver (e.g Authorization material), or for the receiver to correctly + interpret the received content (e.g. the Link URL to fetch an @context). + + Additionally, it can include another generic{key, value} array, named + notifierInfo, which contains the configuration that the broker needs to + know in order to correctly set up the communication channel towards the + receiver + + Example of "receiverInfo" + "receiverInfo": [ + { + "key": "H1", + "value": "123" + }, + { + "key": "H2", + "value": "456" + } + ] + + Example of "notifierInfo" + "notifierInfo": [ + { + "key": "MQTT-Version", + "value": "mqtt5.0" + } + ] + """ + uri: AnyUrl = Field( + description="Dereferenceable URI" + ) + accept: Optional[str] = Field( + default=None, + description="MIME type for the notification payload body " + "(application/json, application/ld+json, " + "application/geo+json)" + ) + receiverInfo: Optional[List[KeyValuePair]] = Field( + default=None, + description="Generic {key, value} array to convey optional information " + "to the receiver" + ) + notifierInfo: Optional[List[KeyValuePair]] = Field( + default=None, + description="Generic {key, value} array to set up the communication " + "channel" + ) + model_config = ConfigDict(populate_by_name=True) + + @field_validator("uri") + @classmethod + def check_uri(cls, uri: AnyUrl): + if uri.scheme not in ("http", "mqtt"): + raise ValueError("NGSI-LD currently only support http and mqtt") + return uri + + @field_validator("notifierInfo") + @classmethod + def check_notifier_info(cls, notifierInfo: List[KeyValuePair]): + # TODO add validation of notifierInfo for MQTT notification + return notifierInfo + + +class NotificationParams(BaseModel): + """ + NGSI-LD Notification model. It contains the parameters that allow to + convey the details of a notification, as described in NGSI-LD Spec section 5.2.14 + """ + attributes: Optional[List[str]] = Field( + default=None, + description="Entity Attribute Names (Properties or Relationships) to be included " + "in the notification payload body. If undefined, it will mean all Attributes" + ) + format: Optional[str] = Field( + default="normalized", + description="Conveys the representation format of the entities delivered at " + "notification time. By default, it will be in normalized format" + ) + endpoint: Endpoint = Field( + ..., + description="Notification endpoint details" + ) + # status can either be "ok" or "failed" + status: Literal["ok", "failed"] = Field( + default="ok", + description="Status of the Notification. It shall be 'ok' if the last attempt " + "to notify the subscriber succeeded. It shall be 'failed' if the last" + " attempt to notify the subscriber failed" + ) + + # Additional members + timesSent: Optional[int] = Field( + default=None, + description="Number of times that the notification was sent. Provided by the " + "system when querying the details of a subscription" + ) + lastNotification: Optional[str] = Field( + default=None, + description="Timestamp corresponding to the instant when the last notification " + "was sent. Provided by the system when querying the details of a subscription" + ) + lastFailure: Optional[str] = Field( + default=None, + description="Timestamp corresponding to the instant when the last notification" + " resulting in failure was sent. Provided by the system when querying the details of a subscription" + ) + lastSuccess: Optional[str] = Field( + default=None, + description="Timestamp corresponding to the instant when the last successful " + "notification was sent. Provided by the system when querying the details of a subscription" + ) + model_config = ConfigDict(populate_by_name=True) + + +class TemporalQuery(BaseModel): + """ + Temporal query according to NGSI-LD Spec section 5.2.21 + + timerel: + Temporal relationship, one of "before", "after" and "between". + "before": before the time specified by timeAt. + "after": after the time specified by timeAt. + "between": after the time specified by timeAt and before the time specified by + endtimeAt + timeAt: + A DateTime object following ISO 8061, e.g. 2007-12-24T18:21Z + endTimeAt (optional): + A DateTime object following ISO 8061, e.g. 2007-12-24T18:21Z + Only required when timerel="between" + timeproperty: str + Representing a Propertyname of the Property that contains the temporal data that + will be used to resolve the temporal query. If not specified, the default is + "observedAt" + + """ + model_config = ConfigDict(populate_by_name=True) + timerel: Literal['before', 'after', 'between'] = Field( + ..., + description="String representing the temporal relationship as defined by clause " + "4.11 (Allowed values: 'before', 'after', and 'between') " + ) + timeAt: str = Field( + ..., + description="String representing the timeAt parameter as defined by clause " + "4.11. It shall be a DateTime " + ) + endTimeAt: Optional[str] = Field( + default=None, + description="String representing the endTimeAt parameter as defined by clause " + "4.11. It shall be a DateTime. Cardinality shall be 1 if timerel is " + "equal to 'between' " + ) + timeproperty: Optional[str] = Field( + default=None, + description="String representing a Property name. The name of the Property that " + "contains the temporal data that will be used to resolve the " + "temporal query. If not specified, " + ) + + @field_validator("timeAt", "endTimeAt") + @classmethod + def check_uri(cls, v: str): + if not v: + return v + else: + try: + dateutil.parser.isoparse(v) + except ValueError: + raise ValueError("timeAt must be in ISO8061 format") + return v + + # when timerel=between, endTimeAt must be specified + @model_validator(mode='after') + def check_passwords_match(self) -> 'TemporalQuery': + if self.timerel == "between" and self.endTimeAt is None: + raise ValueError('When timerel="between", endTimeAt must be specified') + return self + + +class SubscriptionLD(BaseModel): + """ + Context Subscription model according to NGSI-LD Spec section 5.2.12 + """ + id: Optional[str] = Field( + default=None, + description="Subscription identifier (JSON-LD @id)" + ) + type: str = Field( + default="Subscription", + description="JSON-LD @type" + ) + subscriptionName: Optional[str] = Field( + default=None + + , + description="A (short) name given to this Subscription" + ) + description: Optional[str] = Field( + default=None, + description="Subscription description" + ) + entities: Optional[List[EntityInfo]] = Field( + default=None, + description="Entities subscribed" + ) + watchedAttributes: Optional[List[str]] = Field( + default=None, + description="Watched Attributes (Properties or Relationships)" + ) + notificationTrigger: Optional[List[str]] = Field( + default=None, + description="Notification triggers" + ) + timeInterval: Optional[int] = Field( + default=None, + description="Time interval in seconds" + ) + q: Optional[str] = Field( + default=None, + description="Query met by subscribed entities to trigger the notification" + ) + @field_validator("q") + @classmethod + def check_q(cls, v: str): + return validate_ngsi_ld_query(v) + geoQ: Optional[GeoQuery] = Field( + default=None, + description="Geoquery met by subscribed entities to trigger the notification" + ) + csf: Optional[str] = Field( + default=None, + description="Context source filter" + ) + isActive: bool = Field( + default=True, + description="Indicates if the Subscription is under operation (True) or paused (False)" + ) + notification: NotificationParams = Field( + ..., + description="Notification details" + ) + expiresAt: Optional[str] = Field( + default=None, + description="Expiration date for the subscription" + ) + throttling: Optional[int] = Field( + default=None, + description="Minimal period of time in seconds between two consecutive notifications" + ) + temporalQ: Optional[TemporalQuery] = Field( + default=None, + description="Temporal Query" + ) + lang: Optional[str] = Field( + default=None, + description="Language filter applied to the query" + ) + model_config = ConfigDict(populate_by_name=True) diff --git a/filip/models/ngsi_v2/__init__.py b/filip/models/ngsi_v2/__init__.py index c86f116d..cb14b31f 100644 --- a/filip/models/ngsi_v2/__init__.py +++ b/filip/models/ngsi_v2/__init__.py @@ -1,3 +1,4 @@ """ This package contains models for FIWAREs NGSI-LD APIs """ +from .context import ContextEntity diff --git a/filip/models/ngsi_v2/base.py b/filip/models/ngsi_v2/base.py index a902bb14..2b6d9fee 100644 --- a/filip/models/ngsi_v2/base.py +++ b/filip/models/ngsi_v2/base.py @@ -238,7 +238,7 @@ class NamedMetadata(Metadata): """ name: str = Field( - titel="metadata name", + title="metadata name", description="a metadata name, describing the role of the metadata in " "the place where it occurs; for example, the metadata name " "accuracy indicates that the metadata value describes how " @@ -354,7 +354,7 @@ class BaseNameAttribute(BaseModel): """ name: str = Field( - titel="Attribute name", + title="Attribute name", description="The attribute name describes what kind of property the " "attribute value represents of the entity, for example " "current_speed. Allowed characters " diff --git a/filip/models/ngsi_v2/context.py b/filip/models/ngsi_v2/context.py index 20d28145..19567d9e 100644 --- a/filip/models/ngsi_v2/context.py +++ b/filip/models/ngsi_v2/context.py @@ -144,10 +144,10 @@ class ContextEntityKeyValues(BaseModel): ..., title="Entity Id", description="Id of an entity in an NGSI context broker. Allowed " - "characters are the ones in the plain ASCII set, except " - "the following ones: control characters, " - "whitespace, &, ?, / and #.", - example="Bcn-Welt", + "characters are the ones in the plain ASCII set, except " + "the following ones: control characters, " + "whitespace, &, ?, / and #.", + json_schema_extra={"example":"Bcn-Welt"}, max_length=256, min_length=1, frozen=True, @@ -157,10 +157,10 @@ class ContextEntityKeyValues(BaseModel): ..., title="Entity Type", description="Id of an entity in an NGSI context broker. " - "Allowed characters are the ones in the plain ASCII set, " - "except the following ones: control characters, " - "whitespace, &, ?, / and #.", - example="Room", + "Allowed characters are the ones in the plain ASCII set, " + "except the following ones: control characters, " + "whitespace, &, ?, / and #.", + json_schema_extra={"example":"Room"}, max_length=256, min_length=1, frozen=True, diff --git a/filip/models/ngsi_v2/iot.py b/filip/models/ngsi_v2/iot.py index d6950954..57a46670 100644 --- a/filip/models/ngsi_v2/iot.py +++ b/filip/models/ngsi_v2/iot.py @@ -225,12 +225,12 @@ def validate_cbHost(cls, value): return str(value) if value else value lazy: Optional[List[LazyDeviceAttribute]] = Field( default=[], - desription="list of common lazy attributes of the device. For each " + description="list of common lazy attributes of the device. For each " "attribute, its name and type must be provided." ) commands: Optional[List[DeviceCommand]] = Field( default=[], - desription="list of common commands attributes of the device. For each " + description="list of common commands attributes of the device. For each " "attribute, its name and type must be provided, additional " "metadata is optional" ) @@ -383,7 +383,7 @@ class Device(DeviceSettings): ) commands: List[DeviceCommand] = Field( default=[], - desription="List of commands of the device" + description="List of commands of the device" ) attributes: List[DeviceAttribute] = Field( default=[], diff --git a/filip/models/ngsi_v2/registrations.py b/filip/models/ngsi_v2/registrations.py index fc6920ae..148bb74c 100644 --- a/filip/models/ngsi_v2/registrations.py +++ b/filip/models/ngsi_v2/registrations.py @@ -101,18 +101,19 @@ class Registration(BaseModel): default=None, description="A free text used by the client to describe the " "registration.", - example="Relative Humidity Context Source" + json_schema_extra={"example":"Relative Humidity Context Source"} ) provider: Provider = Field( description="Object that describes the context source registered.", - example='"http": {"url": "http://localhost:1234"}' + json_schema_extra={"example": '"http": {"url": "http://localhost:1234"}'} ) dataProvided: DataProvided = Field( description="Object that describes the data provided by this source", - example='{' + json_schema_extra={"example": '{' ' "entities": [{"id": "room2", "type": "Room"}],' ' "attrs": ["relativeHumidity"]' '},' + } ) status: Optional[Status] = Field( default=Status.ACTIVE, diff --git a/filip/models/ngsi_v2/subscriptions.py b/filip/models/ngsi_v2/subscriptions.py index 1e94b777..59925be3 100644 --- a/filip/models/ngsi_v2/subscriptions.py +++ b/filip/models/ngsi_v2/subscriptions.py @@ -497,21 +497,21 @@ class Subscription(BaseModel): ) subject: Subject = Field( description="An object that describes the subject of the subscription.", - examples=[{ + json_schema_extra={'example':{ 'entities': [{'idPattern': '.*', 'type': 'Room'}], 'condition': { 'attrs': ['temperature'], 'expression': {'q': 'temperature>40'}, - }, - }], + }, + }} ) notification: Notification = Field( description="An object that describes the notification to send when " "the subscription is triggered.", - examples=[{ + json_schema_extra={'example':{ 'http': {'url': 'http://localhost:1234'}, 'attrs': ['temperature', 'humidity'], - }], + }} ) expires: Optional[datetime] = Field( default=None, diff --git a/filip/utils/cleanup.py b/filip/utils/cleanup.py index 6cda8402..764d9762 100644 --- a/filip/utils/cleanup.py +++ b/filip/utils/cleanup.py @@ -7,11 +7,60 @@ from pydantic import AnyHttpUrl, AnyUrl from requests import RequestException from typing import Callable, List, Union -from filip.models import FiwareHeader +from filip.models import FiwareHeader, FiwareLDHeader from filip.clients.ngsi_v2 import \ ContextBrokerClient, \ IoTAClient, \ QuantumLeapClient +from filip.clients.ngsi_ld.cb import ContextBrokerLDClient +from filip.models.ngsi_ld.context import ActionTypeLD +import logging + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +def clear_context_broker_ld(url: str = None, + fiware_ld_header: FiwareLDHeader = None, + cb_ld_client: ContextBrokerLDClient = None + ): + """ + Function deletes all entities and subscriptions for a tenant in an LD context broker. + + Args: + url: Url of the context broker LD + fiware_ld_header: header of the NGSI-LD tenant + cb_ld_client: NGSI-LD context broker client object + + Returns: + + """ + assert url or cb_ld_client, "Either url or client object must be given" + # create client + if cb_ld_client is None: + client = ContextBrokerLDClient(url=url, fiware_header=fiware_ld_header) + else: + client = cb_ld_client + # clean entities iteratively + try: + entity_list = True + while entity_list: + entity_list = client.get_entity_list(limit=100) + if entity_list: + client.entity_batch_operation(action_type=ActionTypeLD.DELETE, + entities=entity_list) + except RequestException as e: + logger.warning("Could not clean entities completely") + raise + + # clean subscriptions + try: + sub_list = cb_ld_client.get_subscription_list() + for sub in sub_list: + cb_ld_client.delete_subscription(sub.id) + except RequestException as e: + logger.warning("Could not clean subscriptions completely") + raise def clear_context_broker(url: str=None, @@ -47,6 +96,12 @@ def clear_context_broker(url: str=None, else: client = cb_client + # clear registrations + if clear_registrations: + for reg in client.get_registration_list(): + client.delete_registration(registration_id=reg.id) + assert len(client.get_registration_list()) == 0 + # clean entities client.delete_entities(entities=client.get_entity_list()) @@ -55,11 +110,6 @@ def clear_context_broker(url: str=None, client.delete_subscription(subscription_id=sub.id) assert len(client.get_subscription_list()) == 0 - # clear registrations - if clear_registrations: - for reg in client.get_registration_list(): - client.delete_registration(registration_id=reg.id) - assert len(client.get_registration_list()) == 0 def clear_iot_agent(url: Union[str, AnyHttpUrl] = None, @@ -154,7 +204,8 @@ def clear_all(*, iota_client: IoTAClient = None, ql_client: QuantumLeapClient = None): """ - Clears all services that a url is provided for + Clears all services that a url is provided for. + If cb_url is provided, the registration will also be deleted. Args: fiware_header: @@ -185,7 +236,8 @@ def clear_all(*, clear_iot_agent(url=url, fiware_header=fiware_header) if cb_url is not None or cb_client is not None: - clear_context_broker(url=cb_url, fiware_header=fiware_header, cb_client=cb_client) + clear_context_broker(url=cb_url, fiware_header=fiware_header, cb_client=cb_client, + clear_registrations=True) if ql_url is not None or ql_client is not None: clear_quantumleap(url=ql_url, fiware_header=fiware_header, ql_client=ql_client) diff --git a/setup.py b/setup.py index 1d9d0368..39bbab41 100644 --- a/setup.py +++ b/setup.py @@ -21,18 +21,19 @@ 'rapidfuzz~=3.4.0', 'geojson-pydantic~=1.0.2', 'wget~=3.2', + 'PyLD~=2.0.4', 'pyjexl~=0.3.0'] SETUP_REQUIRES = INSTALL_REQUIRES.copy() -VERSION = '0.5.0' +VERSION = '0.6.0' setuptools.setup( name='filip', version=VERSION, author='RWTH Aachen University, E.ON Energy Research Center, Institute\ of Energy Efficient Buildings and Indoor Climate', - author_email='tstorek@eonerc.rwth-aachen.de', + author_email='junsong.du@eonerc.rwth-aachen.de', description='[FI]WARE [Li]brary for [P]ython', long_description=LONG_DESCRIPTION, long_description_content_type="text/markdown", diff --git a/tests/TEMPLATE_ENV b/tests/TEMPLATE_ENV index a06df4eb..6fb1b5a7 100644 --- a/tests/TEMPLATE_ENV +++ b/tests/TEMPLATE_ENV @@ -4,9 +4,14 @@ LOG_LEVEL="INFO" CB_URL="http://localhost:1026" +ORION_LD_URL="http://localhost:1027" +IOTA_URL="http://localhost:4041" IOTA_JSON_URL="http://localhost:4041" IOTA_UL_URL="http://localhost:4061" QL_URL="http://localhost:8668" MQTT_BROKER_URL="mqtt://localhost:1883" +MQTT_BROKER_URL_INTERNAL="mqtt://localhost:1883" +LD_MQTT_BROKER_URL="mqtt://localhost:1884" +LD_MQTT_BROKER_URL_INTERNAL="mqtt://localhost:1884" FIWARE_SERVICE="filip" FIWARE_SERVICEPATH="/testing" \ No newline at end of file diff --git a/tests/clients/test_ngsi_ld_cb.py b/tests/clients/test_ngsi_ld_cb.py new file mode 100644 index 00000000..c88a6ed4 --- /dev/null +++ b/tests/clients/test_ngsi_ld_cb.py @@ -0,0 +1,739 @@ +""" +Tests for filip.cb.client +""" +import unittest +import logging +import pyld +from requests import RequestException, Session +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry +from filip.clients.ngsi_ld.cb import ContextBrokerLDClient +from filip.models.base import FiwareLDHeader, core_context +from filip.models.ngsi_ld.context import ActionTypeLD, ContextLDEntity, ContextProperty, \ + NamedContextProperty +from tests.config import settings +import requests +from filip.utils.cleanup import clear_context_broker_ld + + +# Setting up logging +logging.basicConfig( + level='ERROR', + format='%(asctime)s %(name)s %(levelname)s: %(message)s') + + +class TestContextBroker(unittest.TestCase): + """ + Test class for ContextBrokerClient + """ + def setUp(self) -> None: + """ + Setup test data + Returns: + None + """ + self.resources = { + "entities_url": "/ngsi-ld/v1/entities", + "types_url": "/ngsi-ld/v1/types" + } + self.attr = { + 'testtemperature': { + 'type': 'Property', + 'value': 20.0} + } + self.entity = ContextLDEntity(id='urn:ngsi-ld:my:id4', type='MyType', **self.attr) + self.entity_2 = ContextLDEntity(id="urn:ngsi-ld:room2", type="room") + self.fiware_header = FiwareLDHeader(ngsild_tenant=settings.FIWARE_SERVICE) + # Set up retry strategy + session = Session() + retry_strategy = Retry( + total=5, # Maximum number of retries + backoff_factor=1, # Exponential backoff (1, 2, 4, 8, etc.) + status_forcelist=[429, 500, 502, 503, 504], # Retry on these HTTP status codes + ) + # Set the HTTP adapter with retry strategy + adapter = HTTPAdapter(max_retries=retry_strategy) + session.mount("https://", adapter) + session.mount("http://", adapter) + self.client = ContextBrokerLDClient(fiware_header=self.fiware_header, + session=session, + url=settings.LD_CB_URL) + clear_context_broker_ld(cb_ld_client=self.client) + + def tearDown(self) -> None: + """ + Cleanup test server + """ + clear_context_broker_ld(cb_ld_client=self.client) + self.client.close() + + + @unittest.skip("Only for local testing environment") + def test_not_existing_tenant(self): + """ + Test the expected behavior of the client when the tenant does not exist + This test will not be included in the CI/CD pipeline. For local testing please + comment out the decorator. + """ + # create uuid for the tenant + import uuid + tenant = str(uuid.uuid4()).split('-')[0] + fiware_header = FiwareLDHeader(ngsild_tenant=tenant) + client = ContextBrokerLDClient(fiware_header=fiware_header, + url=settings.LD_CB_URL) + entities = client.get_entity_list() + self.assertEqual(len(entities), 0) + + + def test_get_entities_pagination(self): + """ + Test pagination of get entities + """ + init_numb = 2000 + entities_a = [ContextLDEntity(id=f"urn:ngsi-ld:test:{str(i)}", + type=f'filip:object:TypeA') for i in + range(0, init_numb)] + + self.client.entity_batch_operation(action_type=ActionTypeLD.CREATE, + entities=entities_a) + + entity_list = self.client.get_entity_list(limit=1) + self.assertEqual(len(entity_list),1) + + entity_list = self.client.get_entity_list(limit=400) + self.assertEqual(len(entity_list),400) + + entity_list = self.client.get_entity_list(limit=800) + self.assertEqual(len(entity_list),800) + + entity_list = self.client.get_entity_list(limit=1000) + self.assertEqual(len(entity_list),1000) + + # currently, there is a limit of 1000 entities per delete request + self.client.entity_batch_operation(action_type=ActionTypeLD.DELETE, + entities=entities_a[0:800]) + self.client.entity_batch_operation(action_type=ActionTypeLD.DELETE, + entities=entities_a[800:1600]) + entity_list = self.client.get_entity_list(limit=1000) + self.assertEqual(len(entity_list), init_numb - 1600) + + def test_get_entites(self): + """ + Retrieve a set of entities which matches a specific query from an NGSI-LD system + Args: + - id(string): Comma separated list of URIs to be retrieved + - idPattern(string): Regular expression that must be matched by Entity ids + - type(string): Comma separated list of Entity type names to be retrieved + - attrs(string): Comma separated list of attribute names (properties or relationships) to be retrieved + - q(string): Query + - georel: Geo-relationship + - geometry(string): Geometry; Available values : Point, MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon + - coordinates: Coordinates serialized as a string + - geoproperty(string): The name of the property that contains the geo-spatial data that will be used to resolve the geoquery + - csf(string): Context Source Filter + - limit(integer): Pagination limit + - options(string): Options dictionary; Available values : keyValues, sysAttrs + """ + entity_list = self.client.get_entity_list() + self.assertEqual(len(entity_list), 0) + + self.client.post_entity(entity=self.entity) + entity_list_idpattern = self.client.get_entity_list( + id_pattern="urn:ngsi-ld:my*") + self.assertEqual(len(entity_list_idpattern), 1) + self.assertEqual(entity_list_idpattern[0].id, self.entity.id) + + entity_list_attrs = self.client.get_entity_list(attrs=["testtemperature"]) + self.assertEqual(len(entity_list_attrs), 1) + self.assertEqual(entity_list_attrs[0].id, self.entity.id) + + def test_post_entity(self): + """ + Post an entity. + Args: + - Entity{ + @context: LdContext{} + location: GeoProperty{} + observationSpace: GeoProperty{} + operationSpace: GeoProperty{} + id: string($uri) required + type: Name(string) required + (NGSI-LD Name) + createdAt: string($date-time) + modifiedAt: string($date_time) + <*>: Property{} + Relationship{} + GeoProperty{} + } + Returns: + - (201) Created. Contains the resource URI of the created Entity + - (400) Bad request. + - (409) Already exists. + - (422) Unprocessable Entity. + Tests: + - Post an entity -> Does it return 201? + - Post an entity again -> Does it return 409? + - Post an entity without requires args -> Does it return 422? + """ + # create entity + self.client.post_entity(entity=self.entity) + entity_list = self.client.get_entity_list(entity_type=self.entity.type) + self.assertEqual(len(entity_list), 1) + self.assertEqual(entity_list[0].id, self.entity.id) + self.assertEqual(entity_list[0].type, self.entity.type) + self.assertEqual(entity_list[0].testtemperature.value, + self.entity.testtemperature.value) + + # existed entity + self.entity_identical = self.entity.model_copy() + with self.assertRaises(requests.exceptions.HTTPError) as contextmanager: + self.client.post_entity(entity=self.entity_identical) + response = contextmanager.exception.response + self.assertEqual(response.status_code, 409) + + entity_list = self.client.get_entity_list( + entity_type=self.entity_identical.type) + self.assertEqual(len(entity_list), 1) + + # append new attribute to existed entity + self.entity_append = self.entity.model_copy() + self.entity_append.delete_properties(['testtemperature']) + self.entity_append.add_properties( + {'humidity': ContextProperty(**{ + 'type': 'Property', + 'value': 50})}) + self.client.post_entity(entity=self.entity_append, append=True) + entity_append_res = self.client.get_entity(entity_id=self.entity_append.id) + self.assertEqual(entity_append_res.humidity.value, + self.entity_append.humidity.value) + self.assertEqual(entity_append_res.testtemperature.value, + self.entity.testtemperature.value) + + # override existed entity + new_attr = {'newattr': + {'type': 'Property', 'value': 999} + } + self.entity_override = ContextLDEntity( + id=self.entity.id, type=self.entity.type, **new_attr) + self.client.post_entity(entity=self.entity_override, update=True) + entity_override_res = self.client.get_entity(entity_id=self.entity.id) + self.assertEqual(entity_override_res.newattr.value, + self.entity_override.newattr.value) + self.assertNotIn('testtemperature', entity_override_res.model_dump()) + + # post without entity type is not allowed + with self.assertRaises(Exception): + self.client.post_entity(ContextLDEntity(id="room2")) + entity_list = self.client.get_entity_list() + self.assertNotIn("room2", entity_list) + + """delete""" + self.client.entity_batch_operation(entities=entity_list, + action_type=ActionTypeLD.DELETE) + + def test_get_entity(self): + """ + Get an entity with an specific ID. + Args: + - entityID(string): Entity ID, required + - attrs(string): Comma separated list of attribute names (properties or relationships) to be retrieved + - type(string): Entity Type + - options(string): Options dictionary; Available values : keyValues, sysAttrs + Returns: + - (200) Entity + - (400) Bad request + - (404) Not found + Tests for get entity: + - Post entity and see if get entity with the same ID returns the entity + with the correct values + - Get entity with an ID that does not exit. See if Not found error is + raised + """ + + """ + Test 1: + post entity_1 with entity_1_ID + get enntity_1 with enity_1_ID + compare if the posted entity_1 is the same as the get_enity_1 + If attributes posted entity.id != ID get entity: + Raise Error + If type posted entity != type get entity: + Raise Error + Test 2: + get enitity with enitity_ID that does not exit + If return != 404: + Raise Error + """ + """Test1""" + self.client.post_entity(entity=self.entity) + ret_entity = self.client.get_entity(entity_id=self.entity.id) + ret_entity_with_type = self.client.get_entity(entity_id=self.entity.id, + entity_type=self.entity.type) + ret_entity_keyValues = self.client.get_entity(entity_id=self.entity.id, + options="keyValues") + ret_entity_sysAttrs = self.client.get_entity(entity_id=self.entity.id, + options="sysAttrs") + + self.assertEqual(ret_entity.id, self.entity.id) + self.assertEqual(ret_entity.type, self.entity.type) + self.assertEqual(ret_entity_with_type.id, self.entity.id) + self.assertEqual(ret_entity_with_type.type, self.entity.type) + self.assertEqual(ret_entity_keyValues.id, self.entity.id) + self.assertEqual(ret_entity_keyValues.type, self.entity.type) + self.assertEqual(ret_entity_sysAttrs.id, self.entity.id) + self.assertEqual(ret_entity_sysAttrs.type, self.entity.type) + self.assertNotEqual(ret_entity_sysAttrs.createdAt, None) + + """Test2""" + with self.assertRaises(requests.exceptions.HTTPError) as contextmanager: + self.client.get_entity("urn:roomDoesnotExist") + response = contextmanager.exception.response + self.assertEqual(response.status_code, 404) + + with self.assertRaises(requests.exceptions.HTTPError) as contextmanager: + self.client.get_entity("roomDoesnotExist") + response = contextmanager.exception.response + self.assertEqual(response.status_code, 400) + + def test_different_context(self): + """ + Get entities with different contexts. + Returns: + """ + temperature_sensor_dict = { + "id": "urn:ngsi-ld:temperatureSensor", + "type": "TemperatureSensor", + "temperature": { + "type": "Property", + "value": 23, + "unitCode": "CEL" + } + } + + # client with custom context + custom_context = "https://n5geh.github.io/n5geh.test-context.io/context_saref.jsonld" + custom_header = FiwareLDHeader( + ngsild_tenant=settings.FIWARE_SERVICE, + ) + custom_header.set_context(custom_context) + client_custom_context = ContextBrokerLDClient( + fiware_header=custom_header, + url=settings.LD_CB_URL) + + # default context + temperature_sensor = ContextLDEntity(**temperature_sensor_dict) + self.client.post_entity(entity=temperature_sensor) + entity_default = self.client.get_entity(entity_id=temperature_sensor.id) + self.assertEqual(entity_default.context, + core_context) + self.assertEqual(entity_default.model_dump(exclude_unset=True, + exclude={"context"}), + temperature_sensor_dict) + entity_custom_context = client_custom_context.get_entity( + entity_id=temperature_sensor.id) + self.assertEqual(entity_custom_context.context, + custom_context) + self.assertEqual(entity_custom_context.model_dump(exclude_unset=True, + exclude={"context"}), + temperature_sensor_dict) + self.client.delete_entity_by_id(entity_id=temperature_sensor.id) + + # custom context in client + temperature_sensor = ContextLDEntity(**temperature_sensor_dict) + client_custom_context.post_entity(entity=temperature_sensor) + entity_custom = client_custom_context.get_entity(entity_id=temperature_sensor.id) + self.assertEqual(entity_custom.context, + custom_context) + self.assertEqual(entity_custom.model_dump(exclude_unset=True, + exclude={"context"}), + temperature_sensor_dict) + entity_default_context = self.client.get_entity(entity_id=temperature_sensor.id) + self.assertEqual(entity_default_context.context, + core_context) + self.assertNotEqual( + entity_default_context.model_dump(exclude_unset=True, + exclude={"context"}), + temperature_sensor_dict) + client_custom_context.delete_entity_by_id(entity_id=temperature_sensor.id) + + # custom context in entity + temperature_sensor = ContextLDEntity( + context=["https://n5geh.github.io/n5geh.test-context.io/context_saref.jsonld"], + **temperature_sensor_dict) + self.client.post_entity(entity=temperature_sensor) + entity_custom = client_custom_context.get_entity(entity_id=temperature_sensor.id) + self.assertEqual(entity_custom.context, + custom_context) + self.assertEqual(entity_custom.model_dump(exclude_unset=True, + exclude={"context"}), + temperature_sensor_dict) + entity_default_context = self.client.get_entity(entity_id=temperature_sensor.id) + self.assertEqual(entity_default_context.context, + core_context) + self.assertNotEqual( + entity_default_context.model_dump(exclude_unset=True, + exclude={"context"}), + temperature_sensor_dict) + self.client.delete_entity_by_id(entity_id=temperature_sensor.id) + + def test_delete_entity(self): + """ + Removes an specific Entity from an NGSI-LD system. + Args: + - entityID(string): Entity ID; required + - type(string): Entity Type + Returns: + - (204) No Content. The entity was removed successfully. + - (400) Bad request. + - (404) Not found. + Tests: + - Try to delete an non existent entity -> Does it return a Not found? + - Post an entity and try to delete the entity -> Does it return 204? + - Try to get to delete an deleted entity -> Does it return 404? + """ + + """ + Test 1: + delete entity with non existent entity_ID + If return != 404: + Raise Error + + Test 2: + post an entity with entity_ID and entity_type + delete entity with entity_ID + get entity list + If entity with entity_ID in entity list: + Raise Error + + Test 3: + delete entity with entity_ID + return != 404 ? + yes: + Raise Error + """ + + """Test1""" + # try to delete nonexistent entity + with self.assertRaises(requests.exceptions.HTTPError) as contextmanager: + self.client.get_entity(entity_id=self.entity.id) + response = contextmanager.exception.response + self.assertEqual(response.status_code, 404) + self.assertEqual(response.json()["title"], "Entity Not Found") + + """Test2""" + self.client.post_entity(entity=self.entity) + self.client.post_entity(entity=self.entity_2) + entity_list = self.client.get_entity_list() + entity_ids = [entity.id for entity in entity_list] + self.assertIn(self.entity.id, entity_ids) + + self.client.delete_entity_by_id(entity_id=self.entity.id) + entity_list = self.client.get_entity_list() + entity_ids = [entity.id for entity in entity_list] + self.assertNotIn(self.entity.id, entity_ids) + self.assertIn(self.entity_2.id, entity_ids) + + """Test3""" + # entity was already deleted + with self.assertRaises(requests.exceptions.HTTPError) as contextmanager: + self.client.get_entity(entity_id=self.entity.id) + response = contextmanager.exception.response + self.assertEqual(response.status_code, 404) + self.assertEqual(response.json()["title"], "Entity Not Found") + + def test_add_attributes_entity(self): + """ + Append new Entity attributes to an existing Entity within an NGSI-LD system. + Args: + - entityID(string): Entity ID; required + - options(string): Indicates that no attribute overwrite shall be performed. + Available values: noOverwrite + Returns: + - (204) No Content + - (207) Partial Success. Only the attributes included in the response payload were successfully appended. + - (400) Bad Request + - (404) Not Found + Tests: + - Post an entity and add an attribute. Test if the attribute is added when Get is done. + - Try to add an attribute to an non existent entity -> Return 404 + - Try to overwrite an attribute even though noOverwrite option is used + """ + """ + Test 1: + post an entity with entity_ID and entity_type + add attribute to the entity with entity_ID + get entity with entity_ID and new attribute + Is new attribute not added to enitity ? + yes: + Raise Error + Test 2: + add attribute to an non existent entity + Raise Error + Test 3: + post an entity with entity_ID, entity_type, entity_attribute + add attribute that already exists with noOverwrite + Raise Error + get entity and compare previous with entity attributes + If attributes are different? + Raise Error + """ + """Test 1""" + self.client.post_entity(self.entity) + attr = ContextProperty(**{'value': 20, 'unitCode': 'Number'}) + + self.entity.add_properties({"test_value": attr}) + self.client.append_entity_attributes(self.entity) + + entity = self.client.get_entity(entity_id=self.entity.id) + self.assertEqual(first=entity.test_value.value, second=attr.value) + self.client.delete_entity_by_id(entity_id=entity.id) + + """Test 2""" + attr = ContextProperty(**{'value': 20, 'type': 'Property'}) + with self.assertRaises(Exception): + self.entity.add_properties({"test_value": attr}) + self.client.append_entity_attributes(self.entity) + + """Test 3""" + self.client.post_entity(self.entity) + # What makes an property/ attribute unique ??? + attr = ContextProperty(**{'value': 20, 'type': 'Property'}) + attr_same = ContextProperty(**{'value': 40, 'type': 'Property'}) + + self.entity.add_properties({"test_value": attr}) + self.client.append_entity_attributes(self.entity) + self.entity.add_properties({"test_value": attr_same}) + # noOverwrite will raise 400, because all attributes exist already. + with self.assertRaises(RequestException): + self.client.append_entity_attributes(self.entity, options="noOverwrite") + entity = self.client.get_entity(entity_id=self.entity.id) + self.assertEqual(first=entity.test_value.value, second=attr.value) + self.assertNotEqual(first=entity.test_value.value, second=attr_same.value) + + def test_patch_entity_attrs(self): + """ + Update existing Entity attributes within an NGSI-LD system + Args: + - entityId(string): Entity ID; required + - Request body; required + Returns: + - (201) Created. Contains the resource URI of the created Entity + - (400) Bad request + - (409) Already exists + - (422) Unprocessable Entity + Tests: + - Post an enitity with specific attributes. Change the attributes with patch. + """ + """ + Test 1: + post an enitity with entity_ID and entity_type and attributes + patch one of the attributes with entity_id by sending request body + get entity list + If new attribute is not added to the entity? + Raise Error + """ + """Test1""" + new_prop = {'new_prop': ContextProperty(value=25)} + newer_prop = NamedContextProperty(value=40, name='new_prop') + + self.entity.add_properties(new_prop) + self.client.post_entity(entity=self.entity) + self.client.update_entity_attribute(entity_id=self.entity.id, attr=newer_prop, + attr_name='new_prop') + entity = self.client.get_entity(entity_id=self.entity.id, options="keyValues") + prop_dict = entity.model_dump() + self.assertIn("new_prop", prop_dict) + self.assertEqual(prop_dict["new_prop"], 40) + + def test_patch_entity_attrs_contextprop(self): + """ + Update existing Entity attributes within an NGSI-LD system + Args: + - entityId(string): Entity ID; required + - Request body; required + Returns: + - (201) Created. Contains the resource URI of the created Entity + - (400) Bad request + - (409) Already exists + - (422) Unprocessable Entity + Tests: + - Post an enitity with specific attributes. Change the attributes with patch. + """ + """ + Test 1: + post an enitity with entity_ID and entity_type and attributes + patch one of the attributes with entity_id by sending request body + get entity list + If new attribute is not added to the entity? + Raise Error + """ + """Test1""" + new_prop = {'new_prop': ContextProperty(value=25)} + newer_prop = ContextProperty(value=55) + + self.entity.add_properties(new_prop) + self.client.post_entity(entity=self.entity) + self.client.update_entity_attribute(entity_id=self.entity.id, attr=newer_prop, + attr_name='new_prop') + entity = self.client.get_entity(entity_id=self.entity.id, options="keyValues") + prop_dict = entity.model_dump() + self.assertIn("new_prop", prop_dict) + self.assertEqual(prop_dict["new_prop"], 55) + + def test_patch_entity_attrs_attrId(self): + """ + Update existing Entity attribute ID within an NGSI-LD system + Args: + - entityId(string): Entity Id; required + - attrId(string): Attribute Id; required + Returns: + - (204) No Content + - (400) Bad Request + - (404) Not Found + Tests: + - Post an enitity with specific attributes. Change the attributes with patch. + """ + """ + Test 1: + post an entity with entity_ID, entity_type and attributes + patch with entity_ID and attribute_ID + return != 204: + yes: + Raise Error + """ + """Test 1""" + attr = NamedContextProperty(name="test_value", + value=20) + self.entity.add_properties(attrs=[attr]) + self.client.post_entity(entity=self.entity) + + attr.value = 40 + self.client.update_entity_attribute(entity_id=self.entity.id, attr=attr, + attr_name="test_value") + entity = self.client.get_entity(entity_id=self.entity.id, options="keyValues") + prop_dict = entity.model_dump() + self.assertIn("test_value", prop_dict) + self.assertEqual(prop_dict["test_value"], 40) + + def test_delete_entity_attribute(self): + """ + Delete existing Entity atrribute within an NGSI-LD system. + Args: + - entityId: Entity Id; required + - attrId: Attribute Id; required + Returns: + - (204) No Content + - (400) Bad Request + - (404) Not Found + Tests: + - Post an entity with attributes. Try to delete non existent attribute with non existent attribute + id. Then check response code. + - Post an entity with attributes. Try to delete one the attributes. Test if the attribute is really + removed by either posting the entity or by trying to delete it again. + """ + """ + Test 1: + post an enitity with entity_ID, entity_type and attribute with attribute_ID + delete an attribute with an non existent attribute_ID of the entity with the entity_ID + Raise Error + Test 2: + post an entity with entity_ID, entitiy_name and attribute with attribute_ID + delete the attribute with the attribute_ID of the entity with the entity_ID + get entity with entity_ID + If attribute with attribute_ID is still there? + Raise Error + delete the attribute with the attribute_ID of the entity with the entity_ID + Raise Error + """ + """Test 1""" + + attr = NamedContextProperty(name="test_value", + value=20) + self.entity.add_properties(attrs=[attr]) + self.client.post_entity(entity=self.entity) + with self.assertRaises(Exception): + self.client.delete_attribute(entity_id=self.entity.id, + attribute_id="does_not_exist") + + entity_list = self.client.get_entity_list() + + for entity in entity_list: + self.client.delete_entity_by_id(entity_id=entity.id) + + """Test 2""" + attr = NamedContextProperty(name="test_value", + value=20) + self.entity.add_properties(attrs=[attr]) + self.client.post_entity(entity=self.entity) + self.client.delete_attribute(entity_id=self.entity.id, + attribute_id="test_value") + + with self.assertRaises(requests.exceptions.HTTPError) as contextmanager: + self.client.delete_attribute(entity_id=self.entity.id, + attribute_id="test_value") + response = contextmanager.exception.response + self.assertEqual(response.status_code, 404) + + def test_replacing_attributes(self): + """ + Patch existing Entity attributes within an NGSI-LD system. + Args: + - entityId: Entity Id; required + Returns: + - (204) No Content + - (400) Bad Request + - (404) Not Found + Tests: + - Post an entity with attribute. Change the attributes with patch. + """ + """ + Test 1: + replace attribute with same name and different value + Test 2: + replace two attributes + """ + + """Test 1""" + attr1 = NamedContextProperty(name="test_value", value=20) + self.entity.add_properties(attrs=[attr1]) + self.client.post_entity(entity=self.entity) + entity = self.client.get_entity(entity_id=self.entity.id, options="keyValues") + prop_dict = entity.model_dump() + self.assertIn("test_value", prop_dict) + self.assertEqual(prop_dict["test_value"], 20) + + attr2 = NamedContextProperty(name="test_value", value=44) + self.entity.delete_properties(props=[attr1]) + self.entity.add_properties(attrs=[attr2]) + self.client.replace_existing_attributes_of_entity(entity=self.entity) + entity = self.client.get_entity(entity_id=self.entity.id, options="keyValues") + prop_dict = entity.model_dump() + self.assertIn("test_value", prop_dict) + self.assertEqual(prop_dict["test_value"], 44) + + self.client.delete_entity_by_id(entity_id=self.entity.id) + + """Test 2""" + attr1 = NamedContextProperty(name="test_value", value=20) + attr2 = NamedContextProperty(name="my_value", value=44) + self.entity.add_properties(attrs=[attr1, attr2]) + self.client.post_entity(entity=self.entity) + entity = self.client.get_entity(entity_id=self.entity.id, options="keyValues") + prop_dict = entity.model_dump() + self.assertIn("test_value", prop_dict) + self.assertEqual(prop_dict["test_value"], 20) + self.assertIn("my_value", prop_dict) + self.assertEqual(prop_dict["my_value"], 44) + + self.entity.delete_properties(props=[attr1]) + self.entity.delete_properties(props=[attr2]) + attr3 = NamedContextProperty(name="test_value", value=25) + attr4 = NamedContextProperty(name="my_value", value=45) + self.entity.add_properties(attrs=[attr3, attr4]) + self.client.replace_existing_attributes_of_entity(entity=self.entity) + entity = self.client.get_entity(entity_id=self.entity.id, options="keyValues") + prop_dict = entity.model_dump() + self.assertIn("test_value", prop_dict) + self.assertEqual(prop_dict["test_value"], 25) + self.assertIn("my_value", prop_dict) + self.assertEqual(prop_dict["my_value"], 45) diff --git a/tests/clients/test_ngsi_ld_entity_batch_operation.py b/tests/clients/test_ngsi_ld_entity_batch_operation.py new file mode 100644 index 00000000..da1397c2 --- /dev/null +++ b/tests/clients/test_ngsi_ld_entity_batch_operation.py @@ -0,0 +1,376 @@ +from random import Random +import unittest +from requests.exceptions import HTTPError +from requests import RequestException +from pydantic import ValidationError + +from filip.models.base import FiwareLDHeader +# FiwareLDHeader issue with pydantic +from filip.clients.ngsi_ld.cb import ContextBrokerLDClient +from filip.models.ngsi_ld.context import ContextLDEntity, ActionTypeLD +from tests.config import settings +from filip.utils.cleanup import clear_context_broker_ld + + +class EntitiesBatchOperations(unittest.TestCase): + """ + Test class for entity endpoints. + Args: + unittest (_type_): _description_ + """ + + def setUp(self) -> None: + """ + Setup test data + Returns: + None + """ + self.r = Random() + self.fiware_header = FiwareLDHeader(ngsild_tenant=settings.FIWARE_SERVICE) + self.cb_client = ContextBrokerLDClient(fiware_header=self.fiware_header, + url=settings.LD_CB_URL) + clear_context_broker_ld(cb_ld_client=self.cb_client) + + def tearDown(self) -> None: + """ + Cleanup test server + """ + clear_context_broker_ld(cb_ld_client=self.cb_client) + self.cb_client.close() + + def test_entity_batch_operations_create(self) -> None: + """ + Batch Entity creation. + Args: + - Request body(Entity List); required + Returns: + - (200) Success + - (400) Bad Request + Tests: + - Post the creation of batch entities. Check if each of the created entities exists and if all attributes exist. + """ + """ + Test 1: + post create batch entity + get entity list + for all elements in entity list: + if entity list element != batch entity element: + Raise Error + Test 2: + post create batch entity with two entities that have the same id + post in try block + no exception raised + check if the entities list only contains one element (No duplicates) + if not raise assert + """ + """Test 1""" + entities_a = [ContextLDEntity(id=f"urn:ngsi-ld:test:{str(i)}", + type=f'filip:object:test') for i in + range(0, 10)] + self.cb_client.entity_batch_operation(entities=entities_a, action_type=ActionTypeLD.CREATE) + entity_list = self.cb_client.get_entity_list(entity_type=f'filip:object:test') + id_list = [entity.id for entity in entity_list] + self.assertEqual(len(entities_a), len(entity_list)) + for entity in entities_a: + self.assertIsInstance(entity, ContextLDEntity) + self.assertIn(entity.id, id_list) + for entity in entity_list: + self.cb_client.delete_entity_by_id(entity_id=entity.id) + + """Test 2""" + entities_b = [ContextLDEntity(id=f"urn:ngsi-ld:test:eins", + type=f'filip:object:test'), + ContextLDEntity(id=f"urn:ngsi-ld:test:eins", + type=f'filip:object:test')] + entity_list_b = [] + try: + self.cb_client.entity_batch_operation(entities=entities_b, action_type=ActionTypeLD.CREATE) + entity_list_b = self.cb_client.get_entity_list( + entity_type=f'filip:object:test') + self.assertEqual(len(entity_list), 1) + except: + pass + finally: + for entity in entity_list_b: + self.cb_client.delete_entity_by_id(entity_id=entity.id) + + def test_entity_batch_operations_update(self) -> None: + """ + Batch Entity update. + Args: + - options(string): Available values: noOverwrite + - Request body(EntityList); required + Returns: + - (200) Success + - (400) Bad Request + Tests: + - Post the update of batch entities. Check if each of the updated entities exists and if the updates appear. + - Try the same with the noOverwrite statement and check if the nooverwrite is acknowledged. + """ + """ + Test 1: + post create entity batches + post update of batch entity + get entities + for all entities in entity list: + if entity list element != updated batch entity element: + Raise Error + Test 2: + post create entity batches + post update of batch entity with no overwrite + get entities + for all entities in entity list: + if entity list element != updated batch entity element but not the existings are overwritten: + Raise Error + + """ + """Test 1""" + entities_a = [ContextLDEntity(id=f"urn:ngsi-ld:test:{str(i)}", + type=f'filip:object:test', + **{'temperature': { + 'value': self.r.randint(20,50), + "type": "Property" + }}) for i in + range(0, 5)] + + self.cb_client.entity_batch_operation(entities=entities_a, action_type=ActionTypeLD.CREATE) + + entities_update = [ContextLDEntity(id=f"urn:ngsi-ld:test:{str(i)}", + type=f'filip:object:test', + **{'temperature': + {'value': self.r.randint(0,20), + "type": "Property" + }}) for i in + range(3, 6)] + self.cb_client.entity_batch_operation(entities=entities_update, action_type=ActionTypeLD.UPDATE) + entity_list = self.cb_client.get_entity_list(entity_type=f'filip:object:test') + self.assertEqual(len(entity_list),5) + updated = [x.model_dump(exclude_unset=True, exclude={"context"}) + for x in entity_list if int(x.id.split(':')[3]) in range(3,5)] + nupdated = [x.model_dump(exclude_unset=True, exclude={"context"}) + for x in entity_list if int(x.id.split(':')[3]) in range(0,3)] + + self.assertCountEqual([entity.model_dump(exclude_unset=True) + for entity in entities_a[0:3]], + nupdated) + + self.assertCountEqual([entity.model_dump(exclude_unset=True) + for entity in entities_update[0:2]], + updated) + + """Test 2""" + # presssure will be appended while the existing temperature will + # not be overwritten + entities_update = [ContextLDEntity(id=f"urn:ngsi-ld:test:{str(i)}", + type=f'filip:object:test', + **{'temperature': + {'value': self.r.randint(50, 100), + "type": "Property"}, + 'pressure': { + 'value': self.r.randint(1,100), + "type": "Property" + } + }) for i in range(0, 5)] + + self.cb_client.entity_batch_operation(entities=entities_update, + action_type=ActionTypeLD.UPDATE, + options="noOverwrite") + + previous = entity_list + previous.sort(key=lambda x: int(x.id.split(':')[3])) + + entity_list = self.cb_client.get_entity_list(entity_type=f'filip:object:test') + entity_list.sort(key=lambda x: int(x.id.split(':')[3])) + + self.assertEqual(len(entity_list),len(entities_update)) + + for (updated,entity,prev) in zip(entities_update,entity_list,previous): + self.assertEqual(updated.model_dump().get('pressure'), + entity.model_dump().get('pressure')) + self.assertNotEqual(updated.model_dump().get('temperature'), + entity.model_dump().get('temperature')) + self.assertEqual(prev.model_dump().get('temperature'), + entity.model_dump().get('temperature')) + + with self.assertRaises(HTTPError): + self.cb_client.entity_batch_operation(entities=[],action_type=ActionTypeLD.UPDATE) + + # according to spec, this should raise bad request data, + # but pydantic is intercepting + with self.assertRaises(ValidationError): + self.cb_client.entity_batch_operation(entities=[None],action_type=ActionTypeLD.UPDATE) + + for entity in entity_list: + self.cb_client.delete_entity_by_id(entity_id=entity.id) + + def test_entity_batch_operations_upsert(self) -> None: + """ + Batch Entity upsert. + Args: + - options(string): Available values: replace, update + - Request body(EntityList); required + Returns: + - (200) Success + - (400) Bad request + Tests: + - Post entity list and then post the upsert with update and replace. + - Get the entitiy list and see if the results are correct. + + """ + """ + Test 1: + post a create entity batch b0 with attr a0 + post entity upsert batch b1 with attr a1 with update, b0 ∩ b1 != ∅ + post entity upsert batch b2 with attr a1 with replace, b0 ∩ b2 != ∅ && b1 ∩ b2 == ∅ + get entity list + for e in entity list: + if e in b0 ∩ b1: + e should contain a1 and a0 + if e in b0 ∩ b2: + e should contain only a1 + else: + e should contain only a0 + """ + """Test 1""" + # create entities 1 -3 + entities_a = [ContextLDEntity(id=f"urn:ngsi-ld:test:{str(i)}", + type=f'filip:object:test', + **{'temperature': + {'value': self.r.randint(0,20), + "type": "Property" + }}) for i in + range(1, 4)] + self.cb_client.entity_batch_operation(entities=entities_a, action_type=ActionTypeLD.CREATE) + + # replace entities 0 - 1 + entities_replace = [ContextLDEntity(id=f"urn:ngsi-ld:test:{str(i)}", + type=f'filip:object:test', + **{'pressure': + {'value': self.r.randint(50,100), + "type": "Property" + }}) for i in + range(0, 2)] + self.cb_client.entity_batch_operation(entities=entities_replace, action_type=ActionTypeLD.UPSERT, + options="replace") + + # update entities 3 - 4, + # pressure will be appended for 3 + # temperature will be appended for 4 + entities_update = [ContextLDEntity(id=f"urn:ngsi-ld:test:{str(i)}", + type=f'filip:object:test', + **{'pressure': + {'value': self.r.randint(50,100), + "type": "Property" + }}) for i in + range(3, 5)] + self.cb_client.entity_batch_operation(entities=entities_update, + action_type=ActionTypeLD.UPSERT, + options="update") + + # 0,1 and 4 should have pressure only + # 2 should have temperature only + # 3 should have both + # can be made modular for variable size batches + entity_list = self.cb_client.get_entity_list() + self.assertEqual(len(entity_list),5) + for _e in entity_list: + _id = int(_e.id.split(':')[3]) + e = _e.model_dump(exclude_unset=True, exclude={'context'}) + if _id in [0,1]: + self.assertIsNone(e.get('temperature',None)) + self.assertIsNotNone(e.get('pressure',None)) + self.assertCountEqual([e], + [x.model_dump(exclude_unset=True) + for x in entities_replace if x.id == _e.id]) + elif _id == 4: + self.assertIsNone(e.get('temperature',None)) + self.assertIsNotNone(e.get('pressure',None)) + self.assertCountEqual([e], + [x.model_dump(exclude_unset=True) + for x in entities_update if x.id == _e.id]) + elif _id == 2: + self.assertIsNone(e.get('pressure',None)) + self.assertIsNotNone(e.get('temperature',None)) + self.assertCountEqual([e], + [x.model_dump(exclude_unset=True) + for x in entities_a if x.id == _e.id]) + elif _id == 3: + self.assertIsNotNone(e.get('temperature',None)) + self.assertIsNotNone(e.get('pressure',None)) + self.assertCountEqual([e.get('temperature')], + [x.model_dump(exclude_unset=True).get('temperature') + for x in entities_a if x.id == _e.id]) + self.assertCountEqual([e.get('pressure')], + [x.model_dump(exclude_unset=True).get('pressure') + for x in entities_update if x.id == _e.id]) + + for entity in entity_list: + self.cb_client.delete_entity_by_id(entity_id=entity.id) + + def test_entity_batch_operations_delete(self) -> None: + """ + Batch entity delete. + Args: + - Request body(string list); required + Returns + - (200) Success + - (400) Bad request + Tests: + - Try to delete non existent entity. + - Try to delete existent entity and check if it is deleted. + """ + """ + Test 1: + delete batch entity that is non existent + if return != 400: + Raise Error + Test 2: + post batch entity + delete batch entity + if return != 200: + Raise Error + get entity list + if batch entities are still on entity list: + Raise Error: + """ + """Test 1""" + entities_delete = [ContextLDEntity(id=f"urn:ngsi-ld:test:{str(i)}", + type=f'filip:object:test') for i in + range(0, 1)] + with self.assertRaises(Exception): + self.cb_client.entity_batch_operation(entities=entities_delete, + action_type=ActionTypeLD.DELETE) + + """Test 2""" + entity_del_type = 'filip:object:test' + entities_ids_a = [f"urn:ngsi-ld:test:{str(i)}" for i in + range(0, 4)] + entities_a = [ContextLDEntity(id=id_a, + type=entity_del_type) for id_a in + entities_ids_a] + + self.cb_client.entity_batch_operation(entities=entities_a, action_type=ActionTypeLD.CREATE) + + entities_delete = [ContextLDEntity(id=id_a, + type=entity_del_type) for id_a in + entities_ids_a[:3]] + entities_delete_ids = [entity.id for entity in entities_delete] + + # send update to delete entities + self.cb_client.entity_batch_operation(entities=entities_delete, action_type=ActionTypeLD.DELETE) + + # get list of entities which is still stored + entity_list = self.cb_client.get_entity_list(entity_type=entity_del_type) + entity_ids = [entity.id for entity in entity_list] + + self.assertEqual(len(entity_list), 1) # all but one entity were deleted + + for entityId in entity_ids: + self.assertIn(entityId, entities_ids_a) + for entityId in entities_delete_ids: + self.assertNotIn(entityId, entity_ids) + for entity in entity_list: + self.cb_client.delete_entity_by_id(entity_id=entity.id) + + entity_list = self.cb_client.get_entity_list(entity_type=entity_del_type) + self.assertEqual(len(entity_list), 0) # all entities were deleted diff --git a/tests/clients/test_ngsi_ld_query.py b/tests/clients/test_ngsi_ld_query.py new file mode 100644 index 00000000..c8050485 --- /dev/null +++ b/tests/clients/test_ngsi_ld_query.py @@ -0,0 +1,313 @@ +""" +Tests for filip.cb.client +""" +import unittest +import logging +import re +import math +import time +from dateutil.parser import parse +from collections.abc import Iterable +from requests import RequestException +from filip.clients.ngsi_ld.cb import ContextBrokerLDClient +from filip.models.base import FiwareLDHeader +from filip.models.ngsi_ld.context import ActionTypeLD, ContextLDEntity, ContextProperty, \ + NamedContextProperty, NamedContextRelationship +from tests.config import settings +from random import Random +from filip.utils.cleanup import clear_context_broker_ld + + +# Setting up logging +logging.basicConfig( + level='ERROR', + format='%(asctime)s %(name)s %(levelname)s: %(message)s') + + +class TestLDQueryLanguage(unittest.TestCase): + """ + Test class for ContextBrokerClient + """ + def setUp(self) -> None: + """ + Setup test data + Returns: + None + """ + #Extra size parameters for modular testing + self.cars_nb = 500 + self.span = 3 + + #client parameters + self.fiware_header = FiwareLDHeader(ngsild_tenant=settings.FIWARE_SERVICE) + self.cb = ContextBrokerLDClient(fiware_header=self.fiware_header, + url=settings.LD_CB_URL) + + #Prep db + clear_context_broker_ld(cb_ld_client=self.cb) + + #base id + self.base='urn:ngsi-ld:' + + #Some entities for relationships + self.garage = ContextLDEntity(id=f"{self.base}garage0",type=f"garage") + self.cam = ContextLDEntity(id=f"{self.base}cam0",type=f"camera") + self.cb.post_entity(entity=self.garage) + self.cb.post_entity(entity=self.cam) + + #Entities to post/test on + self.cars = [ContextLDEntity(id=f"{self.base}car0{i}",type=f"{self.base}car") for i in range(0,self.cars_nb-1)] + + #Some dictionaries for randomizing properties + self.brands = ["Batmobile","DeLorean","Knight 2000"] + self.timestamps = ["2020-12-24T11:00:00Z","2020-12-24T12:00:00Z","2020-12-24T13:00:00Z"] + self.addresses = [ + { + "country": "Germany", + "street-address": { + "street":"Mathieustr.", + "number":10}, + "postal-code": 52072 + }, + { + "country": "USA", + "street-address": { + "street":"Goosetown Drive", + "number":810}, + "postal-code": 27320 + }, + { + "country": "Nigeria", + "street-address": { + "street":"Mustapha Street", + "number":46}, + "postal-code": 65931 + }, + ] + + #base properties/relationships + self.humidity = NamedContextProperty(name="humidity",value=1) + self.temperature = NamedContextProperty(name="temperature",value=0) + self.isParked = NamedContextRelationship(name="isParked",object="placeholder") + self.isMonitoredBy = NamedContextRelationship(name="isMonitoredBy",object="placeholder") + + #q Expressions to test + self.qs = [ + 'temperature > 0', + 'brand != "Batmobile"', + 'isParked | isMonitoredBy', + 'isParked == "urn:ngsi-ld:garage0"', + 'temperature < 60; isParked == "urn:ngsi-ld:garage0"', + '(temperature >= 59 | humidity < 3); brand == "DeLorean"', + '(isMonitoredBy; temperature<30) | isParked', + '(temperature > 30; temperature < 90)| humidity <= 5', + 'temperature.observedAt >= "2020-12-24T12:00:00Z"', + 'address[country] == "Germany"', + 'address[street-address.number] == 810', + 'address[street-address.number]', + 'address[street-address.extra]', + ] + + self.post() + + + def tearDown(self) -> None: + """ + Cleanup test server + """ + clear_context_broker_ld(cb_ld_client=self.cb) + self.cb.close() + + def test_ld_query_language(self): + #Itertools product actually interferes with test results here + for q in self.qs: + entities = self.cb.get_entity_list(q=q,limit=1000) + tokenized,keys_dict = self.extract_keys(q) + + #Replace logical ops with python ones + tokenized = tokenized.replace("|"," or ") + tokenized = tokenized.replace(";"," and ") + size = len([x for x in self.cars if self.search_predicate(x,tokenized,keys_dict)]) + #Check we get the same number of entities + self.assertEqual(size,len(entities),msg=q) + for e in entities: + copy = tokenized + for token,keylist in keys_dict.items(): + copy = self.sub_key_with_val(copy,e,keylist,token) + + #Check each obtained entity obeys the q expression + self.assertTrue(eval(copy),msg=q) + + def extract_keys(self,q:str): + ''' + Extract substring from string expression that is likely to be the name of a + property/relationship of a given entity + Returns: + str,dict + ''' + #Trim empty spaces + n=q.replace(" ","") + + #Find all literals that are not logical operators or parentheses -> keys/values + res = re.findall('[^<>=)()|;!]*', n) + keys = {} + i=0 + for r in res: + #Skip empty string from the regex search result + if len(r) == 0: + continue + + #Skip anything purely numeric -> Definitely a value + if r.isnumeric(): + continue + #Skip anything with a double quote -> string or date + if '"' in r: + try: + #replace date with unix ts + timestamp = r.replace("\"","") + date = parse(timestamp) + timestamp = str(time.mktime(date.timetuple())) + n = n.replace(r,timestamp) + except Exception as e: + r=f'\"{r}\"' + continue + + #Skip keys we already encountered + if [r] in keys.values(): + continue + + #Replace the key name with a custom token in the string + token=f'${i}' + n= n.replace(r,token) + i+=1 + + #Flatten composite keys by chaining them together + l = [] + #Composite of the form x[...] + if '[' in r: + idx_st = r.index('[') + idx_e = r.index(']') + outer_key = r[:idx_st] + l.append(outer_key) + inner_key = r[idx_st+1:idx_e] + + #Composite of the form x[y.z...] + if '.' in inner_key: + rest = inner_key.split('.') + #Composite of the form x[y] + else : + rest = [inner_key] + l+=rest + #Composite of the form x.y... + elif '.' in r: + l+=r.split('.') + #Simple key + else: + l=[r] + + #Finalize incomplete key presence check + idx_next = n.index(token)+len(token) + if idx_next>=len(n) or n[idx_next] not in ['>','<','=','!']: + n = n.replace(token,f'{token} != None') + + #Associate each chain of nested keys with the token it was replaced with + keys[token] = l + return n,keys + + def sub_key_with_val(self,q:str,entity:ContextLDEntity,keylist,token:str): + ''' + Substitute key names in q expression with corresponding entity property/ + relationship values. All while accounting for access of nested properties + Returns: + str + ''' + obj = entity.model_dump() + for key in keylist: + if key in obj: + obj = obj[key] + elif 'value' in obj and key in obj['value']: + obj = obj['value'][key] + else: + obj = None + break + + if isinstance(obj,Iterable): + if 'value' in obj: + obj=obj['value'] + elif 'object' in obj: + obj=obj['object'] + + if obj is not None and re.compile('[a-zA-Z]+').search(str(obj)) is not None: + try: + date = parse(obj) + obj = str(time.mktime(date.timetuple())) #convert to unix ts + except Exception as e: + obj = f'"{str(obj)}"' + + + #replace key names with entity values + n = q.replace(token,str(obj)) + return n + + def search_predicate(self,e,tokenized,keys_dict): + ''' + Search function to search our posted data for checks + This function is needed because , whereas the context broker will not return + an entity with no nested key if that key is given as a filter, our eval attempts + to compare None values using logical operators + ''' + copy = tokenized + for token,keylist in keys_dict.items(): + copy = self.sub_key_with_val(copy,e,keylist,token) + + try: + return eval(copy) + except: + return False + + def post(self): + ''' + Somewhat randomized generation of data. Can be made further random by + Choosing a bigger number of cars, and a more irregular number for remainder + Calculations (self.cars_nb & self.span) + Returns: + None + ''' + for i in range(len(self.cars)): + #Big number rnd generator + r = Random().randint(1,self.span) + tri_rnd = Random().randint(0,(10*self.span)**2) + r = math.trunc(tri_rnd/r) % self.span + r_2 = Random().randint(0,r) + + a=r_2*30 + b=a+30 + + #Every car will have temperature, humidity, brand and address + t = self.temperature.model_copy() + t.value = Random().randint(a,b) + t.observedAt = self.timestamps[r] + + h = self.humidity.model_copy() + h.value = Random().randint(math.trunc(a/10),math.trunc(b/10)) + + self.cars[i].add_properties([t,h,NamedContextProperty(name="brand",value=self.brands[r]), + NamedContextProperty(name="address",value=self.addresses[r])]) + + p = self.isParked.model_copy() + p.object = self.garage.id + + m = self.isMonitoredBy.model_copy() + m.object = self.cam.id + + #Every car is endowed with a set of relationships/nested key + if r==0: + self.cars[i].add_relationships([p]) + elif r==1: + self.cars[i].add_relationships([m]) + elif r==2: + self.cars[i].add_relationships([p,m]) + + #Post everything + self.cb.entity_batch_operation(action_type=ActionTypeLD.CREATE, + entities=self.cars) diff --git a/tests/clients/test_ngsi_ld_subscription.py b/tests/clients/test_ngsi_ld_subscription.py new file mode 100644 index 00000000..c097d1cb --- /dev/null +++ b/tests/clients/test_ngsi_ld_subscription.py @@ -0,0 +1,460 @@ +""" +Test the endpoint for subscription related task of NGSI-LD for ContextBrokerClient +""" +import json +import time +import urllib.parse +from unittest import TestCase +import threading +from paho.mqtt.enums import CallbackAPIVersion +import paho.mqtt.client as mqtt +from requests import RequestException +from filip.clients.ngsi_ld.cb import ContextBrokerLDClient +from filip.models.base import FiwareLDHeader +from filip.models.ngsi_ld.context import \ + NamedContextProperty, \ + ContextLDEntity, ActionTypeLD +from filip.models.ngsi_ld.subscriptions import \ + Endpoint, \ + NotificationParams, \ + SubscriptionLD +from tests.config import settings +from filip.utils.cleanup import clear_context_broker_ld + + +class TestSubscriptions(TestCase): + """ + Test class for context broker models + """ + + def setUp(self) -> None: + """ + Setup test data + Returns: + None + """ + self.fiware_header = FiwareLDHeader(ngsild_tenant=settings.FIWARE_SERVICE) + self.cb_client = ContextBrokerLDClient(fiware_header=self.fiware_header, + url=settings.LD_CB_URL) + clear_context_broker_ld(cb_ld_client=self.cb_client) + + self.mqtt_topic = ''.join([settings.FIWARE_SERVICE, + settings.FIWARE_SERVICEPATH]) + self.endpoint_mqtt = Endpoint(**{ + "uri": str(settings.LD_MQTT_BROKER_URL) + "/my/test/topic", + "accept": "application/json", + }) + self.endpoint_http = Endpoint(**{ + "uri": urllib.parse.urljoin(str(settings.LD_CB_URL), + "/ngsi-ld/v1/subscriptions"), + "accept": "application/json" + }) + + def tearDown(self) -> None: + clear_context_broker_ld(cb_ld_client=self.cb_client) + self.cb_client.close() + + def test_post_subscription_http(self): + """ + Create a new HTTP subscription. + Args: + - Request body: required + Returns: + - (201) successfully created subscription + Tests: + - Create a HTTP subscription and post it + """ + attr_id = "attr" + id = "urn:ngsi-ld:Subscription:" + "test_sub0" + notification_param = NotificationParams(attributes=[attr_id], endpoint=self.endpoint_http) + sub = SubscriptionLD(id=id, notification=notification_param, entities=[{"type": "Room"}]) + self.cb_client.post_subscription(sub) + sub_list = [x for x in self.cb_client.get_subscription_list() + if x.id == 'urn:ngsi-ld:Subscription:test_sub0'] + self.assertEqual(len(sub_list),1) + + def test_post_subscription_http_check_broker(self): + """ + Create a new HTTP subscription and check whether messages are received. + Args: + - Request body: required + Returns: + - (201) successfully created subscription + Tests: + - Create a subscription and post something from this subscription + to see if the subscribed broker gets the message. + - Create a subscription twice to one message and see if the message is + received twice or just once. + """ + pass + + def test_get_subscription(self): + """ + Returns the subscription if it exists. + Args: + - subscriptionId(string): required + Returns: + - (200) subscription or empty list if successful + - Error Code + Tests: + - Get Subscription and check if the subscription is the same as the one posted + """ + attr_id = "attr" + id = "urn:ngsi-ld:Subscription:" + "test_sub0" + notification_param = NotificationParams(attributes=[attr_id], endpoint=self.endpoint_http) + sub = SubscriptionLD(id=id, notification=notification_param, entities=[{"type": "Room"}]) + self.cb_client.post_subscription(sub) + sub_get = self.cb_client.get_subscription(subscription_id=id) + self.assertEqual(sub.entities, sub_get.entities) + self.assertEqual(sub.notification.attributes, sub_get.notification.attributes) + self.assertEqual(sub.notification.endpoint.uri, sub_get.notification.endpoint.uri) + + def test_get_subscription_list(self): + """ + Get a list of all current subscriptions the broker has subscribed to. + Args: + - limit(number($double)): Limits the number of subscriptions retrieved + Returns: + - (200) list of subscriptions + Tests for get subscription list: + - Create list of subscriptions and get the list of subscriptions -> compare the lists + - Set a limit for the subscription number and compare the count of subscriptions + """ + sub_post_list = list() + for i in range(10): + attr_id = "attr" + str(i) + id = "urn:ngsi-ld:Subscription:" + "test_sub" + str(i) + notification_param = NotificationParams(attributes=[attr_id], endpoint=self.endpoint_http) + sub = SubscriptionLD(id=id, notification=notification_param, entities=[{"type": "Room"}]) + sub_post_list.append(sub) + self.cb_client.post_subscription(sub) + + sub_list = self.cb_client.get_subscription_list() + sub_id_list = [sub.id for sub in sub_list] + + for sub in sub_post_list: + self.assertIn(sub.id, sub_id_list) + + sub_limit = 5 + sub_list2 = self.cb_client.get_subscription_list(limit=sub_limit) + self.assertEqual(len(sub_list2), sub_limit) + + def test_delete_subscription(self): + """ + Cancels subscription. + Args: + - subscriptionID(string): required + Returns: + - Successful: 204, no content + Tests: + - Post and delete subscription then get all subscriptions and check whether deleted subscription is still there. + """ + for i in range(10): + attr_id = "attr" + str(i) + notification_param = NotificationParams( + attributes=[attr_id], endpoint=self.endpoint_http) + id = "urn:ngsi-ld:Subscription:" + "test_sub" + str(i) + sub = SubscriptionLD(id=id, + notification=notification_param, + entities=[{"type": "Room"}] + ) + + if i == 0: + del_sub = sub + del_id = id + self.cb_client.post_subscription(sub) + + sub_list = self.cb_client.get_subscription_list(limit=10) + sub_id_list = [sub.id for sub in sub_list] + self.assertIn(del_sub.id, sub_id_list) + + self.cb_client.delete_subscription(subscription_id=del_id) + sub_list = self.cb_client.get_subscription_list(limit=10) + sub_id_list = [sub.id for sub in sub_list] + self.assertNotIn(del_sub.id, sub_id_list) + + for sub in sub_list: + self.cb_client.delete_subscription(subscription_id=sub.id) + + def test_update_subscription(self): + """ + Update a subscription. + Only the fields included in the request are updated in the subscription. + Args: + - subscriptionID(string): required + - body(body): required + Returns: + - Successful: 204, no content + Tests: + - Patch existing subscription and read out if the subscription got patched. + - Try to patch non-existent subscriptions. + - Try to patch more than one subscription at once. + """ + attr_id = "attr" + id = "urn:ngsi-ld:Subscription:" + "test_sub77" + notification_param = NotificationParams(attributes=[attr_id], endpoint=self.endpoint_http) + sub = SubscriptionLD(id=id, notification=notification_param, entities=[{"type": "Room"}]) + self.cb_client.post_subscription(sub) + sub_list = self.cb_client.get_subscription_list() + self.assertEqual(len(sub_list), 1) + print(self.endpoint_http.model_dump()) + sub_changed = SubscriptionLD(id=id, notification=notification_param, entities=[{"type": "House"}]) + self.cb_client.update_subscription(sub_changed) + u_sub= self.cb_client.get_subscription(subscription_id=id) + self.assertNotEqual(u_sub,sub_list[0]) + self.maxDiff = None + self.assertDictEqual(sub_changed.model_dump(), + u_sub.model_dump()) + non_sub = SubscriptionLD(id="urn:ngsi-ld:Subscription:nonexist", + notification=notification_param, + entities=[{"type":"house"}]) + with self.assertRaises(Exception): + self.cb_client.update_subscription(non_sub) + + +class TestSubsCheckBroker(TestCase): + """ + These tests are more oriented towards testing the actual broker. + Some functionality in Orion LD may not be consistent at times. + """ + def timeout_func(self): + self.last_test_timeout =[False] + + def cleanup(self): + """ + Cleanup test subscriptions + """ + sub_list = self.cb_client.get_subscription_list() + for sub in sub_list: + if sub.id.startswith('urn:ngsi-ld:Subscription:test_sub'): + self.cb_client.delete_subscription(sub.id) + try: + entity_list = True + while entity_list: + entity_list = self.cb_client.get_entity_list(limit=100) + self.cb_client.entity_batch_operation(action_type=ActionTypeLD.DELETE, + entities=entity_list) + except RequestException: + pass + + def setUp(self) -> None: + """ + Setup test data + Returns: + None + """ + self.entity_dict = { + 'id': 'urn:ngsi-ld:Entity:test_entity03', + 'type': 'Room', + 'temperature': { + 'type': 'Property', + 'value': 30 + } + } + + self.sub_dict = { + 'description': 'Test Subscription', + 'id': 'urn:ngsi-ld:Subscription:test_sub25', + 'type': 'Subscription', + 'entities': [ + { + 'type': 'Room' + } + ], + 'watchedAttributes': [ + 'temperature' + ], + 'q': 'temperature<30', + 'notification': { + 'attributes': [ + 'temperature' + ], + 'format': 'normalized', + 'endpoint': { + 'uri': f'mqtt://' # change uri + f'{settings.LD_MQTT_BROKER_URL_INTERNAL.host}:' + f'{settings.LD_MQTT_BROKER_URL_INTERNAL.port}/my/test/topic', + 'Accept': 'application/json' + }, + 'notifierInfo': [ + { + "key": "MQTT-Version", + "value": "mqtt5.0" + } + ] + } + } + self.fiware_header = FiwareLDHeader( + ngsild_tenant=settings.FIWARE_SERVICE) + self.cb_client = ContextBrokerLDClient(url=settings.LD_CB_URL, + fiware_header=self.fiware_header) + # initial tenant + self.cb_client.post_entity(ContextLDEntity(id="Dummy:1", type="Dummy"), + update=True) + self.cb_client.delete_entity_by_id("Dummy:1") + self.mqtt_client = mqtt.Client(callback_api_version=CallbackAPIVersion.VERSION2) + #on_message callbacks differ from test to test, but on connect callbacks dont + def on_connect_fail(client,userdata): + self.fail("Test failed due to broker being down") + + def on_connect(client,userdata,flags,reason_code,properties): + self.mqtt_client.subscribe("my/test/topic") + + self.mqtt_client.on_connect_fail = on_connect_fail + self.mqtt_client.on_connect = on_connect + self.cleanup() + #posting one single entity to check subscription existence/triggers + self.cb_client.post_entity(entity=ContextLDEntity(**self.entity_dict)) + + #All broker tests rely on awaiting a message. This timer helps with: + # -Avoiding hang ups in the case of a lost connection + # -Avoid ending the tests early, in the case a notification takes longer + + self.timeout = 5 # in seconds + self.last_test_timeout = [True] + self.timeout_proc = threading.Timer(self.timeout, + self.timeout_func) + + def tearDown(self) -> None: + self.cleanup() + self.cb_client.close() + + def test_post_subscription_mqtt(self): + """ + Tests: + - Subscribe using an mqtt topic as endpoint and see if notification is received + """ + def on_message(client,userdata,msg): + #the callback cancels the timer if a message comes through + self.timeout_proc.cancel() + updated_entity = self.entity_dict.copy() + updated_entity.update({'temperature':{'type':'Property','value':25}}) + self.mqtt_client.loop_stop() + self.mqtt_client.disconnect() + #extra sanity check on the contents of the notification(in case we are + #catching a rogue one) + self.assertEqual(updated_entity, + json.loads(msg.payload.decode())['body']['data'][0]) + + self.mqtt_client.on_message = on_message + self.mqtt_client.connect(settings.LD_MQTT_BROKER_URL.host, + settings.LD_MQTT_BROKER_URL.port, + 60) + self.mqtt_client.loop_start() + #post subscription then start timer + self.cb_client.post_subscription(subscription=SubscriptionLD(**self.sub_dict)) + self.timeout_proc.start() + #update entity to (ideally) get a notification + self.cb_client.update_entity_attribute(entity_id='urn:ngsi-ld:Entity:test_entity03', + attr=NamedContextProperty(type="Property", + value=25, + name='temperature'), + attr_name='temperature') + #this loop is necessary otherwise the test does not fail when the time runs out + while(self.timeout_proc.is_alive()): + continue + #if all goes well, the callback is triggered, and cancels the timer before + #it gets to change the timeout variable to False, making the following assertion True + self.assertTrue(self.last_test_timeout[0],"Operation timed out") + + def test_update_subscription_check_broker(self): + """ + Update a subscription and check changes in received messages. + Args: + - subscriptionID(string): required + - body(body): required + Returns: + - Successful: 204, no content + Tests: + - Patch existing subscription and read out if the subscription got patched. + Steps: + - Create Subscription with q = x + - Update entity to trigger sub with valid condition x + - Update subscription to q = x̄ + - Update entity to trigger sub with opposite condition x̄ + """ + current_vals = [25,33] + + #re-assigning a variable inside an inline function does not work => hence generator + def idx_generator(n): + while(n<2): + yield n + n+=1 + + gen = idx_generator(0) + + def on_message(client,userdata,msg): + idx = next(gen) + self.timeout_proc.cancel() + print(json.loads(msg.payload.decode()) + ['body']['data'][0]['temperature']['value']) + self.assertEqual(current_vals[idx], + json.loads(msg.payload.decode()) + ['body']['data'][0]['temperature']['value']) + + self.mqtt_client.on_message = on_message + + self.mqtt_client.connect(settings.LD_MQTT_BROKER_URL.host, + settings.LD_MQTT_BROKER_URL.port, + 60) + self.mqtt_client.loop_start() + self.cb_client.post_subscription(subscription=SubscriptionLD(**self.sub_dict)) + self.timeout_proc.start() + + self.cb_client.update_entity_attribute(entity_id='urn:ngsi-ld:Entity:test_entity03', + attr=NamedContextProperty(type="Property", + value=current_vals[0], + name='temperature'), + attr_name='temperature') + while(self.timeout_proc.is_alive()): + continue + self.assertTrue(self.last_test_timeout[0], + "Operation timed out") + + self.last_test_timeout = [True] + self.timeout_proc = threading.Timer(self.timeout,self.timeout_func) + + self.sub_dict.update({'q':'temperature>30'}) + self.cb_client.update_subscription(subscription=SubscriptionLD(**self.sub_dict)) + time.sleep(5) + updated = self.cb_client.get_subscription(self.sub_dict['id']) + self.assertEqual(updated.q,'temperature>30') + self.timeout_proc.start() + self.cb_client.update_entity_attribute(entity_id='urn:ngsi-ld:Entity:test_entity03', + attr=NamedContextProperty(type="Property", + value=current_vals[1], + name='temperature'), + attr_name='temperature') + while(self.timeout_proc.is_alive()): + continue + self.assertTrue(self.last_test_timeout[0], + "Operation timed out") + self.mqtt_client.loop_stop() + self.mqtt_client.disconnect() + + def test_delete_subscription_check_broker(self): + """ + Cancels subscription and checks on subscribed values. + Args: + - subscriptionID(string): required + Returns: + - Successful: 204, no content + Tests: + - Post and delete subscription then see if the broker still gets subscribed values. + + """ + pass + + def test_get_subscription_check_broker(self): + """ + Returns the subscription if it exists. + Args: + - subscriptionId(string): required + Returns: + - (200) subscription or empty list if successful + - Error Code + Tests: + - Subscribe to a message and see if it appears when the message is subscribed to + - Choose a non-existent ID and see if the return is an empty array + """ + pass diff --git a/tests/clients/test_ngsi_v2_cb.py b/tests/clients/test_ngsi_v2_cb.py index f44d25a2..c38e8dec 100644 --- a/tests/clients/test_ngsi_v2_cb.py +++ b/tests/clients/test_ngsi_v2_cb.py @@ -18,7 +18,6 @@ from filip.utils.simple_ql import QueryString from filip.clients.ngsi_v2 import ContextBrokerClient, IoTAClient from filip.clients.ngsi_v2 import HttpClient, HttpClientConfig -from filip.config import settings from filip.models.ngsi_v2.context import \ ContextEntity, \ ContextAttribute, \ diff --git a/tests/config.py b/tests/config.py index 097e0c24..5de6f12e 100644 --- a/tests/config.py +++ b/tests/config.py @@ -31,6 +31,12 @@ class TestSettings(BaseSettings): 'CB_HOST', 'CONTEXTBROKER_URL', 'OCB_URL')) + LD_CB_URL: AnyHttpUrl = Field(default="http://localhost:1026", + validation_alias=AliasChoices('LD_ORION_URL', + 'LD_CB_URL', + 'ORION_LD_URL', + 'SCORPIO_URL', + 'STELLIO_URL')) IOTA_URL: AnyHttpUrl = Field(default="http://localhost:4041", validation_alias='IOTA_URL') IOTA_JSON_URL: AnyHttpUrl = Field(default="http://localhost:4041", @@ -54,6 +60,17 @@ class TestSettings(BaseSettings): 'MQTT_BROKER_URL_INTERNAL', 'MQTT_URL_INTERNAL')) + LD_MQTT_BROKER_URL: AnyUrl = Field(default="mqtt://127.0.0.1:1884", + validation_alias=AliasChoices( + 'LD_MQTT_BROKER_URL', + 'LD_MQTT_URL', + 'LD_MQTT_BROKER')) + + LD_MQTT_BROKER_URL_INTERNAL: AnyUrl = Field(default="mqtt://mqtt-broker-ld:1883", + validation_alias=AliasChoices( + 'LD_MQTT_BROKER_URL_INTERNAL', + 'LD_MQTT_URL_INTERNAL')) + # IF CI_JOB_ID is present it will always overwrite the service path CI_JOB_ID: Optional[str] = Field(default=None, validation_alias=AliasChoices('CI_JOB_ID')) diff --git a/tests/models/test_ngsi_ld_context.py b/tests/models/test_ngsi_ld_context.py new file mode 100644 index 00000000..be95f6a7 --- /dev/null +++ b/tests/models/test_ngsi_ld_context.py @@ -0,0 +1,438 @@ +""" +Test module for context broker models +""" +import unittest + +from geojson_pydantic import Point, MultiPoint, LineString, Polygon, GeometryCollection +from pydantic import ValidationError + +from filip.models.ngsi_ld.context import \ + ContextLDEntity, ContextProperty, NamedContextProperty, \ + ContextGeoPropertyValue, ContextGeoProperty, NamedContextGeoProperty, \ + NamedContextRelationship + + +class TestLDContextModels(unittest.TestCase): + """ + Test class for context broker models + """ + def setUp(self) -> None: + """ + Setup test data + Returns: + None + """ + self.entity1_dict = { + "id": "urn:ngsi-ld:OffStreetParking:Downtown1", + "type": "OffStreetParking", + "name": { + "type": "Property", + "value": "Downtown One" + }, + "availableSpotNumber": { + "type": "Property", + "value": 121, + "observedAt": "2017-07-29T12:05:02Z", + "reliability": { + "type": "Property", + "value": 0.7 + }, + "providedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Camera:C1" + } + }, + "totalSpotNumber": { + "type": "Property", + "value": 200 + }, + "location": { + "type": "GeoProperty", + "value": { + "type": "Point", + "coordinates": (-8.5, 41.2) # coordinates are normally a tuple + } + }, + "@context": [ + "http://example.org/ngsi-ld/latest/parking.jsonld", + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.3.jsonld" + ] + } + self.entity1_props_dict = { + "location": { + "type": "GeoProperty", + "value": { + "type": "Point", + "coordinates": (-8.5, 41.2) + } + }, + "totalSpotNumber": { + "type": "Property", + "value": 200 + }, + "availableSpotNumber": { + "type": "Property", + "value": 121, + "observedAt": "2017-07-29T12:05:02Z", + "reliability": { + "type": "Property", + "value": 0.7 + }, + "providedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Camera:C1" + } + }, + "name": { + "type": "Property", + "value": "Downtown One" + }, + } + self.entity1_context = [ + "http://example.org/ngsi-ld/latest/parking.jsonld", + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.3.jsonld" + ] + self.entity2_dict = { + "id": "urn:ngsi-ld:Vehicle:A4567", + "type": "Vehicle", + "@context": [ + "http://example.org/ngsi-ld/latest/commonTerms.jsonld", + "http://example.org/ngsi-ld/latest/vehicle.jsonld", + "http://example.org/ngsi-ld/latest/parking.jsonld", + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.3.jsonld" + ] + } + self.entity2_props_dict = { + "brandName": { + "type": "Property", + "value": "Mercedes" + } + } + self.entity2_rel_dict = { + "isParked": { + "type": "Relationship", + "object": "urn:ngsi-ld:OffStreetParking:Downtown1", + "observedAt": "2017-07-29T12:00:04Z", + "providedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Person:Bob" + } + } + } + self.entity2_dict.update(self.entity2_props_dict) + self.entity2_dict.update(self.entity2_rel_dict) + self.entity3_dict = { + "id": "urn:ngsi-ld:Vehicle:test1243", + "type": "Vehicle", + "isParked": { + "type": "Relationship", + "object": "urn:ngsi-ld:OffStreetParking:Downtown1", + "observedAt": "2017-07-29T12:00:04Z", + "providedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Person:Bob" + } + } + } + # # The entity for testing the nested structure of properties + # self.entity_sub_props_dict_wrong = { + # "id": "urn:ngsi-ld:OffStreetParking:Downtown1", + # "type": "OffStreetParking", + # "name": { + # "type": "Property", + # "value": "Downtown One" + # }, + # "totalSpotNumber": { + # "type": "Property", + # "value": 200 + # }, + # "location": { + # "type": "GeoProperty", + # "value": { + # "type": "Point", + # "coordinates": [-8.5, 41.2] + # } + # }, + # "@context": [ + # "http://example.org/ngsi-ld/latest/parking.jsonld", + # "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.3.jsonld" + # ] + # } + self.testpoint_value = { + "type": "Point", + "coordinates": (-8.5, 41.2) + } + self.testmultipoint_value = { + "type": "MultiPoint", + "coordinates": ( + (-3.80356167695194, 43.46296641666926), + (-3.804056, 43.464638) + ) + } + self.testlinestring_value = { + "type": "LineString", + "coordinates": ( + (-3.80356167695194, 43.46296641666926), + (-3.804056, 43.464638) + ) + } + self.testpolygon_value = { + "type": "Polygon", + "coordinates": [ + [ + (-3.80356167695194, 43.46296641666926), + (-3.804056, 43.464638), + (-3.805056, 43.463638), + (-3.80356167695194, 43.46296641666926) + ] + ] + } + self.testgeometrycollection_value = { + "type": "GeometryCollection", + "geometries": [ + { + "type": "Point", + "coordinates": (-3.80356167695194, 43.46296641666926) + }, + { + "type": "LineString", + "coordinates": ( + (-3.804056, 43.464638), + (-3.805056, 43.463638) + ) + } + ] + } + self.entity_geo_dict = { + "id": "urn:ngsi-ld:Geometry:001", + "type": "MyGeometry", + "testpoint": { + "type": "GeoProperty", + "value": self.testpoint_value + }, + "testmultipoint": { + "type": "GeoProperty", + "value": self.testmultipoint_value, + "observedAt": "2023-09-12T12:35:00Z" + }, + "testlinestring": { + "type": "GeoProperty", + "value": self.testlinestring_value, + "observedAt": "2023-09-12T12:35:30Z" + }, + "testpolygon": { + "type": "GeoProperty", + "value": self.testpolygon_value, + "observedAt": "2023-09-12T12:36:00Z" + } + } + + def test_cb_property(self) -> None: + """ + Test context property models + Returns: + None + """ + prop = ContextProperty(**{'value': "20"}) + self.assertIsInstance(prop.value, str) + prop = ContextProperty(**{'value': 20.53}) + self.assertIsInstance(prop.value, float) + prop = ContextProperty(**{'value': 20}) + self.assertIsInstance(prop.value, int) + + def test_geo_property(self) -> None: + """ + Test ContextGeoPropertyValue models + Returns: + None + """ + geo_entity = ContextLDEntity(**self.entity_geo_dict) + new_entity = ContextLDEntity(id="urn:ngsi-ld:Geometry:002", type="MyGeometry") + test_point = NamedContextGeoProperty( + name="testpoint", + type="GeoProperty", + value=Point(**self.testpoint_value) + ) + test_MultiPoint = NamedContextGeoProperty( + name="testmultipoint", + type="GeoProperty", + value=MultiPoint(**self.testmultipoint_value) + ) + test_LineString = NamedContextGeoProperty( + name="testlinestring", + type="GeoProperty", + value=LineString(**self.testlinestring_value) + ) + test_Polygon = NamedContextGeoProperty( + name="testpolygon", + type="GeoProperty", + value=Polygon(**self.testpolygon_value) + ) + with self.assertRaises(ValidationError): + test_GeometryCollection = NamedContextGeoProperty( + name="testgeometrycollection", + type="GeoProperty", + value=GeometryCollection(**self.testgeometrycollection_value) + ) + new_entity.add_geo_properties([test_point, test_MultiPoint, test_LineString, + test_Polygon]) + + def test_cb_entity(self) -> None: + """ + Test context entity models + Returns: + None + """ + test = ContextLDEntity.get_model_fields_set() + entity1 = ContextLDEntity(**self.entity1_dict) + entity2 = ContextLDEntity(**self.entity2_dict) + + self.assertEqual(self.entity1_dict, + entity1.model_dump(exclude_unset=True)) + entity1 = ContextLDEntity.model_validate(self.entity1_dict) + + self.assertEqual(self.entity2_dict, + entity2.model_dump(exclude_unset=True)) + entity2 = ContextLDEntity.model_validate(self.entity2_dict) + + # check all properties can be returned by get_properties + properties_1 = entity1.get_properties(response_format='list') + for prop in properties_1: + self.assertEqual(self.entity1_props_dict[prop.name], + prop.model_dump( + exclude={'name'}, + exclude_unset=True)) + + properties_2 = entity2.get_properties(response_format='list') + for prop in properties_2: + self.assertEqual(self.entity2_props_dict[prop.name], + prop.model_dump( + exclude={'name'}, + exclude_unset=True)) + + # check all relationships can be returned by get_relationships + relationships = entity2.get_relationships(response_format='list') + for relationship in relationships: + self.assertEqual(self.entity2_rel_dict[relationship.name], + relationship.model_dump( + exclude={'name'}, + exclude_unset=True)) + + # test add properties + new_prop = {'new_prop': ContextProperty(value=25)} + entity2.add_properties(new_prop) + properties = entity2.get_properties(response_format='list') + self.assertIn("new_prop", [prop.name for prop in properties]) + + def test_validate_subproperties_dict(self) -> None: + """ + Test the validation of multi-level properties in entities + Returns: + None + """ + entity4 = ContextLDEntity(**self.entity1_dict) + + def test_validate_subproperties_dict_wrong(self) -> None: + """ + Test the validation of multi-level properties in entities + Returns: + None + """ + entity_sub_props_dict_wrong_1 = self.entity1_dict.copy() + entity_sub_props_dict_wrong_1[ + "availableSpotNumber"]["reliability"]["type"] = "NotProperty" + with self.assertRaises(ValueError): + entity5 = ContextLDEntity(**entity_sub_props_dict_wrong_1) + entity_sub_props_dict_wrong_2 = self.entity1_dict.copy() + entity_sub_props_dict_wrong_2[ + "availableSpotNumber"]["providedBy"]["type"] = "NotRelationship" + with self.assertRaises(ValueError): + entity5 = ContextLDEntity(**entity_sub_props_dict_wrong_2) + + def test_get_properties(self): + """ + Test the get_properties method + """ + entity = ContextLDEntity(id="urn:ngsi-ld:test", + type="Tester", + hasLocation={ + "type": "Relationship", + "object": "urn:ngsi-ld:test2" + }) + + properties = [ + NamedContextProperty(name="prop1"), + NamedContextProperty(name="prop2"), + ] + entity.add_properties(properties) + entity.get_properties(response_format="list") + self.assertEqual(entity.get_properties(response_format="list"), + properties) + + def test_entity_delete_properties(self): + """ + Test the delete_properties method + """ + prop = ContextProperty(**{'value': 20, 'type': 'Property'}) + named_prop = NamedContextProperty(**{'name': 'test2', + 'value': 20, + 'type': 'Property'}) + prop3 = ContextProperty(**{'value': 20, 'type': 'Property'}) + + entity = ContextLDEntity(id="urn:ngsi-ld:12", type="Test") + + entity.add_properties({"test1": prop, "test3": prop3}) + entity.add_properties([named_prop]) + + entity.delete_properties({"test1": prop}) + self.assertEqual(set([_prop.name for _prop in entity.get_properties()]), + {"test2", "test3"}) + + entity.delete_properties([named_prop]) + self.assertEqual(set([_prop.name for _prop in entity.get_properties()]), + {"test3"}) + + entity.delete_properties(["test3"]) + self.assertEqual(set([_prop.name for _prop in entity.get_properties()]), + set()) + + def test_entity_relationships(self): + entity = ContextLDEntity(**self.entity3_dict) + + # test get relationships + relationships_list = entity.get_relationships(response_format='list') + self.assertEqual(len(relationships_list), 1) + relationships_dict = entity.get_relationships(response_format='dict') + self.assertIn("isParked", relationships_dict) + + # test add relationships + new_rel_dict = { + "name": "new_rel", + "type": "Relationship", + "obejct": 'urn:ngsi-ld:test'} + new_rel = NamedContextRelationship(**new_rel_dict) + entity.add_relationships([new_rel]) + relationships_list = entity.get_relationships(response_format='list') + self.assertEqual(len(relationships_list), 2) + + # test delete relationships + entity.delete_relationships(["isParked"]) + relationships_list = entity.get_relationships(response_format='list') + self.assertEqual(len(relationships_list), 1) + relationships_dict = entity.get_relationships(response_format='dict') + self.assertNotIn("isParked", relationships_dict) + + def test_get_context(self): + entity1 = ContextLDEntity(**self.entity1_dict) + context_entity1 = entity1.get_context() + + self.assertEqual(self.entity1_context, + context_entity1) + + # test here if entity without context can be validated and get_context + # works accordingly: + entity3 = ContextLDEntity(**self.entity3_dict) + context_entity3 = entity3.get_context() + + self.assertEqual(None, + context_entity3) diff --git a/tests/models/test_ngsi_ld_subscriptions.py b/tests/models/test_ngsi_ld_subscriptions.py new file mode 100644 index 00000000..61b6d434 --- /dev/null +++ b/tests/models/test_ngsi_ld_subscriptions.py @@ -0,0 +1,179 @@ +""" +Test module for context subscriptions and notifications +""" +import unittest +from pydantic import ValidationError +from filip.models.ngsi_ld.subscriptions import \ + Endpoint, NotificationParams, EntityInfo, TemporalQuery +from filip.models.base import FiwareHeader +from tests.config import settings + + +class TestLDSubscriptions(unittest.TestCase): + """ + Test class for context broker models + """ + + def setUp(self) -> None: + """ + Setup test data + Returns: + None + """ + self.fiware_header = FiwareHeader( + service=settings.FIWARE_SERVICE, + service_path=settings.FIWARE_SERVICEPATH) + self.http_url = "https://test.de:80" + self.mqtt_url = "mqtt://test.de:1883" + self.mqtt_topic = '/filip/testing' + self.notification = { + "attributes": ["speed"], + "format": "keyValues", + "endpoint": { + "uri": "http://my.endpoint.org/notify", + "accept": "application/json" + } + } + self.sub_dict = { + "id": "urn:ngsi-ld:Subscription:mySubscription", + "type": "Subscription", + "entities": [ + { + "type": "Vehicle" + } + ], + "watchedAttributes": ["speed"], + "q": "speed>50", + "geoQ": { + "georel": "near;maxDistance==2000", + "geometry": "Point", + "coordinates": [-1, 100] + }, + "notification": { + "attributes": ["speed"], + "format": "keyValues", + "endpoint": { + "uri": "http://my.endpoint.org/notify", + "accept": "application/json" + } + }, + "@context": [ + "http://example.org/ngsi-ld/latest/vehicle.jsonld", + "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.3.jsonld" + ] + } + + def test_endpoint_models(self): + """ + According to NGSI-LD Spec section 5.2.15 + Returns: + + """ + endpoint_http = Endpoint(**{ + "uri": "http://my.endpoint.org/notify", + "accept": "application/json" + }) + endpoint_mqtt = Endpoint(**{ + "uri": "mqtt://my.host.org:1883/my/test/topic", + "accept": "application/json", + "notifierInfo": [ + { + "key": "MQTT-Version", + "value": "mqtt5.0" + } + ] + }) + with self.assertRaises(ValidationError): + endpoint_https = Endpoint(**{ + "uri": "https://my.endpoint.org/notify", + "accept": "application/json" + }) + with self.assertRaises(ValidationError): + endpoint_amqx = Endpoint(**{ + "uri": "amqx://my.endpoint.org/notify", + "accept": "application/json" + }) + + def test_notification_models(self): + """ + Test notification models + According to NGSI-LD Spec section 5.2.14 + """ + # Test validator for conflicting fields + notification = NotificationParams.model_validate(self.notification) + + def test_entity_selector_models(self): + """ + According to NGSI-LD Spec section 5.2.33 + Returns: + + """ + entity_info = EntityInfo.model_validate({ + "type": "Vehicle" + }) + with self.assertRaises(ValueError): + entity_info = EntityInfo.model_validate({ + "id": "test:001" + }) + with self.assertRaises(ValueError): + entity_info = EntityInfo.model_validate({ + "idPattern": ".*" + }) + + def test_temporal_query_models(self): + """ + According to NGSI-LD Spec section 5.2.21 + Returns: + + """ + example0_temporalQ = { + "timerel": "before", + "timeAt": "2017-12-13T14:20:00Z" + } + self.assertEqual(example0_temporalQ, + TemporalQuery.model_validate(example0_temporalQ).model_dump( + exclude_unset=True) + ) + + example1_temporalQ = { + "timerel": "after", + "timeAt": "2017-12-13T14:20:00Z" + } + self.assertEqual(example1_temporalQ, + TemporalQuery.model_validate(example1_temporalQ).model_dump( + exclude_unset=True) + ) + + example2_temporalQ = { + "timerel": "between", + "timeAt": "2017-12-13T14:20:00Z", + "endTimeAt": "2017-12-13T14:40:00Z", + "timeproperty": "modifiedAt" + } + self.assertEqual(example2_temporalQ, + TemporalQuery.model_validate(example2_temporalQ).model_dump( + exclude_unset=True) + ) + + example3_temporalQ = { + "timerel": "between", + "timeAt": "2017-12-13T14:20:00Z" + } + with self.assertRaises(ValueError): + TemporalQuery.model_validate(example3_temporalQ) + + example4_temporalQ = { + "timerel": "before", + "timeAt": "14:20:00Z" + } + with self.assertRaises(ValueError): + TemporalQuery.model_validate(example4_temporalQ) + + example5_temporalQ = { + "timerel": "between", + "timeAt": "2017-12-13T14:20:00Z", + "endTimeAt": "14:40:00Z", + "timeproperty": "modifiedAt" + } + with self.assertRaises(ValueError): + TemporalQuery.model_validate(example5_temporalQ) diff --git a/tests/utils/test_clear.py b/tests/utils/test_clear.py index e43e6fa8..992ec20d 100644 --- a/tests/utils/test_clear.py +++ b/tests/utils/test_clear.py @@ -4,18 +4,22 @@ import random import time import unittest +import urllib.parse from datetime import datetime from typing import List from uuid import uuid4 - from requests import RequestException - +from filip.clients.ngsi_ld.cb import ContextBrokerLDClient from filip.clients.ngsi_v2 import ContextBrokerClient, IoTAClient, QuantumLeapClient -from filip.models.base import FiwareHeader +from filip.models.base import FiwareHeader, FiwareLDHeader +from filip.models.ngsi_ld.context import ContextLDEntity, ActionTypeLD +from filip.models.ngsi_ld.subscriptions import SubscriptionLD, NotificationParams, \ + Endpoint from filip.models.ngsi_v2.context import ContextEntity from filip.models.ngsi_v2.iot import Device, ServiceGroup from filip.models.ngsi_v2.subscriptions import Subscription, Message -from filip.utils.cleanup import clear_context_broker, clear_iot_agent, clear_quantumleap +from filip.utils.cleanup import clear_context_broker, clear_iot_agent, clear_quantumleap, \ + clear_context_broker_ld from tests.config import settings @@ -35,6 +39,10 @@ def setUp(self) -> None: self.cb_url = settings.CB_URL self.cb_client = ContextBrokerClient(url=self.cb_url, fiware_header=self.fiware_header) + self.cb_client_ld = ContextBrokerLDClient( + fiware_header=FiwareLDHeader(ngsild_tenant=settings.FIWARE_SERVICE), + url=settings.LD_CB_URL) + self.iota_url = settings.IOTA_URL self.iota_client = IoTAClient(url=self.iota_url, fiware_header=self.fiware_header) @@ -86,6 +94,30 @@ def test_clear_context_broker(self): self.assertEqual(0, len(self.cb_client.get_entity_list()) or len(self.cb_client.get_subscription_list())) + def test_clear_context_broker_ld(self): + """ + Test for clearing context broker LD using context broker client + """ + random_list = [random.randint(0, 100) for _ in range(10)] + entities = [ContextLDEntity(id=f"urn:ngsi-ld:clear_test:{str(i)}", + type='clear_test') for i in random_list] + self.cb_client_ld.entity_batch_operation(action_type=ActionTypeLD.CREATE, + entities=entities) + notification_param = NotificationParams(attributes=["attr"], + endpoint=Endpoint(**{ + "uri": urllib.parse.urljoin( + str(settings.LD_CB_URL), + "/ngsi-ld/v1/subscriptions"), + "accept": "application/json" + })) + sub = SubscriptionLD(id=f"urn:ngsi-ld:Subscription:clear_test:{random.randint(0, 100)}", + notification=notification_param, + entities=[{"type": "clear_test"}]) + self.cb_client_ld.post_subscription(subscription=sub) + clear_context_broker_ld(cb_ld_client=self.cb_client_ld) + self.assertEqual(0, len(self.cb_client_ld.get_entity_list())) + self.assertEqual(0, len(self.cb_client_ld.get_subscription_list())) + def test_clear_context_broker_with_url(self): """ Test for clearing context broker using context broker url and fiware header as parameters @@ -170,7 +202,7 @@ def create_attr(): client.post_notification(notification_message) create_data_points() - time.sleep(1) + time.sleep(2) self.assertEqual(len(self.ql_client.get_entities()), rec_numbs) clear_quantumleap(url=self.ql_url, fiware_header=self.fiware_header) @@ -178,7 +210,7 @@ def create_attr(): self.ql_client.get_entities() create_data_points() - time.sleep(1) + time.sleep(2) self.assertEqual(len(self.ql_client.get_entities()), rec_numbs) clear_quantumleap(ql_client=self.ql_client) with self.assertRaises(RequestException):