diff --git a/ingest/api/datastore.py b/ingest/api/datastore.py index 1c876d90..4290bb49 100644 --- a/ingest/api/datastore.py +++ b/ingest/api/datastore.py @@ -3,6 +3,10 @@ from datetime import datetime from dateutil import parser +from api.utilities import convert_to_meter +from api.utilities import seconds_to_iso_8601_duration + + import datastore_pb2 as dstore from google.protobuf.timestamp_pb2 import Timestamp @@ -33,7 +37,14 @@ def build_grpc_messages(msg: str) -> None: period = str(ts_metadata.period) function = ts_metadata.function standard_name = ts_metadata.standard_name - parameter_name = ":".join([standard_name, level, function, period]) + parameter_name = ":".join( + [ + standard_name, + convert_to_meter(level), + function, + seconds_to_iso_8601_duration(int(period)), + ] + ) setattr(ts_metadata, "parameter_name", parameter_name) observation_data = dstore.ObsMetadata() diff --git a/ingest/api/ingest.py b/ingest/api/ingest.py index e76f20ce..e1780f13 100644 --- a/ingest/api/ingest.py +++ b/ingest/api/ingest.py @@ -1,7 +1,6 @@ import logging from typing import Union -import isodate import grpc import json @@ -14,6 +13,9 @@ import datastore_pb2 as dstore +from api.utilities import seconds_to_iso_8601_duration +from api.utilities import convert_to_meter + logger = logging.getLogger(__name__) @@ -36,25 +38,6 @@ def __init__( logger.error("Failed to establish connection to mqtt, " + "\n" + str(e)) raise e - def seconds_to_iso_8601_duration(self, seconds: int) -> str: - duration = isodate.Duration(seconds=seconds) - iso_duration = isodate.duration_isoformat(duration) - - # TODO: find a better way to format these - # Use PT24H instead of P1D - if iso_duration == "P1D": - iso_duration = "PT24H" - - # iso_duration defaults to P0D when seconds is 0 - if iso_duration == "P0D": - iso_duration = "PT0S" - - return iso_duration - - def convert_to_meter(self, level: int) -> str: - level = str(float(level) / 100) - return level - async def ingest(self, message: Union[str, object]): """ This method will interpret call all methods for deciding input type, build the mqtt messages, and @@ -88,8 +71,8 @@ async def publish_messages(self, messages: list): ) # modify the period back to iso format and level back to meter - period_iso = self.seconds_to_iso_8601_duration(msg["properties"]["period"]) - level_string = self.convert_to_meter(msg["properties"]["level"]) + period_iso = seconds_to_iso_8601_duration(msg["properties"]["period"]) + level_string = convert_to_meter(msg["properties"]["level"]) msg["properties"]["level"] = level_string msg["properties"]["period"] = period_iso try: diff --git a/ingest/api/utilities.py b/ingest/api/utilities.py new file mode 100644 index 00000000..ac3dd9df --- /dev/null +++ b/ingest/api/utilities.py @@ -0,0 +1,22 @@ +import isodate + + +def seconds_to_iso_8601_duration(seconds: int) -> str: + duration = isodate.Duration(seconds=seconds) + iso_duration = isodate.duration_isoformat(duration) + + # TODO: find a better way to format these + # Use PT24H instead of P1D + if iso_duration == "P1D": + iso_duration = "PT24H" + + # iso_duration defaults to P0D when seconds is 0 + if iso_duration == "P0D": + iso_duration = "PT0S" + + return iso_duration + + +def convert_to_meter(level: int) -> str: + level = str(float(level) / 100) + return level diff --git a/ingest/test/test_data/test_payload.json b/ingest/test/test_data/test_payload.json index a6e39945..514e2f2f 100644 --- a/ingest/test/test_data/test_payload.json +++ b/ingest/test/test_data/test_payload.json @@ -1,54 +1,56 @@ -{ - "properties": { - "title": "JAN MAYEN", - "institution": "Norwegian Meteorological Institute (MET Norway)", - "Conventions": "CF-1.10, ACDD-1.3", - "history": "2023-10-09 10:19:00: Created from kvalobs.", - "keywords": "Meteoroligical observations from ground stations", - "keywords_vocabulary": "placeholder", - "summary": "Ground observation from kvalobs, stationJAN MAYEN", - "source": "kvalobs", - "creator_name": "Norwegian Meteorological Institute", - "creator_url": "https//www.met.no", - "creator_email": "info@met.no", - "creator_type": "institution", - "integrity": { - "method": "sha256", - "value": "91.0" +[ + { + "properties": { + "title": "JAN MAYEN", + "institution": "Norwegian Meteorological Institute (MET Norway)", + "Conventions": "CF-1.10, ACDD-1.3", + "history": "2023-10-09 10:19:00: Created from kvalobs.", + "keywords": "Meteoroligical observations from ground stations", + "keywords_vocabulary": "placeholder", + "summary": "Ground observation from kvalobs, stationJAN MAYEN", + "source": "kvalobs", + "creator_name": "Norwegian Meteorological Institute", + "creator_url": "https//www.met.no", + "creator_email": "info@met.no", + "creator_type": "institution", + "integrity": { + "method": "sha256", + "value": "91.0" + }, + "processing_level": "q1", + "license": "http//spdx.org/licenses/CC-BY-4.0(CC-BY-4.0)", + "platform": "0-20000-0-01001", + "platform_vocabulary": "placeholder", + "naming_authority": "no.met", + "datetime": "2024-04-24T10:19:00+00:00", + "level": "12", + "function": "point", + "period": "PT0S", + "content": { + "encoding": "utf-8", + "standard_name": "relative_humidity", + "unit": "percent", + "size": 4, + "value": "91.0" + }, + "metadata_id": "urn:x-wmo:md:norway:no.met:bb8c10a1-ba20-4676-a193-1e1929de9d13", + "data_id": "urn:x-wmo:md:norway:no.met:bb8c10a1-ba20-4676-a193-1e1929de9d13", + "pubtime": "2023-10-10T07:50:09.607843+00:00" }, - "processing_level": "q1", - "license": "http//spdx.org/licenses/CC-BY-4.0(CC-BY-4.0)", - "platform": "0-20000-0-01001", - "platform_vocabulary": "placeholder", - "naming_authority": "no.met", - "datetime": "2024-04-24T10:19:00+00:00", - "level": "12", - "function": "point", - "period": "P3DT12H30M5S", - "content": { - "encoding": "utf-8", - "standard_name": "relative_humidity", - "unit": "percent", - "size": 4, - "value": "91.0" + "links": [ + { + "href": "Insert documentation about E-SOH datastore", + "rel": "canonical" + } + ], + "geometry": { + "type": "Point", + "coordinates": { + "lat": "70.9394", + "lon": "-8.669" + } }, - "metadata_id": "urn:x-wmo:md:norway:no.met:bb8c10a1-ba20-4676-a193-1e1929de9d13", - "data_id": "urn:x-wmo:md:norway:no.met:bb8c10a1-ba20-4676-a193-1e1929de9d13", - "pubtime": "2023-10-10T07:50:09.607843+00:00" - }, - "links": [ - { - "href": "Insert documentation about E-SOH datastore", - "rel": "canonical" - } - ], - "geometry": { - "type": "Point", - "coordinates": { - "lat": "70.9394", - "lon": "-8.669" - } - }, - "type": "Feature", - "version": "v4.0" -} + "type": "Feature", + "version": "v4.0" + } +]