Merge remote-tracking branch 'origin/main' into update-landingpage-info

EURODEO · Oct 15, 2024 · 6198707 · 6198707
2 parents a2ded62 + d8f3f44
commit 6198707
Show file tree

Hide file tree

Showing 4 changed files with 93 additions and 75 deletions.
diff --git a/ingest/api/datastore.py b/ingest/api/datastore.py
@@ -3,6 +3,10 @@
 from datetime import datetime
 from dateutil import parser
 
+from api.utilities import convert_to_meter
+from api.utilities import seconds_to_iso_8601_duration
+
+
 import datastore_pb2 as dstore
 
 from google.protobuf.timestamp_pb2 import Timestamp
@@ -33,7 +37,14 @@ def build_grpc_messages(msg: str) -> None:
     period = str(ts_metadata.period)
     function = ts_metadata.function
     standard_name = ts_metadata.standard_name
-    parameter_name = ":".join([standard_name, level, function, period])
+    parameter_name = ":".join(
+        [
+            standard_name,
+            convert_to_meter(level),
+            function,
+            seconds_to_iso_8601_duration(int(period)),
+        ]
+    )
     setattr(ts_metadata, "parameter_name", parameter_name)
 
     observation_data = dstore.ObsMetadata()

diff --git a/ingest/api/ingest.py b/ingest/api/ingest.py
@@ -1,7 +1,6 @@
 import logging
 from typing import Union
 
-import isodate
 import grpc
 import json
 
@@ -14,6 +13,9 @@
 
 import datastore_pb2 as dstore
 
+from api.utilities import seconds_to_iso_8601_duration
+from api.utilities import convert_to_meter
+
 logger = logging.getLogger(__name__)
 
 
@@ -36,25 +38,6 @@ def __init__(
                 logger.error("Failed to establish connection to mqtt, " + "\n" + str(e))
                 raise e
 
-    def seconds_to_iso_8601_duration(self, seconds: int) -> str:
-        duration = isodate.Duration(seconds=seconds)
-        iso_duration = isodate.duration_isoformat(duration)
-
-        # TODO: find a better way to format these
-        # Use PT24H instead of P1D
-        if iso_duration == "P1D":
-            iso_duration = "PT24H"
-
-        # iso_duration defaults to P0D when seconds is 0
-        if iso_duration == "P0D":
-            iso_duration = "PT0S"
-
-        return iso_duration
-
-    def convert_to_meter(self, level: int) -> str:
-        level = str(float(level) / 100)
-        return level
-
     async def ingest(self, message: Union[str, object]):
         """
         This method will interpret call all methods for deciding input type, build the mqtt messages, and
@@ -88,8 +71,8 @@ async def publish_messages(self, messages: list):
                 )
 
                 # modify the period back to iso format and level back to meter
-                period_iso = self.seconds_to_iso_8601_duration(msg["properties"]["period"])
-                level_string = self.convert_to_meter(msg["properties"]["level"])
+                period_iso = seconds_to_iso_8601_duration(msg["properties"]["period"])
+                level_string = convert_to_meter(msg["properties"]["level"])
                 msg["properties"]["level"] = level_string
                 msg["properties"]["period"] = period_iso
                 try:

diff --git a/ingest/api/utilities.py b/ingest/api/utilities.py
@@ -0,0 +1,22 @@
+import isodate
+
+
+def seconds_to_iso_8601_duration(seconds: int) -> str:
+    duration = isodate.Duration(seconds=seconds)
+    iso_duration = isodate.duration_isoformat(duration)
+
+    # TODO: find a better way to format these
+    # Use PT24H instead of P1D
+    if iso_duration == "P1D":
+        iso_duration = "PT24H"
+
+    # iso_duration defaults to P0D when seconds is 0
+    if iso_duration == "P0D":
+        iso_duration = "PT0S"
+
+    return iso_duration
+
+
+def convert_to_meter(level: int) -> str:
+    level = str(float(level) / 100)
+    return level
diff --git a/ingest/test/test_data/test_payload.json b/ingest/test/test_data/test_payload.json
@@ -1,54 +1,56 @@
-{
-    "properties": {
-        "title": "JAN MAYEN",
-        "institution": "Norwegian Meteorological Institute (MET Norway)",
-        "Conventions": "CF-1.10, ACDD-1.3",
-        "history": "2023-10-09 10:19:00: Created from kvalobs.",
-        "keywords": "Meteoroligical observations from ground stations",
-        "keywords_vocabulary": "placeholder",
-        "summary": "Ground observation from kvalobs, stationJAN MAYEN",
-        "source": "kvalobs",
-        "creator_name": "Norwegian Meteorological Institute",
-        "creator_url": "https//www.met.no",
-        "creator_email": "[email protected]",
-        "creator_type": "institution",
-        "integrity": {
-            "method": "sha256",
-            "value": "91.0"
+[
+    {
+        "properties": {
+            "title": "JAN MAYEN",
+            "institution": "Norwegian Meteorological Institute (MET Norway)",
+            "Conventions": "CF-1.10, ACDD-1.3",
+            "history": "2023-10-09 10:19:00: Created from kvalobs.",
+            "keywords": "Meteoroligical observations from ground stations",
+            "keywords_vocabulary": "placeholder",
+            "summary": "Ground observation from kvalobs, stationJAN MAYEN",
+            "source": "kvalobs",
+            "creator_name": "Norwegian Meteorological Institute",
+            "creator_url": "https//www.met.no",
+            "creator_email": "[email protected]",
+            "creator_type": "institution",
+            "integrity": {
+                "method": "sha256",
+                "value": "91.0"
+            },
+            "processing_level": "q1",
+            "license": "http//spdx.org/licenses/CC-BY-4.0(CC-BY-4.0)",
+            "platform": "0-20000-0-01001",
+            "platform_vocabulary": "placeholder",
+            "naming_authority": "no.met",
+            "datetime": "2024-04-24T10:19:00+00:00",
+            "level": "12",
+            "function": "point",
+            "period": "PT0S",
+            "content": {
+                "encoding": "utf-8",
+                "standard_name": "relative_humidity",
+                "unit": "percent",
+                "size": 4,
+                "value": "91.0"
+            },
+            "metadata_id": "urn:x-wmo:md:norway:no.met:bb8c10a1-ba20-4676-a193-1e1929de9d13",
+            "data_id": "urn:x-wmo:md:norway:no.met:bb8c10a1-ba20-4676-a193-1e1929de9d13",
+            "pubtime": "2023-10-10T07:50:09.607843+00:00"
         },
-        "processing_level": "q1",
-        "license": "http//spdx.org/licenses/CC-BY-4.0(CC-BY-4.0)",
-        "platform": "0-20000-0-01001",
-        "platform_vocabulary": "placeholder",
-        "naming_authority": "no.met",
-        "datetime": "2024-04-24T10:19:00+00:00",
-        "level": "12",
-        "function": "point",
-        "period": "P3DT12H30M5S",
-        "content": {
-            "encoding": "utf-8",
-            "standard_name": "relative_humidity",
-            "unit": "percent",
-            "size": 4,
-            "value": "91.0"
+        "links": [
+            {
+                "href": "Insert documentation about E-SOH datastore",
+                "rel": "canonical"
+            }
+        ],
+        "geometry": {
+            "type": "Point",
+            "coordinates": {
+                "lat": "70.9394",
+                "lon": "-8.669"
+            }
         },
-        "metadata_id": "urn:x-wmo:md:norway:no.met:bb8c10a1-ba20-4676-a193-1e1929de9d13",
-        "data_id": "urn:x-wmo:md:norway:no.met:bb8c10a1-ba20-4676-a193-1e1929de9d13",
-        "pubtime": "2023-10-10T07:50:09.607843+00:00"
-    },
-    "links": [
-        {
-            "href": "Insert documentation about E-SOH datastore",
-            "rel": "canonical"
-        }
-    ],
-    "geometry": {
-        "type": "Point",
-        "coordinates": {
-            "lat": "70.9394",
-            "lon": "-8.669"
-        }
-    },
-    "type": "Feature",
-    "version": "v4.0"
-}
+        "type": "Feature",
+        "version": "v4.0"
+    }
+]