Merge branch 'main' of github.com:EURODEO/e-soh into making_ingest_api

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
   #        pass_filenames: false
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: "v17.0.6"
+    rev: "v18.1.2"
     hooks:
       - id: clang-format
         entry: clang-format -i
@@ -43,15 +43,16 @@ repos:
         # https://stackoverflow.com/questions/59413979/how-exclude-ref-tag-from-check-yaml-git-hook
         args: ["--unsafe"] # Parse the yaml files for syntax.
 
-  # reorder-python-imports ~ sort python imports
-  - repo: https://github.com/asottile/reorder_python_imports
-    rev: v3.12.0
-    hooks:
-      - id: reorder-python-imports
+  # TODO: Re-enable when the issue https://github.com/psf/black/issues/4175 is solved for Black.
+  ## reorder-python-imports ~ sort python imports
+  #- repo: https://github.com/asottile/reorder_python_imports
+  #  rev: v3.12.0
+  #  hooks:
+  #    - id: reorder-python-imports
 
   # black ~ Formats Python code
   - repo: https://github.com/psf/black
-    rev: 23.12.1
+    rev: 24.3.0
     hooks:
       - id: black
         args: ["--line-length=120"]
@@ -74,6 +75,6 @@ repos:
 
   # ShellCheck ~ Gives warnings and suggestions for bash/sh shell scripts
   - repo: https://github.com/koalaman/shellcheck-precommit
-    rev: v0.9.0
+    rev: v0.10.0
     hooks:
       - id: shellcheck

api/README.md

@@ -0,0 +1,6 @@
+# E-SOH API
+
+## Enviorment variable
+### FORWARDED_ALLOW_IPS
+
+Environment variable used to set the `forwarded-allow-ips` in gunicorn. If this API is set behind a proxy, `FORWARDED_ALLOW_IPS` should be set to the proxy IP. Setting this to `*` is possible, but should only be set if you have ensured the API is only reachable from the proxy, and not directly from the internet. If not using docker compose this have to be passed to docker using the `-e` argument.

api/formatters/covjson.py

@@ -1,6 +1,8 @@
 import math
+import operator
 from collections import namedtuple
 from datetime import timezone
+from functools import reduce
 from itertools import groupby
 
 from covjson_pydantic.coverage import Coverage
@@ -88,9 +90,8 @@ def convert_to_covjson(response):
     elif len(coverages) == 1:
         return coverages[0]
     else:
-        return CoverageCollection(
-            coverages=coverages, parameters=coverages[0].parameters
-        )  # HACK to take parameters from first one
+        parameter_union = reduce(operator.ior, (c.parameters for c in coverages), {})
+        return CoverageCollection(coverages=coverages, parameters=dict(sorted(parameter_union.items())))
 
 
 def _collect_data(ts_mdata, obs_mdata):

api/formatters/geojson.py

@@ -8,13 +8,13 @@ def _make_properties(ts):
     ts_metadata = {key.name: value for key, value in ts.ts_mdata.ListFields() if value}
 
     ts_metadata["platform_vocabulary"] = (
-        "https://oscar.wmo.int/surface/#/search/station/stationReportDetails/" + ts.ts_mdata.platform
+        "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=" + ts.ts_mdata.platform
         if not ts.ts_mdata.platform_vocabulary
         else ts.ts_mdata.platform_vocabulary
     )
-    # This should also be compatible with future when name is added to datastore
-    if "name" not in ts_metadata:
-        ts_metadata["name"] = ts.ts_mdata.platform  # TODO: grab proper name when implemented in proto
+
+    if "platform_name" not in ts_metadata:
+        ts_metadata["platform_name"] = f'platform-{ts_metadata["platform"]}'
 
     return ts_metadata
 

api/response_classes.py

@@ -0,0 +1,9 @@
+from starlette.responses import JSONResponse
+
+
+class CoverageJsonResponse(JSONResponse):
+    media_type = "application/prs.coverage+json"
+
+
+class GeoJsonResponse(JSONResponse):
+    media_type = "application/geo+json"

api/routers/edr.py

@@ -20,6 +20,8 @@
 from geojson_pydantic import Feature
 from geojson_pydantic import Point
 from grpc_getter import get_obs_request
+from response_classes import CoverageJsonResponse
+from response_classes import GeoJsonResponse
 from shapely import buffer
 from shapely import geometry
 from shapely import wkt
@@ -50,17 +52,31 @@
     tags=["Collection data queries"],
     response_model=EDRFeatureCollection,
     response_model_exclude_none=True,
+    response_class=GeoJsonResponse,
 )
 # We can currently only query data, even if we only need metadata like for this endpoint
 # Maybe it would be better to only query a limited set of data instead of everything (meaning 24 hours)
 async def get_locations(
-    bbox: Annotated[str | None, Query(example="5.0,52.0,6.0,52.1")] = None
+    bbox: Annotated[str | None, Query(example="5.0,52.0,6.0,52.1")] = None,
+    datetime: Annotated[str | None, Query(example="2022-12-31T00:00Z/2023-01-01T00:00Z")] = None,
+    parameter_name: Annotated[
+        str | None,
+        Query(
+            alias="parameter-name",
+            example="wind_from_direction:2.0:mean:PT10M,"
+            "wind_speed:10:mean:PT10M,"
+            "relative_humidity:2.0:mean:PT1M,"
+            "air_pressure_at_sea_level:1:mean:PT1M,"
+            "air_temperature:1.5:maximum:PT10M",
+        ),
+    ] = None,
 ) -> EDRFeatureCollection:  # Hack to use string
     ts_request = dstore.GetObsRequest(
         temporal_latest=True,
         included_response_fields=[
             "parameter_name",
             "platform",
+            "platform_name",
             "geo_point",
             "standard_name",
             "unit",
@@ -77,12 +93,32 @@ async def get_locations(
             [dstore.Point(lat=coord[1], lon=coord[0]) for coord in poly.exterior.coords],
         )
 
+    if parameter_name:
+        parameter_name = split_and_strip(parameter_name)
+        await verify_parameter_names(parameter_name)
+        ts_request.filter["parameter_name"].values.extend(parameter_name)
+
+    if datetime:
+        start, end = get_datetime_range(datetime)
+        ts_request.temporal_interval.start.CopyFrom(start)
+        ts_request.temporal_interval.end.CopyFrom(end)
+
     ts_response = await get_obs_request(ts_request)
 
+    if len(ts_response.observations) == 0:
+        raise HTTPException(
+            status_code=404,
+            detail="Query did not return any features.",
+        )
+
     platform_parameters: DefaultDict[str, Set[str]] = defaultdict(set)
+    platform_names: Dict[str, Set[str]] = defaultdict(set)
     platform_coordinates: Dict[str, Set[Tuple[float, float]]] = defaultdict(set)
     all_parameters: Dict[str, Parameter] = {}
     for obs in ts_response.observations:
+        platform_names[obs.ts_mdata.platform].add(
+            obs.ts_mdata.platform_name if obs.ts_mdata.platform_name else f"platform-{obs.ts_mdata.platform}"
+        )
         parameter = make_parameter(obs.ts_mdata)
         platform_parameters[obs.ts_mdata.platform].add(obs.ts_mdata.parameter_name)
         # Take last point
@@ -101,7 +137,7 @@ async def get_locations(
             )
         all_parameters[obs.ts_mdata.parameter_name] = parameter
 
-    # Check for multiple coordinates on one station
+    # Check for multiple coordinates or names on one station
     for station_id in platform_parameters.keys():
         if len(platform_coordinates[station_id]) > 1:
             raise HTTPException(
@@ -111,15 +147,22 @@ async def get_locations(
                     f"has multiple coordinates: {platform_coordinates[station_id]}"
                 },
             )
+        if len(platform_names[station_id]) > 1:
+            raise HTTPException(
+                status_code=500,
+                detail={
+                    "platform_name": f"Station with id `{station_id} "
+                    f"has multiple names: {platform_names[station_id]}"
+                },
+            )
 
     features = [
         Feature(
             type="Feature",
             id=station_id,
             properties={
-                # TODO: Change to platform_name to correct one when its available, this is only for geoweb demo
-                "name": f"platform-{station_id}",
-                "detail": f"https://oscar.wmo.int/surface/#/search/station/stationReportDetails/{station_id}",
+                "name": list(platform_names[station_id])[0],
+                "detail": f"https://oscar.wmo.int/surface/rest/api/search/station?wigosId={station_id}",
                 "parameter-name": sorted(platform_parameters[station_id]),
             },
             geometry=Point(
@@ -139,6 +182,7 @@ async def get_locations(
     tags=["Collection data queries"],
     response_model=Coverage | CoverageCollection,
     response_model_exclude_none=True,
+    response_class=CoverageJsonResponse,
 )
 async def get_data_location_id(
     location_id: Annotated[str, Path(example="0-20000-0-06260")],
@@ -158,18 +202,23 @@ async def get_data_location_id(
 ):
     # TODO: There is no error handling of any kind at the moment!
     #  This is just a quick and dirty demo
-    range = get_datetime_range(datetime)
-    if parameter_name:
-        parameter_name = split_and_strip(parameter_name)
-        await verify_parameter_names(parameter_name)
     request = dstore.GetObsRequest(
         filter=dict(
-            parameter_name=dstore.Strings(values=parameter_name),
             platform=dstore.Strings(values=[location_id]),
         ),
-        temporal_interval=(dstore.TimeInterval(start=range[0], end=range[1]) if range else None),
         included_response_fields=response_fields_needed_for_data_api,
     )
+
+    if parameter_name:
+        parameter_name = split_and_strip(parameter_name)
+        await verify_parameter_names(parameter_name)
+        request.filter["parameter_name"].values.extend(parameter_name)
+
+    if datetime:
+        start, end = get_datetime_range(datetime)
+        request.temporal_interval.start.CopyFrom(start)
+        request.temporal_interval.end.CopyFrom(end)
+
     response = await get_obs_request(request)
     return formatters.formatters[f](response)
 
@@ -179,6 +228,7 @@ async def get_data_location_id(
     tags=["Collection data queries"],
     response_model=Coverage | CoverageCollection,
     response_model_exclude_none=True,
+    response_class=CoverageJsonResponse,
 )
 async def get_data_position(
     coords: Annotated[str, Query(example="POINT(5.179705 52.0988218)")],
@@ -225,6 +275,7 @@ async def get_data_position(
     tags=["Collection data queries"],
     response_model=Coverage | CoverageCollection,
     response_model_exclude_none=True,
+    response_class=CoverageJsonResponse,
 )
 async def get_data_area(
     coords: Annotated[str, Query(example="POLYGON((5.0 52.0, 6.0 52.0,6.0 52.1,5.0 52.1, 5.0 52.0))")],
@@ -262,18 +313,23 @@ async def get_data_area(
             detail={"coords": f"Unexpected error occurred during wkt parsing: {coords}"},
         )
 
-    range = get_datetime_range(datetime)
-    if parameter_name:
-        parameter_name = split_and_strip(parameter_name)
-        await verify_parameter_names(parameter_name)
     request = dstore.GetObsRequest(
-        filter=dict(parameter_name=dstore.Strings(values=parameter_name if parameter_name else None)),
         spatial_area=dstore.Polygon(
             points=[dstore.Point(lat=coord[1], lon=coord[0]) for coord in poly.exterior.coords]
         ),
-        temporal_interval=dstore.TimeInterval(start=range[0], end=range[1]) if range else None,
         included_response_fields=response_fields_needed_for_data_api,
     )
+
+    if parameter_name:
+        parameter_name = split_and_strip(parameter_name)
+        await verify_parameter_names(parameter_name)
+        request.filter["parameter_name"].values.extend(parameter_name)
+
+    if datetime:
+        start, end = get_datetime_range(datetime)
+        request.temporal_interval.start.CopyFrom(start)
+        request.temporal_interval.end.CopyFrom(end)
+
     coverages = await get_obs_request(request)
     coverages = formatters.formatters[f](coverages)
     return coverages

api/routers/feature.py

@@ -14,6 +14,7 @@
 from jinja2 import Environment
 from jinja2 import FileSystemLoader
 from jinja2 import select_autoescape
+from response_classes import GeoJsonResponse
 from shapely import geometry
 from utilities import get_datetime_range
 from utilities import split_and_strip
@@ -24,7 +25,11 @@
 
 
 @router.get(
-    "/items", tags=["Collection items"], response_model=Feature | FeatureCollection, response_model_exclude_none=True
+    "/items",
+    tags=["Collection items"],
+    response_model=Feature | FeatureCollection,
+    response_model_exclude_none=True,
+    response_class=GeoJsonResponse,
 )
 async def search_timeseries(
     bbox: Annotated[str | None, Query(example="5.0,52.0,6.0,52.1")] = None,
@@ -103,7 +108,13 @@ async def search_timeseries(
     return formatters.metadata_formatters[f](time_series)
 
 
-@router.get("/items/{item_id}", tags=["Collection items"], response_model=Feature, response_model_exclude_none=True)
+@router.get(
+    "/items/{item_id}",
+    tags=["Collection items"],
+    response_model=Feature,
+    response_model_exclude_none=True,
+    response_class=GeoJsonResponse,
+)
 async def get_time_series_by_id(
     item_id: Annotated[str, Path()],
     f: Annotated[