Merge main into branch

.github/workflows/ci.yml

@@ -1,5 +1,9 @@
 name: CI
 
+defaults:
+  run:
+    shell: bash
+
 on:
   push:
     branches:
@@ -23,17 +27,14 @@ jobs:
           python-version: "3.11"
       - name: Install pre-commit
         run: python -m pip install pre-commit
-        shell: bash
       - name: Show pre-commit requirements
         run: python -m pip freeze --local
-        shell: bash
       - uses: actions/cache@v4
         with:
           path: ~/.cache/pre-commit
           key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
       - name: Run pre-commit for all files
         run: pre-commit run --config './.pre-commit-config.yaml' --all-files --color=always --show-diff-on-failure
-        shell: bash
 
   test-datastore:
     runs-on: ubuntu-latest
@@ -129,9 +130,7 @@ jobs:
       - name: Install Dependencies
         run: |
           pip install --upgrade pip
-          pip install pytest-timeout
-          pip install pytest-cov
-          pip install httpx
+          pip install pytest-timeout pytest-cov httpx deepdiff
           pip install -r ./api/requirements.txt
 
       - name: Copy Protobuf file to api directory and build

.gitignore

@@ -50,6 +50,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+*/pytest-coverage*
 cover/
 
 # Translations

api/formatters/__init__.py

@@ -2,6 +2,7 @@
 from enum import Enum
 
 from . import covjson
+from . import geojson
 
 logger = logging.getLogger(__name__)
 
@@ -10,4 +11,11 @@ class Formats(str, Enum):
     covjson = "CoverageJSON"  # According to EDR spec
 
 
-formatters = {"CoverageJSON": covjson.convert_to_covjson}
+class Metadata_Formats(str, Enum):
+    geojson = "GeoJSON"
+
+
+formatters = {
+    "CoverageJSON": covjson.convert_to_covjson,
+}  # observations
+metadata_formatters = {"GeoJSON": geojson.convert_to_geojson}  # metadata

api/formatters/covjson.py

@@ -27,10 +27,12 @@
 
 def make_parameter(ts_mdata):
     return Parameter(
-        description={"en": ts_mdata.title},
+        description={
+            "en": f"{ts_mdata.standard_name} at {ts_mdata.level}m {ts_mdata.period} {ts_mdata.function}",
+        },
         observedProperty=ObservedProperty(
             id=f"https://vocab.nerc.ac.uk/standard_name/{ts_mdata.standard_name}",
-            label={"en": ts_mdata.instrument},
+            label={"en": ts_mdata.parameter_name},
         ),
         unit=Unit(label={"en": ts_mdata.unit}),
     )

api/formatters/geojson.py

@@ -0,0 +1,43 @@
+from fastapi import HTTPException
+from geojson_pydantic import Feature
+from geojson_pydantic import FeatureCollection
+from geojson_pydantic import Point
+
+
+def _make_properties(ts):
+    ts_metadata = {key.name: value for key, value in ts.ts_mdata.ListFields() if value}
+
+    ts_metadata["platform_vocabulary"] = (
+        "https://oscar.wmo.int/surface/#/search/station/stationReportDetails/" + ts.ts_mdata.platform
+        if not ts.ts_mdata.platform_vocabulary
+        else ts.ts_mdata.platform_vocabulary
+    )
+    # This should also be compatible with future when name is added to datastore
+    if "name" not in ts_metadata:
+        ts_metadata["name"] = ts.ts_mdata.platform  # TODO: grab proper name when implemented in proto
+
+    return ts_metadata
+
+
+def convert_to_geojson(response):
+    """
+    Will only generate geoJSON for stationary timeseries
+    """
+    features = [
+        Feature(
+            type="Feature",
+            id=ts.ts_mdata.timeseries_id,
+            properties=_make_properties(ts=ts),
+            geometry=Point(
+                type="Point",
+                coordinates=[
+                    ts.obs_mdata[0].geo_point.lon,
+                    ts.obs_mdata[0].geo_point.lat,
+                ],
+            ),
+        )
+        for ts in sorted(response.observations, key=lambda ts: ts.ts_mdata.timeseries_id)
+    ]
+    if not features:
+        raise HTTPException(404, detail="Query did not return any time series.")
+    return FeatureCollection(features=features, type="FeatureCollection") if len(features) > 1 else features[0]

api/grpc_getter.py

@@ -23,7 +23,13 @@ async def get_obs_request(request):
     return response
 
 
-async def getTSAGRequest(request):
+async def get_ts_ag_request(request):
     grpc_stub = get_grpc_stub()
     response = await grpc_stub.GetTSAttrGroups(request)
     return response
+
+
+async def get_extents_request(request):
+    grpc_stub = get_grpc_stub()
+    response = await grpc_stub.GetExtents(request)
+    return response

api/main.py

@@ -4,13 +4,14 @@
 
 import metadata_endpoints
 from brotli_asgi import BrotliMiddleware
-from dependencies import create_url_from_request
 from edr_pydantic.capabilities import LandingPageModel
 from edr_pydantic.collections import Collection
 from edr_pydantic.collections import Collections
 from fastapi import FastAPI
 from fastapi import Request
-from routers import edr  # , records
+from routers import edr
+from routers import feature
+from utilities import create_url_from_request
 
 
 def setup_logging():
@@ -64,4 +65,4 @@ async def get_collection_metadata(request: Request) -> Collection:
 
 # Include all routes
 app.include_router(edr.router)
-# app.include(records.router)
+app.include_router(feature.router)

api/metadata_endpoints.py

@@ -1,4 +1,6 @@
 import logging
+from datetime import datetime
+from datetime import timezone
 from typing import Dict
 
 import datastore_pb2 as dstore
@@ -11,20 +13,44 @@
 from edr_pydantic.data_queries import EDRQuery
 from edr_pydantic.extent import Extent
 from edr_pydantic.extent import Spatial
+from edr_pydantic.extent import Temporal
 from edr_pydantic.link import EDRQueryLink
 from edr_pydantic.link import Link
 from edr_pydantic.observed_property import ObservedProperty
 from edr_pydantic.parameter import Parameter
 from edr_pydantic.unit import Unit
 from edr_pydantic.variables import Variables
 from fastapi import HTTPException
-from grpc_getter import getTSAGRequest
+from grpc_getter import get_extents_request
+from grpc_getter import get_ts_ag_request
 
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
 
+def datetime_to_iso_string(value: datetime) -> str:
+    """Returns the datetime as ISO 8601 string.
+    Changes timezone +00:00 to the military time zone indicator (Z).
+
+    Keyword arguments:
+    value -- A datetime
+
+    Returns:
+    datetime string -- Returns the datetime as an ISO 8601 string with the military indicator.
+    """
+    if value.tzinfo is None:
+        # This sort of replicates the functionality of Pydantic's AwareDatetime type
+        raise ValueError("Datetime object is not timezone aware")
+
+    iso_8601_str = value.isoformat()
+    tz_offset_utc = "+00:00"
+    if iso_8601_str.endswith(tz_offset_utc):
+        return f"{iso_8601_str[:-len(tz_offset_utc)]}Z"
+    else:
+        return iso_8601_str
+
+
 def get_landing_page(request):
     return LandingPageModel(
         title="E-SOH EDR API",
@@ -43,8 +69,8 @@ def get_landing_page(request):
 
 
 async def get_collection_metadata(base_url: str, is_self) -> Collection:
-    ts_request = dstore.GetTSAGRequest(attrs=["parameter_name", "title", "standard_name", "instrument", "unit"])
-    ts_response = await getTSAGRequest(ts_request)
+    ts_request = dstore.GetTSAGRequest(attrs=["parameter_name", "standard_name", "unit", "level", "period", "function"])
+    ts_response = await get_ts_ag_request(ts_request)
     # logger.info(ts_response.ByteSize())
     # logger.info(len(ts_response.groups))
 
@@ -55,10 +81,10 @@ async def get_collection_metadata(base_url: str, is_self) -> Collection:
     for group in ts_response.groups:
         ts = group.combo
         parameter = Parameter(
-            description=ts.title,
+            description=f"{ts.standard_name} at {ts.level}m {ts.period} {ts.function}",
             observedProperty=ObservedProperty(
                 id=f"https://vocab.nerc.ac.uk/standard_name/{ts.standard_name}",
-                label=ts.instrument,
+                label=ts.parameter_name,
             ),
             unit=Unit(label=ts.unit),
         )
@@ -73,12 +99,28 @@ async def get_collection_metadata(base_url: str, is_self) -> Collection:
 
         all_parameters[ts.parameter_name] = parameter
 
+    extent_request = dstore.GetExtentsRequest()
+    extent_response = await get_extents_request(extent_request)
+    spatial_extent = extent_response.spatial_extent
+    interval_start = extent_response.temporal_extent.start.ToDatetime(tzinfo=timezone.utc)
+    interval_end = extent_response.temporal_extent.end.ToDatetime(tzinfo=timezone.utc)
+
     collection = Collection(
         id="observations",
         links=[
             Link(href=f"{base_url}/observations", rel="self" if is_self else "data"),
         ],
-        extent=Extent(spatial=Spatial(bbox=[[3.0, 50.0, 8.0, 55.0]], crs="WGS84")),  # TODO: Get this from database
+        extent=Extent(
+            spatial=Spatial(
+                bbox=[[spatial_extent.left, spatial_extent.bottom, spatial_extent.right, spatial_extent.top]],
+                crs="EPSG:4326",
+            ),
+            temporal=Temporal(
+                interval=[[interval_start, interval_end]],
+                values=[f"{datetime_to_iso_string(interval_start)}/{datetime_to_iso_string(interval_end)}"],
+                trs="datetime",
+            ),
+        ),
         data_queries=DataQueries(
             position=EDRQuery(
                 link=EDRQueryLink(

api/requirements.in

@@ -12,3 +12,5 @@ covjson-pydantic~=0.2.0
 edr-pydantic~=0.2.0
 shapely~=2.0
 geojson-pydantic~=1.0
+aiocached~=0.3.0
+jinja2~=3.1

api/requirements.txt

@@ -1,12 +1,14 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile --no-emit-index-url
+#    pip-compile --no-emit-index-url --strip-extras requirements.in
 #
+aiocached==0.3
+    # via -r requirements.in
 annotated-types==0.6.0
     # via pydantic
-anyio==4.2.0
+anyio==4.3.0
     # via
     #   starlette
     #   watchfiles
@@ -20,13 +22,15 @@ covjson-pydantic==0.2.1
     # via -r requirements.in
 edr-pydantic==0.2.1
     # via -r requirements.in
+exceptiongroup==1.2.0
+    # via anyio
 fastapi==0.109.2
     # via -r requirements.in
 geojson-pydantic==1.0.2
     # via -r requirements.in
-grpcio==1.60.1
+grpcio==1.62.1
     # via grpcio-tools
-grpcio-tools==1.60.1
+grpcio-tools==1.62.1
     # via -r requirements.in
 gunicorn==21.2.0
     # via -r requirements.in
@@ -36,41 +40,45 @@ httptools==0.6.1
     # via uvicorn
 idna==3.6
     # via anyio
+jinja2==3.1.3
+    # via -r requirements.in
+markupsafe==2.1.5
+    # via jinja2
 numpy==1.26.4
     # via shapely
-packaging==23.2
+packaging==24.0
     # via gunicorn
-protobuf==4.25.2
+protobuf==4.25.3
     # via grpcio-tools
-pydantic==2.6.1
+pydantic==2.6.4
     # via
     #   covjson-pydantic
     #   edr-pydantic
     #   fastapi
     #   geojson-pydantic
-pydantic-core==2.16.2
+pydantic-core==2.16.3
     # via pydantic
 python-dotenv==1.0.1
     # via uvicorn
 pyyaml==6.0.1
     # via uvicorn
-shapely==2.0.2
+shapely==2.0.3
     # via -r requirements.in
-sniffio==1.3.0
+sniffio==1.3.1
     # via anyio
 starlette==0.36.3
     # via
     #   brotli-asgi
     #   fastapi
-typing-extensions==4.9.0
+typing-extensions==4.10.0
     # via
+    #   anyio
     #   fastapi
     #   pydantic
     #   pydantic-core
-uvicorn[standard]==0.23.2
-    # via
-    #   -r requirements.in
     #   uvicorn
+uvicorn==0.23.2
+    # via -r requirements.in
 uvloop==0.19.0
     # via uvicorn
 watchfiles==0.21.0