Merge pull request #31 from EURODEO/simple-format

Simplify output format handling

api/formatters/__init__.py

@@ -1,32 +1,13 @@
-import importlib
 import logging
-import pkgutil
+from enum import Enum
 
-import formatters
+from . import covjson
 
 logger = logging.getLogger(__name__)
 
 
-def get_EDR_formatters() -> dict:
-    """
-    This method should grab all available formatters and make them reachable in a dict
-    This way we can dynamicly grab all available formats and skip configuring this.
-    Should aliases be made available, and how do one make formatters present in openapi doc?
-    """
-    available_formatters = {}
+class Formats(str, Enum):
+    covjson = "CoverageJSON"  # According to EDR spec
 
-    formatter_plugins = [
-        importlib.import_module("formatters." + i.name)
-        for i in pkgutil.iter_modules(formatters.__path__)
-        if i.name != "base_formatter"
-    ]
-    logger.info(f"Loaded plugins : {formatter_plugins}")
-    for formatter_module in formatter_plugins:
-        # Make instance of formatter and save
-        available_formatters[formatter_module.__name__.split(".")[-1]] = getattr(
-            formatter_module, formatter_module.formatter_name
-        )()
 
-    # Should also setup dict for alias discover
-
-    return available_formatters
+formatters = {"CoverageJSON": covjson.convert_to_covjson}

api/formatters/covjson.py

@@ -15,87 +15,77 @@
 from covjson_pydantic.reference_system import ReferenceSystemConnectionObject
 from covjson_pydantic.unit import Unit
 from fastapi import HTTPException
-from formatters.base_formatter import EDRFormatter
 from pydantic import AwareDatetime
 
-# Requierd for pugin discovery
-# Need to be available at top level of formatter plugin
-formatter_name = "Covjson"
 
+# mime_type = "application/prs.coverage+json"
 
-class Covjson(EDRFormatter):
-    """
-    Class for converting protobuf object to coverage json
-    """
 
-    def __init__(self):
-        self.alias = ["covjson", "coveragejson"]
-        self.mime_type = "application/json"  # find the type for covjson
+def convert_to_covjson(response):
+    # Collect data
+    coverages = []
+    data = [_collect_data(md.ts_mdata, md.obs_mdata) for md in response.observations]
 
-    def convert(self, response):
-        # Collect data
-        coverages = []
-        data = [self._collect_data(md.ts_mdata, md.obs_mdata) for md in response.observations]
+    # Need to sort before using groupBy. Also sort on param_id to get consistently sorted output
+    data.sort(key=lambda x: (x[0], x[1]))
+    # The multiple coverage logic is not needed for this endpoint,
+    # but we want to share this code between endpoints
+    for (lat, lon, times), group in groupby(data, lambda x: x[0]):
+        referencing = [
+            ReferenceSystemConnectionObject(
+                coordinates=["y", "x"],
+                system=ReferenceSystem(type="GeographicCRS", id="http://www.opengis.net/def/crs/EPSG/0/4326"),
+            ),
+            ReferenceSystemConnectionObject(
+                coordinates=["z"],
+                system=ReferenceSystem(type="TemporalRS", calendar="Gregorian"),
+            ),
+        ]
+        domain = Domain(
+            domainType=DomainType.point_series,
+            axes=Axes(
+                x=ValuesAxis[float](values=[lon]),
+                y=ValuesAxis[float](values=[lat]),
+                t=ValuesAxis[AwareDatetime](values=times),
+            ),
+            referencing=referencing,
+        )
 
-        # Need to sort before using groupBy. Also sort on param_id to get consistently sorted output
-        data.sort(key=lambda x: (x[0], x[1]))
-        # The multiple coverage logic is not needed for this endpoint,
-        # but we want to share this code between endpoints
-        for (lat, lon, times), group in groupby(data, lambda x: x[0]):
-            referencing = [
-                ReferenceSystemConnectionObject(
-                    coordinates=["y", "x"],
-                    system=ReferenceSystem(type="GeographicCRS", id="http://www.opengis.net/def/crs/EPSG/0/4326"),
-                ),
-                ReferenceSystemConnectionObject(
-                    coordinates=["z"],
-                    system=ReferenceSystem(type="TemporalRS", calendar="Gregorian"),
-                ),
-            ]
-            domain = Domain(
-                domainType=DomainType.point_series,
-                axes=Axes(
-                    x=ValuesAxis[float](values=[lon]),
-                    y=ValuesAxis[float](values=[lat]),
-                    t=ValuesAxis[AwareDatetime](values=times),
-                ),
-                referencing=referencing,
+        parameters = {}
+        ranges = {}
+        for (_, _, _), param_id, unit, values in group:
+            if all(math.isnan(v) for v in values):
+                continue  # Drop ranges if completely nan.
+                # TODO: Drop the whole coverage if it becomes empty?
+            values_no_nan = [v if not math.isnan(v) else None for v in values]
+            # TODO: Improve this based on "standard name", etc.
+            parameters[param_id] = Parameter(
+                observedProperty=ObservedProperty(label={"en": param_id}), unit=Unit(label={"en": unit})
+            )  # TODO: Also fill symbol?
+            ranges[param_id] = NdArray(
+                values=values_no_nan, axisNames=["t", "y", "x"], shape=[len(values_no_nan), 1, 1]
             )
 
-            parameters = {}
-            ranges = {}
-            for (_, _, _), param_id, unit, values in group:
-                if all(math.isnan(v) for v in values):
-                    continue  # Drop ranges if completely nan.
-                    # TODO: Drop the whole coverage if it becomes empty?
-                values_no_nan = [v if not math.isnan(v) else None for v in values]
-                # TODO: Improve this based on "standard name", etc.
-                parameters[param_id] = Parameter(
-                    observedProperty=ObservedProperty(label={"en": param_id}), unit=Unit(label={"en": unit})
-                )  # TODO: Also fill symbol?
-                ranges[param_id] = NdArray(
-                    values=values_no_nan, axisNames=["t", "y", "x"], shape=[len(values_no_nan), 1, 1]
-                )
+        coverages.append(Coverage(domain=domain, parameters=parameters, ranges=ranges))
 
-            coverages.append(Coverage(domain=domain, parameters=parameters, ranges=ranges))
+    if len(coverages) == 0:
+        raise HTTPException(status_code=404, detail="No data found")
+    elif len(coverages) == 1:
+        return coverages[0]
+    else:
+        return CoverageCollection(
+            coverages=coverages, parameters=coverages[0].parameters
+        )  # HACK to take parameters from first one
 
-        if len(coverages) == 0:
-            raise HTTPException(status_code=404, detail="No data found")
-        elif len(coverages) == 1:
-            return coverages[0]
-        else:
-            return CoverageCollection(
-                coverages=coverages, parameters=coverages[0].parameters
-            )  # HACK to take parameters from first one
 
-    def _collect_data(self, ts_mdata, obs_mdata):
-        lat = obs_mdata[0].geo_point.lat  # HACK: For now assume they all have the same position
-        lon = obs_mdata[0].geo_point.lon
-        tuples = (
-            (o.obstime_instant.ToDatetime(tzinfo=timezone.utc), float(o.value)) for o in obs_mdata
-        )  # HACK: str -> float
-        (times, values) = zip(*tuples)
-        param_id = ts_mdata.instrument
-        unit = ts_mdata.unit
+def _collect_data(ts_mdata, obs_mdata):
+    lat = obs_mdata[0].geo_point.lat  # HACK: For now assume they all have the same position
+    lon = obs_mdata[0].geo_point.lon
+    tuples = (
+        (o.obstime_instant.ToDatetime(tzinfo=timezone.utc), float(o.value)) for o in obs_mdata
+    )  # HACK: str -> float
+    (times, values) = zip(*tuples)
+    param_id = ts_mdata.instrument
+    unit = ts_mdata.unit
 
-        return (lat, lon, times), param_id, unit, values
+    return (lat, lon, times), param_id, unit, values

api/requirements.in

@@ -5,7 +5,7 @@
 
 grpcio-tools~=1.56
 brotli-asgi~=1.4
-fastapi~=0.103.1
+fastapi~=0.109.2
 gunicorn~=21.2
 uvicorn[standard]~=0.23.2
 covjson-pydantic~=0.2.0

api/requirements.txt

@@ -6,9 +6,8 @@
 #
 annotated-types==0.6.0
     # via pydantic
-anyio==3.7.1
+anyio==4.2.0
     # via
-    #   fastapi
     #   starlette
     #   watchfiles
 brotli==1.1.0
@@ -17,59 +16,61 @@ brotli-asgi==1.4.0
     # via -r requirements.in
 click==8.1.7
     # via uvicorn
-covjson-pydantic==0.2.0
+covjson-pydantic==0.2.1
     # via -r requirements.in
-edr-pydantic==0.2.0
+edr-pydantic==0.2.1
     # via -r requirements.in
-fastapi==0.103.2
+fastapi==0.109.2
     # via -r requirements.in
-geojson-pydantic==1.0.1
+geojson-pydantic==1.0.2
     # via -r requirements.in
-grpcio==1.59.0
+grpcio==1.60.1
     # via grpcio-tools
-grpcio-tools==1.59.0
+grpcio-tools==1.60.1
     # via -r requirements.in
 gunicorn==21.2.0
     # via -r requirements.in
 h11==0.14.0
     # via uvicorn
 httptools==0.6.1
     # via uvicorn
-idna==3.4
+idna==3.6
     # via anyio
-numpy==1.26.1
+numpy==1.26.4
     # via shapely
 packaging==23.2
     # via gunicorn
-protobuf==4.24.4
+protobuf==4.25.2
     # via grpcio-tools
-pydantic==2.4.2
+pydantic==2.6.1
     # via
     #   covjson-pydantic
     #   edr-pydantic
     #   fastapi
     #   geojson-pydantic
-pydantic-core==2.10.1
+pydantic-core==2.16.2
     # via pydantic
-python-dotenv==1.0.0
+python-dotenv==1.0.1
     # via uvicorn
 pyyaml==6.0.1
     # via uvicorn
 shapely==2.0.2
     # via -r requirements.in
 sniffio==1.3.0
     # via anyio
-starlette==0.27.0
+starlette==0.36.3
     # via
     #   brotli-asgi
     #   fastapi
-typing-extensions==4.8.0
+typing-extensions==4.9.0
     # via
     #   fastapi
     #   pydantic
     #   pydantic-core
 uvicorn[standard]==0.23.2
-    # via -r requirements.in
+    # via
+    #   -r requirements.in
+    #   uvicorn
 uvloop==0.19.0
     # via uvicorn
 watchfiles==0.21.0

api/routers/edr.py

@@ -21,8 +21,6 @@
 
 router = APIRouter(prefix="/collections/observations")
 
-edr_formatter = formatters.get_EDR_formatters()
-
 
 @router.get(
     "/locations",
@@ -71,7 +69,7 @@ async def get_data_location_id(
     location_id: Annotated[str, Path(example="06260")],
     parameter_name: Annotated[str | None, Query(alias="parameter-name", example="dd,ff,rh,pp,tn")] = None,
     datetime: Annotated[str | None, Query(example="2022-12-31T00:00Z/2023-01-01T00:00Z")] = None,
-    f: Annotated[str, Query(description="Specify return format.")] = "covjson",
+    f: Annotated[formatters.Formats, Query(description="Specify return format.")] = formatters.Formats.covjson,
 ):
     # TODO: There is no error handling of any kind at the moment!
     #  This is just a quick and dirty demo
@@ -84,7 +82,7 @@ async def get_data_location_id(
         temporal_interval=dstore.TimeInterval(start=range[0], end=range[1]) if range else None,
     )
     response = await getObsRequest(get_obs_request)
-    return edr_formatter[f].convert(response)
+    return formatters.formatters[f](response)
 
 
 @router.get(
@@ -97,7 +95,7 @@ async def get_data_position(
     coords: Annotated[str, Query(example="POINT(5.179705 52.0988218)")],
     parameter_name: Annotated[str | None, Query(alias="parameter-name", example="dd,ff,rh,pp,tn")] = None,
     datetime: Annotated[str | None, Query(example="2022-12-31T00:00Z/2023-01-01T00:00Z")] = None,
-    f: Annotated[str, Query(description="Specify return format.")] = "covjson",
+    f: Annotated[formatters.Formats, Query(description="Specify return format.")] = formatters.Formats.covjson,
 ):
     try:
         point = wkt.loads(coords)
@@ -126,7 +124,7 @@ async def get_data_area(
     coords: Annotated[str, Query(example="POLYGON((5.0 52.0, 6.0 52.0,6.0 52.1,5.0 52.1, 5.0 52.0))")],
     parameter_name: Annotated[str | None, Query(alias="parameter-name", example="dd,ff,rh,pp,tn")] = None,
     datetime: Annotated[str | None, Query(example="2022-12-31T00:00Z/2023-01-01T00:00Z")] = None,
-    f: Annotated[str, Query(description="Specify return format.")] = "covjson",
+    f: Annotated[formatters.Formats, Query(description="Specify return format.")] = formatters.Formats.covjson,
 ):
     try:
         poly = wkt.loads(coords)
@@ -153,5 +151,5 @@ async def get_data_area(
         temporal_interval=dstore.TimeInterval(start=range[0], end=range[1]) if range else None,
     )
     coverages = await getObsRequest(get_obs_request)
-    coverages = edr_formatter[f].convert(coverages)
+    coverages = formatters.formatters[f](coverages)
     return coverages

api/test/test_covjson.py

@@ -4,7 +4,7 @@
 from covjson_pydantic.coverage import Coverage
 from covjson_pydantic.coverage import CoverageCollection
 from fastapi import HTTPException
-from formatters.covjson import Covjson
+from formatters.covjson import convert_to_covjson
 from test.utilities import create_mock_obs_response
 from test.utilities import load_json
 
@@ -14,7 +14,7 @@ def test_single_parameter_convert():
     compare_data = load_json("test/test_data/test_single_covjson.json")
 
     response = create_mock_obs_response(test_data)
-    coverage_collection = Covjson().convert(response)
+    coverage_collection = convert_to_covjson(response)
 
     assert coverage_collection is not None
 
@@ -44,7 +44,7 @@ def test_multiple_parameter_convert():
 
     response = create_mock_obs_response(test_data)
 
-    coverage_collection = Covjson().convert(response)
+    coverage_collection = convert_to_covjson(response)
 
     assert coverage_collection is not None
 
@@ -70,7 +70,7 @@ def test_single_parameter_area_convert():
 
     response = create_mock_obs_response(test_data)
 
-    coverage_collection = Covjson().convert(response)
+    coverage_collection = convert_to_covjson(response)
 
     assert coverage_collection is not None
 
@@ -96,7 +96,7 @@ def test_empty_response_convert():
     # Expect to get an HTTPException with status code of 404 and detail of
     # "No data found" when converting an empty response
     with pytest.raises(HTTPException) as exception_info:
-        Covjson().convert(response)
+        convert_to_covjson(response)
 
     assert exception_info.value.detail == "No data found"
     assert exception_info.value.status_code == 404