diff --git a/api/formatters/__init__.py b/api/formatters/__init__.py index 55afb0d3..5c8f640c 100644 --- a/api/formatters/__init__.py +++ b/api/formatters/__init__.py @@ -1,32 +1,13 @@ -import importlib import logging -import pkgutil +from enum import Enum -import formatters +from . import covjson logger = logging.getLogger(__name__) -def get_EDR_formatters() -> dict: - """ - This method should grab all available formatters and make them reachable in a dict - This way we can dynamicly grab all available formats and skip configuring this. - Should aliases be made available, and how do one make formatters present in openapi doc? - """ - available_formatters = {} +class Formats(str, Enum): + covjson = "CoverageJSON" # According to EDR spec - formatter_plugins = [ - importlib.import_module("formatters." + i.name) - for i in pkgutil.iter_modules(formatters.__path__) - if i.name != "base_formatter" - ] - logger.info(f"Loaded plugins : {formatter_plugins}") - for formatter_module in formatter_plugins: - # Make instance of formatter and save - available_formatters[formatter_module.__name__.split(".")[-1]] = getattr( - formatter_module, formatter_module.formatter_name - )() - # Should also setup dict for alias discover - - return available_formatters +formatters = {"CoverageJSON": covjson.convert_to_covjson} diff --git a/api/formatters/base_formatter.py b/api/formatters/base_formatter.py deleted file mode 100644 index 59386508..00000000 --- a/api/formatters/base_formatter.py +++ /dev/null @@ -1,18 +0,0 @@ -from abc import ABC -from abc import abstractmethod - - -class EDRFormatter(ABC): - """ - This is the abstract class for implementing a formatter in the E-SOH EDR formatter - Name of class should represent expected output format. - """ - - pass - - @abstractmethod - def convert(self, datastore_reply): - """ - Main method for converting protobuf object to given format. - """ - pass diff --git a/api/formatters/covjson.py b/api/formatters/covjson.py index 5cf69af0..eed9b3ae 100644 --- a/api/formatters/covjson.py +++ b/api/formatters/covjson.py @@ -15,87 +15,77 @@ from covjson_pydantic.reference_system import ReferenceSystemConnectionObject from covjson_pydantic.unit import Unit from fastapi import HTTPException -from formatters.base_formatter import EDRFormatter from pydantic import AwareDatetime -# Requierd for pugin discovery -# Need to be available at top level of formatter plugin -formatter_name = "Covjson" +# mime_type = "application/prs.coverage+json" -class Covjson(EDRFormatter): - """ - Class for converting protobuf object to coverage json - """ - def __init__(self): - self.alias = ["covjson", "coveragejson"] - self.mime_type = "application/json" # find the type for covjson +def convert_to_covjson(response): + # Collect data + coverages = [] + data = [_collect_data(md.ts_mdata, md.obs_mdata) for md in response.observations] - def convert(self, response): - # Collect data - coverages = [] - data = [self._collect_data(md.ts_mdata, md.obs_mdata) for md in response.observations] + # Need to sort before using groupBy. Also sort on param_id to get consistently sorted output + data.sort(key=lambda x: (x[0], x[1])) + # The multiple coverage logic is not needed for this endpoint, + # but we want to share this code between endpoints + for (lat, lon, times), group in groupby(data, lambda x: x[0]): + referencing = [ + ReferenceSystemConnectionObject( + coordinates=["y", "x"], + system=ReferenceSystem(type="GeographicCRS", id="http://www.opengis.net/def/crs/EPSG/0/4326"), + ), + ReferenceSystemConnectionObject( + coordinates=["z"], + system=ReferenceSystem(type="TemporalRS", calendar="Gregorian"), + ), + ] + domain = Domain( + domainType=DomainType.point_series, + axes=Axes( + x=ValuesAxis[float](values=[lon]), + y=ValuesAxis[float](values=[lat]), + t=ValuesAxis[AwareDatetime](values=times), + ), + referencing=referencing, + ) - # Need to sort before using groupBy. Also sort on param_id to get consistently sorted output - data.sort(key=lambda x: (x[0], x[1])) - # The multiple coverage logic is not needed for this endpoint, - # but we want to share this code between endpoints - for (lat, lon, times), group in groupby(data, lambda x: x[0]): - referencing = [ - ReferenceSystemConnectionObject( - coordinates=["y", "x"], - system=ReferenceSystem(type="GeographicCRS", id="http://www.opengis.net/def/crs/EPSG/0/4326"), - ), - ReferenceSystemConnectionObject( - coordinates=["z"], - system=ReferenceSystem(type="TemporalRS", calendar="Gregorian"), - ), - ] - domain = Domain( - domainType=DomainType.point_series, - axes=Axes( - x=ValuesAxis[float](values=[lon]), - y=ValuesAxis[float](values=[lat]), - t=ValuesAxis[AwareDatetime](values=times), - ), - referencing=referencing, + parameters = {} + ranges = {} + for (_, _, _), param_id, unit, values in group: + if all(math.isnan(v) for v in values): + continue # Drop ranges if completely nan. + # TODO: Drop the whole coverage if it becomes empty? + values_no_nan = [v if not math.isnan(v) else None for v in values] + # TODO: Improve this based on "standard name", etc. + parameters[param_id] = Parameter( + observedProperty=ObservedProperty(label={"en": param_id}), unit=Unit(label={"en": unit}) + ) # TODO: Also fill symbol? + ranges[param_id] = NdArray( + values=values_no_nan, axisNames=["t", "y", "x"], shape=[len(values_no_nan), 1, 1] ) - parameters = {} - ranges = {} - for (_, _, _), param_id, unit, values in group: - if all(math.isnan(v) for v in values): - continue # Drop ranges if completely nan. - # TODO: Drop the whole coverage if it becomes empty? - values_no_nan = [v if not math.isnan(v) else None for v in values] - # TODO: Improve this based on "standard name", etc. - parameters[param_id] = Parameter( - observedProperty=ObservedProperty(label={"en": param_id}), unit=Unit(label={"en": unit}) - ) # TODO: Also fill symbol? - ranges[param_id] = NdArray( - values=values_no_nan, axisNames=["t", "y", "x"], shape=[len(values_no_nan), 1, 1] - ) + coverages.append(Coverage(domain=domain, parameters=parameters, ranges=ranges)) - coverages.append(Coverage(domain=domain, parameters=parameters, ranges=ranges)) + if len(coverages) == 0: + raise HTTPException(status_code=404, detail="No data found") + elif len(coverages) == 1: + return coverages[0] + else: + return CoverageCollection( + coverages=coverages, parameters=coverages[0].parameters + ) # HACK to take parameters from first one - if len(coverages) == 0: - raise HTTPException(status_code=404, detail="No data found") - elif len(coverages) == 1: - return coverages[0] - else: - return CoverageCollection( - coverages=coverages, parameters=coverages[0].parameters - ) # HACK to take parameters from first one - def _collect_data(self, ts_mdata, obs_mdata): - lat = obs_mdata[0].geo_point.lat # HACK: For now assume they all have the same position - lon = obs_mdata[0].geo_point.lon - tuples = ( - (o.obstime_instant.ToDatetime(tzinfo=timezone.utc), float(o.value)) for o in obs_mdata - ) # HACK: str -> float - (times, values) = zip(*tuples) - param_id = ts_mdata.instrument - unit = ts_mdata.unit +def _collect_data(ts_mdata, obs_mdata): + lat = obs_mdata[0].geo_point.lat # HACK: For now assume they all have the same position + lon = obs_mdata[0].geo_point.lon + tuples = ( + (o.obstime_instant.ToDatetime(tzinfo=timezone.utc), float(o.value)) for o in obs_mdata + ) # HACK: str -> float + (times, values) = zip(*tuples) + param_id = ts_mdata.instrument + unit = ts_mdata.unit - return (lat, lon, times), param_id, unit, values + return (lat, lon, times), param_id, unit, values diff --git a/api/requirements.in b/api/requirements.in index 2c3ecea2..7ef74df0 100644 --- a/api/requirements.in +++ b/api/requirements.in @@ -5,7 +5,7 @@ grpcio-tools~=1.56 brotli-asgi~=1.4 -fastapi~=0.103.1 +fastapi~=0.109.2 gunicorn~=21.2 uvicorn[standard]~=0.23.2 covjson-pydantic~=0.2.0 diff --git a/api/requirements.txt b/api/requirements.txt index 91c39dc2..222d11b9 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -6,9 +6,8 @@ # annotated-types==0.6.0 # via pydantic -anyio==3.7.1 +anyio==4.2.0 # via - # fastapi # starlette # watchfiles brotli==1.1.0 @@ -17,17 +16,17 @@ brotli-asgi==1.4.0 # via -r requirements.in click==8.1.7 # via uvicorn -covjson-pydantic==0.2.0 +covjson-pydantic==0.2.1 # via -r requirements.in -edr-pydantic==0.2.0 +edr-pydantic==0.2.1 # via -r requirements.in -fastapi==0.103.2 +fastapi==0.109.2 # via -r requirements.in -geojson-pydantic==1.0.1 +geojson-pydantic==1.0.2 # via -r requirements.in -grpcio==1.59.0 +grpcio==1.60.1 # via grpcio-tools -grpcio-tools==1.59.0 +grpcio-tools==1.60.1 # via -r requirements.in gunicorn==21.2.0 # via -r requirements.in @@ -35,23 +34,23 @@ h11==0.14.0 # via uvicorn httptools==0.6.1 # via uvicorn -idna==3.4 +idna==3.6 # via anyio -numpy==1.26.1 +numpy==1.26.4 # via shapely packaging==23.2 # via gunicorn -protobuf==4.24.4 +protobuf==4.25.2 # via grpcio-tools -pydantic==2.4.2 +pydantic==2.6.1 # via # covjson-pydantic # edr-pydantic # fastapi # geojson-pydantic -pydantic-core==2.10.1 +pydantic-core==2.16.2 # via pydantic -python-dotenv==1.0.0 +python-dotenv==1.0.1 # via uvicorn pyyaml==6.0.1 # via uvicorn @@ -59,17 +58,19 @@ shapely==2.0.2 # via -r requirements.in sniffio==1.3.0 # via anyio -starlette==0.27.0 +starlette==0.36.3 # via # brotli-asgi # fastapi -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via # fastapi # pydantic # pydantic-core uvicorn[standard]==0.23.2 - # via -r requirements.in + # via + # -r requirements.in + # uvicorn uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 diff --git a/api/routers/edr.py b/api/routers/edr.py index 910d3b0c..58a909a2 100644 --- a/api/routers/edr.py +++ b/api/routers/edr.py @@ -21,8 +21,6 @@ router = APIRouter(prefix="/collections/observations") -edr_formatter = formatters.get_EDR_formatters() - @router.get( "/locations", @@ -71,7 +69,7 @@ async def get_data_location_id( location_id: Annotated[str, Path(example="06260")], parameter_name: Annotated[str | None, Query(alias="parameter-name", example="dd,ff,rh,pp,tn")] = None, datetime: Annotated[str | None, Query(example="2022-12-31T00:00Z/2023-01-01T00:00Z")] = None, - f: Annotated[str, Query(description="Specify return format.")] = "covjson", + f: Annotated[formatters.Formats, Query(description="Specify return format.")] = formatters.Formats.covjson, ): # TODO: There is no error handling of any kind at the moment! # This is just a quick and dirty demo @@ -84,7 +82,7 @@ async def get_data_location_id( temporal_interval=dstore.TimeInterval(start=range[0], end=range[1]) if range else None, ) response = await getObsRequest(get_obs_request) - return edr_formatter[f].convert(response) + return formatters.formatters[f](response) @router.get( @@ -97,7 +95,7 @@ async def get_data_position( coords: Annotated[str, Query(example="POINT(5.179705 52.0988218)")], parameter_name: Annotated[str | None, Query(alias="parameter-name", example="dd,ff,rh,pp,tn")] = None, datetime: Annotated[str | None, Query(example="2022-12-31T00:00Z/2023-01-01T00:00Z")] = None, - f: Annotated[str, Query(description="Specify return format.")] = "covjson", + f: Annotated[formatters.Formats, Query(description="Specify return format.")] = formatters.Formats.covjson, ): try: point = wkt.loads(coords) @@ -126,7 +124,7 @@ async def get_data_area( coords: Annotated[str, Query(example="POLYGON((5.0 52.0, 6.0 52.0,6.0 52.1,5.0 52.1, 5.0 52.0))")], parameter_name: Annotated[str | None, Query(alias="parameter-name", example="dd,ff,rh,pp,tn")] = None, datetime: Annotated[str | None, Query(example="2022-12-31T00:00Z/2023-01-01T00:00Z")] = None, - f: Annotated[str, Query(description="Specify return format.")] = "covjson", + f: Annotated[formatters.Formats, Query(description="Specify return format.")] = formatters.Formats.covjson, ): try: poly = wkt.loads(coords) @@ -153,5 +151,5 @@ async def get_data_area( temporal_interval=dstore.TimeInterval(start=range[0], end=range[1]) if range else None, ) coverages = await getObsRequest(get_obs_request) - coverages = edr_formatter[f].convert(coverages) + coverages = formatters.formatters[f](coverages) return coverages diff --git a/api/test/test_covjson.py b/api/test/test_covjson.py index df6eb6ca..50f780c0 100644 --- a/api/test/test_covjson.py +++ b/api/test/test_covjson.py @@ -4,7 +4,7 @@ from covjson_pydantic.coverage import Coverage from covjson_pydantic.coverage import CoverageCollection from fastapi import HTTPException -from formatters.covjson import Covjson +from formatters.covjson import convert_to_covjson from test.utilities import create_mock_obs_response from test.utilities import load_json @@ -14,7 +14,7 @@ def test_single_parameter_convert(): compare_data = load_json("test/test_data/test_single_covjson.json") response = create_mock_obs_response(test_data) - coverage_collection = Covjson().convert(response) + coverage_collection = convert_to_covjson(response) assert coverage_collection is not None @@ -44,7 +44,7 @@ def test_multiple_parameter_convert(): response = create_mock_obs_response(test_data) - coverage_collection = Covjson().convert(response) + coverage_collection = convert_to_covjson(response) assert coverage_collection is not None @@ -70,7 +70,7 @@ def test_single_parameter_area_convert(): response = create_mock_obs_response(test_data) - coverage_collection = Covjson().convert(response) + coverage_collection = convert_to_covjson(response) assert coverage_collection is not None @@ -96,7 +96,7 @@ def test_empty_response_convert(): # Expect to get an HTTPException with status code of 404 and detail of # "No data found" when converting an empty response with pytest.raises(HTTPException) as exception_info: - Covjson().convert(response) + convert_to_covjson(response) assert exception_info.value.detail == "No data found" assert exception_info.value.status_code == 404 diff --git a/api/test/test_edr.py b/api/test/test_edr.py index cac854f4..c24a3490 100644 --- a/api/test/test_edr.py +++ b/api/test/test_edr.py @@ -11,7 +11,7 @@ def test_get_locations_id_with_single_parameter_query_without_format(): - with patch("routers.edr.getObsRequest") as mock_getObsRequest: + with patch("routers.edr.getObsRequest") as mock_getObsRequest: # noqa: N806 test_data = load_json("test/test_data/test_single_proto.json") compare_data = load_json("test/test_data/test_single_covjson.json") @@ -37,13 +37,13 @@ def test_get_locations_id_with_single_parameter_query_without_format(): def test_get_locations_id_without_parameter_names_query(): - with patch("routers.edr.getObsRequest") as mock_getObsRequest: + with patch("routers.edr.getObsRequest") as mock_getObsRequest: # noqa: N806 test_data = load_json("test/test_data/test_multiple_proto.json") compare_data = load_json("test/test_data/test_multiple_covjson.json") mock_getObsRequest.return_value = create_mock_obs_response(test_data) - response = client.get("/collections/observations/locations/06260?f=covjson") + response = client.get("/collections/observations/locations/06260?f=CoverageJSON") # Check that getObsRequest gets called with correct arguments when no parameter names are given # in query @@ -73,19 +73,19 @@ def test_get_locations_id_with_incorrect_datetime_range(): def test_get_locations_id_with_empty_response(): - with patch("routers.edr.getObsRequest") as mock_getObsRequest: + with patch("routers.edr.getObsRequest") as mock_getObsRequest: # noqa: N806 test_data = load_json("test/test_data/test_empty_proto.json") mock_getObsRequest.return_value = create_mock_obs_response(test_data) - response = client.get("/collections/observations/locations/10000?f=covjson") + response = client.get("/collections/observations/locations/10000?f=CoverageJSON") assert response.status_code == 404 assert response.json() == {"detail": "No data found"} def test_get_area_with_normal_query(): - with patch("routers.edr.getObsRequest") as mock_getObsRequest: + with patch("routers.edr.getObsRequest") as mock_getObsRequest: # noqa: N806 test_data = load_json("test/test_data/test_coverages_proto.json") compare_data = load_json("test/test_data/test_coverages_covjson.json") @@ -112,7 +112,7 @@ def test_get_area_with_normal_query(): def test_get_area_with_without_parameter_names_query(): - with patch("routers.edr.getObsRequest") as mock_getObsRequest: + with patch("routers.edr.getObsRequest") as mock_getObsRequest: # noqa: N806 test_data = load_json("test/test_data/test_coverages_proto.json") compare_data = load_json("test/test_data/test_coverages_covjson.json") @@ -151,7 +151,7 @@ def test_get_position_with_normal_query(): # Wrap the original get_data_area to a mock so we can assert against the call values with patch("routers.edr.get_data_area", wraps=edr.get_data_area) as mock_get_data_area, patch( "routers.edr.getObsRequest" - ) as mock_getObsRequest: + ) as mock_getObsRequest: # noqa: N806 test_data = load_json("test/test_data/test_coverages_proto.json") compare_data = load_json("test/test_data/test_coverages_covjson.json") @@ -168,7 +168,7 @@ def test_get_position_with_normal_query(): "5.179705 52.09892180000001, 5.179805 52.0988218))", "TA_P1D_AVG", "2022-12-31T00:00Z/2022-12-31T00:00Z", - "covjson", + "CoverageJSON", ) assert response.status_code == 200