Merge branch 'main' into issue-21-parameter-name

api/routers/edr.py

@@ -31,9 +31,10 @@
 # Maybe it would be better to only query a limited set of data instead of everything (meaning 24 hours)
 async def get_locations(bbox: str = Query(..., example="5.0,52.0,6.0,52.1")) -> FeatureCollection:  # Hack to use string
     left, bottom, right, top = map(str.strip, bbox.split(","))
+    print("bbox: {}".format(bbox))
     poly = geometry.Polygon([(left, bottom), (right, bottom), (right, top), (left, top)])
     ts_request = dstore.GetObsRequest(
-        instruments=["tn"],  # Hack
+        filter=dict(instrument=dstore.Strings(values=["tn"])),  # Hack
         inside=dstore.Polygon(points=[dstore.Point(lat=coord[1], lon=coord[0]) for coord in poly.exterior.coords]),
     )
 
@@ -71,7 +72,10 @@ async def get_data_location_id(
     range = get_datetime_range(datetime)
     standard_name, level, func, period = parse_parameter_name(parameter_name)
     get_obs_request = dstore.GetObsRequest(
-        platforms=[location_id],
+        filter=dict(
+            platform=dstore.Strings(values=[location_id]),
+            instrument=dstore.Strings(values=list(map(str.strip, parameter_name.split(",")))),
+        ),
         interval=dstore.TimeInterval(start=range[0], end=range[1]) if range else None,
         standard_name=standard_name,
         level=level,
@@ -117,10 +121,7 @@ async def get_data_area(
     range = get_datetime_range(datetime)
     standard_name, level, func, period = parse_parameter_name(parameter_name)
     get_obs_request = dstore.GetObsRequest(
-        standard_name=standard_name,
-        level=level,
-        func=func,
-        period=period,
+        filter=dict(instrument=dstore.Strings(values=list(map(str.strip, parameter_name.split(","))))),
         inside=dstore.Polygon(points=[dstore.Point(lat=coord[1], lon=coord[0]) for coord in poly.exterior.coords]),
         interval=dstore.TimeInterval(start=range[0], end=range[1]) if range else None,
     )

datastore/datastore/README.md

@@ -141,19 +141,19 @@ go mod tidy
 
 The following environment variables are supported:
 
-Variable | Mandatory | Default value | Description
-:--      | :--       | :--           | :--
-`SERVERPORT`      | No  | `50050`            | Server port number.
-`PGHOST`          | No  | `localhost`        | PostgreSQL host.
-`PGPORT`          | No  | `5433`             | PostgreSQL port number.
-`PGBUSER`         | No  | `postgres`         | PostgreSQL user name.
-`PGPASSWORD`      | No  | `mysecretpassword` | PostgreSQL password.
-`PGDBNAME`        | No  | `data`             | PostgreSQL database name.
-`DYNAMICTIME`     | No  | `true`             | Whether the valid time range is _dynamic_ or _static_ (defined below).
-`LOTIME`          | No  | `86400`            | The _earliest_ valid time as seconds to be either [1] subtracted from the current time (if the valid time range is _dynamic_) or [2] added to UNIX epoch (1970-01-01T00:00:00Z) (if the valid time range is _static_). In the case of a _static_ valid time range, the `LOTIME` can optionally be specified as an ISO-8601 datetime of the exact form `2023-10-10T00:00:00Z`.
-`HITIME`          | No  | `-2`               | Same as `LOTIME`, but for the _latest_ valid time. Note a default leeway of 2 seconds into the future to reduce risk of missing the newest observations.
-`CLEANUPINTERVAL` | No  | `86400`            | The minimum time duration in seconds between automatic cleanups (like removing obsolete observations from the physical store).
-`PUTOBSLIMIT`     | No  | `100000`           | Maximum number of observations allowed in a single call to `PutObservations`.
+| Variable | Mandatory | Default value | Description |
+| :--      | :--       | :--           | :-- |
+| `SERVERPORT`      | No  | `50050`            | Server port number. |
+| `PGHOST`          | No  | `localhost`        | PostgreSQL host. |
+| `PGPORT`          | No  | `5433`             | PostgreSQL port number. |
+| `PGBUSER`         | No  | `postgres`         | PostgreSQL user name. |
+| `PGPASSWORD`      | No  | `mysecretpassword` | PostgreSQL password. |
+| `PGDBNAME`        | No  | `data`             | PostgreSQL database name. |
+| `DYNAMICTIME`     | No  | `true`             | Whether the valid time range is _dynamic_ or _static_ (defined below). |
+| `LOTIME`          | No  | `86400`            | The _earliest_ valid time as seconds to be either [1] subtracted from the current time (if the valid time range is _dynamic_) or [2] added to UNIX epoch (1970-01-01T00:00:00Z) (if the valid time range is _static_). In the case of a _static_ valid time range, the `LOTIME` can optionally be specified as an ISO-8601 datetime of the exact form `2023-10-10T00:00:00Z`. |
+| `HITIME`          | No  | `-2`               | Same as `LOTIME`, but for the _latest_ valid time. Note a default leeway of 2 seconds into the future to reduce risk of missing the newest observations. |
+| `CLEANUPINTERVAL` | No  | `86400`            | The minimum time duration in seconds between automatic cleanups (like removing obsolete observations from the physical store). |
+| `PUTOBSLIMIT`     | No  | `100000`           | Maximum number of observations allowed in a single call to `PutObservations`. |
 
 **TODO:** Ensure that these variables are [passed properly](https://docs.docker.com/compose/environment-variables/set-environment-variables/) to the relevant `docker compose`
 commands. Any secrets should be passed using a [special mechanism](https://docs.docker.com/compose/use-secrets/), etc.
@@ -175,8 +175,9 @@ datastore.Datastore
 $ grpcurl -plaintext -proto protobuf/datastore.proto describe
 datastore.Datastore is a service:
 service Datastore {
-  rpc GetObservations ( .datastore.GetObsRequest ) returns ( .datastore.GetObsResponse );
-  rpc PutObservations ( .datastore.PutObsRequest ) returns ( .datastore.PutObsResponse );
+  // get temporal and spatial extents of current storage contents
+  rpc GetExtents ( .datastore.GetExtentsRequest ) returns ( .datastore.GetExtentsResponse );
+  ...
 }
 ```
 
@@ -185,6 +186,7 @@ service Datastore {
 ```text
 $ grpcurl -plaintext -proto protobuf/datastore.proto describe datastore.Datastore.PutObservations
 datastore.Datastore.PutObservations is a method:
+// insert observations into the storage (or update existing ones)
 rpc PutObservations ( .datastore.PutObsRequest ) returns ( .datastore.PutObsResponse );
 ```
 
@@ -234,9 +236,8 @@ $ grpcurl -d '{"interval": {"start": "2023-01-01T00:00:00Z", "end": "2023-01-01T
 ```
 
 ### Retrieve wind speed and air temperature observations in a time range and a polygon
-
 ```text
-$ grpcurl -d '{"standard_names": ["wind_speed", "air_temperature"], "interval": {"start": "2023-01-01T00:00:00Z", "end": "2023-01-01T00:00:10Z"}, "inside": {"points": [{"lat": 59.90, "lon": 10.70}, {"lat": 59.90, "lon": 10.80}, {"lat": 60, "lon": 10.80}, {"lat": 60, "lon": 10.70}]}}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetObservations
+$ grpcurl -d '{"filter": {"standard_name": {"values": ["wind_speed", "air_temperature"]}}, "interval": {"start": "2023-01-01T00:00:00Z", "end": "2023-01-01T00:00:10Z"}, "inside": {"points": [{"lat": 59.90, "lon": 10.70}, {"lat": 59.90, "lon": 10.80}, {"lat": 60, "lon": 10.80}, {"lat": 60, "lon": 10.70}]}}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetObservations
 ...
 ```