Skip to content

Commit

Permalink
Renamed interval & inside to temporal_interval & spatial_area
Browse files Browse the repository at this point in the history
  • Loading branch information
jo-asplin-met-no committed Feb 5, 2024
1 parent 4e89b76 commit 0671176
Show file tree
Hide file tree
Showing 9 changed files with 33 additions and 24 deletions.
12 changes: 8 additions & 4 deletions api/routers/edr.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@ async def get_locations(bbox: str = Query(..., example="5.0,52.0,6.0,52.1")) ->
poly = geometry.Polygon([(left, bottom), (right, bottom), (right, top), (left, top)])
ts_request = dstore.GetObsRequest(
instruments=["tn"], # Hack
inside=dstore.Polygon(points=[dstore.Point(lat=coord[1], lon=coord[0]) for coord in poly.exterior.coords]),
spatial_area=dstore.Polygon(
points=[dstore.Point(lat=coord[1], lon=coord[0]) for coord in poly.exterior.coords]
),
)

ts_response = await getObsRequest(ts_request)
Expand Down Expand Up @@ -70,7 +72,7 @@ async def get_data_location_id(
get_obs_request = dstore.GetObsRequest(
platforms=[location_id],
instruments=list(map(str.strip, parameter_name.split(","))),
interval=dstore.TimeInterval(start=range[0], end=range[1]) if range else None,
temporal_interval=dstore.TimeInterval(start=range[0], end=range[1]) if range else None,
)
response = await getObsRequest(get_obs_request)
return edr_formatter[f].convert(response)
Expand Down Expand Up @@ -111,8 +113,10 @@ async def get_data_area(
range = get_datetime_range(datetime)
get_obs_request = dstore.GetObsRequest(
instruments=list(map(str.strip, parameter_name.split(","))),
inside=dstore.Polygon(points=[dstore.Point(lat=coord[1], lon=coord[0]) for coord in poly.exterior.coords]),
interval=dstore.TimeInterval(start=range[0], end=range[1]) if range else None,
spatial_area=dstore.Polygon(
points=[dstore.Point(lat=coord[1], lon=coord[0]) for coord in poly.exterior.coords]
),
temporal_interval=dstore.TimeInterval(start=range[0], end=range[1]) if range else None,
)
coverages = await getObsRequest(get_obs_request)
coverages = edr_formatter[f].convert(coverages)
Expand Down
8 changes: 4 additions & 4 deletions datastore/datastore/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -215,28 +215,28 @@ $ grpcurl -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.D
### Retrieve observations in a time range

```text
$ grpcurl -d '{"interval": {"start": "2023-01-01T00:00:00Z", "end": "2023-01-01T00:00:10Z"}}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetObservations
$ grpcurl -d '{"temporal_interval": {"start": "2023-01-01T00:00:00Z", "end": "2023-01-01T00:00:10Z"}}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetObservations
...
```

### Retrieve observations in a polygon

```text
$ grpcurl -d '{"inside": {"points": [{"lat": 59.90, "lon": 10.70}, {"lat": 59.90, "lon": 10.80}, {"lat": 60, "lon": 10.80}, {"lat": 60, "lon": 10.70}]}}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetObservations
$ grpcurl -d '{"spatial_area": {"points": [{"lat": 59.90, "lon": 10.70}, {"lat": 59.90, "lon": 10.80}, {"lat": 60, "lon": 10.80}, {"lat": 60, "lon": 10.70}]}}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetObservations
...
```

### Retrieve observations in both a time range and a polygon

```text
$ grpcurl -d '{"interval": {"start": "2023-01-01T00:00:00Z", "end": "2023-01-01T00:00:10Z"}, "inside": {"points": [{"lat": 59.90, "lon": 10.70}, {"lat": 59.90, "lon": 10.80}, {"lat": 60, "lon": 10.80}, {"lat": 60, "lon": 10.70}]}}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetObservations
$ grpcurl -d '{"temporal_interval": {"start": "2023-01-01T00:00:00Z", "end": "2023-01-01T00:00:10Z"}, "spatial_area": {"points": [{"lat": 59.90, "lon": 10.70}, {"lat": 59.90, "lon": 10.80}, {"lat": 60, "lon": 10.80}, {"lat": 60, "lon": 10.70}]}}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetObservations
...
```

### Retrieve wind speed and air temperature observations in a time range and a polygon

```text
$ grpcurl -d '{"standard_names": ["wind_speed", "air_temperature"], "interval": {"start": "2023-01-01T00:00:00Z", "end": "2023-01-01T00:00:10Z"}, "inside": {"points": [{"lat": 59.90, "lon": 10.70}, {"lat": 59.90, "lon": 10.80}, {"lat": 60, "lon": 10.80}, {"lat": 60, "lon": 10.70}]}}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetObservations
$ grpcurl -d '{"standard_names": ["wind_speed", "air_temperature"], "temporal_interval": {"start": "2023-01-01T00:00:00Z", "end": "2023-01-01T00:00:10Z"}, "spatial_area": {"points": [{"lat": 59.90, "lon": 10.70}, {"lat": 59.90, "lon": 10.80}, {"lat": 60, "lon": 10.80}, {"lat": 60, "lon": 10.70}]}}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetObservations
...
```

Expand Down
2 changes: 1 addition & 1 deletion datastore/datastore/dsimpl/getobservations.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ func (svcInfo *ServiceInfo) GetObservations(
*datastore.GetObsResponse, error) {

// do general validation of any obs time interval
if ti := request.Interval; ti != nil {
if ti := request.TemporalInterval; ti != nil {
if ti.Start != nil && ti.End != nil {
if ti.End.AsTime().Before(ti.Start.AsTime()) {
return nil, fmt.Errorf("end(%v) < start(%v)", ti.End, ti.Start)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ func getObs(db *sql.DB, request *datastore.GetObsRequest, obs *[]*datastore.Meta

phVals := []interface{}{} // placeholder values

timeExpr := getTimeFilter(request.GetInterval())
timeExpr := getTimeFilter(request.GetTemporalInterval())

tsMdataExpr := getMdataFilter([]filterInfo{
{"platform", request.GetPlatforms()},
Expand All @@ -239,7 +239,7 @@ func getObs(db *sql.DB, request *datastore.GetObsRequest, obs *[]*datastore.Meta
// TODO: add search filters for more columns in table 'observation'
}, &phVals)

geoExpr, err := getGeoFilter(request.Inside, &phVals)
geoExpr, err := getGeoFilter(request.GetSpatialArea(), &phVals)
if err != nil {
return fmt.Errorf("getGeoFilter() failed: %v", err)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ func getObsTime(obsMdata *datastore.ObsMetadata) (*timestamppb.Timestamp, error)
return nil, fmt.Errorf("obsMdata.GetObstimeInstant()is nil")
}

// --- BEGIN a variant of getObsTime that also supports intervals ---------------------------------
// --- BEGIN a variant of getObsTime that also supports temporal intervals ------------------------
// getObsTime extracts the obs time from obsMdata as either an instant time or the end of
// an interval.
// Returns (obs time, nil) upon success, otherwise (..., error).
Expand All @@ -201,13 +201,14 @@ func getObsTime(obsMdata *datastore.ObsMetadata) (*timestamppb.Timestamp, error)
if obsTime := obsMdata.GetInstant(); obsTime != nil {
return obsTime, nil
}
if obsTime := obsMdata.GetInterval().GetEnd(); obsTime != nil {
if obsTime := obsMdata.GetTemporalInterval().GetEnd(); obsTime != nil {
return obsTime, nil
}
return nil, fmt.Errorf("obsMdata.GetInstant() and obsMdata.GetInterval().GetEnd() are both nil")
return nil, fmt.Errorf(
"obsMdata.GetInstant() and obsMdata.GetTemporalInterval().GetEnd() are both nil")
}
*/
// --- END a variant of getObsTime that also supports intervals ---------------------------------
// --- END a variant of getObsTime that also supports temporal intervals --------------------------

// getGeoPointID retrieves the ID of the row in table geo_point that matches point,
// inserting a new row if necessary. The ID is first looked up in a cache in order to save
Expand Down
4 changes: 2 additions & 2 deletions datastore/examples/clients/python/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def call_put_obs(stub, version, type, standard_name, unit, value):
# obs time range.
def call_get_obs_in_time_range(stub):
request = dstore.GetObsRequest(
interval=dstore.TimeInterval(
temporal_interval=dstore.TimeInterval(
start=dtime2tstamp(datetime(2023, 1, 1, 0, 0, 0, 0, tzinfo=timezone.utc)),
end=dtime2tstamp(datetime(2023, 1, 2, 0, 0, 0, 0, tzinfo=timezone.utc)),
)
Expand All @@ -77,7 +77,7 @@ def call_get_obs_in_polygon(stub):
points.append(dstore.Point(lat=60, lon=10.80))
points.append(dstore.Point(lat=60, lon=10.70))

request = dstore.GetObsRequest(inside=dstore.Polygon(points=points))
request = dstore.GetObsRequest(spatial_area=dstore.Polygon(points=points))
response = stub.GetObservations(request)

return response
Expand Down
6 changes: 4 additions & 2 deletions datastore/integration-test/test_knmi.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,9 @@ def test_get_values_single_station_single_parameter_one_hour(grpc_stub):
end_datetime.FromDatetime(datetime(2022, 12, 31, 12))

ts_request = dstore.GetObsRequest(
platforms=["06260"], instruments=["rh"], interval=dstore.TimeInterval(start=start_datetime, end=end_datetime)
platforms=["06260"],
instruments=["rh"],
temporal_interval=dstore.TimeInterval(start=start_datetime, end=end_datetime),
)
response = grpc_stub.GetObservations(ts_request)

Expand Down Expand Up @@ -147,7 +149,7 @@ def test_get_values_single_station_single_parameter_one_hour(grpc_stub):
@pytest.mark.parametrize("coords,param_ids,expected_station_ids", input_params_polygon)
def test_get_observations_with_polygon(grpc_stub, coords, param_ids, expected_station_ids):
polygon = dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords])
get_obs_request = dstore.GetObsRequest(inside=polygon, instruments=param_ids)
get_obs_request = dstore.GetObsRequest(spatial_area=polygon, instruments=param_ids)
get_obs_response = grpc_stub.GetObservations(get_obs_request)

actual_station_ids = sorted({ts.ts_mdata.platform for ts in get_obs_response.observations})
Expand Down
8 changes: 5 additions & 3 deletions datastore/load-test/locustfile_read.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def get_data_for_single_timeserie(self):
to_time.FromDatetime(datetime(2023, 1, 1))

request = dstore.GetObsRequest(
interval=dstore.TimeInterval(start=from_time, end=to_time),
temporal_interval=dstore.TimeInterval(start=from_time, end=to_time),
platforms=[random.choice(stations)],
instruments=[random.choice(parameters)],
)
Expand All @@ -62,9 +62,11 @@ def get_data_single_station_through_bbox(self):
poly = buffer(point, 0.0001, quad_segs=1) # Roughly 10 meters around the point

request = dstore.GetObsRequest(
interval=dstore.TimeInterval(start=from_time, end=to_time),
temporal_interval=dstore.TimeInterval(start=from_time, end=to_time),
instruments=[random.choice(parameters)],
inside=dstore.Polygon(points=[dstore.Point(lat=coord[1], lon=coord[0]) for coord in poly.exterior.coords]),
spatial_area=dstore.Polygon(
points=[dstore.Point(lat=coord[1], lon=coord[0]) for coord in poly.exterior.coords]
),
)
response = self.stub.GetObservations(request)
assert len(response.observations) == 1
Expand Down
4 changes: 2 additions & 2 deletions protobuf/datastore.proto
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,8 @@ message PutObsResponse {
//---------------------------------------------------------------------------

message GetObsRequest {
TimeInterval interval = 1; // only return observations in this time range
Polygon inside = 2; // if specified, only return observations in this area
TimeInterval temporal_interval = 1; // only return observations in this time range
Polygon spatial_area = 2; // if specified, only return observations inside this area
repeated string platforms = 3; // if specified, only return observations matching any of these platform patterns
repeated string standard_names = 4 [json_name = "standard_names"]; // if specified, only return observations matching any of these standard names
repeated string instruments = 5; // if specified, only return observations matching any of these instruments
Expand Down

1 comment on commit 0671176

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

API Unit Test Coverage Report
FileStmtsMissCoverMissing
\_\_init\_\_.py00100% 
datastore_pb2.py524023%23–62
datastore_pb2_grpc.py43430%2–222
dependencies.py22220%1–33
grpc_getter.py880%1–16
locustfile.py15150%1–31
main.py22220%3–51
metadata_endpoints.py19190%1–70
formatters
   \_\_init\_\_.py12650%16–32
   base_formatter.py7186%18
   covjson.py52198%69
routers
   \_\_init\_\_.py00100% 
   edr.py46460%2–123
   records.py00100% 
test
   test_covjson.py600100% 
TOTAL35822338% 

API Unit Test Coverage Summary

Tests Skipped Failures Errors Time
4 0 💤 0 ❌ 0 🔥 1.001s ⏱️

Please sign in to comment.