From 07ef5d63f6247768ea087c370b94eb1332a75da0 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 13 Sep 2023 12:05:42 +0200 Subject: [PATCH 01/14] Initial delete test. --- integration-test/test_fake_data.py | 76 ++++++++++++++++++++++++++++++ integration-test/test_knmi.py | 23 +++------ 2 files changed, 82 insertions(+), 17 deletions(-) create mode 100644 integration-test/test_fake_data.py diff --git a/integration-test/test_fake_data.py b/integration-test/test_fake_data.py new file mode 100644 index 0000000..d2d599b --- /dev/null +++ b/integration-test/test_fake_data.py @@ -0,0 +1,76 @@ +# Note that this assumes that the KNMI test data is loader (using loader container) +import os +from datetime import datetime +from datetime import timezone + +import datastore_pb2 as dstore +import datastore_pb2_grpc as dstore_grpc +import grpc +import pytest +from google.protobuf.timestamp_pb2 import Timestamp + + +@pytest.fixture +def grpc_stub(): + with grpc.insecure_channel(f"{os.getenv('DSHOST', 'localhost')}:{os.getenv('DSPORT', '50050')}") as channel: + yield dstore_grpc.DatastoreStub(channel) + + +def test_delete_timeseries(grpc_stub): + test_value = 999999999 + test_value_str = str(test_value) + ts_metadata = dstore.TSMetadata( + station_id=test_value_str, + param_id=test_value_str, + pos=dstore.Point(lat=9999.9999, lon=9999.9999), + other1="test_value1", + other2="test_value2", + other3="test_value3", + ) + ts_add_request = dstore.AddTSRequest( + id=test_value, + metadata=ts_metadata, + ) + ts_add_response = grpc_stub.AddTimeSeries(ts_add_request) + assert str(ts_add_response) == "status: -1\n" + + obs_metadata = dstore.ObsMetadata(field1="test_value1", field2="test_value2") + time_1 = Timestamp() + time_1.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=9, tzinfo=timezone.utc)) + time_2 = Timestamp() + time_2.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=10, tzinfo=timezone.utc)) + time_3 = Timestamp() + time_3.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=11, tzinfo=timezone.utc)) + obs = [ + dstore.Observation(time=time_1, value=test_value, metadata=obs_metadata), + dstore.Observation(time=time_2, value=test_value, metadata=obs_metadata), + dstore.Observation(time=time_3, value=test_value, metadata=obs_metadata), + ] + obs_put_request = dstore.PutObsRequest( + tsobs=[dstore.TSObservations(tsid=test_value, obs=obs)], + ) + obs_response = grpc_stub.PutObservations(obs_put_request) + assert str(obs_response) == "status: -1\n" + + ts_find_request = dstore.FindTSRequest(station_ids=[test_value_str], param_ids=[test_value_str]) + ts_find_response = grpc_stub.FindTimeSeries(ts_find_request) + assert len(ts_find_response.tseries) == 1 + assert ts_find_response.tseries[0].id == test_value + + to_time = Timestamp() + to_time.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=11, second=1)) + obs_get_request = dstore.GetObsRequest(tsids=[test_value], fromtime=time_1, totime=to_time) + obs_get_response = grpc_stub.GetObservations(obs_get_request) + assert obs_get_response.tsobs[0].tsid == test_value + assert len(obs_get_response.tsobs[0].obs) == 3 + + ts_delete_request = dstore.DeleteTSRequest(ids=[test_value]) + ts_delete_response = grpc_stub.DeleteTimeSeries(ts_delete_request) + assert str(ts_delete_response) == "status: -1\n" + + ts_find_response = grpc_stub.FindTimeSeries(ts_find_request) + assert len(ts_find_response.tseries) == 0 + + obs_get_response = grpc_stub.GetObservations(obs_get_request) + assert obs_get_response.tsobs[0].tsid == test_value + assert len(obs_get_response.tsobs[0].obs) == 0 diff --git a/integration-test/test_knmi.py b/integration-test/test_knmi.py index ccf63b8..bb34a3d 100644 --- a/integration-test/test_knmi.py +++ b/integration-test/test_knmi.py @@ -2,14 +2,11 @@ import os from datetime import datetime - -import pytest - -from google.protobuf.timestamp_pb2 import Timestamp - import datastore_pb2 as dstore import datastore_pb2_grpc as dstore_grpc import grpc +import pytest +from google.protobuf.timestamp_pb2 import Timestamp @pytest.fixture @@ -19,10 +16,7 @@ def grpc_stub(): def test_find_series_single_station_single_parameter(grpc_stub): - request = dstore.FindTSRequest( - station_ids=["06260"], - param_ids=["rh"] - ) + request = dstore.FindTSRequest(station_ids=["06260"], param_ids=["rh"]) response = grpc_stub.FindTimeSeries(request) assert len(response.tseries) == 1 @@ -31,9 +25,7 @@ def test_find_series_single_station_single_parameter(grpc_stub): def test_find_series_all_stations_single_parameter(grpc_stub): - request = dstore.FindTSRequest( - param_ids=["rh"] - ) + request = dstore.FindTSRequest(param_ids=["rh"]) response = grpc_stub.FindTimeSeries(request) assert len(response.tseries) == 55 @@ -48,11 +40,8 @@ def test_find_series_single_station_all_parameters(grpc_stub): assert len(response.tseries) == 44 -def test_get_values_single_station_single_paramters(grpc_stub): - ts_request = dstore.FindTSRequest( - station_ids=["06260"], - param_ids=["rh"] - ) +def test_get_values_single_station_single_parameters(grpc_stub): + ts_request = dstore.FindTSRequest(station_ids=["06260"], param_ids=["rh"]) ts_response = grpc_stub.FindTimeSeries(ts_request) assert len(ts_response.tseries) == 1 ts_id = ts_response.tseries[0].id From 015a373972fcfc03ae92278fdccd01d87c4a59fc Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 13 Sep 2023 12:28:00 +0200 Subject: [PATCH 02/14] Move dummy data to functions. --- integration-test/test_fake_data.py | 51 +++++++++++++++++++----------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/integration-test/test_fake_data.py b/integration-test/test_fake_data.py index d2d599b..3b53e68 100644 --- a/integration-test/test_fake_data.py +++ b/integration-test/test_fake_data.py @@ -16,24 +16,23 @@ def grpc_stub(): yield dstore_grpc.DatastoreStub(channel) -def test_delete_timeseries(grpc_stub): - test_value = 999999999 - test_value_str = str(test_value) +def timeseries_add_request(fake_id, fake_str_id): ts_metadata = dstore.TSMetadata( - station_id=test_value_str, - param_id=test_value_str, + station_id=fake_str_id, + param_id=fake_str_id, pos=dstore.Point(lat=9999.9999, lon=9999.9999), other1="test_value1", other2="test_value2", other3="test_value3", ) ts_add_request = dstore.AddTSRequest( - id=test_value, + id=fake_id, metadata=ts_metadata, ) - ts_add_response = grpc_stub.AddTimeSeries(ts_add_request) - assert str(ts_add_response) == "status: -1\n" + return ts_add_request + +def observations_put_request(fake_id): obs_metadata = dstore.ObsMetadata(field1="test_value1", field2="test_value2") time_1 = Timestamp() time_1.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=9, tzinfo=timezone.utc)) @@ -42,29 +41,43 @@ def test_delete_timeseries(grpc_stub): time_3 = Timestamp() time_3.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=11, tzinfo=timezone.utc)) obs = [ - dstore.Observation(time=time_1, value=test_value, metadata=obs_metadata), - dstore.Observation(time=time_2, value=test_value, metadata=obs_metadata), - dstore.Observation(time=time_3, value=test_value, metadata=obs_metadata), + dstore.Observation(time=time_1, value=1111.1111, metadata=obs_metadata), + dstore.Observation(time=time_2, value=2222.2222, metadata=obs_metadata), + dstore.Observation(time=time_3, value=3333.3333, metadata=obs_metadata), ] obs_put_request = dstore.PutObsRequest( - tsobs=[dstore.TSObservations(tsid=test_value, obs=obs)], + tsobs=[dstore.TSObservations(tsid=fake_id, obs=obs)], ) + return obs_put_request + + +def test_delete_timeseries(grpc_stub): + fake_id = 999999999 + fake_str_id = "999999999" + ts_add_request = timeseries_add_request(fake_id=fake_id, fake_str_id=fake_str_id) + obs_put_request = observations_put_request(fake_id=fake_id) + + ts_add_response = grpc_stub.AddTimeSeries(ts_add_request) + assert str(ts_add_response) == "status: -1\n" + obs_response = grpc_stub.PutObservations(obs_put_request) assert str(obs_response) == "status: -1\n" - ts_find_request = dstore.FindTSRequest(station_ids=[test_value_str], param_ids=[test_value_str]) + ts_find_request = dstore.FindTSRequest(station_ids=[fake_str_id], param_ids=[fake_str_id]) ts_find_response = grpc_stub.FindTimeSeries(ts_find_request) assert len(ts_find_response.tseries) == 1 - assert ts_find_response.tseries[0].id == test_value + assert ts_find_response.tseries[0].id == fake_id to_time = Timestamp() - to_time.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=11, second=1)) - obs_get_request = dstore.GetObsRequest(tsids=[test_value], fromtime=time_1, totime=to_time) + to_time.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=11, second=1, tzinfo=timezone.utc)) + obs_get_request = dstore.GetObsRequest( + tsids=[fake_id], fromtime=obs_put_request.tsobs[0].obs[0].time, totime=to_time + ) obs_get_response = grpc_stub.GetObservations(obs_get_request) - assert obs_get_response.tsobs[0].tsid == test_value + assert obs_get_response.tsobs[0].tsid == fake_id assert len(obs_get_response.tsobs[0].obs) == 3 - ts_delete_request = dstore.DeleteTSRequest(ids=[test_value]) + ts_delete_request = dstore.DeleteTSRequest(ids=[fake_id]) ts_delete_response = grpc_stub.DeleteTimeSeries(ts_delete_request) assert str(ts_delete_response) == "status: -1\n" @@ -72,5 +85,5 @@ def test_delete_timeseries(grpc_stub): assert len(ts_find_response.tseries) == 0 obs_get_response = grpc_stub.GetObservations(obs_get_request) - assert obs_get_response.tsobs[0].tsid == test_value + assert obs_get_response.tsobs[0].tsid == fake_id assert len(obs_get_response.tsobs[0].obs) == 0 From ca8339b925f19c222cb0bdd2d6aea2cb14751fb4 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Mon, 18 Sep 2023 15:40:26 +0200 Subject: [PATCH 03/14] Add integration test for find timeseries with a polygon. --- .../timescaledb/findtimeseries.go | 5 +- integration-test/test_fake_data.py | 115 +++++++++++++++++- 2 files changed, 113 insertions(+), 7 deletions(-) diff --git a/datastore/storagebackend/timescaledb/findtimeseries.go b/datastore/storagebackend/timescaledb/findtimeseries.go index b3e89f7..9ee504e 100644 --- a/datastore/storagebackend/timescaledb/findtimeseries.go +++ b/datastore/storagebackend/timescaledb/findtimeseries.go @@ -69,7 +69,7 @@ func insidePolygonCond( return fmt.Sprintf( " AND ST_DWITHIN(%s, ST_GeomFromText($%d, %s)::geography, 0.0)", - name, srid, len(*phVals), srid), nil + name, len(*phVals), srid), nil } // FindTimeSeries ... (see documentation in StorageBackend interface) @@ -94,6 +94,9 @@ func (sbe *TimescaleDB) FindTimeSeries(request *datastore.FindTSRequest) ( // TODO: add more filters + fmt.Println("Query: ", query) + fmt.Println("phVals: ", phVals) + rows, err := sbe.Db.Query(query, phVals...) if err != nil { return nil, fmt.Errorf("sbe.Db.Query() failed: %v", err) diff --git a/integration-test/test_fake_data.py b/integration-test/test_fake_data.py index 3b53e68..75d78e4 100644 --- a/integration-test/test_fake_data.py +++ b/integration-test/test_fake_data.py @@ -1,4 +1,5 @@ # Note that this assumes that the KNMI test data is loader (using loader container) +import logging import os from datetime import datetime from datetime import timezone @@ -10,17 +11,33 @@ from google.protobuf.timestamp_pb2 import Timestamp -@pytest.fixture +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +def _handle_grpc_error(func, *args, **kwargs): + try: + return func(*args, **kwargs) + except grpc.RpcError as rpc_error: + for error in rpc_error.args: + if error.details.endswith("already exists"): + logger.info(error) + else: + logger.error(rpc_error) + raise rpc_error + + +@pytest.fixture(scope="session") def grpc_stub(): with grpc.insecure_channel(f"{os.getenv('DSHOST', 'localhost')}:{os.getenv('DSPORT', '50050')}") as channel: yield dstore_grpc.DatastoreStub(channel) -def timeseries_add_request(fake_id, fake_str_id): +def dummy_timeseries_for_delete(fake_id, fake_str_id): ts_metadata = dstore.TSMetadata( station_id=fake_str_id, param_id=fake_str_id, - pos=dstore.Point(lat=9999.9999, lon=9999.9999), + pos=dstore.Point(lat=-60, lon=-160), other1="test_value1", other2="test_value2", other3="test_value3", @@ -32,7 +49,7 @@ def timeseries_add_request(fake_id, fake_str_id): return ts_add_request -def observations_put_request(fake_id): +def dummy_observations_for_delete(fake_id): obs_metadata = dstore.ObsMetadata(field1="test_value1", field2="test_value2") time_1 = Timestamp() time_1.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=9, tzinfo=timezone.utc)) @@ -54,8 +71,8 @@ def observations_put_request(fake_id): def test_delete_timeseries(grpc_stub): fake_id = 999999999 fake_str_id = "999999999" - ts_add_request = timeseries_add_request(fake_id=fake_id, fake_str_id=fake_str_id) - obs_put_request = observations_put_request(fake_id=fake_id) + ts_add_request = dummy_timeseries_for_delete(fake_id=fake_id, fake_str_id=fake_str_id) + obs_put_request = dummy_observations_for_delete(fake_id=fake_id) ts_add_response = grpc_stub.AddTimeSeries(ts_add_request) assert str(ts_add_response) == "status: -1\n" @@ -87,3 +104,89 @@ def test_delete_timeseries(grpc_stub): obs_get_response = grpc_stub.GetObservations(obs_get_request) assert obs_get_response.tsobs[0].tsid == fake_id assert len(obs_get_response.tsobs[0].obs) == 0 + + +@pytest.fixture(scope="module") +def setup_fake_data_for_polygon(grpc_stub): + fake_id = 999999990 + fake_str_id = "999999990" + ts_metadata = dstore.TSMetadata( + station_id=fake_str_id, + param_id=fake_str_id, + pos=dstore.Point(lat=80, lon=170), + other1="test_value1", + other2="test_value2", + other3="test_value3", + ) + ts_add_request_1 = dstore.AddTSRequest( + id=fake_id, + metadata=ts_metadata, + ) + # grpc_stub.AddTimeSeries(ts_add_request_1) + _handle_grpc_error(grpc_stub.AddTimeSeries, ts_add_request_1) + + fake_id = 999999991 + fake_str_id = "999999991" + ts_metadata = dstore.TSMetadata( + station_id=fake_str_id, + param_id=fake_str_id, + pos=dstore.Point(lat=90, lon=180), + other1="test_value4", + other2="test_value5", + other3="test_value6", + ) + ts_add_request_2 = dstore.AddTSRequest( + id=fake_id, + metadata=ts_metadata, + ) + # grpc_stub.AddTimeSeries(ts_add_request_2) + _handle_grpc_error(grpc_stub.AddTimeSeries, ts_add_request_2) + + yield # Hereafter teardown + + grpc_stub.DeleteTimeSeries(dstore.DeleteTSRequest(ids=[ts_add_request_1.id, ts_add_request_2.id])) + + +input_params_polygon = [ + ( + # Multiple stations within + ((80, 170), (90, 170), (90, 180), (80, 180)), + {"number_of_stations": 2, "station_ids": [999999990, 999999991]}, + ), + ( + # One station within + ((80, 175), (90, 175), (90, 180), (80, 180)), + {"number_of_stations": 1, "station_ids": [999999991]}, + ), + ( + # Nothing within + ((80, 175), (85, 175), (85, 180), (80, 180)), + {"number_of_stations": 0, "station_ids": []}, + ), + ( + # middle top + ((85, 120), (90, 120), (90, -120), (85, -120)), + {"number_of_stations": 1, "station_ids": [999999991]}, + ), + ( + # Middle bottom, should fall outside since polygon is curved because the earth is round (postgres geography). + ((85, 120), (80, 120), (85, -120), (80, -120)), + {"number_of_stations": 0, "station_ids": []}, + ), + ( + # Complex polygon + ((79, 169), (81, 169), (80, 171), (89, 171), (89, 180), (75, 180), (75, 171)), + {"number_of_stations": 1, "station_ids": [999999990]}, + ), +] + + +@pytest.mark.parametrize("coords,expected", input_params_polygon) +def test_get_observations_with_polygon(grpc_stub, setup_fake_data_for_polygon, coords, expected): + ts_request = dstore.FindTSRequest( + inside=dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords]) + ) + ts_response = grpc_stub.FindTimeSeries(ts_request) + + assert len(ts_response.tseries) == expected["number_of_stations"] + assert sorted(ts.id for ts in ts_response.tseries) == expected["station_ids"] From 97f6da5087bfa16e26f0f9c270e25cd61c50794c Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 20 Sep 2023 09:23:22 +0200 Subject: [PATCH 04/14] Changed functions to fixtures for the delete. --- integration-test/test_fake_data.py | 71 ++++++++++++++++-------------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/integration-test/test_fake_data.py b/integration-test/test_fake_data.py index 75d78e4..d9da8b0 100644 --- a/integration-test/test_fake_data.py +++ b/integration-test/test_fake_data.py @@ -33,23 +33,30 @@ def grpc_stub(): yield dstore_grpc.DatastoreStub(channel) -def dummy_timeseries_for_delete(fake_id, fake_str_id): +@pytest.fixture(scope="function") +def dummy_timeseries_for_delete(): + dummy_id = 999999999 + dummy_str_id = "999999999" + ts_metadata = dstore.TSMetadata( - station_id=fake_str_id, - param_id=fake_str_id, + station_id=dummy_str_id, + param_id=dummy_str_id, pos=dstore.Point(lat=-60, lon=-160), other1="test_value1", other2="test_value2", other3="test_value3", ) ts_add_request = dstore.AddTSRequest( - id=fake_id, + id=dummy_id, metadata=ts_metadata, ) return ts_add_request -def dummy_observations_for_delete(fake_id): +@pytest.fixture(scope="function") +def dummy_observations_for_delete(): + dummy_id = 999999999 + obs_metadata = dstore.ObsMetadata(field1="test_value1", field2="test_value2") time_1 = Timestamp() time_1.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=9, tzinfo=timezone.utc)) @@ -63,38 +70,38 @@ def dummy_observations_for_delete(fake_id): dstore.Observation(time=time_3, value=3333.3333, metadata=obs_metadata), ] obs_put_request = dstore.PutObsRequest( - tsobs=[dstore.TSObservations(tsid=fake_id, obs=obs)], + tsobs=[dstore.TSObservations(tsid=dummy_id, obs=obs)], ) return obs_put_request -def test_delete_timeseries(grpc_stub): - fake_id = 999999999 - fake_str_id = "999999999" - ts_add_request = dummy_timeseries_for_delete(fake_id=fake_id, fake_str_id=fake_str_id) - obs_put_request = dummy_observations_for_delete(fake_id=fake_id) - - ts_add_response = grpc_stub.AddTimeSeries(ts_add_request) +def test_delete_timeseries(grpc_stub, dummy_timeseries_for_delete, dummy_observations_for_delete): + ts_add_response = grpc_stub.AddTimeSeries(dummy_timeseries_for_delete) assert str(ts_add_response) == "status: -1\n" - obs_response = grpc_stub.PutObservations(obs_put_request) + obs_response = grpc_stub.PutObservations(dummy_observations_for_delete) assert str(obs_response) == "status: -1\n" - ts_find_request = dstore.FindTSRequest(station_ids=[fake_str_id], param_ids=[fake_str_id]) + ts_find_request = dstore.FindTSRequest( + station_ids=[dummy_timeseries_for_delete.metadata.station_id], + param_ids=[dummy_timeseries_for_delete.metadata.param_id], + ) ts_find_response = grpc_stub.FindTimeSeries(ts_find_request) assert len(ts_find_response.tseries) == 1 - assert ts_find_response.tseries[0].id == fake_id + assert ts_find_response.tseries[0].id == dummy_timeseries_for_delete.id to_time = Timestamp() to_time.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=11, second=1, tzinfo=timezone.utc)) obs_get_request = dstore.GetObsRequest( - tsids=[fake_id], fromtime=obs_put_request.tsobs[0].obs[0].time, totime=to_time + tsids=[dummy_timeseries_for_delete.id], + fromtime=dummy_observations_for_delete.tsobs[0].obs[0].time, + totime=to_time, ) obs_get_response = grpc_stub.GetObservations(obs_get_request) - assert obs_get_response.tsobs[0].tsid == fake_id + assert obs_get_response.tsobs[0].tsid == dummy_timeseries_for_delete.id assert len(obs_get_response.tsobs[0].obs) == 3 - ts_delete_request = dstore.DeleteTSRequest(ids=[fake_id]) + ts_delete_request = dstore.DeleteTSRequest(ids=[dummy_timeseries_for_delete.id]) ts_delete_response = grpc_stub.DeleteTimeSeries(ts_delete_request) assert str(ts_delete_response) == "status: -1\n" @@ -102,41 +109,41 @@ def test_delete_timeseries(grpc_stub): assert len(ts_find_response.tseries) == 0 obs_get_response = grpc_stub.GetObservations(obs_get_request) - assert obs_get_response.tsobs[0].tsid == fake_id + assert obs_get_response.tsobs[0].tsid == dummy_timeseries_for_delete.id assert len(obs_get_response.tsobs[0].obs) == 0 @pytest.fixture(scope="module") -def setup_fake_data_for_polygon(grpc_stub): - fake_id = 999999990 - fake_str_id = "999999990" +def setup_dummy_data_for_polygon(grpc_stub): + dummy_id = 999999990 + dummy_str_id = "999999990" ts_metadata = dstore.TSMetadata( - station_id=fake_str_id, - param_id=fake_str_id, + station_id=dummy_str_id, + param_id=dummy_str_id, pos=dstore.Point(lat=80, lon=170), other1="test_value1", other2="test_value2", other3="test_value3", ) ts_add_request_1 = dstore.AddTSRequest( - id=fake_id, + id=dummy_id, metadata=ts_metadata, ) # grpc_stub.AddTimeSeries(ts_add_request_1) _handle_grpc_error(grpc_stub.AddTimeSeries, ts_add_request_1) - fake_id = 999999991 - fake_str_id = "999999991" + dummy_id = 999999991 + dummy_str_id = "999999991" ts_metadata = dstore.TSMetadata( - station_id=fake_str_id, - param_id=fake_str_id, + station_id=dummy_str_id, + param_id=dummy_str_id, pos=dstore.Point(lat=90, lon=180), other1="test_value4", other2="test_value5", other3="test_value6", ) ts_add_request_2 = dstore.AddTSRequest( - id=fake_id, + id=dummy_id, metadata=ts_metadata, ) # grpc_stub.AddTimeSeries(ts_add_request_2) @@ -182,7 +189,7 @@ def setup_fake_data_for_polygon(grpc_stub): @pytest.mark.parametrize("coords,expected", input_params_polygon) -def test_get_observations_with_polygon(grpc_stub, setup_fake_data_for_polygon, coords, expected): +def test_get_observations_with_polygon(grpc_stub, setup_dummy_data_for_polygon, coords, expected): ts_request = dstore.FindTSRequest( inside=dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords]) ) From 00a1cbd0db29542a5fa98e6395ef4d8ddc6f576d Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 20 Sep 2023 09:26:10 +0200 Subject: [PATCH 05/14] Remove _handle_grpc_error() because it was nice when writing the tests, but it is no longer necessary as the teardown should prevent the record from already existing. --- integration-test/test_fake_data.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/integration-test/test_fake_data.py b/integration-test/test_fake_data.py index d9da8b0..49fc2ad 100644 --- a/integration-test/test_fake_data.py +++ b/integration-test/test_fake_data.py @@ -15,18 +15,6 @@ logger.setLevel(logging.INFO) -def _handle_grpc_error(func, *args, **kwargs): - try: - return func(*args, **kwargs) - except grpc.RpcError as rpc_error: - for error in rpc_error.args: - if error.details.endswith("already exists"): - logger.info(error) - else: - logger.error(rpc_error) - raise rpc_error - - @pytest.fixture(scope="session") def grpc_stub(): with grpc.insecure_channel(f"{os.getenv('DSHOST', 'localhost')}:{os.getenv('DSPORT', '50050')}") as channel: @@ -129,8 +117,7 @@ def setup_dummy_data_for_polygon(grpc_stub): id=dummy_id, metadata=ts_metadata, ) - # grpc_stub.AddTimeSeries(ts_add_request_1) - _handle_grpc_error(grpc_stub.AddTimeSeries, ts_add_request_1) + grpc_stub.AddTimeSeries(ts_add_request_1) dummy_id = 999999991 dummy_str_id = "999999991" @@ -146,8 +133,7 @@ def setup_dummy_data_for_polygon(grpc_stub): id=dummy_id, metadata=ts_metadata, ) - # grpc_stub.AddTimeSeries(ts_add_request_2) - _handle_grpc_error(grpc_stub.AddTimeSeries, ts_add_request_2) + grpc_stub.AddTimeSeries(ts_add_request_2) yield # Hereafter teardown From c013c5b361e68466639f78238888512087da0a1d Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 20 Sep 2023 09:31:48 +0200 Subject: [PATCH 06/14] Remove print statements used for debugging. --- datastore/storagebackend/timescaledb/findtimeseries.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/datastore/storagebackend/timescaledb/findtimeseries.go b/datastore/storagebackend/timescaledb/findtimeseries.go index 9ee504e..c12d121 100644 --- a/datastore/storagebackend/timescaledb/findtimeseries.go +++ b/datastore/storagebackend/timescaledb/findtimeseries.go @@ -94,9 +94,6 @@ func (sbe *TimescaleDB) FindTimeSeries(request *datastore.FindTSRequest) ( // TODO: add more filters - fmt.Println("Query: ", query) - fmt.Println("phVals: ", phVals) - rows, err := sbe.Db.Query(query, phVals...) if err != nil { return nil, fmt.Errorf("sbe.Db.Query() failed: %v", err) From ffbb8f3759cc00b9a9e9b46c78031c6990e5e12a Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 20 Sep 2023 09:46:28 +0200 Subject: [PATCH 07/14] Renamed test file. --- integration-test/{test_fake_data.py => test_dummy_data.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename integration-test/{test_fake_data.py => test_dummy_data.py} (100%) diff --git a/integration-test/test_fake_data.py b/integration-test/test_dummy_data.py similarity index 100% rename from integration-test/test_fake_data.py rename to integration-test/test_dummy_data.py From 41c71474e28e30cd3b08c6ac5acd6d6e5b9722c9 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 20 Sep 2023 09:57:31 +0200 Subject: [PATCH 08/14] Add test file to the integration docker image. --- integration-test/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-test/Dockerfile b/integration-test/Dockerfile index d8bae28..bdee650 100644 --- a/integration-test/Dockerfile +++ b/integration-test/Dockerfile @@ -28,6 +28,7 @@ RUN python -m grpc_tools.protoc \ --grpc_python_out="${DOCKER_PATH}" COPY "${PROJECT_PYTHON_PATH}/test_knmi.py" "${DOCKER_PATH}/test_knmi.py" +COPY "${PROJECT_PYTHON_PATH}/test_dummy_data.py" "${DOCKER_PATH}/test_dummy_data.py" WORKDIR "${DOCKER_PATH}" CMD ["pytest"] From 93e305ab35a44a86f5fa7deb4e7aaeecc7edb4e6 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 20 Sep 2023 10:05:51 +0200 Subject: [PATCH 09/14] Add depends on to integration so that it starts after the loader as it requires some of the data for testing. --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 46a992d..94a6830 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,6 +77,8 @@ services: depends_on: store: condition: service_healthy + loader: + condition: service_completed_successfully volumes: ts-data: From 2f0d0450042897961c70948960a75e886681fd15 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 20 Sep 2023 10:14:43 +0200 Subject: [PATCH 10/14] Remove depends on as it breaks the github ci actions. --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 94a6830..d302d16 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,8 +77,8 @@ services: depends_on: store: condition: service_healthy - loader: - condition: service_completed_successfully +# loader: +# condition: service_completed_successfully volumes: ts-data: From f374ecb4b4affb3271325c6efb58f8f5404841b0 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 20 Sep 2023 16:47:41 +0200 Subject: [PATCH 11/14] Process feedback on merge request: Polygon test on KNMI data; Rename delete test; Cleanup. --- docker-compose.yml | 2 - integration-test/Dockerfile | 2 +- .../{test_dummy_data.py => test_delete.py} | 96 +------------------ integration-test/test_knmi.py | 50 +++++++++- 4 files changed, 54 insertions(+), 96 deletions(-) rename integration-test/{test_dummy_data.py => test_delete.py} (51%) diff --git a/docker-compose.yml b/docker-compose.yml index d302d16..46a992d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,8 +77,6 @@ services: depends_on: store: condition: service_healthy -# loader: -# condition: service_completed_successfully volumes: ts-data: diff --git a/integration-test/Dockerfile b/integration-test/Dockerfile index bdee650..7f56e04 100644 --- a/integration-test/Dockerfile +++ b/integration-test/Dockerfile @@ -28,7 +28,7 @@ RUN python -m grpc_tools.protoc \ --grpc_python_out="${DOCKER_PATH}" COPY "${PROJECT_PYTHON_PATH}/test_knmi.py" "${DOCKER_PATH}/test_knmi.py" -COPY "${PROJECT_PYTHON_PATH}/test_dummy_data.py" "${DOCKER_PATH}/test_dummy_data.py" +COPY "${PROJECT_PYTHON_PATH}/test_delete.py" "${DOCKER_PATH}/test_delete.py" WORKDIR "${DOCKER_PATH}" CMD ["pytest"] diff --git a/integration-test/test_dummy_data.py b/integration-test/test_delete.py similarity index 51% rename from integration-test/test_dummy_data.py rename to integration-test/test_delete.py index 49fc2ad..ab7b13a 100644 --- a/integration-test/test_dummy_data.py +++ b/integration-test/test_delete.py @@ -1,4 +1,3 @@ -# Note that this assumes that the KNMI test data is loader (using loader container) import logging import os from datetime import datetime @@ -24,7 +23,7 @@ def grpc_stub(): @pytest.fixture(scope="function") def dummy_timeseries_for_delete(): dummy_id = 999999999 - dummy_str_id = "999999999" + dummy_str_id = "test_delete-1" ts_metadata = dstore.TSMetadata( station_id=dummy_str_id, @@ -64,11 +63,9 @@ def dummy_observations_for_delete(): def test_delete_timeseries(grpc_stub, dummy_timeseries_for_delete, dummy_observations_for_delete): - ts_add_response = grpc_stub.AddTimeSeries(dummy_timeseries_for_delete) - assert str(ts_add_response) == "status: -1\n" + grpc_stub.AddTimeSeries(dummy_timeseries_for_delete) - obs_response = grpc_stub.PutObservations(dummy_observations_for_delete) - assert str(obs_response) == "status: -1\n" + grpc_stub.PutObservations(dummy_observations_for_delete) ts_find_request = dstore.FindTSRequest( station_ids=[dummy_timeseries_for_delete.metadata.station_id], @@ -90,8 +87,7 @@ def test_delete_timeseries(grpc_stub, dummy_timeseries_for_delete, dummy_observa assert len(obs_get_response.tsobs[0].obs) == 3 ts_delete_request = dstore.DeleteTSRequest(ids=[dummy_timeseries_for_delete.id]) - ts_delete_response = grpc_stub.DeleteTimeSeries(ts_delete_request) - assert str(ts_delete_response) == "status: -1\n" + grpc_stub.DeleteTimeSeries(ts_delete_request) ts_find_response = grpc_stub.FindTimeSeries(ts_find_request) assert len(ts_find_response.tseries) == 0 @@ -99,87 +95,3 @@ def test_delete_timeseries(grpc_stub, dummy_timeseries_for_delete, dummy_observa obs_get_response = grpc_stub.GetObservations(obs_get_request) assert obs_get_response.tsobs[0].tsid == dummy_timeseries_for_delete.id assert len(obs_get_response.tsobs[0].obs) == 0 - - -@pytest.fixture(scope="module") -def setup_dummy_data_for_polygon(grpc_stub): - dummy_id = 999999990 - dummy_str_id = "999999990" - ts_metadata = dstore.TSMetadata( - station_id=dummy_str_id, - param_id=dummy_str_id, - pos=dstore.Point(lat=80, lon=170), - other1="test_value1", - other2="test_value2", - other3="test_value3", - ) - ts_add_request_1 = dstore.AddTSRequest( - id=dummy_id, - metadata=ts_metadata, - ) - grpc_stub.AddTimeSeries(ts_add_request_1) - - dummy_id = 999999991 - dummy_str_id = "999999991" - ts_metadata = dstore.TSMetadata( - station_id=dummy_str_id, - param_id=dummy_str_id, - pos=dstore.Point(lat=90, lon=180), - other1="test_value4", - other2="test_value5", - other3="test_value6", - ) - ts_add_request_2 = dstore.AddTSRequest( - id=dummy_id, - metadata=ts_metadata, - ) - grpc_stub.AddTimeSeries(ts_add_request_2) - - yield # Hereafter teardown - - grpc_stub.DeleteTimeSeries(dstore.DeleteTSRequest(ids=[ts_add_request_1.id, ts_add_request_2.id])) - - -input_params_polygon = [ - ( - # Multiple stations within - ((80, 170), (90, 170), (90, 180), (80, 180)), - {"number_of_stations": 2, "station_ids": [999999990, 999999991]}, - ), - ( - # One station within - ((80, 175), (90, 175), (90, 180), (80, 180)), - {"number_of_stations": 1, "station_ids": [999999991]}, - ), - ( - # Nothing within - ((80, 175), (85, 175), (85, 180), (80, 180)), - {"number_of_stations": 0, "station_ids": []}, - ), - ( - # middle top - ((85, 120), (90, 120), (90, -120), (85, -120)), - {"number_of_stations": 1, "station_ids": [999999991]}, - ), - ( - # Middle bottom, should fall outside since polygon is curved because the earth is round (postgres geography). - ((85, 120), (80, 120), (85, -120), (80, -120)), - {"number_of_stations": 0, "station_ids": []}, - ), - ( - # Complex polygon - ((79, 169), (81, 169), (80, 171), (89, 171), (89, 180), (75, 180), (75, 171)), - {"number_of_stations": 1, "station_ids": [999999990]}, - ), -] - - -@pytest.mark.parametrize("coords,expected", input_params_polygon) -def test_get_observations_with_polygon(grpc_stub, setup_dummy_data_for_polygon, coords, expected): - ts_request = dstore.FindTSRequest( - inside=dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords]) - ) - ts_response = grpc_stub.FindTimeSeries(ts_request) - - assert len(ts_response.tseries) == expected["number_of_stations"] - assert sorted(ts.id for ts in ts_response.tseries) == expected["station_ids"] diff --git a/integration-test/test_knmi.py b/integration-test/test_knmi.py index bb34a3d..1079afd 100644 --- a/integration-test/test_knmi.py +++ b/integration-test/test_knmi.py @@ -9,7 +9,7 @@ from google.protobuf.timestamp_pb2 import Timestamp -@pytest.fixture +@pytest.fixture(scope="session") def grpc_stub(): with grpc.insecure_channel(f"{os.getenv('DSHOST', 'localhost')}:{os.getenv('DSPORT', '50050')}") as channel: yield dstore_grpc.DatastoreStub(channel) @@ -62,3 +62,51 @@ def test_get_values_single_station_single_parameters(grpc_stub): assert len(response.tsobs[0].obs) == 144 assert response.tsobs[0].obs[0].value == 95.0 assert response.tsobs[0].obs[-1].value == 59.0 + + +input_params_polygon = [ + ( + # Multiple stations within + ((52.15, 4.90), (52.15, 5.37), (51.66, 5.37), (51.66, 4.90)), + {"number_of_timeseries": 132, "station_ids": ["06260", "06348", "06356"]}, + ), + ( + # One station within + ((52.11, 5.15), (52.11, 5.204), (52.08, 5.204), (52.08, 5.15)), + {"number_of_timeseries": 44, "station_ids": ["06260"]}, + ), + ( + # Nothing within + ((51.82, 5.07), (51.82, 5.41), (51.73, 5.41), (51.73, 5.07)), + {"number_of_timeseries": 0, "station_ids": []}, + ), + ( + # Middle top + ((52.0989, 4.17), (52.0989, 6.18), (52.09, 6.18), (52.09, 4.17)), + {"number_of_timeseries": 44, "station_ids": ["06260"]}, + ), + ( + # Middle bottom, should fall outside since polygon is curved because the earth is round (postgres geography). + ((52.1, 4.17), (52.1, 6.18), (52.0989, 6.18), (52.0989, 4.17)), + {"number_of_timeseries": 0, "station_ids": []}, + ), + ( + # Complex polygon + ((51.45, 3.47), (51.39, 3.67), (51.39, 4.28), (51.52, 4.96), (51.89, 5.46), (52.18, 5.30), (51.75, 3.68)), + { + "number_of_timeseries": 352, + "station_ids": ["06260", "06310", "06323", "06340", "06343", "06348", "06350", "06356"], + }, + ), +] + + +@pytest.mark.parametrize("coords,expected", input_params_polygon) +def test_get_observations_with_polygon(grpc_stub, coords, expected): + ts_request = dstore.FindTSRequest( + inside=dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords]) + ) + ts_response = grpc_stub.FindTimeSeries(ts_request) + + assert len(ts_response.tseries) == expected["number_of_timeseries"] # parameters * number_of_stations + assert sorted({ts.metadata.station_id for ts in ts_response.tseries}) == expected["station_ids"] From 05e2e3b1f6acded3582109716728f589d02faa33 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 20 Sep 2023 17:17:02 +0200 Subject: [PATCH 12/14] Add integration test for all stations in the Netherlands. --- integration-test/test_knmi.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/integration-test/test_knmi.py b/integration-test/test_knmi.py index 1079afd..b885a35 100644 --- a/integration-test/test_knmi.py +++ b/integration-test/test_knmi.py @@ -68,36 +68,54 @@ def test_get_values_single_station_single_parameters(grpc_stub): ( # Multiple stations within ((52.15, 4.90), (52.15, 5.37), (51.66, 5.37), (51.66, 4.90)), - {"number_of_timeseries": 132, "station_ids": ["06260", "06348", "06356"]}, + {"number_of_timeseries": 132, "number_of_stations": 3, "station_ids": ["06260", "06348", "06356"]}, ), ( # One station within ((52.11, 5.15), (52.11, 5.204), (52.08, 5.204), (52.08, 5.15)), - {"number_of_timeseries": 44, "station_ids": ["06260"]}, + {"number_of_timeseries": 44, "number_of_stations": 1, "station_ids": ["06260"]}, ), ( # Nothing within ((51.82, 5.07), (51.82, 5.41), (51.73, 5.41), (51.73, 5.07)), - {"number_of_timeseries": 0, "station_ids": []}, + {"number_of_timeseries": 0, "number_of_stations": 0, "station_ids": []}, ), ( # Middle top ((52.0989, 4.17), (52.0989, 6.18), (52.09, 6.18), (52.09, 4.17)), - {"number_of_timeseries": 44, "station_ids": ["06260"]}, + {"number_of_timeseries": 44, "number_of_stations": 1, "station_ids": ["06260"]}, ), ( # Middle bottom, should fall outside since polygon is curved because the earth is round (postgres geography). ((52.1, 4.17), (52.1, 6.18), (52.0989, 6.18), (52.0989, 4.17)), - {"number_of_timeseries": 0, "station_ids": []}, + {"number_of_timeseries": 0, "number_of_stations": 0, "station_ids": []}, ), ( # Complex polygon ((51.45, 3.47), (51.39, 3.67), (51.39, 4.28), (51.52, 4.96), (51.89, 5.46), (52.18, 5.30), (51.75, 3.68)), { "number_of_timeseries": 352, + "number_of_stations": 8, "station_ids": ["06260", "06310", "06323", "06340", "06343", "06348", "06350", "06356"], }, ), + ( + # All stations in the Netherlands + ((56.00, 2.85), (56.00, 7.22), (50.75, 7.22), (50.75, 2.85)), + { + "number_of_timeseries": 2288, + "number_of_stations": 52, + # fmt: off + "station_ids": [ + "06201", "06203", "06204", "06205", "06207", "06208", "06211", "06214", "06215", "06225", "06229", + "06235", "06239", "06240", "06242", "06248", "06249", "06251", "06252", "06257", "06258", "06260", + "06267", "06269", "06270", "06273", "06275", "06277", "06278", "06279", "06280", "06283", "06286", + "06290", "06310", "06317", "06319", "06320", "06321", "06323", "06330", "06340", "06343", "06344", + "06348", "06350", "06356", "06370", "06375", "06377", "06380", "06391" + ], + # fmt: on + }, + ), ] @@ -109,4 +127,6 @@ def test_get_observations_with_polygon(grpc_stub, coords, expected): ts_response = grpc_stub.FindTimeSeries(ts_request) assert len(ts_response.tseries) == expected["number_of_timeseries"] # parameters * number_of_stations - assert sorted({ts.metadata.station_id for ts in ts_response.tseries}) == expected["station_ids"] + stations = sorted({ts.metadata.station_id for ts in ts_response.tseries}) + assert len(stations) == expected["number_of_stations"] + assert stations == expected["station_ids"] From 68bbb6a3dee1d8b45c629db2ca5f32913f3504a5 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Fri, 22 Sep 2023 09:54:57 +0200 Subject: [PATCH 13/14] Add test for multiple parameters for a polygon. --- integration-test/test_knmi.py | 36 ++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/integration-test/test_knmi.py b/integration-test/test_knmi.py index b885a35..e2e38dd 100644 --- a/integration-test/test_knmi.py +++ b/integration-test/test_knmi.py @@ -9,6 +9,10 @@ from google.protobuf.timestamp_pb2 import Timestamp +def _strip_empty_parameters(**kwargs): + return {k: v for k, v in kwargs.items() if v} + + @pytest.fixture(scope="session") def grpc_stub(): with grpc.insecure_channel(f"{os.getenv('DSHOST', 'localhost')}:{os.getenv('DSPORT', '50050')}") as channel: @@ -68,31 +72,49 @@ def test_get_values_single_station_single_parameters(grpc_stub): ( # Multiple stations within ((52.15, 4.90), (52.15, 5.37), (51.66, 5.37), (51.66, 4.90)), + None, {"number_of_timeseries": 132, "number_of_stations": 3, "station_ids": ["06260", "06348", "06356"]}, ), + ( + # Multiple stations with a single parameter + ((52.15, 4.90), (52.15, 5.37), (51.66, 5.37), (51.66, 4.90)), + ["rh"], + {"number_of_timeseries": 3, "number_of_stations": 3, "station_ids": ["06260", "06348", "06356"]}, + ), + ( + # Multiple stations with multiple parameters + ((52.15, 4.90), (52.15, 5.37), (51.66, 5.37), (51.66, 4.90)), + ["dd", "rh", "tx"], + {"number_of_timeseries": 9, "number_of_stations": 3, "station_ids": ["06260", "06348", "06356"]}, + ), ( # One station within ((52.11, 5.15), (52.11, 5.204), (52.08, 5.204), (52.08, 5.15)), + None, {"number_of_timeseries": 44, "number_of_stations": 1, "station_ids": ["06260"]}, ), ( # Nothing within ((51.82, 5.07), (51.82, 5.41), (51.73, 5.41), (51.73, 5.07)), + None, {"number_of_timeseries": 0, "number_of_stations": 0, "station_ids": []}, ), ( # Middle top ((52.0989, 4.17), (52.0989, 6.18), (52.09, 6.18), (52.09, 4.17)), + None, {"number_of_timeseries": 44, "number_of_stations": 1, "station_ids": ["06260"]}, ), ( # Middle bottom, should fall outside since polygon is curved because the earth is round (postgres geography). ((52.1, 4.17), (52.1, 6.18), (52.0989, 6.18), (52.0989, 4.17)), + None, {"number_of_timeseries": 0, "number_of_stations": 0, "station_ids": []}, ), ( # Complex polygon ((51.45, 3.47), (51.39, 3.67), (51.39, 4.28), (51.52, 4.96), (51.89, 5.46), (52.18, 5.30), (51.75, 3.68)), + None, { "number_of_timeseries": 352, "number_of_stations": 8, @@ -102,6 +124,7 @@ def test_get_values_single_station_single_parameters(grpc_stub): ( # All stations in the Netherlands ((56.00, 2.85), (56.00, 7.22), (50.75, 7.22), (50.75, 2.85)), + None, { "number_of_timeseries": 2288, "number_of_stations": 52, @@ -119,11 +142,14 @@ def test_get_values_single_station_single_parameters(grpc_stub): ] -@pytest.mark.parametrize("coords,expected", input_params_polygon) -def test_get_observations_with_polygon(grpc_stub, coords, expected): - ts_request = dstore.FindTSRequest( - inside=dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords]) - ) +@pytest.mark.parametrize("coords,param_ids,expected", input_params_polygon) +def test_get_observations_with_polygon(grpc_stub, coords, param_ids, expected): + inside = None + if coords: + inside = dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords]) + + kwargs = _strip_empty_parameters(inside=inside, param_ids=param_ids) + ts_request = dstore.FindTSRequest(**kwargs) ts_response = grpc_stub.FindTimeSeries(ts_request) assert len(ts_response.tseries) == expected["number_of_timeseries"] # parameters * number_of_stations From 9e1dff8f1c3c2e0d04313781265a732dcb94a273 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Fri, 22 Sep 2023 14:12:41 +0200 Subject: [PATCH 14/14] Reduce code. --- integration-test/test_knmi.py | 69 +++++++++++++++-------------------- 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/integration-test/test_knmi.py b/integration-test/test_knmi.py index e2e38dd..2637350 100644 --- a/integration-test/test_knmi.py +++ b/integration-test/test_knmi.py @@ -9,8 +9,7 @@ from google.protobuf.timestamp_pb2 import Timestamp -def _strip_empty_parameters(**kwargs): - return {k: v for k, v in kwargs.items() if v} +NUMBER_OF_PARAMETERS = 44 @pytest.fixture(scope="session") @@ -41,7 +40,7 @@ def test_find_series_single_station_all_parameters(grpc_stub): ) response = grpc_stub.FindTimeSeries(request) - assert len(response.tseries) == 44 + assert len(response.tseries) == NUMBER_OF_PARAMETERS def test_get_values_single_station_single_parameters(grpc_stub): @@ -73,86 +72,76 @@ def test_get_values_single_station_single_parameters(grpc_stub): # Multiple stations within ((52.15, 4.90), (52.15, 5.37), (51.66, 5.37), (51.66, 4.90)), None, - {"number_of_timeseries": 132, "number_of_stations": 3, "station_ids": ["06260", "06348", "06356"]}, + ["06260", "06348", "06356"], ), ( # Multiple stations with a single parameter ((52.15, 4.90), (52.15, 5.37), (51.66, 5.37), (51.66, 4.90)), ["rh"], - {"number_of_timeseries": 3, "number_of_stations": 3, "station_ids": ["06260", "06348", "06356"]}, + ["06260", "06348", "06356"], ), ( # Multiple stations with multiple parameters ((52.15, 4.90), (52.15, 5.37), (51.66, 5.37), (51.66, 4.90)), ["dd", "rh", "tx"], - {"number_of_timeseries": 9, "number_of_stations": 3, "station_ids": ["06260", "06348", "06356"]}, + ["06260", "06348", "06356"], ), ( # One station within ((52.11, 5.15), (52.11, 5.204), (52.08, 5.204), (52.08, 5.15)), None, - {"number_of_timeseries": 44, "number_of_stations": 1, "station_ids": ["06260"]}, + ["06260"], ), ( # Nothing within ((51.82, 5.07), (51.82, 5.41), (51.73, 5.41), (51.73, 5.07)), None, - {"number_of_timeseries": 0, "number_of_stations": 0, "station_ids": []}, + [], ), ( # Middle top ((52.0989, 4.17), (52.0989, 6.18), (52.09, 6.18), (52.09, 4.17)), None, - {"number_of_timeseries": 44, "number_of_stations": 1, "station_ids": ["06260"]}, + ["06260"], ), ( # Middle bottom, should fall outside since polygon is curved because the earth is round (postgres geography). ((52.1, 4.17), (52.1, 6.18), (52.0989, 6.18), (52.0989, 4.17)), None, - {"number_of_timeseries": 0, "number_of_stations": 0, "station_ids": []}, + [], ), ( # Complex polygon ((51.45, 3.47), (51.39, 3.67), (51.39, 4.28), (51.52, 4.96), (51.89, 5.46), (52.18, 5.30), (51.75, 3.68)), None, - { - "number_of_timeseries": 352, - "number_of_stations": 8, - "station_ids": ["06260", "06310", "06323", "06340", "06343", "06348", "06350", "06356"], - }, + ["06260", "06310", "06323", "06340", "06343", "06348", "06350", "06356"], ), ( # All stations in the Netherlands ((56.00, 2.85), (56.00, 7.22), (50.75, 7.22), (50.75, 2.85)), None, - { - "number_of_timeseries": 2288, - "number_of_stations": 52, - # fmt: off - "station_ids": [ - "06201", "06203", "06204", "06205", "06207", "06208", "06211", "06214", "06215", "06225", "06229", - "06235", "06239", "06240", "06242", "06248", "06249", "06251", "06252", "06257", "06258", "06260", - "06267", "06269", "06270", "06273", "06275", "06277", "06278", "06279", "06280", "06283", "06286", - "06290", "06310", "06317", "06319", "06320", "06321", "06323", "06330", "06340", "06343", "06344", - "06348", "06350", "06356", "06370", "06375", "06377", "06380", "06391" - ], - # fmt: on - }, + # fmt: off + [ + "06201", "06203", "06204", "06205", "06207", "06208", "06211", "06214", "06215", "06225", "06229", + "06235", "06239", "06240", "06242", "06248", "06249", "06251", "06252", "06257", "06258", "06260", + "06267", "06269", "06270", "06273", "06275", "06277", "06278", "06279", "06280", "06283", "06286", + "06290", "06310", "06317", "06319", "06320", "06321", "06323", "06330", "06340", "06343", "06344", + "06348", "06350", "06356", "06370", "06375", "06377", "06380", "06391" + ], + # fmt: on ), ] -@pytest.mark.parametrize("coords,param_ids,expected", input_params_polygon) -def test_get_observations_with_polygon(grpc_stub, coords, param_ids, expected): - inside = None - if coords: - inside = dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords]) - - kwargs = _strip_empty_parameters(inside=inside, param_ids=param_ids) - ts_request = dstore.FindTSRequest(**kwargs) +@pytest.mark.parametrize("coords,param_ids,expected_station_ids", input_params_polygon) +def test_get_observations_with_polygon(grpc_stub, coords, param_ids, expected_station_ids): + ts_request = dstore.FindTSRequest( + inside=dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords]), param_ids=param_ids + ) ts_response = grpc_stub.FindTimeSeries(ts_request) - assert len(ts_response.tseries) == expected["number_of_timeseries"] # parameters * number_of_stations - stations = sorted({ts.metadata.station_id for ts in ts_response.tseries}) - assert len(stations) == expected["number_of_stations"] - assert stations == expected["station_ids"] + actual_station_ids = sorted({ts.metadata.station_id for ts in ts_response.tseries}) + assert actual_station_ids == expected_station_ids + number_of_parameters = len(param_ids) if param_ids else NUMBER_OF_PARAMETERS + expected_number_of_timeseries = number_of_parameters * len(expected_station_ids) + assert len(ts_response.tseries) == expected_number_of_timeseries