From f374ecb4b4affb3271325c6efb58f8f5404841b0 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 20 Sep 2023 16:47:41 +0200 Subject: [PATCH] Process feedback on merge request: Polygon test on KNMI data; Rename delete test; Cleanup. --- docker-compose.yml | 2 - integration-test/Dockerfile | 2 +- .../{test_dummy_data.py => test_delete.py} | 96 +------------------ integration-test/test_knmi.py | 50 +++++++++- 4 files changed, 54 insertions(+), 96 deletions(-) rename integration-test/{test_dummy_data.py => test_delete.py} (51%) diff --git a/docker-compose.yml b/docker-compose.yml index d302d16..46a992d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,8 +77,6 @@ services: depends_on: store: condition: service_healthy -# loader: -# condition: service_completed_successfully volumes: ts-data: diff --git a/integration-test/Dockerfile b/integration-test/Dockerfile index bdee650..7f56e04 100644 --- a/integration-test/Dockerfile +++ b/integration-test/Dockerfile @@ -28,7 +28,7 @@ RUN python -m grpc_tools.protoc \ --grpc_python_out="${DOCKER_PATH}" COPY "${PROJECT_PYTHON_PATH}/test_knmi.py" "${DOCKER_PATH}/test_knmi.py" -COPY "${PROJECT_PYTHON_PATH}/test_dummy_data.py" "${DOCKER_PATH}/test_dummy_data.py" +COPY "${PROJECT_PYTHON_PATH}/test_delete.py" "${DOCKER_PATH}/test_delete.py" WORKDIR "${DOCKER_PATH}" CMD ["pytest"] diff --git a/integration-test/test_dummy_data.py b/integration-test/test_delete.py similarity index 51% rename from integration-test/test_dummy_data.py rename to integration-test/test_delete.py index 49fc2ad..ab7b13a 100644 --- a/integration-test/test_dummy_data.py +++ b/integration-test/test_delete.py @@ -1,4 +1,3 @@ -# Note that this assumes that the KNMI test data is loader (using loader container) import logging import os from datetime import datetime @@ -24,7 +23,7 @@ def grpc_stub(): @pytest.fixture(scope="function") def dummy_timeseries_for_delete(): dummy_id = 999999999 - dummy_str_id = "999999999" + dummy_str_id = "test_delete-1" ts_metadata = dstore.TSMetadata( station_id=dummy_str_id, @@ -64,11 +63,9 @@ def dummy_observations_for_delete(): def test_delete_timeseries(grpc_stub, dummy_timeseries_for_delete, dummy_observations_for_delete): - ts_add_response = grpc_stub.AddTimeSeries(dummy_timeseries_for_delete) - assert str(ts_add_response) == "status: -1\n" + grpc_stub.AddTimeSeries(dummy_timeseries_for_delete) - obs_response = grpc_stub.PutObservations(dummy_observations_for_delete) - assert str(obs_response) == "status: -1\n" + grpc_stub.PutObservations(dummy_observations_for_delete) ts_find_request = dstore.FindTSRequest( station_ids=[dummy_timeseries_for_delete.metadata.station_id], @@ -90,8 +87,7 @@ def test_delete_timeseries(grpc_stub, dummy_timeseries_for_delete, dummy_observa assert len(obs_get_response.tsobs[0].obs) == 3 ts_delete_request = dstore.DeleteTSRequest(ids=[dummy_timeseries_for_delete.id]) - ts_delete_response = grpc_stub.DeleteTimeSeries(ts_delete_request) - assert str(ts_delete_response) == "status: -1\n" + grpc_stub.DeleteTimeSeries(ts_delete_request) ts_find_response = grpc_stub.FindTimeSeries(ts_find_request) assert len(ts_find_response.tseries) == 0 @@ -99,87 +95,3 @@ def test_delete_timeseries(grpc_stub, dummy_timeseries_for_delete, dummy_observa obs_get_response = grpc_stub.GetObservations(obs_get_request) assert obs_get_response.tsobs[0].tsid == dummy_timeseries_for_delete.id assert len(obs_get_response.tsobs[0].obs) == 0 - - -@pytest.fixture(scope="module") -def setup_dummy_data_for_polygon(grpc_stub): - dummy_id = 999999990 - dummy_str_id = "999999990" - ts_metadata = dstore.TSMetadata( - station_id=dummy_str_id, - param_id=dummy_str_id, - pos=dstore.Point(lat=80, lon=170), - other1="test_value1", - other2="test_value2", - other3="test_value3", - ) - ts_add_request_1 = dstore.AddTSRequest( - id=dummy_id, - metadata=ts_metadata, - ) - grpc_stub.AddTimeSeries(ts_add_request_1) - - dummy_id = 999999991 - dummy_str_id = "999999991" - ts_metadata = dstore.TSMetadata( - station_id=dummy_str_id, - param_id=dummy_str_id, - pos=dstore.Point(lat=90, lon=180), - other1="test_value4", - other2="test_value5", - other3="test_value6", - ) - ts_add_request_2 = dstore.AddTSRequest( - id=dummy_id, - metadata=ts_metadata, - ) - grpc_stub.AddTimeSeries(ts_add_request_2) - - yield # Hereafter teardown - - grpc_stub.DeleteTimeSeries(dstore.DeleteTSRequest(ids=[ts_add_request_1.id, ts_add_request_2.id])) - - -input_params_polygon = [ - ( - # Multiple stations within - ((80, 170), (90, 170), (90, 180), (80, 180)), - {"number_of_stations": 2, "station_ids": [999999990, 999999991]}, - ), - ( - # One station within - ((80, 175), (90, 175), (90, 180), (80, 180)), - {"number_of_stations": 1, "station_ids": [999999991]}, - ), - ( - # Nothing within - ((80, 175), (85, 175), (85, 180), (80, 180)), - {"number_of_stations": 0, "station_ids": []}, - ), - ( - # middle top - ((85, 120), (90, 120), (90, -120), (85, -120)), - {"number_of_stations": 1, "station_ids": [999999991]}, - ), - ( - # Middle bottom, should fall outside since polygon is curved because the earth is round (postgres geography). - ((85, 120), (80, 120), (85, -120), (80, -120)), - {"number_of_stations": 0, "station_ids": []}, - ), - ( - # Complex polygon - ((79, 169), (81, 169), (80, 171), (89, 171), (89, 180), (75, 180), (75, 171)), - {"number_of_stations": 1, "station_ids": [999999990]}, - ), -] - - -@pytest.mark.parametrize("coords,expected", input_params_polygon) -def test_get_observations_with_polygon(grpc_stub, setup_dummy_data_for_polygon, coords, expected): - ts_request = dstore.FindTSRequest( - inside=dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords]) - ) - ts_response = grpc_stub.FindTimeSeries(ts_request) - - assert len(ts_response.tseries) == expected["number_of_stations"] - assert sorted(ts.id for ts in ts_response.tseries) == expected["station_ids"] diff --git a/integration-test/test_knmi.py b/integration-test/test_knmi.py index bb34a3d..1079afd 100644 --- a/integration-test/test_knmi.py +++ b/integration-test/test_knmi.py @@ -9,7 +9,7 @@ from google.protobuf.timestamp_pb2 import Timestamp -@pytest.fixture +@pytest.fixture(scope="session") def grpc_stub(): with grpc.insecure_channel(f"{os.getenv('DSHOST', 'localhost')}:{os.getenv('DSPORT', '50050')}") as channel: yield dstore_grpc.DatastoreStub(channel) @@ -62,3 +62,51 @@ def test_get_values_single_station_single_parameters(grpc_stub): assert len(response.tsobs[0].obs) == 144 assert response.tsobs[0].obs[0].value == 95.0 assert response.tsobs[0].obs[-1].value == 59.0 + + +input_params_polygon = [ + ( + # Multiple stations within + ((52.15, 4.90), (52.15, 5.37), (51.66, 5.37), (51.66, 4.90)), + {"number_of_timeseries": 132, "station_ids": ["06260", "06348", "06356"]}, + ), + ( + # One station within + ((52.11, 5.15), (52.11, 5.204), (52.08, 5.204), (52.08, 5.15)), + {"number_of_timeseries": 44, "station_ids": ["06260"]}, + ), + ( + # Nothing within + ((51.82, 5.07), (51.82, 5.41), (51.73, 5.41), (51.73, 5.07)), + {"number_of_timeseries": 0, "station_ids": []}, + ), + ( + # Middle top + ((52.0989, 4.17), (52.0989, 6.18), (52.09, 6.18), (52.09, 4.17)), + {"number_of_timeseries": 44, "station_ids": ["06260"]}, + ), + ( + # Middle bottom, should fall outside since polygon is curved because the earth is round (postgres geography). + ((52.1, 4.17), (52.1, 6.18), (52.0989, 6.18), (52.0989, 4.17)), + {"number_of_timeseries": 0, "station_ids": []}, + ), + ( + # Complex polygon + ((51.45, 3.47), (51.39, 3.67), (51.39, 4.28), (51.52, 4.96), (51.89, 5.46), (52.18, 5.30), (51.75, 3.68)), + { + "number_of_timeseries": 352, + "station_ids": ["06260", "06310", "06323", "06340", "06343", "06348", "06350", "06356"], + }, + ), +] + + +@pytest.mark.parametrize("coords,expected", input_params_polygon) +def test_get_observations_with_polygon(grpc_stub, coords, expected): + ts_request = dstore.FindTSRequest( + inside=dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords]) + ) + ts_response = grpc_stub.FindTimeSeries(ts_request) + + assert len(ts_response.tseries) == expected["number_of_timeseries"] # parameters * number_of_stations + assert sorted({ts.metadata.station_id for ts in ts_response.tseries}) == expected["station_ids"]