Process feedback on merge request: Polygon test on KNMI data; Rename …

…delete test; Cleanup.

docker-compose.yml

@@ -77,8 +77,6 @@ services:
     depends_on:
       store:
         condition: service_healthy
-#      loader:
-#        condition: service_completed_successfully
 
 volumes:
   ts-data:

integration-test/Dockerfile

@@ -28,7 +28,7 @@ RUN python -m grpc_tools.protoc  \
     --grpc_python_out="${DOCKER_PATH}"
 
 COPY "${PROJECT_PYTHON_PATH}/test_knmi.py" "${DOCKER_PATH}/test_knmi.py"
-COPY "${PROJECT_PYTHON_PATH}/test_dummy_data.py" "${DOCKER_PATH}/test_dummy_data.py"
+COPY "${PROJECT_PYTHON_PATH}/test_delete.py" "${DOCKER_PATH}/test_delete.py"
 
 WORKDIR "${DOCKER_PATH}"
 CMD ["pytest"]

integration-test/test_dummy_data.py → integration-test/test_delete.py

@@ -1,4 +1,3 @@
-# Note that this assumes that the KNMI test data is loader (using loader container)
 import logging
 import os
 from datetime import datetime
@@ -24,7 +23,7 @@ def grpc_stub():
 @pytest.fixture(scope="function")
 def dummy_timeseries_for_delete():
     dummy_id = 999999999
-    dummy_str_id = "999999999"
+    dummy_str_id = "test_delete-1"
 
     ts_metadata = dstore.TSMetadata(
         station_id=dummy_str_id,
@@ -64,11 +63,9 @@ def dummy_observations_for_delete():
 
 
 def test_delete_timeseries(grpc_stub, dummy_timeseries_for_delete, dummy_observations_for_delete):
-    ts_add_response = grpc_stub.AddTimeSeries(dummy_timeseries_for_delete)
-    assert str(ts_add_response) == "status: -1\n"
+    grpc_stub.AddTimeSeries(dummy_timeseries_for_delete)
 
-    obs_response = grpc_stub.PutObservations(dummy_observations_for_delete)
-    assert str(obs_response) == "status: -1\n"
+    grpc_stub.PutObservations(dummy_observations_for_delete)
 
     ts_find_request = dstore.FindTSRequest(
         station_ids=[dummy_timeseries_for_delete.metadata.station_id],
@@ -90,96 +87,11 @@ def test_delete_timeseries(grpc_stub, dummy_timeseries_for_delete, dummy_observa
     assert len(obs_get_response.tsobs[0].obs) == 3
 
     ts_delete_request = dstore.DeleteTSRequest(ids=[dummy_timeseries_for_delete.id])
-    ts_delete_response = grpc_stub.DeleteTimeSeries(ts_delete_request)
-    assert str(ts_delete_response) == "status: -1\n"
+    grpc_stub.DeleteTimeSeries(ts_delete_request)
 
     ts_find_response = grpc_stub.FindTimeSeries(ts_find_request)
     assert len(ts_find_response.tseries) == 0
 
     obs_get_response = grpc_stub.GetObservations(obs_get_request)
     assert obs_get_response.tsobs[0].tsid == dummy_timeseries_for_delete.id
     assert len(obs_get_response.tsobs[0].obs) == 0
-
-
-@pytest.fixture(scope="module")
-def setup_dummy_data_for_polygon(grpc_stub):
-    dummy_id = 999999990
-    dummy_str_id = "999999990"
-    ts_metadata = dstore.TSMetadata(
-        station_id=dummy_str_id,
-        param_id=dummy_str_id,
-        pos=dstore.Point(lat=80, lon=170),
-        other1="test_value1",
-        other2="test_value2",
-        other3="test_value3",
-    )
-    ts_add_request_1 = dstore.AddTSRequest(
-        id=dummy_id,
-        metadata=ts_metadata,
-    )
-    grpc_stub.AddTimeSeries(ts_add_request_1)
-
-    dummy_id = 999999991
-    dummy_str_id = "999999991"
-    ts_metadata = dstore.TSMetadata(
-        station_id=dummy_str_id,
-        param_id=dummy_str_id,
-        pos=dstore.Point(lat=90, lon=180),
-        other1="test_value4",
-        other2="test_value5",
-        other3="test_value6",
-    )
-    ts_add_request_2 = dstore.AddTSRequest(
-        id=dummy_id,
-        metadata=ts_metadata,
-    )
-    grpc_stub.AddTimeSeries(ts_add_request_2)
-
-    yield  # Hereafter teardown
-
-    grpc_stub.DeleteTimeSeries(dstore.DeleteTSRequest(ids=[ts_add_request_1.id, ts_add_request_2.id]))
-
-
-input_params_polygon = [
-    (
-        # Multiple stations within
-        ((80, 170), (90, 170), (90, 180), (80, 180)),
-        {"number_of_stations": 2, "station_ids": [999999990, 999999991]},
-    ),
-    (
-        # One station within
-        ((80, 175), (90, 175), (90, 180), (80, 180)),
-        {"number_of_stations": 1, "station_ids": [999999991]},
-    ),
-    (
-        # Nothing within
-        ((80, 175), (85, 175), (85, 180), (80, 180)),
-        {"number_of_stations": 0, "station_ids": []},
-    ),
-    (
-        # middle top
-        ((85, 120), (90, 120), (90, -120), (85, -120)),
-        {"number_of_stations": 1, "station_ids": [999999991]},
-    ),
-    (
-        # Middle bottom, should fall outside since polygon is curved because the earth is round (postgres geography).
-        ((85, 120), (80, 120), (85, -120), (80, -120)),
-        {"number_of_stations": 0, "station_ids": []},
-    ),
-    (
-        # Complex polygon
-        ((79, 169), (81, 169), (80, 171), (89, 171), (89, 180), (75, 180), (75, 171)),
-        {"number_of_stations": 1, "station_ids": [999999990]},
-    ),
-]
-
-
-@pytest.mark.parametrize("coords,expected", input_params_polygon)
-def test_get_observations_with_polygon(grpc_stub, setup_dummy_data_for_polygon, coords, expected):
-    ts_request = dstore.FindTSRequest(
-        inside=dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords])
-    )
-    ts_response = grpc_stub.FindTimeSeries(ts_request)
-
-    assert len(ts_response.tseries) == expected["number_of_stations"]
-    assert sorted(ts.id for ts in ts_response.tseries) == expected["station_ids"]

integration-test/test_knmi.py

@@ -9,7 +9,7 @@
 from google.protobuf.timestamp_pb2 import Timestamp
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def grpc_stub():
     with grpc.insecure_channel(f"{os.getenv('DSHOST', 'localhost')}:{os.getenv('DSPORT', '50050')}") as channel:
         yield dstore_grpc.DatastoreStub(channel)
@@ -62,3 +62,51 @@ def test_get_values_single_station_single_parameters(grpc_stub):
     assert len(response.tsobs[0].obs) == 144
     assert response.tsobs[0].obs[0].value == 95.0
     assert response.tsobs[0].obs[-1].value == 59.0
+
+
+input_params_polygon = [
+    (
+        # Multiple stations within
+        ((52.15, 4.90), (52.15, 5.37), (51.66, 5.37), (51.66, 4.90)),
+        {"number_of_timeseries": 132, "station_ids": ["06260", "06348", "06356"]},
+    ),
+    (
+        # One station within
+        ((52.11, 5.15), (52.11, 5.204), (52.08, 5.204), (52.08, 5.15)),
+        {"number_of_timeseries": 44, "station_ids": ["06260"]},
+    ),
+    (
+        # Nothing within
+        ((51.82, 5.07), (51.82, 5.41), (51.73, 5.41), (51.73, 5.07)),
+        {"number_of_timeseries": 0, "station_ids": []},
+    ),
+    (
+        # Middle top
+        ((52.0989, 4.17), (52.0989, 6.18), (52.09, 6.18), (52.09, 4.17)),
+        {"number_of_timeseries": 44, "station_ids": ["06260"]},
+    ),
+    (
+        # Middle bottom, should fall outside since polygon is curved because the earth is round (postgres geography).
+        ((52.1, 4.17), (52.1, 6.18), (52.0989, 6.18), (52.0989, 4.17)),
+        {"number_of_timeseries": 0, "station_ids": []},
+    ),
+    (
+        # Complex polygon
+        ((51.45, 3.47), (51.39, 3.67), (51.39, 4.28), (51.52, 4.96), (51.89, 5.46), (52.18, 5.30), (51.75, 3.68)),
+        {
+            "number_of_timeseries": 352,
+            "station_ids": ["06260", "06310", "06323", "06340", "06343", "06348", "06350", "06356"],
+        },
+    ),
+]
+
+
+@pytest.mark.parametrize("coords,expected", input_params_polygon)
+def test_get_observations_with_polygon(grpc_stub, coords, expected):
+    ts_request = dstore.FindTSRequest(
+        inside=dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords])
+    )
+    ts_response = grpc_stub.FindTimeSeries(ts_request)
+
+    assert len(ts_response.tseries) == expected["number_of_timeseries"]  # parameters * number_of_stations
+    assert sorted({ts.metadata.station_id for ts in ts_response.tseries}) == expected["station_ids"]