Merge pull request #38 from EURODEO/integration-tests

Integration tests

datastore/storagebackend/timescaledb/findtimeseries.go

@@ -69,7 +69,7 @@ func insidePolygonCond(
 
 	return fmt.Sprintf(
 		" AND ST_DWITHIN(%s, ST_GeomFromText($%d, %s)::geography, 0.0)",
-		name, srid, len(*phVals), srid), nil
+		name, len(*phVals), srid), nil
 }
 
 // FindTimeSeries ... (see documentation in StorageBackend interface)

integration-test/Dockerfile

@@ -28,6 +28,7 @@ RUN python -m grpc_tools.protoc  \
     --grpc_python_out="${DOCKER_PATH}"
 
 COPY "${PROJECT_PYTHON_PATH}/test_knmi.py" "${DOCKER_PATH}/test_knmi.py"
+COPY "${PROJECT_PYTHON_PATH}/test_delete.py" "${DOCKER_PATH}/test_delete.py"
 
 WORKDIR "${DOCKER_PATH}"
 CMD ["pytest"]

integration-test/test_delete.py

@@ -0,0 +1,97 @@
+import logging
+import os
+from datetime import datetime
+from datetime import timezone
+
+import datastore_pb2 as dstore
+import datastore_pb2_grpc as dstore_grpc
+import grpc
+import pytest
+from google.protobuf.timestamp_pb2 import Timestamp
+
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+@pytest.fixture(scope="session")
+def grpc_stub():
+    with grpc.insecure_channel(f"{os.getenv('DSHOST', 'localhost')}:{os.getenv('DSPORT', '50050')}") as channel:
+        yield dstore_grpc.DatastoreStub(channel)
+
+
+@pytest.fixture(scope="function")
+def dummy_timeseries_for_delete():
+    dummy_id = 999999999
+    dummy_str_id = "test_delete-1"
+
+    ts_metadata = dstore.TSMetadata(
+        station_id=dummy_str_id,
+        param_id=dummy_str_id,
+        pos=dstore.Point(lat=-60, lon=-160),
+        other1="test_value1",
+        other2="test_value2",
+        other3="test_value3",
+    )
+    ts_add_request = dstore.AddTSRequest(
+        id=dummy_id,
+        metadata=ts_metadata,
+    )
+    return ts_add_request
+
+
+@pytest.fixture(scope="function")
+def dummy_observations_for_delete():
+    dummy_id = 999999999
+
+    obs_metadata = dstore.ObsMetadata(field1="test_value1", field2="test_value2")
+    time_1 = Timestamp()
+    time_1.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=9, tzinfo=timezone.utc))
+    time_2 = Timestamp()
+    time_2.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=10, tzinfo=timezone.utc))
+    time_3 = Timestamp()
+    time_3.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=11, tzinfo=timezone.utc))
+    obs = [
+        dstore.Observation(time=time_1, value=1111.1111, metadata=obs_metadata),
+        dstore.Observation(time=time_2, value=2222.2222, metadata=obs_metadata),
+        dstore.Observation(time=time_3, value=3333.3333, metadata=obs_metadata),
+    ]
+    obs_put_request = dstore.PutObsRequest(
+        tsobs=[dstore.TSObservations(tsid=dummy_id, obs=obs)],
+    )
+    return obs_put_request
+
+
+def test_delete_timeseries(grpc_stub, dummy_timeseries_for_delete, dummy_observations_for_delete):
+    grpc_stub.AddTimeSeries(dummy_timeseries_for_delete)
+
+    grpc_stub.PutObservations(dummy_observations_for_delete)
+
+    ts_find_request = dstore.FindTSRequest(
+        station_ids=[dummy_timeseries_for_delete.metadata.station_id],
+        param_ids=[dummy_timeseries_for_delete.metadata.param_id],
+    )
+    ts_find_response = grpc_stub.FindTimeSeries(ts_find_request)
+    assert len(ts_find_response.tseries) == 1
+    assert ts_find_response.tseries[0].id == dummy_timeseries_for_delete.id
+
+    to_time = Timestamp()
+    to_time.FromDatetime(datetime(year=1999, month=9, day=9, hour=9, minute=11, second=1, tzinfo=timezone.utc))
+    obs_get_request = dstore.GetObsRequest(
+        tsids=[dummy_timeseries_for_delete.id],
+        fromtime=dummy_observations_for_delete.tsobs[0].obs[0].time,
+        totime=to_time,
+    )
+    obs_get_response = grpc_stub.GetObservations(obs_get_request)
+    assert obs_get_response.tsobs[0].tsid == dummy_timeseries_for_delete.id
+    assert len(obs_get_response.tsobs[0].obs) == 3
+
+    ts_delete_request = dstore.DeleteTSRequest(ids=[dummy_timeseries_for_delete.id])
+    grpc_stub.DeleteTimeSeries(ts_delete_request)
+
+    ts_find_response = grpc_stub.FindTimeSeries(ts_find_request)
+    assert len(ts_find_response.tseries) == 0
+
+    obs_get_response = grpc_stub.GetObservations(obs_get_request)
+    assert obs_get_response.tsobs[0].tsid == dummy_timeseries_for_delete.id
+    assert len(obs_get_response.tsobs[0].obs) == 0

integration-test/test_knmi.py

@@ -2,27 +2,24 @@
 import os
 from datetime import datetime
 
-
-import pytest
-
-from google.protobuf.timestamp_pb2 import Timestamp
-
 import datastore_pb2 as dstore
 import datastore_pb2_grpc as dstore_grpc
 import grpc
+import pytest
+from google.protobuf.timestamp_pb2 import Timestamp
+
+
+NUMBER_OF_PARAMETERS = 44
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def grpc_stub():
     with grpc.insecure_channel(f"{os.getenv('DSHOST', 'localhost')}:{os.getenv('DSPORT', '50050')}") as channel:
         yield dstore_grpc.DatastoreStub(channel)
 
 
 def test_find_series_single_station_single_parameter(grpc_stub):
-    request = dstore.FindTSRequest(
-            station_ids=["06260"],
-            param_ids=["rh"]
-        )
+    request = dstore.FindTSRequest(station_ids=["06260"], param_ids=["rh"])
     response = grpc_stub.FindTimeSeries(request)
 
     assert len(response.tseries) == 1
@@ -31,9 +28,7 @@ def test_find_series_single_station_single_parameter(grpc_stub):
 
 
 def test_find_series_all_stations_single_parameter(grpc_stub):
-    request = dstore.FindTSRequest(
-        param_ids=["rh"]
-    )
+    request = dstore.FindTSRequest(param_ids=["rh"])
     response = grpc_stub.FindTimeSeries(request)
 
     assert len(response.tseries) == 55
@@ -45,14 +40,11 @@ def test_find_series_single_station_all_parameters(grpc_stub):
     )
     response = grpc_stub.FindTimeSeries(request)
 
-    assert len(response.tseries) == 44
+    assert len(response.tseries) == NUMBER_OF_PARAMETERS
 
 
-def test_get_values_single_station_single_paramters(grpc_stub):
-    ts_request = dstore.FindTSRequest(
-        station_ids=["06260"],
-        param_ids=["rh"]
-    )
+def test_get_values_single_station_single_parameters(grpc_stub):
+    ts_request = dstore.FindTSRequest(station_ids=["06260"], param_ids=["rh"])
     ts_response = grpc_stub.FindTimeSeries(ts_request)
     assert len(ts_response.tseries) == 1
     ts_id = ts_response.tseries[0].id
@@ -73,3 +65,83 @@ def test_get_values_single_station_single_paramters(grpc_stub):
     assert len(response.tsobs[0].obs) == 144
     assert response.tsobs[0].obs[0].value == 95.0
     assert response.tsobs[0].obs[-1].value == 59.0
+
+
+input_params_polygon = [
+    (
+        # Multiple stations within
+        ((52.15, 4.90), (52.15, 5.37), (51.66, 5.37), (51.66, 4.90)),
+        None,
+        ["06260", "06348", "06356"],
+    ),
+    (
+        # Multiple stations with a single parameter
+        ((52.15, 4.90), (52.15, 5.37), (51.66, 5.37), (51.66, 4.90)),
+        ["rh"],
+        ["06260", "06348", "06356"],
+    ),
+    (
+        # Multiple stations with multiple parameters
+        ((52.15, 4.90), (52.15, 5.37), (51.66, 5.37), (51.66, 4.90)),
+        ["dd", "rh", "tx"],
+        ["06260", "06348", "06356"],
+    ),
+    (
+        # One station within
+        ((52.11, 5.15), (52.11, 5.204), (52.08, 5.204), (52.08, 5.15)),
+        None,
+        ["06260"],
+    ),
+    (
+        # Nothing within
+        ((51.82, 5.07), (51.82, 5.41), (51.73, 5.41), (51.73, 5.07)),
+        None,
+        [],
+    ),
+    (
+        # Middle top
+        ((52.0989, 4.17), (52.0989, 6.18), (52.09, 6.18), (52.09, 4.17)),
+        None,
+        ["06260"],
+    ),
+    (
+        # Middle bottom, should fall outside since polygon is curved because the earth is round (postgres geography).
+        ((52.1, 4.17), (52.1, 6.18), (52.0989, 6.18), (52.0989, 4.17)),
+        None,
+        [],
+    ),
+    (
+        # Complex polygon
+        ((51.45, 3.47), (51.39, 3.67), (51.39, 4.28), (51.52, 4.96), (51.89, 5.46), (52.18, 5.30), (51.75, 3.68)),
+        None,
+        ["06260", "06310", "06323", "06340", "06343", "06348", "06350", "06356"],
+    ),
+    (
+        # All stations in the Netherlands
+        ((56.00, 2.85), (56.00, 7.22), (50.75, 7.22), (50.75, 2.85)),
+        None,
+        # fmt: off
+        [
+            "06201", "06203", "06204", "06205", "06207", "06208", "06211", "06214", "06215", "06225", "06229",
+            "06235", "06239", "06240", "06242", "06248", "06249", "06251", "06252", "06257", "06258", "06260",
+            "06267", "06269", "06270", "06273", "06275", "06277", "06278", "06279", "06280", "06283", "06286",
+            "06290", "06310", "06317", "06319", "06320", "06321", "06323", "06330", "06340", "06343", "06344",
+            "06348", "06350", "06356", "06370", "06375", "06377", "06380", "06391"
+        ],
+        # fmt: on
+    ),
+]
+
+
+@pytest.mark.parametrize("coords,param_ids,expected_station_ids", input_params_polygon)
+def test_get_observations_with_polygon(grpc_stub, coords, param_ids, expected_station_ids):
+    ts_request = dstore.FindTSRequest(
+        inside=dstore.Polygon(points=[dstore.Point(lat=lat, lon=lon) for lat, lon in coords]), param_ids=param_ids
+    )
+    ts_response = grpc_stub.FindTimeSeries(ts_request)
+
+    actual_station_ids = sorted({ts.metadata.station_id for ts in ts_response.tseries})
+    assert actual_station_ids == expected_station_ids
+    number_of_parameters = len(param_ids) if param_ids else NUMBER_OF_PARAMETERS
+    expected_number_of_timeseries = number_of_parameters * len(expected_station_ids)
+    assert len(ts_response.tseries) == expected_number_of_timeseries