From 3f05f0db046fb68a3c2134afb5116ec5f7fb6da3 Mon Sep 17 00:00:00 2001 From: jukkap Date: Tue, 19 Nov 2024 14:01:35 +0200 Subject: [PATCH 1/5] feat: levels filtering with combination --- api/utilities.py | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/api/utilities.py b/api/utilities.py index e56d3bc4..9fe83908 100644 --- a/api/utilities.py +++ b/api/utilities.py @@ -5,6 +5,7 @@ from datetime import datetime from datetime import timedelta from typing import Tuple +from itertools import chain from isodate import ISO8601Error import datastore_pb2 as dstore @@ -181,25 +182,46 @@ def get_periods_or_range(periods: str) -> list[str]: def get_z_levels_or_range(z: str) -> list[str]: """ - Function for getting the levels as a list or a range + Function for getting the levels filters as a list of levels, ranges + or range intervals """ # it can be z=value1,value2,value3: z=2,10,80 # or z=minimum value/maximum value: z=10/100 # or z=Rn/min height/height interval: z=R20/100/50 + # or a combination of the above: z=10,30/100,200,300,R20/100/50 + + values = [level_or_range.split("/") for level_or_range in split_and_strip(z)] + try: - split_on_slash = z.split("/") - if len(split_on_slash) == 2: - z_min = convert_m_to_cm(split_on_slash[0]) if split_on_slash[0] != ".." else -sys.maxsize - 1 - z_max = convert_m_to_cm(split_on_slash[1]) if split_on_slash[1] != ".." else sys.maxsize - return [f"{z_min}/{z_max}"] - elif len(split_on_slash) > 2: - return get_z_values_from_interval(split_on_slash) - else: - return [convert_m_to_cm(level) for level in z.split(",")] + nested_filters = [ + ( + get_z_values_from_interval(level_or_range) + if len(level_or_range) > 2 + else ( + get_z_values_from_range(level_or_range) + if len(level_or_range) == 2 + else [convert_m_to_cm(level_or_range[0])] + ) + ) + for level_or_range in values + ] + # Return the flattened list of level filters + return list(chain(*nested_filters)) except ValueError: raise HTTPException(status_code=400, detail=f"Invalid levels value: {z}") +def get_z_values_from_range(range: list[str]) -> list[str]: + """ + Function for getting the z values from a range. + """ + start, end = range[0], range[1] + z_min = convert_m_to_cm(start) if start != ".." else start + z_max = convert_m_to_cm(end) if end != ".." else end + + return [f"{z_min}/{z_max}"] + + def get_z_values_from_interval(interval: list[str]) -> list[str]: """ Function for getting the z values from a repeating-interval pattern. From 71c92cf7e937f29366f238db299fa8bd35d87470 Mon Sep 17 00:00:00 2001 From: jukkap Date: Tue, 19 Nov 2024 14:02:11 +0200 Subject: [PATCH 2/5] docs: update openapi examples --- api/openapi/custom_dimension_examples.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/openapi/custom_dimension_examples.py b/api/openapi/custom_dimension_examples.py index 00ea623a..de7c0c7a 100644 --- a/api/openapi/custom_dimension_examples.py +++ b/api/openapi/custom_dimension_examples.py @@ -36,6 +36,10 @@ "summary": "Open start range", "value": "../10.0", }, + "Combination": { + "summary": "Combination", + "value": "1.0, 1.5/1.8, R5/2.0/2.0", + }, } periods = { @@ -54,6 +58,10 @@ "summary": "Open end range", "value": "PT0S/..", }, + "Combination": { + "summary": "Combination", + "value": "PT0S, PT1M/PT10M", + }, } methods = { From 779c6a4da8ffac7b1daff9d13c640e2fd21fe7a1 Mon Sep 17 00:00:00 2001 From: jukkap Date: Tue, 19 Nov 2024 14:02:39 +0200 Subject: [PATCH 3/5] test: add unit tests for levels combination --- api/test/test_edr.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/api/test/test_edr.py b/api/test/test_edr.py index 5931689e..ee741965 100644 --- a/api/test/test_edr.py +++ b/api/test/test_edr.py @@ -410,3 +410,33 @@ def test_get_data_with_lowercase_period_range_without_existing_data(): assert response.status_code == 404 assert response.json() == {"detail": "Requested data not found."} + + +def test_get_data_with_combination_levels_filtering(): + with patch("routers.edr.get_obs_request") as mock_get_obs_request: + + # Load with random test data for making a mock_obs_request + test_data = load_json("test/test_data/test_coverages_proto.json") + mock_get_obs_request.return_value = create_mock_obs_response(test_data) + + response = client.get("/collections/observations/locations/0-20000-0-06260?levels=R6/0.0/0.1, 1.5/1.8, 10") + + m_args = mock_get_obs_request.call_args[0][0] + + assert response.status_code == 200 + assert m_args.filter["level"].values == ["0", "10", "20", "30", "40", "50", "150/180", "1000"] + + +def test_get_data_with_combination_periods_filtering(): + with patch("routers.edr.get_obs_request") as mock_get_obs_request: + + # Load with random test data for making a mock_obs_request + test_data = load_json("test/test_data/test_coverages_proto.json") + mock_get_obs_request.return_value = create_mock_obs_response(test_data) + + response = client.get("/collections/observations/locations/0-20000-0-06260?periods=PT0S, PT1M/PT10M") + + m_args = mock_get_obs_request.call_args[0][0] + + assert response.status_code == 200 + assert m_args.filter["period"].values == ["0", "60/600"] From f119e88e7f8435f72a3718fd7b372bd3e8fc2121 Mon Sep 17 00:00:00 2001 From: jukkap Date: Fri, 22 Nov 2024 14:39:42 +0200 Subject: [PATCH 4/5] refactor: levels filtering to a nested function --- api/utilities.py | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/api/utilities.py b/api/utilities.py index 9fe83908..9a9e60d6 100644 --- a/api/utilities.py +++ b/api/utilities.py @@ -156,7 +156,7 @@ async def add_request_parameters( request.filter["standard_name"].values.extend(split_and_strip(standard_names)) if levels: - request.filter["level"].values.extend(get_z_levels_or_range(levels)) + request.filter["level"].values.extend(get_levels_values(levels)) if methods: request.filter["function"].values.extend(split_and_strip(methods)) @@ -180,35 +180,29 @@ def get_periods_or_range(periods: str) -> list[str]: raise HTTPException(status_code=400, detail=f"{err}") -def get_z_levels_or_range(z: str) -> list[str]: +def get_levels_values(levels: str) -> list[str]: """ - Function for getting the levels filters as a list of levels, ranges - or range intervals + Function for getting the levels filters as a list of levels, ranges, + range intervals or combination of the previous """ # it can be z=value1,value2,value3: z=2,10,80 # or z=minimum value/maximum value: z=10/100 # or z=Rn/min height/height interval: z=R20/100/50 # or a combination of the above: z=10,30/100,200,300,R20/100/50 - values = [level_or_range.split("/") for level_or_range in split_and_strip(z)] + def get_level_or_range(z: str) -> list[str]: + try: + split_on_slash = z.split("/") + if len(split_on_slash) == 2: + return get_z_values_from_range(split_on_slash) + elif len(split_on_slash) > 2: + return get_z_values_from_interval(split_on_slash) + else: + return [convert_m_to_cm(z)] + except ValueError: + raise HTTPException(status_code=400, detail=f"Invalid levels value: {z}") - try: - nested_filters = [ - ( - get_z_values_from_interval(level_or_range) - if len(level_or_range) > 2 - else ( - get_z_values_from_range(level_or_range) - if len(level_or_range) == 2 - else [convert_m_to_cm(level_or_range[0])] - ) - ) - for level_or_range in values - ] - # Return the flattened list of level filters - return list(chain(*nested_filters)) - except ValueError: - raise HTTPException(status_code=400, detail=f"Invalid levels value: {z}") + return list(chain(*[get_level_or_range(z) for z in split_and_strip(levels)])) def get_z_values_from_range(range: list[str]) -> list[str]: From cd4cb0199490b9dd0e735ccf014927aa0605ce23 Mon Sep 17 00:00:00 2001 From: jukkap Date: Fri, 22 Nov 2024 15:49:52 +0200 Subject: [PATCH 5/5] test: unit test for get_levels_values --- api/test/test_api_utilities.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 api/test/test_api_utilities.py diff --git a/api/test/test_api_utilities.py b/api/test/test_api_utilities.py new file mode 100644 index 00000000..d0667f91 --- /dev/null +++ b/api/test/test_api_utilities.py @@ -0,0 +1,19 @@ +import pytest + +from utilities import get_levels_values + +levels_in_out = [ + ("1", ["100"]), + ("1,2", ["100", "200"]), + ("1,2, 3", ["100", "200", "300"]), + ("1/3", ["100/300"]), + ("../3", ["../300"]), + ("1/..", ["100/.."]), + ("R3/1.2/0.3", ["120", "150", "180"]), + ("1, 3/5, R3/5/0.1,11", ["100", "300/500", "500", "510", "520", "1100"]), +] + + +@pytest.mark.parametrize("levels_in, levels_out", levels_in_out) +def test_get_levels_values(levels_in, levels_out): + assert get_levels_values(levels_in) == levels_out