From 76ace7d58b36bb5c2a6e4d4eeca09f117711cdac Mon Sep 17 00:00:00 2001 From: Lukas Phaf Date: Fri, 13 Dec 2024 15:11:13 +0100 Subject: [PATCH 1/3] Fix local locust loadtest, and add EWC loadtest code to repo. --- api/load-test/locust-ewc.py | 88 +++++++++++++++++++++++++++++++++++++ api/load-test/locustfile.py | 41 +++++++++++++++++ api/locustfile.py | 36 --------------- api/test/.coveragerc | 4 +- 4 files changed, 132 insertions(+), 37 deletions(-) create mode 100644 api/load-test/locust-ewc.py create mode 100644 api/load-test/locustfile.py delete mode 100644 api/locustfile.py diff --git a/api/load-test/locust-ewc.py b/api/load-test/locust-ewc.py new file mode 100644 index 00000000..fede5c08 --- /dev/null +++ b/api/load-test/locust-ewc.py @@ -0,0 +1,88 @@ +from datetime import datetime, timedelta, UTC +import random + +from locust import HttpUser +from locust import task + +headers = {"Accept-Encoding": "br"} + +common_standard_names = [ + "wind_speed", + "wind_from_direction", + "air_temperature", +] + +polygon_size = [0.5, 1.0, 2.0, 4.0] +hours_choice = [1, 3, 6, 12, 24] + + +class ESohUser(HttpUser): + def on_start(self): + response = self.client.get("/collections/observations/locations", headers=headers) + stations = response.json()["features"] + self.stations = {s["id"]: s["properties"]["parameter-name"] for s in stations} + self.station_ids = list(self.stations.keys()) + self.stations_by_location = {(s["geometry"]["coordinates"][0],s["geometry"]["coordinates"][1]): s["properties"]["parameter-name"] for s in stations} + self.station_locations = list(self.stations_by_location.keys()) + + # @task + # def get_data_single_station_single_parameter(self): + # station_id = random.choice(self.station_ids) + # parameter = random.choice(self.stations[station_id]) + # response = self.client.get( + # f"/collections/observations/locations/{station_id}?parameter-name={parameter}", + # name="location", + # headers=headers + # ) + + @task + def get_data_single_station_single_parameter_last_x_hours(self): + hours = random.choice(hours_choice) + date_time = datetime.now(UTC) - timedelta(hours=hours) + dt_string = date_time.strftime('%Y-%m-%dT%H:%M:%SZ') + station_id = random.choice(self.station_ids) + parameter = random.choice(self.stations[station_id]) + response = self.client.get( + f"/collections/observations/locations/{station_id}?parameter-name={parameter}&datetime={dt_string}/..", + name=f"location {hours:02d} hours", + headers=headers + ) + + @task + def get_data_single_position_single_parameter(self): + (lon, lat) = random.choice(self.station_locations) + parameter = random.choice(self.stations_by_location[(lon, lat)]) + response = self.client.get( + f"/collections/observations/position?coords=POINT({lon} {lat})¶meter-name={parameter}", + name="position", + headers=headers + ) + + @task + def get_data_area_single_parameter_last_hour(self): + date_time = datetime.now(UTC) - timedelta(hours=1) + dt_string = date_time.strftime('%Y-%m-%dT%H:%M:%SZ') + standard_name = random.choice(common_standard_names) + (cx, cy) = random.choice(self.station_locations) + sz = random.choice(polygon_size)/2.0 + left = cx - sz + bottom = cy - sz + right = cx + sz + top = cy + sz + polygon = f"POLYGON(({left} {bottom},{right} {bottom},{right} {top},{left} {top},{left} {bottom}))" + url = f"/collections/observations/area?coords={polygon}&standard_names={standard_name}&datetime={dt_string}/.." + response = self.client.get( + url, + name=f"area {sz*2.0}deg x {sz*2.0}deg x 1h", + headers=headers + ) + # if sz == 2.0: + # j = response.json() + # # print(sz*2.0) + # if response.status_code != 200: + # print(0) + # elif j["type"] == "CoverageCollection": + # print(len(j["coverages"])) + # else: + # print(1) + # # print(j) diff --git a/api/load-test/locustfile.py b/api/load-test/locustfile.py new file mode 100644 index 00000000..fc87145c --- /dev/null +++ b/api/load-test/locustfile.py @@ -0,0 +1,41 @@ +import random + +from locust import HttpUser +from locust import task + +parameters = ["wind_speed:10.0:mean:PT10M", "wind_speed:10.0:mean:PT10M", "relative_humidity:2.0:mean:PT1M", + "air_temperature:2.0:mean:PT1M"] +# fmt: off +stations = [ + "0-20000-0-06203", "0-20000-0-06204", "0-20000-0-06205", "0-20000-0-06207", "0-20000-0-06208", "0-20000-0-06211", + "0-20000-0-06214", "0-20000-0-06215", "0-20000-0-06235", "0-20000-0-06239", "0-20000-0-06242", "0-20000-0-06251", + "0-20000-0-06260", "0-20000-0-06269", "0-20000-0-06270", "0-20000-0-06275", "0-20000-0-06279", "0-20000-0-06280", + "0-20000-0-06290", "0-20000-0-06310", "0-20000-0-06317", "0-20000-0-06319", "0-20000-0-06323", "0-20000-0-06330", + "0-20000-0-06340", "0-20000-0-06344", "0-20000-0-06348", "0-20000-0-06350", "0-20000-0-06356", "0-20000-0-06370", + "0-20000-0-06375", "0-20000-0-06380", +] +# fmt: on +headers = {"Accept-Encoding": "br"} + + +class WebsiteUser(HttpUser): + @task + def get_data_single_station_single_parameter(self): + parameter = random.choice(parameters) + station_id = random.choice(stations) + response = self.client.get( + f"/collections/observations/locations/{station_id}?parameter-name={parameter}", + name=f"single station {parameter}", + headers=headers, + ) + if response.status_code != 200: + print(station_id, parameter) + + @task + def get_data_bbox_three_parameters(self): + self.client.get( + "/collections/observations/area?parameter-name=wind_speed:10.0:mean:PT10M,wind_speed:10.0:mean:PT10M,relative_humidity:2.0:mean:PT1M&" + "coords=POLYGON((5.0 52.0,6.0 52.0,6.0 52.1,5.0 52.1,5.0 52.0))", + name="bbox", + headers=headers, + ) diff --git a/api/locustfile.py b/api/locustfile.py deleted file mode 100644 index 4d24ba94..00000000 --- a/api/locustfile.py +++ /dev/null @@ -1,36 +0,0 @@ -import random - -from locust import HttpUser -from locust import task - -parameters = ["ff", "dd", "rh", "pp", "tn"] -# fmt: off -stations = [ - "06203", "06204", "06205", "06207", "06208", "06211", "06214", "06215", "06235", "06239", - "06242", "06251", "06260", "06269", "06270", "06275", "06279", "06280", "06290", "06310", - "06317", "06319", "06323", "06330", "06340", "06344", "06348", "06350", "06356", "06370", - "06375", "06380", "78871", "78873", -] -# fmt: on -headers = {"Accept-Encoding": "br"} - - -class WebsiteUser(HttpUser): - @task - def get_data_single_station_single_parameter(self): - parameter = random.choice(parameters) - station_id = random.choice(stations) - self.client.get( - f"/collections/observations/locations/{station_id}?parameter-name={parameter}", - name=f"single station {parameter}", - headers=headers, - ) - - @task - def get_data_bbox_three_parameters(self): - self.client.get( - "/collections/observations/area?parameter-name=dd,ff,rh&" - "coords=POLYGON((5.0 52.0,6.0 52.0,6.0 52.1,5.0 52.1,5.0 52.0))", - name="bbox", - headers=headers, - ) diff --git a/api/test/.coveragerc b/api/test/.coveragerc index d057a8b5..a5a0d189 100644 --- a/api/test/.coveragerc +++ b/api/test/.coveragerc @@ -1,2 +1,4 @@ [run] -omit = test/* +omit = + test/* + load-test/* From 49c1944fa29b7995e8f5e6293feaa0267fa8d553 Mon Sep 17 00:00:00 2001 From: Lukas Phaf Date: Fri, 13 Dec 2024 15:15:22 +0100 Subject: [PATCH 2/3] Linting. --- api/load-test/locust-ewc.py | 21 ++++++++++----------- api/load-test/locustfile.py | 8 ++++++-- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/api/load-test/locust-ewc.py b/api/load-test/locust-ewc.py index fede5c08..f0861a98 100644 --- a/api/load-test/locust-ewc.py +++ b/api/load-test/locust-ewc.py @@ -22,7 +22,10 @@ def on_start(self): stations = response.json()["features"] self.stations = {s["id"]: s["properties"]["parameter-name"] for s in stations} self.station_ids = list(self.stations.keys()) - self.stations_by_location = {(s["geometry"]["coordinates"][0],s["geometry"]["coordinates"][1]): s["properties"]["parameter-name"] for s in stations} + self.stations_by_location = { + (s["geometry"]["coordinates"][0], s["geometry"]["coordinates"][1]): s["properties"]["parameter-name"] + for s in stations + } self.station_locations = list(self.stations_by_location.keys()) # @task @@ -39,13 +42,13 @@ def on_start(self): def get_data_single_station_single_parameter_last_x_hours(self): hours = random.choice(hours_choice) date_time = datetime.now(UTC) - timedelta(hours=hours) - dt_string = date_time.strftime('%Y-%m-%dT%H:%M:%SZ') + dt_string = date_time.strftime("%Y-%m-%dT%H:%M:%SZ") station_id = random.choice(self.station_ids) parameter = random.choice(self.stations[station_id]) response = self.client.get( f"/collections/observations/locations/{station_id}?parameter-name={parameter}&datetime={dt_string}/..", name=f"location {hours:02d} hours", - headers=headers + headers=headers, ) @task @@ -55,27 +58,23 @@ def get_data_single_position_single_parameter(self): response = self.client.get( f"/collections/observations/position?coords=POINT({lon} {lat})¶meter-name={parameter}", name="position", - headers=headers + headers=headers, ) @task def get_data_area_single_parameter_last_hour(self): date_time = datetime.now(UTC) - timedelta(hours=1) - dt_string = date_time.strftime('%Y-%m-%dT%H:%M:%SZ') + dt_string = date_time.strftime("%Y-%m-%dT%H:%M:%SZ") standard_name = random.choice(common_standard_names) (cx, cy) = random.choice(self.station_locations) - sz = random.choice(polygon_size)/2.0 + sz = random.choice(polygon_size) / 2.0 left = cx - sz bottom = cy - sz right = cx + sz top = cy + sz polygon = f"POLYGON(({left} {bottom},{right} {bottom},{right} {top},{left} {top},{left} {bottom}))" url = f"/collections/observations/area?coords={polygon}&standard_names={standard_name}&datetime={dt_string}/.." - response = self.client.get( - url, - name=f"area {sz*2.0}deg x {sz*2.0}deg x 1h", - headers=headers - ) + response = self.client.get(url, name=f"area {sz*2.0}deg x {sz*2.0}deg x 1h", headers=headers) # if sz == 2.0: # j = response.json() # # print(sz*2.0) diff --git a/api/load-test/locustfile.py b/api/load-test/locustfile.py index fc87145c..01698252 100644 --- a/api/load-test/locustfile.py +++ b/api/load-test/locustfile.py @@ -3,8 +3,12 @@ from locust import HttpUser from locust import task -parameters = ["wind_speed:10.0:mean:PT10M", "wind_speed:10.0:mean:PT10M", "relative_humidity:2.0:mean:PT1M", - "air_temperature:2.0:mean:PT1M"] +parameters = [ + "wind_speed:10.0:mean:PT10M", + "wind_speed:10.0:mean:PT10M", + "relative_humidity:2.0:mean:PT1M", + "air_temperature:2.0:mean:PT1M", +] # fmt: off stations = [ "0-20000-0-06203", "0-20000-0-06204", "0-20000-0-06205", "0-20000-0-06207", "0-20000-0-06208", "0-20000-0-06211", From 64e76a5dd58ea909ef82dcbbdc3cdf5da4698d3f Mon Sep 17 00:00:00 2001 From: Lukas Phaf Date: Fri, 13 Dec 2024 15:21:09 +0100 Subject: [PATCH 3/3] Linting 2... --- api/load-test/locust-ewc.py | 6 +++--- api/load-test/locustfile.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/api/load-test/locust-ewc.py b/api/load-test/locust-ewc.py index f0861a98..35ead0f2 100644 --- a/api/load-test/locust-ewc.py +++ b/api/load-test/locust-ewc.py @@ -45,7 +45,7 @@ def get_data_single_station_single_parameter_last_x_hours(self): dt_string = date_time.strftime("%Y-%m-%dT%H:%M:%SZ") station_id = random.choice(self.station_ids) parameter = random.choice(self.stations[station_id]) - response = self.client.get( + self.client.get( f"/collections/observations/locations/{station_id}?parameter-name={parameter}&datetime={dt_string}/..", name=f"location {hours:02d} hours", headers=headers, @@ -55,7 +55,7 @@ def get_data_single_station_single_parameter_last_x_hours(self): def get_data_single_position_single_parameter(self): (lon, lat) = random.choice(self.station_locations) parameter = random.choice(self.stations_by_location[(lon, lat)]) - response = self.client.get( + self.client.get( f"/collections/observations/position?coords=POINT({lon} {lat})¶meter-name={parameter}", name="position", headers=headers, @@ -74,7 +74,7 @@ def get_data_area_single_parameter_last_hour(self): top = cy + sz polygon = f"POLYGON(({left} {bottom},{right} {bottom},{right} {top},{left} {top},{left} {bottom}))" url = f"/collections/observations/area?coords={polygon}&standard_names={standard_name}&datetime={dt_string}/.." - response = self.client.get(url, name=f"area {sz*2.0}deg x {sz*2.0}deg x 1h", headers=headers) + self.client.get(url, name=f"area {sz*2.0}deg x {sz*2.0}deg x 1h", headers=headers) # if sz == 2.0: # j = response.json() # # print(sz*2.0) diff --git a/api/load-test/locustfile.py b/api/load-test/locustfile.py index 01698252..2a268c60 100644 --- a/api/load-test/locustfile.py +++ b/api/load-test/locustfile.py @@ -38,7 +38,8 @@ def get_data_single_station_single_parameter(self): @task def get_data_bbox_three_parameters(self): self.client.get( - "/collections/observations/area?parameter-name=wind_speed:10.0:mean:PT10M,wind_speed:10.0:mean:PT10M,relative_humidity:2.0:mean:PT1M&" + "/collections/observations/area?parameter-name=" + "wind_speed:10.0:mean:PT10M,wind_speed:10.0:mean:PT10M,relative_humidity:2.0:mean:PT1M&" "coords=POLYGON((5.0 52.0,6.0 52.0,6.0 52.1,5.0 52.1,5.0 52.0))", name="bbox", headers=headers,