test: adjust test data generation & test responses

EURODEO · Nov 11, 2024 · fef62df · fef62df · github-actions · Nov 11, 2024
1 parent 23103c4
commit fef62df
Show file tree

Hide file tree

Showing 7 changed files with 22 additions and 205 deletions.
diff --git a/datastore/data-loader/client_knmi_station.py b/datastore/data-loader/client_knmi_station.py
@@ -38,16 +38,16 @@ def netcdf_file_to_requests(file_path: Path | str) -> Tuple[List, List]:
 
             for param_id in knmi_parameter_names:
                 param_file = station_slice[param_id]
-                standard_name, level, function, period = generate_parameter_name(
-                    (param_file.standard_name if "standard_name" in param_file.attrs else None),
-                    param_file.long_name,
-                    station_id,
-                    station_name,
-                    param_id,
-                )
 
-                # Drop parameters that are not CF compliant
-                if not standard_name or standard_name in ["precipitation_duration", "rainfall_duration"]:
+                if "standard_name" in param_file.attrs:
+                    standard_name, level, function, period = generate_parameter_name(
+                        param_file.standard_name,
+                        param_file.long_name,
+                        station_id,
+                        station_name,
+                        param_id,
+                    )
+                else:
                     continue
 
                 platform = f"0-20000-0-{station_id}"

diff --git a/datastore/data-loader/client_knmi_station_ingest.py b/datastore/data-loader/client_knmi_station_ingest.py
@@ -34,15 +34,16 @@ def netcdf_file_to_requests(file_path: Path | str) -> Tuple[List, List]:
 
             for param_id in knmi_parameter_names:
                 param_file = station_slice[param_id]
-                standard_name, level, function, period = generate_parameter_name(
-                    (param_file.standard_name if "standard_name" in param_file.attrs else None),
-                    param_file.long_name,
-                    station_id,
-                    station_name,
-                    param_id,
-                )
-
-                if not standard_name or standard_name in ["precipitation_duration", "rainfall_duration"]:
+
+                if "standard_name" in param_file.attrs:
+                    standard_name, level, function, period = generate_parameter_name(
+                        param_file.standard_name,
+                        param_file.long_name,
+                        station_id,
+                        station_name,
+                        param_id,
+                    )
+                else:
                     continue
 
                 platform = f"0-20000-0-{station_id}"

diff --git a/datastore/data-loader/parameters.py b/datastore/data-loader/parameters.py
@@ -1,18 +1,14 @@
 knmi_parameter_names = (
-    "hc3",
-    "nc2",
+    "hc",
+    "nc",
     "zm",
     "R1H",
     "tgn",
     "Tn12",
-    "pr",
     "pg",
     "tn",
     "rg",
-    "hc1",
-    "nc1",
     "ts1",
-    "nc3",
     "ts2",
     "qg",
     "ff",
@@ -24,17 +20,14 @@
     "Tgn12",
     "ss",
     "Tn6",
-    "dr",
     "rh",
-    "hc2",
     "Tgn6",
     "R12H",
     "R24H",
     "Tx6",
     "Tx24",
     "Tx12",
     "Tgn14",
-    "D1H",
     "R6H",
     "pwc",
     "tx",

diff --git a/datastore/data-loader/utilities.py b/datastore/data-loader/utilities.py
@@ -78,15 +78,6 @@ def generate_parameter_name(standard_name, long_name, station_id, station_name,
         else:
             level = "1.8"
 
-    # We want "level" to be numeric, so get rid of "first", "second" and "third".
-    # Note that this level has no meaning, it is just a hack for this test dataset
-    if level == "first":
-        level = "0"
-    if level == "second":
-        level = "1.0"
-    if level == "third":
-        level = "2.0"
-
     if "minimum" in long_name:
         function = "minimum"
     elif "maximum" in long_name:

diff --git a/...tegration-test/response/data_locations_two_points_with_multiple_parameters.json b/...tegration-test/response/data_locations_two_points_with_multiple_parameters.json
@@ -28,11 +28,7 @@
                     "air_temperature:2.0:minimum:PT12H",
                     "air_temperature:2.0:minimum:PT14H",
                     "air_temperature:2.0:minimum:PT6H",
-                    "cloud_area_fraction:0.0:point:PT0S",
-                    "cloud_area_fraction:1.0:point:PT0S",
                     "cloud_area_fraction:2.0:point:PT0S",
-                    "cloud_base_altitude:0.0:point:PT0S",
-                    "cloud_base_altitude:1.0:point:PT0S",
                     "cloud_base_altitude:2.0:point:PT0S",
                     "dew_point_temperature:1.5:mean:PT1M",
                     "duration_of_sunshine:2.0:point:PT0S",
@@ -79,11 +75,7 @@
                     "air_temperature:2.0:minimum:PT12H",
                     "air_temperature:2.0:minimum:PT14H",
                     "air_temperature:2.0:minimum:PT6H",
-                    "cloud_area_fraction:0.0:point:PT0S",
-                    "cloud_area_fraction:1.0:point:PT0S",
                     "cloud_area_fraction:2.0:point:PT0S",
-                    "cloud_base_altitude:0.0:point:PT0S",
-                    "cloud_base_altitude:1.0:point:PT0S",
                     "cloud_base_altitude:2.0:point:PT0S",
                     "dew_point_temperature:1.5:mean:PT1M",
                     "duration_of_sunshine:2.0:point:PT0S",
@@ -427,52 +419,6 @@
             "rodeo:standard_name": "air_temperature",
             "rodeo:level": 2.0
         },
-        "cloud_area_fraction:0.0:point:PT0S": {
-            "type": "Parameter",
-            "description": {
-                "en": "Cloud area fraction at 0.0m, aggregated over PT0S with method 'point'"
-            },
-            "observedProperty": {
-                "id": "https://vocab.nerc.ac.uk/standard_name/cloud_area_fraction",
-                "label": {
-                    "en": "Cloud area fraction"
-                }
-            },
-            "unit": {
-                "label": {
-                    "en": "oktas"
-                }
-            },
-            "measurementType": {
-                "method": "point",
-                "period": "PT0S"
-            },
-            "rodeo:standard_name": "cloud_area_fraction",
-            "rodeo:level": 0.0
-        },
-        "cloud_area_fraction:1.0:point:PT0S": {
-            "type": "Parameter",
-            "description": {
-                "en": "Cloud area fraction at 1.0m, aggregated over PT0S with method 'point'"
-            },
-            "observedProperty": {
-                "id": "https://vocab.nerc.ac.uk/standard_name/cloud_area_fraction",
-                "label": {
-                    "en": "Cloud area fraction"
-                }
-            },
-            "unit": {
-                "label": {
-                    "en": "oktas"
-                }
-            },
-            "measurementType": {
-                "method": "point",
-                "period": "PT0S"
-            },
-            "rodeo:standard_name": "cloud_area_fraction",
-            "rodeo:level": 1.0
-        },
         "cloud_area_fraction:2.0:point:PT0S": {
             "type": "Parameter",
             "description": {
@@ -496,52 +442,6 @@
             "rodeo:standard_name": "cloud_area_fraction",
             "rodeo:level": 2.0
         },
-        "cloud_base_altitude:0.0:point:PT0S": {
-            "type": "Parameter",
-            "description": {
-                "en": "Cloud base altitude at 0.0m, aggregated over PT0S with method 'point'"
-            },
-            "observedProperty": {
-                "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude",
-                "label": {
-                    "en": "Cloud base altitude"
-                }
-            },
-            "unit": {
-                "label": {
-                    "en": "m"
-                }
-            },
-            "measurementType": {
-                "method": "point",
-                "period": "PT0S"
-            },
-            "rodeo:standard_name": "cloud_base_altitude",
-            "rodeo:level": 0.0
-        },
-        "cloud_base_altitude:1.0:point:PT0S": {
-            "type": "Parameter",
-            "description": {
-                "en": "Cloud base altitude at 1.0m, aggregated over PT0S with method 'point'"
-            },
-            "observedProperty": {
-                "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude",
-                "label": {
-                    "en": "Cloud base altitude"
-                }
-            },
-            "unit": {
-                "label": {
-                    "en": "m"
-                }
-            },
-            "measurementType": {
-                "method": "point",
-                "period": "PT0S"
-            },
-            "rodeo:standard_name": "cloud_base_altitude",
-            "rodeo:level": 1.0
-        },
         "cloud_base_altitude:2.0:point:PT0S": {
             "type": "Parameter",
             "description": {

diff --git a/datastore/integration-test/response/metadata_collections_all_collections.json b/datastore/integration-test/response/metadata_collections_all_collections.json
@@ -406,40 +406,6 @@
                     "rodeo:standard_name": "air_temperature",
                     "rodeo:level": 2.0
                 },
-                "cloud_area_fraction:0.0:point:PT0S": {
-                    "type": "Parameter",
-                    "description": "Cloud area fraction at 0.0m, aggregated over PT0S with method 'point'",
-                    "unit": {
-                        "label": "oktas"
-                    },
-                    "observedProperty": {
-                        "id": "https://vocab.nerc.ac.uk/standard_name/cloud_area_fraction",
-                        "label": "Cloud area fraction"
-                    },
-                    "measurementType": {
-                        "method": "point",
-                        "period": "PT0S"
-                    },
-                    "rodeo:standard_name": "cloud_area_fraction",
-                    "rodeo:level": 0.0
-                },
-                "cloud_area_fraction:1.0:point:PT0S": {
-                    "type": "Parameter",
-                    "description": "Cloud area fraction at 1.0m, aggregated over PT0S with method 'point'",
-                    "unit": {
-                        "label": "oktas"
-                    },
-                    "observedProperty": {
-                        "id": "https://vocab.nerc.ac.uk/standard_name/cloud_area_fraction",
-                        "label": "Cloud area fraction"
-                    },
-                    "measurementType": {
-                        "method": "point",
-                        "period": "PT0S"
-                    },
-                    "rodeo:standard_name": "cloud_area_fraction",
-                    "rodeo:level": 1.0
-                },
                 "cloud_area_fraction:2.0:point:PT0S": {
                     "type": "Parameter",
                     "description": "Cloud area fraction at 2.0m, aggregated over PT0S with method 'point'",
@@ -457,40 +423,6 @@
                     "rodeo:standard_name": "cloud_area_fraction",
                     "rodeo:level": 2.0
                 },
-                "cloud_base_altitude:0.0:point:PT0S": {
-                    "type": "Parameter",
-                    "description": "Cloud base altitude at 0.0m, aggregated over PT0S with method 'point'",
-                    "unit": {
-                        "label": "m"
-                    },
-                    "observedProperty": {
-                        "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude",
-                        "label": "Cloud base altitude"
-                    },
-                    "measurementType": {
-                        "method": "point",
-                        "period": "PT0S"
-                    },
-                    "rodeo:standard_name": "cloud_base_altitude",
-                    "rodeo:level": 0.0
-                },
-                "cloud_base_altitude:1.0:point:PT0S": {
-                    "type": "Parameter",
-                    "description": "Cloud base altitude at 1.0m, aggregated over PT0S with method 'point'",
-                    "unit": {
-                        "label": "m"
-                    },
-                    "observedProperty": {
-                        "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude",
-                        "label": "Cloud base altitude"
-                    },
-                    "measurementType": {
-                        "method": "point",
-                        "period": "PT0S"
-                    },
-                    "rodeo:standard_name": "cloud_base_altitude",
-                    "rodeo:level": 1.0
-                },
                 "cloud_base_altitude:2.0:point:PT0S": {
                     "type": "Parameter",
                     "description": "Cloud base altitude at 2.0m, aggregated over PT0S with method 'point'",

diff --git a/datastore/integration-test/test_knmi.py b/datastore/integration-test/test_knmi.py
@@ -47,7 +47,7 @@ def test_find_series_single_station_all_parameters(grpc_stub):
     response = grpc_stub.GetObservations(request)
 
     assert (
-        len([obs.ts_mdata.parameter_name for obs in response.observations]) == 34
+        len([obs.ts_mdata.parameter_name for obs in response.observations]) == 30
     )  # Station 06260 doesn't have all parameters
File	Stmts	Miss	Cover	Missing
__init__.py	0	0	100%
datastore_pb2.py	61	48	21%	34–81
datastore_pb2_grpc.py	54	27	50%	15–16, 19, 65–80, 121–123, 128–130, 135–137, 142–144, 148–173, 219, 246, 273, 300
export_metrics.py	10	0	100%
grpc_getter.py	20	11	45%	15–19, 23–26, 30–32, 36–38
locustfile.py	15	15	0%	1–31
main.py	43	7	84%	45, 50, 60, 70–71, 81–82
metadata_endpoints.py	66	32	52%	45–54, 58, 85, 100–220, 224
response_classes.py	5	0	100%
utilities.py	174	46	74%	20, 38, 45, 67–70, 78–89, 94–101, 121, 125, 127, 155, 161, 179, 193–194, 198, 214–218, 222–228, 232–234, 264, 268, 290, 295
custom_geo_json
edr_feature_collection.py	6	0	100%
formatters
__init__.py	11	0	100%
covjson.py	60	1	98%	91
geojson.py	21	2	90%	27, 52
openapi
custom_dimension_examples.py	4	0	100%
edr_query_parameter_descriptions.py	11	0	100%
openapi_examples.py	13	0	100%
routers
__init__.py	0	0	100%
edr.py	101	4	96%	348–349, 438–439
feature.py	47	19	60%	99–132, 148–153, 159–181
TOTAL	722	212	71%
Title	Coverage	Tests	Skipped	Failures	Errors	Time
API Unit Tests		30	0 💤	0 ❌	0 🔥	1.836s ⏱️
Ingest Unit Tests		16	0 💤	0 ❌	0 🔥	1m 46s ⏱️