diff --git a/datastore/data-loader/client_knmi_station.py b/datastore/data-loader/client_knmi_station.py index a53367bb..1d91877b 100755 --- a/datastore/data-loader/client_knmi_station.py +++ b/datastore/data-loader/client_knmi_station.py @@ -38,16 +38,16 @@ def netcdf_file_to_requests(file_path: Path | str) -> Tuple[List, List]: for param_id in knmi_parameter_names: param_file = station_slice[param_id] - standard_name, level, function, period = generate_parameter_name( - (param_file.standard_name if "standard_name" in param_file.attrs else None), - param_file.long_name, - station_id, - station_name, - param_id, - ) - # Drop parameters that are not CF compliant - if not standard_name or standard_name in ["precipitation_duration", "rainfall_duration"]: + if "standard_name" in param_file.attrs: + standard_name, level, function, period = generate_parameter_name( + param_file.standard_name, + param_file.long_name, + station_id, + station_name, + param_id, + ) + else: continue platform = f"0-20000-0-{station_id}" diff --git a/datastore/data-loader/client_knmi_station_ingest.py b/datastore/data-loader/client_knmi_station_ingest.py index 4811d310..752a1978 100755 --- a/datastore/data-loader/client_knmi_station_ingest.py +++ b/datastore/data-loader/client_knmi_station_ingest.py @@ -34,15 +34,16 @@ def netcdf_file_to_requests(file_path: Path | str) -> Tuple[List, List]: for param_id in knmi_parameter_names: param_file = station_slice[param_id] - standard_name, level, function, period = generate_parameter_name( - (param_file.standard_name if "standard_name" in param_file.attrs else None), - param_file.long_name, - station_id, - station_name, - param_id, - ) - - if not standard_name or standard_name in ["precipitation_duration", "rainfall_duration"]: + + if "standard_name" in param_file.attrs: + standard_name, level, function, period = generate_parameter_name( + param_file.standard_name, + param_file.long_name, + station_id, + station_name, + param_id, + ) + else: continue platform = f"0-20000-0-{station_id}" diff --git a/datastore/data-loader/parameters.py b/datastore/data-loader/parameters.py index 2b11d65a..8e344653 100644 --- a/datastore/data-loader/parameters.py +++ b/datastore/data-loader/parameters.py @@ -1,18 +1,14 @@ knmi_parameter_names = ( - "hc3", - "nc2", + "hc", + "nc", "zm", "R1H", "tgn", "Tn12", - "pr", "pg", "tn", "rg", - "hc1", - "nc1", "ts1", - "nc3", "ts2", "qg", "ff", @@ -24,9 +20,7 @@ "Tgn12", "ss", "Tn6", - "dr", "rh", - "hc2", "Tgn6", "R12H", "R24H", @@ -34,7 +28,6 @@ "Tx24", "Tx12", "Tgn14", - "D1H", "R6H", "pwc", "tx", diff --git a/datastore/data-loader/utilities.py b/datastore/data-loader/utilities.py index 5fa06ab6..48ba655d 100644 --- a/datastore/data-loader/utilities.py +++ b/datastore/data-loader/utilities.py @@ -78,15 +78,6 @@ def generate_parameter_name(standard_name, long_name, station_id, station_name, else: level = "1.8" - # We want "level" to be numeric, so get rid of "first", "second" and "third". - # Note that this level has no meaning, it is just a hack for this test dataset - if level == "first": - level = "0" - if level == "second": - level = "1.0" - if level == "third": - level = "2.0" - if "minimum" in long_name: function = "minimum" elif "maximum" in long_name: diff --git a/datastore/integration-test/response/data_locations_two_points_with_multiple_parameters.json b/datastore/integration-test/response/data_locations_two_points_with_multiple_parameters.json index 98c706b9..125ab218 100644 --- a/datastore/integration-test/response/data_locations_two_points_with_multiple_parameters.json +++ b/datastore/integration-test/response/data_locations_two_points_with_multiple_parameters.json @@ -28,11 +28,7 @@ "air_temperature:2.0:minimum:PT12H", "air_temperature:2.0:minimum:PT14H", "air_temperature:2.0:minimum:PT6H", - "cloud_area_fraction:0.0:point:PT0S", - "cloud_area_fraction:1.0:point:PT0S", "cloud_area_fraction:2.0:point:PT0S", - "cloud_base_altitude:0.0:point:PT0S", - "cloud_base_altitude:1.0:point:PT0S", "cloud_base_altitude:2.0:point:PT0S", "dew_point_temperature:1.5:mean:PT1M", "duration_of_sunshine:2.0:point:PT0S", @@ -79,11 +75,7 @@ "air_temperature:2.0:minimum:PT12H", "air_temperature:2.0:minimum:PT14H", "air_temperature:2.0:minimum:PT6H", - "cloud_area_fraction:0.0:point:PT0S", - "cloud_area_fraction:1.0:point:PT0S", "cloud_area_fraction:2.0:point:PT0S", - "cloud_base_altitude:0.0:point:PT0S", - "cloud_base_altitude:1.0:point:PT0S", "cloud_base_altitude:2.0:point:PT0S", "dew_point_temperature:1.5:mean:PT1M", "duration_of_sunshine:2.0:point:PT0S", @@ -427,52 +419,6 @@ "rodeo:standard_name": "air_temperature", "rodeo:level": 2.0 }, - "cloud_area_fraction:0.0:point:PT0S": { - "type": "Parameter", - "description": { - "en": "Cloud area fraction at 0.0m, aggregated over PT0S with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_area_fraction", - "label": { - "en": "Cloud area fraction" - } - }, - "unit": { - "label": { - "en": "oktas" - } - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_area_fraction", - "rodeo:level": 0.0 - }, - "cloud_area_fraction:1.0:point:PT0S": { - "type": "Parameter", - "description": { - "en": "Cloud area fraction at 1.0m, aggregated over PT0S with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_area_fraction", - "label": { - "en": "Cloud area fraction" - } - }, - "unit": { - "label": { - "en": "oktas" - } - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_area_fraction", - "rodeo:level": 1.0 - }, "cloud_area_fraction:2.0:point:PT0S": { "type": "Parameter", "description": { @@ -496,52 +442,6 @@ "rodeo:standard_name": "cloud_area_fraction", "rodeo:level": 2.0 }, - "cloud_base_altitude:0.0:point:PT0S": { - "type": "Parameter", - "description": { - "en": "Cloud base altitude at 0.0m, aggregated over PT0S with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude", - "label": { - "en": "Cloud base altitude" - } - }, - "unit": { - "label": { - "en": "m" - } - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_base_altitude", - "rodeo:level": 0.0 - }, - "cloud_base_altitude:1.0:point:PT0S": { - "type": "Parameter", - "description": { - "en": "Cloud base altitude at 1.0m, aggregated over PT0S with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude", - "label": { - "en": "Cloud base altitude" - } - }, - "unit": { - "label": { - "en": "m" - } - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_base_altitude", - "rodeo:level": 1.0 - }, "cloud_base_altitude:2.0:point:PT0S": { "type": "Parameter", "description": { diff --git a/datastore/integration-test/response/metadata_collections_all_collections.json b/datastore/integration-test/response/metadata_collections_all_collections.json index 812c477c..5fea4e89 100644 --- a/datastore/integration-test/response/metadata_collections_all_collections.json +++ b/datastore/integration-test/response/metadata_collections_all_collections.json @@ -406,40 +406,6 @@ "rodeo:standard_name": "air_temperature", "rodeo:level": 2.0 }, - "cloud_area_fraction:0.0:point:PT0S": { - "type": "Parameter", - "description": "Cloud area fraction at 0.0m, aggregated over PT0S with method 'point'", - "unit": { - "label": "oktas" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_area_fraction", - "label": "Cloud area fraction" - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_area_fraction", - "rodeo:level": 0.0 - }, - "cloud_area_fraction:1.0:point:PT0S": { - "type": "Parameter", - "description": "Cloud area fraction at 1.0m, aggregated over PT0S with method 'point'", - "unit": { - "label": "oktas" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_area_fraction", - "label": "Cloud area fraction" - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_area_fraction", - "rodeo:level": 1.0 - }, "cloud_area_fraction:2.0:point:PT0S": { "type": "Parameter", "description": "Cloud area fraction at 2.0m, aggregated over PT0S with method 'point'", @@ -457,40 +423,6 @@ "rodeo:standard_name": "cloud_area_fraction", "rodeo:level": 2.0 }, - "cloud_base_altitude:0.0:point:PT0S": { - "type": "Parameter", - "description": "Cloud base altitude at 0.0m, aggregated over PT0S with method 'point'", - "unit": { - "label": "m" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude", - "label": "Cloud base altitude" - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_base_altitude", - "rodeo:level": 0.0 - }, - "cloud_base_altitude:1.0:point:PT0S": { - "type": "Parameter", - "description": "Cloud base altitude at 1.0m, aggregated over PT0S with method 'point'", - "unit": { - "label": "m" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude", - "label": "Cloud base altitude" - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_base_altitude", - "rodeo:level": 1.0 - }, "cloud_base_altitude:2.0:point:PT0S": { "type": "Parameter", "description": "Cloud base altitude at 2.0m, aggregated over PT0S with method 'point'", diff --git a/datastore/integration-test/test_knmi.py b/datastore/integration-test/test_knmi.py index e8d25d11..fa786780 100644 --- a/datastore/integration-test/test_knmi.py +++ b/datastore/integration-test/test_knmi.py @@ -47,7 +47,7 @@ def test_find_series_single_station_all_parameters(grpc_stub): response = grpc_stub.GetObservations(request) assert ( - len([obs.ts_mdata.parameter_name for obs in response.observations]) == 34 + len([obs.ts_mdata.parameter_name for obs in response.observations]) == 30 ) # Station 06260 doesn't have all parameters