diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c4c39dd..5a739cd6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: run: just services - name: Load the data into the database - run: just load + run: just ingest-load - name: Run the integration test run: just integration @@ -113,56 +113,56 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Checkout Source - uses: actions/checkout@v4 - - name: Install Dependencies - run: | - pip install --upgrade pip - pip install pytest-timeout - pip install pytest-cov - pip install httpx - pip install -r ./ingest/requirements.txt - pip install ./ingest - cd ./ingest && python3 api/generate_standard_name.py + - name: Install just + run: ./ci/scripts/install-just.sh + - name: Copy protobuf files + run: just copy-proto - - name: Copy Protobuf file to api directory and build - run: | - mkdir ./ingest/protobuf - cp ./protobuf/datastore.proto ./ingest/protobuf/datastore.proto - python -m grpc_tools.protoc --proto_path=./ingest/protobuf --python_out=./ingest --grpc_python_out=./ingest ./ingest/protobuf/datastore.proto + - name: Run the unit test + run: just ingest-unit - - name: Run Tests - run: | - cd ingest - mkdir -p /tmp/metrics - PROMETHEUS_MULTIPROC_DIR=/tmp/metrics python -m pytest -v --timeout=60 + - name: Archive test artifacts + uses: actions/upload-artifact@v4 + with: + name: ingest-test-results-artifact + path: | + ingest/test/output/pytest-coverage.txt + ingest/test/output/pytest.xml + + - name: Cleanup + if: always() + run: just destroy publish-test-results: - needs: test-datastore + needs: + - test-datastore + - test-ingest runs-on: ubuntu-latest - if: github.event.ref_type != 'tag' + if: github.event_name != 'push' || github.event.ref_type != 'tag' permissions: contents: write issues: write pull-requests: write steps: - - name: Download test results so that they can be published + - name: Download test-datastore results so that they can be published uses: actions/download-artifact@v4 with: name: test-results-artifact + path: ./artifacts/test-results + + - name: Download test-ingest results so that they can be published + uses: actions/download-artifact@v4 + with: + name: ingest-test-results-artifact + path: ./artifacts/ingest-results + - name: Comment coverage uses: MishaKav/pytest-coverage-comment@main with: - pytest-coverage-path: api/test/output/pytest-coverage.txt - coverage-path-prefix: api/test/output/ - title: API Unit Test Coverage Report - hide-badge: true - hide-report: false - create-new-comment: false - hide-comment: false - report-only-changed-files: false - remove-link-from-badge: false - junitxml-path: api/test/output/pytest.xml - junitxml-title: API Unit Test Coverage Summary + title: Unit Test Coverage Report + pytest-coverage-path: ./artifacts/test-results/api/test/output/pytest-coverage.txt + multiple-files: | + API Unit Tests, ./artifacts/test-results/api/test/output/pytest-coverage.txt, ./artifacts/test-results/api/test/output/pytest.xml + Ingest Unit Tests, ./artifacts/ingest-results/pytest-coverage.txt, ./artifacts/ingest-results/pytest.xml diff --git a/.gitignore b/.gitignore index d6a7061c..34c7051a 100644 --- a/.gitignore +++ b/.gitignore @@ -212,3 +212,4 @@ cf_standard_names_v84.txt # API api/test/output/ datastore/load-test/output/ +ingest/test/output/ diff --git a/api/formatters/covjson.py b/api/formatters/covjson.py index e3acf7f4..f4036bdb 100644 --- a/api/formatters/covjson.py +++ b/api/formatters/covjson.py @@ -32,6 +32,7 @@ def make_parameter(ts_mdata): level = convert_cm_to_m(ts_mdata.level) period = seconds_to_iso_8601_duration(ts_mdata.period) + label = " ".join(ts_mdata.standard_name.capitalize().split("_")) custom_fields = { "rodeo:standard_name": ts_mdata.standard_name, @@ -40,12 +41,11 @@ def make_parameter(ts_mdata): return Parameter( description={ - "en": f"{ts_mdata.standard_name} at {level}m, " - f"aggregated over {period} with method '{ts_mdata.function}'", + "en": f"{label} at {level}m, aggregated over {period} with method '{ts_mdata.function}'", }, observedProperty=ObservedProperty( id=f"https://vocab.nerc.ac.uk/standard_name/{ts_mdata.standard_name}", - label={"en": ts_mdata.parameter_name}, + label={"en": label}, ), measurementType=MeasurementType( method=ts_mdata.function, diff --git a/api/metadata_endpoints.py b/api/metadata_endpoints.py index 6b65cc7e..34eb3312 100644 --- a/api/metadata_endpoints.py +++ b/api/metadata_endpoints.py @@ -110,6 +110,7 @@ async def get_collection_metadata(base_url: str, is_self) -> Collection: ts = group.combo level = convert_cm_to_m(ts.level) period = seconds_to_iso_8601_duration(ts.period) + label = " ".join(ts.standard_name.capitalize().split("_")) custom_fields = { "rodeo:standard_name": ts.standard_name, @@ -117,10 +118,10 @@ async def get_collection_metadata(base_url: str, is_self) -> Collection: } parameter = Parameter( - description=f"{ts.standard_name} at {level}m, aggregated over {period} with method '{ts.function}'", + description=f"{label} at {level}m, aggregated over {period} with method '{ts.function}'", observedProperty=ObservedProperty( id=f"https://vocab.nerc.ac.uk/standard_name/{ts.standard_name}", - label=ts.parameter_name, + label=label, ), measurementType=MeasurementType( method=ts.function, diff --git a/api/test/test_data/test_coverages_covjson.json b/api/test/test_data/test_coverages_covjson.json index 9728b4f8..f0d354f0 100644 --- a/api/test/test_data/test_coverages_covjson.json +++ b/api/test/test_data/test_coverages_covjson.json @@ -49,12 +49,12 @@ "air_temperature:2.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT1M with method 'mean'" + "en": "Air temperature at 2.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:mean:PT1M" + "en": "Air temperature" } }, "unit": { @@ -139,12 +139,12 @@ "air_temperature:2.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT1M with method 'mean'" + "en": "Air temperature at 2.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:mean:PT1M" + "en": "Air temperature" } }, "unit": { @@ -186,12 +186,12 @@ "air_temperature:2.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT1M with method 'mean'" + "en": "Air temperature at 2.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:mean:PT1M" + "en": "Air temperature" } }, "unit": { diff --git a/api/test/test_data/test_feature_collection.json b/api/test/test_data/test_feature_collection.json index 2b6d8204..cded9712 100644 --- a/api/test/test_data/test_feature_collection.json +++ b/api/test/test_data/test_feature_collection.json @@ -44,12 +44,12 @@ "air_pressure_at_sea_level:1.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "air_pressure_at_sea_level at 1.0m, aggregated over PT1M with method 'mean'" + "en": "Air pressure at sea level at 1.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_pressure_at_sea_level", "label": { - "en": "air_pressure_at_sea_level:1.0:mean:PT1M" + "en": "Air pressure at sea level" } }, "unit": { @@ -67,12 +67,12 @@ "air_temperature:0.1:minimum:PT10M": { "type": "Parameter", "description": { - "en": "air_temperature at 0.1m, aggregated over PT10M with method 'minimum'" + "en": "Air temperature at 0.1m, aggregated over PT10M with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:0.1:minimum:PT10M" + "en": "Air temperature" } }, "unit": { diff --git a/api/test/test_data/test_multiple_covjson.json b/api/test/test_data/test_multiple_covjson.json index eeda17dd..0a53abe4 100644 --- a/api/test/test_data/test_multiple_covjson.json +++ b/api/test/test_data/test_multiple_covjson.json @@ -48,12 +48,12 @@ "relative_humidity:2.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "relative_humidity at 2.0m, aggregated over PT1M with method 'mean'" + "en": "Relative humidity at 2.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/relative_humidity", "label": { - "en": "relative_humidity:2.0:mean:PT1M" + "en": "Relative humidity" } }, "unit": { @@ -71,12 +71,12 @@ "wind_from_direction:2.0:mean:PT10M": { "type": "Parameter", "description": { - "en": "wind_from_direction at 2.0m, aggregated over PT10M with method 'mean'" + "en": "Wind from direction at 2.0m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_from_direction", "label": { - "en": "wind_from_direction:2.0:mean:PT10M" + "en": "Wind from direction" } }, "unit": { @@ -94,12 +94,12 @@ "wind_speed:10.0:mean:PT10M": { "type": "Parameter", "description": { - "en": "wind_speed at 10.0m, aggregated over PT10M with method 'mean'" + "en": "Wind speed at 10.0m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_speed", "label": { - "en": "wind_speed:10.0:mean:PT10M" + "en": "Wind speed" } }, "unit": { diff --git a/api/test/test_data/test_single_covjson.json b/api/test/test_data/test_single_covjson.json index 8013f580..68a9bf56 100644 --- a/api/test/test_data/test_single_covjson.json +++ b/api/test/test_data/test_single_covjson.json @@ -52,12 +52,12 @@ "wind_speed:10.0:mean:PT10M": { "type": "Parameter", "description": { - "en": "wind_speed at 10.0m, aggregated over PT10M with method 'mean'" + "en": "Wind speed at 10.0m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_speed", "label": { - "en": "wind_speed:10.0:mean:PT10M" + "en": "Wind speed" } }, "unit": { diff --git a/datastore/data-loader/Dockerfile b/datastore/data-loader/Dockerfile index 6be8f9b1..660c02e5 100644 --- a/datastore/data-loader/Dockerfile +++ b/datastore/data-loader/Dockerfile @@ -2,6 +2,7 @@ FROM python:3.11-slim-bookworm SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"] +ARG THROUGH_INGEST=false ENV DOCKER_PATH="/clients/python" COPY "test-data/KNMI/20221231.nc" "${DOCKER_PATH}/test-data/KNMI/20221231.nc" @@ -29,7 +30,17 @@ RUN python -m grpc_tools.protoc \ --grpc_python_out="${DOCKER_PATH}" COPY "./parameters.py" "${DOCKER_PATH}/parameters.py" +COPY "./utilities.py" "${DOCKER_PATH}/utilities.py" + +# Copy both loaders and conditionally use the wanted one +COPY "./client_knmi_station_ingest.py" "${DOCKER_PATH}/client_knmi_station_ingest.py" COPY "./client_knmi_station.py" "${DOCKER_PATH}/client_knmi_station.py" +RUN if [ "${THROUGH_INGEST}" = "true" ]; then \ + mv "${DOCKER_PATH}/client_knmi_station_ingest.py" "${DOCKER_PATH}/client_knmi_station.py"; \ + else \ + rm "${DOCKER_PATH}/client_knmi_station_ingest.py"; \ + fi + WORKDIR "${DOCKER_PATH}" CMD ["python", "-u", "./client_knmi_station.py"] diff --git a/datastore/data-loader/client_knmi_station.py b/datastore/data-loader/client_knmi_station.py index 477a1f5f..1d91877b 100755 --- a/datastore/data-loader/client_knmi_station.py +++ b/datastore/data-loader/client_knmi_station.py @@ -3,10 +3,8 @@ import concurrent import math import os -import re import uuid -import isodate -from datetime import timedelta + from hashlib import md5 from multiprocessing import cpu_count from pathlib import Path @@ -14,7 +12,6 @@ from typing import List from typing import Tuple -from isodate import ISO8601Error import datastore_pb2 as dstore import datastore_pb2_grpc as dstore_grpc import grpc @@ -22,25 +19,7 @@ import xarray as xr from google.protobuf.timestamp_pb2 import Timestamp from parameters import knmi_parameter_names - - -regex_level = re.compile(r"first|second|third|[0-9]+(\.[0-9]+)?(?=m)|(?<=Level )[0-9]+", re.IGNORECASE) -regex_level_centimeters = re.compile(r"[0-9]+(\.[0-9]+)?(?=cm)") -regex_time_period = re.compile(r"(\d+) (Hours|Min)", re.IGNORECASE) - - -def iso_8601_duration_to_seconds(period: str) -> int: - try: - duration = isodate.parse_duration(period) - except ISO8601Error: - raise ValueError(f"Invalid ISO 8601 duration: {period}") - - if isinstance(duration, timedelta): - total_seconds = duration.total_seconds() - else: - raise ValueError("Duration not convertable to seconds.") - - return int(total_seconds) +from utilities import convert_unit_names, iso_8601_duration_to_seconds, generate_parameter_name def netcdf_file_to_requests(file_path: Path | str) -> Tuple[List, List]: @@ -58,42 +37,54 @@ def netcdf_file_to_requests(file_path: Path | str) -> Tuple[List, List]: station_slice = file.sel(station=station_id) for param_id in knmi_parameter_names: - # print(station_id, param_id) param_file = station_slice[param_id] - standard_name, level, function, period, period_as_seconds = generate_parameter_name( - (param_file.standard_name if "standard_name" in param_file.attrs else "placeholder"), - param_file.long_name, - station_id, - station_name, - param_id, - ) + + if "standard_name" in param_file.attrs: + standard_name, level, function, period = generate_parameter_name( + param_file.standard_name, + param_file.long_name, + station_id, + station_name, + param_id, + ) + else: + continue + platform = f"0-20000-0-{station_id}" + period_as_seconds = iso_8601_duration_to_seconds(period) + level_as_centimeters = int(float(level) * 100) ts_mdata = dstore.TSMetadata( platform=platform, instrument=param_id, platform_name=station_name, title=param_file.long_name, + license="CC BY 4.0", standard_name=standard_name, - unit=param_file.units if "units" in param_file.attrs else None, - level=level, + unit=convert_unit_names(param_file.units) if "units" in param_file.attrs else None, + level=level_as_centimeters, period=period_as_seconds, function=function, - parameter_name=":".join([standard_name, str(float(level / 100)), function, period]), + parameter_name=":".join([standard_name, str(float(level)), function, period]), naming_authority="nl.knmi", keywords=file["iso_dataset"].attrs["keyword"], + summary=file["iso_dataset"].attrs["abstract"], keywords_vocabulary=file.attrs["references"], source=file.attrs["source"], creator_name="KNMI", creator_email=file["iso_dataset"].attrs["email_dataset"], creator_url=file["iso_dataset"].attrs["url_metadata"], - creator_type="Institution", + creator_type="institution", institution=file.attrs["institution"], timeseries_id=md5( "".join( [ - station_id, - platform + standard_name + str(float(level / 100)) + period + function + "nl.knmi", + "nl.knmi", + platform, + standard_name, + str(level_as_centimeters), + function, + str(period_as_seconds), ] ).encode() ).hexdigest(), @@ -120,76 +111,6 @@ def netcdf_file_to_requests(file_path: Path | str) -> Tuple[List, List]: return observation_request_messages -def generate_parameter_name(standard_name, long_name, station_id, station_name, param_id): - # TODO: HACK To let the loader have a unique parameter ID and make the parameters distinguishable. - level = 200 - long_name = long_name.lower() - station_name = station_name.lower() - if level_raw := re.search(regex_level, long_name): - try: - level = int(float(level_raw[0]) * 100) - except ValueError: - level = level_raw[0] - if level_raw := re.search(regex_level_centimeters, long_name): - level = int(level_raw[0]) - elif "grass" in long_name: - level = 0 - elif param_id in ["pg", "pr", "pwc", "vv", "W10", "W10-10", "ww", "ww-10", "za", "zm"]: - # https://english.knmidata.nl/open-data/actuele10mindataknmistations - # Comments code: 2, 3, 11 - # Note: The sensor is not installed at equal heights at all types of measurement sites: - # At 'AWS' sites the device is installed at 1.80m. At 'AWS/Aerodrome' and 'Mistpost' - # (note that this includes site Voorschoten (06215) which is 'AWS/Mistpost') - # the device is installed at 2.50m elevation. Exceptions are Berkhout AWS (06249), - # De Bilt AWS (06260) and Twenthe AWS (06290) where the sensor is installed at 2.50m. - # Since WaWa is automatic detection I asssumed that the others stations are AWS, thus 1.80m - if ( - station_id in ["06215", "06249", "06260", "06290"] - or "aerodrome" in station_name - or "mistpost" in station_name - ): - level = 250 - else: - level = 180 - - # We want "level" to be numeric, so get rid of "first", "second" and "third". - # Note that this level has no meaning, it is just a hack for this test dataset - if level == "first": - level = 0 - if level == "second": - level = 100 - if level == "third": - level = 200 - - if "minimum" in long_name: - function = "minimum" - elif "maximum" in long_name: - function = "maximum" - elif "average" in long_name: - function = "mean" - else: - function = "point" - - period = "PT0S" - if period_raw := re.findall(regex_time_period, long_name): - if len(period_raw) == 1: - period_raw = period_raw[0] - else: - raise Exception(f"{period_raw}, {long_name}") - time, scale = period_raw - if scale == "hours": - period = f"PT{time}H" - elif scale == "min": - period = f"PT{time}M" - elif param_id == "ww-10": - period = "PT10M" - elif param_id == "ww": - period = "PT1H" - - period_as_seconds = iso_8601_duration_to_seconds(period) - return standard_name, level, function, period, period_as_seconds - - def insert_data(observation_request_messages: List): workers = int(cpu_count()) @@ -204,7 +125,7 @@ def insert_data(observation_request_messages: List): print(f"Finished observations bulk insert {perf_counter() - obs_insert_start}.") -if __name__ == "__main__": +def main(): total_time_start = perf_counter() print("Starting with creating the time series and observations requests.") @@ -218,3 +139,7 @@ def insert_data(observation_request_messages: List): ) print(f"Finished, total time elapsed: {perf_counter() - total_time_start}") + + +if __name__ == "__main__": + main() diff --git a/datastore/data-loader/client_knmi_station_ingest.py b/datastore/data-loader/client_knmi_station_ingest.py new file mode 100755 index 00000000..752a1978 --- /dev/null +++ b/datastore/data-loader/client_knmi_station_ingest.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# tested with Python 3.11 +import math +import os +import requests +import json +from multiprocessing import cpu_count, Pool +from pathlib import Path +from time import perf_counter +from typing import List +from typing import Tuple +from functools import partial + +import pandas as pd +import xarray as xr +from google.protobuf.timestamp_pb2 import Timestamp +from parameters import knmi_parameter_names +from utilities import generate_parameter_name, convert_unit_names + + +def netcdf_file_to_requests(file_path: Path | str) -> Tuple[List, List]: + observation_request_messages = [] + + with xr.open_dataset(file_path, engine="netcdf4", chunks=None) as file: # chunks=None to disable dask + for station_id, station_name, latitude, longitude, height in zip( + file["station"].values, + file["stationname"].values[0], + file["lat"].values[0], + file["lon"].values[0], + file["height"].values[0], + ): + observations = [] + station_slice = file.sel(station=station_id) + + for param_id in knmi_parameter_names: + param_file = station_slice[param_id] + + if "standard_name" in param_file.attrs: + standard_name, level, function, period = generate_parameter_name( + param_file.standard_name, + param_file.long_name, + station_id, + station_name, + param_id, + ) + else: + continue + + platform = f"0-20000-0-{station_id}" + + properties = { + "platform": platform, + "instrument": param_id, + "platform_name": station_name, + "title": param_file.long_name, + "Conventions": "CF-1.8", + "license": "CC BY 4.0", + "level": level, + "period": period, + "function": function, + "parameter_name": ":".join([standard_name, level, function, period]), + "naming_authority": "nl.knmi", + "keywords": file["iso_dataset"].attrs["keyword"], + "summary": file["iso_dataset"].attrs["abstract"], + "keywords_vocabulary": file.attrs["references"], + "source": file.attrs["source"], + "creator_name": "KNMI", + "creator_email": file["iso_dataset"].attrs["email_dataset"], + "creator_url": file["iso_dataset"].attrs["url_metadata"], + "creator_type": "institution", + "institution": file.attrs["institution"], + } + + geometry = { + "type": "Point", + "coordinates": { + "lon": longitude, + "lat": latitude, + }, + } + + for time, obs_value in zip( + pd.to_datetime(param_file["time"].data).to_pydatetime(), + param_file.data, + ): + + ts = Timestamp() + ts.FromDatetime(time) + if not math.isnan(obs_value): # Stations that don't have a parameter give them all as nan + content = { + "encoding": "utf-8", + "standard_name": standard_name, + "unit": convert_unit_names(param_file.units), + "value": str(obs_value), + } + + observations.append( + { + "properties": { + **properties, + "content": content, + "datetime": ts.ToJsonString(), + }, + "geometry": geometry, + "version": "4.0", + "type": "Feature", + "links": [ + {"href": "Insert documentation about E-SOH datastore", "rel": "canonical"}, + ], + } + ) + + if len(observations) > 0: + observation_request_messages.append(observations) + + return observation_request_messages + + +def send_request_to_ingest(msg, url): + try: + response = requests.post(url, data=json.dumps(msg)) + response.raise_for_status() + return response.status_code, response.json() + except requests.RequestException as e: + return response.status_code, e + except Exception as e: + return 500, e + + +def insert_data(observation_request_messages: List, url): + print(f"Sending {len(observation_request_messages)} bulk observations requests to ingest.") + obs_insert_start = perf_counter() + + partial_send_request = partial(send_request_to_ingest, url=url) + + with Pool(cpu_count()) as pool: + results = pool.map(partial_send_request, observation_request_messages) + + for status_code, response in results: + if status_code != 200: + print(status_code, response) + return + print(f"Finished observations bulk insert {perf_counter() - obs_insert_start}.") + + +def main(): + total_time_start = perf_counter() + print("Starting with creating the time series and observations requests.") + create_requests_start = perf_counter() + file_path = Path(Path(__file__).parent / "test-data" / "KNMI" / "20221231.nc") + observation_request_messages = netcdf_file_to_requests(file_path=file_path) + print("Finished creating the time series and observation requests " f"{perf_counter() - create_requests_start}.") + + insert_data( + observation_request_messages=observation_request_messages, + url=os.getenv("INGEST_URL", "http://localhost:8009/json"), + ) + + print(f"Finished, total time elapsed: {perf_counter() - total_time_start}") + + +if __name__ == "__main__": + main() diff --git a/datastore/data-loader/parameters.py b/datastore/data-loader/parameters.py index 14af459e..8e344653 100644 --- a/datastore/data-loader/parameters.py +++ b/datastore/data-loader/parameters.py @@ -1,19 +1,14 @@ knmi_parameter_names = ( - "hc3", - "nc2", + "hc", + "nc", "zm", "R1H", - "hc", "tgn", "Tn12", - "pr", "pg", "tn", "rg", - "hc1", - "nc1", "ts1", - "nc3", "ts2", "qg", "ff", @@ -25,9 +20,7 @@ "Tgn12", "ss", "Tn6", - "dr", "rh", - "hc2", "Tgn6", "R12H", "R24H", @@ -35,11 +28,9 @@ "Tx24", "Tx12", "Tgn14", - "D1H", "R6H", "pwc", "tx", - "nc", "pp", "Tn14", "ta", diff --git a/datastore/data-loader/requirements.in b/datastore/data-loader/requirements.in index eb0fd406..6e821874 100644 --- a/datastore/data-loader/requirements.in +++ b/datastore/data-loader/requirements.in @@ -7,3 +7,4 @@ grpcio-tools~=1.56 netCDF4~=1.6 xarray~=2023.7 isodate~=0.6.1 +requests~=2.32 diff --git a/datastore/data-loader/requirements.txt b/datastore/data-loader/requirements.txt index 6d780a1f..024b7e4d 100644 --- a/datastore/data-loader/requirements.txt +++ b/datastore/data-loader/requirements.txt @@ -2,44 +2,54 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --no-emit-index-url ./requirements.in +# pip-compile --no-emit-index-url # -certifi==2024.2.2 - # via netcdf4 -cftime==1.6.3 +certifi==2024.8.30 + # via + # netcdf4 + # requests +cftime==1.6.4 # via netcdf4 -grpcio==1.62.2 +charset-normalizer==3.4.0 + # via requests +grpcio==1.67.0 # via grpcio-tools -grpcio-tools==1.62.2 - # via -r ./requirements.in +grpcio-tools==1.67.0 + # via -r requirements.in +idna==3.10 + # via requests isodate==0.6.1 - # via -r ./requirements.in -netcdf4==1.6.5 - # via -r ./requirements.in -numpy==1.26.4 + # via -r requirements.in +netcdf4==1.7.1.post2 + # via -r requirements.in +numpy==2.1.2 # via # cftime # netcdf4 # pandas # xarray -packaging==24.0 +packaging==24.1 # via xarray -pandas==2.2.2 +pandas==2.2.3 # via xarray -protobuf==4.25.3 +protobuf==5.28.2 # via grpcio-tools python-dateutil==2.9.0.post0 # via pandas -pytz==2024.1 +pytz==2024.2 # via pandas +requests==2.32.3 + # via -r requirements.in six==1.16.0 # via # isodate # python-dateutil -tzdata==2024.1 +tzdata==2024.2 # via pandas +urllib3==2.2.3 + # via requests xarray==2023.12.0 - # via -r ./requirements.in + # via -r requirements.in # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/datastore/data-loader/utilities.py b/datastore/data-loader/utilities.py new file mode 100644 index 00000000..48ba655d --- /dev/null +++ b/datastore/data-loader/utilities.py @@ -0,0 +1,107 @@ +import isodate +import re +from datetime import timedelta +from isodate import ISO8601Error + + +regex_level = re.compile(r"first|second|third|[0-9]+(\.[0-9]+)?(?=m)|(?<=Level )[0-9]+", re.IGNORECASE) +regex_level_centimeters = re.compile(r"[0-9]+(\.[0-9]+)?(?=cm)") +regex_time_period = re.compile(r"(\d+) (Hours|Min)", re.IGNORECASE) + + +def convert_standard_names_to_cf(standard_name): + standard_name_mapping = { + "cloud_cover": "cloud_area_fraction", + "total_downwelling_shortwave_flux_in_air": "surface_downwelling_shortwave_flux_in_air", + "precipitation_rate": "rainfall_rate", + "air_pressure_at_sea_level": "air_pressure_at_mean_sea_level", + } + return standard_name_mapping.get(standard_name, standard_name) + + +# NOTE: Only units are converted currently, not values. +def convert_unit_names(unit): + unit_mapping = { + "degrees Celsius": "degC", + "ft": "m", + "min": "s", + "degree": "degrees", + "%": "percent", + "mm": "kg/m2", + "m s-1": "m/s", + "octa": "oktas", + "W m-2": "W/m2", + } + return unit_mapping.get(unit, unit) + + +def iso_8601_duration_to_seconds(period: str) -> int: + try: + duration = isodate.parse_duration(period) + except ISO8601Error: + raise ValueError(f"Invalid ISO 8601 duration: {period}") + + if isinstance(duration, timedelta): + total_seconds = duration.total_seconds() + else: + raise ValueError("Duration not convertable to seconds.") + + return int(total_seconds) + + +def generate_parameter_name(standard_name, long_name, station_id, station_name, param_id): + # TODO: HACK To let the loader have a unique parameter ID and make the parameters distinguishable. + level = "2.0" + long_name = long_name.lower() + station_name = station_name.lower() + if level_raw := re.search(regex_level, long_name): + level = level_raw[0] + if level_raw := re.search(regex_level_centimeters, long_name): + level = str(float(level_raw[0]) / 100.0) + elif "grass" in long_name: + level = "0" + elif param_id in ["pg", "pr", "pwc", "vv", "W10", "W10-10", "ww", "ww-10", "za", "zm"]: + # https://english.knmidata.nl/open-data/actuele10mindataknmistations + # Comments code: 2, 3, 11 + # Note: The sensor is not installed at equal heights at all types of measurement sites: + # At 'AWS' sites the device is installed at 1.80m. At 'AWS/Aerodrome' and 'Mistpost' + # (note that this includes site Voorschoten (06215) which is 'AWS/Mistpost') + # the device is installed at 2.50m elevation. Exceptions are Berkhout AWS (06249), + # De Bilt AWS (06260) and Twenthe AWS (06290) where the sensor is installed at 2.50m. + # Since WaWa is automatic detection I asssumed that the others stations are AWS, thus 1.80m + if ( + station_id in ["06215", "06249", "06260", "06290"] + or "aerodrome" in station_name + or "mistpost" in station_name + ): + level = "2.5" + else: + level = "1.8" + + if "minimum" in long_name: + function = "minimum" + elif "maximum" in long_name: + function = "maximum" + elif "average" in long_name: + function = "mean" + else: + function = "point" + + period = "PT0S" + if period_raw := re.findall(regex_time_period, long_name): + if len(period_raw) == 1: + period_raw = period_raw[0] + else: + raise Exception(f"{period_raw}, {long_name}") + time, scale = period_raw + if scale == "hours": + period = f"PT{time}H" + elif scale == "min": + period = f"PT{time}M" + elif param_id == "ww-10": + period = "PT10M" + elif param_id == "ww": + period = "PT01H" + + standard_name = convert_standard_names_to_cf(standard_name) + return standard_name, level, function, period diff --git a/datastore/integration-test/response/data_area_one_location_with_parameter_with_wildcard.json b/datastore/integration-test/response/data_area_one_location_with_parameter_with_wildcard.json index 78b7c959..953eb33c 100644 --- a/datastore/integration-test/response/data_area_one_location_with_parameter_with_wildcard.json +++ b/datastore/integration-test/response/data_area_one_location_with_parameter_with_wildcard.json @@ -46,17 +46,17 @@ "air_temperature:2.0:maximum:PT6H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT6H with method 'maximum'" + "en": "Air temperature at 2.0m, aggregated over PT6H with method 'maximum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:maximum:PT6H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -69,17 +69,17 @@ "air_temperature:2.0:minimum:PT6H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT6H with method 'minimum'" + "en": "Air temperature at 2.0m, aggregated over PT6H with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:minimum:PT6H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { diff --git a/datastore/integration-test/response/data_area_two_locations_with_two_parameters.json b/datastore/integration-test/response/data_area_two_locations_with_two_parameters.json index 50b1732b..8ecfb03c 100644 --- a/datastore/integration-test/response/data_area_two_locations_with_two_parameters.json +++ b/datastore/integration-test/response/data_area_two_locations_with_two_parameters.json @@ -55,17 +55,17 @@ "relative_humidity:2.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "relative_humidity at 2.0m, aggregated over PT1M with method 'mean'" + "en": "Relative humidity at 2.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/relative_humidity", "label": { - "en": "relative_humidity:2.0:mean:PT1M" + "en": "Relative humidity" } }, "unit": { "label": { - "en": "%" + "en": "percent" } }, "measurementType": { @@ -78,17 +78,17 @@ "wind_speed:10.0:mean:PT10M": { "type": "Parameter", "description": { - "en": "wind_speed at 10.0m, aggregated over PT10M with method 'mean'" + "en": "Wind speed at 10.0m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_speed", "label": { - "en": "wind_speed:10.0:mean:PT10M" + "en": "Wind speed" } }, "unit": { "label": { - "en": "m s-1" + "en": "m/s" } }, "measurementType": { @@ -203,17 +203,17 @@ "relative_humidity:2.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "relative_humidity at 2.0m, aggregated over PT1M with method 'mean'" + "en": "Relative humidity at 2.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/relative_humidity", "label": { - "en": "relative_humidity:2.0:mean:PT1M" + "en": "Relative humidity" } }, "unit": { "label": { - "en": "%" + "en": "percent" } }, "measurementType": { @@ -226,17 +226,17 @@ "wind_speed:10.0:mean:PT10M": { "type": "Parameter", "description": { - "en": "wind_speed at 10.0m, aggregated over PT10M with method 'mean'" + "en": "Wind speed at 10.0m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_speed", "label": { - "en": "wind_speed:10.0:mean:PT10M" + "en": "Wind speed" } }, "unit": { "label": { - "en": "m s-1" + "en": "m/s" } }, "measurementType": { @@ -302,17 +302,17 @@ "relative_humidity:2.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "relative_humidity at 2.0m, aggregated over PT1M with method 'mean'" + "en": "Relative humidity at 2.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/relative_humidity", "label": { - "en": "relative_humidity:2.0:mean:PT1M" + "en": "Relative humidity" } }, "unit": { "label": { - "en": "%" + "en": "percent" } }, "measurementType": { @@ -325,17 +325,17 @@ "wind_speed:10.0:mean:PT10M": { "type": "Parameter", "description": { - "en": "wind_speed at 10.0m, aggregated over PT10M with method 'mean'" + "en": "Wind speed at 10.0m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_speed", "label": { - "en": "wind_speed:10.0:mean:PT10M" + "en": "Wind speed" } }, "unit": { "label": { - "en": "m s-1" + "en": "m/s" } }, "measurementType": { diff --git a/datastore/integration-test/response/data_locations_one_location_with_multiple_parameters.json b/datastore/integration-test/response/data_locations_one_location_with_multiple_parameters.json index 8913da3c..7543ad48 100644 --- a/datastore/integration-test/response/data_locations_one_location_with_multiple_parameters.json +++ b/datastore/integration-test/response/data_locations_one_location_with_multiple_parameters.json @@ -46,17 +46,17 @@ "air_temperature:0.0:minimum:PT12H": { "type": "Parameter", "description": { - "en": "air_temperature at 0.0m, aggregated over PT12H with method 'minimum'" + "en": "Air temperature at 0.0m, aggregated over PT12H with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:0.0:minimum:PT12H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -69,17 +69,17 @@ "air_temperature:0.0:minimum:PT14H": { "type": "Parameter", "description": { - "en": "air_temperature at 0.0m, aggregated over PT14H with method 'minimum'" + "en": "Air temperature at 0.0m, aggregated over PT14H with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:0.0:minimum:PT14H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -92,17 +92,17 @@ "air_temperature:0.0:minimum:PT6H": { "type": "Parameter", "description": { - "en": "air_temperature at 0.0m, aggregated over PT6H with method 'minimum'" + "en": "Air temperature at 0.0m, aggregated over PT6H with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:0.0:minimum:PT6H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -115,17 +115,17 @@ "air_temperature:2.0:maximum:PT12H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT12H with method 'maximum'" + "en": "Air temperature at 2.0m, aggregated over PT12H with method 'maximum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:maximum:PT12H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -138,17 +138,17 @@ "air_temperature:2.0:maximum:PT24H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT24H with method 'maximum'" + "en": "Air temperature at 2.0m, aggregated over PT24H with method 'maximum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:maximum:PT24H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -161,17 +161,17 @@ "air_temperature:2.0:maximum:PT6H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT6H with method 'maximum'" + "en": "Air temperature at 2.0m, aggregated over PT6H with method 'maximum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:maximum:PT6H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -184,17 +184,17 @@ "air_temperature:2.0:minimum:PT12H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT12H with method 'minimum'" + "en": "Air temperature at 2.0m, aggregated over PT12H with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:minimum:PT12H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -207,17 +207,17 @@ "air_temperature:2.0:minimum:PT14H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT14H with method 'minimum'" + "en": "Air temperature at 2.0m, aggregated over PT14H with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:minimum:PT14H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -230,17 +230,17 @@ "air_temperature:2.0:minimum:PT6H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT6H with method 'minimum'" + "en": "Air temperature at 2.0m, aggregated over PT6H with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:minimum:PT6H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -253,17 +253,17 @@ "rainfall_amount:2.0:point:PT12H": { "type": "Parameter", "description": { - "en": "rainfall_amount at 2.0m, aggregated over PT12H with method 'point'" + "en": "Rainfall amount at 2.0m, aggregated over PT12H with method 'point'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", "label": { - "en": "rainfall_amount:2.0:point:PT12H" + "en": "Rainfall amount" } }, "unit": { "label": { - "en": "mm" + "en": "kg/m2" } }, "measurementType": { @@ -276,17 +276,17 @@ "rainfall_amount:2.0:point:PT24H": { "type": "Parameter", "description": { - "en": "rainfall_amount at 2.0m, aggregated over PT24H with method 'point'" + "en": "Rainfall amount at 2.0m, aggregated over PT24H with method 'point'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", "label": { - "en": "rainfall_amount:2.0:point:PT24H" + "en": "Rainfall amount" } }, "unit": { "label": { - "en": "mm" + "en": "kg/m2" } }, "measurementType": { @@ -299,17 +299,17 @@ "rainfall_amount:2.0:point:PT6H": { "type": "Parameter", "description": { - "en": "rainfall_amount at 2.0m, aggregated over PT6H with method 'point'" + "en": "Rainfall amount at 2.0m, aggregated over PT6H with method 'point'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", "label": { - "en": "rainfall_amount:2.0:point:PT6H" + "en": "Rainfall amount" } }, "unit": { "label": { - "en": "mm" + "en": "kg/m2" } }, "measurementType": { diff --git a/datastore/integration-test/response/data_locations_one_location_with_three_parameters.json b/datastore/integration-test/response/data_locations_one_location_with_three_parameters.json index 7ec813e6..68b76051 100644 --- a/datastore/integration-test/response/data_locations_one_location_with_three_parameters.json +++ b/datastore/integration-test/response/data_locations_one_location_with_three_parameters.json @@ -53,17 +53,17 @@ "air_temperature:1.5:maximum:PT10M": { "type": "Parameter", "description": { - "en": "air_temperature at 1.5m, aggregated over PT10M with method 'maximum'" + "en": "Air temperature at 1.5m, aggregated over PT10M with method 'maximum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:1.5:maximum:PT10M" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -76,17 +76,17 @@ "relative_humidity:2.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "relative_humidity at 2.0m, aggregated over PT1M with method 'mean'" + "en": "Relative humidity at 2.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/relative_humidity", "label": { - "en": "relative_humidity:2.0:mean:PT1M" + "en": "Relative humidity" } }, "unit": { "label": { - "en": "%" + "en": "percent" } }, "measurementType": { @@ -99,17 +99,17 @@ "wind_from_direction:2.0:mean:PT10M": { "type": "Parameter", "description": { - "en": "wind_from_direction at 2.0m, aggregated over PT10M with method 'mean'" + "en": "Wind from direction at 2.0m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_from_direction", "label": { - "en": "wind_from_direction:2.0:mean:PT10M" + "en": "Wind from direction" } }, "unit": { "label": { - "en": "degree" + "en": "degrees" } }, "measurementType": { diff --git a/datastore/integration-test/response/data_locations_two_points_with_five_parameters.json b/datastore/integration-test/response/data_locations_two_points_with_five_parameters.json index a8a96a1d..0294a6b3 100644 --- a/datastore/integration-test/response/data_locations_two_points_with_five_parameters.json +++ b/datastore/integration-test/response/data_locations_two_points_with_five_parameters.json @@ -50,17 +50,17 @@ "air_temperature:0.1:minimum:PT10M": { "type": "Parameter", "description": { - "en": "air_temperature at 0.1m, aggregated over PT10M with method 'minimum'" + "en": "Air temperature at 0.1m, aggregated over PT10M with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:0.1:minimum:PT10M" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -73,17 +73,17 @@ "air_temperature:1.5:maximum:PT10M": { "type": "Parameter", "description": { - "en": "air_temperature at 1.5m, aggregated over PT10M with method 'maximum'" + "en": "Air temperature at 1.5m, aggregated over PT10M with method 'maximum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:1.5:maximum:PT10M" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -96,17 +96,17 @@ "air_temperature:1.5:minimum:PT10M": { "type": "Parameter", "description": { - "en": "air_temperature at 1.5m, aggregated over PT10M with method 'minimum'" + "en": "Air temperature at 1.5m, aggregated over PT10M with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:1.5:minimum:PT10M" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -119,17 +119,17 @@ "air_temperature:2.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT1M with method 'mean'" + "en": "Air temperature at 2.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:mean:PT1M" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -142,17 +142,17 @@ "wind_from_direction:2.0:mean:PT10M": { "type": "Parameter", "description": { - "en": "wind_from_direction at 2.0m, aggregated over PT10M with method 'mean'" + "en": "Wind from direction at 2.0m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_from_direction", "label": { - "en": "wind_from_direction:2.0:mean:PT10M" + "en": "Wind from direction" } }, "unit": { "label": { - "en": "degree" + "en": "degrees" } }, "measurementType": { diff --git a/datastore/integration-test/response/data_locations_two_points_with_multiple_parameters.json b/datastore/integration-test/response/data_locations_two_points_with_multiple_parameters.json index f8a25387..125ab218 100644 --- a/datastore/integration-test/response/data_locations_two_points_with_multiple_parameters.json +++ b/datastore/integration-test/response/data_locations_two_points_with_multiple_parameters.json @@ -14,7 +14,7 @@ "name": "DE BILT AWS", "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06260", "parameter-name": [ - "air_pressure_at_sea_level:1.0:mean:PT1M", + "air_pressure_at_mean_sea_level:1.0:mean:PT1M", "air_temperature:0.0:minimum:PT12H", "air_temperature:0.0:minimum:PT14H", "air_temperature:0.0:minimum:PT6H", @@ -28,28 +28,18 @@ "air_temperature:2.0:minimum:PT12H", "air_temperature:2.0:minimum:PT14H", "air_temperature:2.0:minimum:PT6H", - "cloud_base_altitude:0.0:point:PT0S", - "cloud_base_altitude:1.0:point:PT0S", + "cloud_area_fraction:2.0:point:PT0S", "cloud_base_altitude:2.0:point:PT0S", - "cloud_cover:0.0:point:PT0S", - "cloud_cover:1.0:point:PT0S", - "cloud_cover:2.0:point:PT0S", "dew_point_temperature:1.5:mean:PT1M", "duration_of_sunshine:2.0:point:PT0S", "lwe_precipitation_rate:2.5:mean:PT10M", - "placeholder:2.5:point:PT0S", - "placeholder:2.5:point:PT10M", - "placeholder:2.5:point:PT1H", - "precipitation_duration:2.0:point:PT10M", - "precipitation_duration:2.5:point:PT10M", - "precipitation_rate:2.0:mean:PT10M", "rainfall_amount:2.0:point:PT0S", "rainfall_amount:2.0:point:PT12H", "rainfall_amount:2.0:point:PT24H", "rainfall_amount:2.0:point:PT6H", - "rainfall_duration:2.0:point:PT0S", + "rainfall_rate:2.0:mean:PT10M", "relative_humidity:2.0:mean:PT1M", - "total_downwelling_shortwave_flux_in_air:2.0:mean:PT10M", + "surface_downwelling_shortwave_flux_in_air:2.0:mean:PT10M", "visibility_in_air:2.5:mean:PT10M", "wind_from_direction:2.0:mean:PT10M", "wind_speed:10.0:mean:PT10M", @@ -71,7 +61,7 @@ "name": "DEELEN", "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06275", "parameter-name": [ - "air_pressure_at_sea_level:1.0:mean:PT1M", + "air_pressure_at_mean_sea_level:1.0:mean:PT1M", "air_temperature:0.0:minimum:PT12H", "air_temperature:0.0:minimum:PT14H", "air_temperature:0.0:minimum:PT6H", @@ -85,28 +75,18 @@ "air_temperature:2.0:minimum:PT12H", "air_temperature:2.0:minimum:PT14H", "air_temperature:2.0:minimum:PT6H", - "cloud_base_altitude:0.0:point:PT0S", - "cloud_base_altitude:1.0:point:PT0S", + "cloud_area_fraction:2.0:point:PT0S", "cloud_base_altitude:2.0:point:PT0S", - "cloud_cover:0.0:point:PT0S", - "cloud_cover:1.0:point:PT0S", - "cloud_cover:2.0:point:PT0S", "dew_point_temperature:1.5:mean:PT1M", "duration_of_sunshine:2.0:point:PT0S", "lwe_precipitation_rate:1.8:mean:PT10M", - "placeholder:1.8:point:PT0S", - "placeholder:1.8:point:PT10M", - "placeholder:1.8:point:PT1H", - "precipitation_duration:1.8:point:PT10M", - "precipitation_duration:2.0:point:PT10M", - "precipitation_rate:2.0:mean:PT10M", "rainfall_amount:2.0:point:PT0S", "rainfall_amount:2.0:point:PT12H", "rainfall_amount:2.0:point:PT24H", "rainfall_amount:2.0:point:PT6H", - "rainfall_duration:2.0:point:PT0S", + "rainfall_rate:2.0:mean:PT10M", "relative_humidity:2.0:mean:PT1M", - "total_downwelling_shortwave_flux_in_air:2.0:mean:PT10M", + "surface_downwelling_shortwave_flux_in_air:2.0:mean:PT10M", "visibility_in_air:1.8:mean:PT10M", "wind_from_direction:2.0:mean:PT10M", "wind_speed:10.0:mean:PT10M", @@ -117,15 +97,15 @@ } ], "parameters": { - "air_pressure_at_sea_level:1.0:mean:PT1M": { + "air_pressure_at_mean_sea_level:1.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "air_pressure_at_sea_level at 1.0m, aggregated over PT1M with method 'mean'" + "en": "Air pressure at mean sea level at 1.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/air_pressure_at_sea_level", + "id": "https://vocab.nerc.ac.uk/standard_name/air_pressure_at_mean_sea_level", "label": { - "en": "air_pressure_at_sea_level:1.0:mean:PT1M" + "en": "Air pressure at mean sea level" } }, "unit": { @@ -137,23 +117,23 @@ "method": "mean", "period": "PT1M" }, - "rodeo:standard_name": "air_pressure_at_sea_level", + "rodeo:standard_name": "air_pressure_at_mean_sea_level", "rodeo:level": 1.0 }, "air_temperature:0.0:minimum:PT12H": { "type": "Parameter", "description": { - "en": "air_temperature at 0.0m, aggregated over PT12H with method 'minimum'" + "en": "Air temperature at 0.0m, aggregated over PT12H with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:0.0:minimum:PT12H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -166,17 +146,17 @@ "air_temperature:0.0:minimum:PT14H": { "type": "Parameter", "description": { - "en": "air_temperature at 0.0m, aggregated over PT14H with method 'minimum'" + "en": "Air temperature at 0.0m, aggregated over PT14H with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:0.0:minimum:PT14H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -189,17 +169,17 @@ "air_temperature:0.0:minimum:PT6H": { "type": "Parameter", "description": { - "en": "air_temperature at 0.0m, aggregated over PT6H with method 'minimum'" + "en": "Air temperature at 0.0m, aggregated over PT6H with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:0.0:minimum:PT6H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -212,17 +192,17 @@ "air_temperature:0.1:minimum:PT10M": { "type": "Parameter", "description": { - "en": "air_temperature at 0.1m, aggregated over PT10M with method 'minimum'" + "en": "Air temperature at 0.1m, aggregated over PT10M with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:0.1:minimum:PT10M" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -235,17 +215,17 @@ "air_temperature:1.5:maximum:PT10M": { "type": "Parameter", "description": { - "en": "air_temperature at 1.5m, aggregated over PT10M with method 'maximum'" + "en": "Air temperature at 1.5m, aggregated over PT10M with method 'maximum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:1.5:maximum:PT10M" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -258,17 +238,17 @@ "air_temperature:1.5:minimum:PT10M": { "type": "Parameter", "description": { - "en": "air_temperature at 1.5m, aggregated over PT10M with method 'minimum'" + "en": "Air temperature at 1.5m, aggregated over PT10M with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:1.5:minimum:PT10M" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -281,17 +261,17 @@ "air_temperature:2.0:maximum:PT12H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT12H with method 'maximum'" + "en": "Air temperature at 2.0m, aggregated over PT12H with method 'maximum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:maximum:PT12H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -304,17 +284,17 @@ "air_temperature:2.0:maximum:PT24H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT24H with method 'maximum'" + "en": "Air temperature at 2.0m, aggregated over PT24H with method 'maximum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:maximum:PT24H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -327,17 +307,17 @@ "air_temperature:2.0:maximum:PT6H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT6H with method 'maximum'" + "en": "Air temperature at 2.0m, aggregated over PT6H with method 'maximum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:maximum:PT6H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -350,17 +330,17 @@ "air_temperature:2.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT1M with method 'mean'" + "en": "Air temperature at 2.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:mean:PT1M" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -373,17 +353,17 @@ "air_temperature:2.0:minimum:PT12H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT12H with method 'minimum'" + "en": "Air temperature at 2.0m, aggregated over PT12H with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:minimum:PT12H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -396,17 +376,17 @@ "air_temperature:2.0:minimum:PT14H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT14H with method 'minimum'" + "en": "Air temperature at 2.0m, aggregated over PT14H with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:minimum:PT14H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -419,17 +399,17 @@ "air_temperature:2.0:minimum:PT6H": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT6H with method 'minimum'" + "en": "Air temperature at 2.0m, aggregated over PT6H with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:minimum:PT6H" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -439,66 +419,43 @@ "rodeo:standard_name": "air_temperature", "rodeo:level": 2.0 }, - "cloud_base_altitude:0.0:point:PT0S": { + "cloud_area_fraction:2.0:point:PT0S": { "type": "Parameter", "description": { - "en": "cloud_base_altitude at 0.0m, aggregated over PT0S with method 'point'" + "en": "Cloud area fraction at 2.0m, aggregated over PT0S with method 'point'" }, "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude", - "label": { - "en": "cloud_base_altitude:0.0:point:PT0S" - } - }, - "unit": { - "label": { - "en": "ft" - } - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_base_altitude", - "rodeo:level": 0.0 - }, - "cloud_base_altitude:1.0:point:PT0S": { - "type": "Parameter", - "description": { - "en": "cloud_base_altitude at 1.0m, aggregated over PT0S with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude", + "id": "https://vocab.nerc.ac.uk/standard_name/cloud_area_fraction", "label": { - "en": "cloud_base_altitude:1.0:point:PT0S" + "en": "Cloud area fraction" } }, "unit": { "label": { - "en": "ft" + "en": "oktas" } }, "measurementType": { "method": "point", "period": "PT0S" }, - "rodeo:standard_name": "cloud_base_altitude", - "rodeo:level": 1.0 + "rodeo:standard_name": "cloud_area_fraction", + "rodeo:level": 2.0 }, "cloud_base_altitude:2.0:point:PT0S": { "type": "Parameter", "description": { - "en": "cloud_base_altitude at 2.0m, aggregated over PT0S with method 'point'" + "en": "Cloud base altitude at 2.0m, aggregated over PT0S with method 'point'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude", "label": { - "en": "cloud_base_altitude:2.0:point:PT0S" + "en": "Cloud base altitude" } }, "unit": { "label": { - "en": "ft" + "en": "m" } }, "measurementType": { @@ -508,89 +465,20 @@ "rodeo:standard_name": "cloud_base_altitude", "rodeo:level": 2.0 }, - "cloud_cover:0.0:point:PT0S": { - "type": "Parameter", - "description": { - "en": "cloud_cover at 0.0m, aggregated over PT0S with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_cover", - "label": { - "en": "cloud_cover:0.0:point:PT0S" - } - }, - "unit": { - "label": { - "en": "octa" - } - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_cover", - "rodeo:level": 0.0 - }, - "cloud_cover:1.0:point:PT0S": { - "type": "Parameter", - "description": { - "en": "cloud_cover at 1.0m, aggregated over PT0S with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_cover", - "label": { - "en": "cloud_cover:1.0:point:PT0S" - } - }, - "unit": { - "label": { - "en": "octa" - } - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_cover", - "rodeo:level": 1.0 - }, - "cloud_cover:2.0:point:PT0S": { - "type": "Parameter", - "description": { - "en": "cloud_cover at 2.0m, aggregated over PT0S with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_cover", - "label": { - "en": "cloud_cover:2.0:point:PT0S" - } - }, - "unit": { - "label": { - "en": "octa" - } - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_cover", - "rodeo:level": 2.0 - }, "dew_point_temperature:1.5:mean:PT1M": { "type": "Parameter", "description": { - "en": "dew_point_temperature at 1.5m, aggregated over PT1M with method 'mean'" + "en": "Dew point temperature at 1.5m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/dew_point_temperature", "label": { - "en": "dew_point_temperature:1.5:mean:PT1M" + "en": "Dew point temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -603,17 +491,17 @@ "duration_of_sunshine:2.0:point:PT0S": { "type": "Parameter", "description": { - "en": "duration_of_sunshine at 2.0m, aggregated over PT0S with method 'point'" + "en": "Duration of sunshine at 2.0m, aggregated over PT0S with method 'point'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/duration_of_sunshine", "label": { - "en": "duration_of_sunshine:2.0:point:PT0S" + "en": "Duration of sunshine" } }, "unit": { "label": { - "en": "min" + "en": "s" } }, "measurementType": { @@ -626,12 +514,12 @@ "lwe_precipitation_rate:1.8:mean:PT10M": { "type": "Parameter", "description": { - "en": "lwe_precipitation_rate at 1.8m, aggregated over PT10M with method 'mean'" + "en": "Lwe precipitation rate at 1.8m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/lwe_precipitation_rate", "label": { - "en": "lwe_precipitation_rate:1.8:mean:PT10M" + "en": "Lwe precipitation rate" } }, "unit": { @@ -649,12 +537,12 @@ "lwe_precipitation_rate:2.5:mean:PT10M": { "type": "Parameter", "description": { - "en": "lwe_precipitation_rate at 2.5m, aggregated over PT10M with method 'mean'" + "en": "Lwe precipitation rate at 2.5m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/lwe_precipitation_rate", "label": { - "en": "lwe_precipitation_rate:2.5:mean:PT10M" + "en": "Lwe precipitation rate" } }, "unit": { @@ -669,250 +557,20 @@ "rodeo:standard_name": "lwe_precipitation_rate", "rodeo:level": 2.5 }, - "placeholder:1.8:point:PT0S": { - "type": "Parameter", - "description": { - "en": "placeholder at 1.8m, aggregated over PT0S with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/placeholder", - "label": { - "en": "placeholder:1.8:point:PT0S" - } - }, - "unit": { - "label": { - "en": "code" - } - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "placeholder", - "rodeo:level": 1.8 - }, - "placeholder:1.8:point:PT10M": { - "type": "Parameter", - "description": { - "en": "placeholder at 1.8m, aggregated over PT10M with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/placeholder", - "label": { - "en": "placeholder:1.8:point:PT10M" - } - }, - "unit": { - "label": { - "en": "code" - } - }, - "measurementType": { - "method": "point", - "period": "PT10M" - }, - "rodeo:standard_name": "placeholder", - "rodeo:level": 1.8 - }, - "placeholder:1.8:point:PT1H": { - "type": "Parameter", - "description": { - "en": "placeholder at 1.8m, aggregated over PT1H with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/placeholder", - "label": { - "en": "placeholder:1.8:point:PT1H" - } - }, - "unit": { - "label": { - "en": "code" - } - }, - "measurementType": { - "method": "point", - "period": "PT1H" - }, - "rodeo:standard_name": "placeholder", - "rodeo:level": 1.8 - }, - "placeholder:2.5:point:PT0S": { - "type": "Parameter", - "description": { - "en": "placeholder at 2.5m, aggregated over PT0S with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/placeholder", - "label": { - "en": "placeholder:2.5:point:PT0S" - } - }, - "unit": { - "label": { - "en": "code" - } - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "placeholder", - "rodeo:level": 2.5 - }, - "placeholder:2.5:point:PT10M": { - "type": "Parameter", - "description": { - "en": "placeholder at 2.5m, aggregated over PT10M with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/placeholder", - "label": { - "en": "placeholder:2.5:point:PT10M" - } - }, - "unit": { - "label": { - "en": "code" - } - }, - "measurementType": { - "method": "point", - "period": "PT10M" - }, - "rodeo:standard_name": "placeholder", - "rodeo:level": 2.5 - }, - "placeholder:2.5:point:PT1H": { - "type": "Parameter", - "description": { - "en": "placeholder at 2.5m, aggregated over PT1H with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/placeholder", - "label": { - "en": "placeholder:2.5:point:PT1H" - } - }, - "unit": { - "label": { - "en": "code" - } - }, - "measurementType": { - "method": "point", - "period": "PT1H" - }, - "rodeo:standard_name": "placeholder", - "rodeo:level": 2.5 - }, - "precipitation_duration:1.8:point:PT10M": { - "type": "Parameter", - "description": { - "en": "precipitation_duration at 1.8m, aggregated over PT10M with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/precipitation_duration", - "label": { - "en": "precipitation_duration:1.8:point:PT10M" - } - }, - "unit": { - "label": { - "en": "sec" - } - }, - "measurementType": { - "method": "point", - "period": "PT10M" - }, - "rodeo:standard_name": "precipitation_duration", - "rodeo:level": 1.8 - }, - "precipitation_duration:2.0:point:PT10M": { - "type": "Parameter", - "description": { - "en": "precipitation_duration at 2.0m, aggregated over PT10M with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/precipitation_duration", - "label": { - "en": "precipitation_duration:2.0:point:PT10M" - } - }, - "unit": { - "label": { - "en": "sec" - } - }, - "measurementType": { - "method": "point", - "period": "PT10M" - }, - "rodeo:standard_name": "precipitation_duration", - "rodeo:level": 2.0 - }, - "precipitation_duration:2.5:point:PT10M": { - "type": "Parameter", - "description": { - "en": "precipitation_duration at 2.5m, aggregated over PT10M with method 'point'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/precipitation_duration", - "label": { - "en": "precipitation_duration:2.5:point:PT10M" - } - }, - "unit": { - "label": { - "en": "sec" - } - }, - "measurementType": { - "method": "point", - "period": "PT10M" - }, - "rodeo:standard_name": "precipitation_duration", - "rodeo:level": 2.5 - }, - "precipitation_rate:2.0:mean:PT10M": { - "type": "Parameter", - "description": { - "en": "precipitation_rate at 2.0m, aggregated over PT10M with method 'mean'" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/precipitation_rate", - "label": { - "en": "precipitation_rate:2.0:mean:PT10M" - } - }, - "unit": { - "label": { - "en": "mm/h" - } - }, - "measurementType": { - "method": "mean", - "period": "PT10M" - }, - "rodeo:standard_name": "precipitation_rate", - "rodeo:level": 2.0 - }, "rainfall_amount:2.0:point:PT0S": { "type": "Parameter", "description": { - "en": "rainfall_amount at 2.0m, aggregated over PT0S with method 'point'" + "en": "Rainfall amount at 2.0m, aggregated over PT0S with method 'point'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", "label": { - "en": "rainfall_amount:2.0:point:PT0S" + "en": "Rainfall amount" } }, "unit": { "label": { - "en": "mm" + "en": "kg/m2" } }, "measurementType": { @@ -925,17 +583,17 @@ "rainfall_amount:2.0:point:PT12H": { "type": "Parameter", "description": { - "en": "rainfall_amount at 2.0m, aggregated over PT12H with method 'point'" + "en": "Rainfall amount at 2.0m, aggregated over PT12H with method 'point'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", "label": { - "en": "rainfall_amount:2.0:point:PT12H" + "en": "Rainfall amount" } }, "unit": { "label": { - "en": "mm" + "en": "kg/m2" } }, "measurementType": { @@ -948,17 +606,17 @@ "rainfall_amount:2.0:point:PT24H": { "type": "Parameter", "description": { - "en": "rainfall_amount at 2.0m, aggregated over PT24H with method 'point'" + "en": "Rainfall amount at 2.0m, aggregated over PT24H with method 'point'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", "label": { - "en": "rainfall_amount:2.0:point:PT24H" + "en": "Rainfall amount" } }, "unit": { "label": { - "en": "mm" + "en": "kg/m2" } }, "measurementType": { @@ -971,17 +629,17 @@ "rainfall_amount:2.0:point:PT6H": { "type": "Parameter", "description": { - "en": "rainfall_amount at 2.0m, aggregated over PT6H with method 'point'" + "en": "Rainfall amount at 2.0m, aggregated over PT6H with method 'point'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", "label": { - "en": "rainfall_amount:2.0:point:PT6H" + "en": "Rainfall amount" } }, "unit": { "label": { - "en": "mm" + "en": "kg/m2" } }, "measurementType": { @@ -991,43 +649,43 @@ "rodeo:standard_name": "rainfall_amount", "rodeo:level": 2.0 }, - "rainfall_duration:2.0:point:PT0S": { + "rainfall_rate:2.0:mean:PT10M": { "type": "Parameter", "description": { - "en": "rainfall_duration at 2.0m, aggregated over PT0S with method 'point'" + "en": "Rainfall rate at 2.0m, aggregated over PT10M with method 'mean'" }, "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_duration", + "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_rate", "label": { - "en": "rainfall_duration:2.0:point:PT0S" + "en": "Rainfall rate" } }, "unit": { "label": { - "en": "min" + "en": "mm/h" } }, "measurementType": { - "method": "point", - "period": "PT0S" + "method": "mean", + "period": "PT10M" }, - "rodeo:standard_name": "rainfall_duration", + "rodeo:standard_name": "rainfall_rate", "rodeo:level": 2.0 }, "relative_humidity:2.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "relative_humidity at 2.0m, aggregated over PT1M with method 'mean'" + "en": "Relative humidity at 2.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/relative_humidity", "label": { - "en": "relative_humidity:2.0:mean:PT1M" + "en": "Relative humidity" } }, "unit": { "label": { - "en": "%" + "en": "percent" } }, "measurementType": { @@ -1037,38 +695,38 @@ "rodeo:standard_name": "relative_humidity", "rodeo:level": 2.0 }, - "total_downwelling_shortwave_flux_in_air:2.0:mean:PT10M": { + "surface_downwelling_shortwave_flux_in_air:2.0:mean:PT10M": { "type": "Parameter", "description": { - "en": "total_downwelling_shortwave_flux_in_air at 2.0m, aggregated over PT10M with method 'mean'" + "en": "Surface downwelling shortwave flux in air at 2.0m, aggregated over PT10M with method 'mean'" }, "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/total_downwelling_shortwave_flux_in_air", + "id": "https://vocab.nerc.ac.uk/standard_name/surface_downwelling_shortwave_flux_in_air", "label": { - "en": "total_downwelling_shortwave_flux_in_air:2.0:mean:PT10M" + "en": "Surface downwelling shortwave flux in air" } }, "unit": { "label": { - "en": "W m-2" + "en": "W/m2" } }, "measurementType": { "method": "mean", "period": "PT10M" }, - "rodeo:standard_name": "total_downwelling_shortwave_flux_in_air", + "rodeo:standard_name": "surface_downwelling_shortwave_flux_in_air", "rodeo:level": 2.0 }, "visibility_in_air:1.8:mean:PT10M": { "type": "Parameter", "description": { - "en": "visibility_in_air at 1.8m, aggregated over PT10M with method 'mean'" + "en": "Visibility in air at 1.8m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/visibility_in_air", "label": { - "en": "visibility_in_air:1.8:mean:PT10M" + "en": "Visibility in air" } }, "unit": { @@ -1086,12 +744,12 @@ "visibility_in_air:2.5:mean:PT10M": { "type": "Parameter", "description": { - "en": "visibility_in_air at 2.5m, aggregated over PT10M with method 'mean'" + "en": "Visibility in air at 2.5m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/visibility_in_air", "label": { - "en": "visibility_in_air:2.5:mean:PT10M" + "en": "Visibility in air" } }, "unit": { @@ -1109,17 +767,17 @@ "wind_from_direction:2.0:mean:PT10M": { "type": "Parameter", "description": { - "en": "wind_from_direction at 2.0m, aggregated over PT10M with method 'mean'" + "en": "Wind from direction at 2.0m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_from_direction", "label": { - "en": "wind_from_direction:2.0:mean:PT10M" + "en": "Wind from direction" } }, "unit": { "label": { - "en": "degree" + "en": "degrees" } }, "measurementType": { @@ -1132,17 +790,17 @@ "wind_speed:10.0:mean:PT10M": { "type": "Parameter", "description": { - "en": "wind_speed at 10.0m, aggregated over PT10M with method 'mean'" + "en": "Wind speed at 10.0m, aggregated over PT10M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_speed", "label": { - "en": "wind_speed:10.0:mean:PT10M" + "en": "Wind speed" } }, "unit": { "label": { - "en": "m s-1" + "en": "m/s" } }, "measurementType": { @@ -1155,17 +813,17 @@ "wind_speed_of_gust:10.0:maximum:PT10M": { "type": "Parameter", "description": { - "en": "wind_speed_of_gust at 10.0m, aggregated over PT10M with method 'maximum'" + "en": "Wind speed of gust at 10.0m, aggregated over PT10M with method 'maximum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_speed_of_gust", "label": { - "en": "wind_speed_of_gust:10.0:maximum:PT10M" + "en": "Wind speed of gust" } }, "unit": { "label": { - "en": "m s-1" + "en": "m/s" } }, "measurementType": { diff --git a/datastore/integration-test/response/data_locations_two_points_with_two_parameters.json b/datastore/integration-test/response/data_locations_two_points_with_two_parameters.json index 2b6d8204..4a5e040e 100644 --- a/datastore/integration-test/response/data_locations_two_points_with_two_parameters.json +++ b/datastore/integration-test/response/data_locations_two_points_with_two_parameters.json @@ -14,7 +14,7 @@ "name": "DE BILT AWS", "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06260", "parameter-name": [ - "air_pressure_at_sea_level:1.0:mean:PT1M", + "air_pressure_at_mean_sea_level:1.0:mean:PT1M", "air_temperature:0.1:minimum:PT10M" ] }, @@ -33,7 +33,7 @@ "name": "DEELEN", "detail": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06275", "parameter-name": [ - "air_pressure_at_sea_level:1.0:mean:PT1M", + "air_pressure_at_mean_sea_level:1.0:mean:PT1M", "air_temperature:0.1:minimum:PT10M" ] }, @@ -41,15 +41,15 @@ } ], "parameters": { - "air_pressure_at_sea_level:1.0:mean:PT1M": { + "air_pressure_at_mean_sea_level:1.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "air_pressure_at_sea_level at 1.0m, aggregated over PT1M with method 'mean'" + "en": "Air pressure at mean sea level at 1.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/air_pressure_at_sea_level", + "id": "https://vocab.nerc.ac.uk/standard_name/air_pressure_at_mean_sea_level", "label": { - "en": "air_pressure_at_sea_level:1.0:mean:PT1M" + "en": "Air pressure at mean sea level" } }, "unit": { @@ -61,23 +61,23 @@ "method": "mean", "period": "PT1M" }, - "rodeo:standard_name": "air_pressure_at_sea_level", + "rodeo:standard_name": "air_pressure_at_mean_sea_level", "rodeo:level": 1.0 }, "air_temperature:0.1:minimum:PT10M": { "type": "Parameter", "description": { - "en": "air_temperature at 0.1m, aggregated over PT10M with method 'minimum'" + "en": "Air temperature at 0.1m, aggregated over PT10M with method 'minimum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:0.1:minimum:PT10M" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { diff --git a/datastore/integration-test/response/data_position_one_location_with_one_parameter.json b/datastore/integration-test/response/data_position_one_location_with_one_parameter.json index 4ea162ff..67eae9f4 100644 --- a/datastore/integration-test/response/data_position_one_location_with_one_parameter.json +++ b/datastore/integration-test/response/data_position_one_location_with_one_parameter.json @@ -54,17 +54,17 @@ "air_temperature:1.5:maximum:PT10M": { "type": "Parameter", "description": { - "en": "air_temperature at 1.5m, aggregated over PT10M with method 'maximum'" + "en": "Air temperature at 1.5m, aggregated over PT10M with method 'maximum'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:1.5:maximum:PT10M" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { diff --git a/datastore/integration-test/response/data_position_one_location_with_three_parameters.json b/datastore/integration-test/response/data_position_one_location_with_three_parameters.json index 74fe305f..d8120014 100644 --- a/datastore/integration-test/response/data_position_one_location_with_three_parameters.json +++ b/datastore/integration-test/response/data_position_one_location_with_three_parameters.json @@ -46,17 +46,17 @@ "air_temperature:2.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "air_temperature at 2.0m, aggregated over PT1M with method 'mean'" + "en": "Air temperature at 2.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", "label": { - "en": "air_temperature:2.0:mean:PT1M" + "en": "Air temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -69,17 +69,17 @@ "dew_point_temperature:1.5:mean:PT1M": { "type": "Parameter", "description": { - "en": "dew_point_temperature at 1.5m, aggregated over PT1M with method 'mean'" + "en": "Dew point temperature at 1.5m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/dew_point_temperature", "label": { - "en": "dew_point_temperature:1.5:mean:PT1M" + "en": "Dew point temperature" } }, "unit": { "label": { - "en": "degrees Celsius" + "en": "degC" } }, "measurementType": { @@ -92,17 +92,17 @@ "relative_humidity:2.0:mean:PT1M": { "type": "Parameter", "description": { - "en": "relative_humidity at 2.0m, aggregated over PT1M with method 'mean'" + "en": "Relative humidity at 2.0m, aggregated over PT1M with method 'mean'" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/relative_humidity", "label": { - "en": "relative_humidity:2.0:mean:PT1M" + "en": "Relative humidity" } }, "unit": { "label": { - "en": "%" + "en": "percent" } }, "measurementType": { diff --git a/datastore/integration-test/response/items_area_with_one_parameter_name.json b/datastore/integration-test/response/items_area_with_one_parameter_name.json index 74bd80bb..4d67952a 100644 --- a/datastore/integration-test/response/items_area_with_one_parameter_name.json +++ b/datastore/integration-test/response/items_area_with_one_parameter_name.json @@ -9,10 +9,12 @@ }, "properties": { "title": "Air Temperature Minimum last 12 Hours", + "summary": "Most recent 10 minute in situ observations of the Dutch meteorological observation network", "keywords": "temperature, pressure, relative humidity, visibility, wind speed, wind direction, wind gust", "keywords_vocabulary": "http://data.knmi.nl", + "license": "CC BY 4.0", "naming_authority": "nl.knmi", - "creator_type": "Institution", + "creator_type": "institution", "creator_name": "KNMI", "creator_email": "datacentrum@knmi.nl", "creator_url": "http://data.knmi.nl", @@ -20,15 +22,15 @@ "source": "Royal Netherlands Meteorological Institute (KNMI)", "platform": "0-20000-0-06275", "standard_name": "air_temperature", - "unit": "degrees Celsius", + "unit": "degC", "level": 2.0, "period": "PT12H", "instrument": "Tn12", "parameter_name": "air_temperature:2.0:minimum:PT12H", - "timeseries_id": "f0d06231a6508f281dbdaea0b5000220", + "timeseries_id": "20bf5cc13b08b4c1d737966b64d0aea5", "platform_vocabulary": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06275", "platform_name": "DEELEN", "method": "minimum" }, - "id": "f0d06231a6508f281dbdaea0b5000220" + "id": "20bf5cc13b08b4c1d737966b64d0aea5" } diff --git a/datastore/integration-test/response/items_within_area_single_platform.json b/datastore/integration-test/response/items_within_area_single_platform.json index 63fb90fb..9267a9cb 100644 --- a/datastore/integration-test/response/items_within_area_single_platform.json +++ b/datastore/integration-test/response/items_within_area_single_platform.json @@ -11,29 +11,31 @@ ] }, "properties": { - "title": "Wind Speed at 10m 10 Min Average", + "title": "Wind Direction 10 Min Average", + "summary": "Most recent 10 minute in situ observations of the Dutch meteorological observation network", "keywords": "temperature, pressure, relative humidity, visibility, wind speed, wind direction, wind gust", "keywords_vocabulary": "http://data.knmi.nl", + "license": "CC BY 4.0", "naming_authority": "nl.knmi", - "creator_type": "Institution", + "creator_type": "institution", "creator_name": "KNMI", "creator_email": "datacentrum@knmi.nl", "creator_url": "http://data.knmi.nl", "institution": "Royal Netherlands Meteorological Institute (KNMI)", "source": "Royal Netherlands Meteorological Institute (KNMI)", "platform": "0-20000-0-06225", - "standard_name": "wind_speed", - "unit": "m s-1", - "level": 10.0, + "standard_name": "wind_from_direction", + "unit": "degrees", + "level": 2.0, "period": "PT10M", - "instrument": "ff", - "parameter_name": "wind_speed:10.0:mean:PT10M", - "timeseries_id": "6987eca5b92eb041ce27ae26d4bff163", + "instrument": "dd", + "parameter_name": "wind_from_direction:2.0:mean:PT10M", + "timeseries_id": "18cd5ffa1c8fbe2c7e4829bde1e66dcf", "platform_vocabulary": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06225", "platform_name": "IJMUIDEN WP", "method": "mean" }, - "id": "6987eca5b92eb041ce27ae26d4bff163" + "id": "18cd5ffa1c8fbe2c7e4829bde1e66dcf" }, { "type": "Feature", @@ -46,10 +48,12 @@ }, "properties": { "title": "Wind Gust at 10m 10 Min Maximum", + "summary": "Most recent 10 minute in situ observations of the Dutch meteorological observation network", "keywords": "temperature, pressure, relative humidity, visibility, wind speed, wind direction, wind gust", "keywords_vocabulary": "http://data.knmi.nl", + "license": "CC BY 4.0", "naming_authority": "nl.knmi", - "creator_type": "Institution", + "creator_type": "institution", "creator_name": "KNMI", "creator_email": "datacentrum@knmi.nl", "creator_url": "http://data.knmi.nl", @@ -57,17 +61,17 @@ "source": "Royal Netherlands Meteorological Institute (KNMI)", "platform": "0-20000-0-06225", "standard_name": "wind_speed_of_gust", - "unit": "m s-1", + "unit": "m/s", "level": 10.0, "period": "PT10M", "instrument": "gff", "parameter_name": "wind_speed_of_gust:10.0:maximum:PT10M", - "timeseries_id": "86b9d3435ec8e5947a78f8d4a3566c47", + "timeseries_id": "1d8f48bf77c47d0ffd36f857b945bb00", "platform_vocabulary": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06225", "platform_name": "IJMUIDEN WP", "method": "maximum" }, - "id": "86b9d3435ec8e5947a78f8d4a3566c47" + "id": "1d8f48bf77c47d0ffd36f857b945bb00" }, { "type": "Feature", @@ -79,29 +83,31 @@ ] }, "properties": { - "title": "Wind Direction 10 Min Average", + "title": "Wind Speed at 10m 10 Min Average", + "summary": "Most recent 10 minute in situ observations of the Dutch meteorological observation network", "keywords": "temperature, pressure, relative humidity, visibility, wind speed, wind direction, wind gust", "keywords_vocabulary": "http://data.knmi.nl", + "license": "CC BY 4.0", "naming_authority": "nl.knmi", - "creator_type": "Institution", + "creator_type": "institution", "creator_name": "KNMI", "creator_email": "datacentrum@knmi.nl", "creator_url": "http://data.knmi.nl", "institution": "Royal Netherlands Meteorological Institute (KNMI)", "source": "Royal Netherlands Meteorological Institute (KNMI)", "platform": "0-20000-0-06225", - "standard_name": "wind_from_direction", - "unit": "degree", - "level": 2.0, + "standard_name": "wind_speed", + "unit": "m/s", + "level": 10.0, "period": "PT10M", - "instrument": "dd", - "parameter_name": "wind_from_direction:2.0:mean:PT10M", - "timeseries_id": "c056de2030d4019d8b6a3efd20d245f0", + "instrument": "ff", + "parameter_name": "wind_speed:10.0:mean:PT10M", + "timeseries_id": "96cca44cbbd697fce89cc8fd8f75cf94", "platform_vocabulary": "https://oscar.wmo.int/surface/rest/api/search/station?wigosId=0-20000-0-06225", "platform_name": "IJMUIDEN WP", "method": "mean" }, - "id": "c056de2030d4019d8b6a3efd20d245f0" + "id": "96cca44cbbd697fce89cc8fd8f75cf94" } ] } diff --git a/datastore/integration-test/response/metadata_collections_all_collections.json b/datastore/integration-test/response/metadata_collections_all_collections.json index 37cfa1f4..5fea4e89 100644 --- a/datastore/integration-test/response/metadata_collections_all_collections.json +++ b/datastore/integration-test/response/metadata_collections_all_collections.json @@ -43,25 +43,22 @@ "id": "standard_names", "interval": [ [ - "air_pressure_at_sea_level", + "air_pressure_at_mean_sea_level", "wind_speed_of_gust" ] ], "values": [ - "air_pressure_at_sea_level", + "air_pressure_at_mean_sea_level", "air_temperature", + "cloud_area_fraction", "cloud_base_altitude", - "cloud_cover", "dew_point_temperature", "duration_of_sunshine", "lwe_precipitation_rate", - "placeholder", - "precipitation_duration", - "precipitation_rate", "rainfall_amount", - "rainfall_duration", + "rainfall_rate", "relative_humidity", - "total_downwelling_shortwave_flux_in_air", + "surface_downwelling_shortwave_flux_in_air", "visibility_in_air", "wind_from_direction", "wind_speed", @@ -117,7 +114,6 @@ "PT0S", "PT1M", "PT10M", - "PT1H", "PT6H", "PT12H", "PT14H", @@ -172,32 +168,32 @@ "CoverageJSON" ], "parameter_names": { - "air_pressure_at_sea_level:1.0:mean:PT1M": { + "air_pressure_at_mean_sea_level:1.0:mean:PT1M": { "type": "Parameter", - "description": "air_pressure_at_sea_level at 1.0m, aggregated over PT1M with method 'mean'", + "description": "Air pressure at mean sea level at 1.0m, aggregated over PT1M with method 'mean'", "unit": { "label": "hPa" }, "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/air_pressure_at_sea_level", - "label": "air_pressure_at_sea_level:1.0:mean:PT1M" + "id": "https://vocab.nerc.ac.uk/standard_name/air_pressure_at_mean_sea_level", + "label": "Air pressure at mean sea level" }, "measurementType": { "method": "mean", "period": "PT1M" }, - "rodeo:standard_name": "air_pressure_at_sea_level", + "rodeo:standard_name": "air_pressure_at_mean_sea_level", "rodeo:level": 1.0 }, "air_temperature:0.1:minimum:PT10M": { "type": "Parameter", - "description": "air_temperature at 0.1m, aggregated over PT10M with method 'minimum'", + "description": "Air temperature at 0.1m, aggregated over PT10M with method 'minimum'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": "air_temperature:0.1:minimum:PT10M" + "label": "Air temperature" }, "measurementType": { "method": "minimum", @@ -208,13 +204,13 @@ }, "air_temperature:0.0:minimum:PT12H": { "type": "Parameter", - "description": "air_temperature at 0.0m, aggregated over PT12H with method 'minimum'", + "description": "Air temperature at 0.0m, aggregated over PT12H with method 'minimum'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": "air_temperature:0.0:minimum:PT12H" + "label": "Air temperature" }, "measurementType": { "method": "minimum", @@ -225,13 +221,13 @@ }, "air_temperature:0.0:minimum:PT14H": { "type": "Parameter", - "description": "air_temperature at 0.0m, aggregated over PT14H with method 'minimum'", + "description": "Air temperature at 0.0m, aggregated over PT14H with method 'minimum'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": "air_temperature:0.0:minimum:PT14H" + "label": "Air temperature" }, "measurementType": { "method": "minimum", @@ -242,13 +238,13 @@ }, "air_temperature:0.0:minimum:PT6H": { "type": "Parameter", - "description": "air_temperature at 0.0m, aggregated over PT6H with method 'minimum'", + "description": "Air temperature at 0.0m, aggregated over PT6H with method 'minimum'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": "air_temperature:0.0:minimum:PT6H" + "label": "Air temperature" }, "measurementType": { "method": "minimum", @@ -259,13 +255,13 @@ }, "air_temperature:1.5:maximum:PT10M": { "type": "Parameter", - "description": "air_temperature at 1.5m, aggregated over PT10M with method 'maximum'", + "description": "Air temperature at 1.5m, aggregated over PT10M with method 'maximum'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": "air_temperature:1.5:maximum:PT10M" + "label": "Air temperature" }, "measurementType": { "method": "maximum", @@ -276,13 +272,13 @@ }, "air_temperature:1.5:minimum:PT10M": { "type": "Parameter", - "description": "air_temperature at 1.5m, aggregated over PT10M with method 'minimum'", + "description": "Air temperature at 1.5m, aggregated over PT10M with method 'minimum'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": "air_temperature:1.5:minimum:PT10M" + "label": "Air temperature" }, "measurementType": { "method": "minimum", @@ -293,13 +289,13 @@ }, "air_temperature:2.0:maximum:PT12H": { "type": "Parameter", - "description": "air_temperature at 2.0m, aggregated over PT12H with method 'maximum'", + "description": "Air temperature at 2.0m, aggregated over PT12H with method 'maximum'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": "air_temperature:2.0:maximum:PT12H" + "label": "Air temperature" }, "measurementType": { "method": "maximum", @@ -310,13 +306,13 @@ }, "air_temperature:2.0:maximum:PT24H": { "type": "Parameter", - "description": "air_temperature at 2.0m, aggregated over PT24H with method 'maximum'", + "description": "Air temperature at 2.0m, aggregated over PT24H with method 'maximum'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": "air_temperature:2.0:maximum:PT24H" + "label": "Air temperature" }, "measurementType": { "method": "maximum", @@ -327,13 +323,13 @@ }, "air_temperature:2.0:maximum:PT6H": { "type": "Parameter", - "description": "air_temperature at 2.0m, aggregated over PT6H with method 'maximum'", + "description": "Air temperature at 2.0m, aggregated over PT6H with method 'maximum'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": "air_temperature:2.0:maximum:PT6H" + "label": "Air temperature" }, "measurementType": { "method": "maximum", @@ -344,13 +340,13 @@ }, "air_temperature:2.0:mean:PT1M": { "type": "Parameter", - "description": "air_temperature at 2.0m, aggregated over PT1M with method 'mean'", + "description": "Air temperature at 2.0m, aggregated over PT1M with method 'mean'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": "air_temperature:2.0:mean:PT1M" + "label": "Air temperature" }, "measurementType": { "method": "mean", @@ -361,13 +357,13 @@ }, "air_temperature:2.0:minimum:PT12H": { "type": "Parameter", - "description": "air_temperature at 2.0m, aggregated over PT12H with method 'minimum'", + "description": "Air temperature at 2.0m, aggregated over PT12H with method 'minimum'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": "air_temperature:2.0:minimum:PT12H" + "label": "Air temperature" }, "measurementType": { "method": "minimum", @@ -378,13 +374,13 @@ }, "air_temperature:2.0:minimum:PT14H": { "type": "Parameter", - "description": "air_temperature at 2.0m, aggregated over PT14H with method 'minimum'", + "description": "Air temperature at 2.0m, aggregated over PT14H with method 'minimum'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": "air_temperature:2.0:minimum:PT14H" + "label": "Air temperature" }, "measurementType": { "method": "minimum", @@ -395,13 +391,13 @@ }, "air_temperature:2.0:minimum:PT6H": { "type": "Parameter", - "description": "air_temperature at 2.0m, aggregated over PT6H with method 'minimum'", + "description": "Air temperature at 2.0m, aggregated over PT6H with method 'minimum'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", - "label": "air_temperature:2.0:minimum:PT6H" + "label": "Air temperature" }, "measurementType": { "method": "minimum", @@ -410,49 +406,32 @@ "rodeo:standard_name": "air_temperature", "rodeo:level": 2.0 }, - "cloud_base_altitude:0.0:point:PT0S": { + "cloud_area_fraction:2.0:point:PT0S": { "type": "Parameter", - "description": "cloud_base_altitude at 0.0m, aggregated over PT0S with method 'point'", + "description": "Cloud area fraction at 2.0m, aggregated over PT0S with method 'point'", "unit": { - "label": "ft" + "label": "oktas" }, "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude", - "label": "cloud_base_altitude:0.0:point:PT0S" + "id": "https://vocab.nerc.ac.uk/standard_name/cloud_area_fraction", + "label": "Cloud area fraction" }, "measurementType": { "method": "point", "period": "PT0S" }, - "rodeo:standard_name": "cloud_base_altitude", - "rodeo:level": 0.0 - }, - "cloud_base_altitude:1.0:point:PT0S": { - "type": "Parameter", - "description": "cloud_base_altitude at 1.0m, aggregated over PT0S with method 'point'", - "unit": { - "label": "ft" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude", - "label": "cloud_base_altitude:1.0:point:PT0S" - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_base_altitude", - "rodeo:level": 1.0 + "rodeo:standard_name": "cloud_area_fraction", + "rodeo:level": 2.0 }, "cloud_base_altitude:2.0:point:PT0S": { "type": "Parameter", - "description": "cloud_base_altitude at 2.0m, aggregated over PT0S with method 'point'", + "description": "Cloud base altitude at 2.0m, aggregated over PT0S with method 'point'", "unit": { - "label": "ft" + "label": "m" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/cloud_base_altitude", - "label": "cloud_base_altitude:2.0:point:PT0S" + "label": "Cloud base altitude" }, "measurementType": { "method": "point", @@ -461,66 +440,15 @@ "rodeo:standard_name": "cloud_base_altitude", "rodeo:level": 2.0 }, - "cloud_cover:0.0:point:PT0S": { - "type": "Parameter", - "description": "cloud_cover at 0.0m, aggregated over PT0S with method 'point'", - "unit": { - "label": "octa" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_cover", - "label": "cloud_cover:0.0:point:PT0S" - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_cover", - "rodeo:level": 0.0 - }, - "cloud_cover:1.0:point:PT0S": { - "type": "Parameter", - "description": "cloud_cover at 1.0m, aggregated over PT0S with method 'point'", - "unit": { - "label": "octa" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_cover", - "label": "cloud_cover:1.0:point:PT0S" - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_cover", - "rodeo:level": 1.0 - }, - "cloud_cover:2.0:point:PT0S": { - "type": "Parameter", - "description": "cloud_cover at 2.0m, aggregated over PT0S with method 'point'", - "unit": { - "label": "octa" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/cloud_cover", - "label": "cloud_cover:2.0:point:PT0S" - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "cloud_cover", - "rodeo:level": 2.0 - }, "dew_point_temperature:1.5:mean:PT1M": { "type": "Parameter", - "description": "dew_point_temperature at 1.5m, aggregated over PT1M with method 'mean'", + "description": "Dew point temperature at 1.5m, aggregated over PT1M with method 'mean'", "unit": { - "label": "degrees Celsius" + "label": "degC" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/dew_point_temperature", - "label": "dew_point_temperature:1.5:mean:PT1M" + "label": "Dew point temperature" }, "measurementType": { "method": "mean", @@ -531,13 +459,13 @@ }, "duration_of_sunshine:2.0:point:PT0S": { "type": "Parameter", - "description": "duration_of_sunshine at 2.0m, aggregated over PT0S with method 'point'", + "description": "Duration of sunshine at 2.0m, aggregated over PT0S with method 'point'", "unit": { - "label": "min" + "label": "s" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/duration_of_sunshine", - "label": "duration_of_sunshine:2.0:point:PT0S" + "label": "Duration of sunshine" }, "measurementType": { "method": "point", @@ -548,13 +476,13 @@ }, "lwe_precipitation_rate:1.8:mean:PT10M": { "type": "Parameter", - "description": "lwe_precipitation_rate at 1.8m, aggregated over PT10M with method 'mean'", + "description": "Lwe precipitation rate at 1.8m, aggregated over PT10M with method 'mean'", "unit": { "label": "mm/h" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/lwe_precipitation_rate", - "label": "lwe_precipitation_rate:1.8:mean:PT10M" + "label": "Lwe precipitation rate" }, "measurementType": { "method": "mean", @@ -565,13 +493,13 @@ }, "lwe_precipitation_rate:2.5:mean:PT10M": { "type": "Parameter", - "description": "lwe_precipitation_rate at 2.5m, aggregated over PT10M with method 'mean'", + "description": "Lwe precipitation rate at 2.5m, aggregated over PT10M with method 'mean'", "unit": { "label": "mm/h" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/lwe_precipitation_rate", - "label": "lwe_precipitation_rate:2.5:mean:PT10M" + "label": "Lwe precipitation rate" }, "measurementType": { "method": "mean", @@ -580,185 +508,15 @@ "rodeo:standard_name": "lwe_precipitation_rate", "rodeo:level": 2.5 }, - "placeholder:1.8:point:PT1H": { - "type": "Parameter", - "description": "placeholder at 1.8m, aggregated over PT1H with method 'point'", - "unit": { - "label": "code" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/placeholder", - "label": "placeholder:1.8:point:PT1H" - }, - "measurementType": { - "method": "point", - "period": "PT1H" - }, - "rodeo:standard_name": "placeholder", - "rodeo:level": 1.8 - }, - "placeholder:1.8:point:PT0S": { - "type": "Parameter", - "description": "placeholder at 1.8m, aggregated over PT0S with method 'point'", - "unit": { - "label": "code" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/placeholder", - "label": "placeholder:1.8:point:PT0S" - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "placeholder", - "rodeo:level": 1.8 - }, - "placeholder:1.8:point:PT10M": { - "type": "Parameter", - "description": "placeholder at 1.8m, aggregated over PT10M with method 'point'", - "unit": { - "label": "code" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/placeholder", - "label": "placeholder:1.8:point:PT10M" - }, - "measurementType": { - "method": "point", - "period": "PT10M" - }, - "rodeo:standard_name": "placeholder", - "rodeo:level": 1.8 - }, - "placeholder:2.5:point:PT1H": { - "type": "Parameter", - "description": "placeholder at 2.5m, aggregated over PT1H with method 'point'", - "unit": { - "label": "code" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/placeholder", - "label": "placeholder:2.5:point:PT1H" - }, - "measurementType": { - "method": "point", - "period": "PT1H" - }, - "rodeo:standard_name": "placeholder", - "rodeo:level": 2.5 - }, - "placeholder:2.5:point:PT0S": { - "type": "Parameter", - "description": "placeholder at 2.5m, aggregated over PT0S with method 'point'", - "unit": { - "label": "code" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/placeholder", - "label": "placeholder:2.5:point:PT0S" - }, - "measurementType": { - "method": "point", - "period": "PT0S" - }, - "rodeo:standard_name": "placeholder", - "rodeo:level": 2.5 - }, - "placeholder:2.5:point:PT10M": { - "type": "Parameter", - "description": "placeholder at 2.5m, aggregated over PT10M with method 'point'", - "unit": { - "label": "code" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/placeholder", - "label": "placeholder:2.5:point:PT10M" - }, - "measurementType": { - "method": "point", - "period": "PT10M" - }, - "rodeo:standard_name": "placeholder", - "rodeo:level": 2.5 - }, - "precipitation_duration:1.8:point:PT10M": { - "type": "Parameter", - "description": "precipitation_duration at 1.8m, aggregated over PT10M with method 'point'", - "unit": { - "label": "sec" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/precipitation_duration", - "label": "precipitation_duration:1.8:point:PT10M" - }, - "measurementType": { - "method": "point", - "period": "PT10M" - }, - "rodeo:standard_name": "precipitation_duration", - "rodeo:level": 1.8 - }, - "precipitation_duration:2.0:point:PT10M": { - "type": "Parameter", - "description": "precipitation_duration at 2.0m, aggregated over PT10M with method 'point'", - "unit": { - "label": "sec" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/precipitation_duration", - "label": "precipitation_duration:2.0:point:PT10M" - }, - "measurementType": { - "method": "point", - "period": "PT10M" - }, - "rodeo:standard_name": "precipitation_duration", - "rodeo:level": 2.0 - }, - "precipitation_duration:2.5:point:PT10M": { - "type": "Parameter", - "description": "precipitation_duration at 2.5m, aggregated over PT10M with method 'point'", - "unit": { - "label": "sec" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/precipitation_duration", - "label": "precipitation_duration:2.5:point:PT10M" - }, - "measurementType": { - "method": "point", - "period": "PT10M" - }, - "rodeo:standard_name": "precipitation_duration", - "rodeo:level": 2.5 - }, - "precipitation_rate:2.0:mean:PT10M": { - "type": "Parameter", - "description": "precipitation_rate at 2.0m, aggregated over PT10M with method 'mean'", - "unit": { - "label": "mm/h" - }, - "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/precipitation_rate", - "label": "precipitation_rate:2.0:mean:PT10M" - }, - "measurementType": { - "method": "mean", - "period": "PT10M" - }, - "rodeo:standard_name": "precipitation_rate", - "rodeo:level": 2.0 - }, "rainfall_amount:2.0:point:PT0S": { "type": "Parameter", - "description": "rainfall_amount at 2.0m, aggregated over PT0S with method 'point'", + "description": "Rainfall amount at 2.0m, aggregated over PT0S with method 'point'", "unit": { - "label": "mm" + "label": "kg/m2" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", - "label": "rainfall_amount:2.0:point:PT0S" + "label": "Rainfall amount" }, "measurementType": { "method": "point", @@ -769,13 +527,13 @@ }, "rainfall_amount:2.0:point:PT12H": { "type": "Parameter", - "description": "rainfall_amount at 2.0m, aggregated over PT12H with method 'point'", + "description": "Rainfall amount at 2.0m, aggregated over PT12H with method 'point'", "unit": { - "label": "mm" + "label": "kg/m2" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", - "label": "rainfall_amount:2.0:point:PT12H" + "label": "Rainfall amount" }, "measurementType": { "method": "point", @@ -786,13 +544,13 @@ }, "rainfall_amount:2.0:point:PT24H": { "type": "Parameter", - "description": "rainfall_amount at 2.0m, aggregated over PT24H with method 'point'", + "description": "Rainfall amount at 2.0m, aggregated over PT24H with method 'point'", "unit": { - "label": "mm" + "label": "kg/m2" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", - "label": "rainfall_amount:2.0:point:PT24H" + "label": "Rainfall amount" }, "measurementType": { "method": "point", @@ -803,13 +561,13 @@ }, "rainfall_amount:2.0:point:PT6H": { "type": "Parameter", - "description": "rainfall_amount at 2.0m, aggregated over PT6H with method 'point'", + "description": "Rainfall amount at 2.0m, aggregated over PT6H with method 'point'", "unit": { - "label": "mm" + "label": "kg/m2" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_amount", - "label": "rainfall_amount:2.0:point:PT6H" + "label": "Rainfall amount" }, "measurementType": { "method": "point", @@ -818,32 +576,32 @@ "rodeo:standard_name": "rainfall_amount", "rodeo:level": 2.0 }, - "rainfall_duration:2.0:point:PT0S": { + "rainfall_rate:2.0:mean:PT10M": { "type": "Parameter", - "description": "rainfall_duration at 2.0m, aggregated over PT0S with method 'point'", + "description": "Rainfall rate at 2.0m, aggregated over PT10M with method 'mean'", "unit": { - "label": "min" + "label": "mm/h" }, "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_duration", - "label": "rainfall_duration:2.0:point:PT0S" + "id": "https://vocab.nerc.ac.uk/standard_name/rainfall_rate", + "label": "Rainfall rate" }, "measurementType": { - "method": "point", - "period": "PT0S" + "method": "mean", + "period": "PT10M" }, - "rodeo:standard_name": "rainfall_duration", + "rodeo:standard_name": "rainfall_rate", "rodeo:level": 2.0 }, "relative_humidity:2.0:mean:PT1M": { "type": "Parameter", - "description": "relative_humidity at 2.0m, aggregated over PT1M with method 'mean'", + "description": "Relative humidity at 2.0m, aggregated over PT1M with method 'mean'", "unit": { - "label": "%" + "label": "percent" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/relative_humidity", - "label": "relative_humidity:2.0:mean:PT1M" + "label": "Relative humidity" }, "measurementType": { "method": "mean", @@ -852,32 +610,32 @@ "rodeo:standard_name": "relative_humidity", "rodeo:level": 2.0 }, - "total_downwelling_shortwave_flux_in_air:2.0:mean:PT10M": { + "surface_downwelling_shortwave_flux_in_air:2.0:mean:PT10M": { "type": "Parameter", - "description": "total_downwelling_shortwave_flux_in_air at 2.0m, aggregated over PT10M with method 'mean'", + "description": "Surface downwelling shortwave flux in air at 2.0m, aggregated over PT10M with method 'mean'", "unit": { - "label": "W m-2" + "label": "W/m2" }, "observedProperty": { - "id": "https://vocab.nerc.ac.uk/standard_name/total_downwelling_shortwave_flux_in_air", - "label": "total_downwelling_shortwave_flux_in_air:2.0:mean:PT10M" + "id": "https://vocab.nerc.ac.uk/standard_name/surface_downwelling_shortwave_flux_in_air", + "label": "Surface downwelling shortwave flux in air" }, "measurementType": { "method": "mean", "period": "PT10M" }, - "rodeo:standard_name": "total_downwelling_shortwave_flux_in_air", + "rodeo:standard_name": "surface_downwelling_shortwave_flux_in_air", "rodeo:level": 2.0 }, "visibility_in_air:1.8:mean:PT10M": { "type": "Parameter", - "description": "visibility_in_air at 1.8m, aggregated over PT10M with method 'mean'", + "description": "Visibility in air at 1.8m, aggregated over PT10M with method 'mean'", "unit": { "label": "m" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/visibility_in_air", - "label": "visibility_in_air:1.8:mean:PT10M" + "label": "Visibility in air" }, "measurementType": { "method": "mean", @@ -888,13 +646,13 @@ }, "visibility_in_air:2.5:mean:PT10M": { "type": "Parameter", - "description": "visibility_in_air at 2.5m, aggregated over PT10M with method 'mean'", + "description": "Visibility in air at 2.5m, aggregated over PT10M with method 'mean'", "unit": { "label": "m" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/visibility_in_air", - "label": "visibility_in_air:2.5:mean:PT10M" + "label": "Visibility in air" }, "measurementType": { "method": "mean", @@ -905,13 +663,13 @@ }, "wind_from_direction:2.0:mean:PT10M": { "type": "Parameter", - "description": "wind_from_direction at 2.0m, aggregated over PT10M with method 'mean'", + "description": "Wind from direction at 2.0m, aggregated over PT10M with method 'mean'", "unit": { - "label": "degree" + "label": "degrees" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_from_direction", - "label": "wind_from_direction:2.0:mean:PT10M" + "label": "Wind from direction" }, "measurementType": { "method": "mean", @@ -922,13 +680,13 @@ }, "wind_speed:10.0:mean:PT10M": { "type": "Parameter", - "description": "wind_speed at 10.0m, aggregated over PT10M with method 'mean'", + "description": "Wind speed at 10.0m, aggregated over PT10M with method 'mean'", "unit": { - "label": "m s-1" + "label": "m/s" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_speed", - "label": "wind_speed:10.0:mean:PT10M" + "label": "Wind speed" }, "measurementType": { "method": "mean", @@ -939,13 +697,13 @@ }, "wind_speed_of_gust:10.0:maximum:PT10M": { "type": "Parameter", - "description": "wind_speed_of_gust at 10.0m, aggregated over PT10M with method 'maximum'", + "description": "Wind speed of gust at 10.0m, aggregated over PT10M with method 'maximum'", "unit": { - "label": "m s-1" + "label": "m/s" }, "observedProperty": { "id": "https://vocab.nerc.ac.uk/standard_name/wind_speed_of_gust", - "label": "wind_speed_of_gust:10.0:maximum:PT10M" + "label": "Wind speed of gust" }, "measurementType": { "method": "maximum", diff --git a/datastore/integration-test/test_api.py b/datastore/integration-test/test_api.py index 61f1888e..422fd24d 100644 --- a/datastore/integration-test/test_api.py +++ b/datastore/integration-test/test_api.py @@ -140,7 +140,7 @@ def test_from_a_single_collection_get_locations_within_bbox_with_levels_range_fi def test_from_a_single_collection_get_locations_within_a_bbox_with_parameter_name_filtering(): collection_id = "observations" bbox = "5.0,52.0,6.0,52.1" - parameters = "air_temperature:0.1:minimum:PT10M, air_pressure_at_sea_level:1.0:mean:PT1M" + parameters = "air_temperature:0.1:minimum:PT10M, air_pressure_at_mean_sea_level:1.0:mean:PT1M" actual_response = requests.get( url=BASE_URL + f"/collections/{collection_id}/locations?bbox={bbox}¶meter-name={parameters}" ) @@ -427,7 +427,7 @@ def test_items_get_area(): def test_items_get_id(): collection_id = "observations" - timeseries_id = "f0d06231a6508f281dbdaea0b5000220" + timeseries_id = "20bf5cc13b08b4c1d737966b64d0aea5" actual_response = requests.get(url=BASE_URL + f"/collections/{collection_id}/items/{timeseries_id}") assert actual_response.status_code == 200 diff --git a/datastore/integration-test/test_knmi.py b/datastore/integration-test/test_knmi.py index 06c720ea..fa786780 100644 --- a/datastore/integration-test/test_knmi.py +++ b/datastore/integration-test/test_knmi.py @@ -46,7 +46,9 @@ def test_find_series_single_station_all_parameters(grpc_stub): request = dstore.GetObsRequest(filter=dict(platform=dstore.Strings(values=["0-20000-0-06260"]))) response = grpc_stub.GetObservations(request) - assert len(response.observations) == 42 # Station 06260 doesn't have all parameters + assert ( + len([obs.ts_mdata.parameter_name for obs in response.observations]) == 30 + ) # Station 06260 doesn't have all parameters def test_get_values_single_station_single_parameter(grpc_stub): diff --git a/docker-compose.yml b/docker-compose.yml index e9c55d50..a31e0b85 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -115,6 +115,14 @@ services: store: condition: service_healthy + ingest-unit: + profiles: ["test"] + build: + context: ingest + dockerfile: unit.Dockerfile + volumes: + - ./ingest/test/output:/app/output + client: profiles: ["test"] build: @@ -138,6 +146,20 @@ services: store: condition: service_healthy + ingest-loader: + profiles: ["test"] + build: + context: datastore/data-loader + args: + THROUGH_INGEST: "true" + environment: + - INGEST_URL=http://ingest:8001/json + depends_on: + store: + condition: service_healthy + ingest: + condition: service_started + integration: profiles: ["test"] build: diff --git a/ingest/requirements-dev.in b/ingest/requirements-dev.in new file mode 100644 index 00000000..aadddc87 --- /dev/null +++ b/ingest/requirements-dev.in @@ -0,0 +1,10 @@ +# Generate requirements-dev.txt using: +# pip-compile --upgrade --no-emit-index-url requirements-dev.in +# Install normal+dev dependencies using: +# pip-sync requirements-dev.txt + +-r requirements.txt +httpx~=0.27.0 +pytest~=8.1 +pytest-cov~=5.0 +pytest-timeout~=2.3 diff --git a/ingest/requirements-dev.txt b/ingest/requirements-dev.txt index 5e0d552e..45e1df76 100644 --- a/ingest/requirements-dev.txt +++ b/ingest/requirements-dev.txt @@ -1,3 +1,157 @@ -build<=1.0.3, <2.0.0 -setuptools<=59.6.0, <60.0.0 -httpx<=0.26.0 +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --no-emit-index-url requirements-dev.in +# +annotated-types==0.7.0 + # via + # -r requirements.txt + # pydantic +anyio==4.4.0 + # via + # -r requirements.txt + # httpx + # starlette +attrs==24.2.0 + # via + # -r requirements.txt + # jsonschema + # referencing +certifi==2024.8.30 + # via + # -r requirements.txt + # httpcore + # httpx + # requests +charset-normalizer==3.3.2 + # via + # -r requirements.txt + # requests +click==8.1.7 + # via + # -r requirements.txt + # uvicorn +coverage[toml]==7.6.3 + # via pytest-cov +exceptiongroup==1.2.2 + # via -r requirements.txt +fastapi==0.108.0 + # via -r requirements.txt +grpcio==1.62.1 + # via + # -r requirements.txt + # grpcio-tools +grpcio-tools==1.62.1 + # via -r requirements.txt +gunicorn==22.0.0 + # via -r requirements.txt +h11==0.14.0 + # via + # -r requirements.txt + # httpcore + # uvicorn +httpcore==1.0.6 + # via httpx +httpx==0.27.2 + # via -r requirements-dev.in +idna==3.10 + # via + # -r requirements.txt + # anyio + # httpx + # requests +iniconfig==2.0.0 + # via pytest +isodate==0.6.1 + # via -r requirements.txt +jsonschema==4.19.2 + # via -r requirements.txt +jsonschema-specifications==2023.12.1 + # via + # -r requirements.txt + # jsonschema +numpy==1.25.2 + # via -r requirements.txt +packaging==24.1 + # via + # -r requirements.txt + # gunicorn + # pytest +paho-mqtt==2.1.0 + # via -r requirements.txt +pluggy==1.5.0 + # via pytest +prometheus-client==0.20.0 + # via + # -r requirements.txt + # prometheus-fastapi-instrumentator +prometheus-fastapi-instrumentator==7.0.0 + # via -r requirements.txt +protobuf==4.25.4 + # via + # -r requirements.txt + # grpcio-tools +pydantic==2.9.1 + # via + # -r requirements.txt + # fastapi +pydantic-core==2.23.3 + # via + # -r requirements.txt + # pydantic +pytest==8.3.3 + # via + # -r requirements-dev.in + # pytest-cov + # pytest-timeout +pytest-cov==5.0.0 + # via -r requirements-dev.in +pytest-timeout==2.3.1 + # via -r requirements-dev.in +python-dateutil==2.8.2 + # via -r requirements-dev.in +python-multipart==0.0.9 + # via -r requirements.txt +referencing==0.35.1 + # via + # -r requirements.txt + # jsonschema + # jsonschema-specifications +requests==2.31.0 + # via -r requirements.txt +rpds-py==0.20.0 + # via + # -r requirements.txt + # jsonschema + # referencing +six==1.16.0 + # via + # -r requirements.txt + # isodate + # python-dateutil +sniffio==1.3.1 + # via + # -r requirements.txt + # anyio + # httpx +starlette==0.32.0.post1 + # via + # -r requirements.txt + # fastapi + # prometheus-fastapi-instrumentator +typing-extensions==4.12.2 + # via + # -r requirements.txt + # fastapi + # pydantic + # pydantic-core +urllib3==2.2.3 + # via + # -r requirements.txt + # requests +uvicorn==0.25.0 + # via -r requirements.txt + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/ingest/test/.coveragerc b/ingest/test/.coveragerc new file mode 100644 index 00000000..d057a8b5 --- /dev/null +++ b/ingest/test/.coveragerc @@ -0,0 +1,2 @@ +[run] +omit = test/* diff --git a/ingest/unit.Dockerfile b/ingest/unit.Dockerfile new file mode 100644 index 00000000..a02f9343 --- /dev/null +++ b/ingest/unit.Dockerfile @@ -0,0 +1,57 @@ +FROM python:3.10-slim-bookworm + +SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"] + +ENV DOCKER_PATH="/app" + +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get install -y --no-install-recommends git libeccodes-data rapidjson-dev pybind11-dev make g++ libudunits2-0\ + # Cleanup + && rm -rf /usr/tmp \ + && apt-get autoremove -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto" +COPY "./api" "${DOCKER_PATH}/api/" +COPY "requirements-dev.txt" "${DOCKER_PATH}/api/" +COPY "./src/" "${DOCKER_PATH}/src/" +COPY "./pyproject.toml" "${DOCKER_PATH}/pyproject.toml" +COPY "./setup.py" "${DOCKER_PATH}/setup.py" +COPY "./test" "${DOCKER_PATH}/test" + +RUN pip install --no-cache-dir --upgrade -r "${DOCKER_PATH}/api/requirements-dev.txt" \ + && pip install --no-cache-dir --upgrade pybind11~="2.11.1" + +# hadolint ignore=DL3013 +WORKDIR "${DOCKER_PATH}"/src/ingest/bufr/ +RUN make +WORKDIR / + +# Compiling the protobuf file +RUN python -m grpc_tools.protoc \ + --proto_path="protobuf" "protobuf/datastore.proto" \ + --python_out="${DOCKER_PATH}" \ + --grpc_python_out="${DOCKER_PATH}" + +WORKDIR "${DOCKER_PATH}" + +RUN python "api/generate_standard_name.py" + +# hadolint ignore=DL3013 +RUN pip install --no-cache-dir --upgrade pip \ + && pip install --no-cache-dir --upgrade "${DOCKER_PATH}/" \ + && mkdir -p /tmp/metrics + +ENV PROMETHEUS_MULTIPROC_DIR=/tmp/metrics + +CMD ["/bin/sh", "-c", "{ python -m pytest \ + --timeout=300 \ + --junitxml=./output/pytest.xml \ + --cov-report=term-missing \ + --cov=. \ + --cov-config=./test/.coveragerc 2>&1; \ + echo $? > ./output/exit-code; } | \ + tee ./output/pytest-coverage.txt; \ + exit $(cat ./output/exit-code)"] diff --git a/justfile b/justfile index 39320f15..8f066dc6 100644 --- a/justfile +++ b/justfile @@ -5,13 +5,17 @@ default: set positional-arguments # Run all docker services. After running the database needs cleanup, run just destroy -all: lint build unit services monitoring load integration performance client +all: lint build unit ingest-unit services monitoring load integration performance client + # Build and run the default docker services up: build services # Build and run the default docker services and start up monitoring local: up monitoring # Build and run the unit, load and integration tests -test: build unit load integration +test: build unit ingest-load integration + +# Build and run the ingest loader and integration tests +ingest-test: build ingest-unit ingest-load integration # ---------------------------------------------------------------------------- # # utility # @@ -101,6 +105,8 @@ integration: unit: docker compose run --rm api-unit +ingest-unit: + docker compose run --rm ingest-unit # Start the monitoring monitoring: @@ -141,6 +147,8 @@ services: load: docker compose --env-file ./ci/config/env.list run --rm loader +ingest-load: + docker compose --env-file ./ci/config/env.list run --rm ingest-loader # Stop all E-SOH containers down: