Merge pull request #209 from EURODEO/issue_194_204-data-loader-to-ingest

ObservedProperty label & test data loader through ingestion
EURODEO · Nov 19, 2024 · 3f3a241 · 3f3a241 · github-actions · Nov 19, 2024
2 parents 5d239ce + 2347b27
commit 3f3a241
Show file tree

Hide file tree

Showing 35 changed files with 1,033 additions and 1,144 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -61,7 +61,7 @@ jobs:
         run: just services
 
       - name: Load the data into the database
-        run: just load
+        run: just ingest-load
 
       - name: Run the integration test
         run: just integration
@@ -113,56 +113,56 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
-      - name: Checkout Source
-        uses: actions/checkout@v4
 
-      - name: Install Dependencies
-        run: |
-          pip install --upgrade pip
-          pip install pytest-timeout
-          pip install pytest-cov
-          pip install httpx
-          pip install -r ./ingest/requirements.txt
-          pip install ./ingest
-          cd ./ingest && python3 api/generate_standard_name.py
+      - name: Install just
+        run: ./ci/scripts/install-just.sh
 
+      - name: Copy protobuf files
+        run: just copy-proto
 
-      - name: Copy Protobuf file to api directory and build
-        run: |
-          mkdir ./ingest/protobuf
-          cp ./protobuf/datastore.proto ./ingest/protobuf/datastore.proto
-          python -m grpc_tools.protoc --proto_path=./ingest/protobuf --python_out=./ingest --grpc_python_out=./ingest ./ingest/protobuf/datastore.proto
+      - name: Run the unit test
+        run: just ingest-unit
 
-      - name: Run Tests
-        run: |
-          cd ingest
-          mkdir -p /tmp/metrics
-          PROMETHEUS_MULTIPROC_DIR=/tmp/metrics python -m pytest -v --timeout=60
+      - name: Archive test artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ingest-test-results-artifact
+          path: |
+            ingest/test/output/pytest-coverage.txt
+            ingest/test/output/pytest.xml
+
+      - name: Cleanup
+        if: always()
+        run: just destroy
 
   publish-test-results:
-    needs: test-datastore
+    needs:
+      - test-datastore
+      - test-ingest
     runs-on: ubuntu-latest
-    if: github.event.ref_type != 'tag'
+    if: github.event_name != 'push' || github.event.ref_type != 'tag'
     permissions:
       contents: write
       issues: write
       pull-requests: write
     steps:
-      - name: Download test results so that they can be published
+      - name: Download test-datastore results so that they can be published
         uses: actions/download-artifact@v4
         with:
           name: test-results-artifact
+          path: ./artifacts/test-results
+
+      - name: Download test-ingest results so that they can be published
+        uses: actions/download-artifact@v4
+        with:
+          name: ingest-test-results-artifact
+          path: ./artifacts/ingest-results
+
       - name: Comment coverage
         uses: MishaKav/pytest-coverage-comment@main
         with:
-          pytest-coverage-path: api/test/output/pytest-coverage.txt
-          coverage-path-prefix: api/test/output/
-          title: API Unit Test Coverage Report
-          hide-badge: true
-          hide-report: false
-          create-new-comment: false
-          hide-comment: false
-          report-only-changed-files: false
-          remove-link-from-badge: false
-          junitxml-path: api/test/output/pytest.xml
-          junitxml-title: API Unit Test Coverage Summary
+          title: Unit Test Coverage Report
+          pytest-coverage-path: ./artifacts/test-results/api/test/output/pytest-coverage.txt
+          multiple-files: |
+            API Unit Tests, ./artifacts/test-results/api/test/output/pytest-coverage.txt, ./artifacts/test-results/api/test/output/pytest.xml
+            Ingest Unit Tests, ./artifacts/ingest-results/pytest-coverage.txt, ./artifacts/ingest-results/pytest.xml
diff --git a/.gitignore b/.gitignore
@@ -212,3 +212,4 @@ cf_standard_names_v84.txt
 # API
 api/test/output/
 datastore/load-test/output/
+ingest/test/output/
diff --git a/api/formatters/covjson.py b/api/formatters/covjson.py
@@ -32,6 +32,7 @@
 def make_parameter(ts_mdata):
     level = convert_cm_to_m(ts_mdata.level)
     period = seconds_to_iso_8601_duration(ts_mdata.period)
+    label = " ".join(ts_mdata.standard_name.capitalize().split("_"))
 
     custom_fields = {
         "rodeo:standard_name": ts_mdata.standard_name,
@@ -40,12 +41,11 @@ def make_parameter(ts_mdata):
 
     return Parameter(
         description={
-            "en": f"{ts_mdata.standard_name} at {level}m, "
-            f"aggregated over {period} with method '{ts_mdata.function}'",
+            "en": f"{label} at {level}m, aggregated over {period} with method '{ts_mdata.function}'",
         },
         observedProperty=ObservedProperty(
             id=f"https://vocab.nerc.ac.uk/standard_name/{ts_mdata.standard_name}",
-            label={"en": ts_mdata.parameter_name},
+            label={"en": label},
         ),
         measurementType=MeasurementType(
             method=ts_mdata.function,

diff --git a/api/metadata_endpoints.py b/api/metadata_endpoints.py
@@ -110,17 +110,18 @@ async def get_collection_metadata(base_url: str, is_self) -> Collection:
         ts = group.combo
         level = convert_cm_to_m(ts.level)
         period = seconds_to_iso_8601_duration(ts.period)
+        label = " ".join(ts.standard_name.capitalize().split("_"))
 
         custom_fields = {
             "rodeo:standard_name": ts.standard_name,
             "rodeo:level": level,
         }
 
         parameter = Parameter(
-            description=f"{ts.standard_name} at {level}m, aggregated over {period} with method '{ts.function}'",
+            description=f"{label} at {level}m, aggregated over {period} with method '{ts.function}'",
             observedProperty=ObservedProperty(
                 id=f"https://vocab.nerc.ac.uk/standard_name/{ts.standard_name}",
-                label=ts.parameter_name,
+                label=label,
             ),
             measurementType=MeasurementType(
                 method=ts.function,

diff --git a/api/test/test_data/test_coverages_covjson.json b/api/test/test_data/test_coverages_covjson.json
@@ -49,12 +49,12 @@
                 "air_temperature:2.0:mean:PT1M": {
                     "type": "Parameter",
                     "description": {
-                        "en": "air_temperature at 2.0m, aggregated over PT1M with method 'mean'"
+                        "en": "Air temperature at 2.0m, aggregated over PT1M with method 'mean'"
                     },
                     "observedProperty": {
                         "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature",
                         "label": {
-                            "en": "air_temperature:2.0:mean:PT1M"
+                            "en": "Air temperature"
                         }
                     },
                     "unit": {
@@ -139,12 +139,12 @@
                 "air_temperature:2.0:mean:PT1M": {
                     "type": "Parameter",
                     "description": {
-                        "en": "air_temperature at 2.0m, aggregated over PT1M with method 'mean'"
+                        "en": "Air temperature at 2.0m, aggregated over PT1M with method 'mean'"
                     },
                     "observedProperty": {
                         "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature",
                         "label": {
-                            "en": "air_temperature:2.0:mean:PT1M"
+                            "en": "Air temperature"
                         }
                     },
                     "unit": {
@@ -186,12 +186,12 @@
         "air_temperature:2.0:mean:PT1M": {
             "type": "Parameter",
             "description": {
-                "en": "air_temperature at 2.0m, aggregated over PT1M with method 'mean'"
+                "en": "Air temperature at 2.0m, aggregated over PT1M with method 'mean'"
             },
             "observedProperty": {
                 "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature",
                 "label": {
-                    "en": "air_temperature:2.0:mean:PT1M"
+                    "en": "Air temperature"
                 }
             },
             "unit": {

diff --git a/api/test/test_data/test_feature_collection.json b/api/test/test_data/test_feature_collection.json
@@ -44,12 +44,12 @@
         "air_pressure_at_sea_level:1.0:mean:PT1M": {
             "type": "Parameter",
             "description": {
-                "en": "air_pressure_at_sea_level at 1.0m, aggregated over PT1M with method 'mean'"
+                "en": "Air pressure at sea level at 1.0m, aggregated over PT1M with method 'mean'"
             },
             "observedProperty": {
                 "id": "https://vocab.nerc.ac.uk/standard_name/air_pressure_at_sea_level",
                 "label": {
-                    "en": "air_pressure_at_sea_level:1.0:mean:PT1M"
+                    "en": "Air pressure at sea level"
                 }
             },
             "unit": {
@@ -67,12 +67,12 @@
         "air_temperature:0.1:minimum:PT10M": {
             "type": "Parameter",
             "description": {
-                "en": "air_temperature at 0.1m, aggregated over PT10M with method 'minimum'"
+                "en": "Air temperature at 0.1m, aggregated over PT10M with method 'minimum'"
             },
             "observedProperty": {
                 "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature",
                 "label": {
-                    "en": "air_temperature:0.1:minimum:PT10M"
+                    "en": "Air temperature"
                 }
             },
             "unit": {

diff --git a/api/test/test_data/test_multiple_covjson.json b/api/test/test_data/test_multiple_covjson.json
@@ -48,12 +48,12 @@
         "relative_humidity:2.0:mean:PT1M": {
             "type": "Parameter",
             "description": {
-                "en": "relative_humidity at 2.0m, aggregated over PT1M with method 'mean'"
+                "en": "Relative humidity at 2.0m, aggregated over PT1M with method 'mean'"
             },
             "observedProperty": {
                 "id": "https://vocab.nerc.ac.uk/standard_name/relative_humidity",
                 "label": {
-                    "en": "relative_humidity:2.0:mean:PT1M"
+                    "en": "Relative humidity"
                 }
             },
             "unit": {
@@ -71,12 +71,12 @@
         "wind_from_direction:2.0:mean:PT10M": {
             "type": "Parameter",
             "description": {
-                "en": "wind_from_direction at 2.0m, aggregated over PT10M with method 'mean'"
+                "en": "Wind from direction at 2.0m, aggregated over PT10M with method 'mean'"
             },
             "observedProperty": {
                 "id": "https://vocab.nerc.ac.uk/standard_name/wind_from_direction",
                 "label": {
-                    "en": "wind_from_direction:2.0:mean:PT10M"
+                    "en": "Wind from direction"
                 }
             },
             "unit": {
@@ -94,12 +94,12 @@
         "wind_speed:10.0:mean:PT10M": {
             "type": "Parameter",
             "description": {
-                "en": "wind_speed at 10.0m, aggregated over PT10M with method 'mean'"
+                "en": "Wind speed at 10.0m, aggregated over PT10M with method 'mean'"
             },
             "observedProperty": {
                 "id": "https://vocab.nerc.ac.uk/standard_name/wind_speed",
                 "label": {
-                    "en": "wind_speed:10.0:mean:PT10M"
+                    "en": "Wind speed"
                 }
             },
             "unit": {

diff --git a/api/test/test_data/test_single_covjson.json b/api/test/test_data/test_single_covjson.json
@@ -52,12 +52,12 @@
         "wind_speed:10.0:mean:PT10M": {
             "type": "Parameter",
             "description": {
-                "en": "wind_speed at 10.0m, aggregated over PT10M with method 'mean'"
+                "en": "Wind speed at 10.0m, aggregated over PT10M with method 'mean'"
             },
             "observedProperty": {
                 "id": "https://vocab.nerc.ac.uk/standard_name/wind_speed",
                 "label": {
-                    "en": "wind_speed:10.0:mean:PT10M"
+                    "en": "Wind speed"
                 }
             },
             "unit": {

diff --git a/datastore/data-loader/Dockerfile b/datastore/data-loader/Dockerfile
@@ -2,6 +2,7 @@ FROM python:3.11-slim-bookworm
 
 SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"]
 
+ARG THROUGH_INGEST=false
 ENV DOCKER_PATH="/clients/python"
 
 COPY "test-data/KNMI/20221231.nc" "${DOCKER_PATH}/test-data/KNMI/20221231.nc"
@@ -29,7 +30,17 @@ RUN python -m grpc_tools.protoc  \
     --grpc_python_out="${DOCKER_PATH}"
 
 COPY "./parameters.py" "${DOCKER_PATH}/parameters.py"
+COPY "./utilities.py" "${DOCKER_PATH}/utilities.py"
+
+# Copy both loaders and conditionally use the wanted one
+COPY "./client_knmi_station_ingest.py" "${DOCKER_PATH}/client_knmi_station_ingest.py"
 COPY "./client_knmi_station.py" "${DOCKER_PATH}/client_knmi_station.py"
 
+RUN if [ "${THROUGH_INGEST}" = "true" ]; then \
+      mv "${DOCKER_PATH}/client_knmi_station_ingest.py" "${DOCKER_PATH}/client_knmi_station.py"; \
+    else \
+      rm "${DOCKER_PATH}/client_knmi_station_ingest.py"; \
+    fi
+
 WORKDIR "${DOCKER_PATH}"
 CMD ["python", "-u", "./client_knmi_station.py"]
File	Stmts	Miss	Cover	Missing
__init__.py	0	0	100%
datastore_pb2.py	61	48	21%	34–81
datastore_pb2_grpc.py	54	27	50%	15–16, 19, 65–80, 121–123, 128–130, 135–137, 142–144, 148–173, 219, 246, 273, 300
export_metrics.py	10	0	100%
grpc_getter.py	20	11	45%	15–19, 23–26, 30–32, 36–38
locustfile.py	15	15	0%	1–31
main.py	43	7	84%	45, 50, 60, 70–71, 81–82
metadata_endpoints.py	66	32	52%	45–54, 58, 85, 100–220, 224
response_classes.py	5	0	100%
utilities.py	174	46	74%	20, 38, 45, 67–70, 78–89, 94–101, 121, 125, 127, 155, 161, 179, 193–194, 198, 214–218, 222–228, 232–234, 264, 268, 290, 295
custom_geo_json
edr_feature_collection.py	6	0	100%
formatters
__init__.py	11	0	100%
covjson.py	60	1	98%	91
geojson.py	21	2	90%	27, 52
openapi
custom_dimension_examples.py	4	0	100%
edr_query_parameter_descriptions.py	11	0	100%
openapi_examples.py	13	0	100%
routers
__init__.py	0	0	100%
edr.py	101	4	96%	348–349, 438–439
feature.py	47	19	60%	99–132, 148–153, 159–181
TOTAL	722	212	71%
Title	Coverage	Tests	Skipped	Failures	Errors	Time
API Unit Tests		30	0 💤	0 ❌	0 🔥	1.815s ⏱️
Ingest Unit Tests		16	0 💤	0 ❌	0 🔥	1m 44s ⏱️