Merge branch 'main' into add_remaining_acdd_attrs_issue_54

.github/workflows/ci.yml

@@ -6,6 +6,9 @@ on:
       - "**"
     tags:
       - "[0-9]+.[0-9]+.[0-9]+"
+  pull_request:
+    branches:
+      - main
 
 permissions:
   contents: read
@@ -58,55 +61,126 @@ jobs:
       - name: Test client runs without errors
         run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose run --rm client
 
-      - name: Run load test
+      - name: Run load test (read only)
         run: |
           python --version
           pip install -r datastore/load-test/requirements.txt
           python -m grpc_tools.protoc --proto_path=./protobuf datastore.proto --python_out=datastore/load-test --grpc_python_out=datastore/load-test
           cd datastore/load-test
-          locust --headless -u 5 -r 1 --run-time 60 --only-summary --csv store
+          locust -f locustfile_read.py --headless -u 5 -r 10 --run-time 60 --only-summary --csv store_read
+
+      - name: Run load test (write + read)
+        run: |
+          pip install -r datastore/load-test/requirements.txt
+          python -m grpc_tools.protoc --proto_path=./protobuf datastore.proto --python_out=datastore/load-test --grpc_python_out=datastore/load-test
+          cd datastore/load-test
+          python schedule_write.py > schedule_write.log 2>&1 &
+          locust -f locustfile_read.py --headless -u 5 -r 10 --run-time 60 --only-summary --csv store_rw
+          kill %1
+          echo Catting schedule_write output...
+          cat schedule_write.log
+          echo Done catting
 
       - name: Archive load test artifacts
         uses: actions/upload-artifact@v3
         with:
           name: performance
-          path: datastore/load-test/store_*.csv
+          path: |
+            datastore/load-test/store_read_*.csv
+            datastore/load-test/store_rw_*.csv
 
       - name: Print results
         run: |
           pip install csvkit
-          echo "## Stats" >> $GITHUB_STEP_SUMMARY
-          csvlook datastore/load-test/store_stats.csv >> $GITHUB_STEP_SUMMARY
-          echo "## Stats history" >> $GITHUB_STEP_SUMMARY
-          csvlook datastore/load-test/store_stats_history.csv >> $GITHUB_STEP_SUMMARY
-          echo "## Failures" >> $GITHUB_STEP_SUMMARY
-          csvlook datastore/load-test/store_failures.csv >> $GITHUB_STEP_SUMMARY
+          echo "## Stats (READ ONLY)" >> $GITHUB_STEP_SUMMARY
+          csvlook datastore/load-test/store_read_stats.csv >> $GITHUB_STEP_SUMMARY
+          echo "## Failures (READ ONLY)" >> $GITHUB_STEP_SUMMARY
+          csvlook datastore/load-test/store_read_failures.csv >> $GITHUB_STEP_SUMMARY
+          echo "## Stats (WRITE + READ)" >> $GITHUB_STEP_SUMMARY
+          csvlook datastore/load-test/store_rw_stats.csv >> $GITHUB_STEP_SUMMARY
+          echo "## Failures (WRITE + READ)" >> $GITHUB_STEP_SUMMARY
+          csvlook datastore/load-test/store_rw_failures.csv >> $GITHUB_STEP_SUMMARY
 
       - name: Cleanup
         if: always()
         run: docker compose down --volumes
-  test-ingest:
+
+  test-api:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      issues: write
+      pull-requests: write
+
     strategy:
       matrix:
-        python-version: ["3.10"] # Add 3.11 back pybind11 bug is fixed
+        python-version: ["3.11"]
     steps:
       - name: Checkout the repo
-        uses: actions/checkout@v3
-      - name: Ubuntu setup
-        run: sudo apt update && sudo apt install libeccodes-data	rapidjson-dev	pybind11-dev libssl-dev
+        uses: actions/checkout@v4
+
       - name: Python Setup
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: x64
-      - name: Checkout Source
-        uses: actions/checkout@v3
+
       - name: Install Dependencies
         run: |
           pip install --upgrade pip
           pip install pytest-timeout
           pip install pytest-cov
-          pip install ./ingest
+          pip install -r ./api/requirements.txt
+
+      - name: Copy Protobuf file to api directory and build
+        run: |
+          mkdir ./api/protobuf
+          cp ./protobuf/datastore.proto ./api/protobuf/datastore.proto
+          python -m grpc_tools.protoc --proto_path=./api/protobuf --python_out=./api --grpc_python_out=./api ./api/protobuf/datastore.proto
+
       - name: Run Tests
-        run: python -m pytest -v --timeout=60 ./ingest
+        run: |
+          cd api
+          python -m pytest --timeout=60 --junitxml=pytest.xml --cov-report=term-missing --cov=. | tee pytest-coverage.txt
+
+      - name: Comment coverage
+        uses: MishaKav/pytest-coverage-comment@main
+        with:
+          pytest-coverage-path: ./api/pytest-coverage.txt
+          coverage-path-prefix: api/
+          title: API Unit Test Coverage Report
+          hide-badge: true
+          hide-report: false
+          create-new-comment: false
+          hide-comment: false
+          report-only-changed-files: false
+          remove-link-from-badge: false
+          junitxml-path: ./api/pytest.xml
+          junitxml-title: API Unit Test Coverage Summary
+
+
+# TODO: These tests don't currently work. Uncomment once this is resolved.
+#  test-ingest:
+#    runs-on: ubuntu-latest
+#    strategy:
+#      matrix:
+#        python-version: ["3.10"] # Add 3.11 back pybind11 bug is fixed
+#    steps:
+#      - name: Checkout the repo
+#        uses: actions/checkout@v3
+#      - name: Ubuntu setup
+#        run: sudo apt update && sudo apt install libeccodes-data	rapidjson-dev	pybind11-dev libssl-dev
+#      - name: Python Setup
+#        uses: actions/setup-python@v4
+#        with:
+#          python-version: ${{ matrix.python-version }}
+#          architecture: x64
+#      - name: Checkout Source
+#        uses: actions/checkout@v3
+#      - name: Install Dependencies
+#        run: |
+#          pip install --upgrade pip
+#          pip install pytest-timeout
+#          pip install pytest-cov
+#          pip install ./ingest
+#      - name: Run Tests
+#        run: python -m pytest -v --timeout=60 ./ingest

api/test/test_covjson.py

@@ -0,0 +1,113 @@
+import json
+
+import datastore_pb2 as dstore
+import pytest
+from covjson_pydantic.coverage import Coverage
+from covjson_pydantic.coverage import CoverageCollection
+from fastapi import HTTPException
+from formatters.covjson import Covjson
+from google.protobuf.json_format import Parse
+
+
+def test_single_parameter_convert():
+    test_data = load_json("test/test_data/test_single_proto.json")
+    compare_data = load_json("test/test_data/test_single_covjson.json")
+
+    response = create_mock_obs_response(test_data)
+    coverage_collection = Covjson().convert(response)
+
+    assert coverage_collection is not None
+
+    assert type(coverage_collection) is Coverage
+
+    # Assert that the coverage collection has the correct parameter
+    # TODO: Change parameter name when parameter names have been decided
+    assert "ff" in coverage_collection.parameters.keys()
+
+    # Check that correct values exist in the coverage collection
+    assert 9.21 in coverage_collection.ranges["ff"].values
+
+    assert len(coverage_collection.domain.axes.t.values) == 7
+
+    # Number of time points should match with the number of observation values
+    assert len(coverage_collection.domain.axes.t.values) == len(coverage_collection.ranges["ff"].values)
+
+    # compare the coverage collection with the compare data
+    # TODO: Modify compare data when parameter names have been decided
+    coverage_collection_json = json.loads(coverage_collection.model_dump_json(exclude_none=True))
+    assert coverage_collection_json == compare_data
+
+
+def test_multiple_parameter_convert():
+    test_data = load_json("test/test_data/test_multiple_proto.json")
+    compare_data = load_json("test/test_data/test_multiple_covjson.json")
+
+    response = create_mock_obs_response(test_data)
+
+    coverage_collection = Covjson().convert(response)
+
+    assert coverage_collection is not None
+
+    assert type(coverage_collection) is Coverage
+
+    # Check that the coverage collection has the correct parameters
+    # TODO: Change parameter names when parameter names have been decided
+    assert set(["dd", "ff", "rh"]) == coverage_collection.parameters.keys()
+
+    # Check that correct values exist in the coverage collection
+    assert 230.7 in coverage_collection.ranges["dd"].values
+    assert 9.19 in coverage_collection.ranges["ff"].values
+    assert 88.0 in coverage_collection.ranges["rh"].values
+
+    # TODO: Modify compare data when parameter names have been decided
+    coverage_collection_json = json.loads(coverage_collection.model_dump_json(exclude_none=True))
+    assert coverage_collection_json == compare_data
+
+
+def test_single_parameter_area_convert():
+    test_data = load_json("test/test_data/test_coverages_proto.json")
+    compare_data = load_json("test/test_data/test_coverages_covjson.json")
+
+    response = create_mock_obs_response(test_data)
+
+    coverage_collection = Covjson().convert(response)
+
+    assert coverage_collection is not None
+
+    assert type(coverage_collection) is CoverageCollection
+
+    assert len(coverage_collection.coverages) > 1
+
+    assert all([type(coverage) is Coverage for coverage in coverage_collection.coverages])
+
+    # Check that each coverage has the correct parameter
+    # TODO: Change parameter name when parameter names have been decided
+    assert all(["TA_P1D_AVG" in coverage.parameters.keys() for coverage in coverage_collection.coverages])
+
+    # TODO: Modify compare data when parameter names have been decided
+    coverage_collection_json = json.loads(coverage_collection.model_dump_json(exclude_none=True))
+    assert coverage_collection_json == compare_data
+
+
+def test_empty_response_convert():
+    test_data = load_json("test/test_data/test_empty_proto.json")
+    response = create_mock_obs_response(test_data)
+
+    # Expect to get an HTTPException with status code of 404 and detail of
+    # "No data found" when converting an empty response
+    with pytest.raises(HTTPException) as exception_info:
+        Covjson().convert(response)
+
+    assert exception_info.value.detail == "No data found"
+    assert exception_info.value.status_code == 404
+
+
+def create_mock_obs_response(json_data):
+    response = dstore.GetObsResponse()
+    Parse(json.dumps(json_data), response)
+    return response
+
+
+def load_json(file_path):
+    with open(file_path, "r") as file:
+        return json.load(file)