Merge pull request #14 from EURODEO/ci

Combine the CI steps and pre-commit hook of the different repositories.

.github/workflows/ci.yml

@@ -3,9 +3,9 @@ name: CI
 on:
   push:
     branches:
-      - '**'
+      - "**"
     tags:
-      - '[0-9]+.[0-9]+.[0-9]+'
+      - "[0-9]+.[0-9]+.[0-9]+"
 
 permissions:
   contents: read
@@ -14,52 +14,55 @@ jobs:
   pre-commit-hook:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v3
-    - name: Install pre-commit
-      run: python -m pip install pre-commit
-      shell: bash
-    - name: Show pre-commit requirements
-      run: python -m pip freeze --local
-      shell: bash
-    - uses: actions/cache@v3
-      with:
-        path: ~/.cache/pre-commit
-        key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
-    - name: Run pre-commit for all files
-      run: pre-commit run --config './.pre-commit-config.yaml' --all-files --color=always --show-diff-on-failure
-      shell: bash
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+      - name: Install pre-commit
+        run: python -m pip install pre-commit
+        shell: bash
+      - name: Show pre-commit requirements
+        run: python -m pip freeze --local
+        shell: bash
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
+      - name: Run pre-commit for all files
+        run: pre-commit run --config './.pre-commit-config.yaml' --all-files --color=always --show-diff-on-failure
+        shell: bash
 
-  test:
+  test-datastore:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
 
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: "3.11"
+
+      - name: Copy Protobuf files to Dockerfile directories
+        run: ./ci/docker/copy-protobuf.sh
 
       - name: Build docker containers
-        run: docker compose --project-directory datastore --profile test build
+        run: docker compose --profile test build
 
       - name: Start docker stack
-        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose --project-directory datastore up -d
+        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose up -d
 
       - name: Test loader runs without errors
-        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose --project-directory datastore run --rm loader
+        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose run --rm loader
 
       - name: Integration test
-        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose --project-directory datastore run --rm integration
+        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose run --rm integration
 
       - name: Test client runs without errors
-        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose --project-directory datastore run --rm client
+        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose run --rm client
 
       - name: Run load test
         run: |
           python --version
           pip install -r datastore/load-test/requirements.txt
-          python -m grpc_tools.protoc --proto_path=datastore/datastore/protobuf datastore.proto --python_out=datastore/load-test --grpc_python_out=datastore/load-test
+          python -m grpc_tools.protoc --proto_path=./protobuf datastore.proto --python_out=datastore/load-test --grpc_python_out=datastore/load-test
           cd datastore/load-test
           locust --headless -u 5 -r 1 --run-time 60 --only-summary --csv store
 
@@ -81,4 +84,29 @@ jobs:
 
       - name: Cleanup
         if: always()
-        run: docker compose --project-directory datastore down --volumes
+        run: docker compose down --volumes
+  test-ingest:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"] # Add 3.11 back pybind11 bug is fixed
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v3
+      - name: Ubuntu setup
+        run: sudo apt update && sudo apt install libeccodes-data	rapidjson-dev	pybind11-dev libssl-dev
+      - name: Python Setup
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - name: Checkout Source
+        uses: actions/checkout@v3
+      - name: Install Dependencies
+        run: |
+          pip install --upgrade pip
+          pip install pytest-timeout
+          pip install pytest-cov
+          pip install ./ingest
+      - name: Run Tests
+        run: python -m pytest -v --timeout=60 ./ingest

.gitignore

@@ -198,3 +198,9 @@ __pycache__/
 venv/
 
 .env
+
+# Ignore generated local protobuf files.
+datastore_pb2.py
+datastore_pb2_grpc.py
+# Ignore all the copied protobuf directories, as the root contains the source of truth.
+**/protobuf

.pre-commit-config.yaml

@@ -5,16 +5,24 @@ repos:
       - id: go-fmt
         name: go-fmt
         description: "Enforces Go standard formatting (whitespace, indentation, et cetera)."
-        entry: ./datastore/ci/go/go-fmt.sh
+        entry: ./ci/go/go-fmt.sh
         language: script
         pass_filenames: false
-#      # go-vet ~ Finds subtle issues in Go where your code may not work as intended
-#      - id: go-vet
-#        name: go-vet
-#        description: "Finds subtle issues in Go where your code may not work as intended."
-#        entry: ./ci/go/go-vet.sh
-#        language: script
-#        pass_filenames: false
+  #      # go-vet ~ Finds subtle issues in Go where your code may not work as intended
+  #      - id: go-vet
+  #        name: go-vet
+  #        description: "Finds subtle issues in Go where your code may not work as intended."
+  #        entry: ./ci/go/go-vet.sh
+  #        language: script
+  #        pass_filenames: false
+
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: ""
+    hooks:
+      - id: clang-format
+        entry: clang-format -i
+        args: [--style=llvm]
+        types_or: [c++]
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
@@ -23,9 +31,11 @@ repos:
       - id: end-of-file-fixer # Makes sure files end in a newline and only a newline.
       - id: pretty-format-json
         args: ["--autofix", "--indent=4", "--no-ensure-ascii", "--no-sort-keys"] # Formats and sorts your JSON files.
+        exclude: "ingest\/test\/test_data\/bufr\/.*"
       - id: trailing-whitespace # Trims trailing whitespace.
       # Checks
       - id: check-json # Attempts to load all json files to verify syntax.
+        exclude: "ingest\/test\/test_data\/bufr\/.*"
       - id: check-merge-conflict # Check for files that contain merge conflict strings.
       - id: check-shebang-scripts-are-executable # Checks that scripts with shebangs are executable.
       - id: check-yaml
@@ -41,21 +51,17 @@ repos:
 
   # black ~ Formats Python code
   - repo: https://github.com/psf/black
-    rev: 23.10.0
+    rev: 23.12.1
     hooks:
       - id: black
         args: ["--line-length=120"]
 
   # flake8 ~ Enforces the Python PEP8 style guide
   - repo: https://github.com/pycqa/flake8
-    rev: 6.1.0
+    rev: 7.0.0
     hooks:
       - id: flake8
-        args:
-          [
-            "--ignore=W503",
-            "--max-line-length=120",
-          ]
+        args: ["--ignore=W503", "--max-line-length=120"]
 
   # hadolint ~ Docker linter
   - repo: https://github.com/hadolint/hadolint

README.md

@@ -16,4 +16,20 @@ The E-SOH service consist of three parts.
       - The records api will serve all metadata for each timeseries available in E-SOH.
 
 ## Usage
-All three parts of the service is kept in this repository and can be built and setup using docker-compose. For more documentation on each part of the service see README.md in each relevant folder in this repository.
+All three parts of the service is kept in this repository and can be built and setup using docker-compose. For more documentation on each part of the service see README.md in each relevant folder in this repository.
+
+## Pre-commit
+
+### Setup
+
+1. Go to the root of the repository.
+2. Install the python pre-commit package with `pip install pre-commit`.
+3. Reinitialize the repository with `git init`.
+4. Install the hooks defined in `.pre-commit-config.yaml` with `pre-commit install`.
+
+### Useful Commands
+
+- To update the pre-commit hooks in `.pre-commit-config.yaml`, run `pre-commit autoupdate`
+- To apply the pre-commit for every file in the repository, run `pre-commit run --config './.pre-commit-config.yaml' --all-files`
+- To see all options to `pre-commit run`, run `pre-commit help run` (in particular, the `--files` option can be used to apply the command to selected files only).
+- To commit without the pre-commit hook, run `git commit -m "Some message" --no-verify`

api/Dockerfile

@@ -2,8 +2,6 @@ FROM python:3.11-slim-bookworm
 
 SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"]
 
-ENV PROJECT_DATASTORE_PATH="datastore"
-ENV PROJECT_PYTHON_PATH="api"
 ENV DOCKER_PATH="/app"
 
 RUN apt-get update \
@@ -15,8 +13,8 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-COPY "${PROJECT_DATASTORE_PATH}/protobuf/datastore.proto" "/protobuf/datastore.proto"
-COPY "${PROJECT_PYTHON_PATH}/requirements.txt" "${DOCKER_PATH}/requirements.txt"
+COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto"
+COPY "./requirements.txt" "${DOCKER_PATH}/requirements.txt"
 
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --upgrade pip \
@@ -28,8 +26,8 @@ RUN python -m grpc_tools.protoc  \
     --python_out="${DOCKER_PATH}"  \
     --grpc_python_out="${DOCKER_PATH}"
 
-COPY "${PROJECT_PYTHON_PATH}/main.py" "${DOCKER_PATH}/main.py"
-COPY "${PROJECT_PYTHON_PATH}/metadata_endpoints.py" "${DOCKER_PATH}/metadata_endpoints.py"
+COPY "./main.py" "${DOCKER_PATH}/main.py"
+COPY "./metadata_endpoints.py" "${DOCKER_PATH}/metadata_endpoints.py"
 
 WORKDIR "${DOCKER_PATH}"
 CMD ["gunicorn", "main:app", "--workers=4", "--worker-class=uvicorn.workers.UvicornWorker", "--bind=0.0.0.0:8000"]

ci/docker/copy-protobuf.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+declare -a destination_paths=(
+  "./api"
+  "./datastore/data-loader"
+  "./datastore/datastore"
+  "./datastore/examples/clients/python"
+  "./datastore/integration-test"
+#  "./ingest"
+)
+
+for path in "${destination_paths[@]}"
+do
+  cp --verbose -r "./protobuf" "$path"
+done

datastore/data-loader/Dockerfile

@@ -2,11 +2,9 @@ FROM python:3.11-slim-bookworm
 
 SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"]
 
-ENV PROJECT_DATASTORE_PATH="datastore"
-ENV PROJECT_PYTHON_PATH="data-loader"
 ENV DOCKER_PATH="/clients/python"
 
-COPY "test-data/KNMI/20221231.nc" "/test-data/KNMI/20221231.nc"
+COPY "test-data/KNMI/20221231.nc" "${DOCKER_PATH}/test-data/KNMI/20221231.nc"
 
 RUN apt-get update \
     && apt-get -y upgrade \
@@ -17,8 +15,8 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-COPY "${PROJECT_DATASTORE_PATH}/protobuf/datastore.proto" "/protobuf/datastore.proto"
-COPY "${PROJECT_PYTHON_PATH}/requirements.txt" "${DOCKER_PATH}/requirements.txt"
+COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto"
+COPY "./requirements.txt" "${DOCKER_PATH}/requirements.txt"
 
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --upgrade pip \
@@ -30,8 +28,8 @@ RUN python -m grpc_tools.protoc  \
     --python_out="${DOCKER_PATH}"  \
     --grpc_python_out="${DOCKER_PATH}"
 
-COPY "${PROJECT_PYTHON_PATH}/parameters.py" "${DOCKER_PATH}/parameters.py"
-COPY "${PROJECT_PYTHON_PATH}/client_knmi_station.py" "${DOCKER_PATH}/client_knmi_station.py"
+COPY "./parameters.py" "${DOCKER_PATH}/parameters.py"
+COPY "./client_knmi_station.py" "${DOCKER_PATH}/client_knmi_station.py"
 
 WORKDIR "${DOCKER_PATH}"
 CMD ["python", "-u", "./client_knmi_station.py"]

datastore/data-loader/client_fmi_station.py

@@ -76,7 +76,7 @@ def insert_data(observation_request_messages: List):
 
     print("Starting with creating the time series and observations requests.")
     create_requests_start = perf_counter()
-    file_path = Path(Path(__file__).parents[2] / "test-data" / "FMI" / "20221231.csv")
+    file_path = Path(Path(__file__).parent / "test-data" / "FMI" / "20221231.csv")
     print(file_path)
     observation_request_messages = csv_file_to_requests(file_path=file_path)
     print("Finished creating the time series and observation requests " f"{perf_counter() - create_requests_start}.")

datastore/data-loader/client_knmi_station.py

@@ -80,7 +80,7 @@ def insert_data(observation_request_messages: List):
 
     print("Starting with creating the time series and observations requests.")
     create_requests_start = perf_counter()
-    file_path = Path(Path(__file__).parents[2] / "test-data" / "KNMI" / "20221231.nc")
+    file_path = Path(Path(__file__).parent / "test-data" / "KNMI" / "20221231.nc")
     observation_request_messages = netcdf_file_to_requests(file_path=file_path)
     print("Finished creating the time series and observation requests " f"{perf_counter() - create_requests_start}.")