CI: Fix the datastore Github Actions before merging them with the ing…

…estion.

.github/workflows/ci.yml

@@ -40,26 +40,29 @@ jobs:
         with:
           python-version: '3.11'
 
+      - name: Copy Protobuf files to Dockerfile directories
+        run: ./ci/docker/copy-protobuf.sh
+
       - name: Build docker containers
-        run: docker compose --project-directory datastore --profile test build
+        run: docker compose --profile test build
 
       - name: Start docker stack
-        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose --project-directory datastore up -d
+        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose up -d
 
       - name: Test loader runs without errors
-        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose --project-directory datastore run --rm loader
+        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose run --rm loader
 
       - name: Integration test
-        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose --project-directory datastore run --rm integration
+        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose run --rm integration
 
       - name: Test client runs without errors
-        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose --project-directory datastore run --rm client
+        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose run --rm client
 
       - name: Run load test
         run: |
           python --version
           pip install -r datastore/load-test/requirements.txt
-          python -m grpc_tools.protoc --proto_path=datastore/datastore/protobuf datastore.proto --python_out=datastore/load-test --grpc_python_out=datastore/load-test
+          python -m grpc_tools.protoc --proto_path=./protobuf datastore.proto --python_out=datastore/load-test --grpc_python_out=datastore/load-test
           cd datastore/load-test
           locust --headless -u 5 -r 1 --run-time 60 --only-summary --csv store
 
@@ -81,4 +84,4 @@ jobs:
 
       - name: Cleanup
         if: always()
-        run: docker compose --project-directory datastore down --volumes
+        run: docker compose down --volumes

.gitignore

@@ -198,3 +198,9 @@ __pycache__/
 venv/
 
 .env
+
+# Ignore generated local protobuf files.
+datastore_pb2.py
+datastore_pb2_grpc.py
+# Ignore all the copied protobuf directories, as the root contains the source of truth.
+**/protobuf

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ repos:
       - id: go-fmt
         name: go-fmt
         description: "Enforces Go standard formatting (whitespace, indentation, et cetera)."
-        entry: ./datastore/ci/go/go-fmt.sh
+        entry: ./ci/go/go-fmt.sh
         language: script
         pass_filenames: false
 #      # go-vet ~ Finds subtle issues in Go where your code may not work as intended

api/Dockerfile

@@ -2,8 +2,6 @@ FROM python:3.11-slim-bookworm
 
 SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"]
 
-ENV PROJECT_DATASTORE_PATH="datastore"
-ENV PROJECT_PYTHON_PATH="api"
 ENV DOCKER_PATH="/app"
 
 RUN apt-get update \
@@ -15,8 +13,8 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-COPY "${PROJECT_DATASTORE_PATH}/protobuf/datastore.proto" "/protobuf/datastore.proto"
-COPY "${PROJECT_PYTHON_PATH}/requirements.txt" "${DOCKER_PATH}/requirements.txt"
+COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto"
+COPY "./requirements.txt" "${DOCKER_PATH}/requirements.txt"
 
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --upgrade pip \
@@ -28,8 +26,8 @@ RUN python -m grpc_tools.protoc  \
     --python_out="${DOCKER_PATH}"  \
     --grpc_python_out="${DOCKER_PATH}"
 
-COPY "${PROJECT_PYTHON_PATH}/main.py" "${DOCKER_PATH}/main.py"
-COPY "${PROJECT_PYTHON_PATH}/metadata_endpoints.py" "${DOCKER_PATH}/metadata_endpoints.py"
+COPY "./main.py" "${DOCKER_PATH}/main.py"
+COPY "./metadata_endpoints.py" "${DOCKER_PATH}/metadata_endpoints.py"
 
 WORKDIR "${DOCKER_PATH}"
 CMD ["gunicorn", "main:app", "--workers=4", "--worker-class=uvicorn.workers.UvicornWorker", "--bind=0.0.0.0:8000"]

ci/docker/copy-protobuf.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+declare -a destination_paths=(
+  "./api"
+  "./datastore/data-loader"
+  "./datastore/datastore"
+  "./datastore/examples/clients/python"
+  "./datastore/integration-test"
+  "./ingest"
+)
+
+for path in "${destination_paths[@]}"
+do
+  cp --verbose -r "./protobuf" "$path"
+done

datastore/data-loader/Dockerfile

@@ -2,11 +2,9 @@ FROM python:3.11-slim-bookworm
 
 SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"]
 
-ENV PROJECT_DATASTORE_PATH="datastore"
-ENV PROJECT_PYTHON_PATH="data-loader"
 ENV DOCKER_PATH="/clients/python"
 
-COPY "test-data/KNMI/20221231.nc" "/test-data/KNMI/20221231.nc"
+COPY "test-data/KNMI/20221231.nc" "${DOCKER_PATH}/test-data/KNMI/20221231.nc"
 
 RUN apt-get update \
     && apt-get -y upgrade \
@@ -17,8 +15,8 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-COPY "${PROJECT_DATASTORE_PATH}/protobuf/datastore.proto" "/protobuf/datastore.proto"
-COPY "${PROJECT_PYTHON_PATH}/requirements.txt" "${DOCKER_PATH}/requirements.txt"
+COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto"
+COPY "./requirements.txt" "${DOCKER_PATH}/requirements.txt"
 
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --upgrade pip \
@@ -30,8 +28,8 @@ RUN python -m grpc_tools.protoc  \
     --python_out="${DOCKER_PATH}"  \
     --grpc_python_out="${DOCKER_PATH}"
 
-COPY "${PROJECT_PYTHON_PATH}/parameters.py" "${DOCKER_PATH}/parameters.py"
-COPY "${PROJECT_PYTHON_PATH}/client_knmi_station.py" "${DOCKER_PATH}/client_knmi_station.py"
+COPY "./parameters.py" "${DOCKER_PATH}/parameters.py"
+COPY "./client_knmi_station.py" "${DOCKER_PATH}/client_knmi_station.py"
 
 WORKDIR "${DOCKER_PATH}"
 CMD ["python", "-u", "./client_knmi_station.py"]

datastore/data-loader/client_fmi_station.py

@@ -76,7 +76,7 @@ def insert_data(observation_request_messages: List):
 
     print("Starting with creating the time series and observations requests.")
     create_requests_start = perf_counter()
-    file_path = Path(Path(__file__).parents[2] / "test-data" / "FMI" / "20221231.csv")
+    file_path = Path(Path(__file__).parent / "test-data" / "FMI" / "20221231.csv")
     print(file_path)
     observation_request_messages = csv_file_to_requests(file_path=file_path)
     print("Finished creating the time series and observation requests " f"{perf_counter() - create_requests_start}.")

datastore/data-loader/client_knmi_station.py

@@ -80,7 +80,7 @@ def insert_data(observation_request_messages: List):
 
     print("Starting with creating the time series and observations requests.")
     create_requests_start = perf_counter()
-    file_path = Path(Path(__file__).parents[2] / "test-data" / "KNMI" / "20221231.nc")
+    file_path = Path(Path(__file__).parent / "test-data" / "KNMI" / "20221231.nc")
     observation_request_messages = netcdf_file_to_requests(file_path=file_path)
     print("Finished creating the time series and observation requests " f"{perf_counter() - create_requests_start}.")
 

datastore/datastore/Dockerfile

@@ -36,6 +36,9 @@ RUN apt-get update \
 COPY . /go-grpc-demo
 
 WORKDIR /go-grpc-demo
+
+#ENTRYPOINT ["tail", "-f", "/dev/null"]
+
 RUN \
     # compile proto file \
     protoc --go_out=. --go-grpc_out=. protobuf/datastore.proto \

datastore/examples/clients/python/Dockerfile

@@ -2,8 +2,6 @@ FROM python:3.11-slim-bookworm
 
 SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"]
 
-ENV PROJECT_DATASTORE_PATH="datastore"
-ENV PROJECT_PYTHON_PATH="examples/clients/python"
 ENV DOCKER_PATH="/clients/python"
 
 RUN apt-get update \
@@ -14,8 +12,8 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-COPY "${PROJECT_DATASTORE_PATH}/protobuf/datastore.proto" "/protobuf/datastore.proto"
-COPY "${PROJECT_PYTHON_PATH}/requirements.txt" "${DOCKER_PATH}/requirements.txt"
+COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto"
+COPY "./requirements.txt" "${DOCKER_PATH}/requirements.txt"
 
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --upgrade pip \
@@ -27,7 +25,7 @@ RUN python -m grpc_tools.protoc  \
     --python_out="${DOCKER_PATH}"  \
     --grpc_python_out="${DOCKER_PATH}"
 
-COPY "${PROJECT_PYTHON_PATH}/client.py" "${DOCKER_PATH}/client.py"
+COPY "./client.py" "${DOCKER_PATH}/client.py"
 
 WORKDIR "${DOCKER_PATH}"
 CMD ["python", "-u", "./client.py"]

datastore/integration-test/Dockerfile

@@ -2,8 +2,6 @@ FROM python:3.11-slim-bookworm
 
 SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"]
 
-ENV PROJECT_DATASTORE_PATH="datastore"
-ENV PROJECT_PYTHON_PATH="integration-test"
 ENV DOCKER_PATH="/clients/python"
 
 RUN apt-get update \
@@ -14,8 +12,8 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-COPY "${PROJECT_DATASTORE_PATH}/protobuf/datastore.proto" "/protobuf/datastore.proto"
-COPY "${PROJECT_PYTHON_PATH}/requirements.txt" "${DOCKER_PATH}/requirements.txt"
+COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto"
+COPY "./requirements.txt" "${DOCKER_PATH}/requirements.txt"
 
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --upgrade pip \
@@ -27,11 +25,11 @@ RUN python -m grpc_tools.protoc  \
     --python_out="${DOCKER_PATH}"  \
     --grpc_python_out="${DOCKER_PATH}"
 
-COPY "${PROJECT_PYTHON_PATH}/test_knmi.py" "${DOCKER_PATH}/test_knmi.py"
-COPY "${PROJECT_PYTHON_PATH}/test_delete.py" "${DOCKER_PATH}/test_delete.py"
-COPY "${PROJECT_PYTHON_PATH}/test_api.py" "${DOCKER_PATH}/test_api.py"
+COPY "./test_knmi.py" "${DOCKER_PATH}/test_knmi.py"
+COPY "./test_delete.py" "${DOCKER_PATH}/test_delete.py"
+COPY "./test_api.py" "${DOCKER_PATH}/test_api.py"
 
-COPY "${PROJECT_PYTHON_PATH}/response/" "${DOCKER_PATH}/response/"
+COPY "./response/" "${DOCKER_PATH}/response/"
 
 WORKDIR "${DOCKER_PATH}"
 CMD ["pytest"]

datastore/docker-compose.yml → docker-compose.yml

@@ -10,7 +10,7 @@ services:
     volumes:
 #      - ts-data:/home/postgres/pgdata/data # for timescale image
       - ts-data:/var/lib/postgresql/data # for postgres image
-      - ./database/healthcheck_postgis_uptime.sh:/healthcheck_postgis_uptime.sh # for the healthcheck
+      - ./datastore/database/healthcheck_postgis_uptime.sh:/healthcheck_postgis_uptime.sh # for the healthcheck
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=mysecretpassword
@@ -27,15 +27,15 @@ services:
   migrate:
     image: migrate/migrate:4
     volumes:
-      - ./migrate/data/migrations:/data/migrations
+      - ./datastore/migrate/data/migrations:/data/migrations
     command: ["-path", "/data/migrations", "-database", "postgres://postgres:mysecretpassword@db:5432/data?sslmode=disable", "up"]
     depends_on:
       db:
         condition: service_healthy
 
   store:
     build:
-      context: datastore
+      context: datastore/datastore
     ports:
       - "50050:50050"
       - "6060:6060" # for flame graphing
@@ -60,8 +60,7 @@ services:
 
   api:
     build:
-      context: .  # TODO: Needed to get proto file. Find a better solution
-      dockerfile: api/Dockerfile
+      context: api
     ports:
         - "8008:8000"
     environment:
@@ -74,8 +73,7 @@ services:
   client:
     profiles: ["test"]
     build:
-      context: .  # TODO: Needed to get proto file. Find a better solution
-      dockerfile: examples/clients/python/Dockerfile
+      context: datastore/examples/clients/python
     environment:
       - DSHOST=store
       - DSPORT=50050
@@ -86,20 +84,19 @@ services:
   loader:
     profiles: ["test"]
     build:
-      context: .  # TODO: Needed to get proto file and test data. Find a better solution
-      dockerfile: data-loader/Dockerfile
+      context: datastore/data-loader
     environment:
       - DSHOST=store
       - DSPORT=50050
+      - BASE_URL=http://api:8000
     depends_on:
       store:
         condition: service_healthy
 
   integration:
     profiles: ["test"]
     build:
-      context: .  # TODO: Needed to get proto file. Find a better solution
-      dockerfile: integration-test/Dockerfile
+      context: datastore/integration-test
     environment:
       - DSHOST=store
       - DSPORT=50050