From 6cd217831ca03c14918b6f8952796437022cd6e2 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Thu, 11 Jan 2024 17:21:53 +0100 Subject: [PATCH] CI: Fix the datastore Github Actions before merging them with the ingestion. --- .github/workflows/ci.yml | 17 +++++++++------- .gitignore | 6 ++++++ .pre-commit-config.yaml | 2 +- api/Dockerfile | 10 ++++----- ci/docker/copy-protobuf.sh | 15 ++++++++++++++ {datastore/ci => ci}/go/go-fmt.sh | 0 {datastore/ci => ci}/go/go-vet.sh | 0 datastore/data-loader/Dockerfile | 12 +++++------ datastore/data-loader/client_fmi_station.py | 2 +- datastore/data-loader/client_knmi_station.py | 2 +- .../test-data/FMI/fmi_test_data.tar.gz | Bin .../test-data/KNMI/20221231.nc | Bin .../test-data/KNMI/20230101.nc | Bin .../test-data/KNMI/20230102.nc | Bin datastore/datastore/Dockerfile | 3 +++ datastore/examples/clients/python/Dockerfile | 8 +++----- datastore/integration-test/Dockerfile | 14 ++++++------- .../docker-compose.yml => docker-compose.yml | 19 ++++++++---------- .../protobuf => protobuf}/datastore.proto | 0 19 files changed, 63 insertions(+), 47 deletions(-) create mode 100755 ci/docker/copy-protobuf.sh rename {datastore/ci => ci}/go/go-fmt.sh (100%) rename {datastore/ci => ci}/go/go-vet.sh (100%) rename datastore/{ => data-loader}/test-data/FMI/fmi_test_data.tar.gz (100%) rename datastore/{ => data-loader}/test-data/KNMI/20221231.nc (100%) rename datastore/{ => data-loader}/test-data/KNMI/20230101.nc (100%) rename datastore/{ => data-loader}/test-data/KNMI/20230102.nc (100%) rename datastore/docker-compose.yml => docker-compose.yml (79%) rename {datastore/datastore/protobuf => protobuf}/datastore.proto (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fd3b26b..7b663364 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,26 +40,29 @@ jobs: with: python-version: '3.11' + - name: Copy Protobuf files to Dockerfile directories + run: ./ci/docker/copy-protobuf.sh + - name: Build docker containers - run: docker compose --project-directory datastore --profile test build + run: docker compose --profile test build - name: Start docker stack - run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose --project-directory datastore up -d + run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose up -d - name: Test loader runs without errors - run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose --project-directory datastore run --rm loader + run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose run --rm loader - name: Integration test - run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose --project-directory datastore run --rm integration + run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose run --rm integration - name: Test client runs without errors - run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose --project-directory datastore run --rm client + run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose run --rm client - name: Run load test run: | python --version pip install -r datastore/load-test/requirements.txt - python -m grpc_tools.protoc --proto_path=datastore/datastore/protobuf datastore.proto --python_out=datastore/load-test --grpc_python_out=datastore/load-test + python -m grpc_tools.protoc --proto_path=./protobuf datastore.proto --python_out=datastore/load-test --grpc_python_out=datastore/load-test cd datastore/load-test locust --headless -u 5 -r 1 --run-time 60 --only-summary --csv store @@ -81,4 +84,4 @@ jobs: - name: Cleanup if: always() - run: docker compose --project-directory datastore down --volumes + run: docker compose down --volumes diff --git a/.gitignore b/.gitignore index 3c02fccf..a41b46fe 100644 --- a/.gitignore +++ b/.gitignore @@ -198,3 +198,9 @@ __pycache__/ venv/ .env + +# Ignore generated local protobuf files. +datastore_pb2.py +datastore_pb2_grpc.py +# Ignore all the copied protobuf directories, as the root contains the source of truth. +**/protobuf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3109f7e1..12d108bc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - id: go-fmt name: go-fmt description: "Enforces Go standard formatting (whitespace, indentation, et cetera)." - entry: ./datastore/ci/go/go-fmt.sh + entry: ./ci/go/go-fmt.sh language: script pass_filenames: false # # go-vet ~ Finds subtle issues in Go where your code may not work as intended diff --git a/api/Dockerfile b/api/Dockerfile index 8d6d6dc1..11443580 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -2,8 +2,6 @@ FROM python:3.11-slim-bookworm SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"] -ENV PROJECT_DATASTORE_PATH="datastore" -ENV PROJECT_PYTHON_PATH="api" ENV DOCKER_PATH="/app" RUN apt-get update \ @@ -15,8 +13,8 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY "${PROJECT_DATASTORE_PATH}/protobuf/datastore.proto" "/protobuf/datastore.proto" -COPY "${PROJECT_PYTHON_PATH}/requirements.txt" "${DOCKER_PATH}/requirements.txt" +COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto" +COPY "./requirements.txt" "${DOCKER_PATH}/requirements.txt" # hadolint ignore=DL3013 RUN pip install --no-cache-dir --upgrade pip \ @@ -28,8 +26,8 @@ RUN python -m grpc_tools.protoc \ --python_out="${DOCKER_PATH}" \ --grpc_python_out="${DOCKER_PATH}" -COPY "${PROJECT_PYTHON_PATH}/main.py" "${DOCKER_PATH}/main.py" -COPY "${PROJECT_PYTHON_PATH}/metadata_endpoints.py" "${DOCKER_PATH}/metadata_endpoints.py" +COPY "./main.py" "${DOCKER_PATH}/main.py" +COPY "./metadata_endpoints.py" "${DOCKER_PATH}/metadata_endpoints.py" WORKDIR "${DOCKER_PATH}" CMD ["gunicorn", "main:app", "--workers=4", "--worker-class=uvicorn.workers.UvicornWorker", "--bind=0.0.0.0:8000"] diff --git a/ci/docker/copy-protobuf.sh b/ci/docker/copy-protobuf.sh new file mode 100755 index 00000000..f3173578 --- /dev/null +++ b/ci/docker/copy-protobuf.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +declare -a destination_paths=( + "./api" + "./datastore/data-loader" + "./datastore/datastore" + "./datastore/examples/clients/python" + "./datastore/integration-test" + "./ingest" +) + +for path in "${destination_paths[@]}" +do + cp --verbose -r "./protobuf" "$path" +done diff --git a/datastore/ci/go/go-fmt.sh b/ci/go/go-fmt.sh similarity index 100% rename from datastore/ci/go/go-fmt.sh rename to ci/go/go-fmt.sh diff --git a/datastore/ci/go/go-vet.sh b/ci/go/go-vet.sh similarity index 100% rename from datastore/ci/go/go-vet.sh rename to ci/go/go-vet.sh diff --git a/datastore/data-loader/Dockerfile b/datastore/data-loader/Dockerfile index 812a6355..6be8f9b1 100644 --- a/datastore/data-loader/Dockerfile +++ b/datastore/data-loader/Dockerfile @@ -2,11 +2,9 @@ FROM python:3.11-slim-bookworm SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"] -ENV PROJECT_DATASTORE_PATH="datastore" -ENV PROJECT_PYTHON_PATH="data-loader" ENV DOCKER_PATH="/clients/python" -COPY "test-data/KNMI/20221231.nc" "/test-data/KNMI/20221231.nc" +COPY "test-data/KNMI/20221231.nc" "${DOCKER_PATH}/test-data/KNMI/20221231.nc" RUN apt-get update \ && apt-get -y upgrade \ @@ -17,8 +15,8 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY "${PROJECT_DATASTORE_PATH}/protobuf/datastore.proto" "/protobuf/datastore.proto" -COPY "${PROJECT_PYTHON_PATH}/requirements.txt" "${DOCKER_PATH}/requirements.txt" +COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto" +COPY "./requirements.txt" "${DOCKER_PATH}/requirements.txt" # hadolint ignore=DL3013 RUN pip install --no-cache-dir --upgrade pip \ @@ -30,8 +28,8 @@ RUN python -m grpc_tools.protoc \ --python_out="${DOCKER_PATH}" \ --grpc_python_out="${DOCKER_PATH}" -COPY "${PROJECT_PYTHON_PATH}/parameters.py" "${DOCKER_PATH}/parameters.py" -COPY "${PROJECT_PYTHON_PATH}/client_knmi_station.py" "${DOCKER_PATH}/client_knmi_station.py" +COPY "./parameters.py" "${DOCKER_PATH}/parameters.py" +COPY "./client_knmi_station.py" "${DOCKER_PATH}/client_knmi_station.py" WORKDIR "${DOCKER_PATH}" CMD ["python", "-u", "./client_knmi_station.py"] diff --git a/datastore/data-loader/client_fmi_station.py b/datastore/data-loader/client_fmi_station.py index 7bf92159..59b7fd43 100755 --- a/datastore/data-loader/client_fmi_station.py +++ b/datastore/data-loader/client_fmi_station.py @@ -76,7 +76,7 @@ def insert_data(observation_request_messages: List): print("Starting with creating the time series and observations requests.") create_requests_start = perf_counter() - file_path = Path(Path(__file__).parents[2] / "test-data" / "FMI" / "20221231.csv") + file_path = Path(Path(__file__).parent / "test-data" / "FMI" / "20221231.csv") print(file_path) observation_request_messages = csv_file_to_requests(file_path=file_path) print("Finished creating the time series and observation requests " f"{perf_counter() - create_requests_start}.") diff --git a/datastore/data-loader/client_knmi_station.py b/datastore/data-loader/client_knmi_station.py index ef266d6d..b8464294 100755 --- a/datastore/data-loader/client_knmi_station.py +++ b/datastore/data-loader/client_knmi_station.py @@ -80,7 +80,7 @@ def insert_data(observation_request_messages: List): print("Starting with creating the time series and observations requests.") create_requests_start = perf_counter() - file_path = Path(Path(__file__).parents[2] / "test-data" / "KNMI" / "20221231.nc") + file_path = Path(Path(__file__).parent / "test-data" / "KNMI" / "20221231.nc") observation_request_messages = netcdf_file_to_requests(file_path=file_path) print("Finished creating the time series and observation requests " f"{perf_counter() - create_requests_start}.") diff --git a/datastore/test-data/FMI/fmi_test_data.tar.gz b/datastore/data-loader/test-data/FMI/fmi_test_data.tar.gz similarity index 100% rename from datastore/test-data/FMI/fmi_test_data.tar.gz rename to datastore/data-loader/test-data/FMI/fmi_test_data.tar.gz diff --git a/datastore/test-data/KNMI/20221231.nc b/datastore/data-loader/test-data/KNMI/20221231.nc similarity index 100% rename from datastore/test-data/KNMI/20221231.nc rename to datastore/data-loader/test-data/KNMI/20221231.nc diff --git a/datastore/test-data/KNMI/20230101.nc b/datastore/data-loader/test-data/KNMI/20230101.nc similarity index 100% rename from datastore/test-data/KNMI/20230101.nc rename to datastore/data-loader/test-data/KNMI/20230101.nc diff --git a/datastore/test-data/KNMI/20230102.nc b/datastore/data-loader/test-data/KNMI/20230102.nc similarity index 100% rename from datastore/test-data/KNMI/20230102.nc rename to datastore/data-loader/test-data/KNMI/20230102.nc diff --git a/datastore/datastore/Dockerfile b/datastore/datastore/Dockerfile index b5e1e716..95a58bc3 100644 --- a/datastore/datastore/Dockerfile +++ b/datastore/datastore/Dockerfile @@ -36,6 +36,9 @@ RUN apt-get update \ COPY . /go-grpc-demo WORKDIR /go-grpc-demo + +#ENTRYPOINT ["tail", "-f", "/dev/null"] + RUN \ # compile proto file \ protoc --go_out=. --go-grpc_out=. protobuf/datastore.proto \ diff --git a/datastore/examples/clients/python/Dockerfile b/datastore/examples/clients/python/Dockerfile index 2d4b486b..a698768a 100644 --- a/datastore/examples/clients/python/Dockerfile +++ b/datastore/examples/clients/python/Dockerfile @@ -2,8 +2,6 @@ FROM python:3.11-slim-bookworm SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"] -ENV PROJECT_DATASTORE_PATH="datastore" -ENV PROJECT_PYTHON_PATH="examples/clients/python" ENV DOCKER_PATH="/clients/python" RUN apt-get update \ @@ -14,8 +12,8 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY "${PROJECT_DATASTORE_PATH}/protobuf/datastore.proto" "/protobuf/datastore.proto" -COPY "${PROJECT_PYTHON_PATH}/requirements.txt" "${DOCKER_PATH}/requirements.txt" +COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto" +COPY "./requirements.txt" "${DOCKER_PATH}/requirements.txt" # hadolint ignore=DL3013 RUN pip install --no-cache-dir --upgrade pip \ @@ -27,7 +25,7 @@ RUN python -m grpc_tools.protoc \ --python_out="${DOCKER_PATH}" \ --grpc_python_out="${DOCKER_PATH}" -COPY "${PROJECT_PYTHON_PATH}/client.py" "${DOCKER_PATH}/client.py" +COPY "./client.py" "${DOCKER_PATH}/client.py" WORKDIR "${DOCKER_PATH}" CMD ["python", "-u", "./client.py"] diff --git a/datastore/integration-test/Dockerfile b/datastore/integration-test/Dockerfile index eab301c6..5c87e39b 100644 --- a/datastore/integration-test/Dockerfile +++ b/datastore/integration-test/Dockerfile @@ -2,8 +2,6 @@ FROM python:3.11-slim-bookworm SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"] -ENV PROJECT_DATASTORE_PATH="datastore" -ENV PROJECT_PYTHON_PATH="integration-test" ENV DOCKER_PATH="/clients/python" RUN apt-get update \ @@ -14,8 +12,8 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY "${PROJECT_DATASTORE_PATH}/protobuf/datastore.proto" "/protobuf/datastore.proto" -COPY "${PROJECT_PYTHON_PATH}/requirements.txt" "${DOCKER_PATH}/requirements.txt" +COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto" +COPY "./requirements.txt" "${DOCKER_PATH}/requirements.txt" # hadolint ignore=DL3013 RUN pip install --no-cache-dir --upgrade pip \ @@ -27,11 +25,11 @@ RUN python -m grpc_tools.protoc \ --python_out="${DOCKER_PATH}" \ --grpc_python_out="${DOCKER_PATH}" -COPY "${PROJECT_PYTHON_PATH}/test_knmi.py" "${DOCKER_PATH}/test_knmi.py" -COPY "${PROJECT_PYTHON_PATH}/test_delete.py" "${DOCKER_PATH}/test_delete.py" -COPY "${PROJECT_PYTHON_PATH}/test_api.py" "${DOCKER_PATH}/test_api.py" +COPY "./test_knmi.py" "${DOCKER_PATH}/test_knmi.py" +COPY "./test_delete.py" "${DOCKER_PATH}/test_delete.py" +COPY "./test_api.py" "${DOCKER_PATH}/test_api.py" -COPY "${PROJECT_PYTHON_PATH}/response/" "${DOCKER_PATH}/response/" +COPY "./response/" "${DOCKER_PATH}/response/" WORKDIR "${DOCKER_PATH}" CMD ["pytest"] diff --git a/datastore/docker-compose.yml b/docker-compose.yml similarity index 79% rename from datastore/docker-compose.yml rename to docker-compose.yml index b406de26..20f696d1 100644 --- a/datastore/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: volumes: # - ts-data:/home/postgres/pgdata/data # for timescale image - ts-data:/var/lib/postgresql/data # for postgres image - - ./database/healthcheck_postgis_uptime.sh:/healthcheck_postgis_uptime.sh # for the healthcheck + - ./datastore/database/healthcheck_postgis_uptime.sh:/healthcheck_postgis_uptime.sh # for the healthcheck environment: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=mysecretpassword @@ -27,7 +27,7 @@ services: migrate: image: migrate/migrate:4 volumes: - - ./migrate/data/migrations:/data/migrations + - ./datastore/migrate/data/migrations:/data/migrations command: ["-path", "/data/migrations", "-database", "postgres://postgres:mysecretpassword@db:5432/data?sslmode=disable", "up"] depends_on: db: @@ -35,7 +35,7 @@ services: store: build: - context: datastore + context: datastore/datastore ports: - "50050:50050" - "6060:6060" # for flame graphing @@ -60,8 +60,7 @@ services: api: build: - context: . # TODO: Needed to get proto file. Find a better solution - dockerfile: api/Dockerfile + context: api ports: - "8008:8000" environment: @@ -74,8 +73,7 @@ services: client: profiles: ["test"] build: - context: . # TODO: Needed to get proto file. Find a better solution - dockerfile: examples/clients/python/Dockerfile + context: datastore/examples/clients/python environment: - DSHOST=store - DSPORT=50050 @@ -86,11 +84,11 @@ services: loader: profiles: ["test"] build: - context: . # TODO: Needed to get proto file and test data. Find a better solution - dockerfile: data-loader/Dockerfile + context: datastore/data-loader environment: - DSHOST=store - DSPORT=50050 + - BASE_URL=http://api:8000 depends_on: store: condition: service_healthy @@ -98,8 +96,7 @@ services: integration: profiles: ["test"] build: - context: . # TODO: Needed to get proto file. Find a better solution - dockerfile: integration-test/Dockerfile + context: datastore/integration-test environment: - DSHOST=store - DSPORT=50050 diff --git a/datastore/datastore/protobuf/datastore.proto b/protobuf/datastore.proto similarity index 100% rename from datastore/datastore/protobuf/datastore.proto rename to protobuf/datastore.proto