Merge pull request #43 from EURODEO/making_ingest_api

Added json endpoint.

.github/workflows/ci.yml

@@ -160,29 +160,40 @@ jobs:
           junitxml-title: API Unit Test Coverage Summary
 
 
-# TODO: These tests don't currently work. Uncomment once this is resolved.
-#  test-ingest:
-#    runs-on: ubuntu-latest
-#    strategy:
-#      matrix:
-#        python-version: ["3.10"] # Add 3.11 back pybind11 bug is fixed
-#    steps:
-#      - name: Checkout the repo
-#        uses: actions/checkout@v4
-#      - name: Ubuntu setup
-#        run: sudo apt update && sudo apt install libeccodes-data	rapidjson-dev	pybind11-dev libssl-dev
-#      - name: Python Setup
-#        uses: actions/setup-python@v5
-#        with:
-#          python-version: ${{ matrix.python-version }}
-#          architecture: x64
-#      - name: Checkout Source
-#        uses: actions/checkout@v4
-#      - name: Install Dependencies
-#        run: |
-#          pip install --upgrade pip
-#          pip install pytest-timeout
-#          pip install pytest-cov
-#          pip install ./ingest
-#      - name: Run Tests
-#        run: python -m pytest -v --timeout=60 ./ingest
+  test-ingest:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"] # Add 3.11 back pybind11 bug is fixed
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
+      - name: Ubuntu setup
+        run: sudo apt update && sudo apt install libeccodes-data rapidjson-dev pybind11-dev libssl-dev
+      - name: Python Setup
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - name: Checkout Source
+        uses: actions/checkout@v4
+
+      - name: Install Dependencies
+        run: |
+          pip install --upgrade pip
+          pip install pytest-timeout
+          pip install pytest-cov
+          pip install httpx
+          pip install -r ./ingest/requirements.txt
+          pip install ./ingest
+
+      - name: Copy Protobuf file to api directory and build
+        run: |
+          mkdir ./ingest/protobuf
+          cp ./protobuf/datastore.proto ./ingest/protobuf/datastore.proto
+          python -m grpc_tools.protoc --proto_path=./ingest/protobuf --python_out=./ingest --grpc_python_out=./ingest ./ingest/protobuf/datastore.proto
+
+      - name: Run Tests
+        run: |
+          cd ingest
+          python -m pytest -v --timeout=60

.gitignore

@@ -205,3 +205,6 @@ datastore_pb2.py
 datastore_pb2_grpc.py
 # Ignore all the copied protobuf directories, as the root contains the source of truth.
 **/protobuf
+
+# Ignore cf standard file, as it is generated while docker build.
+cf_standard_names_v84.txt

ci/docker/copy-protobuf.sh

@@ -6,7 +6,7 @@ declare -a destination_paths=(
   "./datastore/datastore"
   "./datastore/examples/clients/python"
   "./datastore/integration-test"
-#  "./ingest"
+  "./ingest"
 )
 
 for path in "${destination_paths[@]}"

docker-compose.yml

@@ -71,6 +71,18 @@ services:
       store:
         condition: service_healthy
 
+  ingest:
+    build:
+      context: ingest
+    ports:
+      - "8009:8001"
+    environment:
+      - DATASTORE_HOST=${DATASTORE_HOST:-store}
+      - DATASTORE_PORT=${DATASTORE_PORT:-50050}
+    depends_on:
+      store:
+        condition: service_healthy
+
   client:
     profiles: ["test"]
     build:

ingest/.dockerignore

@@ -0,0 +1 @@
+**/datastore_pb2*.py

ingest/Dockerfile

@@ -0,0 +1,44 @@
+FROM python:3.10-slim-bookworm
+
+SHELL ["/bin/bash", "-eux", "-o", "pipefail", "-c"]
+
+ENV DOCKER_PATH="/app"
+
+# hadolint ignore=DL3008
+RUN apt-get update \
+	&& apt-get -y upgrade \
+	&& apt-get install -y --no-install-recommends git libeccodes-data rapidjson-dev pybind11-dev make g++\
+	# Cleanup
+	&& rm -rf /usr/tmp  \
+	&& apt-get autoremove -y \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists/*
+COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto"
+COPY "./api" "${DOCKER_PATH}/api/"
+COPY "requirements.txt" "${DOCKER_PATH}/api/"
+COPY "./src/" "${DOCKER_PATH}/src/"
+COPY "./pyproject.toml" "${DOCKER_PATH}/pyproject.toml"
+COPY "./setup.py" "${DOCKER_PATH}/setup.py"
+
+RUN pip install --no-cache-dir --upgrade -r "${DOCKER_PATH}/api/requirements.txt" \
+	&& pip install --no-cache-dir --upgrade pybind11~="2.11.1"
+
+# hadolint ignore=DL3013
+WORKDIR "${DOCKER_PATH}"/src/ingest/bufr/
+RUN make
+WORKDIR /
+
+# Compiling the protobuf file
+RUN python -m grpc_tools.protoc  \
+	--proto_path="protobuf" "protobuf/datastore.proto" \
+	--python_out="${DOCKER_PATH}"  \
+	--grpc_python_out="${DOCKER_PATH}"
+
+WORKDIR "${DOCKER_PATH}"
+
+RUN python "api/generate_standard_name.py"
+
+# hadolint ignore=DL3013
+RUN pip install --no-cache-dir --upgrade pip \
+	&& pip install --no-cache-dir --upgrade "${DOCKER_PATH}/"
+CMD ["gunicorn", "api.main:app", "--workers=4", "--worker-class=uvicorn.workers.UvicornWorker", "--bind=0.0.0.0:8001"]