Dockerfile update and package restructuring

.gitignore

@@ -204,3 +204,6 @@ datastore_pb2.py
 datastore_pb2_grpc.py
 # Ignore all the copied protobuf directories, as the root contains the source of truth.
 **/protobuf
+
+# Ignore cf standard file, as it is generated while docker build.
+cf_standard_names_v84.txt

docker-compose.yml

@@ -76,8 +76,8 @@ services:
     ports:
       - "8009:8001"
     environment:
-      - DSHOST=${DSHOST:-store}
-      - DSPORT=${DSPORT:-50050}
+      - DATASTORE_HOST=${DATASTORE_HOST:-store}
+      - DATASTORE_PORT=${DATASTORE_PORT:-50050}
     depends_on:
       store:
         condition: service_healthy

ingest/Dockerfile

@@ -13,32 +13,32 @@ RUN apt-get update \
 	&& apt-get autoremove -y \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/*
-
 COPY "./protobuf/datastore.proto" "/protobuf/datastore.proto"
 COPY "./api" "${DOCKER_PATH}/api/"
-COPY "requirements.txt" "${DOCKER_PATH}/"
-COPY "./src/" "/ingest/src"
-COPY "./pyproject.toml" "/ingest/pyproject.toml"
-COPY "./setup.py" "/ingest/"
+COPY "requirements.txt" "${DOCKER_PATH}/api/"
+COPY "./src/" "${DOCKER_PATH}/src/"
+COPY "./pyproject.toml" "${DOCKER_PATH}/pyproject.toml"
+COPY "./setup.py" "${DOCKER_PATH}/setup.py"
 
-RUN pip install --no-cache-dir --upgrade -r "${DOCKER_PATH}/requirements.txt"
+RUN pip install --no-cache-dir --upgrade -r "${DOCKER_PATH}/api/requirements.txt"
 RUN pip install --no-cache-dir --upgrade pybind11
 
 # hadolint ignore=DL3013
-WORKDIR /ingest/src/ingest/bufr/
+WORKDIR "${DOCKER_PATH}"/src/ingest/bufr/
 RUN make
 WORKDIR /
 
-# hadolint ignore=DL3013
-RUN pip install --no-cache-dir --upgrade pip \
-	&& pip install --no-cache-dir --upgrade "/ingest/"
 # Compiling the protobuf file
 RUN python -m grpc_tools.protoc  \
 	--proto_path="protobuf" "protobuf/datastore.proto" \
 	--python_out="${DOCKER_PATH}"  \
 	--grpc_python_out="${DOCKER_PATH}"
 
+WORKDIR "${DOCKER_PATH}"
 
+RUN python "${DOCKER_PATH}/api/generate_standard_name.py"
 
-WORKDIR "${DOCKER_PATH}"
+# hadolint ignore=DL3013
+RUN pip install --no-cache-dir --upgrade pip \
+	&& pip install --no-cache-dir --upgrade "${DOCKER_PATH}/"
 CMD ["gunicorn", "api.main:app", "--workers=4", "--worker-class=uvicorn.workers.UvicornWorker", "--bind=0.0.0.0:8001"]

ingest/api/generate_standard_name.py

@@ -9,6 +9,6 @@
 )
 
 # Find all 'entry' elements and get their 'id' attribute
-with open("cf_standard_names_v84.txt", "w") as file:
+with open("api/cf_standard_names_v84.txt", "w") as file:
     for entry in root.findall("entry"):
         file.write(f'{entry.get("id")}\n')

ingest/src/ingest/main.py → ingest/api/ingest.py

@@ -8,9 +8,9 @@
 import pkg_resources
 from jsonschema import Draft202012Validator
 
-from ingest.datastore import DatastoreConnection
-from ingest.messages import messages
-from ingest.send_mqtt import MQTTConnection
+from api.datastore import DatastoreConnection
+from api.messages import messages
+from api.send_mqtt import MQTTConnection
 
 logger = logging.getLogger(__name__)
 
@@ -33,15 +33,15 @@ def __init__(
         self.uuid_prefix = uuid_prefix
 
         if not schema_path:
-            self.schema_path = pkg_resources.resource_filename("esoh", "schemas")
+            self.schema_path = pkg_resources.resource_filename("src", "schemas")
         else:
             self.schema_path = schema_path
         if not schema_file:
             self.schema_file = "e-soh-message-spec.json"
         else:
             self.schema_file = schema_file
-
         esoh_mqtt_schema = os.path.join(self.schema_path, self.schema_file)
+
         with open(esoh_mqtt_schema, "r") as file:
             self.esoh_mqtt_schema = json.load(file)
         self.schema_validator = Draft202012Validator(self.esoh_mqtt_schema)

ingest/api/main.py

@@ -7,8 +7,8 @@
 from fastapi import UploadFile
 from pydantic import BaseModel
 
+from api.ingest import IngestToPipeline
 from api.model import JsonMessageSchema
-from ingest.main import IngestToPipeline
 
 
 logger = logging.getLogger(__name__)
@@ -37,7 +37,7 @@ class Response(BaseModel):
 }
 
 
-@app.post("/upload/nc")
+@app.post("/nc")
 async def upload_netcdf_file(files: UploadFile, input_type: str = "nc"):
     try:
         ingester = IngestToPipeline(
@@ -54,7 +54,7 @@ async def upload_netcdf_file(files: UploadFile, input_type: str = "nc"):
         return Response(status_message=str(e), status_code=500)
 
 
-@app.post("/upload/bufr")
+@app.post("/bufr")
 async def upload_bufr_file(files: UploadFile, input_type: str = "bufr"):
     try:
         ingester = IngestToPipeline(

ingest/api/model.py

@@ -83,7 +83,7 @@ class Content(BaseModel):
 
     @model_validator(mode="after")
     def check_standard_name_match(self) -> "Content":
-        with open("cf_standard_names_v84.txt", "r") as file:
+        with open("api/cf_standard_names_v84.txt", "r") as file:
             standard_names = {line.strip() for line in file}
         if self.standard_name not in standard_names:
             raise ValueError(f"{self.standard_name} not a CF Standard name")

ingest/requirements.txt

@@ -8,3 +8,4 @@ fastapi == 0.108.0
 grpcio-tools==1.58.0
 uvicorn==0.25.0
 python-multipart~=0.0.9
+requests == 2.31.0

ingest/test/test_automatic_filetype_detection.py

@@ -1,5 +1,6 @@
 import pytest
-from esoh.ingest.main import IngestToPipeline
+
+from api.ingest import IngestToPipeline
 
 
 @pytest.mark.parametrize(

ingest/test/test_datastore_ingest.py

@@ -1,6 +1,6 @@
 import json
 
-from esoh.ingest.datastore import DatastoreConnection
+from api.datastore import DatastoreConnection
 
 
 def test_datastore_ingest():

ingest/test/test_ingest_to_pipeline_errors.py

@@ -1,6 +1,7 @@
 import pytest
 import xarray as xr
-from esoh.ingest.main import IngestToPipeline
+
+from api.ingest import IngestToPipeline
 
 
 def test_build_message_errors():

ingest/test/test_main.py

@@ -0,0 +1,33 @@
+import json
+
+import pytest
+from fastapi.testclient import TestClient
+
+from api.main import app
+
+
+@pytest.fixture(scope="module")
+def test_app():
+    client = TestClient(app)
+    yield client
+
+
+def test_post_json_success(test_app, monkeypatch):
+    with open("test/test_data/test_payload.json", "r") as file:
+        json_data = json.load(file)
+    monkeypatch.setattr("api.main.IngestToPipeline.ingest", lambda *a: ("Success", 200))
+
+    response = test_app.post("/json?input_type=json", json=json_data)
+    assert response.status_code == 200
+
+    assert response.json() == {"status_message": "Success", "status_code": 200}
+
+
+def test_post_json_failure(test_app, monkeypatch):
+    with open("test/test_data/test_payload.json", "r") as file:
+        json_data = json.load(file)
+    monkeypatch.setattr("api.main.IngestToPipeline.ingest", lambda *a: ("Success", 200))
+
+    response = test_app.post("/json?input_type=json", json=json_data)
+    assert response.status_code == 200
+    assert response.json() == {"status_message": "Success", "status_code": 200}

ingest/test/test_verify_json_payloads.py

@@ -3,24 +3,25 @@
 
 import pytest
 import xarray as xr
-from esoh.ingest.bufr.bufresohmsg_py import init_bufr_schema_py
-from esoh.ingest.bufr.bufresohmsg_py import init_bufrtables_py
-from esoh.ingest.bufr.bufresohmsg_py import init_oscar_py
-from esoh.ingest.main import IngestToPipeline
 from jsonschema import Draft202012Validator
 from jsonschema import ValidationError
 
+from api.ingest import IngestToPipeline
+from ingest.bufr.bufresohmsg_py import init_bufr_schema_py
+from ingest.bufr.bufresohmsg_py import init_bufrtables_py
+from ingest.bufr.bufresohmsg_py import init_oscar_py
+
 
 @pytest.mark.timeout(1000)
 @pytest.mark.parametrize("bufr_file_path", glob.glob("test/test_data/bufr/*.buf*"))
 def test_verify_json_payload_bufr(bufr_file_path):
     # Load the schema
-    with open("src/esoh/schemas/e-soh-message-spec.json", "r") as file:
+    with open("src/schemas/e-soh-message-spec.json", "r") as file:
         e_soh_mqtt_message_schema = json.load(file)
 
     init_bufrtables_py("")
-    init_oscar_py("./src/esoh/ingest/bufr/oscar/oscar_stations_all.json")
-    init_bufr_schema_py("./src/esoh/schemas/bufr_to_e_soh_message.json")
+    init_oscar_py("./src/ingest/bufr/oscar/oscar_stations_all.json")
+    init_bufr_schema_py("./src/schemas/bufr_to_e_soh_message.json")
     msg_build = IngestToPipeline(None, None, "testing", testing=True)
 
     json_payloads = msg_build._build_messages(bufr_file_path, input_type="bufr")
@@ -36,7 +37,7 @@ def test_verify_json_payload_bufr(bufr_file_path):
 @pytest.mark.parametrize("netcdf_file_path", glob.glob("test/test_data/met_norway/*.nc"))
 def test_verify_json_payload_metno_netcdf(netcdf_file_path):
     # Load the schema
-    with open("src/esoh/schemas/e-soh-message-spec.json", "r") as file:
+    with open("src/schemas/e-soh-message-spec.json", "r") as file:
         e_soh_mqtt_message_schema = json.load(file)
 
     ds = xr.load_dataset(netcdf_file_path)
@@ -55,7 +56,7 @@ def test_verify_json_payload_metno_netcdf(netcdf_file_path):
 
 @pytest.mark.parametrize("netcdf_file_path", glob.glob("test/test_data/knmi/*.nc"))
 def test_verify_json_payload_knmi_netcdf(netcdf_file_path):
-    with open("src/esoh/schemas/e-soh-message-spec.json", "r") as file:
+    with open("src/schemas/e-soh-message-spec.json", "r") as file:
         e_soh_mqtt_message_schema = json.load(file)
 
     ds = xr.load_dataset(netcdf_file_path)