Add metrics endpoint to edr api (#170)

* Add metrics endpoint to edr api * Add metrics folder to unit dockerfile * Update dev_req * Recompiled with python3.11 * Recompiled with python3.11
EURODEO · Sep 19, 2024 · 0f08e47 · 0f08e47 · github-actions · Sep 19, 2024
1 parent 4ebfd56
commit 0f08e47
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 29 deletions.
diff --git a/api/Dockerfile b/api/Dockerfile
@@ -28,6 +28,9 @@ RUN python -m grpc_tools.protoc  \
 
 COPY "." "${DOCKER_PATH}/"
 
+# Create folder for metrics
+ENV PROMETHEUS_MULTIPROC_DIR=/tmp/metrics
+RUN mkdir -p /tmp/metrics
 
 WORKDIR "${DOCKER_PATH}"
 CMD ["gunicorn", "main:app", "--worker-class=uvicorn.workers.UvicornWorker"]
diff --git a/api/dev_requirements.txt b/api/dev_requirements.txt
@@ -40,13 +40,13 @@ edr-pydantic==0.4.0
     # via -r requirements.txt
 fastapi==0.110.3
     # via -r requirements.txt
-geojson-pydantic==1.1.0
+geojson-pydantic==1.1.1
     # via -r requirements.txt
-grpcio==1.66.0
+grpcio==1.66.1
     # via
     #   -r requirements.txt
     #   grpcio-tools
-grpcio-tools==1.66.0
+grpcio-tools==1.66.1
     # via -r requirements.txt
 gunicorn==22.0.0
     # via -r requirements.txt
@@ -63,7 +63,7 @@ httptools==0.6.1
     #   uvicorn
 httpx==0.27.0
     # via -r dev_requirements.in
-idna==3.7
+idna==3.10
     # via
     #   -r requirements.txt
     #   anyio
@@ -78,7 +78,7 @@ markupsafe==2.1.5
     # via
     #   -r requirements.txt
     #   jinja2
-numpy==2.1.0
+numpy==2.1.1
     # via
     #   -r requirements.txt
     #   shapely
@@ -91,18 +91,24 @@ packaging==24.1
     #   pytest
 pluggy==1.5.0
     # via pytest
-protobuf==5.27.3
+prometheus-client==0.20.0
+    # via
+    #   -r requirements.txt
+    #   prometheus-fastapi-instrumentator
+prometheus-fastapi-instrumentator==7.0.0
+    # via -r requirements.txt
+protobuf==5.28.1
     # via
     #   -r requirements.txt
     #   grpcio-tools
-pydantic==2.8.2
+pydantic==2.9.1
     # via
     #   -r requirements.txt
     #   covjson-pydantic
     #   edr-pydantic
     #   fastapi
     #   geojson-pydantic
-pydantic-core==2.20.1
+pydantic-core==2.23.3
     # via
     #   -r requirements.txt
     #   pydantic
@@ -139,6 +145,7 @@ starlette==0.37.2
     #   -r requirements.txt
     #   brotli-asgi
     #   fastapi
+    #   prometheus-fastapi-instrumentator
 typing-extensions==4.12.2
     # via
     #   -r requirements.txt
@@ -151,11 +158,11 @@ uvloop==0.20.0
     # via
     #   -r requirements.txt
     #   uvicorn
-watchfiles==0.23.0
+watchfiles==0.24.0
     # via
     #   -r requirements.txt
     #   uvicorn
-websockets==13.0
+websockets==13.0.1
     # via
     #   -r requirements.txt
     #   uvicorn

diff --git a/api/export_metrics.py b/api/export_metrics.py
@@ -0,0 +1,13 @@
+from fastapi import FastAPI
+from prometheus_client import CollectorRegistry
+from prometheus_client import multiprocess
+from prometheus_fastapi_instrumentator import Instrumentator
+
+
+def add_metrics(app: FastAPI):
+    registry = CollectorRegistry()
+    multiprocess.MultiProcessCollector(registry)
+
+    instrumentator = Instrumentator(excluded_handlers=[".*admin.*", "/metrics", "/health"])
+    instrumentator.add()
+    instrumentator.instrument(app).expose(app, include_in_schema=False)
diff --git a/api/main.py b/api/main.py
@@ -14,6 +14,8 @@
 from routers import feature
 from utilities import create_url_from_request
 
+from export_metrics import add_metrics
+
 
 def setup_logging():
     logger = logging.getLogger()
@@ -30,6 +32,7 @@ def setup_logging():
 
 app = FastAPI(swagger_ui_parameters={"tryItOutEnabled": True})
 app.add_middleware(BrotliMiddleware)
+add_metrics(app)
 
 
 @app.get(

diff --git a/api/requirements.in b/api/requirements.in
@@ -15,3 +15,4 @@ geojson-pydantic~=1.0
 aiocached~=0.3.0
 jinja2~=3.1
 isodate~=0.6.1
+prometheus-fastapi-instrumentator~=7.0.0
diff --git a/api/requirements.txt b/api/requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --no-emit-index-url
+#    pip-compile --no-emit-index-url --output-file=requirements.txt
 #
 aiocached==0.3
     # via -r requirements.in
@@ -24,39 +24,43 @@ edr-pydantic==0.4.0
     # via -r requirements.in
 fastapi==0.110.3
     # via -r requirements.in
-geojson-pydantic==1.1.0
+geojson-pydantic==1.1.1
     # via -r requirements.in
-grpcio==1.66.0
+grpcio==1.66.1
     # via grpcio-tools
-grpcio-tools==1.66.0
+grpcio-tools==1.66.1
     # via -r requirements.in
 gunicorn==22.0.0
     # via -r requirements.in
 h11==0.14.0
     # via uvicorn
 httptools==0.6.1
     # via uvicorn
-idna==3.7
+idna==3.10
     # via anyio
 isodate==0.6.1
     # via -r requirements.in
 jinja2==3.1.4
     # via -r requirements.in
 markupsafe==2.1.5
     # via jinja2
-numpy==2.1.0
+numpy==2.1.1
     # via shapely
 packaging==24.1
     # via gunicorn
-protobuf==5.27.3
+prometheus-client==0.20.0
+    # via prometheus-fastapi-instrumentator
+prometheus-fastapi-instrumentator==7.0.0
+    # via -r requirements.in
+protobuf==5.28.1
     # via grpcio-tools
-pydantic==2.8.2
+pydantic==2.9.1
     # via
     #   covjson-pydantic
     #   edr-pydantic
     #   fastapi
     #   geojson-pydantic
-pydantic-core==2.20.1
+pydantic-core==2.23.3
     # via pydantic
 python-dotenv==1.0.1
     # via uvicorn
@@ -72,6 +76,7 @@ starlette==0.37.2
     # via
     #   brotli-asgi
     #   fastapi
+    #   prometheus-fastapi-instrumentator
 typing-extensions==4.12.2
     # via
     #   fastapi
@@ -81,9 +86,9 @@ uvicorn[standard]==0.29.0
     # via -r requirements.in
 uvloop==0.20.0
     # via uvicorn
-watchfiles==0.23.0
+watchfiles==0.24.0
     # via uvicorn
-websockets==13.0
+websockets==13.0.1
     # via uvicorn
 
 # The following packages are considered to be unsafe in a requirements file:

diff --git a/api/unit.Dockerfile b/api/unit.Dockerfile
@@ -29,13 +29,17 @@ RUN python -m grpc_tools.protoc  \
 
 COPY "." "${DOCKER_PATH}/"
 
+# Create folder for metrics
+ENV PROMETHEUS_MULTIPROC_DIR=/tmp/metrics
+RUN mkdir -p /tmp/metrics
+
 WORKDIR "${DOCKER_PATH}"
 CMD ["/bin/sh", "-c", "{ python -m pytest \
-        --timeout=60 \
-        --junitxml=./output/pytest.xml \
-        --cov-report=term-missing \
-        --cov=. \
-        --cov-config=./test/.coveragerc 2>&1; \
-            echo $? > ./output/exit-code; } | \
-            tee ./output/pytest-coverage.txt; \
-            exit $(cat ./output/exit-code)"]
+    --timeout=60 \
+    --junitxml=./output/pytest.xml \
+    --cov-report=term-missing \
+    --cov=. \
+    --cov-config=./test/.coveragerc 2>&1; \
+    echo $? > ./output/exit-code; } | \
+    tee ./output/pytest-coverage.txt; \
+    exit $(cat ./output/exit-code)"]
File	Stmts	Miss	Cover	Missing
\_\_init\_\_.py	0	0	100%
datastore_pb2.py	65	52	20%	34–85
datastore_pb2_grpc.py	54	27	50%	15–16, 19, 57–72, 105–107, 112–114, 119–121, 126–128, 132–157, 195, 222, 249, 276
export_metrics.py	10	0	100%
grpc_getter.py	20	11	45%	15–19, 23–26, 30–32, 36–38
locustfile.py	15	15	0%	1–31
main.py	43	7	84%	45, 50, 60, 70–71, 81–82
metadata_endpoints.py	63	29	54%	45–54, 58, 85, 100–216, 220
response_classes.py	5	0	100%
utilities.py	181	56	69%	20, 38, 45, 67–70, 78–89, 94–101, 121, 125, 127, 154, 157, 166, 184–185, 189, 205–210, 215, 219–220, 224–226, 232–240, 250–253, 259–260, 272–276, 299, 304, 317
custom_geo_json
edr_feature_collection.py	6	0	100%
formatters
\_\_init\_\_.py	11	0	100%
covjson.py	57	1	98%	88
geojson.py	17	2	88%	21, 46
openapi
custom_dimension_examples.py	4	0	100%
edr_query_parameter_descriptions.py	11	0	100%
openapi_examples.py	13	0	100%
routers
\_\_init\_\_.py	0	0	100%
edr.py	102	4	96%	350–351, 441–442
feature.py	47	19	60%	99–132, 148–153, 159–181
TOTAL	724	223	69%