From 0f08e47efc2cdd561336f869d42f31fb933a84b0 Mon Sep 17 00:00:00 2001 From: Amund Isaksen <31344542+Teddy-1000@users.noreply.github.com> Date: Thu, 19 Sep 2024 11:07:03 +0200 Subject: [PATCH] Add metrics endpoint to edr api (#170) * Add metrics endpoint to edr api * Add metrics folder to unit dockerfile * Update dev_req * Recompiled with python3.11 * Recompiled with python3.11 --- api/Dockerfile | 3 +++ api/dev_requirements.txt | 27 +++++++++++++++++---------- api/export_metrics.py | 13 +++++++++++++ api/main.py | 3 +++ api/requirements.in | 1 + api/requirements.txt | 27 ++++++++++++++++----------- api/unit.Dockerfile | 20 ++++++++++++-------- 7 files changed, 65 insertions(+), 29 deletions(-) create mode 100644 api/export_metrics.py diff --git a/api/Dockerfile b/api/Dockerfile index 7be5016c..840e9bfd 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -28,6 +28,9 @@ RUN python -m grpc_tools.protoc \ COPY "." "${DOCKER_PATH}/" +# Create folder for metrics +ENV PROMETHEUS_MULTIPROC_DIR=/tmp/metrics +RUN mkdir -p /tmp/metrics WORKDIR "${DOCKER_PATH}" CMD ["gunicorn", "main:app", "--worker-class=uvicorn.workers.UvicornWorker"] diff --git a/api/dev_requirements.txt b/api/dev_requirements.txt index 53a9b083..06df5bfb 100644 --- a/api/dev_requirements.txt +++ b/api/dev_requirements.txt @@ -40,13 +40,13 @@ edr-pydantic==0.4.0 # via -r requirements.txt fastapi==0.110.3 # via -r requirements.txt -geojson-pydantic==1.1.0 +geojson-pydantic==1.1.1 # via -r requirements.txt -grpcio==1.66.0 +grpcio==1.66.1 # via # -r requirements.txt # grpcio-tools -grpcio-tools==1.66.0 +grpcio-tools==1.66.1 # via -r requirements.txt gunicorn==22.0.0 # via -r requirements.txt @@ -63,7 +63,7 @@ httptools==0.6.1 # uvicorn httpx==0.27.0 # via -r dev_requirements.in -idna==3.7 +idna==3.10 # via # -r requirements.txt # anyio @@ -78,7 +78,7 @@ markupsafe==2.1.5 # via # -r requirements.txt # jinja2 -numpy==2.1.0 +numpy==2.1.1 # via # -r requirements.txt # shapely @@ -91,18 +91,24 @@ packaging==24.1 # pytest pluggy==1.5.0 # via pytest -protobuf==5.27.3 +prometheus-client==0.20.0 + # via + # -r requirements.txt + # prometheus-fastapi-instrumentator +prometheus-fastapi-instrumentator==7.0.0 + # via -r requirements.txt +protobuf==5.28.1 # via # -r requirements.txt # grpcio-tools -pydantic==2.8.2 +pydantic==2.9.1 # via # -r requirements.txt # covjson-pydantic # edr-pydantic # fastapi # geojson-pydantic -pydantic-core==2.20.1 +pydantic-core==2.23.3 # via # -r requirements.txt # pydantic @@ -139,6 +145,7 @@ starlette==0.37.2 # -r requirements.txt # brotli-asgi # fastapi + # prometheus-fastapi-instrumentator typing-extensions==4.12.2 # via # -r requirements.txt @@ -151,11 +158,11 @@ uvloop==0.20.0 # via # -r requirements.txt # uvicorn -watchfiles==0.23.0 +watchfiles==0.24.0 # via # -r requirements.txt # uvicorn -websockets==13.0 +websockets==13.0.1 # via # -r requirements.txt # uvicorn diff --git a/api/export_metrics.py b/api/export_metrics.py new file mode 100644 index 00000000..542c40ca --- /dev/null +++ b/api/export_metrics.py @@ -0,0 +1,13 @@ +from fastapi import FastAPI +from prometheus_client import CollectorRegistry +from prometheus_client import multiprocess +from prometheus_fastapi_instrumentator import Instrumentator + + +def add_metrics(app: FastAPI): + registry = CollectorRegistry() + multiprocess.MultiProcessCollector(registry) + + instrumentator = Instrumentator(excluded_handlers=[".*admin.*", "/metrics", "/health"]) + instrumentator.add() + instrumentator.instrument(app).expose(app, include_in_schema=False) diff --git a/api/main.py b/api/main.py index 49328c61..25490db4 100644 --- a/api/main.py +++ b/api/main.py @@ -14,6 +14,8 @@ from routers import feature from utilities import create_url_from_request +from export_metrics import add_metrics + def setup_logging(): logger = logging.getLogger() @@ -30,6 +32,7 @@ def setup_logging(): app = FastAPI(swagger_ui_parameters={"tryItOutEnabled": True}) app.add_middleware(BrotliMiddleware) +add_metrics(app) @app.get( diff --git a/api/requirements.in b/api/requirements.in index a17a8bdd..c882648c 100644 --- a/api/requirements.in +++ b/api/requirements.in @@ -15,3 +15,4 @@ geojson-pydantic~=1.0 aiocached~=0.3.0 jinja2~=3.1 isodate~=0.6.1 +prometheus-fastapi-instrumentator~=7.0.0 diff --git a/api/requirements.txt b/api/requirements.txt index 2137bfca..e6b38f35 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --no-emit-index-url +# pip-compile --no-emit-index-url --output-file=requirements.txt # aiocached==0.3 # via -r requirements.in @@ -24,11 +24,11 @@ edr-pydantic==0.4.0 # via -r requirements.in fastapi==0.110.3 # via -r requirements.in -geojson-pydantic==1.1.0 +geojson-pydantic==1.1.1 # via -r requirements.in -grpcio==1.66.0 +grpcio==1.66.1 # via grpcio-tools -grpcio-tools==1.66.0 +grpcio-tools==1.66.1 # via -r requirements.in gunicorn==22.0.0 # via -r requirements.in @@ -36,7 +36,7 @@ h11==0.14.0 # via uvicorn httptools==0.6.1 # via uvicorn -idna==3.7 +idna==3.10 # via anyio isodate==0.6.1 # via -r requirements.in @@ -44,19 +44,23 @@ jinja2==3.1.4 # via -r requirements.in markupsafe==2.1.5 # via jinja2 -numpy==2.1.0 +numpy==2.1.1 # via shapely packaging==24.1 # via gunicorn -protobuf==5.27.3 +prometheus-client==0.20.0 + # via prometheus-fastapi-instrumentator +prometheus-fastapi-instrumentator==7.0.0 + # via -r requirements.in +protobuf==5.28.1 # via grpcio-tools -pydantic==2.8.2 +pydantic==2.9.1 # via # covjson-pydantic # edr-pydantic # fastapi # geojson-pydantic -pydantic-core==2.20.1 +pydantic-core==2.23.3 # via pydantic python-dotenv==1.0.1 # via uvicorn @@ -72,6 +76,7 @@ starlette==0.37.2 # via # brotli-asgi # fastapi + # prometheus-fastapi-instrumentator typing-extensions==4.12.2 # via # fastapi @@ -81,9 +86,9 @@ uvicorn[standard]==0.29.0 # via -r requirements.in uvloop==0.20.0 # via uvicorn -watchfiles==0.23.0 +watchfiles==0.24.0 # via uvicorn -websockets==13.0 +websockets==13.0.1 # via uvicorn # The following packages are considered to be unsafe in a requirements file: diff --git a/api/unit.Dockerfile b/api/unit.Dockerfile index f80573b8..29426e68 100644 --- a/api/unit.Dockerfile +++ b/api/unit.Dockerfile @@ -29,13 +29,17 @@ RUN python -m grpc_tools.protoc \ COPY "." "${DOCKER_PATH}/" +# Create folder for metrics +ENV PROMETHEUS_MULTIPROC_DIR=/tmp/metrics +RUN mkdir -p /tmp/metrics + WORKDIR "${DOCKER_PATH}" CMD ["/bin/sh", "-c", "{ python -m pytest \ - --timeout=60 \ - --junitxml=./output/pytest.xml \ - --cov-report=term-missing \ - --cov=. \ - --cov-config=./test/.coveragerc 2>&1; \ - echo $? > ./output/exit-code; } | \ - tee ./output/pytest-coverage.txt; \ - exit $(cat ./output/exit-code)"] + --timeout=60 \ + --junitxml=./output/pytest.xml \ + --cov-report=term-missing \ + --cov=. \ + --cov-config=./test/.coveragerc 2>&1; \ + echo $? > ./output/exit-code; } | \ + tee ./output/pytest-coverage.txt; \ + exit $(cat ./output/exit-code)"]