From 9f3ed76b365d9495e17dd74c33e70b8aba31abff Mon Sep 17 00:00:00 2001 From: Victor Perron Date: Thu, 18 Apr 2024 15:36:38 +0200 Subject: [PATCH 1/4] chore(pipeline) : Use uv to resolve dependencies https://github.com/astral-sh/uv --- pipeline/requirements/Makefile | 22 ++++++++++++---------- pipeline/requirements/dev/requirements.in | 5 +++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/pipeline/requirements/Makefile b/pipeline/requirements/Makefile index 3d83f3ee..46d3798f 100644 --- a/pipeline/requirements/Makefile +++ b/pipeline/requirements/Makefile @@ -1,6 +1,4 @@ -# pip-compile~=7.3 - -PIP_COMPILE := pip-compile --quiet +PIP_COMPILE := pipx run uv pip compile --quiet ifeq ($(filter upgrade,$(MAKECMDGOALS)),upgrade) PIP_COMPILE += --upgrade @@ -8,11 +6,15 @@ endif AIRFLOW_VERSION := 2.9.0 PYTHON_VERSION := 3.11 +UV_VERSION := 0.1.33 CONSTRAINTS_FILE_URL := https://raw.githubusercontent.com/apache/airflow/constraints-$(AIRFLOW_VERSION)/constraints-$(PYTHON_VERSION).txt -.PHONY: all airflow dbt pipx python dev upgrade constraints +.PHONY: all airflow dbt pipx python dev upgrade constraints uv + +all: uv airflow dbt pipx python dev constraints -all: airflow dbt pipx python dev constraints +uv: + pip install uv==$(UV_VERSION) # https://airflow.apache.org/docs/apache-airflow/stable/installation/installing-from-pypi.html#constraints-files constraints: @@ -20,21 +22,21 @@ constraints: # airflow requirements for scalingo deployments # unlike docker based deployments, scalingo requires airflow to be installed from pypi -airflow: airflow/requirements.in constraints +airflow: airflow/requirements.in constraints uv $(PIP_COMPILE) --constraint airflow/constraints.txt airflow/requirements.in # isolated env for dbt, which should be invoked using a bash operator -dbt: tasks/dbt/requirements.in +dbt: tasks/dbt/requirements.in uv $(PIP_COMPILE) tasks/dbt/requirements.in # isolated env for pipx, which should be invoked using a bash operator -pipx: tasks/pipx/requirements.in +pipx: tasks/pipx/requirements.in uv $(PIP_COMPILE) tasks/pipx/requirements.in # python based tasks requirements must be compatible with airflow requirements # therefore these requirements are compiled with airflow constraints -python: airflow tasks/python/requirements.in +python: airflow tasks/python/requirements.in uv $(PIP_COMPILE) --constraint airflow/constraints.txt tasks/python/requirements.in -dev: python dev/requirements.in +dev: python dev/requirements.in uv $(PIP_COMPILE) --constraint airflow/constraints.txt dev/requirements.in diff --git a/pipeline/requirements/dev/requirements.in b/pipeline/requirements/dev/requirements.in index 97bef4da..61e6d095 100644 --- a/pipeline/requirements/dev/requirements.in +++ b/pipeline/requirements/dev/requirements.in @@ -1,9 +1,10 @@ -r ../airflow/requirements.in -r ../tasks/python/requirements.in -ruff pre-commit pytest pytest-dotenv -tox requests-mock +ruff +tox +uv From 964c0b47ca5fcc2aa958a9a26df85e25f9ae0050 Mon Sep 17 00:00:00 2001 From: Victor Perron Date: Thu, 18 Apr 2024 15:46:31 +0200 Subject: [PATCH 2/4] chore(api) : Use uv and add a Makefile --- api/CONTRIBUTING.md | 10 ++-- api/Makefile | 23 +++++++++ api/requirements/dev-requirements.txt | 67 +++++------------------- api/requirements/requirements.txt | 55 ++++---------------- api/requirements/test-requirements.txt | 71 +++++--------------------- api/setup.py | 5 +- 6 files changed, 63 insertions(+), 168 deletions(-) create mode 100644 api/Makefile diff --git a/api/CONTRIBUTING.md b/api/CONTRIBUTING.md index a2872556..a9f6cd30 100644 --- a/api/CONTRIBUTING.md +++ b/api/CONTRIBUTING.md @@ -45,16 +45,12 @@ tox # 1. add/remove packages from the requirements in setup.py # 2. compile dependencies -pip-compile --output-file=requirements/requirements.txt && \ - pip-compile --extra=dev --output-file=requirements/dev-requirements.txt && \ - pip-compile --extra=test --output-file=requirements/test-requirements.txt + +make ``` ### 2. Upgrading packages ```bash -# 1. compile dependencies with the upgrade flag set -pip-compile --upgrade --output-file=requirements/requirements.txt && \ - pip-compile --upgrade --extra=dev --output-file=requirements/dev-requirements.txt && \ - pip-compile --upgrade --extra=test --output-file=requirements/test-requirements.txt +make upgrade all ``` diff --git a/api/Makefile b/api/Makefile new file mode 100644 index 00000000..94302b30 --- /dev/null +++ b/api/Makefile @@ -0,0 +1,23 @@ +UV_VERSION := 0.1.33 + +PIP_COMPILE := pipx run uv pip compile setup.py --quiet + +ifeq ($(filter upgrade,$(MAKECMDGOALS)),upgrade) +PIP_COMPILE += --upgrade +endif + +.PHONY: all dev base test uv upgrade + +all: base dev test + +uv: + pip install uv==$(UV_VERSION) + +base: uv + $(PIP_COMPILE) --output-file=requirements/requirements.txt + +dev: uv + $(PIP_COMPILE) --extra=dev --output-file=requirements/dev-requirements.txt + +test: uv + $(PIP_COMPILE) --extra=test --output-file=requirements/test-requirements.txt diff --git a/api/requirements/dev-requirements.txt b/api/requirements/dev-requirements.txt index d460933e..3eb9bb75 100644 --- a/api/requirements/dev-requirements.txt +++ b/api/requirements/dev-requirements.txt @@ -1,11 +1,6 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --extra=dev --output-file=requirements/dev-requirements.txt -# +# This file was autogenerated by uv via the following command: +# uv pip compile setup.py --extra=dev --output-file=requirements/dev-requirements.txt alembic==1.13.1 - # via data-inclusion-api (setup.py) annotated-types==0.6.0 # via pydantic anyio==4.3.0 @@ -26,7 +21,6 @@ cachetools==5.3.3 # via tox certifi==2024.2.2 # via - # data-inclusion-api (setup.py) # fiona # httpcore # httpx @@ -48,7 +42,6 @@ click==8.1.7 # via # click-plugins # cligj - # data-inclusion-api (setup.py) # fiona # uvicorn click-plugins==1.1.1 @@ -58,11 +51,8 @@ cligj==0.7.2 colorama==0.4.6 # via tox cryptography==42.0.5 - # via - # data-inclusion-api (setup.py) - # python-jose + # via python-jose data-inclusion-schema==0.14.0 - # via data-inclusion-api (setup.py) distlib==0.3.8 # via virtualenv dnspython==2.6.1 @@ -72,17 +62,13 @@ ecdsa==0.18.0 email-validator==2.1.1 # via pydantic faker==23.3.0 - # via data-inclusion-api (setup.py) fastapi==0.110.0 # via - # data-inclusion-api (setup.py) # fastapi-debug-toolbar # fastapi-pagination # sentry-sdk fastapi-debug-toolbar==0.6.2 - # via data-inclusion-api (setup.py) fastapi-pagination==0.12.17 - # via data-inclusion-api (setup.py) filelock==3.13.1 # via # tox @@ -90,15 +76,11 @@ filelock==3.13.1 fiona==1.9.6 # via geopandas furl==2.1.3 - # via data-inclusion-api (setup.py) geoalchemy2==0.14.6 - # via data-inclusion-api (setup.py) geopandas==0.14.3 - # via data-inclusion-api (setup.py) greenlet==3.0.3 # via sqlalchemy gunicorn==21.2.0 - # via data-inclusion-api (setup.py) h11==0.14.0 # via # httpcore @@ -108,7 +90,6 @@ httpcore==1.0.4 httptools==0.6.1 # via uvicorn httpx==0.27.0 - # via data-inclusion-api (setup.py) identify==2.5.35 # via pre-commit idna==3.6 @@ -128,14 +109,12 @@ markupsafe==2.1.5 # jinja2 # mako minio==7.2.5 - # via data-inclusion-api (setup.py) multivolumefile==0.2.3 # via py7zr nodeenv==1.8.0 # via pre-commit numpy==1.26.4 # via - # data-inclusion-api (setup.py) # pandas # pyarrow # shapely @@ -149,9 +128,7 @@ packaging==23.2 # pyproject-api # tox pandas==2.2.1 - # via - # data-inclusion-api (setup.py) - # geopandas + # via geopandas platformdirs==4.2.0 # via # tox @@ -159,15 +136,11 @@ platformdirs==4.2.0 pluggy==1.4.0 # via tox pre-commit==3.6.2 - # via data-inclusion-api (setup.py) psutil==5.9.8 # via py7zr psycopg2==2.9.9 - # via data-inclusion-api (setup.py) py7zr==0.21.0 - # via data-inclusion-api (setup.py) pyarrow==15.0.0 - # via data-inclusion-api (setup.py) pyasn1==0.5.1 # via # python-jose @@ -180,14 +153,12 @@ pycryptodome==3.20.0 # via minio pycryptodomex==3.20.0 # via py7zr -pydantic[email]==2.6.3 +pydantic==2.6.3 # via - # data-inclusion-api (setup.py) # data-inclusion-schema # fastapi # fastapi-debug-toolbar # fastapi-pagination - # pydantic # pydantic-extra-types # pydantic-settings pydantic-core==2.16.3 @@ -195,9 +166,7 @@ pydantic-core==2.16.3 pydantic-extra-types==2.6.0 # via fastapi-debug-toolbar pydantic-settings==2.2.1 - # via - # data-inclusion-api (setup.py) - # fastapi-debug-toolbar + # via fastapi-debug-toolbar pyinstrument==4.6.2 # via fastapi-debug-toolbar pyppmd==1.1.0 @@ -212,15 +181,11 @@ python-dateutil==2.9.0.post0 # pandas python-dotenv==1.0.1 # via - # data-inclusion-api (setup.py) # pydantic-settings # uvicorn -python-jose[cryptography]==3.3.0 - # via data-inclusion-api (setup.py) +python-jose==3.3.0 pytz==2024.1 - # via - # data-inclusion-api (setup.py) - # pandas + # via pandas pyyaml==6.0.1 # via # pre-commit @@ -228,13 +193,12 @@ pyyaml==6.0.1 pyzstd==0.15.10 # via py7zr requests==2.31.0 - # via data-inclusion-api (setup.py) rsa==4.9 # via python-jose ruff==0.3.0 - # via data-inclusion-api (setup.py) -sentry-sdk[fastapi]==1.40.6 - # via data-inclusion-api (setup.py) +sentry-sdk==1.40.6 +setuptools==69.5.1 + # via nodeenv shapely==2.0.3 # via geopandas six==1.16.0 @@ -251,7 +215,6 @@ sniffio==1.3.1 sqlalchemy==2.0.28 # via # alembic - # data-inclusion-api (setup.py) # geoalchemy2 sqlparse==0.4.4 # via fastapi-debug-toolbar @@ -260,9 +223,7 @@ starlette==0.36.3 texttable==1.7.0 # via py7zr tox==4.13.0 - # via data-inclusion-api (setup.py) tqdm==4.66.2 - # via data-inclusion-api (setup.py) typing-extensions==4.10.0 # via # alembic @@ -279,8 +240,7 @@ urllib3==2.2.1 # minio # requests # sentry-sdk -uvicorn[standard]==0.27.1 - # via data-inclusion-api (setup.py) +uvicorn==0.27.1 uvloop==0.19.0 # via uvicorn virtualenv==20.25.1 @@ -291,6 +251,3 @@ watchfiles==0.21.0 # via uvicorn websockets==12.0 # via uvicorn - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/api/requirements/requirements.txt b/api/requirements/requirements.txt index a9dcb8a3..1ec9884e 100644 --- a/api/requirements/requirements.txt +++ b/api/requirements/requirements.txt @@ -1,11 +1,6 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --output-file=requirements/requirements.txt -# +# This file was autogenerated by uv via the following command: +# uv pip compile setup.py --output-file=requirements/requirements.txt alembic==1.13.1 - # via data-inclusion-api (setup.py) annotated-types==0.6.0 # via pydantic anyio==4.3.0 @@ -23,7 +18,6 @@ brotli==1.1.0 # via py7zr certifi==2024.2.2 # via - # data-inclusion-api (setup.py) # fiona # httpcore # httpx @@ -41,7 +35,6 @@ click==8.1.7 # via # click-plugins # cligj - # data-inclusion-api (setup.py) # fiona # uvicorn click-plugins==1.1.1 @@ -49,11 +42,8 @@ click-plugins==1.1.1 cligj==0.7.2 # via fiona cryptography==42.0.5 - # via - # data-inclusion-api (setup.py) - # python-jose + # via python-jose data-inclusion-schema==0.14.0 - # via data-inclusion-api (setup.py) dnspython==2.6.1 # via email-validator ecdsa==0.18.0 @@ -61,26 +51,19 @@ ecdsa==0.18.0 email-validator==2.1.1 # via pydantic faker==23.3.0 - # via data-inclusion-api (setup.py) fastapi==0.110.0 # via - # data-inclusion-api (setup.py) # fastapi-pagination # sentry-sdk fastapi-pagination==0.12.17 - # via data-inclusion-api (setup.py) fiona==1.9.6 # via geopandas furl==2.1.3 - # via data-inclusion-api (setup.py) geoalchemy2==0.14.6 - # via data-inclusion-api (setup.py) geopandas==0.14.3 - # via data-inclusion-api (setup.py) greenlet==3.0.3 # via sqlalchemy gunicorn==21.2.0 - # via data-inclusion-api (setup.py) h11==0.14.0 # via # httpcore @@ -90,7 +73,6 @@ httpcore==1.0.4 httptools==0.6.1 # via uvicorn httpx==0.27.0 - # via data-inclusion-api (setup.py) idna==3.6 # via # anyio @@ -104,12 +86,10 @@ mako==1.3.2 markupsafe==2.1.5 # via mako minio==7.2.5 - # via data-inclusion-api (setup.py) multivolumefile==0.2.3 # via py7zr numpy==1.26.4 # via - # data-inclusion-api (setup.py) # pandas # pyarrow # shapely @@ -121,17 +101,12 @@ packaging==23.2 # geopandas # gunicorn pandas==2.2.1 - # via - # data-inclusion-api (setup.py) - # geopandas + # via geopandas psutil==5.9.8 # via py7zr psycopg2==2.9.9 - # via data-inclusion-api (setup.py) py7zr==0.21.0 - # via data-inclusion-api (setup.py) pyarrow==15.0.0 - # via data-inclusion-api (setup.py) pyasn1==0.5.1 # via # python-jose @@ -144,18 +119,15 @@ pycryptodome==3.20.0 # via minio pycryptodomex==3.20.0 # via py7zr -pydantic[email]==2.6.3 +pydantic==2.6.3 # via - # data-inclusion-api (setup.py) # data-inclusion-schema # fastapi # fastapi-pagination - # pydantic # pydantic-settings pydantic-core==2.16.3 # via pydantic pydantic-settings==2.2.1 - # via data-inclusion-api (setup.py) pyppmd==1.1.0 # via py7zr pyproj==3.6.1 @@ -166,25 +138,19 @@ python-dateutil==2.9.0.post0 # pandas python-dotenv==1.0.1 # via - # data-inclusion-api (setup.py) # pydantic-settings # uvicorn -python-jose[cryptography]==3.3.0 - # via data-inclusion-api (setup.py) +python-jose==3.3.0 pytz==2024.1 - # via - # data-inclusion-api (setup.py) - # pandas + # via pandas pyyaml==6.0.1 # via uvicorn pyzstd==0.15.10 # via py7zr requests==2.31.0 - # via data-inclusion-api (setup.py) rsa==4.9 # via python-jose -sentry-sdk[fastapi]==1.40.6 - # via data-inclusion-api (setup.py) +sentry-sdk==1.40.6 shapely==2.0.3 # via geopandas six==1.16.0 @@ -201,14 +167,12 @@ sniffio==1.3.1 sqlalchemy==2.0.28 # via # alembic - # data-inclusion-api (setup.py) # geoalchemy2 starlette==0.36.3 # via fastapi texttable==1.7.0 # via py7zr tqdm==4.66.2 - # via data-inclusion-api (setup.py) typing-extensions==4.10.0 # via # alembic @@ -225,8 +189,7 @@ urllib3==2.2.1 # minio # requests # sentry-sdk -uvicorn[standard]==0.27.1 - # via data-inclusion-api (setup.py) +uvicorn==0.27.1 uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 diff --git a/api/requirements/test-requirements.txt b/api/requirements/test-requirements.txt index 4c46a0b3..ae603afc 100644 --- a/api/requirements/test-requirements.txt +++ b/api/requirements/test-requirements.txt @@ -1,11 +1,6 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --extra=test --output-file=requirements/test-requirements.txt -# +# This file was autogenerated by uv via the following command: +# uv pip compile setup.py --extra=test --output-file=requirements/test-requirements.txt alembic==1.13.1 - # via data-inclusion-api (setup.py) annotated-types==0.6.0 # via pydantic anyio==4.3.0 @@ -24,7 +19,6 @@ brotli==1.1.0 # via py7zr certifi==2024.2.2 # via - # data-inclusion-api (setup.py) # fiona # httpcore # httpx @@ -42,7 +36,6 @@ click==8.1.7 # via # click-plugins # cligj - # data-inclusion-api (setup.py) # fiona # uvicorn click-plugins==1.1.1 @@ -50,11 +43,8 @@ click-plugins==1.1.1 cligj==0.7.2 # via fiona cryptography==42.0.5 - # via - # data-inclusion-api (setup.py) - # python-jose + # via python-jose data-inclusion-schema==0.14.0 - # via data-inclusion-api (setup.py) dnspython==2.6.1 # via email-validator ecdsa==0.18.0 @@ -62,33 +52,23 @@ ecdsa==0.18.0 email-validator==2.1.1 # via pydantic factory-boy==3.3.0 - # via data-inclusion-api (setup.py) faker==23.3.0 - # via - # data-inclusion-api (setup.py) - # factory-boy + # via factory-boy fastapi==0.110.0 # via - # data-inclusion-api (setup.py) # fastapi-debug-toolbar # fastapi-pagination # sentry-sdk fastapi-debug-toolbar==0.6.2 - # via data-inclusion-api (setup.py) fastapi-pagination==0.12.17 - # via data-inclusion-api (setup.py) fiona==1.9.5 # via geopandas furl==2.1.3 - # via data-inclusion-api (setup.py) geoalchemy2==0.14.6 - # via data-inclusion-api (setup.py) geopandas==0.14.3 - # via data-inclusion-api (setup.py) greenlet==3.0.3 # via sqlalchemy gunicorn==21.2.0 - # via data-inclusion-api (setup.py) h11==0.14.0 # via # httpcore @@ -98,7 +78,6 @@ httpcore==1.0.4 httptools==0.6.1 # via uvicorn httpx==0.27.0 - # via data-inclusion-api (setup.py) idna==3.6 # via # anyio @@ -118,12 +97,10 @@ markupsafe==2.1.5 # jinja2 # mako minio==7.2.5 - # via data-inclusion-api (setup.py) multivolumefile==0.2.3 # via py7zr numpy==1.26.4 # via - # data-inclusion-api (setup.py) # pandas # pyarrow # shapely @@ -136,19 +113,14 @@ packaging==23.2 # gunicorn # pytest pandas==2.2.1 - # via - # data-inclusion-api (setup.py) - # geopandas + # via geopandas pluggy==1.4.0 # via pytest psutil==5.9.8 # via py7zr psycopg2==2.9.9 - # via data-inclusion-api (setup.py) py7zr==0.21.0 - # via data-inclusion-api (setup.py) pyarrow==15.0.0 - # via data-inclusion-api (setup.py) pyasn1==0.5.1 # via # python-jose @@ -161,14 +133,12 @@ pycryptodome==3.20.0 # via minio pycryptodomex==3.20.0 # via py7zr -pydantic[email]==2.6.3 +pydantic==2.6.3 # via - # data-inclusion-api (setup.py) # data-inclusion-schema # fastapi # fastapi-debug-toolbar # fastapi-pagination - # pydantic # pydantic-extra-types # pydantic-settings pydantic-core==2.16.3 @@ -176,9 +146,7 @@ pydantic-core==2.16.3 pydantic-extra-types==2.6.0 # via fastapi-debug-toolbar pydantic-settings==2.2.1 - # via - # data-inclusion-api (setup.py) - # fastapi-debug-toolbar + # via fastapi-debug-toolbar pyinstrument==4.6.2 # via fastapi-debug-toolbar pyppmd==1.1.0 @@ -187,37 +155,31 @@ pyproj==3.6.1 # via geopandas pytest==8.1.0 # via - # data-inclusion-api (setup.py) # pytest-dotenv # syrupy pytest-dotenv==0.5.2 - # via data-inclusion-api (setup.py) python-dateutil==2.9.0.post0 # via # faker # pandas python-dotenv==1.0.1 # via - # data-inclusion-api (setup.py) # pydantic-settings # pytest-dotenv # uvicorn -python-jose[cryptography]==3.3.0 - # via data-inclusion-api (setup.py) +python-jose==3.3.0 pytz==2024.1 - # via - # data-inclusion-api (setup.py) - # pandas + # via pandas pyyaml==6.0.1 # via uvicorn pyzstd==0.15.10 # via py7zr requests==2.31.0 - # via data-inclusion-api (setup.py) rsa==4.9 # via python-jose -sentry-sdk[fastapi]==1.40.6 - # via data-inclusion-api (setup.py) +sentry-sdk==1.40.6 +setuptools==69.5.1 + # via fiona shapely==2.0.3 # via geopandas six==1.16.0 @@ -234,18 +196,15 @@ sniffio==1.3.1 sqlalchemy==2.0.28 # via # alembic - # data-inclusion-api (setup.py) # geoalchemy2 sqlparse==0.4.4 # via fastapi-debug-toolbar starlette==0.36.3 # via fastapi syrupy==4.6.1 - # via data-inclusion-api (setup.py) texttable==1.7.0 # via py7zr tqdm==4.66.2 - # via data-inclusion-api (setup.py) typing-extensions==4.10.0 # via # alembic @@ -262,14 +221,10 @@ urllib3==2.2.1 # minio # requests # sentry-sdk -uvicorn[standard]==0.27.1 - # via data-inclusion-api (setup.py) +uvicorn==0.27.1 uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 # via uvicorn websockets==12.0 # via uvicorn - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/api/setup.py b/api/setup.py index a13e61bc..db5f734c 100644 --- a/api/setup.py +++ b/api/setup.py @@ -50,10 +50,11 @@ "syrupy", ], "dev": [ + "fastapi-debug-toolbar", + "pre-commit", "ruff", "tox", - "pre-commit", - "fastapi-debug-toolbar", + "uv", ], }, ) From cccbba1a905432fee3283a3f731204972b1c249c Mon Sep 17 00:00:00 2001 From: Victor Perron Date: Thu, 18 Apr 2024 15:46:54 +0200 Subject: [PATCH 3/4] chore(api) : Bump the dependencies --- api/requirements/dev-requirements.txt | 61 +++++++++++++------------- api/requirements/requirements.txt | 44 +++++++++---------- api/requirements/test-requirements.txt | 54 +++++++++++------------ 3 files changed, 79 insertions(+), 80 deletions(-) diff --git a/api/requirements/dev-requirements.txt b/api/requirements/dev-requirements.txt index 3eb9bb75..2174a2f0 100644 --- a/api/requirements/dev-requirements.txt +++ b/api/requirements/dev-requirements.txt @@ -57,42 +57,42 @@ distlib==0.3.8 # via virtualenv dnspython==2.6.1 # via email-validator -ecdsa==0.18.0 +ecdsa==0.19.0 # via python-jose email-validator==2.1.1 # via pydantic -faker==23.3.0 -fastapi==0.110.0 +faker==24.11.0 +fastapi==0.110.2 # via # fastapi-debug-toolbar # fastapi-pagination # sentry-sdk fastapi-debug-toolbar==0.6.2 -fastapi-pagination==0.12.17 -filelock==3.13.1 +fastapi-pagination==0.12.23 +filelock==3.13.4 # via # tox # virtualenv fiona==1.9.6 # via geopandas furl==2.1.3 -geoalchemy2==0.14.6 +geoalchemy2==0.14.7 geopandas==0.14.3 greenlet==3.0.3 # via sqlalchemy -gunicorn==21.2.0 +gunicorn==22.0.0 h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.4 +httpcore==1.0.5 # via httpx httptools==0.6.1 # via uvicorn httpx==0.27.0 -identify==2.5.35 +identify==2.5.36 # via pre-commit -idna==3.6 +idna==3.7 # via # anyio # email-validator @@ -102,7 +102,7 @@ inflate64==1.0.0 # via py7zr jinja2==3.1.3 # via fastapi-debug-toolbar -mako==1.3.2 +mako==1.3.3 # via alembic markupsafe==2.1.5 # via @@ -120,40 +120,40 @@ numpy==1.26.4 # shapely orderedmultidict==1.0.1 # via furl -packaging==23.2 +packaging==24.0 # via # geoalchemy2 # geopandas # gunicorn # pyproject-api # tox -pandas==2.2.1 +pandas==2.2.2 # via geopandas platformdirs==4.2.0 # via # tox # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via tox -pre-commit==3.6.2 +pre-commit==3.7.0 psutil==5.9.8 # via py7zr psycopg2==2.9.9 py7zr==0.21.0 -pyarrow==15.0.0 -pyasn1==0.5.1 +pyarrow==16.0.0 +pyasn1==0.6.0 # via # python-jose # rsa pybcj==1.0.2 # via py7zr -pycparser==2.21 +pycparser==2.22 # via cffi pycryptodome==3.20.0 # via minio pycryptodomex==3.20.0 # via py7zr -pydantic==2.6.3 +pydantic==2.7.0 # via # data-inclusion-schema # fastapi @@ -161,7 +161,7 @@ pydantic==2.6.3 # fastapi-pagination # pydantic-extra-types # pydantic-settings -pydantic-core==2.16.3 +pydantic-core==2.18.1 # via pydantic pydantic-extra-types==2.6.0 # via fastapi-debug-toolbar @@ -195,11 +195,11 @@ pyzstd==0.15.10 requests==2.31.0 rsa==4.9 # via python-jose -ruff==0.3.0 -sentry-sdk==1.40.6 +ruff==0.4.1 +sentry-sdk==1.45.0 setuptools==69.5.1 # via nodeenv -shapely==2.0.3 +shapely==2.0.4 # via geopandas six==1.16.0 # via @@ -212,19 +212,19 @@ sniffio==1.3.1 # via # anyio # httpx -sqlalchemy==2.0.28 +sqlalchemy==2.0.29 # via # alembic # geoalchemy2 -sqlparse==0.4.4 +sqlparse==0.5.0 # via fastapi-debug-toolbar -starlette==0.36.3 +starlette==0.37.2 # via fastapi texttable==1.7.0 # via py7zr -tox==4.13.0 +tox==4.14.2 tqdm==4.66.2 -typing-extensions==4.10.0 +typing-extensions==4.11.0 # via # alembic # fastapi @@ -240,10 +240,11 @@ urllib3==2.2.1 # minio # requests # sentry-sdk -uvicorn==0.27.1 +uv==0.1.35 +uvicorn==0.29.0 uvloop==0.19.0 # via uvicorn -virtualenv==20.25.1 +virtualenv==20.25.3 # via # pre-commit # tox diff --git a/api/requirements/requirements.txt b/api/requirements/requirements.txt index 1ec9884e..25edd520 100644 --- a/api/requirements/requirements.txt +++ b/api/requirements/requirements.txt @@ -46,34 +46,34 @@ cryptography==42.0.5 data-inclusion-schema==0.14.0 dnspython==2.6.1 # via email-validator -ecdsa==0.18.0 +ecdsa==0.19.0 # via python-jose email-validator==2.1.1 # via pydantic -faker==23.3.0 -fastapi==0.110.0 +faker==24.11.0 +fastapi==0.110.2 # via # fastapi-pagination # sentry-sdk -fastapi-pagination==0.12.17 +fastapi-pagination==0.12.23 fiona==1.9.6 # via geopandas furl==2.1.3 -geoalchemy2==0.14.6 +geoalchemy2==0.14.7 geopandas==0.14.3 greenlet==3.0.3 # via sqlalchemy -gunicorn==21.2.0 +gunicorn==22.0.0 h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.4 +httpcore==1.0.5 # via httpx httptools==0.6.1 # via uvicorn httpx==0.27.0 -idna==3.6 +idna==3.7 # via # anyio # email-validator @@ -81,7 +81,7 @@ idna==3.6 # requests inflate64==1.0.0 # via py7zr -mako==1.3.2 +mako==1.3.3 # via alembic markupsafe==2.1.5 # via mako @@ -95,37 +95,37 @@ numpy==1.26.4 # shapely orderedmultidict==1.0.1 # via furl -packaging==23.2 +packaging==24.0 # via # geoalchemy2 # geopandas # gunicorn -pandas==2.2.1 +pandas==2.2.2 # via geopandas psutil==5.9.8 # via py7zr psycopg2==2.9.9 py7zr==0.21.0 -pyarrow==15.0.0 -pyasn1==0.5.1 +pyarrow==16.0.0 +pyasn1==0.6.0 # via # python-jose # rsa pybcj==1.0.2 # via py7zr -pycparser==2.21 +pycparser==2.22 # via cffi pycryptodome==3.20.0 # via minio pycryptodomex==3.20.0 # via py7zr -pydantic==2.6.3 +pydantic==2.7.0 # via # data-inclusion-schema # fastapi # fastapi-pagination # pydantic-settings -pydantic-core==2.16.3 +pydantic-core==2.18.1 # via pydantic pydantic-settings==2.2.1 pyppmd==1.1.0 @@ -150,8 +150,8 @@ pyzstd==0.15.10 requests==2.31.0 rsa==4.9 # via python-jose -sentry-sdk==1.40.6 -shapely==2.0.3 +sentry-sdk==1.45.0 +shapely==2.0.4 # via geopandas six==1.16.0 # via @@ -164,16 +164,16 @@ sniffio==1.3.1 # via # anyio # httpx -sqlalchemy==2.0.28 +sqlalchemy==2.0.29 # via # alembic # geoalchemy2 -starlette==0.36.3 +starlette==0.37.2 # via fastapi texttable==1.7.0 # via py7zr tqdm==4.66.2 -typing-extensions==4.10.0 +typing-extensions==4.11.0 # via # alembic # fastapi @@ -189,7 +189,7 @@ urllib3==2.2.1 # minio # requests # sentry-sdk -uvicorn==0.27.1 +uvicorn==0.29.0 uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 diff --git a/api/requirements/test-requirements.txt b/api/requirements/test-requirements.txt index ae603afc..26894f89 100644 --- a/api/requirements/test-requirements.txt +++ b/api/requirements/test-requirements.txt @@ -47,38 +47,38 @@ cryptography==42.0.5 data-inclusion-schema==0.14.0 dnspython==2.6.1 # via email-validator -ecdsa==0.18.0 +ecdsa==0.19.0 # via python-jose email-validator==2.1.1 # via pydantic factory-boy==3.3.0 -faker==23.3.0 +faker==24.11.0 # via factory-boy -fastapi==0.110.0 +fastapi==0.110.2 # via # fastapi-debug-toolbar # fastapi-pagination # sentry-sdk fastapi-debug-toolbar==0.6.2 -fastapi-pagination==0.12.17 -fiona==1.9.5 +fastapi-pagination==0.12.23 +fiona==1.9.6 # via geopandas furl==2.1.3 -geoalchemy2==0.14.6 +geoalchemy2==0.14.7 geopandas==0.14.3 greenlet==3.0.3 # via sqlalchemy -gunicorn==21.2.0 +gunicorn==22.0.0 h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.4 +httpcore==1.0.5 # via httpx httptools==0.6.1 # via uvicorn httpx==0.27.0 -idna==3.6 +idna==3.7 # via # anyio # email-validator @@ -90,7 +90,7 @@ iniconfig==2.0.0 # via pytest jinja2==3.1.3 # via fastapi-debug-toolbar -mako==1.3.2 +mako==1.3.3 # via alembic markupsafe==2.1.5 # via @@ -106,34 +106,34 @@ numpy==1.26.4 # shapely orderedmultidict==1.0.1 # via furl -packaging==23.2 +packaging==24.0 # via # geoalchemy2 # geopandas # gunicorn # pytest -pandas==2.2.1 +pandas==2.2.2 # via geopandas -pluggy==1.4.0 +pluggy==1.5.0 # via pytest psutil==5.9.8 # via py7zr psycopg2==2.9.9 py7zr==0.21.0 -pyarrow==15.0.0 -pyasn1==0.5.1 +pyarrow==16.0.0 +pyasn1==0.6.0 # via # python-jose # rsa pybcj==1.0.2 # via py7zr -pycparser==2.21 +pycparser==2.22 # via cffi pycryptodome==3.20.0 # via minio pycryptodomex==3.20.0 # via py7zr -pydantic==2.6.3 +pydantic==2.7.0 # via # data-inclusion-schema # fastapi @@ -141,7 +141,7 @@ pydantic==2.6.3 # fastapi-pagination # pydantic-extra-types # pydantic-settings -pydantic-core==2.16.3 +pydantic-core==2.18.1 # via pydantic pydantic-extra-types==2.6.0 # via fastapi-debug-toolbar @@ -153,7 +153,7 @@ pyppmd==1.1.0 # via py7zr pyproj==3.6.1 # via geopandas -pytest==8.1.0 +pytest==8.1.1 # via # pytest-dotenv # syrupy @@ -177,10 +177,8 @@ pyzstd==0.15.10 requests==2.31.0 rsa==4.9 # via python-jose -sentry-sdk==1.40.6 -setuptools==69.5.1 - # via fiona -shapely==2.0.3 +sentry-sdk==1.45.0 +shapely==2.0.4 # via geopandas six==1.16.0 # via @@ -193,19 +191,19 @@ sniffio==1.3.1 # via # anyio # httpx -sqlalchemy==2.0.28 +sqlalchemy==2.0.29 # via # alembic # geoalchemy2 -sqlparse==0.4.4 +sqlparse==0.5.0 # via fastapi-debug-toolbar -starlette==0.36.3 +starlette==0.37.2 # via fastapi syrupy==4.6.1 texttable==1.7.0 # via py7zr tqdm==4.66.2 -typing-extensions==4.10.0 +typing-extensions==4.11.0 # via # alembic # fastapi @@ -221,7 +219,7 @@ urllib3==2.2.1 # minio # requests # sentry-sdk -uvicorn==0.27.1 +uvicorn==0.29.0 uvloop==0.19.0 # via uvicorn watchfiles==0.21.0 From 1238ba2c3018188d9e722a9fef1cfa0b0b48eda8 Mon Sep 17 00:00:00 2001 From: Victor Perron Date: Thu, 18 Apr 2024 18:36:23 +0200 Subject: [PATCH 4/4] chore(api) : Use typed pagination fatsapi-pagination introduced a deprecating change in 0.12.18 https://uriyyo-fastapi-pagination.netlify.app/tutorials_advanced/customization/ In order to keep the same API and be future-proof, let's update. --- .../api/inclusion_data/routes.py | 14 ++++------- .../data_inclusion/api/utils/pagination.py | 25 +++++++++++++------ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/api/src/data_inclusion/api/inclusion_data/routes.py b/api/src/data_inclusion/api/inclusion_data/routes.py index e89c1d47..7ec85918 100644 --- a/api/src/data_inclusion/api/inclusion_data/routes.py +++ b/api/src/data_inclusion/api/inclusion_data/routes.py @@ -30,7 +30,7 @@ @router.get( "/structures", - response_model=pagination.Page[schemas.Structure], + response_model=pagination.BigPage[schemas.Structure], summary="Lister les structures consolidées", deprecated=True, dependencies=[auth.authenticated_dependency] if settings.TOKEN_ENABLED else [], @@ -45,9 +45,7 @@ def list_structures_endpoint( ] = None, thematique: Annotated[Optional[di_schema.Thematique], fastapi.Query()] = None, departement: Annotated[Optional[DepartementCOG], fastapi.Query()] = None, - departement_slug: Annotated[ - Optional[DepartementSlug], fastapi.Query() - ] = None, + departement_slug: Annotated[Optional[DepartementSlug], fastapi.Query()] = None, code_postal: Annotated[Optional[di_schema.CodePostal], fastapi.Query()] = None, db_session=fastapi.Depends(db.get_session), ): @@ -94,7 +92,7 @@ def list_sources_endpoint( @router.get( "/services", - response_model=pagination.Page[schemas.Service], + response_model=pagination.BigPage[schemas.Service], summary="Lister les services consolidés", deprecated=True, dependencies=[auth.authenticated_dependency] if settings.TOKEN_ENABLED else [], @@ -105,9 +103,7 @@ def list_services_endpoint( source: Annotated[Optional[str], fastapi.Query()] = None, thematique: Annotated[Optional[di_schema.Thematique], fastapi.Query()] = None, departement: Annotated[Optional[DepartementCOG], fastapi.Query()] = None, - departement_slug: Annotated[ - Optional[DepartementSlug], fastapi.Query() - ] = None, + departement_slug: Annotated[Optional[DepartementSlug], fastapi.Query()] = None, code_insee: Annotated[Optional[di_schema.CodeCommune], fastapi.Query()] = None, ): return services.list_services( @@ -176,7 +172,7 @@ def redirect_service_endpoint( @router.get( "/search/services", - response_model=pagination.Page[schemas.ServiceSearchResult], + response_model=pagination.BigPage[schemas.ServiceSearchResult], summary="Rechercher des services", dependencies=[auth.authenticated_dependency] if settings.TOKEN_ENABLED else [], ) diff --git a/api/src/data_inclusion/api/utils/pagination.py b/api/src/data_inclusion/api/utils/pagination.py index 86616f8a..6d0f04e3 100644 --- a/api/src/data_inclusion/api/utils/pagination.py +++ b/api/src/data_inclusion/api/utils/pagination.py @@ -1,13 +1,22 @@ import fastapi -import fastapi_pagination +from fastapi_pagination import Page +from fastapi_pagination.customization import ( + CustomizedPage, + UseName, + UseParamsFields, +) from data_inclusion.api.config import settings -Page = fastapi_pagination.Page.with_custom_options( - size=fastapi.Query( - default=settings.DEFAULT_PAGE_SIZE, - ge=1, - le=settings.MAX_PAGE_SIZE, - description="Page size", +BigPage = CustomizedPage[ + Page, + UseName("CustomizedPage"), + UseParamsFields( + size=fastapi.Query( + default=settings.DEFAULT_PAGE_SIZE, + ge=1, + le=settings.MAX_PAGE_SIZE, + description="Page size", + ) ), -) +]