diff --git a/api/CONTRIBUTING.md b/api/CONTRIBUTING.md index cc27ed61..90881cc5 100644 --- a/api/CONTRIBUTING.md +++ b/api/CONTRIBUTING.md @@ -61,7 +61,7 @@ After running the main dag: source .venv/bin/activate # Launch command to import the Admin Express database -python src/data_inclusion/api/cli.py import_admin_express +python src/data_inclusion/api/cli.py import_communes # Launch command to import data python src/data_inclusion/api/cli.py load_inclusion_data diff --git a/api/requirements/dev-requirements.txt b/api/requirements/dev-requirements.txt index 2b323a9b..155a39a4 100644 --- a/api/requirements/dev-requirements.txt +++ b/api/requirements/dev-requirements.txt @@ -44,7 +44,7 @@ colorama==0.4.6 # via tox cryptography==43.0.3 # via data-inclusion-api (setup.py) -data-inclusion-schema==0.17.0 +data-inclusion-schema==0.20.0.dev1 # via data-inclusion-api (setup.py) distlib==0.3.9 # via virtualenv @@ -128,6 +128,8 @@ pandas==2.2.3 # via # data-inclusion-api (setup.py) # geopandas +pendulum==3.0.0 + # via data-inclusion-schema platformdirs==4.3.6 # via # tox @@ -172,7 +174,10 @@ pyproj==3.7.0 pyproject-api==1.8.0 # via tox python-dateutil==2.9.0.post0 - # via pandas + # via + # pandas + # pendulum + # time-machine python-dotenv==1.0.1 # via # data-inclusion-api (setup.py) @@ -212,6 +217,8 @@ sqlparse==0.5.1 # via fastapi-debug-toolbar starlette==0.41.2 # via fastapi +time-machine==2.16.0 + # via pendulum tox==4.23.2 # via data-inclusion-api (setup.py) tqdm==4.66.6 @@ -226,7 +233,9 @@ typing-extensions==4.12.2 # pydantic-core # sqlalchemy tzdata==2024.2 - # via pandas + # via + # pandas + # pendulum urllib3==2.2.3 # via # minio diff --git a/api/requirements/requirements.txt b/api/requirements/requirements.txt index 6d34ea38..a78ac49c 100644 --- a/api/requirements/requirements.txt +++ b/api/requirements/requirements.txt @@ -35,7 +35,7 @@ click==8.1.7 # uvicorn cryptography==43.0.3 # via data-inclusion-api (setup.py) -data-inclusion-schema==0.17.0 +data-inclusion-schema==0.20.0.dev1 # via data-inclusion-api (setup.py) dnspython==2.7.0 # via email-validator @@ -102,6 +102,8 @@ pandas==2.2.3 # via # data-inclusion-api (setup.py) # geopandas +pendulum==3.0.0 + # via data-inclusion-schema psycopg2==2.9.10 # via data-inclusion-api (setup.py) pyarrow==18.0.0 @@ -128,7 +130,10 @@ pyogrio==0.10.0 pyproj==3.7.0 # via geopandas python-dateutil==2.9.0.post0 - # via pandas + # via + # pandas + # pendulum + # time-machine python-dotenv==1.0.1 # via # data-inclusion-api (setup.py) @@ -162,6 +167,8 @@ sqlalchemy==2.0.36 # geoalchemy2 starlette==0.41.2 # via fastapi +time-machine==2.16.0 + # via pendulum tqdm==4.66.6 # via data-inclusion-api (setup.py) typing-extensions==4.12.2 @@ -174,7 +181,9 @@ typing-extensions==4.12.2 # pydantic-core # sqlalchemy tzdata==2024.2 - # via pandas + # via + # pandas + # pendulum urllib3==2.2.3 # via # minio diff --git a/api/requirements/test-requirements.txt b/api/requirements/test-requirements.txt index b3493a49..b8d4f203 100644 --- a/api/requirements/test-requirements.txt +++ b/api/requirements/test-requirements.txt @@ -36,7 +36,7 @@ click==8.1.7 # uvicorn cryptography==43.0.3 # via data-inclusion-api (setup.py) -data-inclusion-schema==0.17.0 +data-inclusion-schema==0.20.0.dev1 # via data-inclusion-api (setup.py) dnspython==2.7.0 # via email-validator @@ -117,6 +117,8 @@ pandas==2.2.3 # via # data-inclusion-api (setup.py) # geopandas +pendulum==3.0.0 + # via data-inclusion-schema pluggy==1.5.0 # via pytest psycopg2==2.9.10 @@ -163,6 +165,8 @@ python-dateutil==2.9.0.post0 # via # faker # pandas + # pendulum + # time-machine python-dotenv==1.0.1 # via # data-inclusion-api (setup.py) @@ -201,6 +205,8 @@ starlette==0.41.2 # via fastapi syrupy==4.7.2 # via data-inclusion-api (setup.py) +time-machine==2.16.0 + # via pendulum tqdm==4.66.6 # via data-inclusion-api (setup.py) typing-extensions==4.12.2 @@ -214,7 +220,9 @@ typing-extensions==4.12.2 # pydantic-core # sqlalchemy tzdata==2024.2 - # via pandas + # via + # pandas + # pendulum urllib3==2.2.3 # via # minio diff --git a/api/setup.py b/api/setup.py index d8ef0364..cc314771 100644 --- a/api/setup.py +++ b/api/setup.py @@ -37,7 +37,7 @@ "sqlalchemy", "tqdm", "uvicorn[standard]", - "data-inclusion-schema==0.17.0", + "data-inclusion-schema==0.20.0-dev1", ], extras_require={ "test": [ diff --git a/api/src/alembic/versions/20241028_172223_c947102bb23f_add_profils_autres_field_in_service.py b/api/src/alembic/versions/20241028_172223_c947102bb23f_add_profils_autres_field_in_service.py new file mode 100644 index 00000000..53c14072 --- /dev/null +++ b/api/src/alembic/versions/20241028_172223_c947102bb23f_add_profils_autres_field_in_service.py @@ -0,0 +1,68 @@ +"""add profils_precisions field in service + +Revision ID: c947102bb23f +Revises: 68fe052dc63c +Create Date: 2024-10-28 17:22:23.374004 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import TSVECTOR + +# revision identifiers, used by Alembic. +revision = "c947102bb23f" +down_revision = "68fe052dc63c" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "api__services", + sa.Column( + "profils_precisions", + sa.String(), + nullable=True, + ), + ) + # can't use ARRAY_TO_STRING mutable function in a generation expression. + # So it must be overiden by an immutable function + op.execute(""" + CREATE OR REPLACE FUNCTION generate_profils_precisions( + profils_precisions TEXT, + profils TEXT[] + ) + RETURNS TSVECTOR AS $$ + BEGIN + RETURN to_tsvector( + 'french', + COALESCE(profils_precisions, '') || + ' '|| + COALESCE(ARRAY_TO_STRING(profils, ' '), '') + ); + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """) + op.add_column( + "api__services", + sa.Column( + "searchable_index_profils_precisions", + TSVECTOR(), + sa.Computed( + "generate_profils_precisions(profils_precisions, profils)", + persisted=True, + ), + ), + ) + op.create_index( + "ix_api__services_searchable_index_profils_precisions", + "api__services", + ["searchable_index_profils_precisions"], + postgresql_using="gin", + ) + + +def downgrade() -> None: + op.drop_column("api__services", "searchable_index_profils_precisions") + op.drop_column("api__services", "profils_precisions") diff --git a/api/src/data_inclusion/api/inclusion_data/models.py b/api/src/data_inclusion/api/inclusion_data/models.py index 894869c9..a6a6f10a 100644 --- a/api/src/data_inclusion/api/inclusion_data/models.py +++ b/api/src/data_inclusion/api/inclusion_data/models.py @@ -1,6 +1,8 @@ from datetime import date import sqlalchemy as sqla +from sqlalchemy import Computed +from sqlalchemy.dialects.postgresql import TSVECTOR from sqlalchemy.orm import Mapped, mapped_column, relationship from data_inclusion.api.core.db import Base @@ -92,6 +94,16 @@ class Service(Base): presentation_resume: Mapped[str | None] prise_rdv: Mapped[str | None] profils: Mapped[list[str] | None] + profils_precisions: Mapped[str | None] + # generate_profils_precisions is a function that generates + # a TSVECTOR from profils_precisions and profils + # cf: 20241028_172223_c947102bb23f_add_profils_autres_field_in_service.py + searchable_index_profils_precisions: Mapped[str | None] = mapped_column( + TSVECTOR, + Computed( + "generate_profils_precisions(profils_precisions, profils)", persisted=True + ), + ) recurrence: Mapped[str | None] source: Mapped[str] structure_id: Mapped[str] diff --git a/api/src/data_inclusion/api/inclusion_data/routes.py b/api/src/data_inclusion/api/inclusion_data/routes.py index a5a2aa73..8b5fdd3a 100644 --- a/api/src/data_inclusion/api/inclusion_data/routes.py +++ b/api/src/data_inclusion/api/inclusion_data/routes.py @@ -354,6 +354,15 @@ def search_services_endpoint( """ ), ] = None, + profils_precisions: Annotated[ + Optional[str], + fastapi.Query( + description="""Une recherche elargie sur les profils. + Chaque résultat renvoyé correspond a la recherche fulltext sur + ce champs. + """ + ), + ] = None, types: Annotated[ Optional[list[di_schema.TypologieService]], fastapi.Query( @@ -420,6 +429,7 @@ def search_services_endpoint( frais=frais, modes_accueil=modes_accueil, profils=profils, + profils_precisions=profils_precisions, types=types, search_point=search_point, include_outdated=inclure_suspendus, diff --git a/api/src/data_inclusion/api/inclusion_data/services.py b/api/src/data_inclusion/api/inclusion_data/services.py index 2c91b660..188e358c 100644 --- a/api/src/data_inclusion/api/inclusion_data/services.py +++ b/api/src/data_inclusion/api/inclusion_data/services.py @@ -7,7 +7,7 @@ import geoalchemy2 import sqlalchemy as sqla -from sqlalchemy import orm +from sqlalchemy import func, orm import fastapi @@ -137,6 +137,17 @@ def filter_services_by_profils( ) +def filter_services_by_profils_precisions( + query: sqla.Select, + profils_precisions: str, +): + return query.filter( + models.Service.searchable_index_profils_precisions.bool_op("@@")( + func.websearch_to_tsquery("french", profils_precisions) + ) + ) + + def filter_services_by_types( query: sqla.Select, types: list[di_schema.TypologieService], @@ -265,6 +276,7 @@ def filter_services( thematiques: list[di_schema.Thematique] | None = None, frais: list[di_schema.Frais] | None = None, profils: list[di_schema.Profil] | None = None, + profils_precisions: str | None = None, modes_accueil: list[di_schema.ModeAccueil] | None = None, types: list[di_schema.TypologieService] | None = None, include_outdated: bool | None = False, @@ -292,6 +304,9 @@ def filter_services( if not include_outdated: query = filter_outdated_services(query) + if profils_precisions is not None: + query = filter_services_by_profils_precisions(query, profils_precisions) + return query @@ -354,6 +369,7 @@ def search_services( frais: list[di_schema.Frais] | None = None, modes_accueil: list[di_schema.ModeAccueil] | None = None, profils: list[di_schema.Profil] | None = None, + profils_precisions: str | None = None, types: list[di_schema.TypologieService] | None = None, search_point: str | None = None, include_outdated: bool | None = False, @@ -454,6 +470,7 @@ def search_services( thematiques=thematiques, frais=frais, profils=profils, + profils_precisions=profils_precisions, modes_accueil=modes_accueil, types=types, include_outdated=include_outdated, diff --git a/api/tests/e2e/api/__snapshots__/test_inclusion_data.ambr b/api/tests/e2e/api/__snapshots__/test_inclusion_data.ambr index 2ad666ba..c2d12bff 100644 --- a/api/tests/e2e/api/__snapshots__/test_inclusion_data.ambr +++ b/api/tests/e2e/api/__snapshots__/test_inclusion_data.ambr @@ -652,6 +652,17 @@ }, "description": "Une liste de profils.\n Chaque résultat renvoyé a (au moins) un profil dans cette liste.\n " }, + { + "name": "profils_precisions", + "in": "query", + "required": false, + "schema": { + "type": "string", + "description": "Une recherche elargie sur les profils.\n Chaque résultat renvoyé correspond a la recherche fulltext sur\n ce champs.\n ", + "title": "Profils Precisions" + }, + "description": "Une recherche elargie sur les profils.\n Chaque résultat renvoyé correspond a la recherche fulltext sur\n ce champs.\n " + }, { "name": "types", "in": "query", @@ -1496,6 +1507,17 @@ ], "title": "Profils" }, + "profils_precisions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Profils Precisions" + }, "pre_requis": { "anyOf": [ { @@ -2415,7 +2437,9 @@ "seniors-65", "sortants-de-detention", "surdite", - "victimes" + "victimes", + "tous-publics", + "personnes-en-situation-durgence" ], "title": "Profil" }, @@ -2590,6 +2614,17 @@ ], "title": "Profils" }, + "profils_precisions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Profils Precisions" + }, "pre_requis": { "anyOf": [ { diff --git a/api/tests/e2e/api/test_inclusion_data.py b/api/tests/e2e/api/test_inclusion_data.py index 5b0950cc..18cbd064 100644 --- a/api/tests/e2e/api/test_inclusion_data.py +++ b/api/tests/e2e/api/test_inclusion_data.py @@ -252,6 +252,7 @@ def test_list_services_all(api_client): "presentation_resume": "Puissant fine.", "prise_rdv": "https://teixeira.fr/", "profils": ["femmes"], + "profils_precisions": "femmes", "recurrence": None, "score_qualite": 0.5, "source": "dora", @@ -369,6 +370,68 @@ def test_list_services_filter_by_categorie_thematique(api_client): assert resp_data["items"][0]["id"] == service.id +@pytest.mark.parametrize( + "profils_precisions,input,found", + [ + ("jeunes moins de 18 ans", "jeunes", True), + ("jeune moins de 18 ans", "jeunes", True), + ("jeunes et personne age", "vieux", False), + ("jeunes et personne age", "personne OR âgée", True), + ("jeunes et personne age", "personne jeune", True), + # FIXME: this test is failing because of the accent in the input + ("jeunes et personne agee", "âgée", False), + ], +) +@pytest.mark.with_token +def test_can_filter_resources_by_profils_precisions( + api_client, profils_precisions, input, found +): + resource = factories.ServiceFactory( + profils=None, profils_precisions=profils_precisions + ) + factories.ServiceFactory(profils=None, profils_precisions="tests") + + response = api_client.get( + "/api/v0/search/services", params={"profils_precisions": input} + ) + + assert response.status_code == 200 + resp_data = response.json() + if found: + assert_paginated_response_data(resp_data, total=1) + assert list_resources_data(resp_data)[0]["id"] in [resource.id] + else: + assert_paginated_response_data(resp_data, total=0) + + +@pytest.mark.parametrize( + "profils,input,found", + [ + ([schema.Profil.FEMMES.value], "femme", True), + ([schema.Profil.JEUNES_16_26.value], "jeune", True), + ([schema.Profil.FEMMES.value], "jeune", False), + ], +) +@pytest.mark.with_token +def test_can_filter_resources_by_profils_precisions_with_only_profils_data( + api_client, profils, input, found +): + resource = factories.ServiceFactory(profils=profils, profils_precisions="") + factories.ServiceFactory(profils=schema.Profil.RETRAITES, profils_precisions="") + + response = api_client.get( + "/api/v0/search/services", params={"profils_precisions": input} + ) + + assert response.status_code == 200 + resp_data = response.json() + if found: + assert_paginated_response_data(resp_data, total=1) + assert list_resources_data(resp_data)[0]["id"] in [resource.id] + else: + assert_paginated_response_data(resp_data, total=0) + + @pytest.mark.parametrize( "thematiques,input,found", [ diff --git a/api/tests/factories.py b/api/tests/factories.py index f0365f2f..7e2ec208 100644 --- a/api/tests/factories.py +++ b/api/tests/factories.py @@ -117,6 +117,7 @@ class Meta: ], getter=lambda v: [v.value], ) + profils_precisions = factory.Faker("text", max_nb_chars=20, locale="fr_FR") pre_requis = [] cumulable = False justificatifs = [] diff --git a/deployment/MIGRATION.md b/deployment/MIGRATION.md index 0c853482..d6831f09 100644 --- a/deployment/MIGRATION.md +++ b/deployment/MIGRATION.md @@ -5,7 +5,7 @@ Here is the corrected and formatted version of your migration process: 1. **Connect via SSH to the instance** ```bash - ssh root@163.172.186.56 + ssh root@ ``` 2. **Install PostgreSQL 17** ```bash