Skip to content

Commit

Permalink
feat(profils): add profils search full text to api
Browse files Browse the repository at this point in the history
  • Loading branch information
@hlecuyer
hlecuyer committed Oct 28, 2024
1 parent b768303 commit 4c7abef
Show file tree
Hide file tree
Showing 28 changed files with 362 additions and 74 deletions.
2 changes: 1 addition & 1 deletion api/CONTRIBUTING.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ After running the main dag:
source .venv/bin/activate

# Launch command to import the Admin Express database
python src/data_inclusion/api/cli.py import_admin_express
python src/data_inclusion/api/cli.py import_communes

# Launch command to import data
python src/data_inclusion/api/cli.py load_inclusion_data
Expand Down
44 changes: 44 additions & 0 deletions api/src/alembic/versions/20241017_105705_9bb9919e6b83_add_profils_autres_field_in_service.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
"""add profils_precisions field in service
Revision ID: 9bb9919e6b83
Revises: e3f3dfa4ad01
Create Date: 2024-10-17 10:57:05.985264
"""

import sqlalchemy as sa
from alembic import op

# revision identifiers, used by Alembic.
revision = "9bb9919e6b83"
down_revision = "e3f3dfa4ad01"
branch_labels = None
depends_on = None


def upgrade() -> None:
op.add_column(
"api__services",
sa.Column(
"profils_precisions",
sa.String(),
nullable=True,
),
)
op.execute(
"""
ALTER TABLE public.api__services
ADD COLUMN searchable_index_profils_precisions tsvector
GENERATED ALWAYS AS (
to_tsvector(
'french'::regconfig,
COALESCE(profils_precisions, ''::character varying)::text
)
) STORED NULL;
"""
)


def downgrade() -> None:
op.drop_column("api__services", "searchable_index_profils_precisions")
op.drop_column("api__services", "profils_precisions")
2 changes: 2 additions & 0 deletions api/src/data_inclusion/api/inclusion_data/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ class Service(Base):
presentation_resume: Mapped[str | None]
prise_rdv: Mapped[str | None]
profils: Mapped[list[str] | None]
profils_precisions: Mapped[str | None]
searchable_index_profils_precisions: Mapped[str | None]
recurrence: Mapped[str | None]
source: Mapped[str]
structure_id: Mapped[str]
Expand Down
10 changes: 10 additions & 0 deletions api/src/data_inclusion/api/inclusion_data/routes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,15 @@ def list_services_endpoint(
"""
),
] = None,
profils_precisions: Annotated[
Optional[str],
fastapi.Query(
description="""Une recherche elargie sur les profils.
Chaque résultat renvoyé correspond a la recherche fulltext sur
ce champs.
"""
),
] = None,
modes_accueil: Annotated[
Optional[list[di_schema.ModeAccueil]],
fastapi.Query(
Expand Down Expand Up @@ -267,6 +276,7 @@ def list_services_endpoint(
code_commune=code_commune,
frais=frais,
profils=profils,
profils_precisions=profils_precisions,
modes_accueil=modes_accueil,
types=types,
include_outdated=inclure_suspendus,
Expand Down
10 changes: 9 additions & 1 deletion api/src/data_inclusion/api/inclusion_data/services.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import geoalchemy2
import sqlalchemy as sqla
from sqlalchemy import orm
from sqlalchemy import func, orm

import fastapi

Expand Down Expand Up @@ -305,6 +305,7 @@ def list_services(
code_commune: di_schema.CodeCommune | None = None,
frais: list[di_schema.Frais] | None = None,
profils: list[di_schema.Profil] | None = None,
profils_precisions: str | None = None,
modes_accueil: list[di_schema.ModeAccueil] | None = None,
types: list[di_schema.TypologieService] | None = None,
include_outdated: bool | None = False,
Expand All @@ -326,6 +327,13 @@ def list_services(
if code_commune is not None:
query = query.filter(models.Service.code_insee == code_commune)

if profils_precisions is not None:
query = query.filter(
models.Service.searchable_index_profils_precisions.bool_op("@@")(
func.websearch_to_tsquery("french", profils_precisions)
)
)

query = filter_services(
query=query,
sources=sources,
Expand Down
2 changes: 1 addition & 1 deletion deployment/MIGRATION.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ Here is the corrected and formatted version of your migration process:

1. **Connect via SSH to the instance**
```bash
ssh [email protected].186.56
ssh [email protected].171.247
```
2. **Install PostgreSQL 17**
```bash
Expand Down
2 changes: 2 additions & 0 deletions pipeline/dbt/macros/domain/checks/check_service.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ CREATE OR REPLACE FUNCTION LIST_SERVICE_ERRORS(
presentation_resume TEXT,
prise_rdv TEXT,
profils TEXT[],
profils_precisions TEXT,
recurrence TEXT,
source TEXT,
structure_id TEXT,
Expand Down Expand Up @@ -111,6 +112,7 @@ WITH final AS (
presentation_resume,
prise_rdv,
profils,
profils_precisions,
recurrence,
source,
structure_id,
Expand Down
1 change: 1 addition & 0 deletions pipeline/dbt/models/intermediate/int__union_services__enhanced.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ valid_services AS (
services.presentation_resume,
services.prise_rdv,
services.profils,
services.profils_precisions,
services.recurrence,
services.source,
services.structure_id,
Expand Down
3 changes: 2 additions & 1 deletion pipeline/dbt/models/intermediate/sources/action_logement/int_action_logement__services.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ final AS (
services.presentation_resume AS "presentation_resume",
services.prise_rdv AS "prise_rdv",
services.profils AS "profils",
ARRAY_TO_STRING(services.profils, '') AS "profils_precisions",
services.pre_requis AS "pre_requis",
services.recurrence AS "recurrence",
services.thematiques AS "thematiques",
Expand All @@ -41,7 +42,7 @@ final AS (
'departement' AS "zone_diffusion_type",
NULL AS "zone_diffusion_code",
NULL AS "zone_diffusion_nom",
CAST(NULL AS DATE) AS "date_suspension",
NULL::DATE AS "date_suspension",
structures.id || '-' || services.id AS "id"
FROM services
CROSS JOIN structures
Expand Down
15 changes: 8 additions & 7 deletions pipeline/dbt/models/intermediate/sources/agefiph/int_agefiph__services.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,13 @@ final AS (
NULL AS "page_web",
'https://www.agefiph.fr' || services.attributes__path__alias AS "modes_orientation_accompagnateur_autres",
'https://www.agefiph.fr' || services.attributes__path__alias AS "modes_orientation_beneficiaire_autres",
CAST(NULL AS TEXT []) AS "justificatifs",
CAST(NULL AS TEXT []) AS "pre_requis",
CAST(NULL AS BOOLEAN) AS "cumulable",
CAST(NULL AS DATE) AS "date_suspension",
NULL::TEXT [] AS "justificatifs",
NULL::TEXT [] AS "pre_requis",
NULL::BOOLEAN AS "cumulable",
NULL::DATE AS "date_suspension",
'https://www.agefiph.fr' || services.attributes__path__alias AS "lien_source",
CAST(services.attributes__created AS DATE) AS "date_creation",
CAST(services.attributes__changed AS DATE) AS "date_maj",
(services.attributes__created)::DATE AS "date_creation",
(services.attributes__changed)::DATE AS "date_maj",
NULLIF(
TRIM(
ARRAY_TO_STRING(
Expand All @@ -107,12 +107,13 @@ final AS (
ARRAY[
'personnes-en-situation-de-handicap'
] AS "profils",
'personnes en situation de handicap' AS "profils_precisions",
ARRAY(
SELECT di_type_by_agefiph_type.type_
FROM di_type_by_agefiph_type
WHERE services.relationships__field_type_aide_service__data__id = di_type_by_agefiph_type.agefiph_type
) AS "types",
CAST(NULL AS TEXT []) AS "frais"
NULL::TEXT [] AS "frais"
FROM
structures
CROSS JOIN services
Expand Down
71 changes: 36 additions & 35 deletions pipeline/dbt/models/intermediate/sources/data_inclusion/int_data_inclusion__services.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,45 +16,46 @@ di_profil_by_dora_profil AS (

final AS (
SELECT
id AS "adresse_id",
contact_public AS "contact_public",
contact_nom AS "contact_nom_prenom",
courriel AS "courriel",
cumulable AS "cumulable",
date_creation::DATE AS "date_creation",
date_maj::DATE AS "date_maj",
date_suspension::DATE AS "date_suspension",
formulaire_en_ligne AS "formulaire_en_ligne",
frais_autres AS "frais_autres",
id AS "id",
justificatifs AS "justificatifs",
NULL AS "lien_source", -- ignored
modes_accueil AS "modes_accueil",
modes_orientation_accompagnateur AS "modes_orientation_accompagnateur",
modes_orientation_accompagnateur_autres AS "modes_orientation_accompagnateur_autres",
modes_orientation_beneficiaire AS "modes_orientation_beneficiaire",
modes_orientation_beneficiaire_autres AS "modes_orientation_beneficiaire_autres",
nom AS "nom",
presentation_resume AS "presentation_resume",
presentation_detail AS "presentation_detail",
prise_rdv AS "prise_rdv",
services.id AS "adresse_id",
services.contact_public AS "contact_public",
services.contact_nom AS "contact_nom_prenom",
services.courriel AS "courriel",
services.cumulable AS "cumulable",
services.date_creation::DATE AS "date_creation",
services.date_maj::DATE AS "date_maj",
services.date_suspension::DATE AS "date_suspension",
services.formulaire_en_ligne AS "formulaire_en_ligne",
services.frais_autres AS "frais_autres",
services.id AS "id",
services.justificatifs AS "justificatifs",
NULL AS "lien_source", -- ignored
services.modes_accueil AS "modes_accueil",
services.modes_orientation_accompagnateur AS "modes_orientation_accompagnateur",
services.modes_orientation_accompagnateur_autres AS "modes_orientation_accompagnateur_autres",
services.modes_orientation_beneficiaire AS "modes_orientation_beneficiaire",
services.modes_orientation_beneficiaire_autres AS "modes_orientation_beneficiaire_autres",
services.nom AS "nom",
services.presentation_resume AS "presentation_resume",
services.presentation_detail AS "presentation_detail",
services.prise_rdv AS "prise_rdv",
ARRAY(
SELECT di_profil_by_dora_profil.di_profil
FROM di_profil_by_dora_profil
WHERE di_profil_by_dora_profil.dora_profil = ANY(services.profils)
)::TEXT [] AS "profils",
recurrence AS "recurrence",
_di_source_id AS "source",
structure_id AS "structure_id",
telephone AS "telephone",
thematiques AS "thematiques",
types AS "types",
zone_diffusion_code AS "zone_diffusion_code",
zone_diffusion_nom AS "zone_diffusion_nom",
zone_diffusion_type AS "zone_diffusion_type",
pre_requis AS "pre_requis",
frais AS "frais",
NULL AS "page_web"
)::TEXT [] AS "profils",
ARRAY_TO_STRING(services.profils, ', ') AS "profils_precisions",
services.recurrence AS "recurrence",
services._di_source_id AS "source",
services.structure_id AS "structure_id",
services.telephone AS "telephone",
services.thematiques AS "thematiques",
services.types AS "types",
services.zone_diffusion_code AS "zone_diffusion_code",
services.zone_diffusion_nom AS "zone_diffusion_nom",
services.zone_diffusion_type AS "zone_diffusion_type",
services.pre_requis AS "pre_requis",
services.frais AS "frais",
NULL AS "page_web"
FROM services
)

Expand Down
1 change: 1 addition & 0 deletions pipeline/dbt/models/intermediate/sources/dora/int_dora__services.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ final AS (
FROM di_profil_by_dora_profil
WHERE di_profil_by_dora_profil.dora_profil = ANY(services.profils)
)::TEXT [] AS "profils",
ARRAY_TO_STRING(services.profils, ', ') AS "profils_precisions",
services.recurrence AS "recurrence",
services._di_source_id AS "source",
services.structure_id AS "structure_id",
Expand Down
3 changes: 2 additions & 1 deletion pipeline/dbt/models/intermediate/sources/france_travail/int_france_travail__services.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ final AS (
services.presentation_resume AS "presentation_resume",
services.prise_rdv AS "prise_rdv",
services.profils AS "profils",
ARRAY_TO_STRING(services.profils, ', ') AS "profils_precisions",
services.pre_requis AS "pre_requis",
services.recurrence AS "recurrence",
services.thematiques AS "thematiques",
Expand All @@ -54,7 +55,7 @@ final AS (
structures.code_insee AS "zone_diffusion_code",
structures.commune AS "zone_diffusion_nom",
NULL AS "page_web",
CAST(NULL AS DATE) AS "date_suspension",
NULL::DATE AS "date_suspension",
structures.id || '-' || services.id AS "id"
FROM services
CROSS JOIN structures_with_commune AS structures
Expand Down
23 changes: 14 additions & 9 deletions pipeline/dbt/models/intermediate/sources/fredo/int_fredo__services.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,10 @@ frais_autres AS (
fredo_frais.structure_id,
STRING_AGG(fredo_frais.value, ', ') AS frais_autres
FROM fredo_frais
WHERE fredo_frais.value NOT IN (SELECT frais_fredo FROM di_frais_by_fredo_frais)
WHERE
fredo_frais.value NOT IN (
SELECT di_frais_by_fredo_frais.frais_fredo FROM di_frais_by_fredo_frais
)
GROUP BY fredo_frais.structure_id
),

Expand All @@ -119,7 +122,8 @@ thematiques AS (
profils AS (
SELECT
fredo_publics.structure_id,
ARRAY_AGG(di_profils_by_fredo_public.profil) AS profils
ARRAY_AGG(di_profils_by_fredo_public.profil) AS profils,
STRING_AGG(fredo_publics.value, ', ') AS profils_precisions
FROM fredo_publics
INNER JOIN di_profils_by_fredo_public ON fredo_publics.value = di_profils_by_fredo_public.public
GROUP BY fredo_publics.structure_id
Expand All @@ -134,6 +138,7 @@ final AS (
NULL AS "prise_rdv",
fredo_structures.site_web AS "page_web",
profils.profils AS "profils",
profils.profils_precisions AS "profils_precisions",
NULL AS "modes_orientation_accompagnateur_autres",
NULL AS "modes_orientation_beneficiaire_autres",
NULL AS "formulaire_en_ligne",
Expand All @@ -144,13 +149,13 @@ final AS (
'La Réunion' AS "zone_diffusion_nom",
NULL AS "recurrence",
fredo_structures.last_update AS "date_maj",
CAST(UUID(MD5(fredo_structures.id || COALESCE(fredo_services.value, ''))) AS TEXT) AS "id",
CAST(NULL AS TEXT []) AS "pre_requis",
CAST(NULL AS BOOLEAN) AS "cumulable",
CAST(NULL AS TEXT []) AS "justificatifs",
CAST(NULL AS DATE) AS "date_creation",
CAST(NULL AS DATE) AS "date_suspension",
CAST(NULL AS BOOLEAN) AS "contact_public",
(UUID(MD5(fredo_structures.id || COALESCE(fredo_services.value, ''))))::TEXT AS "id",
NULL::TEXT [] AS "pre_requis",
NULL::BOOLEAN AS "cumulable",
NULL::TEXT [] AS "justificatifs",
NULL::DATE AS "date_creation",
NULL::DATE AS "date_suspension",
NULL::BOOLEAN AS "contact_public",
CASE
WHEN
fredo_structures.adresse IS NOT NULL
Expand Down
Loading

0 comments on commit 4c7abef

Please sign in to comment.