From 0a7858d25f897dd23d3c68734efea1b814c369a7 Mon Sep 17 00:00:00 2001 From: Hugo Lecuyer Date: Tue, 30 Jul 2024 15:46:19 +0200 Subject: [PATCH] fix after review --- .../sources/fredo/int_fredo__services.sql | 10 +++-- .../sources/fredo/int_fredo__structures.sql | 42 ++++++++++--------- .../staging/sources/fredo/_fredo__models.yml | 2 +- .../sources/fredo/stg_fredo__structures.sql | 7 +++- .../dbt/seeds/schema/labels_nationaux.csv | 3 ++ pipeline/dbt/seeds/schema/modes_accueil.csv | 4 +- .../modes_orientation_accompagnateur.csv | 1 - pipeline/dbt/seeds/schema/thematiques.csv | 1 - .../seeds/schema/typologies_de_structures.csv | 1 - 9 files changed, 40 insertions(+), 31 deletions(-) diff --git a/pipeline/dbt/models/intermediate/sources/fredo/int_fredo__services.sql b/pipeline/dbt/models/intermediate/sources/fredo/int_fredo__services.sql index 12fdcfe3..ef2a5ec8 100644 --- a/pipeline/dbt/models/intermediate/sources/fredo/int_fredo__services.sql +++ b/pipeline/dbt/models/intermediate/sources/fredo/int_fredo__services.sql @@ -132,14 +132,13 @@ final AS ( fredo_structures.id AS "adresse_id", thematiques.thematiques AS "thematiques", NULL AS "prise_rdv", - site_web AS "page_web", + fredo_structures.site_web AS "page_web", profils.profils AS "profils", NULL AS "modes_orientation_accompagnateur_autres", NULL AS "modes_orientation_beneficiaire_autres", NULL AS "formulaire_en_ligne", NULL AS "lien_source", - fredo_structures.telephone AS "telephone", - NULL AS "courriel", + fredo_structures.courriel AS "courriel", NULL AS "contact_nom_prenom", 'departement' AS "zone_diffusion_type", '974' AS "zone_diffusion_code", @@ -156,6 +155,9 @@ final AS ( CAST(NULL AS TEXT []) AS "modes_accueil", LEFT(fredo_structures.presentation_resume, 280) AS "presentation_resume", fredo_structures.presentation_resume AS "presentation_detail", + CASE + WHEN ARRAY_LENGTH(fredo_structures.telephone, 1) > 0 THEN fredo_structures.telephone[1] + END AS "telephone", ARRAY[( SELECT di_types_by_fredo_services.di_type FROM di_types_by_fredo_services @@ -168,7 +170,7 @@ final AS ( ) AS "nom", CASE WHEN 'gratuit' = ANY(frais.frais) AND 'payant' = ANY(frais.frais) - THEN ARRAY['gratuit-sous-conditions'] + THEN ARRAY['gratuit-sous-conditions'] ELSE frais.frais END AS "frais", CASE diff --git a/pipeline/dbt/models/intermediate/sources/fredo/int_fredo__structures.sql b/pipeline/dbt/models/intermediate/sources/fredo/int_fredo__structures.sql index 7cd98bc5..a66bb1b1 100644 --- a/pipeline/dbt/models/intermediate/sources/fredo/int_fredo__structures.sql +++ b/pipeline/dbt/models/intermediate/sources/fredo/int_fredo__structures.sql @@ -35,32 +35,34 @@ di_typologie_by_fredo_type_structure AS ( final AS ( SELECT - id AS "id", - id AS "adresse_id", - _di_source_id AS "source", - siret AS "siret", - NULL AS "rna", - nom AS "nom", - last_update AS "date_maj", + structures.id AS "id", + structures.id AS "adresse_id", + structures._di_source_id AS "source", + structures.siret AS "siret", + NULL AS "rna", + structures.nom AS "nom", + structures.last_update AS "date_maj", ( SELECT di_typologie_by_fredo_type_structure.typologie FROM di_typologie_by_fredo_type_structure INNER JOIN fredo_types AS stg_type ON stg_type.structure_id = structures.id WHERE stg_type.value = di_typologie_by_fredo_type_structure.type_structure LIMIT 1 - ) AS "typologie", - telephone AS "telephone", - NULL AS "courriel", - site_web AS "site_web", - LEFT(presentation_resume, 280) AS "presentation_resume", - presentation_resume AS "presentation_detail", - CAST(NULL AS BOOLEAN) AS "antenne", - NULL AS "lien_source", - horaires_ouverture AS "horaires_ouverture", - NULL AS "accessibilite", - CAST(NULL AS TEXT []) AS "labels_nationaux", - CAST(NULL AS TEXT []) AS "labels_autres", - thematiques.thematiques AS "thematiques" + ) AS "typologie", + CASE + WHEN ARRAY_LENGTH(structures.telephone, 1) > 0 THEN structures.telephone[1] + END AS "telephone", + structures.courriel AS "courriel", + structures.site_web AS "site_web", + LEFT(structures.presentation_resume, 280) AS "presentation_resume", + structures.presentation_resume AS "presentation_detail", + CAST(NULL AS BOOLEAN) AS "antenne", + NULL AS "lien_source", + structures.horaires_ouverture AS "horaires_ouverture", + NULL AS "accessibilite", + CAST(NULL AS TEXT []) AS "labels_nationaux", + CAST(NULL AS TEXT []) AS "labels_autres", + thematiques.thematiques AS "thematiques" FROM structures LEFT JOIN thematiques ON structures.id = thematiques.structure_id ) diff --git a/pipeline/dbt/models/staging/sources/fredo/_fredo__models.yml b/pipeline/dbt/models/staging/sources/fredo/_fredo__models.yml index 956fa393..2ed9408d 100644 --- a/pipeline/dbt/models/staging/sources/fredo/_fredo__models.yml +++ b/pipeline/dbt/models/staging/sources/fredo/_fredo__models.yml @@ -33,7 +33,7 @@ models: - name: horaires_ouverture data_tests: - dbt_utils.not_empty_string - - name: lien_source + - name: site_web data_tests: - dbt_utils.not_empty_string - name: nom diff --git a/pipeline/dbt/models/staging/sources/fredo/stg_fredo__structures.sql b/pipeline/dbt/models/staging/sources/fredo/stg_fredo__structures.sql index 8eca9bcd..b7b12b80 100644 --- a/pipeline/dbt/models/staging/sources/fredo/stg_fredo__structures.sql +++ b/pipeline/dbt/models/staging/sources/fredo/stg_fredo__structures.sql @@ -7,6 +7,7 @@ final AS ( _di_source_id AS "_di_source_id", data ->> 'id' AS "id", NULLIF(TRIM(data ->> 'adresse'), '') AS "adresse", + NULLIF(TRIM(data ->> 'email'), '') AS "courriel", NULLIF(TRIM(data ->> 'code_postal'), '') AS "code_postal", NULLIF(TRIM(data ->> 'commune'), '') AS "commune", NULLIF(TRIM(data ->> 'frais'), '') AS "frais", @@ -16,9 +17,13 @@ final AS ( NULLIF(TRIM(data ->> 'nom'), '') AS "nom", NULLIF(TRIM(data ->> 'presentation_resume'), '') AS "presentation_resume", NULLIF(TRIM(data ->> 'siret'), '') AS "siret", - NULLIF(TRIM(data ->> 'telephone'), '') AS "telephone", CAST(NULLIF(TRIM(data ->> 'latitude'), '') AS FLOAT) AS "latitude", CAST(NULLIF(TRIM(data ->> 'longitude'), '') AS FLOAT) AS "longitude", + CASE + WHEN data ->> 'telephone' IS NOT NULL + THEN CAST(ARRAY(SELECT * FROM JSONB_ARRAY_ELEMENTS_TEXT(data -> 'telephone')) AS TEXT []) + ELSE CAST(NULL AS TEXT []) + END AS "telephone", CASE WHEN data ->> 'categories' IS NOT NULL THEN CAST(ARRAY(SELECT * FROM JSONB_ARRAY_ELEMENTS_TEXT(data -> 'categories')) AS TEXT []) diff --git a/pipeline/dbt/seeds/schema/labels_nationaux.csv b/pipeline/dbt/seeds/schema/labels_nationaux.csv index 509edbd3..ccdb442a 100644 --- a/pipeline/dbt/seeds/schema/labels_nationaux.csv +++ b/pipeline/dbt/seeds/schema/labels_nationaux.csv @@ -34,6 +34,9 @@ emmaus,Emmaus, envie,Envie, epide,EPIDE, espace-emploi-agric-arrco,Espace Emploi Agirc Arrco, +etal-pri,Établissement privé, +etal-pub,Établissement public, +etat,Service de l'État, etcld,Expérimentation territoriale contre le chômage de longue durée, fabrique-de-territoire,Fabrique de Territoire, face,Fondation FACE, diff --git a/pipeline/dbt/seeds/schema/modes_accueil.csv b/pipeline/dbt/seeds/schema/modes_accueil.csv index fd3535e2..135bd3f4 100644 --- a/pipeline/dbt/seeds/schema/modes_accueil.csv +++ b/pipeline/dbt/seeds/schema/modes_accueil.csv @@ -1,3 +1,3 @@ value,label,description -a-distance,À distance, -en-presentiel,En présentiel, +a-distance,À distance,"Le service est proposé à distance, par téléphone ou internet." +en-presentiel,En présentiel,"Le service est proposé en présentiel, à l'adresse associée à ce service." diff --git a/pipeline/dbt/seeds/schema/modes_orientation_accompagnateur.csv b/pipeline/dbt/seeds/schema/modes_orientation_accompagnateur.csv index 3ed61799..d36b8ca3 100644 --- a/pipeline/dbt/seeds/schema/modes_orientation_accompagnateur.csv +++ b/pipeline/dbt/seeds/schema/modes_orientation_accompagnateur.csv @@ -2,7 +2,6 @@ value,label,description autre,Autre, completer-le-formulaire-dadhesion,Compléter le formulaire d’adhésion,Compléter le formulaire d’adhésion associé. envoyer-un-mail,Envoyer un courriel,Envoyer un courriel à l’adresse indiquée. -envoyer-un-mail-avec-des-documents-a-completer,Envoyer un courriel avec des documents à compléter,Envoyer un courriel à l’adresse indiquée avec des documents à compléter. envoyer-un-mail-avec-une-fiche-de-prescription,Envoyer un courriel avec une fiche de prescription,Envoyer un courriel à l’adresse indiquée avec une fiche de prescription. prendre-rdv,Prendre rendez-vous,Prendre rendez-vous auprès de la structure telephoner,Téléphoner,Téléphoner au numéro indiqué. diff --git a/pipeline/dbt/seeds/schema/thematiques.csv b/pipeline/dbt/seeds/schema/thematiques.csv index fa7efb3b..177bbb74 100644 --- a/pipeline/dbt/seeds/schema/thematiques.csv +++ b/pipeline/dbt/seeds/schema/thematiques.csv @@ -133,7 +133,6 @@ sante--diagnostic-et-accompagnement-a-lemployabilite,Diagnostic et accompagnemen sante--faire-face-a-une-situation-daddiction,Faire face à une situation d’addiction, sante--obtenir-la-prise-en-charge-de-frais-medicaux,Obtenir la prise en charge de frais médicaux, sante--prevention-et-acces-aux-soins,"Prévention et accès aux soins (vaccination, éducation à la santé, lutte contre la tuberculose…).", -sante--sante-mental,Accompagnement des aidants, sante--se-soigner-et-prevenir-la-maladie,Se soigner et prévenir la maladie, sante--vie-relationnelle-et-affective,"Vie relationnelle et affective, dépistage et prévention des IST/VIH…", se-former,Se former, diff --git a/pipeline/dbt/seeds/schema/typologies_de_structures.csv b/pipeline/dbt/seeds/schema/typologies_de_structures.csv index 303ef5bd..bbe0efdb 100644 --- a/pipeline/dbt/seeds/schema/typologies_de_structures.csv +++ b/pipeline/dbt/seeds/schema/typologies_de_structures.csv @@ -71,7 +71,6 @@ OF,Organisme de formations, OIL,Opérateur d’intermédiation locative, OPCS,Organisation porteuse de la clause sociale, PAD,Point d’Accès au Droit, -PE,Pôle emploi, PENSION,Pension de famille / résidence accueil, PI,Point Information (PI), PIJ_BIJ,Points et bureaux information jeunesse (PIJ/BIJ),