From f3fb6893b8ada55e9519d8d7f15b03aab658d7f4 Mon Sep 17 00:00:00 2001 From: Hugo Lecuyer Date: Thu, 31 Oct 2024 16:58:15 +0100 Subject: [PATCH] fix-valentin-review --- deployment/MIGRATION.md | 2 +- .../sources/agefiph/int_agefiph__services.sql | 2 +- .../int_france_travail__services.sql | 2 +- .../sources/fredo/int_fredo__services.sql | 2 +- .../int_mediation_numerique__services.sql | 4 +- .../monenfant/int_monenfant__services.sql | 2 +- .../int_reseau_alpha__services.sql | 2 +- .../soliguide/int_soliguide__services.sql | 109 +++++------------- .../sources/soliguide/_soliguide__models.yml | 6 +- ...guide__lieux__publics__administrative.sql} | 0 ...soliguide__lieux__publics__familliale.sql} | 0 ...stg_soliguide__lieux__publics__gender.sql} | 0 12 files changed, 39 insertions(+), 92 deletions(-) rename pipeline/dbt/models/staging/sources/soliguide/{stg_soliguide__administrative.sql => stg_soliguide__lieux__publics__administrative.sql} (100%) rename pipeline/dbt/models/staging/sources/soliguide/{stg_soliguide__familliale.sql => stg_soliguide__lieux__publics__familliale.sql} (100%) rename pipeline/dbt/models/staging/sources/soliguide/{stg_soliguide__gender.sql => stg_soliguide__lieux__publics__gender.sql} (100%) diff --git a/deployment/MIGRATION.md b/deployment/MIGRATION.md index 00c588c3..debd39ee 100644 --- a/deployment/MIGRATION.md +++ b/deployment/MIGRATION.md @@ -5,7 +5,7 @@ Here is the corrected and formatted version of your migration process: 1. **Connect via SSH to the instance** ```bash - ssh root@163.172.171.247 + ssh root@@ ``` 2. **Install PostgreSQL 17** ```bash diff --git a/pipeline/dbt/models/intermediate/sources/agefiph/int_agefiph__services.sql b/pipeline/dbt/models/intermediate/sources/agefiph/int_agefiph__services.sql index eb752b3f..98d2db07 100644 --- a/pipeline/dbt/models/intermediate/sources/agefiph/int_agefiph__services.sql +++ b/pipeline/dbt/models/intermediate/sources/agefiph/int_agefiph__services.sql @@ -107,7 +107,7 @@ final AS ( ARRAY[ 'personnes-en-situation-de-handicap' ] AS "profils", - 'personnes en situation de handicap' AS "profils_precisions", + NULL AS "profils_precisions", ARRAY( SELECT di_type_by_agefiph_type.type_ FROM di_type_by_agefiph_type diff --git a/pipeline/dbt/models/intermediate/sources/france_travail/int_france_travail__services.sql b/pipeline/dbt/models/intermediate/sources/france_travail/int_france_travail__services.sql index 787adc17..a27080c6 100644 --- a/pipeline/dbt/models/intermediate/sources/france_travail/int_france_travail__services.sql +++ b/pipeline/dbt/models/intermediate/sources/france_travail/int_france_travail__services.sql @@ -44,7 +44,7 @@ final AS ( services.presentation_resume AS "presentation_resume", services.prise_rdv AS "prise_rdv", services.profils AS "profils", - ARRAY_TO_STRING(services.profils, ', ') AS "profils_precisions", + NULL AS "profils_precisions", services.pre_requis AS "pre_requis", services.recurrence AS "recurrence", services.thematiques AS "thematiques", diff --git a/pipeline/dbt/models/intermediate/sources/fredo/int_fredo__services.sql b/pipeline/dbt/models/intermediate/sources/fredo/int_fredo__services.sql index 87bbcfd1..95ffb5ba 100644 --- a/pipeline/dbt/models/intermediate/sources/fredo/int_fredo__services.sql +++ b/pipeline/dbt/models/intermediate/sources/fredo/int_fredo__services.sql @@ -149,7 +149,7 @@ final AS ( 'La Réunion' AS "zone_diffusion_nom", NULL AS "recurrence", fredo_structures.last_update AS "date_maj", - CAST((UUID(MD5(fredo_structures.id || COALESCE(fredo_services.value, '')))) AS TEXT) AS "id", + CAST(UUID(MD5(fredo_structures.id || COALESCE(fredo_services.value, ''))) AS TEXT) AS "id", CAST(NULL AS TEXT []) AS "pre_requis", CAST(NULL AS BOOLEAN) AS "cumulable", CAST(NULL AS TEXT []) AS "justificatifs", diff --git a/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__services.sql b/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__services.sql index 8b6c8a4f..6342d924 100644 --- a/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__services.sql +++ b/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__services.sql @@ -28,7 +28,7 @@ final AS ( services.prise_rdv AS "prise_rdv", services.frais AS "frais", services.profils AS "profils", - ARRAY_TO_STRING(services.profils, ', ') AS "profils_precisions", + NULL AS "profils_precisions", services.structure_id AS "structure_id", services.thematiques AS "thematiques", services._di_source_id AS "source", @@ -47,7 +47,7 @@ final AS ( CAST(NULL AS BOOLEAN) AS "cumulable", CAST(NULL AS TEXT []) AS "justificatifs", CAST(NULL AS DATE) AS "date_suspension", - CAST((structures.date_maj) AS DATE) AS "date_maj", + CAST(structures.date_maj AS DATE) AS "date_maj", CAST(NULL AS DATE) AS "date_creation", CASE WHEN structures.code_insee LIKE '97%' THEN LEFT(structures.code_insee, 3) diff --git a/pipeline/dbt/models/intermediate/sources/monenfant/int_monenfant__services.sql b/pipeline/dbt/models/intermediate/sources/monenfant/int_monenfant__services.sql index eb32b675..eafee4f3 100644 --- a/pipeline/dbt/models/intermediate/sources/monenfant/int_monenfant__services.sql +++ b/pipeline/dbt/models/intermediate/sources/monenfant/int_monenfant__services.sql @@ -24,7 +24,7 @@ final AS ( NULL AS "prise_rdv", NULL AS "frais_autres", ARRAY['familles-enfants'] AS "profils", - 'famille enfant' AS "profils_precisions", + NULL AS "profils_precisions", id AS "structure_id", _di_source_id AS "source", TRUE AS "cumulable", diff --git a/pipeline/dbt/models/intermediate/sources/reseau_alpha/int_reseau_alpha__services.sql b/pipeline/dbt/models/intermediate/sources/reseau_alpha/int_reseau_alpha__services.sql index 192db146..fbddc8d0 100644 --- a/pipeline/dbt/models/intermediate/sources/reseau_alpha/int_reseau_alpha__services.sql +++ b/pipeline/dbt/models/intermediate/sources/reseau_alpha/int_reseau_alpha__services.sql @@ -86,7 +86,7 @@ final AS ( NULL ) AS "modes_orientation_beneficiaire", ARRAY['public-langues-etrangeres'] AS "profils", - 'public langues etrangeres' AS "profils_precisions", + NULL AS "profils_precisions", ARRAY['formation'] AS "types", CAST(NULL AS TEXT []) AS "frais" FROM formations diff --git a/pipeline/dbt/models/intermediate/sources/soliguide/int_soliguide__services.sql b/pipeline/dbt/models/intermediate/sources/soliguide/int_soliguide__services.sql index 2578433c..17b2a656 100644 --- a/pipeline/dbt/models/intermediate/sources/soliguide/int_soliguide__services.sql +++ b/pipeline/dbt/models/intermediate/sources/soliguide/int_soliguide__services.sql @@ -14,16 +14,12 @@ phones AS ( SELECT * FROM {{ ref('stg_soliguide__phones') }} ), -administrative AS ( - SELECT * FROM {{ ref('stg_soliguide__administrative') }} -), - -gender AS ( - SELECT * FROM {{ ref('stg_soliguide__gender') }} -), - -familliale AS ( - SELECT * FROM {{ ref('stg_soliguide__familliale') }} +publics AS ( + SELECT * FROM {{ ref('stg_soliguide__lieux__publics__administrative') }} + UNION ALL + SELECT * FROM {{ ref('stg_soliguide__lieux__publics__gender') }} + UNION ALL + SELECT * FROM {{ ref('stg_soliguide__lieux__publics__familliale') }} ), thematiques AS ( @@ -74,70 +70,30 @@ di_thematique_by_soliguide_category_code AS ( ) AS x (category, thematique) ), -mapping_administrative AS ( - SELECT x.* - FROM ( +profils AS ( + SELECT + publics.lieu_id, + ARRAY_TO_STRING(ARRAY_AGG(DISTINCT di_mapping.traduction), ',') AS traduction, + ARRAY_REMOVE(ARRAY_AGG(di_mapping.profils), NULL) AS profils + FROM + publics + LEFT JOIN ( VALUES - ('regular', 'en situation reguliere', NULL), + -- administrative status + ('regular', 'en situation régulière', NULL), ('asylum', 'demandeur asile', 'personnes-de-nationalite-etrangere'), - ('undocumented', 'sans papier', 'personnes-de-nationalite-etrangere') - ) AS x (category, traduction, profils) -), - -mapping_gender AS ( - SELECT x.* - FROM ( - VALUES - ('men', 'homme', 'adultes'), - ('women', 'femme', 'femmes') - ) AS x (category, traduction, profils) -), - -mapping_familliale AS ( - SELECT x.* - FROM ( - VALUES - ('isolated', 'isole', NULL), + ('undocumented', 'sans-papiers', 'personnes-de-nationalite-etrangere'), + -- family status + ('isolated', 'isolé', NULL), ('family', 'famille', 'familles-enfants'), ('couple', 'couple', 'familles-enfants'), - ('pregnent', 'enceinte', 'familles-enfants') - ) AS x (category, traduction, profils) -), - -profils_administrative AS ( - SELECT - administrative.lieu_id, - ARRAY_TO_STRING(ARRAY_AGG(mapping_administrative.traduction), ',') AS traduction, - ARRAY_AGG(mapping_administrative.profils) AS profils - FROM - administrative - LEFT JOIN mapping_administrative ON administrative.value = mapping_administrative.category - GROUP BY - administrative.lieu_id -), - -profils_familliale AS ( - SELECT - familliale.lieu_id, - ARRAY_TO_STRING(ARRAY_AGG(mapping_familliale.traduction), ',') AS traduction, - ARRAY_AGG(mapping_familliale.profils) AS profils - FROM - familliale - LEFT JOIN mapping_familliale ON familliale.value = mapping_familliale.category - GROUP BY - familliale.lieu_id -), - -profils_gender AS ( - SELECT - gender.lieu_id, - ARRAY_TO_STRING(ARRAY_AGG(mapping_gender.traduction), ',') AS traduction, - ARRAY_AGG(mapping_gender.profils) AS profils - FROM - gender - LEFT JOIN mapping_gender ON gender.value = mapping_gender.category + ('pregnent', 'enceinte', 'familles-enfants'), + -- gender status + ('men', 'homme', NULL), + ('women', 'femme', 'femmes') + ) AS di_mapping (category, traduction, profils) ON publics.value = di_mapping.category GROUP BY - gender.lieu_id + publics.lieu_id ), filtered_phones AS ( @@ -191,15 +147,8 @@ final AS ( open_services._di_source_id AS "source", CAST(NULL AS TEXT []) AS "types", NULL AS "prise_rdv", - ARRAY_REMOVE( - profils_administrative.profils - || profils_familliale.profils - || profils_gender.profils, - NULL - ) AS "profils", - profils_administrative.traduction || ' ' - || profils_familliale.traduction || ' ' - || profils_gender.traduction AS "profils_precisions", + profils.profils AS "profils", + profils.traduction AS "profils_precisions", CAST(NULL AS TEXT []) AS "pre_requis", TRUE AS "cumulable", CAST(NULL AS TEXT []) AS "justificatifs", @@ -292,9 +241,7 @@ final AS ( LEFT JOIN lieux ON open_services.lieu_id = lieux.id LEFT JOIN categories ON open_services.category = categories.code LEFT JOIN filtered_phones ON open_services.lieu_id = filtered_phones.lieu_id - LEFT JOIN profils_administrative ON open_services.lieu_id = profils_administrative.lieu_id - LEFT JOIN profils_familliale ON open_services.lieu_id = profils_familliale.lieu_id - LEFT JOIN profils_gender ON open_services.lieu_id = profils_gender.lieu_id + LEFT JOIN profils AS profils ON lieux.id = profils.lieu_id ORDER BY open_services.id ) diff --git a/pipeline/dbt/models/staging/sources/soliguide/_soliguide__models.yml b/pipeline/dbt/models/staging/sources/soliguide/_soliguide__models.yml index 96e96293..45565711 100644 --- a/pipeline/dbt/models/staging/sources/soliguide/_soliguide__models.yml +++ b/pipeline/dbt/models/staging/sources/soliguide/_soliguide__models.yml @@ -89,7 +89,7 @@ models: field: lieu_id - - name: stg_soliguide__familliale + - name: stg_soliguide__lieux__publics__familliale columns: - name: lieu_id data_tests: @@ -103,7 +103,7 @@ models: - dbt_utils.not_empty_string - accepted_values: values: ['isolated', 'family', 'pregnant', 'couple'] - - name: stg_soliguide__gender + - name: stg_soliguide__lieux__publics__gender columns: - name: lieu_id data_tests: @@ -117,7 +117,7 @@ models: - dbt_utils.not_empty_string - accepted_values: values: ['men', 'women'] - - name: stg_soliguide__administrative + - name: stg_soliguide__lieux__publics__administrative columns: - name: lieu_id data_tests: diff --git a/pipeline/dbt/models/staging/sources/soliguide/stg_soliguide__administrative.sql b/pipeline/dbt/models/staging/sources/soliguide/stg_soliguide__lieux__publics__administrative.sql similarity index 100% rename from pipeline/dbt/models/staging/sources/soliguide/stg_soliguide__administrative.sql rename to pipeline/dbt/models/staging/sources/soliguide/stg_soliguide__lieux__publics__administrative.sql diff --git a/pipeline/dbt/models/staging/sources/soliguide/stg_soliguide__familliale.sql b/pipeline/dbt/models/staging/sources/soliguide/stg_soliguide__lieux__publics__familliale.sql similarity index 100% rename from pipeline/dbt/models/staging/sources/soliguide/stg_soliguide__familliale.sql rename to pipeline/dbt/models/staging/sources/soliguide/stg_soliguide__lieux__publics__familliale.sql diff --git a/pipeline/dbt/models/staging/sources/soliguide/stg_soliguide__gender.sql b/pipeline/dbt/models/staging/sources/soliguide/stg_soliguide__lieux__publics__gender.sql similarity index 100% rename from pipeline/dbt/models/staging/sources/soliguide/stg_soliguide__gender.sql rename to pipeline/dbt/models/staging/sources/soliguide/stg_soliguide__lieux__publics__gender.sql