From 61806d9ab3fa808749b9a0e64888ec98a711a5be Mon Sep 17 00:00:00 2001 From: Hugo Lecuyer Date: Tue, 17 Sep 2024 16:26:44 +0200 Subject: [PATCH] feat(mes_aides): add permis velos from mes_aides source --- pipeline/dags/dag_utils/sources/__init__.py | 5 + pipeline/dbt/models/_sources.yml | 3 + .../sources/mes_aides/_mes_aides__models.yml | 11 -- .../garage/_mes_aides_garage__models.yml | 67 +++++++ .../garage/int_mes_aides__adresses_garage.sql | 19 ++ .../{ => garage}/int_mes_aides__contacts.sql | 0 .../garage/int_mes_aides__services_garage.sql | 103 ++++++++++ .../int_mes_aides__structures_garage.sql | 32 +++ .../mes_aides/int_mes_aides__adresses.sql | 40 ++-- .../mes_aides/int_mes_aides__services.sql | 172 ++++++++-------- .../mes_aides/int_mes_aides__structures.sql | 75 ++++--- .../permis_velo/_mes_aides__models.yml | 57 ++++++ .../int_mes_aides__adresses_permis_velo.sql | 19 ++ .../int_mes_aides__services_permis_velo.sql | 186 ++++++++++++++++++ .../int_mes_aides__structures_permis_velo.sql | 56 ++++++ .../_decoupage_administratif__models.yml | 2 +- .../sources/mes_aides/_mes_aides__models.yml | 1 + .../mes_aides/stg_mes_aides__permis_velo.sql | 39 ++++ pipeline/defaults.env | 1 + 19 files changed, 745 insertions(+), 143 deletions(-) create mode 100644 pipeline/dbt/models/intermediate/sources/mes_aides/garage/_mes_aides_garage__models.yml create mode 100644 pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__adresses_garage.sql rename pipeline/dbt/models/intermediate/sources/mes_aides/{ => garage}/int_mes_aides__contacts.sql (100%) create mode 100644 pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__services_garage.sql create mode 100644 pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__structures_garage.sql create mode 100644 pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/_mes_aides__models.yml create mode 100644 pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__adresses_permis_velo.sql create mode 100644 pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__services_permis_velo.sql create mode 100644 pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__structures_permis_velo.sql create mode 100644 pipeline/dbt/models/staging/sources/mes_aides/stg_mes_aides__permis_velo.sql diff --git a/pipeline/dags/dag_utils/sources/__init__.py b/pipeline/dags/dag_utils/sources/__init__.py index 40b42c168..f43b90b2d 100644 --- a/pipeline/dags/dag_utils/sources/__init__.py +++ b/pipeline/dags/dag_utils/sources/__init__.py @@ -54,6 +54,11 @@ "url": Variable.get("MES_AIDES_AIDES_URL", None), "token": Variable.get("MES_AIDES_AIRTABLE_KEY", None), }, + "permis_velo": { + "filename": "aides.json", + "url": Variable.get("MES_AIDES_PERMIS_VELO_URL", None), + "token": Variable.get("MES_AIDES_AIRTABLE_KEY", None), + }, }, }, "siao": { diff --git a/pipeline/dbt/models/_sources.yml b/pipeline/dbt/models/_sources.yml index e7204723b..34d939ab3 100644 --- a/pipeline/dbt/models/_sources.yml +++ b/pipeline/dbt/models/_sources.yml @@ -75,6 +75,9 @@ sources: - name: garages meta: kind: structure + - name: permis_velo + meta: + kind: structure - name: aides meta: kind: service diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/_mes_aides__models.yml b/pipeline/dbt/models/intermediate/sources/mes_aides/_mes_aides__models.yml index 2aa113ead..1fc9b0c92 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/_mes_aides__models.yml +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/_mes_aides__models.yml @@ -13,17 +13,6 @@ models: - not_null - dbt_utils.not_empty_string - - name: int_mes_aides__contacts - columns: - - name: contact_uid - data_tests: - - unique - - not_null - - name: courriel - data_tests: - - not_null - - dbt_utils.not_empty_string - - name: int_mes_aides__structures data_tests: - check_structure: diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/garage/_mes_aides_garage__models.yml b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/_mes_aides_garage__models.yml new file mode 100644 index 000000000..13c37645c --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/_mes_aides_garage__models.yml @@ -0,0 +1,67 @@ +version: 2 + +models: + - name: int_mes_aides__adresses_garage + data_tests: + - check_adresse: + config: + severity: warn + columns: + - name: id + data_tests: + - unique + - not_null + - dbt_utils.not_empty_string + + - name: int_mes_aides__contacts + columns: + - name: contact_uid + data_tests: + - unique + - not_null + - name: courriel + data_tests: + - not_null + - dbt_utils.not_empty_string + + - name: int_mes_aides__structures_garages + data_tests: + - check_structure: + config: + severity: warn + columns: + - name: id + data_tests: + - unique + - not_null + - dbt_utils.not_empty_string + - name: adresse_id + data_tests: + - not_null + - relationships: + to: ref('int_mes_aides__adresses_garage') + field: id + + - name: int_mes_aides__services_garage + data_tests: + - check_service: + config: + severity: warn + columns: + - name: id + data_tests: + - unique + - not_null + - dbt_utils.not_empty_string + - name: structure_id + data_tests: + - not_null + - relationships: + to: ref('int_mes_aides__structures_garage') + field: id + - name: adresse_id + data_tests: + - not_null + - relationships: + to: ref('int_mes_aides__adresses_garage') + field: id \ No newline at end of file diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__adresses_garage.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__adresses_garage.sql new file mode 100644 index 000000000..92c8ae89f --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__adresses_garage.sql @@ -0,0 +1,19 @@ +WITH garages AS ( + SELECT * FROM {{ ref('stg_mes_aides__garages') }} +), + +final AS ( + SELECT + id AS "id", + ville_nom AS "commune", + code_postal AS "code_postal", + code_insee AS "code_insee", + adresse AS "adresse", + NULL AS "complement_adresse", + longitude AS "longitude", + latitude AS "latitude", + _di_source_id AS "source" + FROM garages +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__contacts.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__contacts.sql similarity index 100% rename from pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__contacts.sql rename to pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__contacts.sql diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__services_garage.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__services_garage.sql new file mode 100644 index 000000000..bf2d7336a --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__services_garage.sql @@ -0,0 +1,103 @@ +{% set check_structure_str = "Veuillez vérifier sur le site internet de la structure" %} + +WITH garages AS ( + SELECT * FROM {{ ref('stg_mes_aides__garages') }} +), + +service_types_by_garage AS ( + SELECT + garages.id, + service_type + FROM garages, + UNNEST(garages.services) AS service_type +), + +final AS ( + SELECT + garages.id AS "adresse_id", + TRUE AS "contact_public", + TRUE AS "cumulable", + garages.cree_le AS "date_creation", + garages.modifie_le AS "date_maj", + CAST(NULL AS DATE) AS "date_suspension", + NULL AS "formulaire_en_ligne", + '{{ check_structure_str }}' AS "frais_autres", + CAST(UUID(MD5(garages.id || COALESCE(service_types_by_garage.service_type, ''))) AS TEXT) AS "id", + CAST(NULL AS TEXT []) AS "justificatifs", + 'https://mes-aides.francetravail.fr/transport-et-mobilite/garages-solidaires' AS "lien_source", + ARRAY['en-presentiel'] AS "modes_accueil", + ARRAY_REMOVE( + ARRAY[ + CASE WHEN garages.telephone IS NOT NULL THEN 'telephoner' END, + CASE WHEN garages.email IS NOT NULL THEN 'envoyer-un-mail' END + ], + NULL + ) AS "modes_orientation_accompagnateur", + CASE + WHEN + garages.telephone IS NULL + AND garages.email IS NULL + THEN '{{ check_structure_str }}' + END AS "modes_orientation_accompagnateur_autres", + ARRAY_REMOVE( + ARRAY[ + 'se-presenter', + CASE WHEN garages.telephone IS NOT NULL THEN 'telephoner' END, + CASE WHEN garages.email IS NOT NULL THEN 'envoyer-un-mail' END + ], + NULL + ) AS "modes_orientation_beneficiaire", + NULL AS "modes_orientation_beneficiaire_autres", + FORMAT( + '%s de %s', + COALESCE(service_types_by_garage.service_type, 'Réparation, vente et location'), + LOWER(COALESCE(NULLIF(ARRAY_TO_STRING(garages.types_de_vehicule, ', '), ''), 'véhicule')) + ) AS "nom", + FORMAT( + '%s de %s à tarif solidaire pour les personnes en difficulté, et selon leur situation', + COALESCE(service_types_by_garage.service_type, 'Réparation, vente et location'), + LOWER(COALESCE(NULLIF(ARRAY_TO_STRING(garages.types_de_vehicule, ', '), ''), 'véhicule')) + ) AS "presentation_resume", + NULL AS "presentation_detail", + NULL AS "prise_rdv", + CAST(NULL AS TEXT []) AS "profils", + NULL AS "recurrence", + garages._di_source_id AS "source", + garages.id AS "structure_id", + ARRAY_REMOVE( + ARRAY[ + CASE service_types_by_garage.service_type + WHEN 'Achat' THEN 'mobilite--acheter-un-vehicule-motorise' + WHEN 'Location' THEN 'mobilite--louer-un-vehicule' + WHEN 'Réparation' THEN 'mobilite--entretenir-reparer-son-vehicule' + ELSE 'mobilite' + END, + CASE + WHEN + service_types_by_garage.service_type = 'Achat' + AND 'Vélo' = ANY(garages.types_de_vehicule) + THEN 'mobilite--acheter-un-velo' + END + ], + NULL + ) AS "thematiques", + ARRAY['aide-materielle', 'aide-financiere'] AS "types", + CASE LEFT(garages.code_insee, 2) + WHEN '97' THEN LEFT(garages.code_insee, 3) + ELSE LEFT(garages.code_insee, 2) + END AS "zone_diffusion_code", + NULL AS "zone_diffusion_nom", + 'departement' AS "zone_diffusion_type", + garages.criteres_eligibilite AS "pre_requis", + NULL AS "contact_nom_prenom", + garages.email AS "courriel", + garages.telephone AS "telephone", + CAST(NULL AS TEXT []) AS "frais", + NULL AS "page_web" + FROM garages + LEFT JOIN service_types_by_garage ON garages.id = service_types_by_garage.id + WHERE + garages.en_ligne +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__structures_garage.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__structures_garage.sql new file mode 100644 index 000000000..084c26a51 --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__structures_garage.sql @@ -0,0 +1,32 @@ +WITH garages AS ( + SELECT * FROM {{ ref('stg_mes_aides__garages') }} +), + +final AS ( + SELECT + id AS "id", + id AS "adresse_id", + siret AS "siret", + NULL::BOOLEAN AS "antenne", + NULL AS "rna", + nom AS "nom", + telephone AS "telephone", + email AS "courriel", + url AS "site_web", + _di_source_id AS "source", + NULL AS "lien_source", + NULL AS "horaires_ouverture", + NULL AS "accessibilite", + NULL::TEXT [] AS "labels_nationaux", + NULL::TEXT [] AS "labels_autres", + NULL AS "typologie", + NULL AS "presentation_resume", + NULL AS "presentation_detail", + modifie_le::DATE AS "date_maj", + ARRAY['mobilite'] AS "thematiques" + FROM garages + WHERE + en_ligne +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__adresses.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__adresses.sql index 92c8ae89f..81cd45944 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__adresses.sql +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__adresses.sql @@ -1,19 +1,35 @@ -WITH garages AS ( - SELECT * FROM {{ ref('stg_mes_aides__garages') }} +WITH adresses_garage AS ( + SELECT * FROM {{ ref('int_mes_aides__adresses_garage') }} +), + +adresses_permis_velo AS ( + SELECT * FROM {{ ref('int_mes_aides__adresses_permis_velo') }} ), final AS ( SELECT - id AS "id", - ville_nom AS "commune", - code_postal AS "code_postal", - code_insee AS "code_insee", - adresse AS "adresse", - NULL AS "complement_adresse", - longitude AS "longitude", - latitude AS "latitude", - _di_source_id AS "source" - FROM garages + adresses_garage.id, + adresses_garage.commune, + adresses_garage.code_insee, + adresses_garage.longitude, + adresses_garage.latitude, + adresses_garage.source, + adresses_garage.code_postal, + adresses_garage.adresse, + adresses_garage.complement_adresse + FROM adresses_garage + UNION ALL + SELECT + adresses_permis_velo.id, + adresses_permis_velo.commune, + adresses_permis_velo.code_insee, + adresses_permis_velo.longitude, + adresses_permis_velo.latitude, + adresses_permis_velo.source, + adresses_permis_velo.code_postal, + adresses_permis_velo.adresse, + adresses_permis_velo.complement_adresse + FROM adresses_permis_velo ) SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__services.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__services.sql index bf2d7336a..8adb2e88d 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__services.sql +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__services.sql @@ -1,103 +1,87 @@ -{% set check_structure_str = "Veuillez vérifier sur le site internet de la structure" %} - -WITH garages AS ( - SELECT * FROM {{ ref('stg_mes_aides__garages') }} +WITH services_garage AS ( + SELECT * FROM {{ ref('int_mes_aides__services_garage') }} ), -service_types_by_garage AS ( - SELECT - garages.id, - service_type - FROM garages, - UNNEST(garages.services) AS service_type +services_permis_velo AS ( + SELECT * FROM {{ ref('int_mes_aides__services_permis_velo') }} ), final AS ( SELECT - garages.id AS "adresse_id", - TRUE AS "contact_public", - TRUE AS "cumulable", - garages.cree_le AS "date_creation", - garages.modifie_le AS "date_maj", - CAST(NULL AS DATE) AS "date_suspension", - NULL AS "formulaire_en_ligne", - '{{ check_structure_str }}' AS "frais_autres", - CAST(UUID(MD5(garages.id || COALESCE(service_types_by_garage.service_type, ''))) AS TEXT) AS "id", - CAST(NULL AS TEXT []) AS "justificatifs", - 'https://mes-aides.francetravail.fr/transport-et-mobilite/garages-solidaires' AS "lien_source", - ARRAY['en-presentiel'] AS "modes_accueil", - ARRAY_REMOVE( - ARRAY[ - CASE WHEN garages.telephone IS NOT NULL THEN 'telephoner' END, - CASE WHEN garages.email IS NOT NULL THEN 'envoyer-un-mail' END - ], - NULL - ) AS "modes_orientation_accompagnateur", - CASE - WHEN - garages.telephone IS NULL - AND garages.email IS NULL - THEN '{{ check_structure_str }}' - END AS "modes_orientation_accompagnateur_autres", - ARRAY_REMOVE( - ARRAY[ - 'se-presenter', - CASE WHEN garages.telephone IS NOT NULL THEN 'telephoner' END, - CASE WHEN garages.email IS NOT NULL THEN 'envoyer-un-mail' END - ], - NULL - ) AS "modes_orientation_beneficiaire", - NULL AS "modes_orientation_beneficiaire_autres", - FORMAT( - '%s de %s', - COALESCE(service_types_by_garage.service_type, 'Réparation, vente et location'), - LOWER(COALESCE(NULLIF(ARRAY_TO_STRING(garages.types_de_vehicule, ', '), ''), 'véhicule')) - ) AS "nom", - FORMAT( - '%s de %s à tarif solidaire pour les personnes en difficulté, et selon leur situation', - COALESCE(service_types_by_garage.service_type, 'Réparation, vente et location'), - LOWER(COALESCE(NULLIF(ARRAY_TO_STRING(garages.types_de_vehicule, ', '), ''), 'véhicule')) - ) AS "presentation_resume", - NULL AS "presentation_detail", - NULL AS "prise_rdv", - CAST(NULL AS TEXT []) AS "profils", - NULL AS "recurrence", - garages._di_source_id AS "source", - garages.id AS "structure_id", - ARRAY_REMOVE( - ARRAY[ - CASE service_types_by_garage.service_type - WHEN 'Achat' THEN 'mobilite--acheter-un-vehicule-motorise' - WHEN 'Location' THEN 'mobilite--louer-un-vehicule' - WHEN 'Réparation' THEN 'mobilite--entretenir-reparer-son-vehicule' - ELSE 'mobilite' - END, - CASE - WHEN - service_types_by_garage.service_type = 'Achat' - AND 'Vélo' = ANY(garages.types_de_vehicule) - THEN 'mobilite--acheter-un-velo' - END - ], - NULL - ) AS "thematiques", - ARRAY['aide-materielle', 'aide-financiere'] AS "types", - CASE LEFT(garages.code_insee, 2) - WHEN '97' THEN LEFT(garages.code_insee, 3) - ELSE LEFT(garages.code_insee, 2) - END AS "zone_diffusion_code", - NULL AS "zone_diffusion_nom", - 'departement' AS "zone_diffusion_type", - garages.criteres_eligibilite AS "pre_requis", - NULL AS "contact_nom_prenom", - garages.email AS "courriel", - garages.telephone AS "telephone", - CAST(NULL AS TEXT []) AS "frais", - NULL AS "page_web" - FROM garages - LEFT JOIN service_types_by_garage ON garages.id = service_types_by_garage.id - WHERE - garages.en_ligne + adresse_id, + contact_public, + cumulable, + date_creation, + date_maj, + date_suspension, + formulaire_en_ligne, + frais_autres, + id, + justificatifs, + lien_source, + modes_accueil, + modes_orientation_accompagnateur, + modes_orientation_accompagnateur_autres, + modes_orientation_beneficiaire, + modes_orientation_beneficiaire_autres, + nom, + presentation_resume, + presentation_detail, + prise_rdv, + profils, + recurrence, + source, + structure_id, + thematiques, + types, + zone_diffusion_code, + zone_diffusion_nom, + zone_diffusion_type, + pre_requis, + contact_nom_prenom, + courriel, + telephone, + frais, + page_web + FROM services_garage + UNION ALL + SELECT + adresse_id, + contact_public, + cumulable, + date_creation, + date_maj, + date_suspension, + formulaire_en_ligne, + frais_autres, + id, + justificatifs, + lien_source, + modes_accueil, + modes_orientation_accompagnateur, + modes_orientation_accompagnateur_autres, + modes_orientation_beneficiaire, + modes_orientation_beneficiaire_autres, + nom, + presentation_resume, + presentation_detail, + prise_rdv, + profils, + recurrence, + source, + structure_id, + thematiques, + types, + zone_diffusion_code, + zone_diffusion_nom, + zone_diffusion_type, + pre_requis, + contact_nom_prenom, + courriel, + telephone, + frais, + page_web + FROM services_permis_velo ) SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__structures.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__structures.sql index 084c26a51..bd27cfcec 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__structures.sql +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__structures.sql @@ -1,32 +1,57 @@ -WITH garages AS ( - SELECT * FROM {{ ref('stg_mes_aides__garages') }} +WITH structures_garage AS ( + SELECT * FROM {{ ref('int_mes_aides__structures_garage') }} +), + +structures_permis_velo AS ( + SELECT * FROM {{ ref('int_mes_aides__structures_permis_velo') }} ), final AS ( SELECT - id AS "id", - id AS "adresse_id", - siret AS "siret", - NULL::BOOLEAN AS "antenne", - NULL AS "rna", - nom AS "nom", - telephone AS "telephone", - email AS "courriel", - url AS "site_web", - _di_source_id AS "source", - NULL AS "lien_source", - NULL AS "horaires_ouverture", - NULL AS "accessibilite", - NULL::TEXT [] AS "labels_nationaux", - NULL::TEXT [] AS "labels_autres", - NULL AS "typologie", - NULL AS "presentation_resume", - NULL AS "presentation_detail", - modifie_le::DATE AS "date_maj", - ARRAY['mobilite'] AS "thematiques" - FROM garages - WHERE - en_ligne + id, + adresse_id, + siret, + antenne, + rna, + nom, + telephone, + courriel, + site_web, + "source", + lien_source, + horaires_ouverture, + accessibilite, + labels_nationaux, + labels_autres, + typologie, + presentation_resume, + presentation_detail, + date_maj, + thematiques + FROM structures_garage + UNION ALL + SELECT + id, + adresse_id, + siret, + antenne, + rna, + nom, + telephone, + courriel, + site_web, + "source", + lien_source, + horaires_ouverture, + accessibilite, + labels_nationaux, + labels_autres, + typologie, + presentation_resume, + presentation_detail, + date_maj, + thematiques + FROM structures_permis_velo ) SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/_mes_aides__models.yml b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/_mes_aides__models.yml new file mode 100644 index 000000000..e1ae612d6 --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/_mes_aides__models.yml @@ -0,0 +1,57 @@ +version: 2 + +models: + - name: int_mes_aides__adresses_permis_velo + data_tests: + - check_adresse: + config: + severity: warn + columns: + - name: id + data_tests: + - unique + - not_null + - dbt_utils.not_empty_string + + + - name: int_mes_aides__structures_permis_velo + data_tests: + - check_structure: + config: + severity: warn + columns: + - name: id + data_tests: + - unique + - not_null + - dbt_utils.not_empty_string + - name: adresse_id + data_tests: + - not_null + - relationships: + to: ref('int_mes_aides__adresses_permis_velo') + field: id + + - name: int_mes_aides__services_permis_velo + data_tests: + - check_service: + config: + severity: warn + columns: + - name: id + data_tests: + - unique + - not_null + - dbt_utils.not_empty_string + - name: structure_id + data_tests: + - not_null + - relationships: + to: ref('int_mes_aides__structures_permis_velo') + field: id + - name: adresse_id + data_tests: + - not_null + - relationships: + to: ref('int_mes_aides__adresses_permis_velo') + field: id diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__adresses_permis_velo.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__adresses_permis_velo.sql new file mode 100644 index 000000000..84ea35ddb --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__adresses_permis_velo.sql @@ -0,0 +1,19 @@ +WITH permis_velo AS ( + SELECT * FROM {{ ref('stg_mes_aides__permis_velo') }} +), + +final AS ( + SELECT + id AS "id", + ville_nom AS "commune", + code_postal AS "code_postal", + NULL AS "code_insee", + NULL AS "adresse", + NULL AS "complement_adresse", + CAST(NULL AS FLOAT) AS "longitude", + CAST(NULL AS FLOAT) AS "latitude", + _di_source_id AS "source" + FROM permis_velo +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__services_permis_velo.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__services_permis_velo.sql new file mode 100644 index 000000000..5e014af0e --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__services_permis_velo.sql @@ -0,0 +1,186 @@ +{% set check_structure_str = "Veuillez vérifier sur le site internet de la structure" %} + +WITH permis_velo AS ( + SELECT * FROM {{ ref('stg_mes_aides__permis_velo') }} +), + +communes AS ( + SELECT * FROM {{ ref('stg_decoupage_administratif__communes') }} +), + +zone_code AS ( + SELECT + permis_velo.id, + communes.code, + communes.siren_epci, + communes.departement, + communes.region + FROM permis_velo + LEFT JOIN communes ON permis_velo.code_postal = ANY(communes.codes_postaux) AND permis_velo.ville_nom = communes.nom +), + +types_mapping AS ( + SELECT x.* + FROM ( + VALUES + ('Aide financière', 'aide-financiere'), + ('Prestation', 'aide-materielle'), + ('Accompagnement', 'accompagnement') + ) AS x (types_mes_aides, types_di) +), + +zone_de_diffusion_mapping AS ( + SELECT x.* + FROM ( + VALUES + ('Échelle communale', 'commune'), + ('Échelle intercommunale', 'epci'), + ('Échelle régionale', 'region'), + ('Échelle départementale', 'departement'), + ('France métropolitaine', 'pays') + ) AS x (zone_mes_aides, zone_di) +), + +zone_diffusion AS ( + SELECT + zone_de_diffusion_mapping.zone_di, + permis_velo.id + FROM permis_velo + INNER JOIN zone_de_diffusion_mapping ON permis_velo.zone_diffusion_type = zone_de_diffusion_mapping.zone_mes_aides +), + +modes_orientation_mapping AS ( + SELECT x.* + FROM ( + VALUES + ('La démarche nécessite un passage ou une prise de contact directe avec votre organisme', ARRAY['envoyer-un-email', 'telephoner'], ARRAY['envoyer-un-email', 'telephoner', 'se-presenter']), + ('La démarche nécessite une connexion sur le site de votre organisme', ARRAY['completer-le-formulaire-dadhesion'], ARRAY['completer-le-formulaire-dadhesion']), + ('La démarche peut être faite par mail', ARRAY['envoyer-un-email'], ARRAY['envoyer-un-email']) + ) AS x (demarche, modes_orientation_accompagnateur, modes_orientation_beneficiaire) +), + +mapping_thematiques AS ( + SELECT x.* + FROM ( + VALUES + ('Financer mon permis', 'transport-et-mobilite', ARRAY['mobilite--financer-mon-projet-mobilite', 'mobilite--preparer-son-permis-de-conduire-se-reentrainer-a-la-conduite']), + ('Financer mes trajets', 'transport-et-mobilite', ARRAY['mobilite--financer-mon-projet-mobilite']), + ('Financer le carburant', 'transport-et-mobilite', ARRAY['mobilite--financer-mon-projet-mobilite']), + ('Financer mon BSR', 'transport-et-mobilite', ARRAY['mobilite--financer-mon-projet-mobilite']), + ('Financer l''achat d''un véhicule', 'transport-et-mobilite', ARRAY['mobilite--acheter-un-vehicule-motorise']), + ('Déménager', 'logement', ARRAY['logement-hebergement--demenagement']), + ('Transports en commun', 'transport-et-mobilite', ARRAY['mobilite--comprendre-et-utiliser-les-transports-en-commun']), + ('Transport à la demande', 'transport-et-mobilite', ARRAY['mobilite--etre-accompagne-dans-son-parcours-mobilite']), + ('Réparer ma voiture', 'transport-et-mobilite', ARRAY['mobilite--entretenir-reparer-son-vehicule']), + ('Louer un vélo électrique', 'transport-et-mobilite', ARRAY['mobilite--louer-un-vehicule']), + ('Ma voiture', 'transport-et-mobilite', ARRAY['mobilite--louer-un-vehicule', 'mobilite--entretenir-reparer-son-vehicule', 'mobilite--acheter-un-vehicule-motorise']), + ('Mon vélo électrique', 'transport-et-mobilite', ARRAY['mobilite--acheter-un-velo', 'mobilite--entretenir-reparer-son-vehicule', 'mobilite--apprendre-a-utiliser-un-deux-roues', 'mobilite--louer-un-vehicule', 'mobilite--acheter-un-vehicule-motorise']), + ('Mon scooter', 'transport-et-mobilite', ARRAY['mobilite--entretenir-reparer-son-vehicule', 'mobilite--apprendre-a-utiliser-un-deux-roues', 'mobilite--louer-un-vehicule']), + ('Acheter un vélo électrique', 'transport-et-mobilite', ARRAY['mobilite--acheter-un-velo']), + ('Acheter ou louer une voiture', 'transport-et-mobilite', ARRAY['mobilite--louer-un-vehicule', 'mobilite--acheter-un-vehicule-motorise']), + ('Accéder au permis de conduire', 'transport-et-mobilite', ARRAY['mobilite--preparer-son-permis-de-conduire-se-reentrainer-a-la-conduite']), + ('Déménager', 'transport-et-mobilite', ARRAY['logement-hebergement--demenagement']), + ('Partir à l''étranger', 'transport-et-mobilite', ARRAY['souvrir-a-linternational--sinformer-sur-les-aides-pour-travailler-a-letranger']), + ('Handicap & mobilité', 'transport-et-mobilite', ARRAY['handicap--favoriser-le-retour-et-le-maintien-dans-lemploi']), + ('Trouver une alternance', 'trouver-un-job', ARRAY['trouver-un-emploi--suivre-ses-candidatures-et-relancer-les-employeurs', 'trouver-un-emploi--convaincre-un-recruteur-en-entretien', 'trouver-un-emploi--faire-des-candidatures-spontanees', 'trouver-un-emploi--repondre-a-des-offres-demploi']), + ('Trouver un contrat d''apprentissage', 'trouver-un-job', ARRAY['trouver-un-emploi--suivre-ses-candidatures-et-relancer-les-employeurs', 'trouver-un-emploi--convaincre-un-recruteur-en-entretien', 'trouver-un-emploi--faire-des-candidatures-spontanees', 'trouver-un-emploi--repondre-a-des-offres-demploi']), + ('Préparer une candidature', 'trouver-un-job', ARRAY['preparer-sa-candidature--valoriser-ses-competences']), + ('Rédiger une lettre de motivation ou un CV', 'trouver-un-job', ARRAY['preparer-sa-candidature--realiser-un-cv-et-ou-une-lettre-de-motivation']), + ('Devenir locataire', 'logement', ARRAY['logement-hebergement--etre-accompagne-pour-se-loger']), + ('Financer mon loyer', 'logement', ARRAY['logement-hebergement--gerer-son-budget']), + ('Faire des travaux', 'logement', ARRAY['logement-hebergement--besoin-dadapter-mon-logement']), + ('Trouver un logement social', 'logement', ARRAY['logement-hebergement--etre-accompagne-pour-se-loger']), + ('Financer une formation', 'formation-et-orientation', ARRAY['se-former--monter-son-dossier-de-formation']), + ('Trouver une formation', 'formation-et-orientation', ARRAY['se-former--trouver-sa-formation']), + ('Rémunération pendant la formation', 'formation-et-orientation', ARRAY['se-former--monter-son-dossier-de-formation']), + ('Reconversion', 'formation-et-orientation', ARRAY['se-former--trouver-sa-formation']), + ('Accompagnement personnalisé à la mobilité', 'transport-et-mobilite', ARRAY['mobilite--etre-accompagne-dans-son-parcours-mobilite']) + ) AS x (besoins, thematiques, correspondance_di) +), + +transformed_types AS ( + SELECT + permis_velo.id, + ARRAY_AGG(types_mapping.types_di) AS transformed_types + FROM + permis_velo, + UNNEST(permis_velo.types) AS mes_aides_types + LEFT JOIN + types_mapping ON mes_aides_types = types_mapping.types_mes_aides + GROUP BY + permis_velo.id +), + +final AS ( + SELECT + permis_velo.id AS "adresse_id", + TRUE AS "contact_public", + TRUE AS "cumulable", + permis_velo.date_creation AS "date_creation", + permis_velo.date_maj AS "date_maj", + CAST(NULL AS DATE) AS "date_suspension", + NULL AS "formulaire_en_ligne", + permis_velo.frais_autres AS "frais_autres", + permis_velo.id AS "id", + CAST(NULL AS TEXT []) AS "justificatifs", + permis_velo.lien_source AS "lien_source", + CASE + WHEN permis_velo.mode_acceuil = TRUE THEN ARRAY['a-distance'] + ELSE ARRAY['en-presentiel'] + END AS "modes_accueil", + ( + SELECT modes_orientation_mapping.modes_orientation_accompagnateur + FROM modes_orientation_mapping + WHERE permis_velo.methode = modes_orientation_mapping.demarche + ) AS "modes_orientation_accompagnateur", + NULL AS "modes_orientation_accompagnateur_autres", + ( + SELECT modes_orientation_mapping.modes_orientation_beneficiaire + FROM modes_orientation_mapping + WHERE permis_velo.methode = modes_orientation_mapping.demarche + ) AS "modes_orientation_beneficiaire", + permis_velo.mode_orientation_beneficiare_autre AS "modes_orientation_beneficiaire_autres", + permis_velo.nom AS "nom", + NULL AS "presentation_resume", + permis_velo.presentation_detail AS "presentation_detail", + NULL AS "prise_rdv", + CAST(NULL AS TEXT []) AS "profils", + NULL AS "recurrence", + permis_velo._di_source_id AS "source", + COALESCE(permis_velo.siret_structure, permis_velo.id) AS "structure_id", + ( + SELECT mapping_thematiques.correspondance_di + FROM mapping_thematiques + WHERE + permis_velo.besoins_mes_aides = mapping_thematiques.besoins + AND permis_velo.thematique_mes_aides = mapping_thematiques.thematiques + ) AS "thematiques", + transformed_types.transformed_types AS "types", + CASE + WHEN zone_diffusion.zone_di = 'commune' THEN zone_code.code + WHEN zone_diffusion.zone_di = 'epci' THEN zone_code.siren_epci + WHEN zone_diffusion.zone_di = 'region' THEN zone_code.region + WHEN zone_diffusion.zone_di = 'departement' THEN zone_code.departement + WHEN zone_diffusion.zone_di = 'pays' THEN NULL + END AS "zone_diffusion_code", + CASE + WHEN zone_diffusion.zone_di = 'commune' THEN permis_velo.ville_nom + WHEN zone_diffusion.zone_di = 'epci' THEN permis_velo.ville_nom + WHEN zone_diffusion.zone_di = 'region' THEN permis_velo.region + WHEN zone_diffusion.zone_di = 'departement' THEN permis_velo.departement + WHEN zone_diffusion.zone_di = 'pays' THEN NULL + END AS "zone_diffusion_nom", + zone_diffusion.zone_di AS "zone_diffusion_type", + CAST(NULL AS TEXT []) AS "pre_requis", + NULL AS "contact_nom_prenom", + permis_velo.courriel AS "courriel", + permis_velo.telephone AS "telephone", + CAST(NULL AS TEXT []) AS "frais", + permis_velo.page_web AS "page_web" + FROM permis_velo + LEFT JOIN transformed_types ON permis_velo.id = transformed_types.id + LEFT JOIN zone_diffusion ON permis_velo.id = zone_diffusion.id + LEFT JOIN zone_code ON permis_velo.id = zone_code.id +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__structures_permis_velo.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__structures_permis_velo.sql new file mode 100644 index 000000000..4e7d70606 --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__structures_permis_velo.sql @@ -0,0 +1,56 @@ +WITH permis_velo AS ( + SELECT * FROM {{ ref('stg_mes_aides__permis_velo') }} +), + +final AS ( + SELECT + siret_structure AS "id", + MAX(id) AS "adresse_id", + siret_structure AS "siret", + CAST(NULL AS BOOLEAN) AS "antenne", + NULL AS "rna", + MAX(nom) AS "nom", + MAX(telephone) AS "telephone", + MAX(courriel) AS "courriel", + MAX(page_web) AS "site_web", + MAX(_di_source_id) AS "source", + MAX(lien_source) AS "lien_source", + NULL AS "horaires_ouverture", + NULL AS "accessibilite", + CAST(NULL AS TEXT []) AS "labels_nationaux", + CAST(NULL AS TEXT []) AS "labels_autres", + NULL AS "typologie", + NULL AS "presentation_resume", + NULL AS "presentation_detail", + MAX(CAST(date_maj AS DATE)) AS "date_maj", + ARRAY['mobilite'] AS "thematiques" + FROM permis_velo + WHERE siret_structure IS NOT NULL + GROUP BY siret_structure + UNION ALL + SELECT + id AS "id", + id AS "adresse_id", + NULL AS "siret", + CAST(NULL AS BOOLEAN) AS "antenne", + NULL AS "rna", + nom AS "nom", + telephone AS "telephone", + courriel AS "courriel", + page_web AS "site_web", + _di_source_id AS "source", + lien_source AS "lien_source", + NULL AS "horaires_ouverture", + NULL AS "accessibilite", + CAST(NULL AS TEXT []) AS "labels_nationaux", + CAST(NULL AS TEXT []) AS "labels_autres", + NULL AS "typologie", + NULL AS "presentation_resume", + NULL AS "presentation_detail", + CAST(date_maj AS DATE) AS "date_maj", + ARRAY['mobilite'] AS "thematiques" + FROM permis_velo + WHERE siret_structure IS NULL +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/staging/decoupage_administratif/_decoupage_administratif__models.yml b/pipeline/dbt/models/staging/decoupage_administratif/_decoupage_administratif__models.yml index 664acfcfa..efd234777 100644 --- a/pipeline/dbt/models/staging/decoupage_administratif/_decoupage_administratif__models.yml +++ b/pipeline/dbt/models/staging/decoupage_administratif/_decoupage_administratif__models.yml @@ -1,4 +1,4 @@ -version: 2 + version: 2 models: - name: stg_decoupage_administratif__regions diff --git a/pipeline/dbt/models/staging/sources/mes_aides/_mes_aides__models.yml b/pipeline/dbt/models/staging/sources/mes_aides/_mes_aides__models.yml index 0970e9b94..3069370ba 100644 --- a/pipeline/dbt/models/staging/sources/mes_aides/_mes_aides__models.yml +++ b/pipeline/dbt/models/staging/sources/mes_aides/_mes_aides__models.yml @@ -3,3 +3,4 @@ version: 2 models: - name: stg_mes_aides__aides - name: stg_mes_aides__garages + - name: stg_mes_aides__permis_velo diff --git a/pipeline/dbt/models/staging/sources/mes_aides/stg_mes_aides__permis_velo.sql b/pipeline/dbt/models/staging/sources/mes_aides/stg_mes_aides__permis_velo.sql new file mode 100644 index 000000000..5db60fa4e --- /dev/null +++ b/pipeline/dbt/models/staging/sources/mes_aides/stg_mes_aides__permis_velo.sql @@ -0,0 +1,39 @@ +WITH source AS ( + {{ stg_source_header('mes_aides', 'permis_velo') }} +), + +final AS ( + SELECT + _di_source_id AS "_di_source_id", + data #>> '{fields,ID}' AS "id", + TRIM(SUBSTRING(data #>> '{fields,Liaisons Villes}' FROM '^(.*) \(')) AS "ville_nom", + TRIM(SUBSTRING(data #>> '{fields,Liaisons Villes}' FROM '\((\d+)\)$')) AS "code_postal", + SUBSTRING( + data #>> '{fields,Liaisons Départements}' FROM POSITION('-' IN data #>> '{fields,Liaisons Départements}') + 1 -- noqa + ) AS "departement", + data #>> '{fields,Liaisons Régions}' AS "region", + data #>> '{fields,Nom}' AS "nom", + data #>> '{fields,Liaisons Besoins}' AS "besoins_mes_aides", + data #>> '{fields,Slug Thématiques}' AS "thematique_mes_aides", + data #>> '{fields,Zone géographique}' AS "zone_diffusion_type", + data #>> '{fields,Nature}' AS "frais_autres", + data #>> '{fields,Démarches}' AS "mode_orientation_beneficiare_autre", + data #>> '{fields,"Modalité et versement"}' AS "presentation_detail", + data #>> '{fields,"Méthode"}' AS "methode", + data #>> '{fields,SIRET}' AS "siret_structure", + data #>> '{fields,Slug Organisme}' AS "lien_source_structure", + data #>> '{fields,Site}' AS "page_web", + data #>> '{fields,Nom Organisme}' AS "nom_structure", + data #>> '{fields,Organisme Type}' AS "typologie_structure", + CAST(data #>> '{fields,Créée le}' AS DATE) AS "date_creation", + CAST(data #>> '{fields,Modifiée le}' AS DATE) AS "date_maj", + CAST(data #>> '{fields,En ligne}' AS BOOLEAN) AS "mode_acceuil", + TRIM(data #>> '{fields,Contact Email}') AS "courriel", + data #>> '{fields,Contact Tel}' AS "telephone", + data #>> '{fields,Autres Conditions}' AS "autres_conditions", + data #>> '{fields,Url Mes Aides}' AS "lien_source", + STRING_TO_ARRAY(data #>> '{fields,Type}', ',') AS "types" + FROM source +) + +SELECT * FROM final diff --git a/pipeline/defaults.env b/pipeline/defaults.env index 73ad9860d..5a4861875 100644 --- a/pipeline/defaults.env +++ b/pipeline/defaults.env @@ -20,6 +20,7 @@ AIRFLOW_VAR_INSEE_FIRSTNAME_FILE_URL=https://www.insee.fr/fr/statistiques/fichie AIRFLOW_VAR_MEDNUM_API_URL=https://cartographie.societenumerique.gouv.fr/api/v0/ AIRFLOW_VAR_MES_AIDES_AIDES_URL=https://airtable.com/appRga7C9USklxYiV/tblcAC5yMV3Ftzv5c/viwMte3unsIYXxY9a AIRFLOW_VAR_MES_AIDES_GARAGES_URL=https://airtable.com/appRga7C9USklxYiV/tblfhYoBpcQoJwGIv/viwoJsw0vsAnU0fAo +AIRFLOW_VAR_MES_AIDES_PERMIS_VELO_URL=https://airtable.com/appRga7C9USklxYiV/tblcAC5yMV3Ftzv5c/viwMte3unsIYXxY9a AIRFLOW_VAR_MONENFANT_BASE_URL=https://monenfant.fr AIRFLOW_VAR_ODSPEP_S3_KEY_PREFIX=sources/odspep/2023-01-23/denormalized/Exports/ AIRFLOW_VAR_RESEAU_ALPHA_URL=https://www.reseau-alpha.org