From 62c010857344c4e5cee57a24b55ad0b7595a3938 Mon Sep 17 00:00:00 2001 From: Hugo Lecuyer Date: Thu, 19 Sep 2024 14:57:19 +0200 Subject: [PATCH] refacto and fixes following val coments --- pipeline/dags/dag_utils/sources/__init__.py | 5 - .../quality/int_quality__stats.sql | 2 +- .../sources/mes_aides/_mes_aides__models.yml | 10 ++ .../garage/_mes_aides_garage__models.yml | 23 ++--- ...ql => int_mes_aides_garages__adresses.sql} | 0 ...ql => int_mes_aides_garages__services.sql} | 0 ... => int_mes_aides_garages__structures.sql} | 0 .../mes_aides/int_mes_aides__adresses.sql | 38 +------- .../{garage => }/int_mes_aides__contacts.sql | 6 +- .../mes_aides/int_mes_aides__services.sql | 90 +---------------- .../mes_aides/int_mes_aides__structures.sql | 60 +----------- .../permis_velo/_mes_aides__models.yml | 12 +-- .../int_mes_aides__adresses_permis_velo.sql | 19 ---- .../int_mes_aides__structures_permis_velo.sql | 56 ----------- .../int_mes_aides_permis_velo__adresses.sql | 19 ++++ ...> int_mes_aides_permis_velo__services.sql} | 96 ++++++++++--------- .../int_mes_aides_permis_velo__structures.sql | 30 ++++++ .../_decoupage_administratif__models.yml | 2 +- .../sources/mes_aides/_mes_aides__models.yml | 1 - .../mes_aides/stg_mes_aides__aides.sql | 22 ----- .../mes_aides/stg_mes_aides__permis_velo.sql | 40 ++++---- pipeline/defaults.env | 1 - 22 files changed, 156 insertions(+), 376 deletions(-) rename pipeline/dbt/models/intermediate/sources/mes_aides/garage/{int_mes_aides__adresses_garage.sql => int_mes_aides_garages__adresses.sql} (100%) rename pipeline/dbt/models/intermediate/sources/mes_aides/garage/{int_mes_aides__services_garage.sql => int_mes_aides_garages__services.sql} (100%) rename pipeline/dbt/models/intermediate/sources/mes_aides/garage/{int_mes_aides__structures_garage.sql => int_mes_aides_garages__structures.sql} (100%) rename pipeline/dbt/models/intermediate/sources/mes_aides/{garage => }/int_mes_aides__contacts.sql (76%) delete mode 100644 pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__adresses_permis_velo.sql delete mode 100644 pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__structures_permis_velo.sql create mode 100644 pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides_permis_velo__adresses.sql rename pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/{int_mes_aides__services_permis_velo.sql => int_mes_aides_permis_velo__services.sql} (70%) create mode 100644 pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides_permis_velo__structures.sql delete mode 100644 pipeline/dbt/models/staging/sources/mes_aides/stg_mes_aides__aides.sql diff --git a/pipeline/dags/dag_utils/sources/__init__.py b/pipeline/dags/dag_utils/sources/__init__.py index 4735464ab..acde0c289 100644 --- a/pipeline/dags/dag_utils/sources/__init__.py +++ b/pipeline/dags/dag_utils/sources/__init__.py @@ -49,11 +49,6 @@ "url": Variable.get("MES_AIDES_GARAGES_URL", None), "token": Variable.get("MES_AIDES_AIRTABLE_KEY", None), }, - "aides": { - "filename": "aides.json", - "url": Variable.get("MES_AIDES_AIDES_URL", None), - "token": Variable.get("MES_AIDES_AIRTABLE_KEY", None), - }, "permis_velo": { "filename": "permis_velo.json", "url": Variable.get("MES_AIDES_PERMIS_VELO_URL", None), diff --git a/pipeline/dbt/models/intermediate/quality/int_quality__stats.sql b/pipeline/dbt/models/intermediate/quality/int_quality__stats.sql index 2ba177fec..b36eab2ad 100644 --- a/pipeline/dbt/models/intermediate/quality/int_quality__stats.sql +++ b/pipeline/dbt/models/intermediate/quality/int_quality__stats.sql @@ -29,7 +29,7 @@ -- depends_on: {{ ref('stg_mediation_numerique__structures') }} -- depends_on: {{ ref('int_mediation_numerique__services') }} -- depends_on: {{ ref('int_mediation_numerique__structures') }} --- depends_on: {{ ref('stg_mes_aides__aides') }} +-- depends_on: {{ ref('stg_mes_aides__permis_velo') }} -- depends_on: {{ ref('stg_mes_aides__garages') }} -- depends_on: {{ ref('int_mes_aides__services') }} -- depends_on: {{ ref('int_mes_aides__structures') }} diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/_mes_aides__models.yml b/pipeline/dbt/models/intermediate/sources/mes_aides/_mes_aides__models.yml index 1fc9b0c92..d71d05380 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/_mes_aides__models.yml +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/_mes_aides__models.yml @@ -54,3 +54,13 @@ models: - relationships: to: ref('int_mes_aides__adresses') field: id + - name: int_mes_aides__contacts + columns: + - name: contact_uid + data_tests: + - unique + - not_null + - name: courriel + data_tests: + - not_null + - dbt_utils.not_empty_string \ No newline at end of file diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/garage/_mes_aides_garage__models.yml b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/_mes_aides_garage__models.yml index 13c37645c..80d6c1820 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/garage/_mes_aides_garage__models.yml +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/_mes_aides_garage__models.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: int_mes_aides__adresses_garage + - name: int_mes_aides_garages__adresses data_tests: - check_adresse: config: @@ -13,18 +13,7 @@ models: - not_null - dbt_utils.not_empty_string - - name: int_mes_aides__contacts - columns: - - name: contact_uid - data_tests: - - unique - - not_null - - name: courriel - data_tests: - - not_null - - dbt_utils.not_empty_string - - - name: int_mes_aides__structures_garages + - name: int_mes_aides_garages__structures data_tests: - check_structure: config: @@ -39,10 +28,10 @@ models: data_tests: - not_null - relationships: - to: ref('int_mes_aides__adresses_garage') + to: ref('int_mes_aides_garages__adresses') field: id - - name: int_mes_aides__services_garage + - name: int_mes_aides_garages__services data_tests: - check_service: config: @@ -57,11 +46,11 @@ models: data_tests: - not_null - relationships: - to: ref('int_mes_aides__structures_garage') + to: ref('int_mes_aides_garages__structures') field: id - name: adresse_id data_tests: - not_null - relationships: - to: ref('int_mes_aides__adresses_garage') + to: ref('int_mes_aides_garages__adresses') field: id \ No newline at end of file diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__adresses_garage.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides_garages__adresses.sql similarity index 100% rename from pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__adresses_garage.sql rename to pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides_garages__adresses.sql diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__services_garage.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides_garages__services.sql similarity index 100% rename from pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__services_garage.sql rename to pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides_garages__services.sql diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__structures_garage.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides_garages__structures.sql similarity index 100% rename from pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__structures_garage.sql rename to pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides_garages__structures.sql diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__adresses.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__adresses.sql index 81cd45944..7effece89 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__adresses.sql +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__adresses.sql @@ -1,35 +1,3 @@ -WITH adresses_garage AS ( - SELECT * FROM {{ ref('int_mes_aides__adresses_garage') }} -), - -adresses_permis_velo AS ( - SELECT * FROM {{ ref('int_mes_aides__adresses_permis_velo') }} -), - -final AS ( - SELECT - adresses_garage.id, - adresses_garage.commune, - adresses_garage.code_insee, - adresses_garage.longitude, - adresses_garage.latitude, - adresses_garage.source, - adresses_garage.code_postal, - adresses_garage.adresse, - adresses_garage.complement_adresse - FROM adresses_garage - UNION ALL - SELECT - adresses_permis_velo.id, - adresses_permis_velo.commune, - adresses_permis_velo.code_insee, - adresses_permis_velo.longitude, - adresses_permis_velo.latitude, - adresses_permis_velo.source, - adresses_permis_velo.code_postal, - adresses_permis_velo.adresse, - adresses_permis_velo.complement_adresse - FROM adresses_permis_velo -) - -SELECT * FROM final +{{ dbt_utils.union_relations( + relations=[ref('int_mes_aides_garages__adresses'), ref('int_mes_aides_permis_velo__adresses')] +) }} diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__contacts.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__contacts.sql similarity index 76% rename from pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__contacts.sql rename to pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__contacts.sql index 2cab5cb20..5e202d3ca 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/garage/int_mes_aides__contacts.sql +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__contacts.sql @@ -8,10 +8,10 @@ WITH structure_contacts AS ( service_contacts AS ( SELECT - email AS "courriel", + contact_email AS "courriel", 'mes-aides:aides:' || id AS contact_uid - FROM {{ ref('stg_mes_aides__aides') }} - WHERE email IS NOT NULL + FROM {{ ref('stg_mes_aides__permis_velo') }} + WHERE contact_email IS NOT NULL ), final AS ( diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__services.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__services.sql index 8adb2e88d..82515118c 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__services.sql +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__services.sql @@ -1,87 +1,3 @@ -WITH services_garage AS ( - SELECT * FROM {{ ref('int_mes_aides__services_garage') }} -), - -services_permis_velo AS ( - SELECT * FROM {{ ref('int_mes_aides__services_permis_velo') }} -), - -final AS ( - SELECT - adresse_id, - contact_public, - cumulable, - date_creation, - date_maj, - date_suspension, - formulaire_en_ligne, - frais_autres, - id, - justificatifs, - lien_source, - modes_accueil, - modes_orientation_accompagnateur, - modes_orientation_accompagnateur_autres, - modes_orientation_beneficiaire, - modes_orientation_beneficiaire_autres, - nom, - presentation_resume, - presentation_detail, - prise_rdv, - profils, - recurrence, - source, - structure_id, - thematiques, - types, - zone_diffusion_code, - zone_diffusion_nom, - zone_diffusion_type, - pre_requis, - contact_nom_prenom, - courriel, - telephone, - frais, - page_web - FROM services_garage - UNION ALL - SELECT - adresse_id, - contact_public, - cumulable, - date_creation, - date_maj, - date_suspension, - formulaire_en_ligne, - frais_autres, - id, - justificatifs, - lien_source, - modes_accueil, - modes_orientation_accompagnateur, - modes_orientation_accompagnateur_autres, - modes_orientation_beneficiaire, - modes_orientation_beneficiaire_autres, - nom, - presentation_resume, - presentation_detail, - prise_rdv, - profils, - recurrence, - source, - structure_id, - thematiques, - types, - zone_diffusion_code, - zone_diffusion_nom, - zone_diffusion_type, - pre_requis, - contact_nom_prenom, - courriel, - telephone, - frais, - page_web - FROM services_permis_velo -) - -SELECT * FROM final +{{ dbt_utils.union_relations( + relations=[ref('int_mes_aides_garages__services'), ref('int_mes_aides_permis_velo__services')] +) }} diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__structures.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__structures.sql index bd27cfcec..f2558ad21 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__structures.sql +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__structures.sql @@ -1,57 +1,3 @@ -WITH structures_garage AS ( - SELECT * FROM {{ ref('int_mes_aides__structures_garage') }} -), - -structures_permis_velo AS ( - SELECT * FROM {{ ref('int_mes_aides__structures_permis_velo') }} -), - -final AS ( - SELECT - id, - adresse_id, - siret, - antenne, - rna, - nom, - telephone, - courriel, - site_web, - "source", - lien_source, - horaires_ouverture, - accessibilite, - labels_nationaux, - labels_autres, - typologie, - presentation_resume, - presentation_detail, - date_maj, - thematiques - FROM structures_garage - UNION ALL - SELECT - id, - adresse_id, - siret, - antenne, - rna, - nom, - telephone, - courriel, - site_web, - "source", - lien_source, - horaires_ouverture, - accessibilite, - labels_nationaux, - labels_autres, - typologie, - presentation_resume, - presentation_detail, - date_maj, - thematiques - FROM structures_permis_velo -) - -SELECT * FROM final +{{ dbt_utils.union_relations( + relations=[ref('int_mes_aides_garages__structures'), ref('int_mes_aides_permis_velo__structures')] +) }} diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/_mes_aides__models.yml b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/_mes_aides__models.yml index e1ae612d6..9a20a02ee 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/_mes_aides__models.yml +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/_mes_aides__models.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: int_mes_aides__adresses_permis_velo + - name: int_mes_aides_permis_velo__adresses data_tests: - check_adresse: config: @@ -14,7 +14,7 @@ models: - dbt_utils.not_empty_string - - name: int_mes_aides__structures_permis_velo + - name: int_mes_aides_permis_velo__structures data_tests: - check_structure: config: @@ -29,10 +29,10 @@ models: data_tests: - not_null - relationships: - to: ref('int_mes_aides__adresses_permis_velo') + to: ref('int_mes_aides_permis_velo__adresses') field: id - - name: int_mes_aides__services_permis_velo + - name: int_mes_aides_permis_velo__services data_tests: - check_service: config: @@ -47,11 +47,11 @@ models: data_tests: - not_null - relationships: - to: ref('int_mes_aides__structures_permis_velo') + to: ref('int_mes_aides_permis_velo__structures') field: id - name: adresse_id data_tests: - not_null - relationships: - to: ref('int_mes_aides__adresses_permis_velo') + to: ref('int_mes_aides_permis_velo__adresses') field: id diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__adresses_permis_velo.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__adresses_permis_velo.sql deleted file mode 100644 index 84ea35ddb..000000000 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__adresses_permis_velo.sql +++ /dev/null @@ -1,19 +0,0 @@ -WITH permis_velo AS ( - SELECT * FROM {{ ref('stg_mes_aides__permis_velo') }} -), - -final AS ( - SELECT - id AS "id", - ville_nom AS "commune", - code_postal AS "code_postal", - NULL AS "code_insee", - NULL AS "adresse", - NULL AS "complement_adresse", - CAST(NULL AS FLOAT) AS "longitude", - CAST(NULL AS FLOAT) AS "latitude", - _di_source_id AS "source" - FROM permis_velo -) - -SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__structures_permis_velo.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__structures_permis_velo.sql deleted file mode 100644 index 4e7d70606..000000000 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__structures_permis_velo.sql +++ /dev/null @@ -1,56 +0,0 @@ -WITH permis_velo AS ( - SELECT * FROM {{ ref('stg_mes_aides__permis_velo') }} -), - -final AS ( - SELECT - siret_structure AS "id", - MAX(id) AS "adresse_id", - siret_structure AS "siret", - CAST(NULL AS BOOLEAN) AS "antenne", - NULL AS "rna", - MAX(nom) AS "nom", - MAX(telephone) AS "telephone", - MAX(courriel) AS "courriel", - MAX(page_web) AS "site_web", - MAX(_di_source_id) AS "source", - MAX(lien_source) AS "lien_source", - NULL AS "horaires_ouverture", - NULL AS "accessibilite", - CAST(NULL AS TEXT []) AS "labels_nationaux", - CAST(NULL AS TEXT []) AS "labels_autres", - NULL AS "typologie", - NULL AS "presentation_resume", - NULL AS "presentation_detail", - MAX(CAST(date_maj AS DATE)) AS "date_maj", - ARRAY['mobilite'] AS "thematiques" - FROM permis_velo - WHERE siret_structure IS NOT NULL - GROUP BY siret_structure - UNION ALL - SELECT - id AS "id", - id AS "adresse_id", - NULL AS "siret", - CAST(NULL AS BOOLEAN) AS "antenne", - NULL AS "rna", - nom AS "nom", - telephone AS "telephone", - courriel AS "courriel", - page_web AS "site_web", - _di_source_id AS "source", - lien_source AS "lien_source", - NULL AS "horaires_ouverture", - NULL AS "accessibilite", - CAST(NULL AS TEXT []) AS "labels_nationaux", - CAST(NULL AS TEXT []) AS "labels_autres", - NULL AS "typologie", - NULL AS "presentation_resume", - NULL AS "presentation_detail", - CAST(date_maj AS DATE) AS "date_maj", - ARRAY['mobilite'] AS "thematiques" - FROM permis_velo - WHERE siret_structure IS NULL -) - -SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides_permis_velo__adresses.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides_permis_velo__adresses.sql new file mode 100644 index 000000000..2c5dd4dc7 --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides_permis_velo__adresses.sql @@ -0,0 +1,19 @@ +WITH permis_velo AS ( + SELECT * FROM {{ ref('stg_mes_aides__permis_velo') }} +), + +final AS ( + SELECT + id AS "id", + liaisons_ville_nom AS "commune", + liaisons_code_postal AS "code_postal", + NULL AS "code_insee", + NULL AS "adresse", + NULL AS "complement_adresse", + CAST(NULL AS FLOAT) AS "longitude", + CAST(NULL AS FLOAT) AS "latitude", + _di_source_id AS "source" + FROM permis_velo +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__services_permis_velo.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides_permis_velo__services.sql similarity index 70% rename from pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__services_permis_velo.sql rename to pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides_permis_velo__services.sql index d4191e5c7..b0ca6a066 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides__services_permis_velo.sql +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides_permis_velo__services.sql @@ -8,6 +8,10 @@ communes AS ( SELECT * FROM {{ ref('stg_decoupage_administratif__communes') }} ), +departements AS ( + SELECT * FROM {{ ref('stg_decoupage_administratif__departements') }} +), + zone_code AS ( SELECT permis_velo.id, @@ -16,7 +20,7 @@ zone_code AS ( communes.code_departement, communes.code_region FROM permis_velo - LEFT JOIN communes ON permis_velo.code_postal = ANY(communes.codes_postaux) AND permis_velo.ville_nom = communes.nom + LEFT JOIN communes ON permis_velo.liaisons_code_postal = ANY(communes.codes_postaux) AND permis_velo.liaisons_ville_nom = communes.nom ), types_mapping AS ( @@ -46,7 +50,7 @@ zone_diffusion AS ( zone_de_diffusion_mapping.zone_di, permis_velo.id FROM permis_velo - INNER JOIN zone_de_diffusion_mapping ON permis_velo.zone_diffusion_type = zone_de_diffusion_mapping.zone_mes_aides + INNER JOIN zone_de_diffusion_mapping ON permis_velo.zone_geographique = zone_de_diffusion_mapping.zone_mes_aides ), modes_orientation_mapping AS ( @@ -113,74 +117,78 @@ transformed_types AS ( final AS ( SELECT - permis_velo.id AS "adresse_id", - TRUE AS "contact_public", - TRUE AS "cumulable", - permis_velo.date_creation AS "date_creation", - permis_velo.date_maj AS "date_maj", - CAST(NULL AS DATE) AS "date_suspension", - NULL AS "formulaire_en_ligne", - permis_velo.frais_autres AS "frais_autres", - permis_velo.id AS "id", - CAST(NULL AS TEXT []) AS "justificatifs", - permis_velo.lien_source AS "lien_source", + permis_velo.id AS "adresse_id", + TRUE AS "contact_public", + TRUE AS "cumulable", + permis_velo.creee_le AS "date_creation", + permis_velo.modifiee_le AS "date_maj", + CAST(NULL AS DATE) AS "date_suspension", + NULL AS "formulaire_en_ligne", + permis_velo.nature AS "frais_autres", + permis_velo.id AS "id", + CAST(NULL AS TEXT []) AS "justificatifs", + permis_velo.url_mes_aides AS "lien_source", CASE - WHEN permis_velo.mode_acceuil = TRUE THEN ARRAY['a-distance'] + WHEN permis_velo.en_ligne = TRUE THEN ARRAY['a-distance'] ELSE ARRAY['en-presentiel'] - END AS "modes_accueil", + END AS "modes_accueil", ( SELECT modes_orientation_mapping.modes_orientation_accompagnateur FROM modes_orientation_mapping WHERE permis_velo.methode = modes_orientation_mapping.demarche - ) AS "modes_orientation_accompagnateur", - NULL AS "modes_orientation_accompagnateur_autres", + ) AS "modes_orientation_accompagnateur", + NULL AS "modes_orientation_accompagnateur_autres", ( SELECT modes_orientation_mapping.modes_orientation_beneficiaire FROM modes_orientation_mapping WHERE permis_velo.methode = modes_orientation_mapping.demarche - ) AS "modes_orientation_beneficiaire", - permis_velo.mode_orientation_beneficiare_autre AS "modes_orientation_beneficiaire_autres", - permis_velo.nom AS "nom", - NULL AS "presentation_resume", - permis_velo.presentation_detail AS "presentation_detail", - NULL AS "prise_rdv", - CAST(NULL AS TEXT []) AS "profils", - NULL AS "recurrence", - permis_velo._di_source_id AS "source", - COALESCE(permis_velo.siret_structure, permis_velo.id) AS "structure_id", + ) AS "modes_orientation_beneficiaire", + permis_velo.demarche AS "modes_orientation_beneficiaire_autres", + permis_velo.nom AS "nom", + CASE + WHEN LENGTH(permis_velo.modalite_versement) > 297 THEN SUBSTRING(permis_velo.modalite_versement FROM 1 FOR 294) || '...' + ELSE permis_velo.modalite_versement + END AS "presentation_resume", + permis_velo.modalite_versement AS "presentation_detail", + NULL AS "prise_rdv", + CAST(NULL AS TEXT []) AS "profils", + NULL AS "recurrence", + permis_velo._di_source_id AS "source", + permis_velo.id AS "structure_id", ( SELECT mapping_thematiques.correspondance_di FROM mapping_thematiques WHERE - permis_velo.besoins_mes_aides = mapping_thematiques.besoins - AND permis_velo.thematique_mes_aides = mapping_thematiques.thematiques - ) AS "thematiques", - transformed_types.transformed_types AS "types", + permis_velo.liaisons_besoins_mes_aides = mapping_thematiques.besoins + AND permis_velo.slug_thematique_mes_aides = mapping_thematiques.thematiques + ) AS "thematiques", + transformed_types.transformed_types AS "types", CASE WHEN zone_diffusion.zone_di = 'commune' THEN zone_code.code WHEN zone_diffusion.zone_di = 'epci' THEN zone_code.code_epci WHEN zone_diffusion.zone_di = 'region' THEN zone_code.code_region WHEN zone_diffusion.zone_di = 'departement' THEN zone_code.code_departement WHEN zone_diffusion.zone_di = 'pays' THEN NULL - END AS "zone_diffusion_code", + END AS "zone_diffusion_code", CASE - WHEN zone_diffusion.zone_di = 'commune' THEN permis_velo.ville_nom - WHEN zone_diffusion.zone_di = 'epci' THEN permis_velo.ville_nom - WHEN zone_diffusion.zone_di = 'region' THEN permis_velo.region - WHEN zone_diffusion.zone_di = 'departement' THEN permis_velo.departement + WHEN zone_diffusion.zone_di = 'commune' THEN permis_velo.liaisons_ville_nom + WHEN zone_diffusion.zone_di = 'epci' THEN permis_velo.liaisons_ville_nom + WHEN zone_diffusion.zone_di = 'region' THEN permis_velo.liaisons_region + WHEN zone_diffusion.zone_di = 'departement' THEN departements.nom WHEN zone_diffusion.zone_di = 'pays' THEN NULL - END AS "zone_diffusion_nom", - zone_diffusion.zone_di AS "zone_diffusion_type", - CAST(NULL AS TEXT []) AS "pre_requis", - NULL AS "contact_nom_prenom", - permis_velo.courriel AS "courriel", - permis_velo.telephone AS "telephone", - CAST(NULL AS TEXT []) AS "frais", - permis_velo.page_web AS "page_web" + END AS "zone_diffusion_nom", + zone_diffusion.zone_di AS "zone_diffusion_type", + CAST(NULL AS TEXT []) AS "pre_requis", + NULL AS "contact_nom_prenom", + permis_velo.contact_email AS "courriel", + permis_velo.contact_telephone AS "telephone", + CAST(NULL AS TEXT []) AS "frais", + permis_velo.site AS "page_web" FROM permis_velo LEFT JOIN transformed_types ON permis_velo.id = transformed_types.id LEFT JOIN zone_diffusion ON permis_velo.id = zone_diffusion.id LEFT JOIN zone_code ON permis_velo.id = zone_code.id + LEFT JOIN departements ON permis_velo.num_departement = departements.code ) SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides_permis_velo__structures.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides_permis_velo__structures.sql new file mode 100644 index 000000000..bde2084db --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/permis_velo/int_mes_aides_permis_velo__structures.sql @@ -0,0 +1,30 @@ +WITH permis_velo AS ( + SELECT * FROM {{ ref('stg_mes_aides__permis_velo') }} +), + +final AS ( + SELECT + id AS "id", + id AS "adresse_id", + siret_structure AS "siret", + CAST(NULL AS BOOLEAN) AS "antenne", + NULL AS "rna", + nom AS "nom", + contact_telephone AS "telephone", + contact_email AS "courriel", + site AS "site_web", + _di_source_id AS "source", + url_mes_aides AS "lien_source", + NULL AS "horaires_ouverture", + NULL AS "accessibilite", + CAST(NULL AS TEXT []) AS "labels_nationaux", + CAST(NULL AS TEXT []) AS "labels_autres", + NULL AS "typologie", + NULL AS "presentation_resume", + NULL AS "presentation_detail", + CAST(modifiee_le AS DATE) AS "date_maj", + ARRAY['mobilite'] AS "thematiques" + FROM permis_velo +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/staging/decoupage_administratif/_decoupage_administratif__models.yml b/pipeline/dbt/models/staging/decoupage_administratif/_decoupage_administratif__models.yml index efd234777..664acfcfa 100644 --- a/pipeline/dbt/models/staging/decoupage_administratif/_decoupage_administratif__models.yml +++ b/pipeline/dbt/models/staging/decoupage_administratif/_decoupage_administratif__models.yml @@ -1,4 +1,4 @@ - version: 2 +version: 2 models: - name: stg_decoupage_administratif__regions diff --git a/pipeline/dbt/models/staging/sources/mes_aides/_mes_aides__models.yml b/pipeline/dbt/models/staging/sources/mes_aides/_mes_aides__models.yml index 3069370ba..34a736cb6 100644 --- a/pipeline/dbt/models/staging/sources/mes_aides/_mes_aides__models.yml +++ b/pipeline/dbt/models/staging/sources/mes_aides/_mes_aides__models.yml @@ -1,6 +1,5 @@ version: 2 models: - - name: stg_mes_aides__aides - name: stg_mes_aides__garages - name: stg_mes_aides__permis_velo diff --git a/pipeline/dbt/models/staging/sources/mes_aides/stg_mes_aides__aides.sql b/pipeline/dbt/models/staging/sources/mes_aides/stg_mes_aides__aides.sql deleted file mode 100644 index b0c5bf126..000000000 --- a/pipeline/dbt/models/staging/sources/mes_aides/stg_mes_aides__aides.sql +++ /dev/null @@ -1,22 +0,0 @@ -WITH source AS ( - {{ stg_source_header('mes_aides', 'aides') }} -), - -final AS ( - SELECT - _di_source_id AS "_di_source_id", - (data #>> '{fields,Ville Longitude}')::FLOAT AS "ville_longitude", - (data #>> '{fields,Ville Latitude}')::FLOAT AS "ville_latitude", - (data #>> '{fields,Modifié le}')::TIMESTAMP WITH TIME ZONE AS "modifie_le", - data #>> '{fields,ID}' AS "id", - data #>> '{fields,Nom}' AS "nom", - data #>> '{fields,Ville Nom}' AS "ville_nom", - data #>> '{fields,Code Postal}' AS "code_postal", - data #>> '{fields,Adresse}' AS "adresse", - data #>> '{fields,Téléphone}' AS "telephone", - data #>> '{fields,Email}' AS "email", - data #>> '{fields,Url}' AS "url" - FROM source -) - -SELECT * FROM final diff --git a/pipeline/dbt/models/staging/sources/mes_aides/stg_mes_aides__permis_velo.sql b/pipeline/dbt/models/staging/sources/mes_aides/stg_mes_aides__permis_velo.sql index 5db60fa4e..13d578678 100644 --- a/pipeline/dbt/models/staging/sources/mes_aides/stg_mes_aides__permis_velo.sql +++ b/pipeline/dbt/models/staging/sources/mes_aides/stg_mes_aides__permis_velo.sql @@ -6,32 +6,30 @@ final AS ( SELECT _di_source_id AS "_di_source_id", data #>> '{fields,ID}' AS "id", - TRIM(SUBSTRING(data #>> '{fields,Liaisons Villes}' FROM '^(.*) \(')) AS "ville_nom", - TRIM(SUBSTRING(data #>> '{fields,Liaisons Villes}' FROM '\((\d+)\)$')) AS "code_postal", - SUBSTRING( - data #>> '{fields,Liaisons Départements}' FROM POSITION('-' IN data #>> '{fields,Liaisons Départements}') + 1 -- noqa - ) AS "departement", - data #>> '{fields,Liaisons Régions}' AS "region", + TRIM(SUBSTRING(data #>> '{fields,Liaisons Villes}' FROM '^(.*) \(')) AS "liaisons_ville_nom", + TRIM(SUBSTRING(data #>> '{fields,Liaisons Villes}' FROM '\((\d+)\)$')) AS "liaisons_code_postal", + data #>> '{fields,N° Départements}' AS "num_departement", + data #>> '{fields,Liaisons Régions}' AS "liaisons_region", data #>> '{fields,Nom}' AS "nom", - data #>> '{fields,Liaisons Besoins}' AS "besoins_mes_aides", - data #>> '{fields,Slug Thématiques}' AS "thematique_mes_aides", - data #>> '{fields,Zone géographique}' AS "zone_diffusion_type", - data #>> '{fields,Nature}' AS "frais_autres", - data #>> '{fields,Démarches}' AS "mode_orientation_beneficiare_autre", - data #>> '{fields,"Modalité et versement"}' AS "presentation_detail", + data #>> '{fields,Liaisons Besoins}' AS "liaisons_besoins_mes_aides", + data #>> '{fields,Slug Thématiques}' AS "slug_thematique_mes_aides", + data #>> '{fields,Zone géographique}' AS "zone_geographique", + data #>> '{fields,Nature}' AS "nature", + data #>> '{fields,Démarches}' AS "demarche", + data #>> '{fields,"Modalité et versement"}' AS "modalite_versement", data #>> '{fields,"Méthode"}' AS "methode", data #>> '{fields,SIRET}' AS "siret_structure", - data #>> '{fields,Slug Organisme}' AS "lien_source_structure", - data #>> '{fields,Site}' AS "page_web", - data #>> '{fields,Nom Organisme}' AS "nom_structure", + data #>> '{fields,Slug Organisme}' AS "slug_organisme_structure", + data #>> '{fields,Site}' AS "site", + data #>> '{fields,Nom Organisme}' AS "nom_organisme_structure", data #>> '{fields,Organisme Type}' AS "typologie_structure", - CAST(data #>> '{fields,Créée le}' AS DATE) AS "date_creation", - CAST(data #>> '{fields,Modifiée le}' AS DATE) AS "date_maj", - CAST(data #>> '{fields,En ligne}' AS BOOLEAN) AS "mode_acceuil", - TRIM(data #>> '{fields,Contact Email}') AS "courriel", - data #>> '{fields,Contact Tel}' AS "telephone", + CAST(data #>> '{fields,Créée le}' AS DATE) AS "creee_le", + CAST(data #>> '{fields,Modifiée le}' AS DATE) AS "modifiee_le", + CAST(data #>> '{fields,En ligne}' AS BOOLEAN) AS "en_ligne", + TRIM(data #>> '{fields,Contact Email}') AS "contact_email", + data #>> '{fields,Contact Tel}' AS "contact_telephone", data #>> '{fields,Autres Conditions}' AS "autres_conditions", - data #>> '{fields,Url Mes Aides}' AS "lien_source", + data #>> '{fields,Url Mes Aides}' AS "url_mes_aides", STRING_TO_ARRAY(data #>> '{fields,Type}', ',') AS "types" FROM source ) diff --git a/pipeline/defaults.env b/pipeline/defaults.env index 5a4861875..45b689d5b 100644 --- a/pipeline/defaults.env +++ b/pipeline/defaults.env @@ -18,7 +18,6 @@ AIRFLOW_VAR_ETAB_PUB_FILE_URL=https://www.data.gouv.fr/fr/datasets/r/73302880-e4 AIRFLOW_VAR_FINESS_FILE_URL=https://www.data.gouv.fr/fr/datasets/r/3dc9b1d5-0157-440d-a7b5-c894fcfdfd45 AIRFLOW_VAR_INSEE_FIRSTNAME_FILE_URL=https://www.insee.fr/fr/statistiques/fichier/2540004/nat2021_csv.zip AIRFLOW_VAR_MEDNUM_API_URL=https://cartographie.societenumerique.gouv.fr/api/v0/ -AIRFLOW_VAR_MES_AIDES_AIDES_URL=https://airtable.com/appRga7C9USklxYiV/tblcAC5yMV3Ftzv5c/viwMte3unsIYXxY9a AIRFLOW_VAR_MES_AIDES_GARAGES_URL=https://airtable.com/appRga7C9USklxYiV/tblfhYoBpcQoJwGIv/viwoJsw0vsAnU0fAo AIRFLOW_VAR_MES_AIDES_PERMIS_VELO_URL=https://airtable.com/appRga7C9USklxYiV/tblcAC5yMV3Ftzv5c/viwMte3unsIYXxY9a AIRFLOW_VAR_MONENFANT_BASE_URL=https://monenfant.fr