Skip to content

Commit

Permalink
refacto and fixes following val coments
Browse files Browse the repository at this point in the history
  • Loading branch information
hlecuyer committed Sep 19, 2024
1 parent 617ab96 commit 62c0108
Show file tree
Hide file tree
Showing 22 changed files with 156 additions and 376 deletions.
5 changes: 0 additions & 5 deletions pipeline/dags/dag_utils/sources/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,6 @@
"url": Variable.get("MES_AIDES_GARAGES_URL", None),
"token": Variable.get("MES_AIDES_AIRTABLE_KEY", None),
},
"aides": {
"filename": "aides.json",
"url": Variable.get("MES_AIDES_AIDES_URL", None),
"token": Variable.get("MES_AIDES_AIRTABLE_KEY", None),
},
"permis_velo": {
"filename": "permis_velo.json",
"url": Variable.get("MES_AIDES_PERMIS_VELO_URL", None),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
-- depends_on: {{ ref('stg_mediation_numerique__structures') }}
-- depends_on: {{ ref('int_mediation_numerique__services') }}
-- depends_on: {{ ref('int_mediation_numerique__structures') }}
-- depends_on: {{ ref('stg_mes_aides__aides') }}
-- depends_on: {{ ref('stg_mes_aides__permis_velo') }}
-- depends_on: {{ ref('stg_mes_aides__garages') }}
-- depends_on: {{ ref('int_mes_aides__services') }}
-- depends_on: {{ ref('int_mes_aides__structures') }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,13 @@ models:
- relationships:
to: ref('int_mes_aides__adresses')
field: id
- name: int_mes_aides__contacts
columns:
- name: contact_uid
data_tests:
- unique
- not_null
- name: courriel
data_tests:
- not_null
- dbt_utils.not_empty_string
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
version: 2

models:
- name: int_mes_aides__adresses_garage
- name: int_mes_aides_garages__adresses
data_tests:
- check_adresse:
config:
Expand All @@ -13,18 +13,7 @@ models:
- not_null
- dbt_utils.not_empty_string

- name: int_mes_aides__contacts
columns:
- name: contact_uid
data_tests:
- unique
- not_null
- name: courriel
data_tests:
- not_null
- dbt_utils.not_empty_string

- name: int_mes_aides__structures_garages
- name: int_mes_aides_garages__structures
data_tests:
- check_structure:
config:
Expand All @@ -39,10 +28,10 @@ models:
data_tests:
- not_null
- relationships:
to: ref('int_mes_aides__adresses_garage')
to: ref('int_mes_aides_garages__adresses')
field: id

- name: int_mes_aides__services_garage
- name: int_mes_aides_garages__services
data_tests:
- check_service:
config:
Expand All @@ -57,11 +46,11 @@ models:
data_tests:
- not_null
- relationships:
to: ref('int_mes_aides__structures_garage')
to: ref('int_mes_aides_garages__structures')
field: id
- name: adresse_id
data_tests:
- not_null
- relationships:
to: ref('int_mes_aides__adresses_garage')
to: ref('int_mes_aides_garages__adresses')
field: id
Original file line number Diff line number Diff line change
@@ -1,35 +1,3 @@
WITH adresses_garage AS (
SELECT * FROM {{ ref('int_mes_aides__adresses_garage') }}
),

adresses_permis_velo AS (
SELECT * FROM {{ ref('int_mes_aides__adresses_permis_velo') }}
),

final AS (
SELECT
adresses_garage.id,
adresses_garage.commune,
adresses_garage.code_insee,
adresses_garage.longitude,
adresses_garage.latitude,
adresses_garage.source,
adresses_garage.code_postal,
adresses_garage.adresse,
adresses_garage.complement_adresse
FROM adresses_garage
UNION ALL
SELECT
adresses_permis_velo.id,
adresses_permis_velo.commune,
adresses_permis_velo.code_insee,
adresses_permis_velo.longitude,
adresses_permis_velo.latitude,
adresses_permis_velo.source,
adresses_permis_velo.code_postal,
adresses_permis_velo.adresse,
adresses_permis_velo.complement_adresse
FROM adresses_permis_velo
)

SELECT * FROM final
{{ dbt_utils.union_relations(
relations=[ref('int_mes_aides_garages__adresses'), ref('int_mes_aides_permis_velo__adresses')]
) }}
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@ WITH structure_contacts AS (

service_contacts AS (
SELECT
email AS "courriel",
contact_email AS "courriel",
'mes-aides:aides:' || id AS contact_uid
FROM {{ ref('stg_mes_aides__aides') }}
WHERE email IS NOT NULL
FROM {{ ref('stg_mes_aides__permis_velo') }}
WHERE contact_email IS NOT NULL
),

final AS (
Expand Down
Original file line number Diff line number Diff line change
@@ -1,87 +1,3 @@
WITH services_garage AS (
SELECT * FROM {{ ref('int_mes_aides__services_garage') }}
),

services_permis_velo AS (
SELECT * FROM {{ ref('int_mes_aides__services_permis_velo') }}
),

final AS (
SELECT
adresse_id,
contact_public,
cumulable,
date_creation,
date_maj,
date_suspension,
formulaire_en_ligne,
frais_autres,
id,
justificatifs,
lien_source,
modes_accueil,
modes_orientation_accompagnateur,
modes_orientation_accompagnateur_autres,
modes_orientation_beneficiaire,
modes_orientation_beneficiaire_autres,
nom,
presentation_resume,
presentation_detail,
prise_rdv,
profils,
recurrence,
source,
structure_id,
thematiques,
types,
zone_diffusion_code,
zone_diffusion_nom,
zone_diffusion_type,
pre_requis,
contact_nom_prenom,
courriel,
telephone,
frais,
page_web
FROM services_garage
UNION ALL
SELECT
adresse_id,
contact_public,
cumulable,
date_creation,
date_maj,
date_suspension,
formulaire_en_ligne,
frais_autres,
id,
justificatifs,
lien_source,
modes_accueil,
modes_orientation_accompagnateur,
modes_orientation_accompagnateur_autres,
modes_orientation_beneficiaire,
modes_orientation_beneficiaire_autres,
nom,
presentation_resume,
presentation_detail,
prise_rdv,
profils,
recurrence,
source,
structure_id,
thematiques,
types,
zone_diffusion_code,
zone_diffusion_nom,
zone_diffusion_type,
pre_requis,
contact_nom_prenom,
courriel,
telephone,
frais,
page_web
FROM services_permis_velo
)

SELECT * FROM final
{{ dbt_utils.union_relations(
relations=[ref('int_mes_aides_garages__services'), ref('int_mes_aides_permis_velo__services')]
) }}
Original file line number Diff line number Diff line change
@@ -1,57 +1,3 @@
WITH structures_garage AS (
SELECT * FROM {{ ref('int_mes_aides__structures_garage') }}
),

structures_permis_velo AS (
SELECT * FROM {{ ref('int_mes_aides__structures_permis_velo') }}
),

final AS (
SELECT
id,
adresse_id,
siret,
antenne,
rna,
nom,
telephone,
courriel,
site_web,
"source",
lien_source,
horaires_ouverture,
accessibilite,
labels_nationaux,
labels_autres,
typologie,
presentation_resume,
presentation_detail,
date_maj,
thematiques
FROM structures_garage
UNION ALL
SELECT
id,
adresse_id,
siret,
antenne,
rna,
nom,
telephone,
courriel,
site_web,
"source",
lien_source,
horaires_ouverture,
accessibilite,
labels_nationaux,
labels_autres,
typologie,
presentation_resume,
presentation_detail,
date_maj,
thematiques
FROM structures_permis_velo
)

SELECT * FROM final
{{ dbt_utils.union_relations(
relations=[ref('int_mes_aides_garages__structures'), ref('int_mes_aides_permis_velo__structures')]
) }}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
version: 2

models:
- name: int_mes_aides__adresses_permis_velo
- name: int_mes_aides_permis_velo__adresses
data_tests:
- check_adresse:
config:
Expand All @@ -14,7 +14,7 @@ models:
- dbt_utils.not_empty_string


- name: int_mes_aides__structures_permis_velo
- name: int_mes_aides_permis_velo__structures
data_tests:
- check_structure:
config:
Expand All @@ -29,10 +29,10 @@ models:
data_tests:
- not_null
- relationships:
to: ref('int_mes_aides__adresses_permis_velo')
to: ref('int_mes_aides_permis_velo__adresses')
field: id

- name: int_mes_aides__services_permis_velo
- name: int_mes_aides_permis_velo__services
data_tests:
- check_service:
config:
Expand All @@ -47,11 +47,11 @@ models:
data_tests:
- not_null
- relationships:
to: ref('int_mes_aides__structures_permis_velo')
to: ref('int_mes_aides_permis_velo__structures')
field: id
- name: adresse_id
data_tests:
- not_null
- relationships:
to: ref('int_mes_aides__adresses_permis_velo')
to: ref('int_mes_aides_permis_velo__adresses')
field: id

This file was deleted.

Loading

0 comments on commit 62c0108

Please sign in to comment.