diff --git a/pipeline/dags/notify_rgpd_contacts.py b/pipeline/dags/notify_rgpd_contacts.py index 081f1bcea..ffa54fef7 100644 --- a/pipeline/dags/notify_rgpd_contacts.py +++ b/pipeline/dags/notify_rgpd_contacts.py @@ -20,10 +20,10 @@ def _sync_new_contacts_to_brevo(): from dag_utils import constants, pg from dag_utils.sources import brevo - dora_contacts = pg.hook().get_records( + known_contacts = pg.hook().get_records( sql=( "SELECT courriel, ARRAY_AGG(contact_uid) as contact_uids " - "FROM public_intermediate.int_dora__contacts " + "FROM public_intermediate.int__union__contacts " "GROUP BY courriel" ) ) @@ -44,7 +44,7 @@ def _sync_new_contacts_to_brevo(): # If the email is not new but linked to a new contact UID, update the associated # list accordingly. new_contacts_map = defaultdict(list) - for email, contact_uids in dora_contacts: + for email, contact_uids in known_contacts: for contact_uid in contact_uids: if contact_uid not in brevo_contacts_uid_set: if email in brevo_contacts_map: diff --git a/pipeline/dbt/models/intermediate/_models.yml b/pipeline/dbt/models/intermediate/_models.yml index c485cf5f1..3b2e923ad 100644 --- a/pipeline/dbt/models/intermediate/_models.yml +++ b/pipeline/dbt/models/intermediate/_models.yml @@ -17,6 +17,19 @@ models: data_tests: - check_adresse: *union-common-check-args + - name: int__union__contacts + description: | + Gathers contacts from all sources + columns: + - name: contact_uid + data_tests: + - unique + - not_null + - name: courriel + data_tests: + - not_null + - dbt_utils.not_empty_string + - name: int__union_services description: | Gathers services from all sources diff --git a/pipeline/dbt/models/intermediate/int__union__contacts.sql b/pipeline/dbt/models/intermediate/int__union__contacts.sql new file mode 100644 index 000000000..8237fbf77 --- /dev/null +++ b/pipeline/dbt/models/intermediate/int__union__contacts.sql @@ -0,0 +1,14 @@ +WITH final AS ( + {{ + dbt_utils.union_relations( + relations=[ + ref('int_dora__contacts'), + ref('int_mediation_numerique__contacts'), + ref('int_mes_aides__contacts'), + ], + source_column_name=None + ) + }} +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mediation_numerique/_mediation_numerique_models.yml b/pipeline/dbt/models/intermediate/sources/mediation_numerique/_mediation_numerique_models.yml index dc99553d9..60ffbf5af 100644 --- a/pipeline/dbt/models/intermediate/sources/mediation_numerique/_mediation_numerique_models.yml +++ b/pipeline/dbt/models/intermediate/sources/mediation_numerique/_mediation_numerique_models.yml @@ -16,6 +16,18 @@ models: - not_null - dbt_utils.not_empty_string + - name: int_mediation_numerique__contacts + columns: + - name: contact_uid + data_tests: + - unique + - not_null + - name: courriel + data_tests: + - not_null + - dbt_utils.not_empty_string + + - name: int_mediation_numerique__services data_tests: - dbt_utils.unique_combination_of_columns: diff --git a/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__contacts.sql b/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__contacts.sql new file mode 100644 index 000000000..e9d63d6eb --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__contacts.sql @@ -0,0 +1,9 @@ +WITH final AS ( + SELECT + courriel AS "courriel", + 'mediation-numerique:structures:' || id AS contact_uid + FROM {{ ref('stg_mediation_numerique__structures') }} + WHERE courriel IS NOT NULL +) + +SELECT * FROM final diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/_mes_aides__models.yml b/pipeline/dbt/models/intermediate/sources/mes_aides/_mes_aides__models.yml index 731972c7c..2aa113ead 100644 --- a/pipeline/dbt/models/intermediate/sources/mes_aides/_mes_aides__models.yml +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/_mes_aides__models.yml @@ -13,6 +13,17 @@ models: - not_null - dbt_utils.not_empty_string + - name: int_mes_aides__contacts + columns: + - name: contact_uid + data_tests: + - unique + - not_null + - name: courriel + data_tests: + - not_null + - dbt_utils.not_empty_string + - name: int_mes_aides__structures data_tests: - check_structure: @@ -53,4 +64,4 @@ models: - not_null - relationships: to: ref('int_mes_aides__adresses') - field: id \ No newline at end of file + field: id diff --git a/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__contacts.sql b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__contacts.sql new file mode 100644 index 000000000..2cab5cb20 --- /dev/null +++ b/pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__contacts.sql @@ -0,0 +1,23 @@ +WITH structure_contacts AS ( + SELECT + email AS "courriel", + 'mes-aides:garages:' || id AS contact_uid + FROM {{ ref('stg_mes_aides__garages') }} + WHERE email IS NOT NULL +), + +service_contacts AS ( + SELECT + email AS "courriel", + 'mes-aides:aides:' || id AS contact_uid + FROM {{ ref('stg_mes_aides__aides') }} + WHERE email IS NOT NULL +), + +final AS ( + SELECT * FROM structure_contacts + UNION ALL + SELECT * FROM service_contacts +) + +SELECT * FROM final