feat(pipeline) : Add mes-aides, mediation-numerique to Brevo contacts

They will also be part of the notification campaign about displaying
their e-mail address in the API.

pipeline/dags/notify_rgpd_contacts.py

@@ -20,10 +20,10 @@ def _sync_new_contacts_to_brevo():
     from dag_utils import constants, pg
     from dag_utils.sources import brevo
 
-    dora_contacts = pg.hook().get_records(
+    known_contacts = pg.hook().get_records(
         sql=(
             "SELECT courriel, ARRAY_AGG(contact_uid) as contact_uids "
-            "FROM public_intermediate.int_dora__contacts "
+            "FROM public_intermediate.int__union__contacts "
             "GROUP BY courriel"
         )
     )
@@ -44,7 +44,7 @@ def _sync_new_contacts_to_brevo():
     # If the email is not new but linked to a new contact UID, update the associated
     # list accordingly.
     new_contacts_map = defaultdict(list)
-    for email, contact_uids in dora_contacts:
+    for email, contact_uids in known_contacts:
         for contact_uid in contact_uids:
             if contact_uid not in brevo_contacts_uid_set:
                 if email in brevo_contacts_map:

pipeline/dbt/models/intermediate/_models.yml

@@ -17,6 +17,19 @@ models:
     data_tests:
       - check_adresse: *union-common-check-args
 
+  - name: int__union__contacts
+    description: |
+      Gathers contacts from all sources
+    columns:
+      - name: contact_uid
+        data_tests:
+          - unique
+          - not_null
+      - name: courriel
+        data_tests:
+          - not_null
+          - dbt_utils.not_empty_string
+
   - name: int__union_services
     description: |
       Gathers services from all sources

pipeline/dbt/models/intermediate/int__union__contacts.sql

@@ -0,0 +1,14 @@
+WITH final AS (
+    {{
+        dbt_utils.union_relations(
+            relations=[
+                ref('int_dora__contacts'),
+                ref('int_mediation_numerique__contacts'),
+                ref('int_mes_aides__contacts'),
+            ],
+            source_column_name=None
+        )
+    }}
+)
+
+SELECT * FROM final

pipeline/dbt/models/intermediate/sources/mediation_numerique/_mediation_numerique_models.yml

@@ -16,6 +16,18 @@ models:
           - not_null
           - dbt_utils.not_empty_string
 
+  - name: int_mediation_numerique__contacts
+    columns:
+      - name: contact_uid
+        data_tests:
+          - unique
+          - not_null
+      - name: courriel
+        data_tests:
+          - not_null
+          - dbt_utils.not_empty_string
+
+
   - name: int_mediation_numerique__services
     data_tests:
       - dbt_utils.unique_combination_of_columns:

pipeline/dbt/models/intermediate/sources/mediation_numerique/int_mediation_numerique__contacts.sql

@@ -0,0 +1,9 @@
+WITH final AS (
+    SELECT
+        courriel                                AS "courriel",
+        'mediation-numerique:structures:' || id AS contact_uid
+    FROM {{ ref('stg_mediation_numerique__structures') }}
+    WHERE courriel IS NOT NULL
+)
+
+SELECT * FROM final

pipeline/dbt/models/intermediate/sources/mes_aides/_mes_aides__models.yml

@@ -13,6 +13,17 @@ models:
           - not_null
           - dbt_utils.not_empty_string
 
+  - name: int_mes_aides__contacts
+    columns:
+      - name: contact_uid
+        data_tests:
+          - unique
+          - not_null
+      - name: courriel
+        data_tests:
+          - not_null
+          - dbt_utils.not_empty_string
+
   - name: int_mes_aides__structures
     data_tests:
       - check_structure:
@@ -53,4 +64,4 @@ models:
           - not_null
           - relationships:
               to: ref('int_mes_aides__adresses')
-              field: id
+              field: id

pipeline/dbt/models/intermediate/sources/mes_aides/int_mes_aides__contacts.sql

@@ -0,0 +1,23 @@
+WITH structure_contacts AS (
+    SELECT
+        email                      AS "courriel",
+        'mes-aides:garages:' || id AS contact_uid
+    FROM {{ ref('stg_mes_aides__garages') }}
+    WHERE email IS NOT NULL
+),
+
+service_contacts AS (
+    SELECT
+        email                    AS "courriel",
+        'mes-aides:aides:' || id AS contact_uid
+    FROM {{ ref('stg_mes_aides__aides') }}
+    WHERE email IS NOT NULL
+),
+
+final AS (
+    SELECT * FROM structure_contacts
+    UNION ALL
+    SELECT * FROM service_contacts
+)
+
+SELECT * FROM final