diff --git a/pipeline/dags/notify_rgpd_contacts.py b/pipeline/dags/notify_rgpd_contacts.py index 0b6c743b..ecf9b1ec 100644 --- a/pipeline/dags/notify_rgpd_contacts.py +++ b/pipeline/dags/notify_rgpd_contacts.py @@ -21,11 +21,13 @@ def _sync_new_contacts_to_brevo(): sql=( """ SELECT - json_build_object('email', _courriel_original), - rgpd_notice_has_hardbounced - FROM public_intermediate.int__union_contacts__enhanced - GROUP BY _courriel_original, rgpd_notice_has_hardbounced - ORDER BY _courriel_original + json_build_object('email', our_contacts.courriel), + brevo_contacts.has_hardbounced + FROM public_intermediate.int__union_contacts AS our_contacts + LEFT JOIN public_intermediate.int_brevo__contacts AS brevo_contacts + ON our_contacts.courriel = brevo_contacts.courriel + GROUP BY our_contacts.courriel, brevo_contacts.has_hardbounced + ORDER BY our_contacts.courriel """ ) ) diff --git a/pipeline/dbt/models/intermediate/_models.yml b/pipeline/dbt/models/intermediate/_models.yml index e839547e..be329273 100644 --- a/pipeline/dbt/models/intermediate/_models.yml +++ b/pipeline/dbt/models/intermediate/_models.yml @@ -39,7 +39,6 @@ models: - name: int__union_contacts__enhanced columns: - name: _di_surrogate_id - - name: _courriel_original - name: id - name: source - name: contact_nom_prenom @@ -245,10 +244,11 @@ unit_tests: - {"source": "autre", "id": "solo", "courriel": "solo@autre.net", "contact_nom_prenom": "Autre Personne", "telephone": "06666666"} - input: ref('int_brevo__contacts') rows: + # note: values can't be NULL. has_hardbounced is set to true/false by brevo, and was_objected to is set to true/false by us. - {"courriel": "truc1@toto.at", "has_hardbounced": true, "was_objected_to": true} - {"courriel": "truc2@toto.at", "has_hardbounced": false, "was_objected_to": true} - - {"courriel": "truc3@toto.at", "has_hardbounced": false, "was_objected_to": null} - - {"courriel": "solo@autre.net", "has_hardbounced": true, "was_objected_to": null} + - {"courriel": "truc3@toto.at", "has_hardbounced": false, "was_objected_to": false} + - {"courriel": "solo@autre.net", "has_hardbounced": true, "was_objected_to": false} expect: rows: - {source: dora, id: foo, courriel: NULL, contact_nom_prenom: NULL, telephone: NULL} # objected to, hardbounce diff --git a/pipeline/dbt/models/intermediate/brevo/int_brevo__contacts.sql b/pipeline/dbt/models/intermediate/brevo/int_brevo__contacts.sql index be209d80..895eeae6 100644 --- a/pipeline/dbt/models/intermediate/brevo/int_brevo__contacts.sql +++ b/pipeline/dbt/models/intermediate/brevo/int_brevo__contacts.sql @@ -4,10 +4,10 @@ WITH contacts AS ( final AS ( SELECT - id AS "id", - email AS "courriel", - email_blacklisted AS "has_hardbounced", - date_di_rgpd_opposition IS NOT NULL AS "was_objected_to" + id AS "id", + email AS "courriel", + email_blacklisted AS "has_hardbounced", + date_di_rgpd_opposition IS NOT NULL AS "was_objected_to" FROM contacts ) diff --git a/pipeline/dbt/models/intermediate/int__union_contacts__enhanced.sql b/pipeline/dbt/models/intermediate/int__union_contacts__enhanced.sql index eba32fe0..05dd559d 100644 --- a/pipeline/dbt/models/intermediate/int__union_contacts__enhanced.sql +++ b/pipeline/dbt/models/intermediate/int__union_contacts__enhanced.sql @@ -6,9 +6,7 @@ rgpd_notices AS ( SELECT courriel, has_hardbounced, - was_objected_to, - was_objected_to IS TRUE AS was_objected_to_truthy, - has_hardbounced IS TRUE AS has_hardbounced_truthy + was_objected_to FROM {{ ref('int_brevo__contacts') }} ), @@ -17,23 +15,22 @@ final AS ( contacts.source || '-' || contacts.id AS "_di_surrogate_id", contacts.id AS "id", contacts.source AS "source", - contacts.courriel AS "_courriel_original", CASE WHEN rgpd_notices.courriel IS NULL - OR NOT rgpd_notices.was_objected_to_truthy + OR NOT rgpd_notices.was_objected_to THEN contacts.contact_nom_prenom END AS "contact_nom_prenom", CASE WHEN rgpd_notices.courriel IS NULL - OR (NOT rgpd_notices.was_objected_to_truthy AND NOT rgpd_notices.has_hardbounced_truthy) + OR (NOT rgpd_notices.was_objected_to AND NOT rgpd_notices.has_hardbounced) THEN contacts.courriel END AS "courriel", CASE WHEN rgpd_notices.courriel IS NULL - OR NOT rgpd_notices.was_objected_to_truthy + OR NOT rgpd_notices.was_objected_to THEN contacts.telephone END AS "telephone", rgpd_notices.was_objected_to AS "rgpd_notice_was_objected_to",