From 175bdbfa0d192ac90f024123693d7b50b6e7ce9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chlo=C3=A9?= Date: Mon, 2 Dec 2024 10:38:40 +0100 Subject: [PATCH 1/6] Ajout de siae.extra_data dans le dictionnaire d'attributs --- lemarche/utils/apis/api_brevo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemarche/utils/apis/api_brevo.py b/lemarche/utils/apis/api_brevo.py index b85317879..4a9fc4dc0 100644 --- a/lemarche/utils/apis/api_brevo.py +++ b/lemarche/utils/apis/api_brevo.py @@ -146,6 +146,7 @@ def create_or_update_company(siae): "geo_range": siae.geo_range, "app_url": get_object_share_url(siae), "app_admin_url": get_object_admin_url(siae), + **siae.extra_data, # includes completion_rate, tender_email_send_count, etc. }, ) From 050b86b3fecab6695c7f5350af39f4d899c35ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chlo=C3=A9?= Date: Mon, 2 Dec 2024 10:39:34 +0100 Subject: [PATCH 2/6] =?UTF-8?q?Ajout=20d'un=20filtre=20'since=5Fdays'=20da?= =?UTF-8?q?ns=20la=20m=C3=A9thode=20'with=5Ftender=5Fstats'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lemarche/siaes/models.py | 60 +++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/lemarche/siaes/models.py b/lemarche/siaes/models.py index bd5ff0fc2..068b5a845 100644 --- a/lemarche/siaes/models.py +++ b/lemarche/siaes/models.py @@ -1,3 +1,4 @@ +from datetime import timedelta from uuid import uuid4 from django.conf import settings @@ -78,6 +79,24 @@ def get_city_filter(perimeter, with_country=False): return filters +def count_field(field_name, date_limit): + """ + Helper method to construct a conditional count annotation. + """ + condition = ( + Q(**{f"tendersiae__{field_name}__gte": date_limit}) + if date_limit + else Q(**{f"tendersiae__{field_name}__isnull": False}) + ) + return Sum( + Case( + When(condition, then=1), + default=0, + output_field=IntegerField(), + ) + ) + + class SiaeGroupQuerySet(models.QuerySet): def with_siae_stats(self): return self.annotate(siae_count_annotated=Count("siaes", distinct=True)) @@ -425,41 +444,20 @@ def filter_with_tender_tendersiae_status(self, tender, tendersiae_status=None): return qs.distinct() - def with_tender_stats(self): + def with_tender_stats(self, since_days=None): """ - Enrich each Siae with stats on their linked Tender + Enrich each Siae with stats on their linked Tender. + Optionally, limit the stats to the last `since_days` days. """ + date_limit = timezone.now() - timedelta(days=since_days) if since_days else None + return self.annotate( tender_count_annotated=Count("tenders", distinct=True), - tender_email_send_count_annotated=Sum( - Case(When(tendersiae__email_send_date__isnull=False, then=1), default=0, output_field=IntegerField()) - ), - tender_email_link_click_count_annotated=Sum( - Case( - When(tendersiae__email_link_click_date__isnull=False, then=1), - default=0, - output_field=IntegerField(), - ) - ), - tender_detail_display_count_annotated=Sum( - Case( - When(tendersiae__detail_display_date__isnull=False, then=1), default=0, output_field=IntegerField() - ) - ), - tender_detail_contact_click_count_annotated=Sum( - Case( - When(tendersiae__detail_contact_click_date__isnull=False, then=1), - default=0, - output_field=IntegerField(), - ) - ), - tender_detail_not_interested_count_annotated=Sum( - Case( - When(tendersiae__detail_not_interested_click_date__isnull=False, then=1), - default=0, - output_field=IntegerField(), - ) - ), + tender_email_send_count_annotated=count_field("email_send_date", date_limit), + tender_email_link_click_count_annotated=count_field("email_link_click_date", date_limit), + tender_detail_display_count_annotated=count_field("detail_display_date", date_limit), + tender_detail_contact_click_count_annotated=count_field("detail_contact_click_date", date_limit), + tender_detail_not_interested_count_annotated=count_field("detail_not_interested_click_date", date_limit), ) def with_brand_or_name(self, with_order_by=False): From 27044f0228c4a010614b50be5c101ddde0ac7f84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chlo=C3=A9?= Date: Mon, 2 Dec 2024 10:40:06 +0100 Subject: [PATCH 3/6] =?UTF-8?q?Mise=20=C3=A0=20jour=20de=20siae.extra=5Fda?= =?UTF-8?q?ta=20lors=20de=20la=20synchronisation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../commands/crm_brevo_sync_companies.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lemarche/crm/management/commands/crm_brevo_sync_companies.py b/lemarche/crm/management/commands/crm_brevo_sync_companies.py index c8f71c99a..350402465 100644 --- a/lemarche/crm/management/commands/crm_brevo_sync_companies.py +++ b/lemarche/crm/management/commands/crm_brevo_sync_companies.py @@ -8,7 +8,7 @@ from lemarche.utils.commands import BaseCommand -ten_days_ago = timezone.now() - timedelta(days=10) +two_weeks_ago = timezone.now() - timedelta(weeks=2) class Command(BaseCommand): @@ -34,11 +34,25 @@ def handle(self, recently_updated: bool, **options): self.stdout.write(f"Sync Siae > Brevo: we have {Siae.objects.count()} siaes") # Update only the recently updated siaes if recently_updated: - siaes_qs = siaes_qs.filter(updated_at__gte=ten_days_ago) + siaes_qs = siaes_qs.filter(updated_at__gte=two_weeks_ago) self.stdout.write(f"Sync Siae > Brevo: {siaes_qs.count()} recently updated") - # Step 2: loop on the siaes + # Step 2: Add the 90-day limited annotations + siaes_qs = siaes_qs.with_tender_stats(since_days=90) + + # Step 3: loop on the siaes for index, siae in enumerate(siaes_qs): + new_extra_data = { + "TAUX DE COMPLÉTION": siae.completion_rate, + "BESOINS REÇUS": siae.tender_email_send_count_annotated, + "BESOINS INTERESSÉS": siae.tender_detail_contact_click_count_annotated, + } + + # extra_data update if needed + if siae.extra_data != new_extra_data: + siae.extra_data = new_extra_data + siae.save(update_fields=["extra_data"]) + api_brevo.create_or_update_company(siae) if (index % 10) == 0: # avoid API rate-limiting time.sleep(1) From c97c8ceea22a2c9112238385c4a3263c1a85c322 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chlo=C3=A9?= Date: Wed, 4 Dec 2024 09:15:33 +0100 Subject: [PATCH 4/6] Tests de la commande 'crm_brevo_sync_companies' --- lemarche/crm/tests.py | 200 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 lemarche/crm/tests.py diff --git a/lemarche/crm/tests.py b/lemarche/crm/tests.py new file mode 100644 index 000000000..4ebacaeea --- /dev/null +++ b/lemarche/crm/tests.py @@ -0,0 +1,200 @@ +from datetime import timedelta +from unittest.mock import patch + +from django.core.management import call_command +from django.test import TestCase +from django.utils import timezone + +from lemarche.siaes.factories import SiaeFactory +from lemarche.siaes.models import Siae +from lemarche.tenders.factories import TenderFactory +from lemarche.tenders.models import TenderSiae +from lemarche.users.factories import UserFactory +from lemarche.users.models import User + + +now = timezone.now() +date_tomorrow = now + timedelta(days=1) +old_date = timezone.now() - timedelta(days=91) +recent_date = now - timedelta(days=10) + + +class CrmBrevoSyncCompaniesCommandTest(TestCase): + @classmethod + def setUpTestData(cls): + """Siae instances initialization""" + cls.user_siae = UserFactory(kind=User.KIND_SIAE) + cls.siae_with_name = SiaeFactory(name="Test Company 1") + cls.siae_with_user = SiaeFactory(users=[cls.user_siae]) + cls.siae_with_brevo_id = SiaeFactory( + brevo_company_id="123456789", + completion_rate=50, + ) + + cls.tender_1 = TenderFactory(deadline_date=date_tomorrow) + cls.tender_2 = TenderFactory(deadline_date=date_tomorrow) + + TenderSiae.objects.create( + tender=cls.tender_1, + siae=cls.siae_with_user, + detail_contact_click_date=recent_date, + ) + + TenderSiae.objects.create( + tender=cls.tender_1, + siae=cls.siae_with_brevo_id, + email_send_date=recent_date, + detail_contact_click_date=old_date, + ) + + cls.siae_with_user_stats = Siae.objects.with_tender_stats().filter(id=cls.siae_with_user.id).first() + cls.siae_with_brevo_id_all_stats = ( + Siae.objects.with_tender_stats().filter(id=cls.siae_with_brevo_id.id).first() + ) + cls.siae_with_brevo_id_recent_stats = ( + Siae.objects.with_tender_stats(since_days=90).filter(id=cls.siae_with_brevo_id.id).first() + ) + + # siae_with_brevo_id.extra_data initialization + cls.siae_with_brevo_id.extra_data = { + "TAUX DE COMPLÉTION": cls.siae_with_brevo_id.completion_rate, + "BESOINS REÇUS": cls.siae_with_brevo_id_recent_stats.tender_email_send_count_annotated, + "BESOINS INTERESSÉS": cls.siae_with_brevo_id_recent_stats.tender_detail_contact_click_count_annotated, + } + cls.siae_with_brevo_id.save() + cls.initial_extra_data = cls.siae_with_brevo_id.extra_data.copy() + + def test_annotated_fields_set_up(self): + """Test annotated fields are correctly set up""" + self.assertEqual( + self.siae_with_user_stats.tender_email_send_count_annotated, + 0, + "Le nombre total de besoins reçus devrait être 0", + ) + self.assertEqual( + self.siae_with_user_stats.tender_detail_contact_click_count_annotated, + 1, + "Le nombre total de besoins intéressés devrait être 1", + ) + self.assertEqual( + self.siae_with_brevo_id_all_stats.tender_email_send_count_annotated, + 1, + "Le nombre total de besoins reçus devrait être 1", + ) + self.assertEqual( + self.siae_with_brevo_id_all_stats.tender_detail_contact_click_count_annotated, + 1, + "Le nombre total de besoins intéressés devrait être 1", + ) + self.assertEqual( + self.siae_with_brevo_id_recent_stats.tender_email_send_count_annotated, + 1, + "Le nombre de besoins reçus dans les 90 derniers jours devrait être 1", + ) + self.assertEqual( + self.siae_with_brevo_id_recent_stats.tender_detail_contact_click_count_annotated, + 0, + "Le nombre de besoins intéressés dans les 90 derniers jours devrait être 0", + ) + + @patch("lemarche.utils.apis.api_brevo.create_or_update_company") + def test_new_siaes_are_synced_in_brevo(self, mock_create_or_update_company): + """Test new siaes are synced in brevo""" + call_command("crm_brevo_sync_companies") + + self.assertEqual(mock_create_or_update_company.call_count, 3) + + def test_siae_has_tender_stats(self): + self.assertIsNotNone( + self.siae_with_user_stats, + "Cette SIAE devrait avoir des statistiques sur les besoins.", + ) + self.assertIsNotNone( + self.siae_with_brevo_id_all_stats, + "Cette SIAE devrait avoir des statistiques sur les besoins.", + ) + + def test_siae_extra_data_is_set_on_first_sync(self): + """Test siae is updated if extra_data is changed.""" + # siae_with_user.extra_data should be empty + initial_extra_data = self.siae_with_user.extra_data.copy() + + call_command("crm_brevo_sync_companies", recently_updated=True) + + self.siae_with_user.refresh_from_db() + + expected_extra_data = { + "TAUX DE COMPLÉTION": self.siae_with_user.completion_rate, + "BESOINS REÇUS": self.siae_with_user_stats.tender_email_send_count_annotated, + "BESOINS INTERESSÉS": self.siae_with_user_stats.tender_detail_contact_click_count_annotated, + } + + self.assertNotEqual(initial_extra_data, expected_extra_data, "siae.extra_data aurait dû être mis à jour.") + self.assertEqual( + self.siae_with_user.extra_data, expected_extra_data, "siae.extra_data n'est pas conforme aux attentes." + ) + + def test_siae_extra_data_is_not_updated_if_no_changes(self): + """Test siae.extra_data is not updated if no changes.""" + call_command("crm_brevo_sync_companies", recently_updated=True) + + self.siae_with_brevo_id.refresh_from_db() + self.assertEqual( + self.initial_extra_data, + self.siae_with_brevo_id.extra_data, + "siae.extra_data a été mis à jour alors qu'il n'y avait pas de changement.", + ) + + def test_fields_update_within_90_days_and_ignore_older_changes(self): + """Test fields update within 90 days and ignore older changes.""" + TenderSiae.objects.create( + tender=self.tender_2, + siae=self.siae_with_brevo_id, + email_send_date=now, + detail_contact_click_date=now, + ) + + call_command("crm_brevo_sync_companies", recently_updated=True) + + self.siae_with_brevo_id_all_stats = ( + Siae.objects.with_tender_stats().filter(id=self.siae_with_brevo_id.id).first() + ) + self.siae_with_brevo_id_recent_stats = ( + Siae.objects.with_tender_stats(since_days=90).filter(id=self.siae_with_brevo_id.id).first() + ) + + # Tender stats without date limit + self.assertEqual( + self.siae_with_brevo_id_all_stats.tender_email_send_count_annotated, + 2, + "Le nombre total des besoins reçus devrait être 2", + ) + self.assertEqual( + self.siae_with_brevo_id_all_stats.tender_detail_contact_click_count_annotated, + 2, + "Le nombre de bsoins interessés devrait être 2", + ) + + # Tender stats with date limit + self.assertEqual( + self.siae_with_brevo_id_recent_stats.tender_email_send_count_annotated, + 2, + "Le nombre de besoins reçus dans les 90 jours devraient être 2", + ) + self.assertEqual( + self.siae_with_brevo_id_recent_stats.tender_detail_contact_click_count_annotated, + 1, + "Les nombre de bsoins interessés dans les 90 jours devraient être 1", + ) + + expected_extra_data = { + "TAUX DE COMPLÉTION": self.siae_with_brevo_id.completion_rate, + "BESOINS REÇUS": self.siae_with_brevo_id_recent_stats.tender_email_send_count_annotated, + "BESOINS INTERESSÉS": self.siae_with_brevo_id_recent_stats.tender_detail_contact_click_count_annotated, + } + + self.assertNotEqual( + self.initial_extra_data, + expected_extra_data, + "Les valeurs récentes dans extra_data devraient être mises à jour en fonction du filtre de 90 jours.", + ) From ae5db363ab8446906952173d7192bc923e5194bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chlo=C3=A9?= Date: Thu, 5 Dec 2024 14:58:57 +0100 Subject: [PATCH 5/6] =?UTF-8?q?Am=C3=A9lioration=20de=20la=20mise=20=C3=A0?= =?UTF-8?q?=20jour=20de=20siae.extra=5Fdata?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../commands/crm_brevo_sync_companies.py | 12 ++++-- lemarche/crm/tests.py | 41 ++++++++++++++----- lemarche/utils/apis/api_brevo.py | 4 +- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/lemarche/crm/management/commands/crm_brevo_sync_companies.py b/lemarche/crm/management/commands/crm_brevo_sync_companies.py index 350402465..1a5014a15 100644 --- a/lemarche/crm/management/commands/crm_brevo_sync_companies.py +++ b/lemarche/crm/management/commands/crm_brevo_sync_companies.py @@ -43,14 +43,18 @@ def handle(self, recently_updated: bool, **options): # Step 3: loop on the siaes for index, siae in enumerate(siaes_qs): new_extra_data = { - "TAUX DE COMPLÉTION": siae.completion_rate, - "BESOINS REÇUS": siae.tender_email_send_count_annotated, - "BESOINS INTERESSÉS": siae.tender_detail_contact_click_count_annotated, + "completion_rate": siae.completion_rate if siae.completion_rate is not None else 0, + "tender_received": siae.tender_email_send_count_annotated, + "tender_interest": siae.tender_detail_contact_click_count_annotated, } # extra_data update if needed if siae.extra_data != new_extra_data: - siae.extra_data = new_extra_data + siae.extra_data.update( + { + "brevo_company_data": new_extra_data, + } + ) siae.save(update_fields=["extra_data"]) api_brevo.create_or_update_company(siae) diff --git a/lemarche/crm/tests.py b/lemarche/crm/tests.py index 4ebacaeea..9dc517fcd 100644 --- a/lemarche/crm/tests.py +++ b/lemarche/crm/tests.py @@ -57,9 +57,13 @@ def setUpTestData(cls): # siae_with_brevo_id.extra_data initialization cls.siae_with_brevo_id.extra_data = { - "TAUX DE COMPLÉTION": cls.siae_with_brevo_id.completion_rate, - "BESOINS REÇUS": cls.siae_with_brevo_id_recent_stats.tender_email_send_count_annotated, - "BESOINS INTERESSÉS": cls.siae_with_brevo_id_recent_stats.tender_detail_contact_click_count_annotated, + "brevo_company_data": { + "completion_rate": cls.siae_with_brevo_id.completion_rate + if cls.siae_with_brevo_id.completion_rate is not None + else 0, + "tender_received": cls.siae_with_brevo_id_recent_stats.tender_email_send_count_annotated, + "tender_interest": cls.siae_with_brevo_id_recent_stats.tender_detail_contact_click_count_annotated, + } } cls.siae_with_brevo_id.save() cls.initial_extra_data = cls.siae_with_brevo_id.extra_data.copy() @@ -115,18 +119,29 @@ def test_siae_has_tender_stats(self): ) def test_siae_extra_data_is_set_on_first_sync(self): - """Test siae is updated if extra_data is changed.""" - # siae_with_user.extra_data should be empty + """ + - Test siae is updated if extra_data is changed. + - Test siae.extra_data update does not erase existing data. + """ initial_extra_data = self.siae_with_user.extra_data.copy() + initial_extra_data["test_data"] = "test value" + + self.siae_with_user.extra_data = initial_extra_data + self.siae_with_user.save(update_fields=["extra_data"]) call_command("crm_brevo_sync_companies", recently_updated=True) self.siae_with_user.refresh_from_db() expected_extra_data = { - "TAUX DE COMPLÉTION": self.siae_with_user.completion_rate, - "BESOINS REÇUS": self.siae_with_user_stats.tender_email_send_count_annotated, - "BESOINS INTERESSÉS": self.siae_with_user_stats.tender_detail_contact_click_count_annotated, + "brevo_company_data": { + "completion_rate": self.siae_with_user.completion_rate + if self.siae_with_user.completion_rate is not None + else 0, + "tender_received": self.siae_with_user_stats.tender_email_send_count_annotated, + "tender_interest": self.siae_with_user_stats.tender_detail_contact_click_count_annotated, + }, + "test_data": "test value", } self.assertNotEqual(initial_extra_data, expected_extra_data, "siae.extra_data aurait dû être mis à jour.") @@ -188,9 +203,13 @@ def test_fields_update_within_90_days_and_ignore_older_changes(self): ) expected_extra_data = { - "TAUX DE COMPLÉTION": self.siae_with_brevo_id.completion_rate, - "BESOINS REÇUS": self.siae_with_brevo_id_recent_stats.tender_email_send_count_annotated, - "BESOINS INTERESSÉS": self.siae_with_brevo_id_recent_stats.tender_detail_contact_click_count_annotated, + "brevo_company_data": { + "completion_rate": self.siae_with_brevo_id.completion_rate + if self.siae_with_brevo_id.completion_rate is not None + else 0, + "tender_received": self.siae_with_brevo_id_recent_stats.tender_email_send_count_annotated, + "tender_interest": self.siae_with_brevo_id_recent_stats.tender_detail_contact_click_count_annotated, + } } self.assertNotEqual( diff --git a/lemarche/utils/apis/api_brevo.py b/lemarche/utils/apis/api_brevo.py index 4a9fc4dc0..fd940eadc 100644 --- a/lemarche/utils/apis/api_brevo.py +++ b/lemarche/utils/apis/api_brevo.py @@ -146,7 +146,9 @@ def create_or_update_company(siae): "geo_range": siae.geo_range, "app_url": get_object_share_url(siae), "app_admin_url": get_object_admin_url(siae), - **siae.extra_data, # includes completion_rate, tender_email_send_count, etc. + "taux_de_completion": siae.extra_data.get("brevo_company_data", {}).get("completion_rate"), + "nombre_de_besoins_recus": siae.extra_data.get("brevo_company_data", {}).get("tender_received"), + "nombre_de_besoins_interesses": siae.extra_data.get("brevo_company_data", {}).get("tender_interest"), }, ) From 0d9af3a0ffb3e33297cf0ab012999b9045db5668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chlo=C3=A9?= Date: Mon, 9 Dec 2024 17:15:11 +0100 Subject: [PATCH 6/6] Fix update siae.extra_data --- lemarche/crm/management/commands/crm_brevo_sync_companies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemarche/crm/management/commands/crm_brevo_sync_companies.py b/lemarche/crm/management/commands/crm_brevo_sync_companies.py index 1a5014a15..4590066e3 100644 --- a/lemarche/crm/management/commands/crm_brevo_sync_companies.py +++ b/lemarche/crm/management/commands/crm_brevo_sync_companies.py @@ -49,7 +49,7 @@ def handle(self, recently_updated: bool, **options): } # extra_data update if needed - if siae.extra_data != new_extra_data: + if siae.extra_data.get("brevo_company_data") != new_extra_data: siae.extra_data.update( { "brevo_company_data": new_extra_data,