Feature(Dépôt de besoin) : Envoi des dépôt de besoins par lot (#1025)

* update tender model

* rename sent_at to first_sent_at and add attrs

* add status for close tenders

* update request to get the validated tenders but not sent

* stop the task when tenders are sent

* limit siaes by the admin paramater

* add params to the admin

* send each every 24 hours

* order by super esi

* improve ordering

* improve by review

lemarche/siaes/models.py

@@ -493,6 +493,11 @@ def with_employees_stats(self):
             ),
         )
 
+    def order_by_super_siaes(self):
+        return self.order_by(
+            "-super_badge", "-tender_detail_contact_click_count", "-tender_detail_display_count", "-completion_rate"
+        )
+
 
 class Siae(models.Model):
     FIELDS_FROM_C1 = [

lemarche/templates/tenders/_detail_admin_extra_info.html

@@ -8,7 +8,7 @@ <h2 class="h3">Informations Admin</h2>
                 </li>
                 <li class="mb-2">
                     {% if tender.is_sent %}
-                        Validé le {{ tender.sent_at|date }}
+                        Validé le {{ tender.first_sent_at|date }}
                     {% else %}
                         Statut : {{ tender.get_status_display }}
                     {% endif %}

lemarche/templates/tenders/_list_item_buyer.html

@@ -36,7 +36,7 @@ <h2 class="py-2">{{ tender.title }}</h2>
                     {% if tender.is_sent %}
                         <div class="col-md-4">
                             <i class="ri-time-line"></i>
-                            Publié le {{ tender.sent_at|date }}
+                            Publié le {{ tender.first_sent_at|date }}
                         </div>
                     {% endif %}
                 </div>

lemarche/templates/tenders/admin_change_form.html

@@ -39,8 +39,8 @@
         <div class="submit-row">
             {% if original.validated_at %}
                 <i>Validé le {{ original.validated_at }}.&nbsp;</i>
-                {% if original.sent_at %}
-                    <i>Envoyé le {{ original.sent_at }}.</i>
+                {% if original.first_sent_at %}
+                    <i>Envoyé le {{ original.first_sent_at }}.</i>
                     <button type="submit" class="button" name="_restart_tender" value="1">Renvoyer aux structures</button>
                 {% endif %}
             {% else %}

lemarche/tenders/admin.py

@@ -135,7 +135,9 @@ class TenderAdmin(FieldsetsInlineMixin, admin.ModelAdmin):
         "siae_transactioned",
         "created_at",
         "validated_at",
-        "sent_at",
+        "first_sent_at",
+        "limit_send_to_siae_batch",
+        "limit_nb_siae_interested",
     ]
 
     list_filter = [
@@ -206,6 +208,7 @@ class TenderAdmin(FieldsetsInlineMixin, admin.ModelAdmin):
             },
         ),
         TenderQuestionInline,
+        ("Paramètres d'envois", {"fields": ("limit_send_to_siae_batch", "limit_nb_siae_interested")}),
         (
             "Filtres",
             {
@@ -312,7 +315,7 @@ class TenderAdmin(FieldsetsInlineMixin, admin.ModelAdmin):
                     "status",
                     "published_at",
                     "validated_at",
-                    "sent_at",
+                    "first_sent_at",
                 )
             },
         ),

lemarche/tenders/factories.py

@@ -44,7 +44,7 @@ class Meta:
     # marche_benefits = factory.fuzzy.FuzzyChoice([key for (key, _) in constants.MARCHE_BENEFIT_CHOICES])
     status = tender_constants.STATUS_SENT
     validated_at = timezone.now()
-    sent_at = timezone.now()
+    first_sent_at = timezone.now()
 
     @factory.post_generation
     def perimeters(self, create, extracted, **kwargs):

lemarche/tenders/management/commands/send_author_incremental_emails.py

@@ -29,8 +29,8 @@ def handle(self, dry_run=False, **options):
         two_days_ago = timezone.now() - timedelta(days=2)
         three_days_ago = timezone.now() - timedelta(days=3)
         tender_sent_incremental = Tender.objects.sent().is_incremental()
-        tender_sent_incremental_2_days = tender_sent_incremental.filter(sent_at__gte=three_days_ago).filter(
-            sent_at__lt=two_days_ago
+        tender_sent_incremental_2_days = tender_sent_incremental.filter(first_sent_at__gte=three_days_ago).filter(
+            first_sent_at__lt=two_days_ago
         )
         self.stdout.write(f"Found {tender_sent_incremental_2_days.count()} Tenders")
 

lemarche/tenders/management/commands/send_validated_tenders.py

@@ -22,4 +22,5 @@ def handle(self, *args, **options):
         if validated_tenders_to_send.count():
             self.stdout.write(f"Found {validated_tenders_to_send.count()} validated tender(s) to send")
             for tender in validated_tenders_to_send:
+                self.stdout.write(f"Found {tender} ")
                 send_validated_tender(tender)

lemarche/tenders/migrations/0065_tender_limit_nb_siae_interested_and_more.py

@@ -0,0 +1,52 @@
+# Generated by Django 4.2.2 on 2023-12-11 15:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("tenders", "0064_tender_status_sent"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="tender",
+            name="limit_nb_siae_interested",
+            field=models.PositiveSmallIntegerField(
+                default=5, help_text="Champ renseigné par un ADMIN", verbose_name="Limite des SIAES intéressées"
+            ),
+        ),
+        migrations.AddField(
+            model_name="tender",
+            name="limit_send_to_siae_batch",
+            field=models.PositiveSmallIntegerField(
+                default=10, help_text="Champ renseigné par un ADMIN", verbose_name="Nombre de SIAES par envoi"
+            ),
+        ),
+        migrations.RenameField(
+            model_name="tender",
+            old_name="sent_at",
+            new_name="first_sent_at",
+        ),
+        migrations.AlterField(
+            model_name="tender",
+            name="first_sent_at",
+            field=models.DateTimeField(blank=True, null=True, verbose_name="Date du premier envoi"),
+        ),
+        migrations.AddField(
+            model_name="tender",
+            name="last_sent_at",
+            field=models.DateTimeField(blank=True, null=True, verbose_name="Date du dernier envoi"),
+        ),
+        migrations.AddField(
+            model_name="tender",
+            name="version",
+            field=models.PositiveIntegerField(default=0, verbose_name="Version"),
+        ),
+        # this is migration to manage the existants stock
+        migrations.AlterField(
+            model_name="tender",
+            name="version",
+            field=models.PositiveIntegerField(default=1, verbose_name="Version"),
+        ),
+    ]

lemarche/tenders/models.py

@@ -45,10 +45,21 @@ def validated(self):
         return self.filter(validated_at__isnull=False)
 
     def validated_but_not_sent(self):
-        return self.filter(validated_at__isnull=False).filter(sent_at__isnull=True)
+        yesterday = timezone.now() - timedelta(days=1)
+
+        return self.with_siae_stats().filter(
+            (Q(version=0) & Q(validated_at__isnull=False) & Q(first_sent_at__isnull=True))
+            | (
+                Q(version=1)
+                & Q(validated_at__isnull=False)
+                & Q(siae_detail_contact_click_count_annotated__lte=F("limit_nb_siae_interested"))
+                & ~Q(siae_count_annotated=F("siae_email_send_count_annotated"))
+                & (Q(first_sent_at__isnull=True) | Q(last_sent_at__lt=yesterday))
+            )
+        )
 
     def sent(self):
-        return self.filter(sent_at__isnull=False)
+        return self.filter(first_sent_at__isnull=False)
 
     def is_incremental(self):
         return self.filter(
@@ -216,7 +227,7 @@ class Tender(models.Model):
     FIELDS_STATS_TIMESTAMPS = [
         "published_at",
         "validated_at",
-        "sent_at",
+        "first_sent_at",
         "siae_list_last_seen_date",
         "created_at",
         "updated_at",
@@ -417,8 +428,8 @@ class Tender(models.Model):
         default=tender_constants.STATUS_DRAFT,
     )
     validated_at = models.DateTimeField("Date de validation", blank=True, null=True)
-    sent_at = models.DateTimeField("Date d'envoi", blank=True, null=True)
-
+    first_sent_at = models.DateTimeField("Date du premier envoi", blank=True, null=True)
+    last_sent_at = models.DateTimeField("Date du dernier envoi", blank=True, null=True)
     # admin
     notes = GenericRelation("notes.Note", related_query_name="tender")
     siae_transactioned = models.BooleanField(
@@ -437,7 +448,17 @@ class Tender(models.Model):
         null=True,
         default=None,
     )
+    limit_send_to_siae_batch = models.PositiveSmallIntegerField(
+        verbose_name="Nombre de SIAES par envoi",
+        help_text="Champ renseigné par un ADMIN",
+        default=10,
+    )
 
+    limit_nb_siae_interested = models.PositiveSmallIntegerField(
+        verbose_name="Limite des SIAES intéressées",
+        help_text="Champ renseigné par un ADMIN",
+        default=5,
+    )
     # stats
     siae_count = models.IntegerField(
         "Nombre de structures concernées", help_text=RECALCULATED_FIELD_HELP_TEXT, default=0
@@ -468,6 +489,7 @@ class Tender(models.Model):
         choices=tender_constants.SOURCE_CHOICES,
         default=tender_constants.SOURCE_FORM,
     )
+    version = models.PositiveIntegerField(verbose_name="Version", default=1)
     extra_data = models.JSONField(verbose_name="Données complémentaires", editable=False, default=dict)
     import_raw_object = models.JSONField(verbose_name="Données d'import", editable=False, null=True)
 
@@ -692,7 +714,7 @@ def is_pending_validation_or_validated(self) -> bool:
 
     @property
     def is_sent(self) -> bool:
-        return bool(self.sent_at) and self.status == tender_constants.STATUS_SENT
+        return bool(self.first_sent_at) and self.status == tender_constants.STATUS_SENT
 
     @property
     def is_validated_or_sent(self) -> bool:
@@ -709,11 +731,14 @@ def set_validated(self):
         self.save()
 
     def set_sent(self):
-        self.sent_at = timezone.now()
-        self.status = tender_constants.STATUS_SENT
+        if not self.first_sent_at:
+            self.first_sent_at = timezone.now()
+            self.status = tender_constants.STATUS_SENT
+
+        self.last_sent_at = timezone.now()
         log_item = {
             "action": "send",
-            "date": self.sent_at.isoformat(),
+            "date": self.last_sent_at.isoformat(),
         }
         self.logs.append(log_item)
         self.save()

lemarche/tenders/tests.py

@@ -109,7 +109,7 @@ def test_status(self):
         tender_pending_validation = TenderFactory(status=tender_constants.STATUS_PUBLISHED)
         tender_validated_half = TenderFactory(status=tender_constants.STATUS_VALIDATED)
         tender_validated_full = TenderFactory(status=tender_constants.STATUS_VALIDATED, validated_at=timezone.now())
-        tender_sent = TenderFactory(status=tender_constants.STATUS_SENT, sent_at=timezone.now())
+        tender_sent = TenderFactory(status=tender_constants.STATUS_SENT, first_sent_at=timezone.now())
         self.assertTrue(tender_draft.is_draft, True)
         self.assertTrue(tender_pending_validation.is_pending_validation, True)
         self.assertTrue(tender_validated_half.is_validated, False)
@@ -198,8 +198,8 @@ def test_validated(self):
         self.assertEqual(Tender.objects.validated().count(), 1)
 
     def test_sent(self):
-        TenderFactory(sent_at=timezone.now())
-        TenderFactory(sent_at=None)
+        TenderFactory(first_sent_at=timezone.now())
+        TenderFactory(first_sent_at=None)
         self.assertEqual(Tender.objects.sent().count(), 1)
 
     def test_is_live(self):
@@ -339,7 +339,7 @@ def setUpTestData(cls):
         TenderQuestionFactory(tender=cls.tender_with_siae_1)
 
     def test_filter_with_siaes(self):
-        self.tender_with_siae_2.sent_at = None
+        self.tender_with_siae_2.first_sent_at = None
         self.tender_with_siae_2.save()
         # tender_with_siae_2 is not sent
         self.assertEqual(Tender.objects.filter_with_siaes(self.user_siae.siaes.all()).count(), 1)

lemarche/www/tenders/tasks.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import timedelta
 
 from django.conf import settings
@@ -14,9 +15,13 @@
 from lemarche.utils.urls import get_admin_url_object, get_domain_url, get_share_url_object
 
 
+logger = logging.getLogger(__name__)
+
+
 def send_validated_tender(tender: Tender):
     # find the matching Siaes? done in Tender post_save signal
     # notify author
+    # TODO: we still notify author for each send ?
     send_confirmation_published_email_to_author(tender, nb_matched_siaes=tender.siaes.count())
     # send the tender to all matching Siaes & Partners
     send_tender_emails_to_siaes(tender)
@@ -54,7 +59,9 @@ def send_tender_emails_to_siaes(tender: Tender):
     siae_users_send_count = 0
 
     # queryset
-    siaes = tender.siaes.all()
+    all_siaes = tender.siaes.filter(tendersiae__email_send_date=None).order_by_super_siaes()
+    logger.info(f"total siaes {all_siaes.count()}")
+    siaes = all_siaes[: tender.limit_send_to_siae_batch]
 
     for siae in siaes:
         # send to siae 'contact_email'
@@ -65,8 +72,9 @@ def send_tender_emails_to_siaes(tender: Tender):
             if user.email != siae.contact_email:
                 send_tender_email_to_siae(tender, siae, email_subject, email_to_override=user.email)
                 siae_users_send_count += 1
-
-    tender.tendersiae_set.update(email_send_date=timezone.now(), updated_at=timezone.now())
+    TenderSiae.objects.filter(tender=tender, siae__in=siaes).update(
+        email_send_date=timezone.now(), updated_at=timezone.now()
+    )
 
     # log email batch
     siaes_log_item = {
@@ -76,6 +84,8 @@ def send_tender_emails_to_siaes(tender: Tender):
         "email_timestamp": timezone.now().isoformat(),
     }
     tender.logs.append(siaes_log_item)
+    logger.info(siaes_log_item)
+
     siae_users_log_item = {
         "action": "email_siae_users_matched",
         "email_subject": email_subject,
@@ -84,6 +94,8 @@ def send_tender_emails_to_siaes(tender: Tender):
         "siae_users_count": siae_users_count,
     }
     tender.logs.append(siae_users_log_item)
+    logger.info(siae_users_log_item)
+
     tender.save()
 
 
@@ -494,7 +506,7 @@ def send_author_incremental_2_days_email(tender: Tender):
         variables = {
             "TENDER_AUTHOR_FIRST_NAME": tender.author.first_name,
             "TENDER_TITLE": tender.title,
-            "TENDER_VALIDATE_AT": tender.sent_at.strftime("%d %B %Y"),  # TODO: TENDER_SENT_AT?
+            "TENDER_VALIDATE_AT": tender.first_sent_at.strftime("%d %B %Y"),  # TODO: TENDER_SENT_AT?
             "TENDER_KIND": tender.get_kind_display(),
         }
 
@@ -528,7 +540,7 @@ def send_tenders_author_feedback_or_survey(tender: Tender, kind="feedback_30d"):
         variables = {
             "TENDER_AUTHOR_FIRST_NAME": tender.author.first_name,
             "TENDER_TITLE": tender.title,
-            "TENDER_VALIDATE_AT": tender.sent_at.strftime("%d %B %Y"),  # TODO: TENDER_SENT_AT?
+            "TENDER_VALIDATE_AT": tender.first_sent_at.strftime("%d %B %Y"),  # TODO: TENDER_SENT_AT?
             "TENDER_KIND": tender.get_kind_display(),
         }
 

lemarche/www/tenders/tests.py

@@ -599,7 +599,7 @@ def setUpTestData(cls):
             sectors=[sector_1],
             location=grenoble_perimeter,
             status=tender_constants.STATUS_SENT,
-            sent_at=timezone.now(),
+            first_sent_at=timezone.now(),
         )
         cls.tendersiae_1_1 = TenderSiae.objects.create(
             tender=cls.tender_1,
@@ -615,7 +615,7 @@ def setUpTestData(cls):
             author=cls.user_buyer_1,
             contact_company_name="Another company",
             status=tender_constants.STATUS_SENT,
-            sent_at=timezone.now(),
+            first_sent_at=timezone.now(),
         )
 
     def test_anyone_can_view_sent_tenders(self):