[Dépôt de besoin] Stats supplémentaires pour Metabase (#939)

* Tender: new stat field siae_count * Tender update_tender_count_fields command * Add other count fields * Add command to cron (daily)
gip-inclusion · Oct 26, 2023 · 8ebd5cd · 8ebd5cd
1 parent 53402d8
commit 8ebd5cd
Show file tree

Hide file tree

Showing 7 changed files with 189 additions and 11 deletions.
diff --git a/clevercloud/cron.json b/clevercloud/cron.json
@@ -2,6 +2,7 @@
     "0 0 * * * $ROOT/clevercloud/siaes_export_all_siae_to_file.sh",
     "15 0 * * * $ROOT/clevercloud/stats_export_user_download_list_to_file.sh",
     "30 0 * * * $ROOT/clevercloud/stats_export_user_search_list_to_file.sh",
+    "0 1 * * * $ROOT/clevercloud/tenders_update_count_fields.sh",
     "0 7 * * 1 $ROOT/clevercloud/siaes_sync_c1_c4.sh",
     "10 7 * * 1 $ROOT/clevercloud/siaes_sync_c2_c4.sh",
     "20 7 * * 1 $ROOT/clevercloud/siaes_update_api_entreprise_fields.sh",

diff --git a/clevercloud/siaes_update_count_fields.sh b/clevercloud/siaes_update_count_fields.sh
@@ -1,10 +1,10 @@
 #!/bin/bash -l
 
-# Update API ZRR fields for new siaes
+# Update siae count fields
 
 # Do not run if this env var is not set:
-if [[ -z "$CRON_UPDATE_COUNT_FIELDS_ENABLED" ]]; then
-    echo "CRON_UPDATE_COUNT_FIELDS_ENABLED not set. Exiting..."
+if [[ -z "$CRON_UPDATE_SIAE_COUNT_FIELDS_ENABLED" ]]; then
+    echo "CRON_UPDATE_SIAE_COUNT_FIELDS_ENABLED not set. Exiting..."
     exit 0
 fi
 
@@ -19,5 +19,5 @@ fi
 # $APP_HOME is set by default by clever cloud.
 cd $APP_HOME
 
-# django-admin update_count_fields
-django-admin update_count_fields --fields etablissement_count
+# django-admin update_siae_count_fields
+django-admin update_siae_count_fields --fields etablissement_count
diff --git a/clevercloud/tenders_update_count_fields.sh b/clevercloud/tenders_update_count_fields.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -l
+
+# Update tender count fields
+
+# Do not run if this env var is not set:
+if [[ -z "$CRON_UPDATE_TENDER_COUNT_FIELDS_ENABLED" ]]; then
+    echo "CRON_UPDATE_TENDER_COUNT_FIELDS_ENABLED not set. Exiting..."
+    exit 0
+fi
+
+# About clever cloud cronjobs:
+# https://www.clever-cloud.com/doc/tools/crons/
+
+if [[ "$INSTANCE_NUMBER" != "0" ]]; then
+    echo "Instance number is ${INSTANCE_NUMBER}. Stop here."
+    exit 0
+fi
+
+# $APP_HOME is set by default by clever cloud.
+cd $APP_HOME
+
+# django-admin update_tender_count_fields
+django-admin update_tender_count_fields
diff --git a/...anagement/commands/update_count_fields.py → ...ment/commands/update_siae_count_fields.py b/...anagement/commands/update_count_fields.py → ...ment/commands/update_siae_count_fields.py
@@ -24,10 +24,10 @@ class Command(BaseCommand):
     Note: these fields should be updated automatically on each Siae save()
 
     Usage:
-    python manage.py update_count_fields
-    python manage.py update_count_fields --id 1
-    python manage.py update_count_fields --id 1 --fields user_count
-    python manage.py update_count_fields --id 1 --fields user_count --fields etablissement_count
+    python manage.py update_siae_count_fields
+    python manage.py update_siae_count_fields --id 1
+    python manage.py update_siae_count_fields --id 1 --fields user_count
+    python manage.py update_siae_count_fields --id 1 --fields user_count --fields etablissement_count
     """
 
     def add_arguments(self, parser):
@@ -39,13 +39,13 @@ def add_arguments(self, parser):
     def handle(self, *args, **options):
         self.stdout_messages_info("Updating Siae count fields...")
 
-        # Step 1a: build Siae queryset
+        # Step 1a: build the queryset
         siae_queryset = Siae.objects.prefetch_related(
             "users", "sectors", "networks", "groups", "offers", "client_references", "labels", "images"
         ).all()
         if options["id"]:
             siae_queryset = siae_queryset.filter(id=options["id"])
-        self.stdout_messages_info(f"Found {siae_queryset.count()} Siae")
+        self.stdout_messages_info(f"Found {siae_queryset.count()} siaes")
 
         # Step 1b: init fields to update
         update_fields = options["fields"] if options["fields"] else SIAE_COUNT_FIELDS

diff --git a/lemarche/tenders/management/commands/update_tender_count_fields.py b/lemarche/tenders/management/commands/update_tender_count_fields.py
@@ -0,0 +1,70 @@
+from lemarche.tenders.models import Tender
+from lemarche.utils.commands import BaseCommand
+
+
+TENDER_COUNT_FIELDS = [
+    "siae_count",
+    "siae_email_send_count",
+    "siae_email_link_click_count",
+    "siae_detail_display_count",
+    "siae_email_link_click_or_detail_display_count",
+    "siae_detail_contact_click_count",
+]
+
+
+class Command(BaseCommand):
+    """
+    Goal: update the '_count' fields of each Tender
+
+    Usage:
+    python manage.py update_tender_count_fields
+    python manage.py update_tender_count_fields --id 1
+    python manage.py update_tender_count_fields --id 1 --fields siae_count
+    python manage.py update_tender_count_fields --id 1 --fields siae_count --fields siae_detail_contact_click_count
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument("--id", type=int, default=None, help="Indiquer l'ID d'un besoin")
+        parser.add_argument(
+            "--fields", action="append", default=[], help="Filtrer sur les champs count à mettre à jour"
+        )
+
+    def handle(self, *args, **options):
+        self.stdout_messages_info("Updating Tender count fields...")
+
+        # Step 1a: build the queryset
+        tender_queryset = Tender.objects.with_siae_stats().all()
+        if options["id"]:
+            tender_queryset = tender_queryset.filter(id=options["id"])
+        self.stdout_messages_info(f"Found {tender_queryset.count()} tenders")
+
+        # Step 1b: init fields to update
+        update_fields = options["fields"] if options["fields"] else TENDER_COUNT_FIELDS
+        self.stdout_messages_info(f"Fields to update: {update_fields}")
+
+        # Step 2: loop on each Tender
+        progress = 0
+        for index, tender in enumerate(tender_queryset):
+            # M2M
+            tender.siae_count = tender.siae_count_annotated
+            tender.siae_email_send_count = tender.siae_email_send_count_annotated
+            tender.siae_email_link_click_count = tender.siae_email_link_click_count_annotated
+            tender.siae_detail_display_count = tender.siae_detail_display_count_annotated
+            tender.siae_email_link_click_or_detail_display_count = (
+                tender.siae_email_link_click_or_detail_display_count_annotated
+            )
+            tender.siae_detail_contact_click_count = tender.siae_detail_contact_click_count_annotated
+
+            # Step 3: update count fields
+            tender.save(update_fields=update_fields)
+
+            progress += 1
+            if (progress % 500) == 0:
+                self.stdout_info(f"{progress}...")
+
+        msg_success = [
+            "----- Tender count fields -----",
+            f"Done! Processed {tender_queryset.count()} tenders",
+            f"Fields updated: {update_fields}",
+        ]
+        self.stdout_messages_success(msg_success)
diff --git a/lemarche/tenders/migrations/0059_tender_siae_count_updated.py b/lemarche/tenders/migrations/0059_tender_siae_count_updated.py
@@ -0,0 +1,66 @@
+# Generated by Django 4.2.2 on 2023-10-25 11:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("tenders", "0058_tenderstepsdata"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="tender",
+            name="siae_count",
+            field=models.IntegerField(
+                default=0,
+                help_text="Champ recalculé à intervalles réguliers",
+                verbose_name="Nombre de structures concernées",
+            ),
+        ),
+        migrations.AddField(
+            model_name="tender",
+            name="siae_detail_contact_click_count",
+            field=models.IntegerField(
+                default=0,
+                help_text="Champ recalculé à intervalles réguliers",
+                verbose_name="Nombre de structures intéressées",
+            ),
+        ),
+        migrations.AddField(
+            model_name="tender",
+            name="siae_detail_display_count",
+            field=models.IntegerField(
+                default=0,
+                help_text="Champ recalculé à intervalles réguliers",
+                verbose_name="Nombre de structures vues",
+            ),
+        ),
+        migrations.AddField(
+            model_name="tender",
+            name="siae_email_link_click_count",
+            field=models.IntegerField(
+                default=0,
+                help_text="Champ recalculé à intervalles réguliers",
+                verbose_name="Nombre de structures cliquées",
+            ),
+        ),
+        migrations.AddField(
+            model_name="tender",
+            name="siae_email_link_click_or_detail_display_count",
+            field=models.IntegerField(
+                default=0,
+                help_text="Champ recalculé à intervalles réguliers",
+                verbose_name="Nombre de structures cliquées ou vues",
+            ),
+        ),
+        migrations.AddField(
+            model_name="tender",
+            name="siae_email_send_count",
+            field=models.IntegerField(
+                default=0,
+                help_text="Champ recalculé à intervalles réguliers",
+                verbose_name="Nombre de structures contactées",
+            ),
+        ),
+    ]
diff --git a/lemarche/tenders/models.py b/lemarche/tenders/models.py
@@ -395,6 +395,24 @@ class Tender(models.Model):
     )
 
     # stats
+    siae_count = models.IntegerField(
+        "Nombre de structures concernées", help_text="Champ recalculé à intervalles réguliers", default=0
+    )
+    siae_email_send_count = models.IntegerField(
+        "Nombre de structures contactées", help_text="Champ recalculé à intervalles réguliers", default=0
+    )
+    siae_email_link_click_count = models.IntegerField(
+        "Nombre de structures cliquées", help_text="Champ recalculé à intervalles réguliers", default=0
+    )
+    siae_detail_display_count = models.IntegerField(
+        "Nombre de structures vues", help_text="Champ recalculé à intervalles réguliers", default=0
+    )
+    siae_email_link_click_or_detail_display_count = models.IntegerField(
+        "Nombre de structures cliquées ou vues", help_text="Champ recalculé à intervalles réguliers", default=0
+    )
+    siae_detail_contact_click_count = models.IntegerField(
+        "Nombre de structures intéressées", help_text="Champ recalculé à intervalles réguliers", default=0
+    )
     published_at = models.DateTimeField("Date de publication", blank=True, null=True)
     siae_list_last_seen_date = models.DateTimeField(
         "Date de dernière visite de l'auteur sur la page 'structures intéressées'", blank=True, null=True