From f24a8b22d253c8afd5e685723f5daeedd095c2dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Thu, 5 Oct 2023 16:57:54 +0200 Subject: [PATCH 1/8] keep steps data --- .../migrations/0058_tenderstepsdata.py | 40 +++++++++++++++++++ lemarche/tenders/models.py | 16 ++++++++ lemarche/www/tenders/views.py | 24 ++++++++++- 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 lemarche/tenders/migrations/0058_tenderstepsdata.py diff --git a/lemarche/tenders/migrations/0058_tenderstepsdata.py b/lemarche/tenders/migrations/0058_tenderstepsdata.py new file mode 100644 index 000000000..3cce3a1fd --- /dev/null +++ b/lemarche/tenders/migrations/0058_tenderstepsdata.py @@ -0,0 +1,40 @@ +# Generated by Django 4.2.2 on 2023-10-05 14:15 + +from django.db import migrations, models +import django.utils.timezone +import django_extensions.db.fields +import shortuuid.main + + +class Migration(migrations.Migration): + + dependencies = [ + ("tenders", "0057_alter_tender_siae_transactioned"), + ] + + operations = [ + migrations.CreateModel( + name="TenderStepsData", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "uuid", + django_extensions.db.fields.ShortUUIDField( + auto_created=True, + blank=True, + db_index=True, + default=shortuuid.main.ShortUUID.uuid, + editable=False, + unique=True, + verbose_name="Identifiant UUID", + ), + ), + ( + "created_at", + models.DateTimeField(default=django.utils.timezone.now, verbose_name="Date de création"), + ), + ("updated_at", models.DateTimeField(auto_now=True, verbose_name="Date de modification")), + ("steps_data", models.JSONField(default=list, editable=False, verbose_name="Données des étapes")), + ], + ), + ] diff --git a/lemarche/tenders/models.py b/lemarche/tenders/models.py index dc261e9ac..68ae4f8ad 100644 --- a/lemarche/tenders/models.py +++ b/lemarche/tenders/models.py @@ -11,6 +11,8 @@ from django.utils.functional import cached_property from django.utils.text import slugify from django_better_admin_arrayfield.models.fields import ArrayField +from django_extensions.db.fields import ShortUUIDField +from shortuuid import uuid from lemarche.perimeters.models import Perimeter from lemarche.siaes import constants as siae_constants @@ -761,3 +763,17 @@ class Meta: @cached_property def perimeters_list_string(self) -> str: return ", ".join(self.perimeters.values_list("name", flat=True)) + + +class TenderStepsData(models.Model): + created_at = models.DateTimeField(verbose_name="Date de création", default=timezone.now) + updated_at = models.DateTimeField(verbose_name="Date de modification", auto_now=True) + uuid = ShortUUIDField( + verbose_name="Identifiant UUID", + default=uuid, + editable=False, + unique=True, + db_index=True, + auto_created=True, + ) + steps_data = models.JSONField(verbose_name="Données des étapes", editable=False, default=list) diff --git a/lemarche/www/tenders/views.py b/lemarche/www/tenders/views.py index c33c2abf3..a48e00533 100644 --- a/lemarche/www/tenders/views.py +++ b/lemarche/www/tenders/views.py @@ -14,7 +14,7 @@ from lemarche.siaes.models import Siae from lemarche.tenders import constants as tender_constants -from lemarche.tenders.models import Tender, TenderSiae +from lemarche.tenders.models import Tender, TenderSiae, TenderStepsData from lemarche.users.models import User from lemarche.utils.data import get_choice from lemarche.utils.mixins import ( @@ -148,6 +148,28 @@ def get_context_data(self, form, **kwargs): context.update({"tender": tender_dict}) return context + def process_step(self, form): + """ + Save step data + """ + data = form.data.copy() + if "csrfmiddlewaretoken" in data: + del data["csrfmiddlewaretoken"] + + uuid = self.request.session.get("tender_steps_data_uuid", None) + if uuid: + try: + tender_steps_data = TenderStepsData.objects.get(uuid=uuid) + tender_steps_data.steps_data.append(data) + tender_steps_data.save() + except TenderStepsData.DoesNotExist: + tender_steps_data = TenderStepsData.objects.create(uuid=uuid, steps_data=[data]) + else: + tender_steps_data = TenderStepsData.objects.create(steps_data=[data]) + self.request.session["tender_steps_data_uuid"] = tender_steps_data.uuid + + return form.data + def save_instance_tender(self, tender_dict: dict, form_dict: dict, is_draft: bool): tender_status = tender_constants.STATUS_DRAFT if is_draft else tender_constants.STATUS_PUBLISHED tender_published_at = None if is_draft else timezone.now() From e65f30d216fe0106f490f159e8fae82e793cdc79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Mon, 9 Oct 2023 14:39:30 +0200 Subject: [PATCH 2/8] add admin for tender steps data --- lemarche/tenders/admin.py | 22 ++++++++++++++++++- .../migrations/0058_tenderstepsdata.py | 6 ++++- lemarche/tenders/models.py | 7 ++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/lemarche/tenders/admin.py b/lemarche/tenders/admin.py index e2cc5286b..88372d08f 100644 --- a/lemarche/tenders/admin.py +++ b/lemarche/tenders/admin.py @@ -16,7 +16,7 @@ from lemarche.perimeters.admin import PerimeterRegionFilter from lemarche.tenders import constants from lemarche.tenders.forms import TenderAdminForm -from lemarche.tenders.models import PartnerShareTender, Tender, TenderQuestion +from lemarche.tenders.models import PartnerShareTender, Tender, TenderQuestion, TenderStepsData from lemarche.utils.admin.admin_site import admin_site from lemarche.utils.apis import api_hubspot from lemarche.utils.fields import ChoiceArrayField, pretty_print_readonly_jsonfield @@ -591,3 +591,23 @@ def logs_display(self, partnersharetender=None): return "-" logs_display.short_description = PartnerShareTender._meta.get_field("logs").verbose_name + + +@admin.register(TenderStepsData, site=admin_site) +class TenderStepsDataAdmin(admin.ModelAdmin): + list_display = ["created_at", "updated_at", "uuid"] + + readonly_fields = [ + "id", + "created_at", + "updated_at", + "uuid", + "steps_data_display", + ] + + def steps_data_display(self, tender_steps_data: TenderStepsData = None): + if tender_steps_data: + return pretty_print_readonly_jsonfield(tender_steps_data.steps_data) + return "-" + + steps_data_display.short_description = "Données saisies dans les étapes" diff --git a/lemarche/tenders/migrations/0058_tenderstepsdata.py b/lemarche/tenders/migrations/0058_tenderstepsdata.py index 3cce3a1fd..d95fd605d 100644 --- a/lemarche/tenders/migrations/0058_tenderstepsdata.py +++ b/lemarche/tenders/migrations/0058_tenderstepsdata.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.2 on 2023-10-05 14:15 +# Generated by Django 4.2.2 on 2023-10-09 12:32 from django.db import migrations, models import django.utils.timezone @@ -36,5 +36,9 @@ class Migration(migrations.Migration): ("updated_at", models.DateTimeField(auto_now=True, verbose_name="Date de modification")), ("steps_data", models.JSONField(default=list, editable=False, verbose_name="Données des étapes")), ], + options={ + "verbose_name": "Besoin d'achat - Données des étapes", + "verbose_name_plural": "Besoins d'achat - Données des étapes", + }, ), ] diff --git a/lemarche/tenders/models.py b/lemarche/tenders/models.py index 68ae4f8ad..9644a6b1a 100644 --- a/lemarche/tenders/models.py +++ b/lemarche/tenders/models.py @@ -777,3 +777,10 @@ class TenderStepsData(models.Model): auto_created=True, ) steps_data = models.JSONField(verbose_name="Données des étapes", editable=False, default=list) + + class Meta: + verbose_name = "Besoin d'achat - Données des étapes" + verbose_name_plural = "Besoins d'achat - Données des étapes" + + def __str__(self): + return f"{self.uuid} - {self.created_at}" From 3757c27ed121a3ab31850efc4756495cf18df971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Mon, 9 Oct 2023 15:09:00 +0200 Subject: [PATCH 3/8] add tests to check steps data --- lemarche/tenders/migrations/0058_tenderstepsdata.py | 13 ++++++------- lemarche/www/tenders/tests.py | 9 ++++++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lemarche/tenders/migrations/0058_tenderstepsdata.py b/lemarche/tenders/migrations/0058_tenderstepsdata.py index d95fd605d..42d9cbda0 100644 --- a/lemarche/tenders/migrations/0058_tenderstepsdata.py +++ b/lemarche/tenders/migrations/0058_tenderstepsdata.py @@ -1,13 +1,12 @@ -# Generated by Django 4.2.2 on 2023-10-09 12:32 +# Generated by Django 4.2.2 on 2023-10-09 13:35 -from django.db import migrations, models import django.utils.timezone -import django_extensions.db.fields -import shortuuid.main +from django.db import migrations, models +from django_extensions.db.fields import ShortUUIDField +from shortuuid import uuid class Migration(migrations.Migration): - dependencies = [ ("tenders", "0057_alter_tender_siae_transactioned"), ] @@ -19,11 +18,11 @@ class Migration(migrations.Migration): ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ( "uuid", - django_extensions.db.fields.ShortUUIDField( + ShortUUIDField( auto_created=True, blank=True, db_index=True, - default=shortuuid.main.ShortUUID.uuid, + default=uuid, editable=False, unique=True, verbose_name="Identifiant UUID", diff --git a/lemarche/www/tenders/tests.py b/lemarche/www/tenders/tests.py index e99dc36e6..ca17813a1 100644 --- a/lemarche/www/tenders/tests.py +++ b/lemarche/www/tenders/tests.py @@ -18,7 +18,7 @@ from lemarche.siaes.models import Siae from lemarche.tenders import constants as tender_constants from lemarche.tenders.factories import TenderFactory, TenderQuestionFactory -from lemarche.tenders.models import Tender, TenderSiae +from lemarche.tenders.models import Tender, TenderSiae, TenderStepsData from lemarche.users.factories import UserFactory from lemarche.users.models import User from lemarche.www.tenders.views import TenderCreateMultiStepView @@ -89,6 +89,13 @@ def _check_every_step(self, tenders_step_data, final_redirect_page: str = revers current_errors = response.context_data["form"].errors self.assertEquals(current_errors, {}) + # Is the step data stored correctly ? + tender_step_data = TenderStepsData.objects.first() + self.assertEqual( + data_step["tender_create_multi_step_view-current_step"], + tender_step_data.steps_data[-1]["tender_create_multi_step_view-current_step"], + ) + def test_anyone_can_access_create_tender(self): # anonymous url = reverse("tenders:create") From 19d19c16b543c628104396777da15906effd3520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Tue, 10 Oct 2023 10:15:29 +0200 Subject: [PATCH 4/8] hide personal data --- lemarche/www/tenders/views.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lemarche/www/tenders/views.py b/lemarche/www/tenders/views.py index a48e00533..0dede4d08 100644 --- a/lemarche/www/tenders/views.py +++ b/lemarche/www/tenders/views.py @@ -156,6 +156,18 @@ def process_step(self, form): if "csrfmiddlewaretoken" in data: del data["csrfmiddlewaretoken"] + # Hide personal data + for field_to_redacted in [ + "contact-contact_email", + "contact-contact_phone", + "contact-contact_last_name", + "contact-contact_first_name", + ]: + if field_to_redacted in data: + data[field_to_redacted] = "[REDACTED]" + + data["timestamp"] = timezone.now().isoformat() + uuid = self.request.session.get("tender_steps_data_uuid", None) if uuid: try: From 8f68d8ec919827b4150f56f1be7f3a985573dc04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Tue, 10 Oct 2023 10:15:49 +0200 Subject: [PATCH 5/8] set right permissions --- lemarche/tenders/admin.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lemarche/tenders/admin.py b/lemarche/tenders/admin.py index 88372d08f..0d36f9d88 100644 --- a/lemarche/tenders/admin.py +++ b/lemarche/tenders/admin.py @@ -611,3 +611,12 @@ def steps_data_display(self, tender_steps_data: TenderStepsData = None): return "-" steps_data_display.short_description = "Données saisies dans les étapes" + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False From a8f4c332286064377ead9e18005d73f82d5d19fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Tue, 10 Oct 2023 10:37:59 +0200 Subject: [PATCH 6/8] clean step datas after publication --- lemarche/www/tenders/tests.py | 2 ++ lemarche/www/tenders/views.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/lemarche/www/tenders/tests.py b/lemarche/www/tenders/tests.py index ca17813a1..020dd3f62 100644 --- a/lemarche/www/tenders/tests.py +++ b/lemarche/www/tenders/tests.py @@ -83,6 +83,8 @@ def _check_every_step(self, tenders_step_data, final_redirect_page: str = revers # make sure that after the create tender we are redirected to ?? self.assertEqual(response.status_code, 200) self.assertRedirects(response, final_redirect_page) + # has the step datas been cleaned ? + self.assertEqual(TenderStepsData.objects.count(), 0) return response else: self.assertEqual(response.status_code, 200) diff --git a/lemarche/www/tenders/views.py b/lemarche/www/tenders/views.py index 0dede4d08..10f557123 100644 --- a/lemarche/www/tenders/views.py +++ b/lemarche/www/tenders/views.py @@ -235,6 +235,11 @@ def done(self, _, form_dict, **kwargs): self.save_instance_tender(tender_dict=tender_dict, form_dict=form_dict, is_draft=is_draft) self.instance.set_siae_found_list() + # remove steps data + uuid = self.request.session.get("tender_steps_data_uuid", None) + if uuid: + TenderStepsData.objects.filter(uuid=uuid).delete() + # we notify the admin team if settings.BITOUBI_ENV == "prod": notify_admin_tender_created(self.instance) From 638359c132b7aed181906f2be3106337f4dc2d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Tue, 10 Oct 2023 16:02:57 +0200 Subject: [PATCH 7/8] also remove steps data in csrf way --- lemarche/www/pages/views.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lemarche/www/pages/views.py b/lemarche/www/pages/views.py index 52fe8cd04..b9ad96b6f 100644 --- a/lemarche/www/pages/views.py +++ b/lemarche/www/pages/views.py @@ -15,7 +15,7 @@ from lemarche.sectors.models import Sector from lemarche.siaes.models import Siae, SiaeGroup from lemarche.tenders import constants as tender_constants -from lemarche.tenders.models import Tender +from lemarche.tenders.models import Tender, TenderStepsData from lemarche.users.models import User from lemarche.utils.tracker import track from lemarche.www.pages.forms import ( @@ -357,6 +357,11 @@ def csrf_failure(request, reason=""): # noqa C901 tender.save() tender.set_siae_found_list() + # remove steps data + uuid = request.session.get("tender_steps_data_uuid", None) + if uuid: + TenderStepsData.objects.filter(uuid=uuid).delete() + if settings.BITOUBI_ENV == "prod": notify_admin_tender_created(tender) From 45787b9f0747bfff600dbf5c6ae51e1641c13680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Tue, 10 Oct 2023 16:09:43 +0200 Subject: [PATCH 8/8] add FIELDS_TO_REDACT list to model --- lemarche/tenders/models.py | 7 +++++++ lemarche/www/tenders/views.py | 7 +------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lemarche/tenders/models.py b/lemarche/tenders/models.py index 9644a6b1a..2c97ecaa0 100644 --- a/lemarche/tenders/models.py +++ b/lemarche/tenders/models.py @@ -766,6 +766,13 @@ def perimeters_list_string(self) -> str: class TenderStepsData(models.Model): + FIELDS_TO_REDACT = [ + "contact-contact_email", + "contact-contact_phone", + "contact-contact_last_name", + "contact-contact_first_name", + ] + created_at = models.DateTimeField(verbose_name="Date de création", default=timezone.now) updated_at = models.DateTimeField(verbose_name="Date de modification", auto_now=True) uuid = ShortUUIDField( diff --git a/lemarche/www/tenders/views.py b/lemarche/www/tenders/views.py index 10f557123..ea11a18f7 100644 --- a/lemarche/www/tenders/views.py +++ b/lemarche/www/tenders/views.py @@ -157,12 +157,7 @@ def process_step(self, form): del data["csrfmiddlewaretoken"] # Hide personal data - for field_to_redacted in [ - "contact-contact_email", - "contact-contact_phone", - "contact-contact_last_name", - "contact-contact_first_name", - ]: + for field_to_redacted in TenderStepsData.FIELDS_TO_REDACT: if field_to_redacted in data: data[field_to_redacted] = "[REDACTED]"