From 67c6c14f3e48f6d2fdf9184506d878215228343f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Thu, 10 Oct 2024 18:15:26 +0200 Subject: [PATCH] add activities matching and the corresponding tests --- Makefile | 2 +- lemarche/siaes/models.py | 189 ++++++++++++++- lemarche/tenders/tests/test_matching.py | 290 ++++++++++++++++++++++++ 3 files changed, 479 insertions(+), 2 deletions(-) create mode 100644 lemarche/tenders/tests/test_matching.py diff --git a/Makefile b/Makefile index 7a83ee0e5..157636976 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # ============================================================================= PYTHON_VERSION := python3.11 -.PHONY: shell_on_django_container shell_on_django_container_as_root shell_on_postgres_container +.PHONY: shell_on_django_container shell_on_postgres_container # DOCKER commands # ============================================================================= diff --git a/lemarche/siaes/models.py b/lemarche/siaes/models.py index d319bd6ce..a98f9111f 100644 --- a/lemarche/siaes/models.py +++ b/lemarche/siaes/models.py @@ -7,7 +7,20 @@ from django.contrib.gis.measure import D from django.contrib.postgres.search import TrigramSimilarity # SearchVector from django.db import IntegrityError, models, transaction -from django.db.models import BooleanField, Case, CharField, Count, F, IntegerField, PositiveIntegerField, Q, Sum, When +from django.db.models import ( + BooleanField, + Case, + CharField, + Count, + F, + IntegerField, + OuterRef, + PositiveIntegerField, + Q, + Subquery, + Sum, + When, +) from django.db.models.functions import Greatest, Round from django.db.models.signals import m2m_changed, post_delete, post_save from django.dispatch import receiver @@ -411,6 +424,54 @@ def filter_with_tender(self, tender, tendersiae_status=None): # noqa C901 return qs.distinct() + def filter_with_tender_through_activities(self, tender, tendersiae_status=None): + """ + Filter Siaes with tenders: + - first we filter the Siae that are live + can be contacted + - then we filter through the SiaeActivity on the presta_type, sectors and perimeters + - then we filter on kind + - finally we filter with the tendersiae_status passed as a parameter + + Nota Bene: create a other filter_with_tender method to manage temporary cohabitation + + Args: + tender (Tender): Tender used to make the matching + """ + qs = self.tender_matching_query_set() + + # Subquery to filter SiaeActivity by presta_type, sector and perimeter + siae_activity_subquery = ( + SiaeActivity.objects.filter_with_tender(tender).filter(siae=OuterRef("pk")).values("pk") + ) + qs = qs.filter(Q(activities__in=Subquery(siae_activity_subquery))) + + # filter by siae_kind + if len(tender.siae_kind): + qs = qs.filter(kind__in=tender.siae_kind) + + # tender status + if tendersiae_status == "INTERESTED": + qs = qs.filter(tendersiae__tender=tender, tendersiae__detail_contact_click_date__isnull=False) + qs = qs.order_by("-tendersiae__detail_contact_click_date") + elif tendersiae_status == "VIEWED": + qs = qs.filter( + Q(tendersiae__tender=tender) + & ( + Q(tendersiae__email_link_click_date__isnull=False) + | Q(tendersiae__detail_display_date__isnull=False) + ) + ) + qs = qs.order_by("-tendersiae__email_link_click_date") + elif tendersiae_status == "COCONTRACTED": + qs = qs.filter(tendersiae__tender=tender, tendersiae__detail_cocontracting_click_date__isnull=False) + qs = qs.order_by("-tendersiae__detail_cocontracting_click_date") + elif tendersiae_status == "ALL": + # why need to filter more ? + qs = qs.filter(tendersiae__tender=tender, tendersiae__email_send_date__isnull=False) + qs = qs.order_by("-tendersiae__email_send_date") + + return qs.distinct() + def filter_with_tender_tendersiae_status(self, tender, tendersiae_status=None): qs = self.prefetch_related("sectors").is_live().has_contact_email() # .filter(tendersiae__tender=tender) # tender status @@ -1405,6 +1466,130 @@ class Meta: ordering = ["-created_at"] +class SiaeActivityQuerySet(models.QuerySet): + def filter_sectors(self, sectors): + return self.filter(sectors__in=sectors) + + def geo_range_in_perimeter_list(self, perimeters: models.QuerySet, include_country_area=False): + """ + Method to filter the Siaes Activities depending on the perimeter filter. + Depending on the type of Perimeter that were chosen, different cases arise: + + - If the Perimeter is a city, we filter the Siae Activities with the following conditions: + - The Siae Activity has a geo_range equal to GEO_RANGE_ZONES and the city is in the locations + - The Siae Activity has a geo_range equal to GEO_RANGE_CUSTOM and the distance between the Siae + address and the city is less than the geo_range_custom_distance + - The Siae Activity has a geo_range equal to GEO_RANGE_ZONES and the department of the city is + in the locations + - The Siae Activity has a geo_range equal to GEO_RANGE_ZONES and the region of the city is in + the locations + - If the Perimeter is a department, we filter the Siae Activities with the following conditions: + - The Siae Activity has a geo_range equal to GEO_RANGE_ZONES and the department is in the locations + - The Siae Activity has a geo_range equal to GEO_RANGE_ZONES and the region of the department is in + the locations + - If the Perimeter is a region, we filter the Siae Activities with the following conditions: + - The Siae Activity has a geo_range equal to GEO_RANGE_ZONES and the region is in the locations + + If include_country_area is True, we also filter the Siae Activities + with the geo_range equal to GEO_RANGE_COUNTRY + """ + + # Initialize an empty Q object to accumulate conditions + conditions = Q() + for perimeter in perimeters: + # Match siae activity with geo range zone and same perimeter + conditions |= Q(Q(geo_range=siae_constants.GEO_RANGE_ZONES) & Q(locations=perimeter)) + + if perimeter.kind == Perimeter.KIND_CITY: + # Match siae activity with geo range custom and siae city is in area + conditions |= Q( + Q(geo_range=siae_constants.GEO_RANGE_CUSTOM) + & Q(geo_range_custom_distance__gte=Distance("siae__coords", perimeter.coords) / 1000) + ) + + # Match the department that includes this city + conditions |= Q( + Q(geo_range=siae_constants.GEO_RANGE_ZONES) + & Q(locations__kind=Perimeter.KIND_DEPARTMENT) + & Q(locations__insee_code=perimeter.department_code) + ) + + # Match the region that includes this city + conditions |= Q( + Q(geo_range=siae_constants.GEO_RANGE_ZONES) + & Q(locations__kind=Perimeter.KIND_REGION) + & Q(locations__insee_code=f"R{perimeter.region_code}") + ) + if perimeter.kind == Perimeter.KIND_DEPARTMENT: + # Match the region that includes this department + conditions |= Q( + Q(geo_range=siae_constants.GEO_RANGE_ZONES) + & Q(locations__kind=Perimeter.KIND_REGION) + & Q(locations__insee_code=f"R{perimeter.region_code}") + ) + + if include_country_area: + conditions = Q(geo_range=siae_constants.GEO_RANGE_COUNTRY) | conditions + return self.filter(conditions) + + def with_country_geo_range(self): + return self.filter(Q(geo_range=siae_constants.GEO_RANGE_COUNTRY)) + + def exclude_country_geo_range(self): + return self.exclude(Q(geo_range=siae_constants.GEO_RANGE_COUNTRY)) + + def siae_within(self, point, distance_km=0, include_country_area=False): + return ( + self.filter( + Q(siae__coords__dwithin=(point, D(km=distance_km))) | Q(geo_range=siae_constants.GEO_RANGE_COUNTRY) + ) + if include_country_area + else self.filter(siae__coords__dwithin=(point, D(km=distance_km))) + ) + + def filter_with_tender(self, tender): + """ + Filter SiaeActivity with tenders: + - first we filter on presta_type + - then we filter on the sectors through the SiaeActivity + - then we filter on the perimeters through the SiaeActivity: + - if tender is made for country area, we filter with siae_geo_range=country + - else we filter on the perimeters + + If tender specify a city and a distance, we filter on the Siae adress that are within the distance of the city. + """ + qs = self.prefetch_related("sectors").prefetch_related("locations") + + # filter by presta_type + if len(tender.presta_type): + qs = qs.filter(presta_type__overlap=tender.presta_type) + + if tender.sectors.count(): + qs = qs.filter_sectors(tender.sectors.all()) + + # filter by perimeters + if tender.is_country_area: # for all country + qs = qs.with_country_geo_range() + else: + if ( + tender.location + and tender.location.kind == Perimeter.KIND_CITY + and tender.distance_location + and tender.distance_location > 0 + ): + # keep this filter on siae activity to handle include_country_area on activity level + qs = qs.siae_within(tender.location.coords, tender.distance_location, tender.include_country_area) + elif tender.perimeters.count() and tender.include_country_area: # perimeters and all country + qs = qs.geo_range_in_perimeter_list(tender.perimeters.all(), include_country_area=True) + elif tender.perimeters.count(): # only perimeters + qs = qs.geo_range_in_perimeter_list(tender.perimeters.all()).exclude_country_geo_range() + pass + elif tender.include_country_area: + qs = qs.filter(Q(geo_range=siae_constants.GEO_RANGE_COUNTRY)) + + return qs + + class SiaeActivity(models.Model): siae = models.ForeignKey( "siaes.Siae", verbose_name="Structure", related_name="activities", on_delete=models.CASCADE @@ -1444,6 +1629,8 @@ class SiaeActivity(models.Model): created_at = models.DateTimeField(verbose_name="Date de création", default=timezone.now) updated_at = models.DateTimeField(verbose_name="Date de modification", auto_now=True) + objects = models.Manager.from_queryset(SiaeActivityQuerySet)() + class Meta: verbose_name = "Activité" verbose_name_plural = "Activités" diff --git a/lemarche/tenders/tests/test_matching.py b/lemarche/tenders/tests/test_matching.py new file mode 100644 index 000000000..264af85d6 --- /dev/null +++ b/lemarche/tenders/tests/test_matching.py @@ -0,0 +1,290 @@ +from django.contrib.gis.geos import Point +from django.test import TestCase + +from lemarche.perimeters.factories import PerimeterFactory +from lemarche.sectors.factories import SectorFactory +from lemarche.siaes import constants as siae_constants +from lemarche.siaes.factories import SiaeActivityFactory, SiaeFactory +from lemarche.tenders.factories import TenderFactory +from lemarche.tenders.models import Perimeter, Siae + + +class TenderMatchingTest(TestCase): + @classmethod + def setUpTestData(cls): + cls.sectors = [SectorFactory() for i in range(10)] + cls.perimeter_paris = PerimeterFactory(department_code="75", post_codes=["75019", "75018"]) + cls.perimeter_marseille = PerimeterFactory(coords=Point(43.35101634452076, 5.379616625955892)) + cls.perimeters = [cls.perimeter_paris, PerimeterFactory()] + # by default is Paris + coords_paris = Point(48.86385199985207, 2.337071483848432) + + cls.siae_one = SiaeFactory( + is_active=True, + kind=siae_constants.KIND_AI, + coords=coords_paris, + ) + cls.siae_one_activity = SiaeActivityFactory( + siae=cls.siae_one, + sector_group=cls.sectors[0].group, + presta_type=[siae_constants.PRESTA_PREST, siae_constants.PRESTA_BUILD], + geo_range=siae_constants.GEO_RANGE_CUSTOM, + geo_range_custom_distance=100, + ) + cls.siae_one_activity.locations.set([cls.perimeter_paris]) + + cls.siae_two = SiaeFactory( + is_active=True, + kind=siae_constants.KIND_ESAT, + coords=coords_paris, + ) + cls.siae_two_activity = SiaeActivityFactory( + siae=cls.siae_two, + sector_group=cls.sectors[5].group, + presta_type=[siae_constants.PRESTA_BUILD], + geo_range=siae_constants.GEO_RANGE_CUSTOM, + geo_range_custom_distance=10, + ) + for i in range(5): + cls.siae_one_activity.sectors.add(cls.sectors[i]) + cls.siae_two_activity.sectors.add(cls.sectors[i + 5]) + + def test_matching_siae_presta_type(self): + tender = TenderFactory(presta_type=[], sectors=self.sectors, perimeters=self.perimeters) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 2) + tender = TenderFactory( + presta_type=[siae_constants.PRESTA_BUILD], sectors=self.sectors, perimeters=self.perimeters + ) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 2) + tender = TenderFactory( + presta_type=[siae_constants.PRESTA_PREST], sectors=self.sectors, perimeters=self.perimeters + ) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 1) + + def test_matching_siae_kind(self): + tender = TenderFactory(siae_kind=[], sectors=self.sectors, perimeters=self.perimeters) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 2) + tender = TenderFactory(siae_kind=[siae_constants.KIND_AI], sectors=self.sectors, perimeters=self.perimeters) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 1) + tender = TenderFactory( + siae_kind=[siae_constants.KIND_ESAT, siae_constants.KIND_AI], + sectors=self.sectors, + perimeters=self.perimeters, + ) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 2) + tender = TenderFactory(siae_kind=[siae_constants.KIND_SEP], sectors=self.sectors, perimeters=self.perimeters) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 0) + + def test_matching_siae_sectors(self): + tender = TenderFactory(sectors=self.sectors) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 2) + + def test_matching_siae_distance_location(self): + # create SIAE in Tours + siae_tours = SiaeFactory( + is_active=True, + kind=siae_constants.KIND_AI, + coords=Point(47.392287, 0.690049), # Tours city + ) + siae_tours_activity = SiaeActivityFactory( + siae=siae_tours, + sector_group=self.sectors[0].group, + presta_type=[siae_constants.PRESTA_PREST, siae_constants.PRESTA_BUILD], + with_custom_distance_perimeter=True, + ) + + siae_tours_activity.sectors.add(self.sectors[0]) + + # create SIAE in Marseille + siae_marseille = SiaeFactory( + is_active=True, + kind=siae_constants.KIND_AI, + coords=self.perimeter_marseille.coords, + ) + siae_marseille_activity = SiaeActivityFactory( + siae=siae_marseille, + sector_group=self.sectors[0].group, + presta_type=[siae_constants.PRESTA_PREST, siae_constants.PRESTA_BUILD], + with_country_perimeter=True, + ) + siae_marseille_activity.sectors.add(self.sectors[0]) + + # create tender in Azay-le-rideau (near Tours ~25km) + perimeter_azaylerideau = PerimeterFactory(coords=Point(47.262352, 0.466372)) + tender = TenderFactory( + location=perimeter_azaylerideau, + distance_location=30, + siae_kind=[siae_constants.KIND_ESAT, siae_constants.KIND_AI], + sectors=self.sectors, + ) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 1) + self.assertIn(siae_tours, siae_found_list) + + # Azay-le-rideau is less than 240km from Paris but more 550km from Marseille + tender = TenderFactory( + location=perimeter_azaylerideau, + distance_location=300, + siae_kind=[siae_constants.KIND_ESAT, siae_constants.KIND_AI], + sectors=self.sectors, + perimeters=[self.perimeter_paris], # test this option without effect when the distance is setted + ) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + # self.assertEqual(len(siae_found_list), 3) + self.assertIn(siae_tours, siae_found_list) + self.assertIn(self.siae_one, siae_found_list) + self.assertIn(self.siae_two, siae_found_list) + + # unset distance location, perimeters is used instead, Paris as it happens + tender.distance_location = None + tender.save() + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 2) + self.assertIn(self.siae_one, siae_found_list) + self.assertIn(self.siae_two, siae_found_list) + + # set distance location and include country + tender = TenderFactory( + location=perimeter_azaylerideau, + distance_location=50, + siae_kind=[siae_constants.KIND_ESAT, siae_constants.KIND_AI], + sectors=self.sectors, + include_country_area=True, + ) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 2) + self.assertIn(siae_tours, siae_found_list) + self.assertIn(siae_marseille, siae_found_list) + + # set a department in location disable distance_location, perimeters is used instead + tender = TenderFactory( + location=PerimeterFactory( + name="Indre-et-loire", kind=Perimeter.KIND_DEPARTMENT, insee_code="37", region_code="24" + ), + distance_location=50, + siae_kind=[siae_constants.KIND_ESAT, siae_constants.KIND_AI], + sectors=self.sectors, + include_country_area=True, # check this option without effect when the distance is setted + perimeters=[self.perimeter_paris], # without effect too + ) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 3) + self.assertIn(self.siae_one, siae_found_list) + self.assertIn(self.siae_two, siae_found_list) + self.assertIn(siae_marseille, siae_found_list) + + def test_matching_siae_perimeters_custom(self): + # add Siae with geo_range_country + siae_country = SiaeFactory(is_active=True) + siae_country_activity = SiaeActivityFactory( + siae=siae_country, + sector_group=self.sectors[0].group, + presta_type=[], + with_country_perimeter=True, + ) + siae_country_activity.sectors.add(self.sectors[0]) + # tender perimeter custom with include_country_area = False + tender_1 = TenderFactory(sectors=self.sectors, perimeters=self.perimeters) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender_1) + self.assertEqual(len(siae_found_list), 2 + 0) + # tender perimeter custom with include_country_area = True + tender_2 = TenderFactory(sectors=self.sectors, perimeters=self.perimeters, include_country_area=True) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender_2) + self.assertEqual(len(siae_found_list), 2 + 1) + + def test_matching_siae_country(self): + # add Siae with geo_range_country + siae_country = SiaeFactory(is_active=True) + siae_country_activity = SiaeActivityFactory( + siae=siae_country, + sector_group=self.sectors[0].group, + with_country_perimeter=True, + ) + siae_country_activity.sectors.add(self.sectors[0]) + + siae_country_2 = SiaeFactory(is_active=True) + siae_country_activity_2 = SiaeActivityFactory( + siae=siae_country_2, + sector_group=self.sectors[0].group, + with_country_perimeter=True, + ) + siae_country_activity_2.sectors.add(self.sectors[0]) + + # tender perimeter custom with is_country_area = False + tender_1 = TenderFactory(sectors=self.sectors, is_country_area=True) + siae_found_list_1 = Siae.objects.filter_with_tender_through_activities(tender_1) + self.assertEqual(len(siae_found_list_1), 2) + # tender perimeter custom with include_country_area = True + tender_2 = TenderFactory(sectors=self.sectors, include_country_area=True) + siae_found_list_2 = Siae.objects.filter_with_tender_through_activities(tender_2) + # we should have the same length of structures + self.assertEqual(len(siae_found_list_1), len(siae_found_list_2)) + # add perimeters + tender_2.perimeters.set(self.perimeters) + siae_found_list_2 = Siae.objects.filter_with_tender_through_activities(tender_2) + self.assertEqual(len(siae_found_list_2), 2 + 2) + tender_2.is_country_area = True + tender_2.save() + siae_found_list_2 = Siae.objects.filter_with_tender_through_activities(tender_2) + # we should have only siaes with country geo range + self.assertEqual(len(siae_found_list_2), 2 + 0) + + def test_matching_siae_perimeters_custom_2(self): + # add Siae with geo_range_department (75) + siae_department = SiaeFactory(is_active=True) + siae_department_activity = SiaeActivityFactory( + siae=siae_department, sector_group=self.sectors[0].group, with_zones_perimeter=True + ) + siae_department_activity.sectors.add(self.sectors[0]) + perimeter_department = PerimeterFactory( + name="Paris", kind=Perimeter.KIND_DEPARTMENT, insee_code="75", region_code="11" + ) + siae_department_activity.locations.set([perimeter_department]) + + # tender perimeter custom + tender = TenderFactory(sectors=self.sectors, perimeters=self.perimeters) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 2 + 1) + + def test_matching_siae_perimeters_france(self): + # tender france + tender = TenderFactory(sectors=self.sectors, is_country_area=True) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 0) + # add Siae with geo_range_country + siae_country = SiaeFactory(is_active=True) + siae_country_activity = SiaeActivityFactory( + siae=siae_country, sector_group=self.sectors[0].group, with_country_perimeter=True + ) + siae_country_activity.sectors.add(self.sectors[0]) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 1) + + def test_no_siaes(self): + # tender with empty sectors list + tender = TenderFactory(sectors=[SectorFactory()], perimeters=self.perimeters) + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 0) + # tender near Marseille + tender_marseille = TenderFactory(sectors=self.sectors, perimeters=[self.perimeter_marseille]) + siae_found_list_marseille = Siae.objects.filter_with_tender_through_activities(tender_marseille) + self.assertEqual(len(siae_found_list_marseille), 0) + + def test_with_no_contact_email(self): + tender = TenderFactory(sectors=self.sectors, perimeters=self.perimeters) + siae = SiaeFactory(is_active=True, contact_email="") + siae_activity = SiaeActivityFactory(siae=siae, sector_group=self.sectors[0].group, with_zones_perimeter=True) + siae_activity.locations.set([self.perimeter_paris]) + siae_activity.sectors.add(self.sectors[0]) + + siae_found_list = Siae.objects.filter_with_tender_through_activities(tender) + self.assertEqual(len(siae_found_list), 2 + 0) + self.assertNotIn(siae, siae_found_list)