From 3505968548412d2968e0130d0358472d4ab4c8d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Reuiller?= Date: Wed, 22 May 2024 14:56:24 +0200 Subject: [PATCH] add support for multiline sep --- .../management/commands/import_new_sep.py | 130 +++++++++++------- 1 file changed, 78 insertions(+), 52 deletions(-) diff --git a/lemarche/siaes/management/commands/import_new_sep.py b/lemarche/siaes/management/commands/import_new_sep.py index 9c43b68f9..b46fe505c 100644 --- a/lemarche/siaes/management/commands/import_new_sep.py +++ b/lemarche/siaes/management/commands/import_new_sep.py @@ -1,3 +1,4 @@ +from lemarche.sectors.models import Sector from lemarche.siaes import constants as siae_constants from lemarche.siaes.management.commands.import_sep import Command as SepCommand, read_csv as read_sep_csv from lemarche.siaes.models import Siae @@ -31,9 +32,7 @@ def add_arguments(self, parser): default="", ) - def handle(self, *args, **options): - sep_command = SepCommand() - + def handle(self, *args, **options): # noqa C901 added = 0 delisted = 0 email_updated = 0 @@ -42,50 +41,65 @@ def handle(self, *args, **options): if options["csv_file"]: self.stdout_info("Importing SEP...") siae_list = read_sep_csv(options["csv_file"]) - progress = 0 + group_by_city = {} # group by city to handle multilines SEP, one line per sector for index, data in enumerate(siae_list): - progress += 1 - if (progress % 10) == 0: - self.stdout_info(f"{progress}...") + if data["Ville"] not in group_by_city: + group_by_city[data["Ville"]] = [] + + group_by_city[data["Ville"]].append(data) + for city, datas in group_by_city.items(): try: - # TODO: fix email and phone update when there are several lines for the same SEP siae_same_city = Siae.objects.get( - siret=data["Siret"], city=data["Ville"], kind=siae_constants.KIND_SEP + siret=datas[0]["Siret"], city=datas[0]["Ville"], kind=siae_constants.KIND_SEP ) - if siae_same_city.address != data["Adresse"]: - self.stdout_error("Different address (but nothing will be done automatically): ") - self.stdout_error(f"Adresse csv {data['Adresse']} <-> {siae_same_city.address}") + if siae_same_city.user_count == 0: # update only if siae has no user + # Update siae in db with the first line related to the same city + data = datas.pop(0) - email_updated += self.update_email_if_different( - siae_same_city, data["Email 1"], options["dry_run"] - ) + sector = Sector.objects.get(name=data["Secteurs d'act list"][0]) + if siae_same_city.sectors.count() > 1 or siae_same_city.sectors.first() != sector: + if options["dry_run"]: + self.stdout_info(f"sector needs to be erase per {sector}") + else: + self.stdout_success(f"sector has been erase per {sector}") + siae_same_city.sectors.set([sector]) - phone_updated += self.update_phone_if_different( - siae_same_city, data["Téléphone"], options["dry_run"] - ) + if siae_same_city.address != data["Adresse"]: + self.stdout_error("Different address (but nothing will be done automatically): ") + self.stdout_error(f"Adresse csv {data['Adresse']} <-> {siae_same_city.address}") + + email_updated += self.update_email_if_different( + siae_same_city, data["Email 1"], options["dry_run"] + ) + + phone_updated += self.update_phone_if_different( + siae_same_city, data["Téléphone"], options["dry_run"] + ) + + # add other + for data in datas: + added += 1 + if options["dry_run"]: + self.stdout_messages_success(["This SEP need to be add for its new sector :", data]) + else: + self.import_sep(data, "sep") except Siae.DoesNotExist: - added += 1 - if options["dry_run"]: - self.stdout_success("This SEP need to be add :") - self.stdout_success(data) - else: - [data.pop(key) for key in USELESS_COLUMN_NAME_LIST if key in data] - sep_command.import_sep(data, source="sep") - self.stdout_success("This SEP has been added :") - self.stdout_success(data) + for data in datas: + added += 1 + if options["dry_run"]: + self.stdout_messages_success(["This SEP need to be add :", data]) + else: + self.import_sep(data, source="sep") + except Siae.MultipleObjectsReturned: + self.stdout_error(f"Mutliple siae in db isn't yet supported (city : {city})") if options["csv_file_externe"]: self.stdout_info("Importing SEP Externe...") siae_list = read_sep_csv(options["csv_file_externe"]) - progress = 0 for index, data in enumerate(siae_list): - progress += 1 - if (progress % 10) == 0: - self.stdout_info(f"{progress}...") - if data["A Supprimer"] == "1": siae_with_siret = Siae.objects.get(siret=data["Siret"], kind=siae_constants.KIND_SEP) if not siae_with_siret.is_delisted: @@ -109,34 +123,46 @@ def handle(self, *args, **options): self.stdout_error(f"Ville csv {data['Ville']} <-> {siae_same_siret.city}") self.stdout_error(f"Adresse csv {data['Adresse']} <-> {siae_same_siret.address}") - email_updated += self.update_email_if_different( - siae_same_siret, data["Email 1"], options["dry_run"] - ) - phone_updated += self.update_phone_if_different( - siae_same_siret, data["Téléphone"], options["dry_run"] - ) + if siae_same_siret.user_count == 0: # update only if siae has no user + email_updated += self.update_email_if_different( + siae_same_siret, data["Email 1"], options["dry_run"] + ) + phone_updated += self.update_phone_if_different( + siae_same_siret, data["Téléphone"], options["dry_run"] + ) except Siae.DoesNotExist: added += 1 if options["dry_run"]: - self.stdout_success("This SEP externe need to be add :") - self.stdout_success(data) + self.stdout_messages_success(["This SEP externe need to be add :", data]) else: - [data.pop(key) for key in USELESS_COLUMN_NAME_LIST if key in data] - sep_command.import_sep(data, source="sep_externe") - self.stdout_success("This SEP externe has been added :") - self.stdout_success(data) + self.import_sep(data, "sep_externe") self.stdout_success("Done !") if options["dry_run"]: - self.stdout_success(f"{added} new SEP needs to be added") - self.stdout_success(f"{delisted} old SEP needs to be delisted") - self.stdout_success(f"Email needs update : {email_updated}") - self.stdout_success(f"Phone needs update : {phone_updated}") + self.stdout_messages_success( + [ + f"{added} new SEP needs to be added", + f"{delisted} old SEP needs to be delisted", + f"Email needs update : {email_updated}", + f"Phone needs update : {phone_updated}", + ] + ) else: - self.stdout_success(f"{added} SIAE has been added") - self.stdout_success(f"{delisted} old SEP has been delisted") - self.stdout_success(f"{email_updated} email updated !") - self.stdout_success(f"{phone_updated} phone updated !") + self.stdout_messages_success( + [ + f"{added} SIAE has been added", + f"{delisted} old SEP has been delisted", + f"{email_updated} email updated !", + f"{phone_updated} phone updated !", + ] + ) + + def import_sep(self, data, source): + sep_command = SepCommand() + + [data.pop(key) for key in USELESS_COLUMN_NAME_LIST if key in data] + sep_command.import_sep(data, source=source) + self.stdout_messages_success(["This SEP has been added :", data]) def update_email_if_different(self, siae, email, dry_run): contact_email_before = siae.contact_email