diff --git a/lemarche/siaes/management/commands/update_api_entreprise_fields.py b/lemarche/siaes/management/commands/update_api_entreprise_fields.py index 8aa36376a..9200aa115 100644 --- a/lemarche/siaes/management/commands/update_api_entreprise_fields.py +++ b/lemarche/siaes/management/commands/update_api_entreprise_fields.py @@ -1,11 +1,18 @@ import time +from datetime import datetime from django.db.models import Q +from django.utils import timezone from sentry_sdk.crons import monitor from lemarche.siaes.models import Siae from lemarche.utils.apis import api_slack -from lemarche.utils.apis.api_entreprise import siae_update_entreprise, siae_update_etablissement, siae_update_exercice +from lemarche.utils.apis.api_entreprise import ( + API_ENTREPRISE_REASON, + entreprise_get_or_error, + etablissement_get_or_error, + exercice_get_or_error, +) from lemarche.utils.commands import BaseCommand @@ -54,124 +61,96 @@ def handle(self, *args, **options): if options["limit"]: siae_queryset = siae_queryset[: options["limit"]] - # self.stdout_info(f"Found {siae_queryset.count()} Siae") + results = { + "entreprise": {"success": 0, "error": 0}, + "etablissement": {"success": 0, "error": 0}, + "exercice": {"success": 0, "error": 0}, + } - if options["scope"] in ("all", "entreprise"): - self.update_api_entreprise_entreprise_fields(siae_queryset) - - if options["scope"] in ("all", "etablissement"): - self.update_api_entreprise_etablissement_fields(siae_queryset) - - if options["scope"] in ("all", "exercice"): - self.update_api_entreprise_exercice_fields(siae_queryset) - - # API Entreprise: entreprises - def update_api_entreprise_entreprise_fields(self, siae_queryset): progress = 0 - results = {"success": 0, "error": 0} - siae_queryset_entreprise = siae_queryset.filter(api_entreprise_entreprise_last_sync_date=None) - self.stdout_info("-" * 80) - self.stdout_info(f"Populating 'entreprise' for {siae_queryset_entreprise.count()} Siae...") - - for siae in siae_queryset_entreprise: - try: - progress += 1 - if (progress % 50) == 0: - self.stdout_info(f"{progress}...") - # self.stdout_info("-" * 80) - # self.stdout_info(f"{siae.id} / {siae.name} / {siae.siret}") - response, message = siae_update_entreprise(siae) - if response: - results["success"] += 1 - else: - self.stdout_error(str(message)) - results["error"] += 1 - # small delay to avoid going above the API limitation - # "max. 250 requêtes/min/jeton cumulées sur tous les endpoints" - time.sleep(0.5) - except Exception as e: - self.stdout_error(str(e)) - api_slack.send_message_to_channel("Erreur lors de la synchronisation API entreprises: entreprises") + for siae in siae_queryset: + progress += 1 + if (progress % 50) == 0: + self.stdout_info(f"{progress}...") + + if siae.siret: + update_data = dict() + if options["scope"] in ("all", "entreprise") and siae.api_entreprise_entreprise_last_sync_date is None: + entreprise, error = entreprise_get_or_error(siae.siret[:9], reason=API_ENTREPRISE_REASON) + if error: + results["entreprise"]["error"] += 1 + self.stdout_error(str(error)) + else: + results["entreprise"]["success"] += 1 + if entreprise: + if entreprise.forme_juridique: + update_data["api_entreprise_forme_juridique"] = entreprise.forme_juridique + if entreprise.forme_juridique_code: + update_data["api_entreprise_forme_juridique_code"] = entreprise.forme_juridique_code + update_data["api_entreprise_entreprise_last_sync_date"] = timezone.now() + + if ( + options["scope"] in ("all", "etablissement") + and siae.api_entreprise_etablissement_last_sync_date is None + ): + etablissement, error = etablissement_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) + if error: + results["etablissement"]["error"] += 1 + self.stdout_error(str(error)) + else: + results["etablissement"]["success"] += 1 + if etablissement: + if etablissement.employees: + update_data["api_entreprise_employees"] = ( + etablissement.employees + if (etablissement.employees != "Unités non employeuses") + else "Non renseigné" + ) + if etablissement.employees_date_reference: + update_data[ + "api_entreprise_employees_year_reference" + ] = etablissement.employees_date_reference + if etablissement.date_constitution: + update_data["api_entreprise_date_constitution"] = etablissement.date_constitution + + update_data["api_entreprise_etablissement_last_sync_date"] = timezone.now() + + if options["scope"] in ("all", "exercice") and siae.api_entreprise_exercice_last_sync_date is None: + exercice, error = exercice_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) + if error: + results["exercice"]["error"] += 1 + self.stdout_error(str(error)) + else: + results["exercice"]["success"] += 1 + if exercice: + if exercice.chiffre_affaires: + update_data["api_entreprise_ca"] = exercice.chiffre_affaires + if exercice.date_fin_exercice: + update_data["api_entreprise_ca_date_fin_exercice"] = datetime.strptime( + exercice.date_fin_exercice, "%Y-%m-%d" + ).date() + + update_data["api_entreprise_exercice_last_sync_date"] = timezone.now() + + Siae.objects.filter(id=siae.id).update(**update_data) - msg_success = [ - "----- Synchronisation API Entreprise (/entreprises) -----", - f"Done! Processed {siae_queryset_entreprise.count()} siae", - f"success count: {results['success']}/{siae_queryset_entreprise.count()}", - f"error count: {results['error']}/{siae_queryset_entreprise.count()} (voir les logs)", - ] - self.stdout_messages_success(msg_success) - api_slack.send_message_to_channel("\n".join(msg_success)) - - # API Entreprise: etablissements - def update_api_entreprise_etablissement_fields(self, siae_queryset): - progress = 0 - results = {"success": 0, "error": 0} - siae_queryset_etablissement = siae_queryset.filter(api_entreprise_etablissement_last_sync_date=None) - self.stdout_info("-" * 80) - self.stdout_info(f"Populating 'etablissement' for {siae_queryset_etablissement.count()} Siae...") - - for siae in siae_queryset_etablissement: - try: - progress += 1 - if (progress % 50) == 0: - self.stdout_info(f"{progress}...") - # self.stdout_info("-" * 80) - # self.stdout_info(f"{siae.id} / {siae.name} / {siae.siret}") - response, message = siae_update_etablissement(siae) - if response: - results["success"] += 1 - else: - self.stdout_error(str(message)) - results["error"] += 1 - # small delay to avoid going above the API limitation - # "max. 250 requêtes/min/jeton cumulées sur tous les endpoints" - time.sleep(0.5) - except Exception as e: - self.stdout_error(str(e)) - api_slack.send_message_to_channel("Erreur lors de la synchronisation API entreprises: etablissements") - - msg_success = [ - "----- Synchronisation API Entreprise (/etablissements) -----", - f"Done! Processed {siae_queryset_etablissement.count()} siae", - f"success count: {results['success']}/{siae_queryset_etablissement.count()}", - f"error count: {results['error']}/{siae_queryset_etablissement.count()} (voir les logs)", - ] - self.stdout_messages_success(msg_success) - api_slack.send_message_to_channel("\n".join(msg_success)) - - # API Entreprise: exercices - def update_api_entreprise_exercice_fields(self, siae_queryset): - progress = 0 - results = {"success": 0, "error": 0} - siae_queryset_exercice = siae_queryset.filter(api_entreprise_exercice_last_sync_date=None) - self.stdout_info("-" * 80) - self.stdout_info(f"Populating 'exercice' for {siae_queryset_exercice.count()} Siae...") - - for siae in siae_queryset_exercice: - try: - progress += 1 - if (progress % 50) == 0: - self.stdout_info(f"{progress}...") - # self.stdout_info("-" * 80) - # self.stdout_info(f"{siae.id} / {siae.name} / {siae.siret}") - response, message = siae_update_exercice(siae) - if response: - results["success"] += 1 - else: - self.stdout_error(str(message)) - results["error"] += 1 # small delay to avoid going above the API limitation # "max. 250 requêtes/min/jeton cumulées sur tous les endpoints" time.sleep(0.5) - except Exception as e: - self.stdout_error(str(e)) - api_slack.send_message_to_channel("Erreur lors de la synchronisation API entreprises: exercices") + else: + self.stdout_error(f"SIAE {siae.id} without SIRET") msg_success = [ - "----- Synchronisation API Entreprise (/exercices) -----", - f"Done! Processed {siae_queryset_exercice.count()} siae", - f"success count: {results['success']}/{siae_queryset_exercice.count()}", - f"error count: {results['error']}/{siae_queryset_exercice.count()} (voir les logs)", + "----- Synchronisation API Entreprise -----", + f"Done! Processed {siae_queryset.count()} siae", + "----- Success -----", + f"entreprise: {results['entreprise']['success']}/{siae_queryset.count()}", + f"etablissement: {results['etablissement']['success']}/{siae_queryset.count()}", + f"exercice: {results['exercice']['success']}/{siae_queryset.count()}", + "----- Error ----- (voir les logs)", + f"entreprise: {results['entreprise']['error']}/{siae_queryset.count()}", + f"etablissement: {results['etablissement']['error']}/{siae_queryset.count()}", + f"exercice: {results['exercice']['error']}/{siae_queryset.count()}", ] self.stdout_messages_success(msg_success) api_slack.send_message_to_channel("\n".join(msg_success)) diff --git a/lemarche/siaes/tests/test_commands.py b/lemarche/siaes/tests/test_commands.py index b78cc1541..9f5eb55c3 100644 --- a/lemarche/siaes/tests/test_commands.py +++ b/lemarche/siaes/tests/test_commands.py @@ -1,11 +1,14 @@ -import factory +import datetime +import json import logging import os from unittest.mock import patch +import factory from django.core.management import call_command from django.db.models import signals from django.test import TransactionTestCase +from django.utils import timezone from lemarche.perimeters.factories import PerimeterFactory from lemarche.perimeters.models import Perimeter @@ -14,6 +17,11 @@ from lemarche.siaes.factories import SiaeActivityFactory, SiaeFactory from lemarche.siaes.models import Siae, SiaeActivity from lemarche.users.factories import UserFactory +from lemarche.utils.mocks.api_entreprise import ( + MOCK_ENTREPRISE_API_DATA, + MOCK_ETABLISSEMENT_API_DATA, + MOCK_EXERCICES_API_DATA, +) class SyncWithEmploisInclusionCommandTest(TransactionTestCase): @@ -524,3 +532,55 @@ def test_update_count_fields_with_id(self): siae_not_updated.refresh_from_db() self.assertEqual(siae_not_updated.user_count, 0) self.assertEqual(siae_not_updated.sector_count, 0) + + +class SiaeUpdateApiEntrepriseFieldsCommandTest(TransactionTestCase): + @patch("requests.get") + def test_siae_update_entreprise(self, mock_api): + mock_response = mock_api.return_value + mock_response.json.return_value = json.loads(MOCK_ENTREPRISE_API_DATA) + mock_response.status_code = 200 + + siae = SiaeFactory(siret="13002526500013") + + call_command("update_api_entreprise_fields", scope="entreprise") + + # Assert the updates + siae.refresh_from_db() + self.assertEqual(siae.api_entreprise_forme_juridique, "Service central d'un ministère") + self.assertEqual(siae.api_entreprise_forme_juridique_code, "7120") + self.assertLess((timezone.now() - siae.api_entreprise_entreprise_last_sync_date).total_seconds(), 60) + + @patch("requests.get") + def test_siae_update_etablissement(self, mock_api): + mock_response = mock_api.return_value + mock_response.json.return_value = json.loads(MOCK_ETABLISSEMENT_API_DATA) + mock_response.status_code = 200 + + siae = SiaeFactory(siret="30613890001294") + + call_command("update_api_entreprise_fields", scope="etablissement") + + # Assert the updates + siae.refresh_from_db() + self.assertEqual(siae.siret, "30613890001294") + self.assertEqual(siae.api_entreprise_employees, "2 000 à 4 999 salariés") + self.assertEqual(siae.api_entreprise_employees_year_reference, "2016") + self.assertEqual(siae.api_entreprise_date_constitution, datetime.date(2021, 10, 13)) + self.assertLess((timezone.now() - siae.api_entreprise_etablissement_last_sync_date).total_seconds(), 60) + + @patch("requests.get") + def test_siae_update_exercice(self, mock_api): + mock_response = mock_api.return_value + mock_response.json.return_value = json.loads(MOCK_EXERCICES_API_DATA) + mock_response.status_code = 200 + + siae = SiaeFactory(siret="30613890001294") + + call_command("update_api_entreprise_fields", scope="exercice") + + # Assert the updates + siae.refresh_from_db() + self.assertEqual(siae.api_entreprise_ca, 900001) + self.assertEqual(siae.api_entreprise_ca_date_fin_exercice, datetime.date(2015, 12, 1)) + self.assertLess((timezone.now() - siae.api_entreprise_exercice_last_sync_date).total_seconds(), 60) diff --git a/lemarche/utils/apis/api_entreprise.py b/lemarche/utils/apis/api_entreprise.py index d00ab6393..eca055df0 100644 --- a/lemarche/utils/apis/api_entreprise.py +++ b/lemarche/utils/apis/api_entreprise.py @@ -2,20 +2,16 @@ import logging from dataclasses import dataclass -from datetime import date, datetime +from datetime import date import requests from django.conf import settings -from django.utils import timezone from django.utils.http import urlencode -from lemarche.siaes.models import Siae - logger = logging.getLogger(__name__) API_ENTREPRISE_REASON = "Mise à jour données Marché de la plateforme de l'Inclusion" -DATE_FORMAT = "%Y-%m-%d" @dataclass @@ -40,6 +36,17 @@ class Exercice: date_fin_exercice: date +def get_url_endpoint(endpoint: str, reason: str) -> str: + query_string = urlencode( + { + "recipient": settings.API_ENTREPRISE_RECIPIENT, + "context": settings.API_ENTREPRISE_CONTEXT, + "object": reason, + } + ) + return f"{settings.API_ENTREPRISE_BASE_URL}/{endpoint}?{query_string}" + + def entreprise_get_or_error(siren, reason="Inscription au marché de l'inclusion"): """ Obtain company data from entreprise.api.gouv.fr @@ -50,14 +57,7 @@ def entreprise_get_or_error(siren, reason="Inscription au marché de l'inclusion """ error = None - query_string = urlencode( - { - "recipient": settings.API_ENTREPRISE_RECIPIENT, - "context": settings.API_ENTREPRISE_CONTEXT, - "object": reason, - } - ) - url = f"{settings.API_ENTREPRISE_BASE_URL}/insee/sirene/unites_legales/{siren}?{query_string}" + url = get_url_endpoint(f"insee/sirene/unites_legales/{siren}", reason) headers = {"Authorization": f"Bearer {settings.API_ENTREPRISE_TOKEN}"} try: @@ -98,27 +98,6 @@ def entreprise_get_or_error(siren, reason="Inscription au marché de l'inclusion return entreprise, None -def siae_update_entreprise(siae): - if siae.siret: - siae_siren = siae.siret[:9] - entreprise, error = entreprise_get_or_error(siae_siren, reason=API_ENTREPRISE_REASON) - if error: - return 0, error - - update_data = dict() - if entreprise: - if entreprise.forme_juridique: - update_data["api_entreprise_forme_juridique"] = entreprise.forme_juridique - if entreprise.forme_juridique_code: - update_data["api_entreprise_forme_juridique_code"] = entreprise.forme_juridique_code - - update_data["api_entreprise_entreprise_last_sync_date"] = timezone.now() - Siae.objects.filter(id=siae.id).update(**update_data) - - return 1, entreprise - return 0, f"SIAE {siae.id} without SIREN" - - def etablissement_get_or_error(siret, reason="Inscription au marché de l'inclusion"): """ Obtain company data from entreprise.api.gouv.fr @@ -132,14 +111,7 @@ def etablissement_get_or_error(siret, reason="Inscription au marché de l'inclus """ error = None - query_string = urlencode( - { - "recipient": settings.API_ENTREPRISE_RECIPIENT, - "context": settings.API_ENTREPRISE_CONTEXT, - "object": reason, - } - ) - url = f"{settings.API_ENTREPRISE_BASE_URL}/insee/sirene/etablissements/{siret}?{query_string}" + url = get_url_endpoint(f"insee/sirene/etablissements/{siret}", reason) headers = {"Authorization": f"Bearer {settings.API_ENTREPRISE_TOKEN}"} try: r = requests.get(url, headers=headers) @@ -187,33 +159,6 @@ def etablissement_get_or_error(siret, reason="Inscription au marché de l'inclus return etablissement, None -def siae_update_etablissement(siae): - if siae.siret: - etablissement, error = etablissement_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) - if error: - return 0, error - - update_data = dict() - if etablissement: - if etablissement.employees: - update_data["api_entreprise_employees"] = ( - etablissement.employees - if (etablissement.employees != "Unités non employeuses") - else "Non renseigné" - ) - if etablissement.employees_date_reference: - update_data["api_entreprise_employees_year_reference"] = etablissement.employees_date_reference - if etablissement.date_constitution: - update_data["api_entreprise_date_constitution"] = etablissement.date_constitution - - update_data["api_entreprise_etablissement_last_sync_date"] = timezone.now() - Siae.objects.filter(id=siae.id).update(**update_data) - - return 1, etablissement - - return 0, f"SIAE {siae.id} without SIRET" - - def exercice_get_or_error(siret, reason="Inscription au marché de l'inclusion"): """ Obtain company data from entreprises.api.gouv.fr @@ -226,14 +171,7 @@ def exercice_get_or_error(siret, reason="Inscription au marché de l'inclusion") """ error = None - query_string = urlencode( - { - "recipient": settings.API_ENTREPRISE_RECIPIENT, - "context": settings.API_ENTREPRISE_CONTEXT, - "object": reason, - } - ) - url = f"{settings.API_ENTREPRISE_BASE_URL}/dgfip/etablissements/{siret}/chiffres_affaires?{query_string}" + url = get_url_endpoint(f"dgfip/etablissements/{siret}/chiffres_affaires", reason) headers = {"Authorization": f"Bearer {settings.API_ENTREPRISE_TOKEN}"} try: @@ -268,27 +206,3 @@ def exercice_get_or_error(siret, reason="Inscription au marché de l'inclusion") date_fin_exercice=response["data"][0]["data"]["date_fin_exercice"], ) return exercice, None - - -def siae_update_exercice(siae): - if siae.siret: - exercice, error = exercice_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON) - if error: - return 0, error - - update_data = dict() - - if exercice: - update_data = dict() - if exercice.chiffre_affaires: - update_data["api_entreprise_ca"] = exercice.chiffre_affaires - if exercice.date_fin_exercice: - update_data["api_entreprise_ca_date_fin_exercice"] = datetime.strptime( - exercice.date_fin_exercice, DATE_FORMAT - ).date() - - update_data["api_entreprise_exercice_last_sync_date"] = timezone.now() - Siae.objects.filter(id=siae.id).update(**update_data) - - return 1, exercice - return 0, f"SIAE {siae.id} without SIRET" diff --git a/lemarche/utils/tests_apis.py b/lemarche/utils/mocks/api_entreprise.py similarity index 65% rename from lemarche/utils/tests_apis.py rename to lemarche/utils/mocks/api_entreprise.py index ec8e7bcaf..170c110f0 100644 --- a/lemarche/utils/tests_apis.py +++ b/lemarche/utils/mocks/api_entreprise.py @@ -1,14 +1,4 @@ -import datetime -import json -from unittest.mock import patch - -from django.test import TestCase -from django.utils import timezone - -from lemarche.siaes.factories import SiaeFactory -from lemarche.utils.apis.api_entreprise import siae_update_entreprise, siae_update_etablissement, siae_update_exercice - - +# Result for a call to: https://entreprises.api.gouv.fr/api/v3/insee/unites_legales/130025265 MOCK_ENTREPRISE_API_DATA = """ { "data": { @@ -66,31 +56,7 @@ } """ - -class TestSiaeUpdateEntreprise(TestCase): - def setUp(self): - self.siae = SiaeFactory(siret="13002526500013") - - @patch("requests.get") - def test_siae_update_entreprise(self, mock_api): - mock_response = mock_api.return_value - mock_response.json.return_value = json.loads(MOCK_ENTREPRISE_API_DATA) - mock_response.status_code = 200 - - result, entreprise = siae_update_entreprise(self.siae) - - self.siae.refresh_from_db() - - # Assert the result - self.assertEqual(result, 1) - self.assertIsNotNone(entreprise) - - # Assert the updates - self.assertEqual(self.siae.api_entreprise_forme_juridique, "Service central d'un ministère") - self.assertEqual(self.siae.api_entreprise_forme_juridique_code, "7120") - self.assertLess((timezone.now() - self.siae.api_entreprise_entreprise_last_sync_date).total_seconds(), 60) - - +# Result for a call to: https://entreprises.api.gouv.fr/api/v3/insee/sirene/etablissements/30613890001294 MOCK_ETABLISSEMENT_API_DATA = """ { "data": { @@ -194,32 +160,7 @@ def test_siae_update_entreprise(self, mock_api): } """ - -class TestSiaeUpdateEtablissement(TestCase): - def setUp(self): - self.siae = SiaeFactory(siret="30613890001294") - - @patch("requests.get") - def test_siae_update_etablissement(self, mock_api): - mock_response = mock_api.return_value - mock_response.json.return_value = json.loads(MOCK_ETABLISSEMENT_API_DATA) - mock_response.status_code = 200 - - result, etablissement = siae_update_etablissement(self.siae) - - # Assert the result - self.assertEqual(result, 1) - self.assertIsNotNone(etablissement) - - # Assert the updates - self.siae.refresh_from_db() - self.assertEqual(self.siae.siret, "30613890001294") - self.assertEqual(self.siae.api_entreprise_employees, "2 000 à 4 999 salariés") - self.assertEqual(self.siae.api_entreprise_employees_year_reference, "2016") - self.assertEqual(self.siae.api_entreprise_date_constitution, datetime.date(2021, 10, 13)) - self.assertLess((timezone.now() - self.siae.api_entreprise_etablissement_last_sync_date).total_seconds(), 60) - - +# Result for a call to: https://entreprises.api.gouv.fr/api/v3/dgfip/etablissements/30613890001294/chiffres_affaires MOCK_EXERCICES_API_DATA = """ { "data": [ @@ -236,21 +177,3 @@ def test_siae_update_etablissement(self, mock_api): "links": {} } """ - - -class TestSiaeUpdateExercice(TestCase): - def setUp(self): - self.siae = SiaeFactory(siret="30613890001294") - - @patch("requests.get") - def test_siae_update_exercice(self, mock_api): - mock_response = mock_api.return_value - mock_response.json.return_value = json.loads(MOCK_EXERCICES_API_DATA) - mock_response.status_code = 200 - - siae_update_exercice(self.siae) - - self.siae.refresh_from_db() - self.assertEqual(self.siae.api_entreprise_ca, 900001) - self.assertEqual(self.siae.api_entreprise_ca_date_fin_exercice, datetime.date(2015, 12, 1)) - self.assertLess((timezone.now() - self.siae.api_entreprise_exercice_last_sync_date).total_seconds(), 60)