Skip to content

Commit

Permalink
refactor: merge the three updates into one
Browse files Browse the repository at this point in the history
  • Loading branch information
@SebastienReuiller
SebastienReuiller committed Dec 19, 2024
1 parent 7732a25 commit 605bea8
Show file tree
Hide file tree
Showing 4 changed files with 169 additions and 293 deletions.
201 changes: 90 additions & 111 deletions lemarche/siaes/management/commands/update_api_entreprise_fields.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
import time
from datetime import datetime

from django.db.models import Q
from django.utils import timezone
from sentry_sdk.crons import monitor

from lemarche.siaes.models import Siae
from lemarche.utils.apis import api_slack
from lemarche.utils.apis.api_entreprise import siae_update_entreprise, siae_update_etablissement, siae_update_exercice
from lemarche.utils.apis.api_entreprise import (
API_ENTREPRISE_REASON,
entreprise_get_or_error,
etablissement_get_or_error,
exercice_get_or_error,
)
from lemarche.utils.commands import BaseCommand


Expand Down Expand Up @@ -54,124 +61,96 @@ def handle(self, *args, **options):
if options["limit"]:
siae_queryset = siae_queryset[: options["limit"]]

# self.stdout_info(f"Found {siae_queryset.count()} Siae")
results = {
"entreprise": {"success": 0, "error": 0},
"etablissement": {"success": 0, "error": 0},
"exercice": {"success": 0, "error": 0},
}

if options["scope"] in ("all", "entreprise"):
self.update_api_entreprise_entreprise_fields(siae_queryset)

if options["scope"] in ("all", "etablissement"):
self.update_api_entreprise_etablissement_fields(siae_queryset)

if options["scope"] in ("all", "exercice"):
self.update_api_entreprise_exercice_fields(siae_queryset)

# API Entreprise: entreprises
def update_api_entreprise_entreprise_fields(self, siae_queryset):
progress = 0
results = {"success": 0, "error": 0}
siae_queryset_entreprise = siae_queryset.filter(api_entreprise_entreprise_last_sync_date=None)
self.stdout_info("-" * 80)
self.stdout_info(f"Populating 'entreprise' for {siae_queryset_entreprise.count()} Siae...")

for siae in siae_queryset_entreprise:
try:
progress += 1
if (progress % 50) == 0:
self.stdout_info(f"{progress}...")
# self.stdout_info("-" * 80)
# self.stdout_info(f"{siae.id} / {siae.name} / {siae.siret}")
response, message = siae_update_entreprise(siae)
if response:
results["success"] += 1
else:
self.stdout_error(str(message))
results["error"] += 1
# small delay to avoid going above the API limitation
# "max. 250 requêtes/min/jeton cumulées sur tous les endpoints"
time.sleep(0.5)
except Exception as e:
self.stdout_error(str(e))
api_slack.send_message_to_channel("Erreur lors de la synchronisation API entreprises: entreprises")
for siae in siae_queryset:
progress += 1
if (progress % 50) == 0:
self.stdout_info(f"{progress}...")

if siae.siret:
update_data = dict()
if options["scope"] in ("all", "entreprise") and siae.api_entreprise_entreprise_last_sync_date is None:
entreprise, error = entreprise_get_or_error(siae.siret[:9], reason=API_ENTREPRISE_REASON)
if error:
results["entreprise"]["error"] += 1
self.stdout_error(str(error))
else:
results["entreprise"]["success"] += 1
if entreprise:
if entreprise.forme_juridique:
update_data["api_entreprise_forme_juridique"] = entreprise.forme_juridique
if entreprise.forme_juridique_code:
update_data["api_entreprise_forme_juridique_code"] = entreprise.forme_juridique_code
update_data["api_entreprise_entreprise_last_sync_date"] = timezone.now()

if (
options["scope"] in ("all", "etablissement")
and siae.api_entreprise_etablissement_last_sync_date is None
):
etablissement, error = etablissement_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON)
if error:
results["etablissement"]["error"] += 1
self.stdout_error(str(error))
else:
results["etablissement"]["success"] += 1
if etablissement:
if etablissement.employees:
update_data["api_entreprise_employees"] = (
etablissement.employees
if (etablissement.employees != "Unités non employeuses")
else "Non renseigné"
)
if etablissement.employees_date_reference:
update_data[
"api_entreprise_employees_year_reference"
] = etablissement.employees_date_reference
if etablissement.date_constitution:
update_data["api_entreprise_date_constitution"] = etablissement.date_constitution

update_data["api_entreprise_etablissement_last_sync_date"] = timezone.now()

if options["scope"] in ("all", "exercice") and siae.api_entreprise_exercice_last_sync_date is None:
exercice, error = exercice_get_or_error(siae.siret, reason=API_ENTREPRISE_REASON)
if error:
results["exercice"]["error"] += 1
self.stdout_error(str(error))
else:
results["exercice"]["success"] += 1
if exercice:
if exercice.chiffre_affaires:
update_data["api_entreprise_ca"] = exercice.chiffre_affaires
if exercice.date_fin_exercice:
update_data["api_entreprise_ca_date_fin_exercice"] = datetime.strptime(
exercice.date_fin_exercice, "%Y-%m-%d"
).date()

update_data["api_entreprise_exercice_last_sync_date"] = timezone.now()

Siae.objects.filter(id=siae.id).update(**update_data)

msg_success = [
"----- Synchronisation API Entreprise (/entreprises) -----",
f"Done! Processed {siae_queryset_entreprise.count()} siae",
f"success count: {results['success']}/{siae_queryset_entreprise.count()}",
f"error count: {results['error']}/{siae_queryset_entreprise.count()} (voir les logs)",
]
self.stdout_messages_success(msg_success)
api_slack.send_message_to_channel("\n".join(msg_success))

# API Entreprise: etablissements
def update_api_entreprise_etablissement_fields(self, siae_queryset):
progress = 0
results = {"success": 0, "error": 0}
siae_queryset_etablissement = siae_queryset.filter(api_entreprise_etablissement_last_sync_date=None)
self.stdout_info("-" * 80)
self.stdout_info(f"Populating 'etablissement' for {siae_queryset_etablissement.count()} Siae...")

for siae in siae_queryset_etablissement:
try:
progress += 1
if (progress % 50) == 0:
self.stdout_info(f"{progress}...")
# self.stdout_info("-" * 80)
# self.stdout_info(f"{siae.id} / {siae.name} / {siae.siret}")
response, message = siae_update_etablissement(siae)
if response:
results["success"] += 1
else:
self.stdout_error(str(message))
results["error"] += 1
# small delay to avoid going above the API limitation
# "max. 250 requêtes/min/jeton cumulées sur tous les endpoints"
time.sleep(0.5)
except Exception as e:
self.stdout_error(str(e))
api_slack.send_message_to_channel("Erreur lors de la synchronisation API entreprises: etablissements")

msg_success = [
"----- Synchronisation API Entreprise (/etablissements) -----",
f"Done! Processed {siae_queryset_etablissement.count()} siae",
f"success count: {results['success']}/{siae_queryset_etablissement.count()}",
f"error count: {results['error']}/{siae_queryset_etablissement.count()} (voir les logs)",
]
self.stdout_messages_success(msg_success)
api_slack.send_message_to_channel("\n".join(msg_success))

# API Entreprise: exercices
def update_api_entreprise_exercice_fields(self, siae_queryset):
progress = 0
results = {"success": 0, "error": 0}
siae_queryset_exercice = siae_queryset.filter(api_entreprise_exercice_last_sync_date=None)
self.stdout_info("-" * 80)
self.stdout_info(f"Populating 'exercice' for {siae_queryset_exercice.count()} Siae...")

for siae in siae_queryset_exercice:
try:
progress += 1
if (progress % 50) == 0:
self.stdout_info(f"{progress}...")
# self.stdout_info("-" * 80)
# self.stdout_info(f"{siae.id} / {siae.name} / {siae.siret}")
response, message = siae_update_exercice(siae)
if response:
results["success"] += 1
else:
self.stdout_error(str(message))
results["error"] += 1
# small delay to avoid going above the API limitation
# "max. 250 requêtes/min/jeton cumulées sur tous les endpoints"
time.sleep(0.5)
except Exception as e:
self.stdout_error(str(e))
api_slack.send_message_to_channel("Erreur lors de la synchronisation API entreprises: exercices")
else:
self.stdout_error(f"SIAE {siae.id} without SIRET")

msg_success = [
"----- Synchronisation API Entreprise (/exercices) -----",
f"Done! Processed {siae_queryset_exercice.count()} siae",
f"success count: {results['success']}/{siae_queryset_exercice.count()}",
f"error count: {results['error']}/{siae_queryset_exercice.count()} (voir les logs)",
"----- Synchronisation API Entreprise -----",
f"Done! Processed {siae_queryset.count()} siae",
"----- Success -----",
f"entreprise: {results['entreprise']['success']}/{siae_queryset.count()}",
f"etablissement: {results['etablissement']['success']}/{siae_queryset.count()}",
f"exercice: {results['exercice']['success']}/{siae_queryset.count()}",
"----- Error ----- (voir les logs)",
f"entreprise: {results['entreprise']['error']}/{siae_queryset.count()}",
f"etablissement: {results['etablissement']['error']}/{siae_queryset.count()}",
f"exercice: {results['exercice']['error']}/{siae_queryset.count()}",
]
self.stdout_messages_success(msg_success)
api_slack.send_message_to_channel("\n".join(msg_success))
62 changes: 61 additions & 1 deletion lemarche/siaes/tests/test_commands.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
import factory
import datetime
import json
import logging
import os
from unittest.mock import patch

import factory
from django.core.management import call_command
from django.db.models import signals
from django.test import TransactionTestCase
from django.utils import timezone

from lemarche.perimeters.factories import PerimeterFactory
from lemarche.perimeters.models import Perimeter
Expand All @@ -14,6 +17,11 @@
from lemarche.siaes.factories import SiaeActivityFactory, SiaeFactory
from lemarche.siaes.models import Siae, SiaeActivity
from lemarche.users.factories import UserFactory
from lemarche.utils.mocks.api_entreprise import (
MOCK_ENTREPRISE_API_DATA,
MOCK_ETABLISSEMENT_API_DATA,
MOCK_EXERCICES_API_DATA,
)


class SyncWithEmploisInclusionCommandTest(TransactionTestCase):
Expand Down Expand Up @@ -524,3 +532,55 @@ def test_update_count_fields_with_id(self):
siae_not_updated.refresh_from_db()
self.assertEqual(siae_not_updated.user_count, 0)
self.assertEqual(siae_not_updated.sector_count, 0)


class SiaeUpdateApiEntrepriseFieldsCommandTest(TransactionTestCase):
@patch("requests.get")
def test_siae_update_entreprise(self, mock_api):
mock_response = mock_api.return_value
mock_response.json.return_value = json.loads(MOCK_ENTREPRISE_API_DATA)
mock_response.status_code = 200

siae = SiaeFactory(siret="13002526500013")

call_command("update_api_entreprise_fields", scope="entreprise")

# Assert the updates
siae.refresh_from_db()
self.assertEqual(siae.api_entreprise_forme_juridique, "Service central d'un ministère")
self.assertEqual(siae.api_entreprise_forme_juridique_code, "7120")
self.assertLess((timezone.now() - siae.api_entreprise_entreprise_last_sync_date).total_seconds(), 60)

@patch("requests.get")
def test_siae_update_etablissement(self, mock_api):
mock_response = mock_api.return_value
mock_response.json.return_value = json.loads(MOCK_ETABLISSEMENT_API_DATA)
mock_response.status_code = 200

siae = SiaeFactory(siret="30613890001294")

call_command("update_api_entreprise_fields", scope="etablissement")

# Assert the updates
siae.refresh_from_db()
self.assertEqual(siae.siret, "30613890001294")
self.assertEqual(siae.api_entreprise_employees, "2 000 à 4 999 salariés")
self.assertEqual(siae.api_entreprise_employees_year_reference, "2016")
self.assertEqual(siae.api_entreprise_date_constitution, datetime.date(2021, 10, 13))
self.assertLess((timezone.now() - siae.api_entreprise_etablissement_last_sync_date).total_seconds(), 60)

@patch("requests.get")
def test_siae_update_exercice(self, mock_api):
mock_response = mock_api.return_value
mock_response.json.return_value = json.loads(MOCK_EXERCICES_API_DATA)
mock_response.status_code = 200

siae = SiaeFactory(siret="30613890001294")

call_command("update_api_entreprise_fields", scope="exercice")

# Assert the updates
siae.refresh_from_db()
self.assertEqual(siae.api_entreprise_ca, 900001)
self.assertEqual(siae.api_entreprise_ca_date_fin_exercice, datetime.date(2015, 12, 1))
self.assertLess((timezone.now() - siae.api_entreprise_exercice_last_sync_date).total_seconds(), 60)
Loading

0 comments on commit 605bea8

Please sign in to comment.