From 6d52305ea6c121bf6773d8e9c3fc0da1fab69365 Mon Sep 17 00:00:00 2001 From: Raphael Odini Date: Thu, 30 Nov 2023 11:41:20 +0100 Subject: [PATCH] Replace C1 with les-emplois --- clevercloud/cron.json | 2 +- ...h => siaes_sync_with_emplois_inclusion.sh} | 8 ++--- ...1_c4.py => sync_with_emplois_inclusion.py} | 32 ++++++++++--------- 3 files changed, 22 insertions(+), 20 deletions(-) rename clevercloud/{siaes_sync_c1_c4.sh => siaes_sync_with_emplois_inclusion.sh} (59%) rename lemarche/siaes/management/commands/{sync_c1_c4.py => sync_with_emplois_inclusion.py} (93%) diff --git a/clevercloud/cron.json b/clevercloud/cron.json index 53b21ae93..b2c546363 100644 --- a/clevercloud/cron.json +++ b/clevercloud/cron.json @@ -3,7 +3,7 @@ "15 0 * * * $ROOT/clevercloud/stats_export_user_download_list_to_file.sh", "30 0 * * * $ROOT/clevercloud/stats_export_user_search_list_to_file.sh", "0 1 * * * $ROOT/clevercloud/tenders_update_count_fields.sh", - "0 7 * * 1 $ROOT/clevercloud/siaes_sync_c1_c4.sh", + "0 7 * * 1 $ROOT/clevercloud/siaes_sync_with_emplois_inclusion.sh", "5 7 * * 1 $ROOT/clevercloud/siaes_sync_c2_c4.sh", "10 7 * * 1 $ROOT/clevercloud/siaes_update_api_entreprise_fields.sh", "15 7 * * 1 $ROOT/clevercloud/siaes_update_api_qpv_fields.sh", diff --git a/clevercloud/siaes_sync_c1_c4.sh b/clevercloud/siaes_sync_with_emplois_inclusion.sh similarity index 59% rename from clevercloud/siaes_sync_c1_c4.sh rename to clevercloud/siaes_sync_with_emplois_inclusion.sh index ce54dd924..577200976 100755 --- a/clevercloud/siaes_sync_c1_c4.sh +++ b/clevercloud/siaes_sync_with_emplois_inclusion.sh @@ -1,10 +1,10 @@ #!/bin/bash -l -# Fetch new siaes from C1 + update existing +# Fetch new siaes from les-emplois + update existing # Do not run if this env var is not set: -if [[ -z "$CRON_SYNC_C1_C4_ENABLED" ]]; then - echo "CRON_SYNC_C1_C4_ENABLED not set. Exiting..." +if [[ -z "$CRON_SYNC_WITH_EMPLOIS_INCLUSION_ENABLED" ]]; then + echo "CRON_SYNC_WITH_EMPLOIS_INCLUSION_ENABLED not set. Exiting..." exit 0 fi @@ -19,4 +19,4 @@ fi # $APP_HOME is set by default by clever cloud. cd $APP_HOME -django-admin sync_c1_c4 +django-admin sync_with_emplois_inclusion diff --git a/lemarche/siaes/management/commands/sync_c1_c4.py b/lemarche/siaes/management/commands/sync_with_emplois_inclusion.py similarity index 93% rename from lemarche/siaes/management/commands/sync_c1_c4.py rename to lemarche/siaes/management/commands/sync_with_emplois_inclusion.py index 7b9b68ade..807b28e66 100644 --- a/lemarche/siaes/management/commands/sync_c1_c4.py +++ b/lemarche/siaes/management/commands/sync_with_emplois_inclusion.py @@ -63,7 +63,7 @@ def map_siae_nature(siae_source): def set_is_active(siae): """ - C1 field + les-emplois field Most Siae have a convention (AI, ACI, EI, ETTI, EITI) We consider the Siae as active if "having a convention" + "this convention is active". @@ -81,7 +81,7 @@ def set_is_active(siae): def set_is_delisted(siae): """ - C4 field + le-marche field Helps to track the number of Siae who were set from active to inactive during a sync. """ is_active = set_is_active(siae) @@ -90,16 +90,18 @@ def set_is_delisted(siae): class Command(BaseCommand): """ - This command syncs the list of siae (creates new, updates existing) from C1 to C4. + This command syncs the list of siae (creates new, updates existing) from les-emplois to le-marché. + C1 = les-emplois + C4 = le-marche Steps: - 1. First we fetch all the siae from C1 + 1. First we fetch all the siae from les-emplois 2. Then we loop on each of them, to create or update it (depends if its c1_id already exists or not) 3. Don't forget to delist the siae who were not updated or inactive Usage: - - poetry run python manage.py sync_c1_c4 --dry-run - - poetry run python manage.py sync_c1_c4 + - poetry run python manage.py sync_with_emplois_inclusion --dry-run + - poetry run python manage.py sync_with_emplois_inclusion """ def add_arguments(self, parser): @@ -110,21 +112,21 @@ def handle(self, dry_run=False, **options): raise CommandError("Missing API_EMPLOIS_INCLUSION_TOKEN in env") self.stdout_info("-" * 80) - self.stdout_info("Sync script between C1 & C4...") + self.stdout_info("Sync script between les-emplois & le-marché...") if dry_run: self.stdout_info("Running in dry run mode !") self.stdout_info("-" * 80) - self.stdout_info("Step 1: fetching C1 data") + self.stdout_info("Step 1: fetching les-emplois data") c1_list = self.c1_export() self.stdout_info("-" * 80) - self.stdout_info("Step 2: filter C1 data") + self.stdout_info("Step 2: filter les-emplois data") c1_list_filtered = self.filter_c1_export(c1_list) self.stdout_info("-" * 80) - self.stdout_info("Step 3: update C4 data") + self.stdout_info("Step 3: update le-marche data") # count before siae_total_before = Siae.objects.all().count() siae_active_before = Siae.objects.filter(is_active=True).count() @@ -142,7 +144,7 @@ def handle(self, dry_run=False, **options): created_count = siae_total_after - siae_total_before updated_count = len(c1_list_filtered) - created_count msg_success = [ - "----- Synchronisation C1/C4 -----", + "----- Synchronisation emplois/marché -----", f"Siae total: before {siae_total_before} / after {siae_total_after} / +{created_count}", f"Siae updated: {updated_count}", f"Siae active: before {siae_active_before} / after {siae_active_after}", @@ -205,10 +207,10 @@ def c1_export(self): # noqa C901 c1_list_cleaned.append(c1_siae_cleaned) - self.stdout_info(f"Found {len(c1_list_cleaned)} Siae in C1") + self.stdout_info(f"Found {len(c1_list_cleaned)} Siae in les-emplois") return c1_list_cleaned except Exception as e: - api_slack.send_message_to_channel("Erreur lors de la synchronisation C1 <-> C4") + api_slack.send_message_to_channel("Erreur lors de la synchronisation emplois <-> marché") self.stdout_error(e) raise Exception(e) @@ -246,7 +248,7 @@ def c4_update(self, c1_list, dry_run): def c4_create_siae(self, c1_siae, dry_run): """ - Here we create a new Siae with C1 data + Here we create a new Siae with les-emplois data """ self.stdout_info("Creating Siae...") @@ -282,7 +284,7 @@ def add_siae_to_contact_list(self, siae: Siae): def c4_update_siae(self, c1_siae, c4_siae, dry_run): """ - Here we update an existing Siae with a subset of C1 data + Here we update an existing Siae with a subset of les-emplois data """ if not dry_run: # other fields