From 8f494143facad78175368dff981ba562dc2d10be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20Dor=C3=A9?= Date: Fri, 5 Jul 2019 12:10:04 +0200 Subject: [PATCH] Add Exception for the daily parser script --- .../daily_json_activity_parser.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/labonneboite/scripts/impact_retour_emploi/daily_json_activity_parser.py b/labonneboite/scripts/impact_retour_emploi/daily_json_activity_parser.py index 68e9832de..dc762796a 100644 --- a/labonneboite/scripts/impact_retour_emploi/daily_json_activity_parser.py +++ b/labonneboite/scripts/impact_retour_emploi/daily_json_activity_parser.py @@ -7,6 +7,10 @@ from labonneboite.importer import settings as importer_settings from labonneboite.importer.jobs.common import logger + +class NoDataException(Exception): + pass + def sql_queries(): create_table_query1 = 'CREATE TABLE IF NOT EXISTS `idpe_connect` ( \ `idutilisateur_peconnect` text, \ @@ -56,14 +60,21 @@ def get_dateheure(row): logger.info('.json files found : {}'.format(json_logs_paths)) + file_used = False + for json_logs_path in json_logs_paths: date = json_logs_path.replace('activity-lbb-','').replace('.json','').replace('.','-') if date >= date_last_recorded_activity: + file_used = True logger.info('.json file used : {}'.format(json_logs_path)) with open(json_logs_folder_path+'/'+json_logs_path, 'r') as json_file: for line in json_file: data.append(line) + if not file_used: + logger.info("Did not find/need any data to parse") + raise NoDataException + activities = {} i = 1 for activity in data: @@ -97,6 +108,9 @@ def insert_id_peconnect(activity_df): engine = import_util.create_sqlalchemy_engine() + nb_lines = activity_idpec.shape[0] + logger.info('Number of lines to insert into idpec : {}'.format(nb_lines)) + activity_idpec.to_sql( con=engine, name='idpe_connect', if_exists='append', index=False, chunksize=10000) @@ -124,6 +138,9 @@ def siret(row): engine = import_util.create_sqlalchemy_engine() + nb_lines = activity_logs_df.shape[0] + logger.info('Number of lines to insert into idpec : {}'.format(nb_lines)) + activity_logs_df.to_sql(con=engine, name='activity_logs', if_exists='append', index=False, chunksize=10000)