From 9d77aa141822570c296de1f937ff8b0d6a630da0 Mon Sep 17 00:00:00 2001 From: Vanya Belyaev Date: Thu, 15 Feb 2024 09:28:17 +0100 Subject: [PATCH] fix? --- ostap/tools/chopping.py | 48 ++++++++++++++++++++++++++++++++--------- ostap/tools/tmva.py | 40 ++++++++++++++++++++++++++++------ 2 files changed, 71 insertions(+), 17 deletions(-) diff --git a/ostap/tools/chopping.py b/ostap/tools/chopping.py index 390dc969..332bb78a 100644 --- a/ostap/tools/chopping.py +++ b/ostap/tools/chopping.py @@ -95,6 +95,7 @@ from ostap.core.core import WSE from ostap.core.pyrouts import hID, h1_axis, Ostap from ostap.core.ostap_types import integer_types +from ostap.utils.cleanup import CleanUp import ostap.trees.trees import ostap.trees.cuts import ostap.utils.utils as Utils @@ -1258,39 +1259,66 @@ def make_tarfile ( self , tarfiles , logfiles = [] ) : tfile = '%s.tgz' % self.name if os.path.exists ( tfile ) : self.logger.verbose ( "Remove existing tar-file %s" % tfile ) + try : + os.remove ( tfile ) + except : + pass + + # create temporary tar-file + tmptar = CleanUp.tempfile ( prefix = 'ostap-tmp-tarfile-' , suffix = '.tgz' ) - with tarfile.open ( tfile , 'w:gz' ) as tar : + with tarfile.open ( tmptar , 'w:gz' ) as tar : for x in tarfiles: tar.add ( x ) - self.logger.info ( "Tar/gz file : %s" % tfile ) - ## if self.verbose : tar.list () + self.logger.debug ( "Tar/gz file : %s" % tmptar ) + if self.verbose : tar.list () + + assert os.path.exists ( tmptar ) and os.path.isfile ( tmptar ) and tarfile.is_tarfile ( tmptar ) , \ + 'Non-existing or invalid temporary tar-file!' + ## copy it + shutil.copy ( tmptar , tfile ) + + assert os.path.exists ( tfile ) and os.path.isfile ( tfile ) and tarfile.is_tarfile ( tfile ) , \ + 'Non-existing or invalid temporary tar-file!' + ## finally set the tar-file if os.path.exists ( tfile ) and tarfile.is_tarfile( tfile ) : self.__tar_file = tfile ; ## os.path.abspath ( tfile ) - + self.logger.debug ( "Tar/gz file : %s" % tfile ) + if not logfiles : return tfile lfile = '%s_logs.tgz' % self.name if os.path.exists ( lfile ) : self.logger.verbose ( "Remove existing tar-logfile %s" % lfile ) - - with tarfile.open ( lfile , 'w:gz' ) as tar : + try : + os.remove ( lfile ) + except : + pass + + # create temporary tar-file + tmptar = CleanUp.tempfile ( prefix = 'ostap-tmp-logfile-' , suffix = '.tgz' ) + with tarfile.open ( tmptar , 'w:gz' ) as tar : for x in logfiles: if os.path.exists ( x ) : tar.add ( x ) - self.logger.info ( "Tar/gz logfile : %s" % lfile ) - ## if self.verbose : tar.list () - + self.logger.debug ( "Tar/gz logfile : %s" % tmptar ) + if self.verbose : tar.list () + + if os.path.exists ( tmptar ) and os.path.isfile ( tmptar ) and tarfile.is_tarfile ( tmptar ) : + shutil.copy ( tmptar , lfile ) + ## finally set the tar/log-file if os.path.exists ( lfile ) and tarfile.is_tarfile( lfile ) : self.__log_file = lfile ## os.path.abspath ( lfile ) + self.logger.debug ( "Tar/gz logfile : %s" % lfile ) + return tfile # ============================================================================= ## @class WeightFiles # helper structure to deal with weights files -from ostap.utils.cleanup import CleanUp class WeightsFiles(CleanUp) : """Helper structure to deal with weights files """ diff --git a/ostap/tools/tmva.py b/ostap/tools/tmva.py index a0fffe4e..736541fe 100755 --- a/ostap/tools/tmva.py +++ b/ostap/tools/tmva.py @@ -37,6 +37,7 @@ from ostap.core.core import items_loop, WSE, Ostap, rootWarning from ostap.core.ostap_types import num_types, string_types, integer_types from ostap.core.meta_info import root_version_int, root_info +from ostap.utils.cleanup import CleanUp import ostap.io.root_file import ROOT, os, glob, math, tarfile, shutil, itertools # ============================================================================= @@ -75,7 +76,6 @@ def dir_name ( name ) : # ============================================================================= ## @class WeightFiles # helper structure to deal with weights files -from ostap.utils.cleanup import CleanUp class WeightsFiles(CleanUp) : """Helper structure to deal with weights files """ @@ -1375,14 +1375,27 @@ def __train ( self ) : tfile = self.name + '.tgz' if os.path.exists ( tfile ) : self.logger.debug ( "Remove existing tar-file %s" % tfile ) - - with tarfile.open ( tfile , 'w:gz' ) as tar : + try : + os.remove ( tfile ) + except : + pass + + # create temporary tar-file + tmptar = CleanUp.tempfile ( prefix = 'ostap-tmp-tarfile-' , suffix = '.tgz' ) + with tarfile.open ( tmptar , 'w:gz' ) as tar : for x in self.weights_files : tar.add ( x ) for x in self. class_files : tar.add ( x ) for x in self. plots : tar.add ( x ) if self.log_file and os.path.exists ( self.log_file ) and os.path.isfile ( self.log_file ) : - tar.add ( self.log_file ) + tar.add ( self.log_file ) + + assert os.path.exists ( tmptar ) and os.path.isfile ( tmptar ) and tarfile.is_tarfile ( tmptar ) , \ + 'Non-existing or invalid temporary tar-file!' + ## copy it + import shutil + shutil.copy ( tmptar , tfile ) + self.__weights_files = tuple ( [ os.path.abspath ( f ) for f in self.weights_files ] ) self.__class_files = tuple ( [ os.path.abspath ( f ) for f in self.class_files ] ) self.__plots = tuple ( [ os.path.abspath ( f ) for f in self.__plots ] ) @@ -1571,11 +1584,24 @@ def make_Plots ( name , output , show_plots = True ) : if plots : ## tarfile with plots - tfile = '%s_plots.tgz' % name - with tarfile.open ( tfile , 'w:gz' ) as tar : + tfile = '%s_plots.tgz' % name + + if os.path.exists ( tfile ) : + self.logger.verbose ( "Remove existing tar-plotsfile %s" % lfile ) + try : + os.remove ( tfile ) + except : + pass + + # create temporary tar-file + tmptar = CleanUp.tempfile ( prefix = 'ostap-tmp-plots-' , suffix = '.tgz' ) + + with tarfile.open ( tmptar , 'w:gz' ) as tar : for x in plots : tar.add ( x ) - tfile = os.path.abspath ( tar.name ) + ## copy it + shutil.copy ( tmptar , tfile ) + if tfile and os.path.exists ( tfile ) and tarfile.is_tarfile ( tfile ) : with tarfile.open ( tfile , 'r' ) as tar : logger.info ( "Tarfile with plots: '%s'" % tfile )