From 39959580a34a82d1475aa983270ad09ffc3dd2cb Mon Sep 17 00:00:00 2001 From: Vanya Belyaev Date: Thu, 8 Feb 2024 09:34:07 +0100 Subject: [PATCH] tiny fix --- ostap/fitting/pyselectors.py | 10 ++++- ostap/frames/tree_reduce.py | 29 +++++++++----- ostap/parallel/parallel_reduce.py | 20 ++++++++-- ostap/plotting/graph_summary.py | 13 +++++-- ostap/tools/chopping.py | 59 ++++++++++++++++++++-------- ostap/tools/tmva.py | 65 ++++++++++++++++++++----------- ostap/trees/data_utils.py | 5 ++- source/CMakeLists.txt | 2 + source/CMakeROOT_6_23.cmake | 34 ++++++++++++++++ source/CMake_OSTAP.cmake | 5 ++- source/src/BreitWigner.cpp | 2 +- 11 files changed, 184 insertions(+), 60 deletions(-) diff --git a/ostap/fitting/pyselectors.py b/ostap/fitting/pyselectors.py index f49b0b57..f5642a73 100644 --- a/ostap/fitting/pyselectors.py +++ b/ostap/fitting/pyselectors.py @@ -1849,8 +1849,14 @@ def make_dataset ( tree , total = len ( tree ) - if not silent : - pb = frame.ProgressBar ( total ) + if not silent : + if ( 6 , 29 ) <= root_info : + from ostap.frames.frames import frame_progress2 + pb = frame_progress2 ( frame ) + else : + from ostap.frames.frames import frame_progress + pb = frame_progress ( frame , total ) + columns = set ( tree.branches() ) | set ( tree.leaves() ) diff --git a/ostap/frames/tree_reduce.py b/ostap/frames/tree_reduce.py index 34ee8ece..1d41e64d 100644 --- a/ostap/frames/tree_reduce.py +++ b/ostap/frames/tree_reduce.py @@ -24,7 +24,8 @@ import ostap.trees.trees from ostap.core.meta_info import root_info from ostap.core.core import cpp, Ostap -from ostap.utils.cleanup import CleanUp +from ostap.utils.cleanup import CleanUp +from ostap.utils.basic import make_dirs import ROOT, os, sys # ============================================================================= # logging @@ -62,14 +63,19 @@ def __init__ ( self , tmp_keep = False , ## keep the temporary file silent = False ): ## silent processing - from ostap.frames.frames import DataFrame, frame_prescale + from ostap.frames.frames import DataFrame, frame_prescale frame = DataFrame ( chain ) report = None self.__frame_main = frame if not silent : - pbar = frame.ProgressBar ( len ( chain ) ) + if ( 6 , 29 ) <= root_info : + from ostap.frames.frames import frame_progress2 + cnt = frame_progress2 ( frame ) + else : + from ostap.frames.frames import frame_progress + cnt = frame_progress ( frame , len ( chain ) ) ## add overall prescale (if requested) if 1 != prescale : @@ -133,8 +139,8 @@ def __init__ ( self , all_vars = list ( bvars ) + [ v for v in nvars if not v in bvars ] save_vars = tuple ( [ v for v in all_vars if not v in no_vars ] ) - nb_ = len ( chain.branches () ) - ne_ = len ( chain ) + nb0 = len ( chain.branches () ) + ne0 = len ( chain ) if not silent and output and os.path.exists ( output ) and os.path.isfile ( output ) : logger.warning ("Existing file %s will be overwritten!" % output ) @@ -153,6 +159,10 @@ def __init__ ( self , output = self.tempfile ( prefix = 'ostap-frame-' , suffix = '.root' ) if not tmp_keep : self.trash.add ( output ) + dirname = os.path.dirname ( output ) + if dirname and not os.path.exists ( dirname ) : + make_dirs ( dirname ) + if not save_vars : snapshot = frame.Snapshot ( name , output ) else : @@ -192,10 +202,11 @@ def __init__ ( self , nb = len ( self.__chain.branches () ) ne = len ( self.__chain ) - - self.__report += '\n# Reduce %d -> %d branches, %d -> %d entries' % ( nb_ , nb , ne_ , ne ) - self.__report += '\n# Output:%s size:%s' % ( self.__output , fs ) - self.__report += '\n# %s' % str ( self.__chain ) + ff = float ( nb * ne * 100 ) / ( nb0 * ne0 ) + + self.__report += '\n# Reduce: (%dx%d) -> (%dx%d) branches x entries => %.1f%% ' % ( nb0 , ne0 , nb , ne , ff ) ) + self.__report += '\n# Output:%s size:%s' % ( self.__output , fs ) + ## self.__report += '\n# %s' % str ( self.__chain ) del self.__frame_main diff --git a/ostap/parallel/parallel_reduce.py b/ostap/parallel/parallel_reduce.py index 4625fee4..c1c712ab 100644 --- a/ostap/parallel/parallel_reduce.py +++ b/ostap/parallel/parallel_reduce.py @@ -17,7 +17,7 @@ ) # ============================================================================= from ostap.parallel.parallel import Task, WorkManager -import ROOT +import ROOT, os # ============================================================================= # logging # ============================================================================= @@ -176,8 +176,22 @@ def reduce ( chain , nb = len ( result.chain.branches() ) ne = len ( result.chain ) - f = float ( nb0 * ne0 ) / ( nb * ne ) - logger.info ( 'reduce: (%dx%d) -> (%dx%d) %.1f (branches x entries) ' % ( nb0 , ne0 , nb , ne , f ) ) + ff = float ( nb * ne * 100 ) / ( nb0 * ne0 ) + logger.info ( 'Reduce: (%dx%d) -> (%dx%d) branches x entries => %.1f%% ' % ( nb0 , ne0 , nb , ne , ff ) ) + + if output and os.path.exists ( output ) and os.path.isfile ( output ) : + fs = os.path.getsize ( output ) + gb , r = divmod ( fs , 1024 * 1024 * 1024 ) + mb , r = divmod ( r , 1024 * 1024 ) + kb , r = divmod ( r , 1024 ) + + if gb : fs = '%.1fGB' % ( float ( fs ) / 1024 / 1024 / 1024 ) + elif mb : fs = '%.1fMB' % ( float ( fs ) / 1024 / 1024 ) + elif kb : fs = '%.1fkB' % ( float ( fs ) / 1024 ) + else : fs = '%sB' % fs + + logger.info ( 'Output:%s size:%s' % ( output , fs ) + return result diff --git a/ostap/plotting/graph_summary.py b/ostap/plotting/graph_summary.py index 1247b0ae..4ad63ab5 100755 --- a/ostap/plotting/graph_summary.py +++ b/ostap/plotting/graph_summary.py @@ -149,16 +149,21 @@ def error_band2 ( value , epos , eneg , min_value , max_value , **kwargs ) : ## fill color fcolor = config.get ( 'fill_color' , ROOT.kOrange ) + + + delta = max_value - min_value + mnv = min_value + 0.001 * delta + mxv = max_value - 0.001 * delta from itertools import count for fc , ep , en in zip ( count ( fcolor , -1 ) , reversed ( epos ) , reversed ( eneg ) ) : if not transpose : - box1 = ROOT.TBox ( value - en , min_value , value + ep , max_value ) - box2 = ROOT.TBox ( value - en , min_value , value + ep , max_value ) + box1 = ROOT.TBox ( value - en , mnv , value + ep , mxv ) + box2 = ROOT.TBox ( value - en , mnv , value + ep , mxv ) else : - box1 = ROOT.TBox ( min_value , value - en , max_value , value + ep ) - box2 = ROOT.TBox ( min_value , value - en , max_value , value + ep ) + box1 = ROOT.TBox ( mnv , value - en , mxv , value + ep ) + box2 = ROOT.TBox ( mnv , value - en , mxv , value + ep ) box1.set_fill_attributes ( **config ) box2.set_fill_attributes ( **config ) diff --git a/ostap/tools/chopping.py b/ostap/tools/chopping.py index 581e1bf0..390dc969 100644 --- a/ostap/tools/chopping.py +++ b/ostap/tools/chopping.py @@ -326,11 +326,6 @@ def __init__ ( self , self.__signal_weight = signal_weight self.__signal_cuts = ROOT.TCut ( signal_cuts ) - self.__signal_vars = {} - if signal_vars : self.__signal_vars.update ( signal_vars ) - self.__background_vars = {} - if background_vars : self.__background_vars.update ( background_vars ) - self.__prefilter = ROOT.TCut ( prefilter ) self.__prefilter_signal = prefilter_signal if prefilter_signal else '' self.__prefilter_background = prefilter_background if prefilter_background else '' @@ -341,6 +336,9 @@ def __init__ ( self , variables = list ( variables ) + _sig_vars = {} + _bkg_vars = {} + vars = [] for v in variables : a , s1 , b = v.partition ( ':' ) ## a : b @@ -363,6 +361,18 @@ def __init__ ( self , variables = vars variables.sort () + + if signal_vars : _sig_vars.update ( signal_vars ) + if background_vars : _bkg_vars.update ( background_vars ) + + signal_vars = _sig_vars + background_vars = _bkg_vars + + self.__signal_vars = {} + if signal_vars : self.__signal_vars.update ( signal_vars ) + self.__background_vars = {} + if background_vars : self.__background_vars.update ( background_vars ) + self.__variables = tuple ( variables ) @@ -402,18 +412,28 @@ def __init__ ( self , ##if self.verbose : - all_vars = [ self.prefilter ] + all_vars = [] for v in self.variables : vv = v if isinstance ( vv , str ) : vv = ( vv , 'F' ) - if vv[0] in self.signal_vars : continue - elif vv[0] in self.background_vars : continue - else : all_vars.append ( vv[0] ) + + varexp = vv [ 0 ].strip() + nick , sep , expr = varexp.partition ( ":=" ) + nick = nick.strip() + expr = expr.strip() + if nick and sep and expr : varexp = expr + else : nick = varexp + + if varexp in self.signal_vars or ( nick and nick in self.signal_vars ) : continue + elif varexp in self.background_vars or ( nick and nick in self.background_vars ) : continue + else : all_vars.append ( varexp ) + for v in self.spectators : vv = v if isinstance ( vv , str ) : vv = ( vv , 'F' ) all_vars.append ( vv[0] ) + if self.prefilter : all_vars.append ( self.prefilter ) ## if self.signal_cuts : all_vars.append ( self.signal_cuts ) ## if self.signal_weight : all_vars.append ( self.signal_weight ) ## if self.background_cuts : all_vars.append ( self.background_cuts ) @@ -429,10 +449,13 @@ def __init__ ( self , import ostap.trees.trees avars = self.signal.the_variables ( all_vars ) + + keys = set ( self.background_vars.keys () ) - set ( self.signal_vars .keys () ) + avars = tuple ( sorted ( set ( avars ).union ( keys ) ) ) scuts = {} if self.prefilter_signal : scuts.update ( { 'PreFilter/Signal' : self.prefilter_signal } ) - if self.prefilter : scuts.update ( { 'PreFilter' : self.prefilter } ) + if self.prefilter : scuts.update ( { 'PreFilter/Common' : self.prefilter } ) if self.signal_cuts : scuts.update ( { 'Signal' : self.signal_cuts } ) if ( 6 , 24 ) <= root_info : @@ -449,7 +472,7 @@ def __init__ ( self , new_vars = self.signal_vars , prescale = self.prescale_signal , silent = False ) - + self.__signal = self.__SigTR self.__signal_cuts = ROOT.TCut() @@ -460,11 +483,14 @@ def __init__ ( self , if self.background_weight : all_vars.append ( self.background_weight ) import ostap.trees.trees - avars = self.background.the_variables ( all_vars ) + bvars = self.background.the_variables ( all_vars ) + keys = set ( self.signal_vars.keys () ) - set ( self.background_vars .keys () ) + bvars = tuple ( sorted ( set ( bvars ).union ( keys ) ) ) + bcuts = {} if self.prefilter_background : bcuts.update ( { 'PreFilter/Background' : self.prefilter_background } ) - if self.prefilter : bcuts.update ( { 'PreFilter' : self.prefilter } ) + if self.prefilter : bcuts.update ( { 'PreFilter/Common' : self.prefilter } ) if self.background_cuts : bcuts.update ( { 'Background' : self.background_cuts } ) if ( 6 , 24 ) <= root_info : @@ -477,10 +503,10 @@ def __init__ ( self , self.logger.info ( 'Pre-filter Background before processing' ) self.__BkgTR = TR.reduce ( self.background , selection = bcuts , - save_vars = avars , + save_vars = bvars , new_vars = self.background_vars , prescale = self.prescale_background , - silent = False ) + silent = False ) self.__background = self.__BkgTR self.__background_cuts = ROOT.TCut() @@ -1249,7 +1275,8 @@ def make_tarfile ( self , tarfiles , logfiles = [] ) : self.logger.verbose ( "Remove existing tar-logfile %s" % lfile ) with tarfile.open ( lfile , 'w:gz' ) as tar : - for x in logfiles: tar.add ( x ) + for x in logfiles: + if os.path.exists ( x ) : tar.add ( x ) self.logger.info ( "Tar/gz logfile : %s" % lfile ) ## if self.verbose : tar.list () diff --git a/ostap/tools/tmva.py b/ostap/tools/tmva.py index 354163dc..a0fffe4e 100755 --- a/ostap/tools/tmva.py +++ b/ostap/tools/tmva.py @@ -257,8 +257,8 @@ def __init__( self , signal , ## signal sample/tree background , ## background sample/tree ## - signal_vars = {} , ## dictionary with new variables for signal sample - background_vars = {} , ## dictionary with new variables for background sample + signal_vars = {} , ## dictionary with new variables for signal sample + background_vars = {} , ## dictionary with new variables for background sample ## signal_cuts = '' , # signal cuts background_cuts = '' , # background cuts @@ -319,7 +319,10 @@ def __init__( self , self.__methods = tuple ( methods ) - variables = list ( variables ) ; + variables = list ( variables ) + + _sig_vars = {} + _bkg_vars = {} vars = [] for v in variables : @@ -332,15 +335,21 @@ def __init__( self , d = d.strip() if c and s2 and d : vars.append ( a ) - signal_vars .update ( { a : c } ) ## ATTENTION HERE - background_vars.update ( { a : d } ) ## ATTENTION HERE + _sig_vars.update ( { a : c } ) ## ATTENTION HERE + _bkg_vars.update ( { a : d } ) ## ATTENTION HERE else : vars.append ( a ) - signal_vars .update ( { a : b } ) ## ATTENTION HERE - background_vars.update ( { a : b } ) ## ATTENTION HERE + _sig_vars.update ( { a : b } ) ## ATTENTION HERE + _bkg_vars.update ( { a : b } ) ## ATTENTION HERE else : vars.append ( v ) - + + if signal_vars : _sig_vars.update ( signal_vars ) + if background_vars : _bkg_vars.update ( background_vars ) + + signal_vars = _sig_vars + background_vars = _bkg_vars + variables = vars variables.sort () self.__variables = tuple ( variables ) @@ -562,11 +571,11 @@ def __init__( self , rows.append ( row ) if 0 < self.background_train_fraction < 1 : - row = 'Backgroundtrain fraction' , '%.1f%%' % ( 100 * self.background_train_fraction ) + row = 'Background train fraction' , '%.1f%%' % ( 100 * self.background_train_fraction ) rows.append ( row ) if 1 != self.prescale_background: - row = 'Backgronud prescale' , '%s' % self.prescale_background + row = 'Background prescale' , '%s' % self.prescale_background rows.append ( row ) ms = [ m[1] for m in self.methods ] @@ -976,13 +985,23 @@ def __train ( self ) : # # ================================================================= - all_vars = [] - for v in itertools.chain ( self.variables , self.spectators ) : + all_vars = [] + + for v in self.variables : + vv = v if isinstance ( vv , str ) : vv = ( vv , 'F' ) - if vv[0] in self.signal_vars : continue - elif vv[0] in self.background_vars : continue - else : all_vars.append ( vv[0] ) + + varexp = vv [ 0 ].strip() + nick , sep , expr = varexp.partition ( ":=" ) + nick = nick.strip() + expr = expr.strip() + if nick and sep and expr : varexp = expr + else : nick = varexp + + if varexp in self.signal_vars or ( nick and nick in self.signal_vars ) : continue + elif varexp in self.background_vars or ( nick and nick in self.background_vars ) : continue + else : all_vars.append ( varexp ) for v in self.spectators : vv = v @@ -996,7 +1015,7 @@ def __train ( self ) : ## if self.background_weight : all_vars.append ( self.background_weight ) # ================================================================= - ## prefilter signal if required + ## prefilter/prescale signal if required if self.prefilter_signal or self.prefilter or 1 != self.prescale_signal or self.signal_vars : if self.signal_weight : all_vars.append ( self.signal_weight ) @@ -1005,9 +1024,9 @@ def __train ( self ) : avars = self.signal.the_variables ( all_vars ) scuts = {} - if self.prefilter : scuts.update ( { 'PreFilterSignal' : self.prefilter_signal } ) - if self.prefilter_signal : scuts.update ( { 'PreFilterCommon' : self.prefilter } ) - if self.signal_cuts : scuts.update ( { 'Signal' : self.signal_cuts } ) + if self.prefilter_signal : scuts.update ( { 'PreFilter/Signal' : self.prefilter_signal } ) + if self.prefilter : scuts.update ( { 'PreFilter/Common' : self.prefilter } ) + if self.signal_cuts : scuts.update ( { 'Signal' : self.signal_cuts } ) if ( 6 , 24 ) <= root_info : import ostap.frames.frames @@ -1028,7 +1047,7 @@ def __train ( self ) : self.__signal_cuts = ROOT.TCut() # ================================================================= - ## prefilter background if required + ## prefilter/prescale background if required if self.prefilter_background or self.prefilter or 1 != self.prescale_background or self.background_vars : if self.background_weight : all_vars.append ( self.background_weight ) @@ -1037,9 +1056,9 @@ def __train ( self ) : avars = self.background.the_variables ( all_vars ) bcuts = {} - if self.prefilter_background : bcuts.update ( { 'PreFilterBackground' : self.prefilter_background } ) - if self.prefilter : bcuts.update ( { 'PreFilterCommon' : self.prefilter } ) - if self.background_cuts : bcuts.update ( { 'Signal' : self.background_cuts } ) + if self.prefilter_background : bcuts.update ( { 'PreFilter/Background' : self.prefilter_background } ) + if self.prefilter : bcuts.update ( { 'PreFilter/Common' : self.prefilter } ) + if self.background_cuts : bcuts.update ( { 'Background' : self.background_cuts } ) if ( 6 , 24 ) <= root_info : import ostap.frames.frames diff --git a/ostap/trees/data_utils.py b/ostap/trees/data_utils.py index e688035b..69fcd7ab 100755 --- a/ostap/trees/data_utils.py +++ b/ostap/trees/data_utils.py @@ -594,7 +594,10 @@ def commonpath ( self ) : files = [] for f in self.__files : files .append ( strip_protocol ( f ) ) - else : files = self.__files + else : files = self.__files + + if not files : return '' + cp = commonpath ( files ) return cp if os.path.isdir ( cp ) else os.path.dirname ( cp ) # ========================================================================= diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 40462e71..1fb6b85e 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -21,3 +21,5 @@ else() message ( "----> Use OSTAP/CMAKE setting for ROOT 6.14") include(CMakeROOT_6_14.cmake) endif() + + diff --git a/source/CMakeROOT_6_23.cmake b/source/CMakeROOT_6_23.cmake index 39b4933d..0da70ff6 100644 --- a/source/CMakeROOT_6_23.cmake +++ b/source/CMakeROOT_6_23.cmake @@ -139,6 +139,8 @@ include(CMake_OSTAP.cmake) ## NO_SYSTEM_FROM_IMPORTED ON ## ) +if(ROOT_VERSION VERSION_LESS_EQUAL "6.31.00") + execute_process( COMMAND "${ROOT_CONFIG_EXECUTABLE}" --has-cxx17 OUTPUT_VARIABLE CXX17_ROOT OUTPUT_STRIP_TRAILING_WHITESPACE ) @@ -157,6 +159,38 @@ elseif ( ${CXX11_ROOT} STREQUAL "yes" ) target_compile_features (ostap PUBLIC cxx_std_11 ) endif() +else() + +execute_process( COMMAND "${ROOT_CONFIG_EXECUTABLE}" --cxxstandard + OUTPUT_VARIABLE CXX_ROOT + OUTPUT_STRIP_TRAILING_WHITESPACE ) + +set_property(TARGET ostap PROPERTY CXX_STANDARD ${CXX_ROOT} ) + +if ( ${CXX_ROOT} STREQUAL "26" ) +target_compile_features (ostap PUBLIC cxx_std_26 ) + message ( '26' ) +elseif ( ${CXX_ROOT} STREQUAL "23" ) +target_compile_features (ostap PUBLIC cxx_std_23 ) + message ( '23' ) +elseif ( ${CXX_ROOT} STREQUAL "20" ) +target_compile_features (ostap PUBLIC cxx_std_20 ) + message ( '20' ) +elseif ( ${CXX_ROOT} STREQUAL "17" ) +## target_compile_features (ostap PUBLIC cxx_std_17 ) +## message ( '17' ) +else() +##target_compile_features (ostap PUBLIC cxx_std_17 ) +## message ( '17/0' ) +endif() + +endif() + + + + + + target_link_libraries ( ostap ROOT::MathMore ROOT::ROOTVecOps ROOT::GenVector root_pyroot ROOT::RooFit ROOT::Hist ROOT::Tree ROOT::TreePlayer ROOT::RIO ROOT::TMVA ROOT::ROOTDataFrame GSL::gsl ) target_include_directories (ostap diff --git a/source/CMake_OSTAP.cmake b/source/CMake_OSTAP.cmake index 5c9e5f41..97f0dd82 100644 --- a/source/CMake_OSTAP.cmake +++ b/source/CMake_OSTAP.cmake @@ -118,7 +118,6 @@ target_compile_features ( ostap PUBLIC cxx_variadic_templates ) target_compile_features ( ostap PUBLIC cxx_delegating_constructors ) target_compile_features ( ostap PUBLIC cxx_defaulted_move_initializers ) target_compile_features ( ostap PUBLIC cxx_decltype ) -target_compile_features ( ostap PUBLIC cxx_decltype_auto ) target_compile_features ( ostap PUBLIC cxx_deleted_functions ) target_compile_features ( ostap PUBLIC cxx_final ) target_compile_features ( ostap PUBLIC cxx_lambdas ) @@ -129,4 +128,8 @@ target_compile_features ( ostap PUBLIC cxx_static_assert ) target_compile_features ( ostap PUBLIC cxx_right_angle_brackets ) target_compile_features ( ostap PUBLIC cxx_nullptr ) target_compile_features ( ostap PUBLIC cxx_auto_type ) + +if(ROOT_VERSION VERSION_LESS_EQUAL "6.31.00") +target_compile_features ( ostap PUBLIC cxx_decltype_auto ) target_compile_features ( ostap PUBLIC cxx_aggregate_default_initializers ) +endif() \ No newline at end of file diff --git a/source/src/BreitWigner.cpp b/source/src/BreitWigner.cpp index fb5c8115..22e5204d 100644 --- a/source/src/BreitWigner.cpp +++ b/source/src/BreitWigner.cpp @@ -366,7 +366,7 @@ double Ostap::Math::FormFactors::BlattWeisskopf::operator() std::size_t Ostap::Math::FormFactors::BlattWeisskopf::tag () const { static const std::string s_name = "BlattWeisskopf"; - return Ostap::Utils::hash_combiner ( s_name , m_what , m_L , m_b ) ; + return Ostap::Utils::hash_combiner ( s_name , m_what , (int) m_L , m_b ) ; } // ============================================================================ // clone operation