From 89954871b0a4d7bec03e40f39a121f1bc0304c3b Mon Sep 17 00:00:00 2001 From: Vanya Belyaev Date: Thu, 25 Jul 2024 15:35:33 +0200 Subject: [PATCH] fix? --- ostap/frames/frames.py | 2110 +++++++++++++--------- ostap/frames/tests/test_frames_frames.py | 7 +- ostap/io/dbase.py | 1 - source/include/Ostap/DataFrameActions.h | 210 ++- source/include/Ostap/DataFrameUtils.h | 7 + source/src/DataFrameUtils.cpp | 31 +- source/src/StatVar.cpp | 26 +- 7 files changed, 1424 insertions(+), 968 deletions(-) diff --git a/ostap/frames/frames.py b/ostap/frames/frames.py index 8a5f71f8..d109f7b2 100644 --- a/ostap/frames/frames.py +++ b/ostap/frames/frames.py @@ -12,31 +12,37 @@ __author__ = "Vanya BELYAEV Ivan.Belyaev@itep.ru" __date__ = "2011-06-07" __all__ = ( - 'DataFrame' , ## RDataFrame object - 'report_print' , ## print the report - 'report_print_table' , ## print the report - 'report_as_table' , ## print the report - 'frame_progress' , ## progress bar for frame - 'frame_prescale' , ## prescale frame (trivial filter) + 'DataFrame' , ## RDataFrame object + ## + 'frame_columns' , ## defined columns + 'frame_branches' , ## defined columns + ## + 'frame_progress1' , ## progress bar for frame (OK for ROOT<6.32) + 'frame_progress2' , ## progress bar for frame (OK for ROOT>6.29) + 'frame_progress' , ## progress bar for frame + 'frame_prescale' , ## prescale frame (trivial filter) + 'frame_project' , ## project data frame to the (1D/2D/3D) histogram + 'frame_draw' , ## draw variable from the frame + 'frame_print' , ## over-simplified print frame + 'frame_table' , ## Print frame as detailed table + ## + 'frame_length' , ## length/size of the frame + 'frame_size' , ## length/size of the frame ## - 'frame_project' , ## project data frame to the (1D/2D/3D) histogram - 'frame_print' , ## print frame - 'frame_draw' , ## draw variable from the frame - 'frame_columns' , ## defined columns + 'frame_nEff' , ## nEff function for frames + 'frame_statVar' , ## stat var for frame + 'frame_statCov' , ## stat cov for frame ## - 'frame_nEff' , ## nEff function for frames - 'frame_statVar' , ## stat var for frame - 'frame_statCov' , ## sta tcov for frame + 'frame_get_moment' , ## get the moment or certain order around defined center + 'frame_moment' , ## get the moment or certain order + 'frame_central_moment' , ## get the central moment or certain order ## 'frame_mean' , ## mean value for the variable 'frame_rms' , ## RMS for the variable 'frame_variance' , ## variance for the variable 'frame_dispersion' , ## dispersion for the variable 'frame_skewness' , ## skewness for the variable - 'frame_kurtosis' , ## kurtosos for the variable - 'frame_get_moment' , ## get the moment - 'frame_moment' , ## moment relative to zero - 'frame_central_moment' , ## central moment + 'frame_kurtosis' , ## kurtosos for the variable ## 'frame_quantile' , ## quantile 'frame_interval' , ## quantile interval @@ -46,7 +52,12 @@ 'frame_quartiles' , ## quartiles 'frame_quintiles' , ## quintiles 'frame_deciles' , ## deciles - ## + ## + 'report_print' , ## print the report + 'report_print_table' , ## print the report + 'report_as_table' , ## print the report + ## + 'frame_types' , ## the basic DataFrame/FeameNode types ) # ============================================================================= from ostap.core.core import ( cpp , Ostap , @@ -76,47 +87,28 @@ ## @see Ostap/DataFrame.h DataFrame = Ostap.DataFrame ## @see Ostap/DataFrame.h -FrameNode = Ostap.FrameNode - +FrameNode = Ostap.FrameNode +# ============================================================================== +if FrameNode is DataFrame : frame_types = DataFrame , +else : frame_types = DataFrame , FrameNode # =============================================================================== +if not hasattr ( Ostap ,'DataFrame' ) : Ostap.DataFrame = DataFrame +if not hasattr ( Ostap ,'FrameNode' ) : Ostap.FrameNode = FrameNode +# ============================================================================= Frames_OK = False std_move = None has_std_move = False try : std_move = ROOT.std.move has_std_move = True - Frames_OK = has_std_move and (6,25)<= root_info ## ATTENTION! + Frames_OK = has_std_move and ( 6 , 25 )<= root_info ## ATTENTION! except AttributeError : std_move = None has_std_move = False Frames_OK = False - - -## try : -## DataFrame = ROOT.ROOT.RDataFrame -## except AttributeError :_new -## DataFrame = ROOT.ROOT.Experimental.TDataFrame -## # ============================================================================= -## try : -## FrameNode = ROOT.ROOT.RDF.RNode -## except AttributeError : -## FrameNode = DataFrame - # ============================================================================= -if not hasattr ( Ostap ,'DataFrame' ) : Ostap.DataFrame = DataFrame -if not hasattr ( Ostap ,'FrameNode' ) : Ostap.FrameNode = FrameNode - -has_std_move = False -try : - std_move = ROOT.std.move - has_std_move = True and (6,25)<= root_info ## ATTENTION! -except AttributeError : - has_std_move = False - - -## type for column names +## type for the column names CNT = DataFrame.ColumnNames_t - # ============================================================================= if ( 6 , 18 ) <= root_info : # ========================================================================= @@ -129,29 +121,12 @@ def as_rnode ( frame ) : ## get frame-like stuff as rnode def as_rnode ( frame ) : """get frame-like stuff as rnode""" - return FrameNode ( frame ) -# ============================================================================= - -# ============================================================================= -## get all column names -# @code -# cols = colums ( frame ) -# @endcode -def columns ( frame ) : - """Get all column names - >>> cols = colums ( frame ) - """ - names = [ str(c) for c in frame.GetColumnNames() ] - if ( 6 , 16 ) <= root_info : - names += [ str ( c ) for c in frame.GetDefinedColumnNames() ] - return tuple ( sorted ( set ( names ) ) ) - -frame_columns = columns + return FrameNode ( frame ) # ============================================================================== -## generate new, unique name for the variable +## Helper method to generate new, unique name for the variable def var_name ( prefix , used_names , *garbage ) : - """Generate new, unique name for the variable + """Helper method to generate new, unique name for the variable """ name = prefix + '%x' % ( hash ( ( prefix , used_names , garbage ) ) % ( 2**32 ) ) while name in used_names : @@ -160,9 +135,84 @@ def var_name ( prefix , used_names , *garbage ) : # ============================================================================== ## Is implicit MC globally enabled? -mt_global = OCC.general.getboolean ( 'ImplicitMT' , fallback = False ) -mt_global = True +mt_global = OCC.general.getboolean ( 'ImplicitMT' , fallback = True ) + +# ============================================================================== +## Helper function that defines expressions and cuts +# @code +# frame = ... +# currnt , vexpr , cexpr, input_string = _fr_helper_ ( frame , 'x*x' , 'z<0' ) +# #endcode +def _fr_helper_ ( frame , expressions , cuts = '' ) : + """Helper function that defin4es expressions and cuts + >>> frame = ... + >>> current, vexpr , cexpr = _fr_helper_ ( frame , 'x*x' , 'z<0' ) + """ + + exprs, cuts, input_string = SV.vars_and_cuts ( expressions , cuts ) + + ## Frame/Tree ? + if isinstance ( frame , ROOT.TTree ) : node = DataFrame ( frame ) + elif isinstance ( frame , frame_types ) : node = frame + else : node = as_rnode ( frame ) + + ## get the list of currently known names + vars = frame_columns ( node ) + all_vars = set ( vars ) + + current = node + vnames = ordered_dict() + for i , expr in enumerate ( exprs , start = 1 ) : + vname = expr + if not expr in all_vars : + used = tuple ( all_vars | set ( frame_columns ( current ) ) ) + vn = var_name ( 'var%d_' % i , used , expr , *vars ) + all_vars.add ( vn ) + current = current.Define ( vn , expr ) + vname = vn + vnames [ expr ] = vname + + ## Filter frame, if needed! + ## if cuts : + ## ## add named filter + ## current = current.Filter ( '(bool) (%s)' % cuts, 'FILTER: %s' % cuts ) + + cname = cuts + if cuts and not cuts in all_vars : + used = tuple ( all_vars | set ( frame_columns ( current ) ) ) + cn = var_name ( 'cut_' , used , cuts , *vars ) + all_vars.add ( cn ) + ncuts = '1.0*(%s)' % cuts + current = current.Define ( cn , ncuts ) + cname = cn + + return current, vnames, cname, input_string + +# ================================================================================== +## The second helper method to implement various "statistics"-related actions +def _fr_helper2_ ( frame , creator , expressions , cuts = '' , lazy = True ) : + """The seocnd helper method to implement various statitic-related actions + """ + + current, var_names, cut_name, input_string = _fr_helper_ ( frame , expressions , cuts ) + + results = {} + for expr, var_name in loop_items ( var_names ) : + results [ expr ] = creator ( current , var_name , cut_name ) + + lasy = False + if not lazy : + for expr, res in loop_items ( results ) : + results [ expr ] = res.GetValue() + rr = results [ expr ] + + if input_string and 1 == len ( results ) : + _ , r = results.popitem() + return r + return results + + # ============================================================================== ## modify constructor for RDataFrame to enable/disable Implicit multithreading # @code @@ -196,381 +246,212 @@ def _fr_new_init_ ( self , name , *args , **kwargs ) : if not hasattr ( DataFrame , '_fr_old_init_' ) : DataFrame._fr_old_init_ = DataFrame.__init__ DataFrame.__init__ = _fr_new_init_ + +# ============================================================================= +if ( 6 , 16 ) <= root_info : + # ========================================================================= + ## get all column names + # @code + # cols = frame_colums ( frame ) + # cols = frame_branches ( frame ) ## ditto + # @endcode + def frame_columns ( frame ) : + """Get all column names + >>> cols = frame_colums ( frame ) + >>> cols = frame_branches ( frame ) ## ditto + """ + names = [ str ( c ) for c in frame.GetColumnNames() ] + names += [ str ( c ) for c in frame.GetDefinedColumnNames() ] + return tuple ( sorted ( set ( names ) ) ) + # ============================================================================= +else : + # ============================================================================= + ## get all column names + # @code + # cols = frame_colums ( frame ) + # cols = frame_branches ( frame ) ## ditto + # @endcode + def frame_columns ( frame ) : + """Get all column names + >>> cols = frame_colums ( frame ) + >>> cols = frame_branches ( frame ) + """ + names = [ str ( c ) for c in frame.GetColumnNames() ] + return tuple ( sorted ( set ( names ) ) ) +# ============================================================================== +## Frame branches, same as frame columns +# @see frame_columns +frame_branches = frame_columns # ============================================================================= -## Get the length/size of the data frame -# @code -# frame = ... -# print len(frame) -# @endcode -def _fr_len_ ( frame ) : - """Get the length/size of the data frame - >>> frame = ... - >>> print len(frame) - """ - node = as_rnode ( frame ) - return node.Count().GetValue() + # ============================================================================= -## Draw (lazy) progress bar for the DataFrame: +## Display the progress bar for the DataFrame: # @code # f = DataFrame ( ... ) -# p = frame_progress ( f , 1000 ) ## number of elements! -# p = f.ProgressBar ( 1000 ) ## number of elements! - +# p = frame_progress1 ( f , 1000 ) ## number of elements! # .... # @endcode -def frame_progress ( frame , - length ) : +# @attention active only for ROOT < 6.32 +def frame_progress1 ( frame , length ) : """ Draw (lazy) progress bar for the DataFrame: >>> f = DataFrame ( ... ) - >>> p = f.ProgressBar ( 1000 ) ## number of elements! + >>> p = frame_progress1 ( f , 1000 ) ## number of elements! >>> .... + - active only for ROOT < 6.32 """ - cnt = frame.Count () - if not isatty () : return cnt - - if not length : length = len ( frame ) + if isinstance ( frame , frame_types ) : pass + elif isinstance ( frame , ROOT.TTree ) : + length = len ( frame ) + frame = DataFrame ( frame ) + else : + frame = as_rnode ( frame ) + cnt = frame.Count () + + ## commented out + if not isatty () : return frame , cnt ## ATTENTION + elif ( 6 , 32 ) <= root_info : return frame , cnt ## ATTENTION + if 2048 < length : nchunks = 2000 elif 1024 < length : nchunks = 1000 elif 512 < length : nchunks = 500 elif 101 < length : nchunks = 100 - else : return cnt ## no progress bar for short frames + else : return frame, cnt ## no progress bar for short frames csize , rr = divmod ( length , nchunks ) csize = max ( csize , 1 ) - ## if rr : nchunks += 1 - - ## commented out - ## fun = Ostap.Utils.frame_progress ( nchunks , progress_conf () ) - ## cnt.OnPartialResultSlot ( csize , fun ) + if rr : nchunks += 1 + + fun = Ostap.Utils.frame_progress ( nchunks , progress_conf () ) + cnt.OnPartialResultSlot ( csize , fun ) + + return frame , cnt + +# ========================================================================= +## make use of new `ROOT::RDF::Experimental::AddProgressbar` utility +# @see ROOT::RDF::Experimental::AddProgressbar +# @attention no action for ROOT < 6.30 +def frame_progress2 ( frame , length = -1 ) : + """Make use of new `ROOT.RDF.Experimental.AddProgressbar` utility + - see ROOT.RDF.Experimental.AddProgressbar + - no action for ROOT < 6.30 + """ + + if isinstance ( frame , frame_types ) : pass + elif isinstance ( frame , ROOT.TTree ) : + length = len ( frame ) + frame = DataFrame ( frame ) + else : + frame = as_rnode ( frame ) + + cnt = frame.Count () - return cnt + if not isatty () :return frame , cnt ## ATTENTION + elif root_info < ( 6, 30 ) : return frame , cnt ## ATTENTION! + ## add progress bar + ROOT.ROOT.RDF.Experimental.AddProgressBar ( frame ) + return frame, cnt -if ( 6 , 29 ) <= root_info : +# ========================================================================= +if ( 6,30 ) < root_info : frame_progress = frame_progress2 +else : frame_progress = frame_progress1 + +# ============================================================================== +## Prescale the frame +# @code +# frame = ... +# prescaled1 = frame_prescale ( frame , 0.15 ) ## 0.0 < prescale < 1.0 +# prescaled2 = frame_prescale ( frame , 20 ) ## 1 < prescale +# @endcode +def frame_prescale ( frame , prescale , name = '' ) : + """Prescale the frame + >>> frame = ... + >>> prescaled1 = frame_prescale ( frame , 0.15 ) ## 0.0 < prescale < 1.0 + >>> prescaled2 = frame_prescale ( frame , 20 ) ## 1 < prescale + """ + node = as_rnode ( frame ) + + if isinstance ( prescale , integer_types ) and 1 < prescale : - # ========================================================================= - ## make use of new `ROOT::RDF::Experimental::AddProgressbar` utility - # @see ROOT::RDF::Experimental::AddProgressbar - def frame_progress2 ( frame , length = -1 ) : - """Make use of new `ROOT.RDF.Experimental.AddProgressbar` utility - - see ROOT.RDF.Experimental.AddProgressbar - """ + name = name if name else 'PRESCALE_%d' % prescale + code = '( 0 == ( ( rdfentry_ + %d * rdfslot_ ) %% %d ) )' \ + if ( 6 , 16 ) < root_info else \ + '( 0 == ( ( tdfentry_ + %d * tdfslot_ ) %% %d ) ) ' + ## 16777213 and 16777199 are just large prime numbers + code = code % ( 16777213 , prescale ) + return node.Filter ( code , name ) - if isinstance ( frame , DataFrame ) : pass - elif isinstance ( frame , FrameNode ) : pass - else : frame = as_rnode ( frame ) + elif isinstance ( prescale , float ) and 0 < prescale < 1 : - ROOT.ROOT.RDF.Experimental.AddProgressBar ( frame ) - return frame + name = name if name else 'PRESCALE_%.6g' % prescale + code = 'gRandom->Rndm() <= %.12g' % prescale + return node.Filter ( code , name ) - __all__ = __all__ + ( 'frame_progress2', ) + elif 1 == prescale : + ## no action + return node -# ============================================================================= -## Get the effective entries in data frame -# @code -# data = ... -# neff = data.nEff('b1*b1') -# @endcode -def frame_nEff ( frame , cuts = '' ) : - """Get the effective entries in data frame - >>> data = ... - >>> neff = data.nEff('b1*b1') - """ - node = as_rnode ( frame ) - return Ostap.StatVar.nEff ( node , cuts ) + raise TypeError ( "Invalid type/value for 'prescale' %s/%s" %( prescale , type ( prescale ) ) ) -# ============================================================================= -## Get statistics for the given expression in data frame -# @code -# data = ... -# c1 = data.statVar( 'S_sw' , 'pt>10' ) -# c2 = data.statVar( 'S_sw' , 'pt>0' ) -# @endcode -def frame_statVar ( frame , expression , cuts = '' ) : - """Get statistics for the given expression in data frame - >>> data = ... - >>> c1 = data.statVar( 'S_sw' , 'pt>10' ) - >>> c2 = data.statVar( 'S_sw' ) +# ============================================================================== +from ostap.fitting.dataset import ds_draw as _ds_draw +# ============================================================================== +## Draw the variables for the frame +def frame_draw ( frame , *args , **kwargs ) : + """Draw the varibale (s) for the frames """ - if isinstance ( frame , ROOT.TTree ) : frame = DataFrame ( frame ) - node = as_rnode ( frame ) - return Ostap.StatVar.statVar ( node , str ( expression ) , str ( cuts ) ) + node = as_rnode ( frame ) + return _ds_draw ( node , *args , **kwargs ) # ============================================================================= -## get the statistic for pair of expressions in DataFrame -# @code -# frame = ... -# stat1 , stat2 , cov2 , len = frame.statCov( 'x' , 'y' ) -# # apply some cuts -# stat1 , stat2 , cov2 , len = frame.statCov( 'x' , 'y' , 'z>0' ) -# @endcode -# @author Vanya BELYAEV Ivan.Belyaev@itep.ru -# @date 2018-06-18 -def frame_statCov ( frame , - expression1 , - expression2 , - cuts = '' ) : - """Get the statistic for pair of expressions in DataFrame - - >>> frame = ... - >>> stat1 , stat2 , cov2 , len = framw.statCov( 'x' , 'y' ) - - Apply some cuts: - >>> stat1 , stat2 , cov2 , len = frame.statCov( 'x' , 'y' , 'z>0' ) +## Simplified print out for the frame +# @code +# frame = ... +# @endcode +def frame_print ( frame ) : + """Simplified print out for the frame + >>> frame = ... + >>> print frame """ - import ostap.math.linalg - stat1 = Ostap.WStatEntity () - stat2 = Ostap.WStatEntity () - cov2 = Ostap.Math.SymMatrix(2) () - - if isinstance ( frame , ROOT.TTree ) : frame = DataFrame ( frame ) - node = as_rnode ( frame ) - length = Ostap.StatVar.statCov ( node , - expression1 , - expression2 , - cuts , - stat1 , - stat2 , - cov2 ) - - return stat1 , stat2 , cov2 , length + ## + if isinstance ( frame , ROOT.TTree ) : frame = DataFrame ( frame ) + ## + node = as_rnode ( frame ) + res = "DataFrame Enries/#%d" % len ( frame ) + ## + cols = frame_columns ( node ) + res += "\nColumns:\n%s" % multicolumn ( cols , indent = 2 , pad = 1 ) + return res + + + # ============================================================================== -## possible types of expressions -expression_types = string_types + ( ROOT.TCut , ) -# ============================================================================== -## helper functionm that defines expression and cuts +## Data frame as table # @code # frame = ... -# currnt , vexpr , cexpr, input_string = _fr_helper_ ( frame , 'x*x' , 'z<0' ) -# #endcode -def _fr_helper_ ( frame , expressions , cuts = '' ) : - """Helper function that defin4es expressions and cuts +# table = frame_table ( frame , '.*PT.*' , cuts = ... , more_vars = [ 'x*x/y' , 'y+z'] ) +# print ( table ) +# @endcode +def frame_table ( frame , pattern = None , cuts = '' , more_vars = () , title = '' , prefix = '' ) : + """Data frame as table >>> frame = ... - >>> current , vexpr , cexpr = _fr_helper_ ( frame , 'x*x' , 'z<0' ) - """ - - exprs, cuts, input_string = SV.vars_and_cuts ( expressions , cuts ) - - ## Frame/Tree ? - if isinstance ( frame , ROOT.TTree ) : node = DataFrame ( frame ) - elif isinstance ( frame , DataFrame ) : node = frame - elif isinstance ( frame , FrameNode ) : node = frame - else : node = as_rnode ( frame ) - - ## get the list of currently known names - vars = frame_columns ( node ) - all_vars = set ( vars ) - - current = node - - vnames = ordered_dict() - for i , expr in enumerate ( exprs , start = 1 ) : - vname = expr - if not expr in all_vars : - used = tuple ( all_vars | set ( frame_columns ( current ) ) ) - vn = var_name ( 'var%d_' % i , used , expr , *vars ) - all_vars.add ( vn ) - current = current.Define ( vn , expr ) - vname = vn - vnames [ expr ] = vname - - ## Filter frame, if needed! - ## if cuts : - ## ## add named filter - ## current = current.Filter ( '(bool) (%s)' % cuts, 'FILTER: %s' % cuts ) - - cname = cuts - if cuts and not cuts in all_vars : - used = tuple ( all_vars | set ( frame_columns ( current ) ) ) - cn = var_name ( 'cut_' , used , cuts , *vars ) - all_vars.add ( cn ) - ncuts = '1.0*(%s)' % cuts - current = current.Define ( cn , ncuts ) - cname = cn - - return current, vnames, cname, input_string - -# ================================================================================== -## the second helper method to implement "statistics"-related actions -def _fr_helper2_ ( frame , creator , expressions , cuts = '' , lazy = True ) : - """The seocnd helper method to implement statitic related actions - """ - - current, var_names, cut_name, input_string = _fr_helper_ ( frame , expressions , cuts ) - - results = {} - for expr, var_name in loop_items ( var_names ) : - results [ expr ] = creator ( current , var_name , cut_name ) - - lasy = False - if not lazy : - for expr, res in loop_items ( results ) : - results [ expr ] = res.GetValue() - rr = results [ expr ] - - if input_string and 1 == len ( results ) : - _ , r = results.popitem() - return r - - return results - -# ============================================================================ -_types_1D = Ostap.Math.LegendreSum , Ostap.Math.Bernstein , Ostap.Math.ChebyshevSum , -_types_2D = Ostap.Math.LegendreSum2 , Ostap.Math.Bernstein2D , -_types_3D = Ostap.Math.LegendreSum3 , Ostap.Math.Bernstein3D , -_types_4D = Ostap.Math.LegendreSum4 , -_types_nD = _types_1D + _types_2D + _types_3D + _types_4D -# ============================================================================= -## "project/parameterise" frame into polynomial structures (lazy action) -# @code -# frame = ... -# -# ls = Ostap.Math.LegendreSum ( ... ) -# res = frame_param ( frame , ls , 'x' , 'y>0' ) -# -# bs = Ostap.Math.Bernstein ( ... ) -# res = frame_param ( frame , bs , 'x' , 'y>0' ) -# -# cs = Ostap.Math.ChebyshevSum ( ... ) -# res = frame_param ( frame , cs , 'x' , 'y>0' ) -# -# ls2 = Ostap.Math.LegendreSum2 ( ... ) -# res = frame_param ( frame , ls2 , 'y' , 'x' , 'z>0' ) -# -# bs2 = Ostap.Math.Bernstein2D ( ... ) -# res = frame_param ( frame , bs2 , ( 'y' , 'x' ) , 'z>0' ) -# -# ls3 = Ostap.Math.LegendreSum3 ( ... ) -# res = frame_param ( frame , ls3 , ( 'z' , 'y' , 'x' ) , 'z>0' ) -# -# bs2 = Ostap.Math.Bernstein2D ( ... ) -# res = frame_param ( frame , bs2 , ( 'y' , 'x' ) , 'z>0' ) -# @endcode -def _fr_param_ ( frame , poly , expressions , cuts = '' , lazy = True ) : - """ `project/parameterise` frame into polynomial structures - - >>> frame = ... - - >>> ls = Ostap.Math.LegendreSum ( ... ) - >>> res = frame_param_lazy ( frame , ls , 'x' , 'y>0' ) - - >>> bs = Ostap.Math.Bernstein ( ... ) - >>> res = frame_param_lazy ( frame , bs , 'x' , 'y>0' ) - - >>> cs = Ostap.Math.ChebyshevSum ( ... ) - >>> res = frame_param_lazy( frame , cs , 'x' , 'y>0' ) - - >>> ls2 = Ostap.Math.LegendreSum2 ( ... ) - >>> res = frame_param_lazy ( frame , ls2 , 'y,x' , 'z>0' ) - - >>> bs2 = Ostap.Math.Bernstein2D ( ... ) - >>> res = frame_param_lazy ( frame , bs2 , 'y,x' , 'z>0' ) - - >>> ls3 = Ostap.Math.LegendreSum3 ( ... ) - >>> res = frame_param_lazy ( frame , ls3 , 'z,y;z' , 'z>0' ) - - >>> bs2 = Ostap.Math.Bernstein2D ( ... ) - >>> res = frame_param_lazy ( frame , bs2 , 'y,x' , 'z>0' ) - """ - - ## the histogram ? - if isinstance ( poly , ROOT.TH1 ) : - return _fr_project_ ( frame , poly , expressions , cuts = cuts , lazy = lazy ) - - ## - current , items, cname , _ = _fr_helper_ ( frame , expressions , cuts ) - - nvars = len ( items ) - - assert \ - ( 1 == nvars and isinstance ( poly , _types_1D ) ) or \ - ( 2 == nvars and isinstance ( poly , _types_2D ) ) or \ - ( 3 == nvars and isinstance ( poly , _types_3D ) ) or \ - ( 4 == nvars and isinstance ( poly , _types_4D ) ) , \ - "Invalid structure of polynomial and variables/cuts!" - - ## variables - uvars = [ k for k in items.values() ] - if cuts : uvars.append ( cname ) - uvars = CNT ( uvars ) - - ## finally book the actions! - if isinstance ( poly , Ostap.Math.LegendreSum ) : - result = current.Book ( ROOT.std.move ( Ostap.Actions.LegendrePoly ( poly ) ) , uvars ) - elif isinstance ( poly , Ostap.Math.LegendreSum2 ) : - result = current.Book ( ROOT.std.move ( Ostap.Actions.LegendrePoly2 ( poly ) ) , uvars ) - elif isinstance ( poly , Ostap.Math.LegendreSum3 ) : - result = current.Book ( ROOT.std.move ( Ostap.Actions.LegendrePoly3 ( poly ) ) , uvars ) - elif isinstance ( poly , Ostap.Math.LegendreSum4 ) : - result = current.Book ( ROOT.std.move ( Ostap.Actions.LegendrePoly4 ( poly ) ) , uvars ) - elif isinstance ( poly , Ostap.Math.ChebyshevSum ) : - result = current.Book ( ROOT.std.move ( Ostap.Actions.ChebyshevPoly ( poly ) ) , uvars ) - elif isinstance ( poly , Ostap.Math.Bernstein ) : - result = current.Book ( ROOT.std.move ( Ostap.Actions.BernsteinPoly ( poly ) ) , uvars ) - elif isinstance ( poly , Ostap.Math.Bernstein2D ) : - result = current.Book ( ROOT.std.move ( Ostap.Actions.BernsteinPoly2 ( poly ) ) , uvars ) - elif isinstance ( poly , Ostap.Math.Bernstein3D ) : - result = current.Book ( ROOT.std.move ( Ostap.Actions.BernsteinPoly3 ( poly ) ) , uvars ) - - print ( 'I AM HERE/1' ) - if not lazy : - result = result.GetValue() - poly = result - print ( 'I AM HERE/2' ) - - print ( 'I AM HERE/3', result ) - return result - - -# ============================================================================= -## Simplified print out for the frame -# @code -# frame = ... -# print frame -# @endcode -def frame_print ( frame ) : - """Simplified print out for the frame - - >>> frame = ... - >>> print frame - """ - ## - if isinstance ( frame , ROOT.TTree ) : frame = DataFrame ( frame ) - ## - node = as_rnode ( frame ) - res = "DataFrame Enries/#%d" % len ( frame ) - ## - cols = frame_columns ( node ) - res += "\nColumns:\n%s" % multicolumn ( cols , indent = 2 , pad = 1 ) - return res - - -# ============================================================================== -## Data frame as table -# @code -# frame = ... -# table = frame_table ( frame , '.*PT.*' , cuts = ... , more_vars = [ 'x*x/y' , 'y+z'] ) -# print ( table ) -# @endcode -def _fr_table_ ( frame , pattern = None , cuts = '' , more_vars = () , prefix = '' ) : - """Data frame as table - >>> frame = ... - >>> table = frame_table ( frame , '.*PT.*' , cuts = ... , more_vars = [ 'x*x/y' , 'y+z'] ) - >>> print ( table ) + >>> table = frame_table ( frame , '.*PT.*' , cuts = ... , more_vars = [ 'x*x/y' , 'y+z'] ) + >>> print ( table ) """ if isinstance ( frame , ROOT.TTree ) : frame = DataFrame ( frame ) - frame = as_rnode ( frame ) + node = as_rnode ( frame ) + ## the basic column type def col_type ( var ) : - + """The basic column type""" if var in cols : t = frame.GetColumnType ( var ) else : return '' @@ -582,14 +463,14 @@ def col_type ( var ) : elif 'Int_t' == t : return 'int' elif 'Long64_t' == t : return 'long' elif 'UChar_t' == t : return 'unsigned char' - elif 'UShort_t' == t : return 'unisgned short' + elif 'UShort_t' == t : return 'unsigned short' elif 'UInt_t' == t : return 'unsigned int' elif 'ULong64_t' == t : return 'unisgned long' return t ## get all variables/columns - cols = tuple ( frame.GetColumnNames() ) + cols = frame_columns ( node ) vcols = cols @@ -599,7 +480,7 @@ def col_type ( var ) : vcols = tuple ( [ v for v in vcols if cpat.match ( v ) ] ) svars = vcols + tuple ( more_vars ) - stats = _fr_statVar_ ( frame , svars , cuts = cuts , lazy = False ) + stats = frame_statVar ( node , svars , cuts = cuts ) if len ( vcols ) < 10 : ifmt = '%1d.' elif len ( vcols ) < 100 : ifmt = '%2d.' @@ -653,86 +534,15 @@ def col_type ( var ) : table.append ( row ) import ostap.logger.table as T - title = 'DataFrame variables' + title = title if title else 'DataFrame variables' if pattern : title = '%s (pattern %s)' % ( title , pattern ) t = T.table ( table , title , prefix = prefix , alignment = 'rllrlrl' ) return t -# =============================================================================== -## Print the frame report data -def report_print_table ( report , title = '' , prefix = '' , more_rows = [] ) : - """Print a frame report data - """ - from ostap.core.core import binomEff - - n0 = -1 - lmax = 5 - table = [] - - for name, passed, all in report : - - n0 = max ( n0 , all , passed ) - - eff1 = binomEff ( passed , all ) * 100 - - eff2 = binomEff ( passed , n0 ) * 100 - - lmax = max ( len ( name ) , lmax , len ( 'Filter ' ) ) - - item = name , passed , all , eff1 , eff2 - table.append ( item ) - - lmax = max ( lmax + 2 , len ( 'Selection' ) + 2 ) - fmt_name = '%%-%ds ' % lmax - fmt_input = '%10d' - fmt_passed = '%-10d' - fmt_eff = '%8.3g +/- %-8.3g' - fmt_cumulated = '%8.3g +/- %-8.3g' - - header = ( ( '{:^%d}' % lmax ).format ( 'Filter' ) , - ( '{:>10}' ).format ( '#input ' ) , - ( '{:<10}' ).format ( '#passed' ) , - ( '{:^20}' ).format ( 'efficiency [%]' ) , - ( '{:^20}' ).format ( 'cumulated efficiency [%]' ) ) - - table_data = [ header ] - for entry in table : - n, p, a , e1 , e2 = entry - table_data.append ( ( fmt_name % n , - fmt_input % a , - fmt_passed % p , - fmt_eff % ( e1.value () , e1.error () ) , - fmt_cumulated % ( e2.value () , e2.error () ) ) ) - for row in more_rows : - table_data.append ( row ) - - import ostap.logger.table as T - return T.table ( table_data , title = title , prefix = prefix , alignment = 'lcccc' ) - -# =============================================================================== -## Print the frame report -def report_as_table ( report ) : - """Print a frame report - """ - table = [] - for c in report: - name = c.GetName () - passed = c.GetPass () - all = c.GetAll () - table.append ( ( name , passed , all ) ) - - return table - -# =============================================================================== -## Print the data frame report -def report_print ( report , title = '' , prefix = '' , more_rows = [] ) : - """Print the data frame report - """ - table = report_as_table ( report ) - return report_print_table ( table , title, prefix , more_rows ) - - +# ============================================================================= +# Frame -> histogram prjections +# ============================================================================= if ( 6 , 14 ) <= root_info : DF_H1Model = ROOT.ROOT.RDF.TH1DModel @@ -779,7 +589,7 @@ def report_print ( report , title = '' , prefix = '' , more_rows = [] ) : # ============================================================================= ## convert 1D-histogram to "model" for usage with DataFrames def _h1_model_ ( histo ) : - """Convert 1D-histogram to 'model' for usage with DataFrames """ + """Convert 1D-histogram to 'model' for usage with DataFrame""" model = histo if not isinstance ( model , DF_H1Type ) : model = DF_H1Type() @@ -788,7 +598,7 @@ def _h1_model_ ( histo ) : # ============================================================================= ## convert 2D-histogram to "model" for usage with DataFrames def _h2_model_ ( histo ) : - """Convert 2D-histogram to 'model' for usage with DataFrames """ + """Convert 2D-histogram to 'model' for usage with DataFrame""" model = histo if not isinstance ( model , DF_H2Type ) : model = DF_H2Type() @@ -797,7 +607,7 @@ def _h2_model_ ( histo ) : # ============================================================================= ## convert 3D-histogram to "model" for usage with DataFrames def _h3_model_ ( histo ) : - """Convert 3D-histogram to 'model' for usage with DataFrames """ + """Convert 3D-histogram to 'model' for usage with DataFrame""" model = histo if not isinstance ( model , DF_H3Type ) : model = DF_H3Type() @@ -806,7 +616,7 @@ def _h3_model_ ( histo ) : # ============================================================================= ## convert 1D-profile to "model" for usage with DataFrames def _p1_model_ ( histo ) : - """Convert 1D-profile to 'model' for usage with DataFrames """ + """Convert 1D-profile to 'model' for usage with DataFrame""" model = histo if not isinstance ( model , DF_P1Type ) : model = DF_P1Type() @@ -815,24 +625,18 @@ def _p1_model_ ( histo ) : # ============================================================================= ## convert 2D-profile to "model" for usage with DataFrames def _p2_model_ ( histo ) : - """Convert 2D-profile to 'model' for usage with DataFrames """ + """Convert 2D-profile to 'model' for usage with DataFrame""" model = histo if not isinstance ( model , DF_P2Type ) : model = DF_P2Type() histo.Copy ( model ) return P2Model ( model ) - - - # ============================================================================== - ROOT.TH1.model = _h1_model_ ROOT.TH2.model = _h2_model_ ROOT.TH3.model = _h3_model_ ROOT.TProfile .model = _p1_model_ ROOT.TProfile2D.model = _p1_model_ - - # ============================================================================== ## Project of the frame # @code @@ -906,7 +710,8 @@ def frame_project ( frame , model , expressions , cuts = '' , lazy = True ) : if isinstance ( model , _types_nD ) : return _fr_param_ ( frame , model , expression, cuts = cuts , laszy = lazy ) - + + ## decode expresisons & cuts current , items, cname , _ = _fr_helper_ ( frame , expressions , cuts ) ## convert histogram-like objects into 'models' @@ -932,7 +737,6 @@ def frame_project ( frame , model , expressions , cuts = '' , lazy = True ) : else : raise TypeError ('Invalid model/what objects %s %s ' % ( type ( model ) , str ( pvars ) ) ) - ## if true histo is specified, the aciton is NOT lazy! if histo : histo += action.GetValue() @@ -940,47 +744,196 @@ def frame_project ( frame , model , expressions , cuts = '' , lazy = True ) : return action if lazy else action.GetValue() -# ============================================================================== -## Prescale the frame +# ============================================================================ +_types_1D = Ostap.Math.LegendreSum , Ostap.Math.Bernstein , Ostap.Math.ChebyshevSum , +_types_2D = Ostap.Math.LegendreSum2 , Ostap.Math.Bernstein2D , +_types_3D = Ostap.Math.LegendreSum3 , Ostap.Math.Bernstein3D , +_types_4D = Ostap.Math.LegendreSum4 , +_types_nD = _types_1D + _types_2D + _types_3D + _types_4D +# ============================================================================= +## "project/parameterise" frame into polynomial structures (lazy action) # @code # frame = ... -# prescaled1 = frame_prescale ( frame , 0.15 ) ## 0.0 < prescale < 1.0 -# prescaled2 = frame_prescale ( frame , 20 ) ## 1 < prescale -# @endcode -def frame_prescale ( frame , prescale , name = '' ) : - """Prescale the frame +# +# ls = Ostap.Math.LegendreSum ( ... ) +# res = frame_the_param ( frame , ls , 'x' , 'y>0' ) +# +# bs = Ostap.Math.Bernstein ( ... ) +# res = frame_the_param ( frame , bs , 'x' , 'y>0' ) +# +# cs = Ostap.Math.ChebyshevSum ( ... ) +# res = frame_the_param ( frame , cs , 'x' , 'y>0' ) +# +# ls2 = Ostap.Math.LegendreSum2 ( ... ) +# res = frame_the_param ( frame , ls2 , 'y' , 'x' , 'z>0' ) +# +# bs2 = Ostap.Math.Bernstein2D ( ... ) +# res = frame_the_param ( frame , bs2 , ( 'y' , 'x' ) , 'z>0' ) +# +# ls3 = Ostap.Math.LegendreSum3 ( ... ) +# res = frame_the_param ( frame , ls3 , ( 'z' , 'y' , 'x' ) , 'z>0' ) +# +# bs2 = Ostap.Math.Bernstein2D ( ... ) +# res = frame_the_param ( frame , bs2 , ( 'y' , 'x' ) , 'z>0' ) +# @endcode +def _fr_param_ ( frame , poly , expressions , cuts = '' , lazy = True ) : + """ `project/parameterise` frame into polynomial structures + >>> frame = ... - >>> prescaled1 = frame_prescale ( frame , 0.15 ) ## 0.0 < prescale < 1.0 - >>> prescaled2 = frame_prescale ( frame , 20 ) ## 1 < prescale + + >>> ls = Ostap.Math.LegendreSum ( ... ) + >>> res = frame_the_param ( frame , ls , 'x' , 'y>0' ) + + >>> bs = Ostap.Math.Bernstein ( ... ) + >>> res = frame_the_param ( frame , bs , 'x' , 'y>0' ) + + >>> cs = Ostap.Math.ChebyshevSum ( ... ) + >>> res = frame_the_param ( frame , cs , 'x' , 'y>0' ) + + >>> ls2 = Ostap.Math.LegendreSum2 ( ... ) + >>> res = frame_the_param ( frame , ls2 , 'y,x' , 'z>0' ) + + >>> bs2 = Ostap.Math.Bernstein2D ( ... ) + >>> res = frame_the_param ( frame , bs2 , 'y,x' , 'z>0' ) + + >>> ls3 = Ostap.Math.LegendreSum3 ( ... ) + >>> res = frame_the_param ( frame , ls3 , 'z,y;z' , 'z>0' ) + + >>> bs2 = Ostap.Math.Bernstein2D ( ... ) + >>> res = frame_the_param ( frame , bs2 , 'y,x' , 'z>0' ) """ - node = as_rnode ( frame ) - if isinstance ( prescale , integer_types ) and 1 < prescale : + ## the histogram ? + if isinstance ( poly , ROOT.TH1 ) : + return _fr_project_ ( frame , poly , expressions , cuts = cuts , lazy = lazy ) - name = name if name else 'PRESCALE_%d' % prescale - - code = '( 0 == ( ( rdfentry_ + %d * rdfslot_ ) %% %d ) )' \ - if ( 6 , 16 ) < root_info else \ - '( 0 == ( ( tdfentry_ + %d * tdfslot_ ) %% %d ) ) ' - - ## 16777213 and 16777199 are just large prime numbers - code = code % ( 16777213 , prescale ) - return node.Filter ( code , name ) - - elif isinstance ( prescale , float ) and 0 < prescale < 1 : + ## + current , items, cname , _ = _fr_helper_ ( frame , expressions , cuts ) + + nvars = len ( items ) + + assert \ + ( 1 == nvars and isinstance ( poly , _types_1D ) ) or \ + ( 2 == nvars and isinstance ( poly , _types_2D ) ) or \ + ( 3 == nvars and isinstance ( poly , _types_3D ) ) or \ + ( 4 == nvars and isinstance ( poly , _types_4D ) ) , \ + "Invalid structure of polynomial and variables/cuts!" - name = name if name else 'PRESCALE_%.6g' % prescale - code = 'gRandom->Rndm() <= %.12g' % prescale - return node.Filter ( code , name ) + ## variables + uvars = [ k for k in items.values() ] + if cuts : uvars.append ( cname ) + uvars = CNT ( uvars ) - elif 1 == prescale : - ## no action - return node + ## finally book the actions! + if isinstance ( poly , Ostap.Math.LegendreSum ) : + result = current.Book ( ROOT.std.move ( Ostap.Actions.LegendrePoly ( poly ) ) , uvars ) + elif isinstance ( poly , Ostap.Math.LegendreSum2 ) : + result = current.Book ( ROOT.std.move ( Ostap.Actions.LegendrePoly2 ( poly ) ) , uvars ) + elif isinstance ( poly , Ostap.Math.LegendreSum3 ) : + result = current.Book ( ROOT.std.move ( Ostap.Actions.LegendrePoly3 ( poly ) ) , uvars ) + elif isinstance ( poly , Ostap.Math.LegendreSum4 ) : + result = current.Book ( ROOT.std.move ( Ostap.Actions.LegendrePoly4 ( poly ) ) , uvars ) + elif isinstance ( poly , Ostap.Math.ChebyshevSum ) : + result = current.Book ( ROOT.std.move ( Ostap.Actions.ChebyshevPoly ( poly ) ) , uvars ) + elif isinstance ( poly , Ostap.Math.Bernstein ) : + result = current.Book ( ROOT.std.move ( Ostap.Actions.BernsteinPoly ( poly ) ) , uvars ) + elif isinstance ( poly , Ostap.Math.Bernstein2D ) : + result = current.Book ( ROOT.std.move ( Ostap.Actions.BernsteinPoly2 ( poly ) ) , uvars ) + elif isinstance ( poly , Ostap.Math.Bernstein3D ) : + result = current.Book ( ROOT.std.move ( Ostap.Actions.BernsteinPoly3 ( poly ) ) , uvars ) + + if not lazy : + result = result.GetValue() + poly *= 0.0 + poly += result - raise TypeError ( "Invalid type/value for 'prescale' %s/%s" %( prescale , type ( prescale ) ) ) - + return result + + +# ============================================================================= +## Get the length/size of the data frame +# @code +# frame = ... +# print ( len(frame) ) +# len = frame_length ( frame ) +# len = frame_size ( frame ) ## ditto +# @endcode +def frame_length ( frame , lazy = False ) : + """Get the length/size of the data frame + >>> frame = ... + >>> print len(frame) + >>> len = frame_length ( frame ) + >>> len = frame_size ( frame ) ## ditto + """ + node = as_rnode ( frame ) + cnt = node.Count () + return cnt if lazy else cnt.GetValue() + # ============================================================================== -## Get the moment for the frame object +## Size of the frame, same as frame length +# @see frame_length +frame_size = frame_length + +# ============================================================================= +## Get the effective entries in data frame +# @code +# data = ... +# neff = data.nEff('b1*b1') +# @endcode +def frame_nEff ( frame , cuts = '' ) : + """Get the effective entries in data frame + >>> data = ... + >>> neff = data.nEff('b1*b1') + """ + node = as_rnode ( frame ) + return SV.data_nEff ( node , cuts ) + +# ============================================================================= +## Get statistics for the given expression in data frame +# @code +# data = ... +# c1 = data.statVar( 'S_sw' , 'pt>10' ) +# c2 = data.statVar( 'S_sw' , 'pt>0' ) +# @endcode +def frame_statVar ( frame , expression , cuts = '' ) : + """Get statistics for the given expression in data frame + >>> data = ... + >>> c1 = data.statVar( 'S_sw' , 'pt>10' ) + >>> c2 = data.statVar( 'S_sw' ) + """ + if isinstance ( frame , ROOT.TTree ) : frame = DataFrame ( frame ) + node = as_rnode ( frame ) + return SV.data_statVar ( node , str ( expression ) , str ( cuts ) ) + +# ============================================================================= +## get the statistic for pair of expressions in DataFrame +# @code +# frame = ... +# stat1 , stat2 , cov2 , len = frame.statCov( 'x' , 'y' ) +# # apply some cuts +# stat1 , stat2 , cov2 , len = frame.statCov( 'x' , 'y' , 'z>0' ) +# @endcode +# @author Vanya BELYAEV Ivan.Belyaev@itep.ru +# @date 2018-06-18 +def frame_statCov ( frame , + expression1 , + expression2 , + cuts = '' ) : + """Get the statistic for pair of expressions in DataFrame + + >>> frame = ... + >>> stat1 , stat2 , cov2 , len = framw.statCov( 'x' , 'y' ) + + Apply some cuts: + >>> stat1 , stat2 , cov2 , len = frame.statCov( 'x' , 'y' , 'z>0' ) + + """ + if isinstance ( frame , ROOT.TTree ) : frame = DataFrame ( frame ) + node = as_rnode ( frame ) + return SV.data_statCov ( node , expression1 , expression2 , cuts ) + +# ============================================================================== +## Get the moment for the frame object # @code # frame = ... # value = frame_get_moment ( 3 , 1.234 , 'b1*b2' , 'b3>0' ) @@ -1028,6 +981,90 @@ def frame_central_moment ( frame , order , expression , cuts = '' ) : return SV.data_central_moment ( node , order = order , expression = expression , cuts = cuts ) +# ============================================================================== +## Get the mean for the frame object +# @code +# frame = ... +# value = frame_mean ( 'b1*b2' , 'b3>0' ) +# @endcode +# @see Ostap::StatVar::moment +def frame_mean ( frame , expression , cuts = '' ) : + """ Get the mean for the frame object + >>> frame = ... + >>> value = frame_mean ( 'b1*b2' , 'b3>0' ) + - see `Ostap.StatVar.moment` + """ + return frame_moment ( frame , order = 1 , expression = expression , cuts = cuts ) + +# ============================================================================== +## Get the variance for the frame object +# @code +# frame = ... +# value = frame_variance ( 'b1*b2' , 'b3>0' , exact = True ) +# value = frame_dispersion ( 'b1*b2' , 'b3>0' , exact = True ) ## ditto +# @endcode +# @see Ostap::StatVar::central_moment +def frame_variance ( frame , expression , cuts = '' ) : + """ Get the variance for the frame object + >>> frame = ... + >>> value = frame_variance ( 'b1*b2' , 'b3>0' ) + >>> value = frame_dispersion ( 'b1*b2' , 'b3>0' ) ## ditto + - see `Ostap.StatVar.moment` + """ + return frame_central_moment ( frame , order = 2 , expression = expression , cuts = cuts ) + +# ============================================================================== +## dispersion, same as variance +frame_dispersion = frame_variance + +# ============================================================================== +## Get the RMS for the frame object +# @code +# frame = ... +# value = frame_rms ( 'b1*b2' , 'b3>0' , exact = True ) +# @endcode +# @see Ostap::StatVar::central_moment +def frame_rms ( frame , expression , cuts = '' ) : + """ Get the RMS for the frame object + >>> frame = ... + >>> value = frame_rms ( 'b1*b2' , 'b3>0' , exact = True ) + >>> value = frame_rms ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm + - see `Ostap.StatVar.moment` + """ + return frame_variance ( frame , expression = expression , cuts = cuts ) ** 0.5 + +# ============================================================================== +## Get the skewness for the frame object +# @code +# frame = ... +# value = frame_skewness ( 'b1*b2' , 'b3>0' ) +# @endcode +# @see Ostap::StatVar::skewness +def frame_skewness ( frame , expression , cuts = '' ) : + """ Get the skewness for the frame object + >>> frame = ... + >>> value = frame_skeness ( 'b1*b2' , 'b3>0' ) + - see `Ostap.StatVar.skewness` + """ + node = as_rnode ( frame ) + return SV.data_skewness ( node , expression = expression , cuts = cuts ) + +# ============================================================================== +## Get the kurtosis for the frame object +# @code +# frame = ... +# value = frame_kurtosis ( 'b1*b2' , 'b3>0' ) +# @endcode +# @see Ostap::StatVar::kurtosis +def frame_kurtosis ( frame , expression , cuts = '' ) : + """ Get the kurtosis for the frame object + >>> frame = ... + >>> value = frame_kurtosis ( 'b1*b2' , 'b3>0' ) + - see `Ostap.StatVar.kurtosis` + """ + node = as_rnode ( frame ) + return SV.data_kurtosis ( node , expression = expression , cuts = cuts ) + # ============================================================================== ## Get the quantile for the frame object # @code @@ -1048,7 +1085,6 @@ def frame_quantile ( frame , q , expression , cuts = '' , exact = True ) : node = as_rnode ( frame ) return SV.data_quantile ( node , q = q , expression = expression , cuts = cuts , exact = exact ) - # ============================================================================== ## Get the interval for the frame object # @code @@ -1116,167 +1152,130 @@ def frame_quantiles ( frame , quantiles , expression , cuts = '' , exact = True # @see Ostap::StatVar::quantiles # @see Ostap::StatVar::p2quantiles def frame_terciles ( frame , expression , cuts = '' , exact = True ) : - """ Get the quantile for the frame object - >>> frame = ... - >>> value = frame_terciles ( 'b1*b2' , 'b3>0' , exact = True ) - >>> value = frame_terciles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm - - see `Ostap.StatVar.quantiles` - - see `Ostap.StatVar.p2quantiles` - """ - return frame_quantiles ( frame , quantiles = 3 , expression = expression , cuts = cuts , exact = exact ) - -# ============================================================================== -## Get the quartiles for the frame object -# @code -# frame = ... -# value = frame_quartiles ( 'b1*b2' , 'b3>0' , exact = True ) -# value = frame_quartiles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm -# @endcode -# @see Ostap::StatVar::quantiles -# @see Ostap::StatVar::p2quantiles -def frame_quartiles ( frame , expression , cuts = '' , exact = True ) : - """ Get the quantile for the frame object - >>> frame = ... - >>> value = frame_quartiles ( 'b1*b2' , 'b3>0' , exact = True ) - >>> value = frame_quartiles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm - - see `Ostap.StatVar.quantiles` - - see `Ostap.StatVar.p2quantiles` - """ - return frame_quantiles ( frame , quantiles = 4 , expression = expression , cuts = cuts , exact = exact ) - -# ============================================================================== -## Get the quintiles for the frame object -# @code -# frame = ... -# value = frame_quintiles ( 'b1*b2' , 'b3>0' , exact = True ) -# value = frame_quintiles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm -# @endcode -# @see Ostap::StatVar::quantiles -# @see Ostap::StatVar::p2quantiles -def frame_quintiles ( frame , expression , cuts = '' , exact = True ) : - """ Get the quintiles for the frame object - >>> frame = ... - >>> value = frame_quintiles ( 'b1*b2' , 'b3>0' , exact = True ) - >>> value = frame_quintiles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm - - see `Ostap.StatVar.quantiles` - - see `Ostap.StatVar.p2quantiles` - """ - return frame_quantiles ( frame , quantiles = 5 , expression = expression , cuts = cuts , exact = exact ) - -# ============================================================================== -## Get the deciles for the frame object -# @code -# frame = ... -# value = frame_deciles ( 'b1*b2' , 'b3>0' , exact = True ) -# value = frame_deciles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm -# @endcode -# @see Ostap::StatVar::quantiles -# @see Ostap::StatVar::p2quantiles -def frame_deciles ( frame , expression , cuts = '' , exact = True ) : - """ Get the quintiles for the frame object - >>> frame = ... - >>> value = frame_deciles ( 'b1*b2' , 'b3>0' , exact = True ) - >>> value = frame_deciles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm - - see `Ostap.StatVar.quantiles` - - see `Ostap.StatVar.p2quantiles` - """ - return frame_quantiles ( frame , quantiles = 10 , expression = expression , cuts = cuts , exact = exact ) - -# ============================================================================== -## Get the mean for the frame object -# @code -# frame = ... -# value = frame_mean ( 'b1*b2' , 'b3>0' ) -# @endcode -# @see Ostap::StatVar::moment -def frame_mean ( frame , expression , cuts = '' ) : - """ Get the mean for the frame object - >>> frame = ... - >>> value = frame_mean ( 'b1*b2' , 'b3>0' ) - - see `Ostap.StatVar.moment` - """ - return frame_moment ( frame , order = 1 , expression = expression , cuts = cuts ) - -# ============================================================================== -## Get the variance for the frame object -# @code -# frame = ... -# value = frame_variance ( 'b1*b2' , 'b3>0' , exact = True ) -# value = frame_dispersion ( 'b1*b2' , 'b3>0' , exact = True ) ## ditto -# @endcode -# @see Ostap::StatVar::central_moment -def frame_variance ( frame , expression , cuts = '' ) : - """ Get the variance for the frame object - >>> frame = ... - >>> value = frame_variance ( 'b1*b2' , 'b3>0' ) - >>> value = frame_dispersion ( 'b1*b2' , 'b3>0' ) ## ditto - - see `Ostap.StatVar.moment` - """ - return frame_central_moment ( frame , order = 2 , expression = expression , cuts = cuts ) - -frame_dispersion = frame_variance + """ Get the quantile for the frame object + >>> frame = ... + >>> value = frame_terciles ( 'b1*b2' , 'b3>0' , exact = True ) + >>> value = frame_terciles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm + - see `Ostap.StatVar.quantiles` + - see `Ostap.StatVar.p2quantiles` + """ + return frame_quantiles ( frame , quantiles = 3 , expression = expression , cuts = cuts , exact = exact ) # ============================================================================== -## Get the RMS for the frame object +## Get the quartiles for the frame object # @code # frame = ... -# value = frame_rms ( 'b1*b2' , 'b3>0' , exact = True ) +# value = frame_quartiles ( 'b1*b2' , 'b3>0' , exact = True ) +# value = frame_quartiles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm # @endcode -# @see Ostap::StatVar::central_moment -def frame_rms ( frame , expression , cuts = '' ) : - """ Get the RMS for the frame object +# @see Ostap::StatVar::quantiles +# @see Ostap::StatVar::p2quantiles +def frame_quartiles ( frame , expression , cuts = '' , exact = True ) : + """ Get the quantile for the frame object >>> frame = ... - >>> value = frame_rms ( 'b1*b2' , 'b3>0' , exact = True ) - >>> value = frame_rms ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm - - see `Ostap.StatVar.moment` - """ - return frame_variance ( frame , expression = expression , cuts = cuts ) ** 0.5 + >>> value = frame_quartiles ( 'b1*b2' , 'b3>0' , exact = True ) + >>> value = frame_quartiles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm + - see `Ostap.StatVar.quantiles` + - see `Ostap.StatVar.p2quantiles` + """ + return frame_quantiles ( frame , quantiles = 4 , expression = expression , cuts = cuts , exact = exact ) # ============================================================================== -## Get the skewness for the frame object +## Get the quintiles for the frame object # @code # frame = ... -# value = frame_skewness ( 'b1*b2' , 'b3>0' ) +# value = frame_quintiles ( 'b1*b2' , 'b3>0' , exact = True ) +# value = frame_quintiles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm # @endcode -# @see Ostap::StatVar::skewness -def frame_skewness ( frame , expression , cuts = '' ) : - """ Get the skewness for the frame object +# @see Ostap::StatVar::quantiles +# @see Ostap::StatVar::p2quantiles +def frame_quintiles ( frame , expression , cuts = '' , exact = True ) : + """ Get the quintiles for the frame object >>> frame = ... - >>> value = frame_skeness ( 'b1*b2' , 'b3>0' ) - - see `Ostap.StatVar.skewness` + >>> value = frame_quintiles ( 'b1*b2' , 'b3>0' , exact = True ) + >>> value = frame_quintiles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm + - see `Ostap.StatVar.quantiles` + - see `Ostap.StatVar.p2quantiles` """ - node = as_rnode ( frame ) - return SV.data_skewness ( node , expression = expression , cuts = cuts ) + return frame_quantiles ( frame , quantiles = 5 , expression = expression , cuts = cuts , exact = exact ) # ============================================================================== -## Get the kurtosis for the frame object +## Get the deciles for the frame object # @code # frame = ... -# value = frame_kurtosis ( 'b1*b2' , 'b3>0' ) +# value = frame_deciles ( 'b1*b2' , 'b3>0' , exact = True ) +# value = frame_deciles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm # @endcode -# @see Ostap::StatVar::kurtosis -def frame_kurtosis ( frame , expression , cuts = '' ) : - """ Get the kurtosis for the frame object +# @see Ostap::StatVar::quantiles +# @see Ostap::StatVar::p2quantiles +def frame_deciles ( frame , expression , cuts = '' , exact = True ) : + """ Get the quintiles for the frame object >>> frame = ... - >>> value = frame_kurtosis ( 'b1*b2' , 'b3>0' ) - - see `Ostap.StatVar.kurtosis` + >>> value = frame_deciles ( 'b1*b2' , 'b3>0' , exact = True ) + >>> value = frame_deciles ( 'b1*b2' , 'b3>0' , exact = False ) ## use P2 algorithm + - see `Ostap.StatVar.quantiles` + - see `Ostap.StatVar.p2quantiles` """ - node = as_rnode ( frame ) - return SV.data_kurtosis ( node , expression = expression , cuts = cuts ) + return frame_quantiles ( frame , quantiles = 10 , expression = expression , cuts = cuts , exact = exact ) -from ostap.fitting.dataset import ds_draw as _ds_draw -# ============================================================================== -## draw for frame -def frame_draw ( frame , *args , **kwargs ) : - """Draw function for the frames +# ================================================================================== +## Action-based methods +# ================================================================================== + +# ================================================================================== +## get statistics of variable(s) +# @code +# frame = .... +# stat = frame_the_tatVar ( 'pt' , lazy = True ) +# stat = frame_the_statVar ( 'pt' , 'eta>0' , lazy = True ) +# @endcode +def _fr_the_statVar_ ( frame , expressions , cuts = '' , lazy = True ) : + """Get statistics of variable(s) + >>> frame = .... + >>> stat = frame_the_statVar ( 'pt' , lazy = True ) + >>> stat = frame_the_statVar ( 'pt' , 'eta>0' , lazy = True ) """ - node = as_rnode ( frame ) - return _ds_draw ( node , *args , **kwargs ) + + def screator ( node , var_name , cut_name ) : + if cut_name: return node.Book( ROOT.std.move ( Ostap.Actions.WStatVar() ) , CNT ( [ var_name , cut_name ] ) ) + else : return node.Book( ROOT.std.move ( Ostap.Actions. StatVar() ) , CNT ( 1 , var_name ) ) + + return _fr_helper2_ ( frame , screator , expressions , cuts , lazy = lazy ) # ================================================================================== -## Action-based methods -# ================================================================================== +## get covariance for variables +# @code +# frame = .... +# stat = frame_the_statCov ( 'pt1' , 'pt2' , lazy = True ) +# stat = frame_the_statCov ( 'pt1' , 'pt2' , 'z<0' , lazy = True ) +# @endcode +def _fr_the_statCov_ ( frame , expression1 , expression2 , cuts = '' , lazy = True ) : + """Get statistics of variable(s) + >>> frame = .... + >>> stat = frame_the_statCov ( 'pt1' , 'pt2' , lazy = True ) + >>> stat = frame_the_statCov ( 'pt1' , 'pt2' , 'z<0' , lazy = True ) + """ + + current , exprs1 , cut_name , input_string1 = _fr_helper_ ( frame , expression1 , cuts ) + assert exprs1 and input_string1, 'Invalid expression1: %s' % expression2 + + current , exprs2 , cut_name , input_string2 = _fr_helper_ ( current , expression2 , cut_name ) + assert exprs2 and input_string2, 'Invalid expression2: %s' % expression2 + + ## variables + uvars = [ k for k in exprs1.values() ] + [ k for k in exprs2.values() ] + if cut_name : uvars.append ( cut_name ) + uvars = CNT ( uvars ) + + if cut_name: + TT = ROOT.Detail.RDF.Stat3Action ( Ostap.Math.WCovariance ) + action = node.Book( ROOT.std.move ( TT () ) , CNT ( uvars ) ) + else : + TT = ROOT.Detail.RDF.Stat2Action ( Ostap.Math.Covariance ) + action = node.Book( ROOT.std.move ( TT () ) , CNT ( uvars ) ) + + return action if lazy else action.GetValue() # ================================================================================== ## get statistics of variable(s) @@ -1285,7 +1284,7 @@ def frame_draw ( frame , *args , **kwargs ) : # stat = frame.statVar ( 'pt' , lazy = True ) # stat = frame.statVar ( 'pt' , 'eta>0' , lazy = True ) # @endcode -def _fr_statVar_ ( frame , expressions , cuts = '' , lazy = True) : +def _fr_the_statVar_ ( frame , expressions , cuts = '' , lazy = True ) : """Get statistics of variable(s) >>> frame = .... >>> stat = frame.statVar ( 'pt' , lazy = True ) @@ -1298,16 +1297,17 @@ def screator ( node , var_name , cut_name ) : return _fr_helper2_ ( frame , screator , expressions , cuts , lazy = lazy ) + # ================================================================================== ## get nEff through action # @code # frame = ... -# bEff = data_the_nEff ( 'x*x' , 'y>0' ) +# nEff = frame_the_nEff ( 'x*x' , 'y>0' ) # @endcode -def _fr_nEff_ ( frame , expressions , cuts = '' , lazy = False ) : +def _fr_the_nEff_ ( frame , expressions , cuts = '' , lazy = False ) : """Get nEff through action >>> frame = ... - >>> nEff = data_get_nEff ( 'x*x' , 'y>0' ) + >>> nEff = frame_the_nEff ( 'x*x' , 'y>0' ) """ return _fr_statVar_ ( frame , expressions , cuts = cuts , lazy = lazy ) @@ -1315,30 +1315,27 @@ def _fr_nEff_ ( frame , expressions , cuts = '' , lazy = False ) : ## get statistics of variable(s) # @code # frame = .... -# stat = frame.the_moment ( 5 , 'pt' ) -# stat = frame.the_moment ( 5 , 'pt' , 'eta>0' ) +# stat = frame_the_moment ( 5 , 'pt' ) +# stat = frame_the_moment ( 5 , 'pt' , 'eta>0' ) # @endcode # @see Ostap::Math::Moment_ # @see Ostap::Math::WMoment_ def _fr_the_moment_ ( frame , N , expressions , cuts = '' , lazy = True ) : """Get statistics of variable(s) >>> frame = .... - >>> stat = frame.the_moment ( 5 , 'pt' ) - >>> stat = frame.the_moment ( 5 , 'pt' , 'eta>0' ) + >>> stat = frame_the_moment ( 5 , 'pt' ) + >>> stat = frame_the_moment ( 5 , 'pt' , 'eta>0' ) """ - assert isinstance ( N , integer_types ) and 0 <= N , 'Invalid order!' current, vname , cname , input_string = _fr_helper_ ( frame , expressions , cuts ) def mcreator ( node , var_name , cut_name ) : if cname : - ## TT = Ostap.Actions.WMoment_(N) - TT = ROOT.Detail.RDF.WStatAction ( Ostap.Math.WMoment_(N) ) + TT = ROOT.Detail.RDF.Stat2Action ( Ostap.Math.WMoment_(N) ) return current.Book ( ROOT.std.move ( TT () ) , CNT ( [ var_name , cut_name ] ) ) else : - ## TT = Ostap.Actions.Moment_(N) - TT = ROOT.Detail.RDF.StatAction ( Ostap.Math.Moment_(N) ) + TT = ROOT.Detail.RDF.Stat1Action ( Ostap.Math.Moment_(N) ) return current.Book ( ROOT.std.move ( TT () ) , CNT ( 1 , var_name ) ) return _fr_helper2_ ( frame , mcreator , expressions , cuts , lazy = lazy ) @@ -1423,21 +1420,21 @@ def _fr_the_kurtosis_ ( frame , expressions , cuts = '' , errors = True , lazy = # @see Ostap::Math::ArithmeticMean # @code # frame = ... -# mean = frame_arithmetic_mean ( frame , 'x*x' , '0>> frame = ... - >>> mean = frame_arithmetic_mean ( frame , 'x*x' , '0>> mean = frame_the_arithmetic_mean ( frame , 'x*x' , '0>> frame = ... - >>> mean = frame_arithmetic_mean ( frame , 'x*x' , '0>> mean = frame_the_geometric_mean ( frame , 'x*x' , '0>> frame = ... - >>> mean = frame_harmonic_mean ( frame , 'x*x' , '0>> mean = frame_the_harmonic_mean ( frame , 'x*x' , '0>> frame = ... - >>> mean = frame_power_mean ( frame , 5 , 'x*x' , '0>> mean = frame_the_power_mean ( frame , 5 , 'x*x' , '0>> frame = ... - >>> mean = frame_lehmer_mean ( frame , 5 , 'x*x' , '0>> mean = frame_the_lehmer_mean ( frame , 5 , 'x*x' , '0 1 : +# =============================================================================== +if Frames_OK : - # ========================================================================== - ## get the mean using moment - # @see frame_the_mean - def frame_mean ( frame , expression , cuts = '' ) : - """Get mean using moment - - see frame_the_mean + ## frame_get_moment + ## frame_central_moment + + frame_the_statVar = _fr_the_statVar_ + frame_the_statCov = _fr_the_statCov_ + frame_the_nEff = _fr_the_nEff_ + frame_the_moment = _fr_the_moment_ + frame_the_mean = _fr_the_mean_ + frame_the_rms = _fr_the_rms_ + frame_the_variance = _fr_the_variance_ + frame_the_dispersion = _fr_the_variance_ + frame_the_skewness = _fr_the_skewness_ + frame_the_kurtosis = _fr_the_kurtosis_ + frame_the_arithmetic_mean = _fr_the_arithmetic_mean_ + frame_the_geometric_mean = _fr_the_geometric_mean_ + frame_the_harmonic_mean = _fr_the_harmonic_mean_ + frame_the_power_mean = _fr_the_power_mean_ + frame_the_lehmer_mean = _fr_the_lehmer_mean_ + frame_the_param = _fr_param_ + + __all__ += ( + 'frame_the_statVar' , + 'frame_the_statCov' , + 'frame_the_nEff' , + 'frame_the_moment' , + 'frame_the_mean' , + 'frame_the_rms' , + 'frame_the_variance' , + 'frame_the_dispersion' , + 'frame_the_skewness' , + 'frame_the_kurtosis' , + 'frame_the_arithmetic_mean' , + 'frame_the_geometric_mean' , + 'frame_the_harmonic_mean' , + 'frame_the_power_mean' , + 'frame_the_lehmer_mean' , + 'frame_the_param' , + ) + + # ================================================================================== + ## get statistics of variable(s) (via actions) + # @code + # frame = .... + # stat = frame_statVar ( frame , 'pt' ) + # stat = frame_statVar ( frame , 'pt' , 'eta>0' ) + # @endcode + def frame_statVar_ ( frame , expressions , cuts = '' ) : + """Get statistics of variable(s) + >>> frame = .... + >>> stat = frame.statVar ( 'pt' ) + >>> stat = frame.statVar ( 'pt' , 'eta>0' ) + """ + return frame_the_statVar ( frame , expressions , cuts = cuts , lazy = False ) + # ================================================================================== + ## get covariance for variables via action + # @code + # frame = .... + # stat = frame_statCov ( 'pt1' , 'pt2' , ) + # stat = frame_statCov ( 'pt1' , 'pt2' , 'z<0' ) + # @endcode + def frame_statCov ( frame , expression1 , expression2 , cuts = '' ) : + """Get statistics of variables via actions + >>> frame = .... + >>> stat = frame_statCov ( 'pt1' , 'pt2' , ) + >>> stat = frame_statCov ( 'pt1' , 'pt2' , 'z<0' ) + """ + return frame_the_statCov ( frame , expression1 , expression2 , cuts = cuts , lazy = False ) + + # ============================================================================= + ## Get the effective entries in data frame (via actions) + # @code + # data = ... + # neff = data.nEff('b1*b1') + # @endcode + def frame_nEff ( frame , cuts = '' ) : + """Get the effective entries in data frame + >>> data = ... + >>> neff = frame_nEff('b1*b1') + """ + return frame_the_nEff ( frame , cuts = cuts , lazy = False ).nEff() + # ================================================================================== + ## get statistics of variable(s) (via actions) + # @code + # frame = .... + # stat = frame_the_moment ( 5 , 'pt' ) + # stat = frame_the_moment ( 5 , 'pt' , 'eta>0' ) + # @endcode + # @see Ostap::Math::Moment_ + # @see Ostap::Math::WMoment_ + def frame_moment ( frame , N , expressions , cuts = '' ) : + """Get statistics of variable(s) (via actions) + >>> frame = .... + >>> stat = frame_moment ( 5 , 'pt' ) + >>> stat = frame_moment ( 5 , 'pt' , 'eta>0' ) """ - return frame_the_mean ( frame , expression , cuts = cuts , errors = True ).mean() - # ========================================================================== - ## get variance using moment - # @see frame_the_variance - def frame_variance ( frame , expression , cuts = '' ) : - """Get variance using moment - - see frame_the_variance + return frame_the_moment ( frame , N , expressions , cuts = cuts , lazy = False ) + # ============================================================================ + ## Get a mean via moment&action + # @code + # frame = ... + # mean = frame_mean ( 'x'x' , 'y<0' ) + # @ndcode + def frame_mean ( frame , expressions , cuts = '' , errors = True ) : + """ Get a mean via moment + >>> frame = ... + >>> mean = frame_mean ( 'x'x' , 'y<0' ) + """ + return frame_the_mean ( frame , expressions , cuts = cuts , errors = errors , lazy = False ).mean() + # ============================================================================ + ## Get a rms via moment&action + # @code + # frame = ... + # rms = frame_rms ( 'x'x' , 'y<0' ) + # @ndcode + def frame_rms ( frame , expressions , cuts = '' , errors = True , lazy = True ) : + """Get a rms via moment&action + >>> frame = ... + >>> rms = frame_rms ( 'x'x' , 'y<0' ) + """ + return frame_the_rms ( frame , expressions , cuts = cuts , errors = errors , lazy = False ).rms() + # ============================================================================ + ## Get a variance via moment&actions + # @code + # frame = ... + # variance = frame_variance ( 'x'x' , 'y<0' ) + # variance = frame_dispersion ( 'x'x' , 'y<0' ) + # @ndcode + def frame_variance ( frame , expressions , cuts = '' , errors = True , lazy = True ) : + """Get a variance via moment&action + >>> frame = ... + >>> variance = frame_variance ( 'x'x' , 'y<0' ) + >>> variance = frame_dispersion ( 'x'x' , 'y<0' ) + """ + return frame_the_variance ( frame , expressions , cuts = cuts , errors = errors , lazy = False ).variance() + # ============================================================================ + ## ditto + frame_dispersion = frame_variance + # ============================================================================ + ## Get a skewness via moment&action + # @code + # frame = ... + # skew = frame_skewness ( 'x'x' , 'y<0' ) + # @ndcode + def frame_skewness ( frame , expressions , cuts = '' , errors = True ) : + """Get a variance via moment + >>> frame = ... + >>> skew = frame_skewness ( 'x'x' , 'y<0' ) + >>> mean = stat.skreness () + """ + return frame_the_skewness ( frame , expressions , cuts = cuts , errros = errors , lazy = lazy ).skewness() + # ============================================================================ + ## Get a kurtosis via moment&actgions + # @code + # frame = ... + # kurt = frame_kurtosis ( 'x'x' , 'y<0' ) + # @ndcode + def frame_kurtosis ( frame , expressions , cuts = '' , errors = True ) : + """Get a kurtosis via moment + >>> frame = ... + >>> stat = frame.the_kurtosis ( 'x'x' , 'y<0' ) + >>> value = stat.kurtosis () + """ + return frame_the_kurtosis ( frame , expressions , cuts = cuts , errors = True , lazy = False ).kurtosis() + # ============================================================================ + ## Get an arithmetic mean + # @see Ostap::Math::ArithmeticMean + # @code + # frame = ... + # mean = frame_arithmetic_mean ( frame , 'x*x' , '0>> frame = ... + >>> mean = frame_arithmetic_mean ( frame , 'x*x' , '0>> frame = ... + >>> mean = frame_geometric_mean ( frame , 'x*x' , '0>> frame = ... + >>> mean = frame_harmonic_mean ( frame , 'x*x' , '0>> frame = ... + >>> mean = frame_power_mean ( frame , 5 , 'x*x' , '0>> frame = ... + >>> mean = frame_lehmer_mean ( frame , 5 , 'x*x' , '00' ) + # + # bs = Ostap.Math.Bernstein ( ... ) + # res = frame_param ( frame , bs , 'x' , 'y>0' ) + # + # cs = Ostap.Math.ChebyshevSum ( ... ) + # res = frame_param ( frame , cs , 'x' , 'y>0' ) + # + # ls2 = Ostap.Math.LegendreSum2 ( ... ) + # res = frame_param ( frame , ls2 , 'y' , 'x' , 'z>0' ) + # + # bs2 = Ostap.Math.Bernstein2D ( ... ) + # res = frame_param ( frame , bs2 , ( 'y' , 'x' ) , 'z>0' ) + # + # ls3 = Ostap.Math.LegendreSum3 ( ... ) + # res = frame_param ( frame , ls3 , ( 'z' , 'y' , 'x' ) , 'z>0' ) + # + # bs2 = Ostap.Math.Bernstein2D ( ... ) + # res = frame_param ( frame , bs2 , ( 'y' , 'x' ) , 'z>0' ) + # @endcode + def frame_param ( frame , poly , expressions , cuts = '' ) : + """ `project/parameterise` frame into polynomial structures + + >>> frame = ... + + >>> ls = Ostap.Math.LegendreSum ( ... ) + >>> res = frame_param ( frame , ls , 'x' , 'y>0' ) + + >>> bs = Ostap.Math.Bernstein ( ... ) + >>> res = frame_param ( frame , bs , 'x' , 'y>0' ) + + >>> cs = Ostap.Math.ChebyshevSum ( ... ) + >>> res = frame_param ( frame , cs , 'x' , 'y>0' ) + + >>> ls2 = Ostap.Math.LegendreSum2 ( ... ) + >>> res = frame_param ( frame , ls2 , 'y,x' , 'z>0' ) + + >>> bs2 = Ostap.Math.Bernstein2D ( ... ) + >>> res = frame_param ( frame , bs2 , 'y,x' , 'z>0' ) + + >>> ls3 = Ostap.Math.LegendreSum3 ( ... ) + >>> res = frame_param ( frame , ls3 , 'z,y;z' , 'z>0' ) + + >>> bs2 = Ostap.Math.Bernstein2D ( ... ) + >>> res = frame_param ( frame , bs2 , 'y,x' , 'z>0' ) """ - return frame_the_rms ( frame , expression , cuts = cuts , errors = True ) .rms () + return frame_the_param ( frame , poly , expressions , cuts = cuts , lazy = False ) + + +# =============================================================================== +## Print the frame report data +def report_print_table ( report , title = '' , prefix = '' , more_rows = [] ) : + """Print a frame report data + """ + from ostap.core.core import binomEff + + n0 = -1 + lmax = 5 + table = [] + + for name, passed, all in report : + n0 = max ( n0 , all , passed ) + eff1 = binomEff ( passed , all ) * 100 + eff2 = binomEff ( passed , n0 ) * 100 + lmax = max ( len ( name ) , lmax , len ( 'Filter ' ) ) + item = name , passed , all , eff1 , eff2 + table.append ( item ) + + lmax = max ( lmax + 2 , len ( 'Selection' ) + 2 ) + fmt_name = '%%-%ds ' % lmax + fmt_input = '%10d' + fmt_passed = '%-10d' + fmt_eff = '%8.3g +/- %-8.3g' + fmt_cumulated = '%8.3g +/- %-8.3g' + + header = ( ( '{:^%d}' % lmax ).format ( 'Filter' ) , + ( '{:>10}' ).format ( '#input ' ) , + ( '{:<10}' ).format ( '#passed' ) , + ( '{:^20}' ).format ( 'efficiency [%]' ) , + ( '{:^20}' ).format ( 'cumulated efficiency [%]' ) ) + + table_data = [ header ] + for entry in table : + n, p, a , e1 , e2 = entry + table_data.append ( ( fmt_name % n , + fmt_input % a , + fmt_passed % p , + fmt_eff % ( e1.value () , e1.error () ) , + fmt_cumulated % ( e2.value () , e2.error () ) ) ) + for row in more_rows : + table_data.append ( row ) + import ostap.logger.table as T + return T.table ( table_data , title = title , prefix = prefix , alignment = 'lcccc' ) + +# =============================================================================== +## Print the frame report +def report_as_table ( report ) : + """Print a frame report + """ + table = [] + for c in report: + name = c.GetName () + passed = c.GetPass () + all = c.GetAll () + table.append ( ( name , passed , all ) ) + return table + +# =============================================================================== +## Print the data frame report +def report_print ( report , title = '' , prefix = '' , more_rows = [] ) : + """Print the data frame report + """ + table = report_as_table ( report ) + return report_print_table ( table , title, prefix , more_rows ) # ============================================================================== # decorate # ============================================================================== _new_methods_ = [ - frame_nEff , - frame_statVar , - frame_statCov , + ## + frame_columns , + frame_branches , frame_progress , - frame_draw , + frame_prescale , frame_project , - frame_columns , + frame_draw , + frame_print , + frame_table , ## + frame_length , + frame_size , + ## + frame_nEff , + frame_statVar , + frame_statCov , + # + frame_get_moment , + frame_moment , + frame_central_moment , + # frame_mean , frame_rms , frame_variance , frame_dispersion , frame_skewness , frame_kurtosis , - frame_get_moment , - frame_moment , - frame_central_moment , ## frame_quantile , frame_interval , @@ -1622,163 +1967,174 @@ def frame_rms ( frame , expression , cuts = '' ) : frame_quartiles , frame_quintiles , frame_deciles , - ## - ] - -# ============================================================================== -if FrameNode is DataFrame : frames = DataFrame , -else : frames = DataFrame , FrameNode - + ## + report_print , + report_print_table , + report_as_table , + ] # ================================================================================ +for f in frame_types : -if Frames_OK : - - frame_table = _fr_table_ - - frame_the_statVar = _fr_statVar_ - frame_the_statVars = _fr_statVar_ - frame_the_moment = _fr_the_moment_ - - frame_the_nEff = _fr_nEff_ - frame_the_mean = _fr_the_mean_ - frame_the_rms = _fr_the_rms_ - frame_the_variance = _fr_the_variance_ - frame_the_dispersion = _fr_the_variance_ - frame_the_skewness = _fr_the_skewness_ - frame_the_kurtosis = _fr_the_kurtosis_ - - frame_arithmetic_mean = _fr_arithmetic_mean_ - frame_geometric_mean = _fr_geometric_mean_ - frame_harmonic_mean = _fr_harmonic_mean_ - frame_power_mean = _fr_power_mean_ - frame_lehmer_mean = _fr_lehmer_mean_ - - for f in frames : - f.statVars = frame_the_statVars - _new_methods_ .append ( f.statVars ) - f.table = frame_table - _new_methods_ .append ( f.table ) - f.the_moment = frame_the_moment - _new_methods_ .append ( f.the_moment ) - f.the_mean = frame_the_mean - _new_methods_ .append ( f.the_mean ) - f.the_rms = frame_the_rms - _new_methods_ .append ( f.the_rms ) - f.the_variance = frame_the_variance - _new_methods_ .append ( f.the_variance ) - f.the_skewness = frame_the_skewness - _new_methods_ .append ( f.the_skewness ) - f.the_kurtosis = frame_the_kurtosis - _new_methods_ .append ( f.the_kurtosis ) - f.arithmetic_mean = frame_arithmetic_mean - _new_methods_ .append ( f.arithmetic_mean ) - f.geometric_mean = frame_geometric_mean - _new_methods_ .append ( f.geometric_mean ) - f.harmonic_mean = frame_harmonic_mean - _new_methods_ .append ( f.harmonic_mean ) - f.power_mean = frame_power_mean - _new_methods_ .append ( f.power_mean ) - f.lehmer_mean = frame_lehmer_mean - _new_methods_ .append ( f.lehmer_mean ) - - _new_methods_ .append ( frame_table ) - _new_methods_ .append ( frame_the_statVar ) - _new_methods_ .append ( frame_the_moment ) - _new_methods_ .append ( frame_arithmetic_mean ) - __all__ = __all__ + ( 'frame_the_statVar' , - 'frame_table' , - 'frame_the_moment' , - 'frame_the_mean' , - 'frame_the_rms' , - 'frame_the_variance' , - 'frame_the_skewness' , - 'frame_the_kurtosis' , - 'frame_arithmetic_mean' , - 'frame_geometric_mean' , - 'frame_harmonic_mean' , - 'frame_power_mean' , - 'frame_lehmer_mean' ) - - ROOT.TTree. fstatVars = frame_the_statVar - ROOT.TTree. fthe_moment = frame_the_statVar - - _new_methods_ .append ( ROOT.TTree. fstatVars ) - _new_methods_ .append ( ROOT.TTree. fthe_moment ) - - -for f in frames : - - if not hasattr ( f , '__len__') : - f.__len__ = _fr_len_ - _new_methods_ .append ( f.__len__ ) - f.__str__ = frame_print f.__repr__ = frame_print + f.__len__ = frame_length + f.columns = frame_columns + f.branches = frame_branches + + f.project = frame_project + f.draw = frame_draw + f.nEff = frame_nEff - f.statVar = frame_statVar + f.statVar = frame_statVar f.statCov = frame_statCov - f.ProgressBar = frame_progress - f.progress = frame_progress - f.prescale = frame_prescale - - f.columns = frame_columns - f.branches = frame_columns - - f.mean = frame_mean - f.rms = frame_rms - f.variance = frame_variance - f.dispersion = frame_dispersion - f.skewness = frame_skewness + # + f.get_moment = frame_get_moment + f.moment = frame_moment + f.central_moment = frame_moment + # + f.mean = frame_mean + f.rms = frame_rms + f.variance = frame_variance + f.dispersion = frame_dispersion + f.skewness = frame_skewness f.kurtosis = frame_kurtosis - f.get_moment = frame_get_moment - f.moment = frame_moment - f.central_moment = frame_central_moment - - f.quantile = frame_quantile - f.interval = frame_interval - f.median = frame_median - f.quantiles = frame_quantiles - f.terciles = frame_terciles - f.quartiles = frame_quartiles - f.quintiles = frame_quintiles - f.deciles = frame_deciles - - _new_methods_ .append ( f.__str__ ) - _new_methods_ .append ( f.__repr__ ) - _new_methods_ .append ( f. nEff ) - _new_methods_ .append ( f. statVar ) - _new_methods_ .append ( f. ProgressBar ) - _new_methods_ .append ( f. progress ) - _new_methods_ .append ( f. columns ) - _new_methods_ .append ( f. branches ) - - _new_methods_ .append ( f.mean ) - _new_methods_ .append ( f.rms ) - _new_methods_ .append ( f.variance ) - _new_methods_ .append ( f.dispersion ) - _new_methods_ .append ( f.skewness ) - _new_methods_ .append ( f.kurtosis ) - _new_methods_ .append ( f.get_moment ) - _new_methods_ .append ( f.moment ) - _new_methods_ .append ( f.central_moment ) - - _new_methods_ .append ( f.quantile ) - _new_methods_ .append ( f.interval ) - _new_methods_ .append ( f.median ) - _new_methods_ .append ( f.quantiles ) - _new_methods_ .append ( f.terciles ) - _new_methods_ .append ( f.quartiles ) - _new_methods_ .append ( f.quintiles ) - _new_methods_ .append ( f.deciles ) - - f.draw = frame_draw - - _new_methods_ .append ( f. draw ) - - - -_decorated_classes_ = frames + # + f.quantile = frame_quantile + f.interval = frame_interval + f.median = frame_median + f.quantiles = frame_quantiles + f.terciles = frame_terciles + f.quartiles = frame_quartiles + f.quintiles = frame_quintiles + f.deciles = frame_deciles + + _new_methods_ += [ + f.columns , + f.branches , + f.project , + f.draw , + # + f.__str__ , + f.__repr__ , + f.__len__ , + # + f.nEff , + f.statVar , + f.statCov , + # + f.get_moment , + f.moment , + f.central_moment , + # + f.mean , + f.rms , + f.variance , + f.dispersion , + f.skewness , + f.kurtosis , + # + f.quantile , + f.interval , + f.median , + f.quantiles , + f.terciles , + f.quartiles , + f.quintiles , + f.deciles + ] + + +if Frames_OK : + + _new_methods_ += [ + frame_the_statVar , + frame_the_statCov , + frame_the_nEff , + frame_the_moment , + frame_the_mean , + frame_the_rms , + frame_the_variance , + frame_the_dispersion , + frame_the_skewness , + frame_the_kurtosis , + frame_the_arithmetic_mean , + frame_the_geometric_mean , + frame_the_harmonic_mean , + frame_the_power_mean , + frame_the_lehmer_mean , + frame_the_param , + ] + + for f in frame_types : + + f.the_statVar = frame_the_statVar + f.the_statCov = frame_the_statCov + f.the_nEff = frame_the_nEff + f.the_moment = frame_the_moment + f.the_mean = frame_the_mean + f.the_rms = frame_the_rms + f.the_variance = frame_the_variance + f.the_dispersion = frame_the_dispersion + f.the_skewness = frame_the_skewness + f.the_kurtosis = frame_the_kurtosis + + f.the_arithmetic_mean = frame_the_arithmetic_mean + f.the_geometric_mean = frame_the_geometric_mean + f.the_harmonic_mean = frame_the_harmonic_mean + f.the_power_mean = frame_the_power_mean + f.the_lehmer_mean = frame_the_lehmer_mean + f.the_param = frame_the_param + + f.arithmetic_mean = frame_arithmetic_mean + f.geometric_mean = frame_geometric_mean + f.harmonic_mean = frame_harmonic_mean + f.power_mean = frame_power_mean + f.lehmer_mean = frame_lehmer_mean + f.param = frame_param + + _new_methods_ += [ + f.arithmetic_mean , + f.geometric_mean , + f.harmonic_mean , + f.power_mean , + f.lehmer_mean , + f.param , + f.the_statVar , + f.the_statCov , + f.the_nEff , + f.the_moment , + f.the_mean , + f.the_rms , + f.the_variance , + f.the_dispersion , + f.the_skewness , + f.the_kurtosis , + + f.the_arithmetic_mean , + f.the_geometric_mean , + f.the_harmonic_mean , + f.the_power_mean , + f.the_lehmer_mean , + f.the_param , + + f.arithmetic_mean , + f.geometric_mean , + f.harmonic_mean , + f.power_mean , + f.lehmer_mean , + f.param , + ] + + ROOT.TTree. fstatVar = frame_statVar + ROOT.TTree. fstatCov = frame_statCov + ROOT.TTree. fmoment = frame_moment + + _new_methods_ .append ( ROOT.TTree. fstatVar ) + _new_methods_ .append ( ROOT.TTree. fstatCov ) + _new_methods_ .append ( ROOT.TTree. fmoment ) + +_decorated_classes_ = frame_types _new_methods_ += [ ROOT.TH1.model , @@ -1825,7 +2181,7 @@ def _rt_fproject_ ( tree , histo , *args ) : frame_param = _fr_param_ __all__ = __all__ + ( 'frame_param' , ) - for f in frames : f.param = frame_param + for f in frame_types : f.param = frame_param ## Project/parameterise the tree to the polynomial using DataFrame machinery # @code @@ -1843,7 +2199,7 @@ def _rt_fparam_ ( tree , poly , *args ) : ## use frame methods result = frame_param ( tree , poly, *args ) ## return - return result.GetValue() + return result _rt_fparam_ .__doc__ += '\n' + frame_param .__doc__ diff --git a/ostap/frames/tests/test_frames_frames.py b/ostap/frames/tests/test_frames_frames.py index fe1a0a3b..ce4852ba 100644 --- a/ostap/frames/tests/test_frames_frames.py +++ b/ostap/frames/tests/test_frames_frames.py @@ -37,9 +37,10 @@ # A simple helper function to fill a test tree def fill_tree ( tname , fname ) : - - tdf = DataFrame ( 10000 ) - a = tdf.ProgressBar ( 10000 ) + + N = 10000 + tdf = DataFrame ( N ) + tdf , pb = frame_progress ( tdf , N ) tdf.Define ("one", "1.0" )\ .Define ("b1" , "tdfentry_ + 1000.0") \ .Define ("b2" , "(1.0+b1)*(1.0+b1)" ) \ diff --git a/ostap/io/dbase.py b/ostap/io/dbase.py index e08285cd..fb8d2407 100644 --- a/ostap/io/dbase.py +++ b/ostap/io/dbase.py @@ -47,7 +47,6 @@ # ============================================================================= use_bsddb3 = False use_berkeleydb = False - # ============================================================================= ## make a try to use berkeleydb if ( 3 , 6 ) <= sys.version_info : diff --git a/source/include/Ostap/DataFrameActions.h b/source/include/Ostap/DataFrameActions.h index 1ba9dc08..08eb2c17 100644 --- a/source/include/Ostap/DataFrameActions.h +++ b/source/include/Ostap/DataFrameActions.h @@ -31,6 +31,8 @@ // ============================================================================ #if ROOT_VERSION(6,16,0) <= ROOT_VERSION_CODE // ============================================================================ +#include "Ostap/DataFrameUtils.h" +// ============================================================================ namespace ROOT { // ========================================================================== @@ -40,7 +42,7 @@ namespace ROOT namespace RDF { // ====================================================================== - /** @class StatAction + /** @class Stat1Action * Helper class to get statitsics for the column in DataFrame * using soem COUNTER class * Requirements for COUNTER: @@ -50,7 +52,7 @@ namespace ROOT * @see Ostap::Math::Moment_ */ template - class StatAction : public RActionImpl< StatAction > + class Stat1Action : public RActionImpl< Stat1Action > { public: // ==================================================================== @@ -61,19 +63,15 @@ namespace ROOT // ==================================================================== /// constructor template - StatAction ( const Args& ...args ) + Stat1Action ( const Args& ...args ) : m_result ( std::make_shared( args ... ) ) -#if ROOT_VERSION_CODE >= ROOT_VERSION(6,22,0) - , m_N ( ROOT::IsImplicitMTEnabled() ? std::max ( 1u , ROOT::GetThreadPoolSize () ) : 1u ) -#else - , m_N ( ROOT::IsImplicitMTEnabled() ? std::max ( 1u , ROOT::GetImplicitMTPoolSize () ) : 1u ) -#endif + , m_N ( std::max ( 1u , Ostap::Utils::mt_pool_size () ) ) , m_slots ( this->m_N , *(this->m_result.get() ) ) {} /// Move constructor - StatAction ( StatAction&& ) = default ; + Stat1Action ( Stat1Action&& ) = default ; /// Copy constructor is disabled - StatAction ( const StatAction& ) = delete ; + Stat1Action ( const Stat1Action& ) = delete ; // ==================================================================== public: // ==================================================================== @@ -89,7 +87,7 @@ namespace ROOT *m_result = sum ; } /// who am I ? - std::string GetActionName() { return "StatAction" ; } + std::string GetActionName() { return "Stat1Action" ; } // ==================================================================== public: // ==================================================================== @@ -124,17 +122,18 @@ namespace ROOT // ==================================================================== } ; // The end of class ROOT::Detail::RDF::StatCounter // ====================================================================== - /** @class WStatAction + /** @class Stat2Action * Helper class to get statitsics for the column in DataFrame * using soem COUNTER class * Requirements for COUNTER: * - counter.add ( value , weight ) * - counter += counter * @see Ostap::WStatEntity + * @see Ostap::Math::Covariance * @see Ostap::Math::WMoment_ */ template - class WStatAction : public RActionImpl< WStatAction > + class Stat2Action : public RActionImpl< Stat2Action > { public: // ==================================================================== @@ -145,19 +144,15 @@ namespace ROOT // ==================================================================== /// default constructor template - WStatAction ( const Args& ...args ) + Stat2Action ( const Args& ...args ) : m_result ( std::make_shared( args ... ) ) -#if ROOT_VERSION_CODE >= ROOT_VERSION(6,22,0) - , m_N ( ROOT::IsImplicitMTEnabled() ? std::max ( 1u , ROOT::GetThreadPoolSize () ) : 1u ) -#else - , m_N ( ROOT::IsImplicitMTEnabled() ? std::max ( 1u , ROOT::GetImplicitMTPoolSize () ) : 1u ) -#endif + , m_N ( std::max ( 1u , Ostap::Utils::mt_pool_size () ) ) , m_slots ( this->m_N , *(this->m_result.get() ) ) {} /// Move constructor - WStatAction ( WStatAction&& ) = default ; + Stat2Action ( Stat2Action&& ) = default ; /// Copy constructor is disabled - WStatAction ( const WStatAction& ) = delete ; + Stat2Action ( const Stat2Action& ) = delete ; // ==================================================================== public: // ==================================================================== @@ -173,7 +168,7 @@ namespace ROOT *m_result = sum ; } /// who am I ? - std::string GetActionName() { return "WStatAction" ; } + std::string GetActionName() { return "Stat2Action" ; } // ==================================================================== public: // ==================================================================== @@ -215,7 +210,120 @@ namespace ROOT /// (current) results per slot std::vector m_slots { 1 } ; // ==================================================================== - } ; // The end of class ROOT::Detail::RDF::WStatAction + } ; // The end of class ROOT::Detail::RDF::Stat2Action + // ====================================================================== + /** @class Stat3Action + * Helper class to get statitsics for the column in DataFrame + * using some COUNTER class + * Requirements for COUNTER: + * - counter.add ( v1 , v2 , weight ) + * - counter += counter + * @see Ostap::Math::WCovariance + */ + template + class Stat3Action : public RActionImpl< Stat3Action > + { + public: + // ==================================================================== + /// define the resutl type + using Result_t = Counter ; + // ==================================================================== + public: + // ==================================================================== + /// default constructor + template + Stat3Action ( const Args& ...args ) + : m_result ( std::make_shared( args ... ) ) + , m_N ( std::max ( 1u , Ostap::Utils::mt_pool_size () ) ) + , m_slots ( this->m_N , *(this->m_result.get() ) ) + {} + /// Move constructor + Stat3Action ( Stat3Action&& ) = default ; + /// Copy constructor is disabled + Stat3Action ( const Stat3Action& ) = delete ; + // ==================================================================== + public: + // ==================================================================== + /// initialize (empty) + void InitTask ( TTreeReader * , unsigned int ) {} ; + /// initialize (empty) + void Initialize () {} ; + /// finalize : sum over the slots + void Finalize () + { + Result_t sum { m_slots [ 0 ] } ; + for ( unsigned int i = 1 ; i < m_N ; ++i ) { sum += m_slots [ i ] ; } + *m_result = sum ; + } + /// who am I ? + std::string GetActionName() { return "Stat3Action" ; } + // ==================================================================== + public: + // ==================================================================== + /// The basic method: increment the counter + void Exec + ( unsigned int slot , + const double v1 , + const double v2 , + const double weight ) + { m_slots [ slot % m_N ].add ( v1 , v2 , weight ) ; } + // ==================================================================== + /// The basic method: increment the counter for the vector-like columns +#if ROOT_VERSION(6,22,0) <= ROOT_VERSION_CODE + template ::value, int>::type = 0> +#else + template ::value, int>::type = 0> +#endif + void Exec + ( unsigned int slot , + const T& vs , + const double v2 , + const double weight = 1 ) + { Result_t& m = m_slots [ slot % m_N ] ; for ( const auto& v1 : vs ) { m.add ( v1 , v2 , weight ) ; } } + // ==================================================================== + /// The basic method: increment the counter for the vector-like columns +#if ROOT_VERSION(6,22,0) <= ROOT_VERSION_CODE + template ::value, int>::type = 0> +#else + template ::value, int>::type = 0> +#endif + void Exec + ( unsigned int slot , + const double v1 , + const T& vs , + const double weight = 1 ) + { Result_t& m = m_slots [ slot % m_N ] ; for ( const auto& v2 : vs ) { m.add ( v1 , v2 , weight ) ; } } + // ==================================================================== + /// The basic method: increment the counter for the vector-like column of weight +#if ROOT_VERSION(6,22,0) <= ROOT_VERSION_CODE + template ::value, int>::type = 0> +#else + template ::value, int>::type = 0> +#endif + void Exec + ( unsigned int slot , + const double v1 , + const double v2 , + const T& ws ) + { Result_t& e = m_slots [ slot % m_N ] ; for ( const auto& w : ws ) { e.add ( v1 , v2 , w ) ; } } + // ==================================================================== + public: + // ==================================================================== + /// Get the result + std::shared_ptr GetResultPtr () const { return m_result ; } + /// get partial result for the given slot + Result_t& PartialUpdate ( unsigned int slot ) { return m_slots [ slot % m_N ] ; } + // ==================================================================== + private: + // ==================================================================== + /// the final result + const std::shared_ptr m_result { } ; + /// size of m_slots + unsigned long m_N { 1 } ; + /// (current) results per slot + std::vector m_slots { 1 } ; + // ==================================================================== + } ; // The end of class ROOT::Detail::RDF::Stat2Action // ====================================================================== /** @class Poly1Action * Helper class to parameterise data as 1D polynomial @@ -244,11 +352,7 @@ namespace ROOT Poly1Action ( const Args& ...args ) : m_result ( std::make_shared( args ... ) ) -#if ROOT_VERSION_CODE >= ROOT_VERSION(6,22,0) - , m_N ( ROOT::IsImplicitMTEnabled() ? std::max ( 1u , ROOT::GetThreadPoolSize () ) : 1u ) -#else - , m_N ( ROOT::IsImplicitMTEnabled() ? std::max ( 1u , ROOT::GetImplicitMTPoolSize () ) : 1u ) -#endif + , m_N ( std::max ( 1u , Ostap::Utils::mt_pool_size () ) ) , m_slots ( this->m_N , *(this->m_result.get() ) ) { (*m_result) *= 0.0 ; // reset the polynomial @@ -344,11 +448,7 @@ namespace ROOT Poly2Action ( const Args& ...args ) : m_result ( std::make_shared( args ... ) ) -#if ROOT_VERSION_CODE >= ROOT_VERSION(6,22,0) - , m_N ( ROOT::IsImplicitMTEnabled() ? std::max ( 1u , ROOT::GetThreadPoolSize () ) : 1u ) -#else - , m_N ( ROOT::IsImplicitMTEnabled() ? std::max ( 1u , ROOT::GetImplicitMTPoolSize () ) : 1u ) -#endif + , m_N ( std::max ( 1u , Ostap::Utils::mt_pool_size () ) ) , m_slots ( this->m_N , *(this->m_result.get() ) ) { (*m_result) *= 0.0 ; // reset the polynomial @@ -448,11 +548,7 @@ namespace ROOT Poly3Action ( const Args& ...args ) : m_result ( std::make_shared( args... ) ) -#if ROOT_VERSION_CODE >= ROOT_VERSION(6,22,0) - , m_N ( ROOT::IsImplicitMTEnabled() ? std::max ( 1u , ROOT::GetThreadPoolSize () ) : 1u ) -#else - , m_N ( ROOT::IsImplicitMTEnabled() ? std::max ( 1u , ROOT::GetImplicitMTPoolSize () ) : 1u ) -#endif + , m_N ( std::max ( 1u , Ostap::Utils::mt_pool_size () ) ) , m_slots ( this->m_N , *(this->m_result.get() ) ) { (*m_result) *= 0.0 ; // reset the polynomial @@ -563,11 +659,7 @@ namespace ROOT Poly4Action ( const Args& ...args ) : m_result ( std::make_shared( args ... ) ) -#if ROOT_VERSION_CODE >= ROOT_VERSION(6,22,0) - , m_N ( ROOT::IsImplicitMTEnabled() ? std::max ( 1u , ROOT::GetThreadPoolSize () ) : 1u ) -#else - , m_N ( ROOT::IsImplicitMTEnabled() ? std::max ( 1u , ROOT::GetImplicitMTPoolSize () ) : 1u ) -#endif + , m_N ( std::max ( 1u , Ostap::Utils::mt_pool_size () ) ) , m_slots ( this->m_N , *(this->m_result.get() ) ) { (*m_result) *= 0.0 ; // reset the polynomial @@ -678,29 +770,29 @@ namespace Ostap // ======================================================================== template - using StatAction = ROOT::Detail::RDF::StatAction ; + using Stat1Action = ROOT::Detail::RDF::Stat1Action ; template - using WStatAction = ROOT::Detail::RDF::WStatAction ; + using Stat2Action = ROOT::Detail::RDF::Stat2Action ; - using StatVar = StatAction ; - using WStatVar = WStatAction ; + using StatVar = Stat1Action ; + using WStatVar = Stat2Action ; template - using Moment_ = StatAction > ; + using Moment_ = Stat1Action > ; template - using WMoment_ = WStatAction > ; + using WMoment_ = Stat2Action > ; - using GeometricMean = StatAction ; - using ArithmeticMean = StatAction ; - using HarmonicMean = StatAction ; - using PowerMean = StatAction ; - using LehmerMean = StatAction ; + using GeometricMean = Stat1Action ; + using ArithmeticMean = Stat1Action ; + using HarmonicMean = Stat1Action ; + using PowerMean = Stat1Action ; + using LehmerMean = Stat1Action ; - using WGeometricMean = WStatAction ; - using WArithmeticMean = WStatAction ; - using WHarmonicMean = WStatAction ; - using WPowerMean = WStatAction ; - using WLehmerMean = WStatAction ; + using WGeometricMean = Stat2Action ; + using WArithmeticMean = Stat2Action ; + using WHarmonicMean = Stat2Action ; + using WPowerMean = Stat2Action ; + using WLehmerMean = Stat2Action ; template using Poly1Action = ROOT::Detail::RDF::Poly1Action ; diff --git a/source/include/Ostap/DataFrameUtils.h b/source/include/Ostap/DataFrameUtils.h index e28aec59..5bc6b31f 100644 --- a/source/include/Ostap/DataFrameUtils.h +++ b/source/include/Ostap/DataFrameUtils.h @@ -54,6 +54,13 @@ namespace Ostap ( const unsigned short nchunks , const ProgressConf& progress ) ; // ======================================================================== + /** get pool size + * @see ROOT::IsImplicitMTEnabled () + * @see ROOT::GetThreadPoolSize () + * @see ROOT::GetImplicitMTPoolSize () + */ + unsigned int mt_pool_size () ; + // ======================================================================== } // The end of namespace Ostap::Utils // ========================================================================== } // The end of namespace Ostap diff --git a/source/src/DataFrameUtils.cpp b/source/src/DataFrameUtils.cpp index 1f423c18..a367441f 100644 --- a/source/src/DataFrameUtils.cpp +++ b/source/src/DataFrameUtils.cpp @@ -60,9 +60,9 @@ namespace } // ======================================================================== /// default move constructor - DataFrameProgress ( DataFrameProgress&& ) = default ; + // DataFrameProgress ( DataFrameProgress&& ) = default ; /// disabled copy constructir - DataFrameProgress ( const DataFrameProgress& ) = default ; + // DataFrameProgress ( const DataFrameProgress& ) = default ; // ======================================================================== /// destructor ~DataFrameProgress() @@ -160,10 +160,10 @@ Ostap::Utils::frame_progress return DataFrameProgress ( nchunks , Ostap::Utils::ProgressConf ( width , - symbol , - blank , - left , - right ) ) ; + symbol , + blank , + left , + right ) ) ; } // =========================================================================== /* helper function to create callable for drawing of progress bar to the frame @@ -179,6 +179,25 @@ Ostap::Utils::frame_progress const Ostap::Utils::ProgressConf& progress ) { return DataFrameProgress ( nchunks , progress ) ; } // ============================================================================ +/* get pool size + * @see ROOT::IsImplicitMTEnabled() + * @see ROOT::GetThreadPoolSize () + * @see ROOT::GetImplicitMTPoolSize () + */ +// ============================================================================ +unsigned int Ostap::Utils::mt_pool_size () +{ + return std::max + ( 1u , + ROOT::IsImplicitMTEnabled() ? +#if ROOT_VERSION_CODE >= ROOT_VERSION(6,22,0) + ROOT::GetThreadPoolSize () : 1u +#else + ROOT::GetImplicitMTPoolSize () : 1u +#endif + ) ; +} +// ============================================================================ // ============================================================================ // The END diff --git a/source/src/StatVar.cpp b/source/src/StatVar.cpp index 2f2626f8..281ee434 100644 --- a/source/src/StatVar.cpp +++ b/source/src/StatVar.cpp @@ -30,6 +30,7 @@ #include "Ostap/Moments.h" #include "Ostap/GetWeight.h" #include "Ostap/Moments.h" +#include "Ostap/DataFrameUtils.h" // ============================================================================ // Local // ============================================================================ @@ -3243,14 +3244,7 @@ Ostap::StatVar::statVar .Define ( var , "1.0*(" + expression + ")" ) .Define ( weight , no_cuts ? "1.0" : "1.0*(" + cuts + ")" ) ; // - const unsigned int nSlots = - std::max ( 1u , -#if ROOT_VERSION_CODE >= ROOT_VERSION(6,22,0) - ROOT::GetThreadPoolSize () -#else - ROOT::GetImplicitMTPoolSize () -#endif - ) ; + const unsigned int nSlots = std::max ( 1u , Ostap::Utils::mt_pool_size() ) ; // std::vector _stat ( nSlots ? nSlots : 1 ) ; // @@ -3284,13 +3278,7 @@ Ostap::StatVar::statCov .Define ( var1 , "1.0*(" + exp1 + ")" ) .Define ( var2 , "1.0*(" + exp2 + ")" ) ; /// - const unsigned int nSlots = std::max ( 1u , -#if ROOT_VERSION_CODE >= ROOT_VERSION(6,22,0) - ROOT::GetThreadPoolSize () -#else - ROOT::GetImplicitMTPoolSize () -#endif - ); + const unsigned int nSlots = std::max ( 1u , Ostap::Utils::mt_pool_size() ) ; // std::vector _covs ( nSlots ? nSlots : 1 ) ; // @@ -3334,13 +3322,7 @@ Ostap::StatVar::statCov .Define ( var2 , "1.0*(" + exp2 + ")" ) .Define ( weight , no_cuts ? "1.0" : "1.0*(" + cuts + ")" ) ; /// - const unsigned int nSlots = std::max ( 1u , -#if ROOT_VERSION_CODE >= ROOT_VERSION(6,22,0) - ROOT::GetThreadPoolSize () -#else - ROOT::GetImplicitMTPoolSize () -#endif - ); + const unsigned int nSlots = std::max ( 1u , Ostap::Utils::mt_pool_size() ) ; // std::vector _covs ( nSlots ? nSlots : 1 ) ; //