From 55cd982ac9d07025bf54c1ff9a0d998e1471cb7a Mon Sep 17 00:00:00 2001 From: Vanya Belyaev Date: Wed, 31 Jul 2024 17:07:11 +0200 Subject: [PATCH] fix progress bar for frames --- ostap/frames/frames.py | 66 +++++++++++++++++++-------- ostap/stats/moment.py | 56 ++++++++++++++++------- source/include/Ostap/DataFrameUtils.h | 27 +++++++++++ source/src/DataFrameUtils.cpp | 56 +++++++++++++---------- source/src/StatEntity.cpp | 4 +- source/src/WStatEntity.cpp | 6 +-- 6 files changed, 152 insertions(+), 63 deletions(-) diff --git a/ostap/frames/frames.py b/ostap/frames/frames.py index e4c6c655..6486f1eb 100644 --- a/ostap/frames/frames.py +++ b/ostap/frames/frames.py @@ -309,8 +309,8 @@ def frame_progress1 ( frame , length ) : cnt = frame.Count () ## commented out - if not isatty () : return frame , cnt ## ATTENTION - elif ( 6 , 32 ) <= root_info : return frame , cnt ## ATTENTION + if not isatty () : return frame , cnt ## ATTENTION + elif length <= 0 : return frame_progress2 ( frame , length ) if 2048 < length : nchunks = 2000 elif 1024 < length : nchunks = 1000 @@ -322,9 +322,12 @@ def frame_progress1 ( frame , length ) : csize = max ( csize , 1 ) if rr : nchunks += 1 - - fun = Ostap.Utils.frame_progress ( nchunks , progress_conf () ) - cnt.OnPartialResultSlot ( csize , fun ) + + if (6,32 ) <= root_info : + cnt = Ostap.Utils.add_progress_bar ( cnt , nchunks , csize , progress_conf () ) + else : + fun = Ostap.Utils.frame_progress ( nchunks , progress_conf () ) + cnt = cnt.OnPartialResultSlot ( csize , fun ) return frame , cnt @@ -347,7 +350,7 @@ def frame_progress2 ( frame , length = -1 ) : cnt = frame.Count () - if not isatty () :return frame , cnt ## ATTENTION + if not isatty () : return frame , cnt ## ATTENTION elif root_info < ( 6, 30 ) : return frame , cnt ## ATTENTION! ## add progress bar @@ -355,8 +358,27 @@ def frame_progress2 ( frame , length = -1 ) : return frame, cnt # ========================================================================= -if ( 6,30 ) < root_info : frame_progress = frame_progress2 -else : frame_progress = frame_progress1 +## Add progress bar to the frame +# @code +# frame = ... +# frame , cnt = frame_progrees ( frame , nevents ) +# @endcode +def frame_progress ( frame , length = -1 ) : + """Add progress bar to the frame + >>> frame = ... + >>> frame , cnt = frame_progrees ( frame , nevents ) + """ + + if isinstance ( frame , frame_types ) : pass + elif isinstance ( frame , ROOT.TTree ) : + length = len ( frame ) + frame = DataFrame ( frame ) + else : + frame = as_rnode ( frame ) + + if 1 <= length : return frame_progress1 ( frame , length ) + else : return frame_progress2 ( frame , length ) + # ============================================================================== ## Prescale the frame @@ -395,15 +417,14 @@ def frame_prescale ( frame , prescale , name = '' ) : raise TypeError ( "Invalid type/value for 'prescale' %s/%s" %( prescale , type ( prescale ) ) ) -# ============================================================================== -from ostap.fitting.dataset import ds_draw as _ds_draw # ============================================================================== ## Draw the variables for the frame def frame_draw ( frame , *args , **kwargs ) : - """Draw the varibale (s) for the frames + """Draw the variable (s) for the frames """ node = as_rnode ( frame ) - return _ds_draw ( node , *args , **kwargs ) + from ostap.fitting.dataset import ds_draw as _ds_draw_ + return _ds_draw_ ( node , *args , **kwargs ) # ============================================================================= ## Simplified print out for the frame @@ -720,8 +741,6 @@ def frame_project ( frame , model , expressions , cuts = '' , lazy = True ) : ## convert histogram-like objects into 'models' - print ( 'FRAME PROJECT-BEFORE/1', type ( current ) , type ( model ) , items , cname ) - histo = None if isinstance ( model , ROOT.TProfile2D ) : histo, model = model, model.model () elif isinstance ( model , ROOT.TProfile ) : histo, model = model, model.model () @@ -729,16 +748,12 @@ def frame_project ( frame , model , expressions , cuts = '' , lazy = True ) : elif isinstance ( model , ROOT.TH2 ) and 2 == model.dim () : histo, model = model, model.model () elif isinstance ( model , ROOT.TH1 ) and 1 == model.dim () : histo, model = model, model.model () - print ( 'FRAME PROJECT-BEFORE/2', type ( current ) , type ( model ) , items , cname ) - if histo : histo.Reset() nvars = len ( items ) pvars = [ v for v in items.values() ] if cname : pvars.append ( cname ) - print ( 'FRAME PROJECT-AFTER', type ( current ) , type ( model ) , pvars ) - if 3 == nvars and isinstance ( model , DF_P2Model ) : action = current.Profile2D ( model , *pvars ) elif 2 == nvars and isinstance ( model , DF_P1Model ) : action = current.Profiel1D ( model , *pvars ) elif 3 == nvars and isinstance ( model , DF_H3Model ) : action = current.Histo3D ( model , *pvars ) @@ -855,7 +870,16 @@ def _fr_param_ ( frame , poly , expressions , cuts = '' , lazy = True ) : # ========================================================================== +## draw the variable(s) from the frame +# @code +# frame = ... +# result = frame_draw ( frame , 'x+12/z' , cut = 'z>1' ) +## @endcode def _fr_draw_ ( frame , expressions , cuts = '' , opts = '' , **kwargs ) : + """Draw the variable(s) from the frame + >>> frame = ... + >>> result = frame_draw ( frame , 'x+12/z' , cut = 'z>1' ) + """ ## decode expressions & cuts current , items, cname , _ = _fr_helper_ ( frame , expressions , cuts ) @@ -1946,7 +1970,7 @@ def frame_param ( frame , poly , expressions , cuts = '' ) : # frame = ... # mean = frame_range ( frame , 'x*x' , '0>> frame = ... @@ -1961,6 +1985,10 @@ def frame_range( frame , expressions , cuts = '' , delta = 0.05 ) : ranges [ k ] = axis_range ( mn , mx , delta = delta ) else : ranges = axis_range ( ranges.min() , ranges.max() , delta = delta ) return ranges + + + ## use more efficient drawing function + frame_draw = _fr_draw_ __all__ += ( 'frame_arithmetic_mean' , diff --git a/ostap/stats/moment.py b/ostap/stats/moment.py index e41b9ba2..b9e4c716 100644 --- a/ostap/stats/moment.py +++ b/ostap/stats/moment.py @@ -827,6 +827,7 @@ def _mn_reduce_ ( cnt ) : Ostap.Math.WHarmonicMean . __reduce__ = _mn_reduce_ Ostap.Math.WArithmeticMean . __reduce__ = _mn_reduce_ + # ============================================================================= ## serialization of Power means # @see Ostap::Math::PoweMean @@ -855,41 +856,64 @@ def _lm_reduce_ ( cnt ) : Ostap.Math.LehmerMean . __reduce__ = _lm_reduce_ Ostap.Math.WLehmerMean . __reduce__ = _lm_reduce_ +# ============================================================================= +## serialization of MinMaxValues +# @see Ostap::Math::MinMaxValue +# @see Ostap::Math::WMinMaxValue +def _nx_reduce_ ( cnt ) : + """Serialization of MinMax values + - see Ostap::Math::MinMaxValue + - see Ostap::Math::WMinMaxValue + """ + return root_factory , ( type ( cnt ) , cnt.min () , cnt.max() , cnt.conuter ()) + +Ostap.Math.MinMaxValue . __reduce__ = _nx_reduce_ +Ostap.Math.WMinMaxValue . __reduce__ = _nx_reduce_ + # ============================================================================== ## equality for the mean # @see Ostap::Math::GeometricMean # @see Ostap::Math::HarmonicMean +# @see Ostap::Math::ArithmeticMean # @see Ostap::Math::WGeometricMean # @see Ostap::Math::WHarmonicMean +# @see Ostap::Math::WArithmeticMean def _mn_eq_ ( cnt , another ) : """Equality Harmonic&Geometric means - see Ostap::Math::GeometricMean - see Ostap::Math::HarmonicMean + - see Ostap::Math::ArithmeticMean - see Ostap::Math::WGeometricMean - see Ostap::Math::WHarmonicMean + - see Ostap::Math::WArithmeticMean """ if not type ( cnt ) is type ( another ) : return NotImplemented return isequal ( cnt.value() , another.value() ) and \ cnt.counter() == another.counter() -Ostap.Math.GeometricMean . __eq__ = _mn_eq_ -Ostap.Math.HarmonicMean . __eq__ = _mn_eq_ -Ostap.Math.WGeometricMean . __eq__ = _mn_eq_ -Ostap.Math.WHarmonicMean . __eq__ = _mn_eq_ - +Ostap.Math.GeometricMean . __eq__ = _mn_eq_ +Ostap.Math.HarmonicMean . __eq__ = _mn_eq_ +Ostap.Math.ArithmeticMean . __eq__ = _mn_eq_ +Ostap.Math.WGeometricMean . __eq__ = _mn_eq_ +Ostap.Math.WHarmonicMean . __eq__ = _mn_eq_ +Ostap.Math.WArithmeticMean . __eq__ = _mn_eq_ + + _decorated_classes = ( - Ostap.Math.Moment , - Ostap.Math.WMoment , - Ostap.Math.Moment_(0) , - Ostap.Math.Moment_(1) , - Ostap.Math.Moment_(0) , - Ostap.Math.Moment_(1) , + Ostap.Math.Moment , + Ostap.Math.WMoment , + Ostap.Math.Moment_(0) , + Ostap.Math.Moment_(1) , + Ostap.Math.Moment_(0) , + Ostap.Math.Moment_(1) , ## - Ostap.Math.GeometricMean , - Ostap.Math.HarmonicMean , - Ostap.Math.WGeometricMean , - Ostap.Math.WHarmonicMean , - ) + Ostap.Math.GeometricMean , + Ostap.Math.HarmonicMean , + Ostap.Math.ArithmeticMean , + Ostap.Math.WGeometricMean , + Ostap.Math.WHarmonicMean , + Ostap.Math.WArithmeticMean , +) _new_methods_ = ( ## diff --git a/source/include/Ostap/DataFrameUtils.h b/source/include/Ostap/DataFrameUtils.h index 5bc6b31f..0d390dc3 100644 --- a/source/include/Ostap/DataFrameUtils.h +++ b/source/include/Ostap/DataFrameUtils.h @@ -9,10 +9,19 @@ #include #include // ============================================================================ +// ROOT +// ============================================================================ +#include "RVersion.h" +// ============================================================================ // Ostap // ============================================================================ #include "Ostap/DataFrame.h" // ============================================================================ +#if ROOT_VERSION(6,32,0) <= ROOT_VERSION_CODE +// ============================================================================ +namespace ROOT { namespace RDF { template class RResultPtr ; } } +// ============================================================================ +#endif namespace Ostap { // ========================================================================== @@ -61,6 +70,24 @@ namespace Ostap */ unsigned int mt_pool_size () ; // ======================================================================== +#if ROOT_VERSION(6,32,0) <= ROOT_VERSION_CODE + // ======================================================================== + /** helper utility to add progress bar + * to "Conut"-action + * @see https://root-forum.cern.ch/t/problems-with-onpartialresultslot-in-new-root-version-6-32-02/60257/3 + * @param count input counter + * @param nchunks + * @param progress progress bar configuration + */ + ROOT::RDF::RResultPtr& + add_progress_bar + ( ROOT::RDF::RResultPtr& result , + const unsigned short nchunks , + const unsigned long how_often , + const ProgressConf& progress ) ; + // ======================================================================== +#endif + // ======================================================================== } // The end of namespace Ostap::Utils // ========================================================================== } // The end of namespace Ostap diff --git a/source/src/DataFrameUtils.cpp b/source/src/DataFrameUtils.cpp index a367441f..3da3a9b1 100644 --- a/source/src/DataFrameUtils.cpp +++ b/source/src/DataFrameUtils.cpp @@ -34,7 +34,7 @@ namespace std::mutex s_mutex_bar {} ; // ========================================================================== /** @class DataFrameProgress - * Helper class to show th eprogree bar for data frame + * Helper class to show the progrees bar for data frame */ class DataFrameProgress : public Ostap::Utils::ProgressConf { @@ -59,37 +59,24 @@ namespace if ( !m_nchunks ) { setWidth ( 0 ) ; } // DISABLE IT! } // ======================================================================== - /// default move constructor - // DataFrameProgress ( DataFrameProgress&& ) = default ; - /// disabled copy constructir - // DataFrameProgress ( const DataFrameProgress& ) = default ; - // ======================================================================== /// destructor ~DataFrameProgress() { - // std::cout << "DESTRUCTOR1 " - // << ( m_done ? "DONE" : "NOT_YET" ) - // << "/" << m_chunks - // << "/" << m_nchunks - // << std::endl ; if ( m_nchunks && m_chunks && !m_done ) - { - m_chunks = m_nchunks ; - ULong64_t dummy = 0 ; - (*this)( 0 , dummy ) ; - } - // std::cout << "DESTRUCTOR2 " - // << ( m_done ? "DONE" : "NOT_YET" ) - // << "/" << m_chunks - // << "/" << m_nchunks - // << std::endl ; + { + m_chunks = m_nchunks ; + ULong64_t dummy = 0 ; + (*this)( 0 , dummy ) ; + } } // ======================================================================== public: // ======================================================================== // the main method // ======================================================================== - void operator() ( unsigned int islot , ULong64_t& /* u */ ) + void operator() + ( unsigned int islot , + ULong64_t& /* u */ ) { std::lock_guard lock ( s_mutex_bar ) ; /// @@ -190,7 +177,7 @@ unsigned int Ostap::Utils::mt_pool_size () return std::max ( 1u , ROOT::IsImplicitMTEnabled() ? -#if ROOT_VERSION_CODE >= ROOT_VERSION(6,22,0) +#if ROOT_VERSION(6,22,0) <= ROOT_VERSION_CODE ROOT::GetThreadPoolSize () : 1u #else ROOT::GetImplicitMTPoolSize () : 1u @@ -198,6 +185,29 @@ unsigned int Ostap::Utils::mt_pool_size () ) ; } // ============================================================================ +#if ROOT_VERSION(6,32,0) <= ROOT_VERSION_CODE +// ============================================================================ +/* helper utility to add progress bar + * to "Conut"-action + * @see https://root-forum.cern.ch/t/problems-with-onpartialresultslot-in-new-root-version-6-32-02/60257/3 + * @param count input counter + * @param progress cporgress bar configuration + */ +// ============================================================================ +ROOT::RDF::RResultPtr& +Ostap::Utils::add_progress_bar +( ROOT::RDF::RResultPtr& result , + const unsigned short nchunks , + const unsigned long howoften , + const Ostap::Utils::ProgressConf& progress ) +{ + auto thefun = frame_progress ( nchunks , progress ) ; + return result.OnPartialResultSlot( howoften , thefun ); +} +// =========================================================================== +#endif + + // ============================================================================ // The END diff --git a/source/src/StatEntity.cpp b/source/src/StatEntity.cpp index 0ead191b..a26aafd1 100644 --- a/source/src/StatEntity.cpp +++ b/source/src/StatEntity.cpp @@ -56,7 +56,7 @@ Ostap::StatEntity::StatEntity , m_max ( maxv ) { // empty counter (ignore min/max) - if ( 0 == m_n ) + if ( empty () ) { Ostap::Assert ( s_zero ( m_mu ) && s_zero ( m_mu2 ) , "Ostap::StatEntity: invalid mu/mu2 for empty counter!" , @@ -76,7 +76,7 @@ Ostap::StatEntity::StatEntity // if ( s_zero ( m_mu2 ) ) { m_mu2 = 0 ; } // - Ostap::Assert ( ( empty() && !m_mu2 ) || ( !empty() && m_mu2 ) , + Ostap::Assert ( ( !empty () ) || ( empty() && !m_mu2 ) , "Ostap::StatEntity: inconsistent mu2/empty!" , "Ostap::StatEntity" ) ; // diff --git a/source/src/WStatEntity.cpp b/source/src/WStatEntity.cpp index 58547f68..da089f75 100644 --- a/source/src/WStatEntity.cpp +++ b/source/src/WStatEntity.cpp @@ -55,18 +55,18 @@ Ostap::WStatEntity::WStatEntity { Ostap::Assert ( s_zero ( m_mu ) && s_zero ( m_mu2 ) , "Ostap::WStatWEntity: invalid mu/mu2 for empty counter!" , - "Ostap::WStatWEntity" ) ; + "Ostap::WStatEntity" ) ; m_mu = 0 ; m_mu2 = 0 ; } // Ostap::Assert ( m_values.n () <= m_weights.n () , "Ostap::WStatWEntity: inconsistent values/weights counters!" , - "Ostap::WStatWEntity" ) ; + "Ostap::WStatEntity" ) ; // if ( s_zero ( m_mu2 ) ) { m_mu2 = 0 ; } // - Ostap::Assert ( ( empty() && !m_mu2 ) || ( !empty() && m_mu2 ) , + Ostap::Assert ( ( !empty () ) || ( empty() && !m_mu2 ) , "Ostap::WStatEntity: inconsistent mu2/emptty!" , "Ostap::WStatEntity" ) ; //