diff --git a/CMakeLists.txt b/CMakeLists.txt index 3efe2c33..dc8d5a59 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,7 +25,7 @@ include(CTest) set(OSTAP_VERSION_MAJOR 1) set(OSTAP_VERSION_MINOR 13) set(OSTAP_VERSION_PATCH 5) -set(OSTAP_VERSION_TWEAK 0) +set(OSTAP_VERSION_TWEAK 1) set(OSTAP_VERSION ${OSTAP_VERSION_MAJOR}.${OSTAP_VERSION_MINOR}.${OSTAP_VERSION_PATCH}.${OSTAP_VERSION_TWEAK}) diff --git a/ReleaseNotes/release_notes.md b/ReleaseNotes/release_notes.md index bed910f4..f4462fe1 100644 --- a/ReleaseNotes/release_notes.md +++ b/ReleaseNotes/release_notes.md @@ -1,3 +1,9 @@ +## New features + +## Backward incompatible + +## Bug fixes + # v1.13.5.0 ## New features diff --git a/ReleaseNotes/v1.13.5.0.md b/ReleaseNotes/v1.13.5.0.md deleted file mode 100644 index 9cac7e85..00000000 --- a/ReleaseNotes/v1.13.5.0.md +++ /dev/null @@ -1,32 +0,0 @@ -# v1.13.5.0 - -## New features - - 1. Add `operator+` for `Ostap::Math.ECDF` - 1. Add correlation coefficient to `Ostap::Math::pow` function - 1. Slight improvements in convolution for fitting - 1. Add the first Goodness-of-Fit method for multidimensional case - 1. Add test for Goddness-of-Fit method for multidimensional case - 1. Add very simple "efficiency-counter" `ostap.stats.counters.EffCounter` - 1. suppress `ostap.core.config.config_goodby` prints for non-interactive sessions - 1. add the most primitive splitter `ostap.utils.utils.splitter` - 1. modify `point-to-point-dissimilarity` GoF method: split into chubnks for large datasets, use parallel processing for permutations - 1. add keyword arguments to `WorkManager.iexecute` method - 1. add an option to run parallel permutations using `WorkManager` parallelisation instead of `joblib` - 1. extend Point-to-Point Dissimilarity GoF test for 1D case - 1. extend 1D GoF test to include Point-to-Point Dissimilarity GoF test - 1. add argument `description` to `WorkManager.iexecute` methods - 1. add parallelisaiton for GoF permutations and toys - 1. implement tests for USTAT & DNN - 1. prepend the default progress-bar for trees/datasets/frames with `Entries:` - 1. add a kind of replacement of `ROOT.RooAbsCollection.assign` for old versions of ROOT - 1. add meaningful `description` argument to all `progress_bar` instance - 1. extend `gof1d` and `gofnd` tests - 1. Add `RooAddPdf::fixCoefNormalization( vars )` for all appearences of `RooAddPdf` - 1. Add (fictive) context mnager protocol to WorkManagers - -## Backward incompatible - -## Bug fixes - - 1. fix the bug in `Ostap::UStat` diff --git a/ostap/io/bz2shelve.py b/ostap/io/bz2shelve.py index 7c495831..0e9e7c4c 100644 --- a/ostap/io/bz2shelve.py +++ b/ostap/io/bz2shelve.py @@ -189,7 +189,7 @@ def __init__( self , # ========================================================================== ## compress (bzip2) the item using bz2.compress def compress_item ( self , value ) : - """Compress (zip) the item using ``bz2.compress'' + """ Compress (zip) the item using ``bz2.compress'' - see bz2.compress """ return bz2.compress ( self.pickle ( value ) , self.compresslevel ) @@ -197,7 +197,7 @@ def compress_item ( self , value ) : # ========================================================================= ## uncompres (bzip2) the item using bz2.decompress def uncompress_item ( self , value ) : - """Uncompress (bzip2) the item using ``bz2.decompress'' + """ Uncompress (bzip2) the item using ``bz2.decompress'' - see bz2.decompress """ return self.unpickle ( bz2.decompress ( value ) ) @@ -207,7 +207,7 @@ def uncompress_item ( self , value ) : # @author Vanya BELYAEV Ivan.Belyaev@cern.ch # @date 2010-04-30 def open ( dbname , mode = 'c' , **kwargs ) : - """Open a persistent dictionary for reading and writing. + """ Open a persistent dictionary for reading and writing. The filename parameter is the base filename for the underlying database. As a side-effect, an extension may be added to the diff --git a/ostap/io/lzshelve.py b/ostap/io/lzshelve.py index f05bacbc..4cedb6da 100644 --- a/ostap/io/lzshelve.py +++ b/ostap/io/lzshelve.py @@ -149,18 +149,23 @@ if '__main__' == __name__ : logger = getLogger ( 'ostap.io.lzshelve' ) else : logger = getLogger ( __name__ ) # ============================================================================= -logger.debug ( "Simple generic (c)Pickle-based ``LZMA''-database" ) +logger.debug ( "Simple generic (c)pickle-based `LZMA'-database" ) # ============================================================================= -if ( 3 , 3 ) <= python_version : - try : +if ( 3 , 3 ) <= python_version : # ============================================ + # ========================================================================= + try : # =================================================================== + # ===================================================================== import lzma - except ImportError : + # ===================================================================== + except ImportError : # ==================================================== + # ===================================================================== logger.error ( "Cannot import 'lzma', lzshelve is disabled" ) lzma = None -else : + # ========================================================================= +else : # ====================================================================== + # ========================================================================= logger.error ( 'lzshelve is disabled for python %s' % str ( python_version ) ) lzma = None - # ============================================================================= ## @class LzShelf # `LZMA'-version of `shelve'-database diff --git a/ostap/io/sqlitedict.py b/ostap/io/sqlitedict.py index d4a2332b..8209eeb5 100755 --- a/ostap/io/sqlitedict.py +++ b/ostap/io/sqlitedict.py @@ -49,6 +49,7 @@ from ostap.io.pickling import dumps, loads, HIGHEST_PROTOCOL as PICKLE_PROTOCOL import logging from ostap.logger.logger import getLogger +# ============================================================================= logger = getLogger( __name__ ) # ============================================================================= ## Is it a sqlite3 file? diff --git a/scripts/ostap-check-dependencies b/scripts/ostap-check-dependencies index 7763353e..02af56e7 100755 --- a/scripts/ostap-check-dependencies +++ b/scripts/ostap-check-dependencies @@ -9,65 +9,266 @@ """Check dependencies for ostap """ # ============================================================================= -import sys +from ostap.core.meta_info import root_info, python_info, ostap_info +from ostap.logger.colorized import attention +from ostap.logger.logger import getLogger # ============================================================================= -from ostap.logger.logger import getLogger -logger = getLogger('ostap-check-dependencies') +logger = getLogger('ostap-check-dependencies') # ============================================================================= -try : +rows = [ ( 'Module' , 'version', 'location' ) ] +# ============================================================================= +row = 'python' , '.'.join ( str ( a ) for a in python_info ) , '' +rows.append ( row ) +# ============================================================================= +try : # ======================================================================= + # ========================================================================= + import IPython + row = 'IPython', IPython.__version__ , IPython.__file__ + rows.append ( row ) + # ========================================================================= +except ImportError : # ======================================================== + # ========================================================================= + pass +# ============================================================================= +try : # ======================================================================= + # ========================================================================= + import ostap + row = 'ostap', ostap.__version__ , ostap.__file__ + rows.append ( row ) + # ========================================================================= +except ImportError : # ======================================================== + # ========================================================================= + logger.fatal ( 'ostap cannot be imported!' ) +# ============================================================================= +try : # ======================================================================= + # ========================================================================= import ROOT - logger.info ( "ROOT is imported from %s" % ROOT.__file__ ) -except ImportError : - logger.fatal ( "ROOT cannot be imported!") -# ============================================================================= -try : + row = 'ROOT', ROOT.__version__ , ROOT.__file__ + rows.append ( row ) + # ========================================================================= +except ImportError : # ======================================================== + # ========================================================================= + logger.fatal ( 'ROOT cannot be imported!' ) + row = attention ( 'ROOT' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +try : # ======================================================================= + # ========================================================================= + import cppyy + row = 'cppyy', cppyy.__version__ , cppyy.__file__ + rows.append ( row ) + # ========================================================================= +except ImportError : # ======================================================== + # ========================================================================= + logger.fatal ( 'cppyy cannot be imported!' ) + row = attention ( 'cpppy' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +try : # ======================================================================= + # ========================================================================= import numpy - logger.info ( "numpy is imported from %s" % numpy.__file__ ) -except ImportError : + row = 'numpy' , numpy.__version__ , numpy.__file__ + rows.append ( row ) + # ========================================================================= +except ImportError : # ======================================================== + # ========================================================================= message = """numpy cannot be imported! Numpy 'is needed for Fast Fourier Transform used for function/histogram parameterisation. - - There is no local replacement + and for (almost all) Googness-of-Fit methods + - There is no local replacement! - see http://numpy.org""" logger.error ( message ) + row = attention ( 'numpy' ) , attention ( '---' ) , '' + rows.append ( row ) # ============================================================================= -try : +try : # ======================================================================= + # ========================================================================= import scipy - logger.info ( "scipy is imported from %s" % scipy.__file__ ) -except ImportError : + row = 'scipy' , scipy.__version__ , scipy.__file__ + rows.append ( row ) + # ========================================================================= +except ImportError : # ======================================================== + # ========================================================================= message = """scipy cannot be imported! + Scipy is needed for (almost all) Goodness-of-Fit methods and Scipy is used for an efficient alternative for - numerical integration - root findings - function minimization - histogram/function parameterization - see http://www.scipy.org""" - logger.warning ( message ) + logger.error ( message ) + row = attention ( 'scipy' ) , attention ( '---' ) , '' + rows.append ( row ) + +# ============================================================================= +try : # ======================================================================= + # ========================================================================= + import numba + row = 'numba' , numba.__version__ , numba.__file__ + rows.append ( row ) + # ========================================================================= +except ImportError : # ======================================================== + # ========================================================================= + message = """numba cannot be imported! + Numba is useful for (almost all) Goodness-of-Fit methods + """ + logger.warning ( message ) + row = attention ( 'numba' ) , attention ( '---' ) , '' + rows.append ( row ) # ============================================================================= -if (3,3) <= sys.version_info < (3,10) : - try : - import bsddb - logger.info ( "bdsdb is imported from %s" % bsddb.__file__ ) +if python_info < ( 3 , 0 ) : # ================================================ + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import dbhash + row = 'dbhash' , '' , dbhash.__file__ + rows.append ( row ) + # ===================================================================== except ImportError : - message = """bdsbd cannot be imported - It is (optionally) used for persistency - """ - logger.warning ( message ) - -if (3,6) <= sys.version_info : - try : - import berkeleydb - logger.info ( "berkeleydb is imported from %s" % berkeleydb.__file__ ) - except ImportError : - message = """berkeleydb cannot be imported - It is (optionally) used for persistency - """ - logger.warning ( message ) - -# ============================================================================= -try : - import pathos - logger.info ( "pathos is imported from %s" % pathos.__file__ ) + # ===================================================================== + row = attention ( 'dbhash' ) , attention ( '---' ) , '' + rows.append ( row ) + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import gdbm + row = 'gdbm' , '' , gdbm.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'gdbm' ) , attention ( '---' ) , '' + rows.append ( row ) + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import dbm + row = 'dbm' , '' , dbm.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'dbm' ) , attention ( '---' ) , '' + rows.append ( row ) + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import dumbdbm + row = 'dumbdbm' , '' , dumbdbm.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'dumbdbm' ) , attention ( '---' ) , '' + rows.append ( row ) + # ========================================================================= +elif ( 3 , 13 ) <= python_info : # ============================================ + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import dbm.sqlite3 + row = 'dbm.sqlite3' , '' , dbm.sqlite3.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'dbm.sqlite3' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +if ( 3 , 0 ) <= python_info : # =============================================== + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import dbm.gnu + row = 'dbm.gnu' , '' , dbm.gnu.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'dbm.gnu' ) , attention ( '---' ) , '' + rows.append ( row ) + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import dbm.ndbm + row = 'dbm.ndbm' , '' , dbm.ndbm.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'dbm.ndbm' ) , attention ( '---' ) , '' + rows.append ( row ) + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import dbm.dumb + row = 'dbm.dumb' , '' , dbm.dumb.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'dbm.dumb' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +if ( 3 , 3 ) <= python_info < ( 3 , 10) : # =================================== + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import bsddb3 + row = 'bsddb3' , '' , bsddb3.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'bsddb3' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +if ( 3 , 6 ) <= python_info : # =============================================== + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import berkeleydb + row = 'berkeleydb' , berkeleydb.__version__ , berkeleydb.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'berkeleydb' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +try : # ======================================================================= + # ========================================================================= + import sqlite3 + row = 'sqlite3' , sqlite3.version , sqlite3.__file__ + rows.append ( row ) + # ========================================================================= except ImportError : + # ========================================================================= + row = attention ( 'sqlite3' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +if ( 3 , 7 ) <= python_info : # =============================================== + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import lmdb + row = 'lmdb' , lmdb.__version__ , lmdb.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'lmdb' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================ +try : # ====================================================================== + # ======================================================================== + import pathos + row = 'pathos' , pathos.__version__ , pathos.__file__ + rows.append ( row ) + # ======================================================================== +except ImportError : # ======================================================= + # ======================================================================== message = """pathos cannot be imported! Pathos is used for the paralellel ``cluster''-processing - see https://github.com/uqfoundation/pathos @@ -78,76 +279,207 @@ except ImportError : - dill - ppft""" logger.warning ( message ) + row = attention ( 'pathos' ) , attention ( '---' ) , '' + rows.append ( row ) # ============================================================================= -try : +try : # ======================================================================= + # ========================================================================= import dill - logger.info ( "dill is imported from %s" % dill.__file__ ) -except ImportError : + row = 'dill' , dill.__version__ , dill.__file__ + rows.append ( row ) + # ========================================================================= +except ImportError : # ======================================================== + # ========================================================================= message = """dill (part of pathos-suite) cannot be imported! Dill provdies a powerful object serialization needed for the parallel processing - https://github.com/uqfoundation/dill - https://pathos.readthedocs.io""" - logger.warning ( message ) -# ============================================================================= -try : + logger.warning ( message ) + row = attention ( 'dill' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +try : # ======================================================================= + # ========================================================================= import multiprocess - logger.info ( "multiprocess is imported from %s" % multiprocess.__file__ ) -except ImportError : + row = 'multiprocess' , multiprocess.__version__ , multiprocess.__file__ + rows.append ( row ) + # ========================================================================= +except ImportError : # ======================================================== + # ========================================================================= message = """multiprocess (part of pathos-suite) cannot be imported! Multiprocess is used for the paralellel processing - https://github.com/uqfoundation/multiprocess - https://pathos.readthedocs.io""" - logger.warning ( message ) + logger.warning ( message ) + row = attention ( 'multiprocess' ) , attention ( '---' ) , '' + rows.append ( row ) # ============================================================================= -try : +try : # ======================================================================= + # ========================================================================= import ppft - logger.info ( "ppft is imported from %s" % ppft.__file__ ) + row = 'ppft' , ppft.__version__ , ppft.__file__ + rows.append ( row ) + # ========================================================================= except ImportError : + # ========================================================================= message = """ppft (part of pathos-suite) cannot be imported! Ppft is a distributed and parallel python - https://github.com/uqfoundation/ppft - https://pathos.readthedocs.io""" - logger.warning ( message ) -# ============================================================================= -try : - import terminaltables - logger.info ( "terminaltables is imported from %s" % terminaltables.__file__ ) -except ImportError : - message = """terminaltables cannot be imported! - Terminaltables allowsvery nice altenrative for table formatting - - https://github.com/Robpol86/terminaltables - - https://robpol86.github.io/terminaltables""" - logger.warning ( message ) -# ============================================================================= -try : + logger.warning ( message ) + row = attention ( 'ppft' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +if ( 3 , 6 ) <= python_info : # =============================================== + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import ipyparallel + row = 'ipyparallel' , ipyparallel.__version__ , ipyparallel.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'ipyparallel' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +if ( 3 , 6 ) <= python_info : # =============================================== + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import joblib + row = 'joblib' , joblib.__version__ , joblib.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + message = """joblib cannot be imported! + Joblb (optional) allows to speedup Goodness-of-Fit tests""" + logger.warning ( message ) + row = attention ( 'joblib' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================ +try : # ====================================================================== + # ======================================================================== import psutil - logger.info ( "psutil is imported from %s" % psutil.__file__ ) -except ImportError : + row = 'psutil' , psutil.__version__ , psutil.__file__ + rows.append ( row ) + # ======================================================================== +except ImportError : # ======================================================= + # ======================================================================== message = """psutil cannot be imported! psutil is recommended for memory monitoring - There is no local replacement - see https://github.com/giampaolo/psutil""" - logger.error ( message ) + logger.error ( message ) + row = attention ( 'psutil' ) , attention ( '---' ) , '' + rows.append ( row ) # ============================================================================= -try : - import more_itertools - logger.info ( "more_itertools is imported from %s" % more_itertools.__file__ ) -except ImportError : +try : # ======================================================================= + # ========================================================================= + import more_itertools + row = 'more_itertools' , '' , more_itertools.__file__ + rows.append ( row ) + # ========================================================================= +except ImportError : # ======================================================== + # ========================================================================= message = """more_itertools cannot be imported! more_itertools is recommended for more efficient job splitting - There is a local replacement for ``chunked/divide/grouper'' https://github.com/more-itertools/more-itertools""" logger.error ( message ) -# ============================================================================= -try : + row = attention ( 'more_itertools' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +if ( 3 , 8 ) <= python_info : # =============================================== + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import zstandard + row = 'zstandard' , zstandard.__version__ , zstandard.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'zstandard' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +if ( 3 , 3 ) <= python_info : # =============================================== + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import lzma + row = 'lzma' , '' , lzma.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'lzma' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +has_tt3 = False +# ============================================================================= +if ( 3 , 9 ) <= python_info : # =============================================== + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import terminaltables3 + has_tt3 = True + row = 'terminaltables3' , terminaltables3.__version__ , terminaltables3.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'terminaltables3' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +if not has_tt3 : # ============================================================ + # ========================================================================= + try : # =================================================================== + # ===================================================================== + import terminaltables + row = 'terminaltables' , terminaltables.__version__ , terminaltables.__file__ + rows.append ( row ) + # ===================================================================== + except ImportError : + # ===================================================================== + row = attention ( 'terminaltables' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +try : # ======================================================================= + # ========================================================================= + import tabulate + row = 'tabulate' , tabulate.__version__ , tabulate.__file__ + rows.append ( row ) + # ========================================================================= +except ImportError : # ======================================================== + # ========================================================================= + row = attention ( 'tabulate' ) , attention ( '---' ) , '' + rows.append ( row ) +# ============================================================================= +try : # ======================================================================= + # ========================================================================= import hep_ml - logger.info ( "hep_ml is imported from %s" % hep_ml.__file__ ) + row = 'hep_ml' , hep_ml.__version__ , hep_ml.__file__ + rows.append ( row ) + # ========================================================================= except ImportError : + # ========================================================================= message = """hep_ml cannot be imported! hep_ml is recommended for very efficitnt multidimensional reweighting""" logger.error ( message ) + row = attention ( 'hep_ml' ) , attention ( '---' ) , '' + rows.append ( row ) # ============================================================================= + +# ============================================================================= +if '__main__' == __name__ : # ================================================= + # ========================================================================= + title = 'Ostap dependencies' + import ostap.logger.table as T + table = T.table ( rows , title = title , prefix = '# ' , alignment = 'lll' ) + logger.info ( '%s:\n%s' % ( title , table ) ) # ============================================================================= ## The END