From c6eeae3132f7a1fe461cb7548fd453b2a25924c0 Mon Sep 17 00:00:00 2001 From: Vanya Belyaev Date: Wed, 11 Sep 2024 13:04:35 +0200 Subject: [PATCH] 1. set `cmake_policy` to fic nuilding for virtual environemnts (conda) 1. more improvements and fixes for `ostap.io.dbase` module --- .aux/test_with_lcg | 2 +- CMakeLists.txt | 11 +-- ReleaseNotes/release_notes.md | 4 +- ostap/io/dbase.py | 110 ++++++++++++++++++++++++------ ostap/io/tests/test_io_shelves.py | 25 ++++--- 5 files changed, 116 insertions(+), 36 deletions(-) diff --git a/.aux/test_with_lcg b/.aux/test_with_lcg index 8923cec5..3949cb55 100755 --- a/.aux/test_with_lcg +++ b/.aux/test_with_lcg @@ -4,4 +4,4 @@ CMTCONFIG=$2 source /cvmfs/sft.cern.ch/lcg/views/${LCG}/${CMTCONFIG}/setup.sh source build/INSTALL/thisostap.sh cd build -ctest -N && cmake .. -DCMAKE_INSTALL_PREFIX=./INSTALL/ && ctest -j4 --output-on-failure --test-output-size-failed=5000000 +ctest -N && cmake .. -DCMAKE_INSTALL_PREFIX=./INSTALL/ && ctest -j4 -R _io_ --output-on-failure --test-output-size-failed=5000000 diff --git a/CMakeLists.txt b/CMakeLists.txt index 9594b3c1..cdcca656 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,12 @@ if ( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.10" ) cmake_host_system_information(RESULT OSTAP_OS_RELEASE QUERY OS_RELEASE) endif() +if ( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.15" ) + ## https://cmake.org/cmake/help/latest/policy/CMP0094.html + cmake_policy ( SET CMP0094 NEW ) +endif() + + project(ostap LANGUAGES CXX) include(CTest) @@ -35,8 +41,7 @@ find_program(CMAKE_C_COMPILER NAMES $ENV{CC} gcc PATHS ENV PATH NO_DEFAULT_PA find_program(CMAKE_CXX_COMPILER NAMES $ENV{CXX} g++ PATHS ENV PATH NO_DEFAULT_PATH) message ( "----> C-compiler : " $ENV{CC} ) -message ( x"----> C++-compiler : " $ENV{CXX} ) - +message ( "----> C++-compiler : " $ENV{CXX} ) # You need to tell CMake where to find the ROOT installation. This can be done in a number of ways: # - ROOT built with classic configure/make use the provided $ROOTSYS/etc/cmake/FindROOT.cmake @@ -54,10 +59,8 @@ add_subdirectory(examples) add_subdirectory(scripts) ## add_subdirectory(docs) - configure_file(${CMAKE_SOURCE_DIR}/CTestCustom.cmake ${CMAKE_BINARY_DIR}) - ### where to put the libraries set(LIBRARY_OUTPUT_PATH "${CMAKE_BINARY_DIR}/lib") ### where to put the executables diff --git a/ReleaseNotes/release_notes.md b/ReleaseNotes/release_notes.md index 2c48ffcd..5e398af5 100644 --- a/ReleaseNotes/release_notes.md +++ b/ReleaseNotes/release_notes.md @@ -4,7 +4,9 @@ 1. ROOT warnings -> Python warning now used RuntimeWarning 1. add new test `test_math_convolution` 1. add `filter='data'` argument for `tarfile` (TMVA&Chopping) - + 1. set `cmake_policy` to fic nuilding for virtual environemnts (conda) + 1. more improvements and fixes for `ostap.io.dbase` module + ## Backward incompatible ## Bug fixes diff --git a/ostap/io/dbase.py b/ostap/io/dbase.py index 0d1ada77..cb2881a4 100644 --- a/ostap/io/dbase.py +++ b/ostap/io/dbase.py @@ -43,6 +43,36 @@ if sys.version_info < ( 3, 7 ) : ordered_dict = collections.OrderedDict +# ============================================================================= +try : # ======================================================================= + # ========================================================================= + if ( 3 , 0 ) <= sys.version_info : import dbm.gnu as db_gnu + else : import gdbm as db_gnu + # ========================================================================= +except ImportError : + # ========================================================================= + db_gnu = None +# ============================================================================= +try : # ======================================================================= + # ========================================================================= + if ( 3 , 0 ) <= sys.version_info : import dbm.ndbm as db_dbm + else : import dbm as db_dbm + # ========================================================================= +except ImportError : + # ========================================================================= + db_dbm = None +# ============================================================================= +try : # ======================================================================= + # ========================================================================= + if sys.version_info < ( 3,0 ) : import dbhash as db_hash + else : db_hash = None + # ========================================================================= +except ImportError : + # ========================================================================= + db_hash = None +# ============================================================================= +if ( 3 , 0 ) <= sys.version_info : import dbm.dumb as db_dumb +else : import dumbdbm as db_dumb # ============================================================================= ## Check for Berkeley DB # ============================================================================= @@ -50,9 +80,9 @@ # ============================================================================= ## make a try to use berkeleydb if ( 3 , 6 ) <= sys.version_info : - - try : - + # ========================================================================= + try : # =================================================================== + # ===================================================================== import berkeleydb use_berkeleydb = True @@ -81,9 +111,10 @@ def berkeleydb_open ( filename , db.open ( filename , dbname , filetype , berkeleydb_open_mode [ flag ] , mode ) return db - - except ImportError : + # ===================================================================== + except ImportError : + # ===================================================================== berkeleydb = None use_berkeleydb = False @@ -93,8 +124,10 @@ def berkeleydb_open ( filename , use_bsddb3 = False # ============================================================================= ## make a try for dbddb3 -if ( 3 , 3 ) <= sys.version_info < ( 3 , 10 ) : - try : +if ( 3 , 3 ) <= sys.version_info < ( 3 , 10 ) : + # ========================================================================= + try : # =================================================================== + # ===================================================================== import bsddb3 ## open bsddb3 database def bsddb3_open ( filelame , @@ -103,8 +136,10 @@ def bsddb3_open ( filelame , """ Open `bsddb3` database """ return bsddb3.hasopen ( filename , flag , mode , **kwargs ) - use_bsddb3 = True - except ImportError : + use_bsddb3 = True + # ===================================================================== + except ImportError : + # ===================================================================== bsddb3 = None use_bsddb3 = False @@ -140,7 +175,7 @@ def bsddb3_open ( filelame , # - Actually it is a bit extended form of dbm.whichdb # that accounts for bsddb3 and sqlite3 def whichdb ( filename ) : - """Guess which db package to use to open a db file. + """ Guess which db package to use to open a db file. Return values: @@ -236,7 +271,7 @@ def dbopen ( file , flag = 'r' , mode = 0o666 , concurrent = True , - dbtype = () , ## preferred dbtype or list of preferences + dbtype = () , ## preferred db-type as list of preferences **kwargs ) : """Open or create database at path given by *file*. @@ -262,26 +297,43 @@ def dbopen ( file , # 'n' flag is specified or dbase does not exist and c flag is specified if 'n' in flag or ( check is None and 'c' in flag ) : - + if isinstance ( dbtype , str ) : db_types = [ dbtype.lower() ] + elif not dbtype : db_types = () else : db_types = [ db.lower() for db in dbtype ] - ## check the preferred database type: for db in db_types : - if use_berkeleydb and db in ( 'berkeleydb' , 'berkeley' ) : + if use_berkeleydb and db in ( 'berkeleydb' , 'berkeley' , 'berkeley-db' ) : return berkeleydb_open ( file , flag , mode , **kwargs ) - elif use_bsddb3 and 'bdsdb3' == db : + elif use_bsddb3 and 'bsddb3' == db : return bsddb3_open ( file , flag , mode , **kwargs ) elif use_lmdb and 'lmdb' == db : return LmdbDict ( path = file , flag = flag , **kwargs ) - elif db in ( 'sqlite' , 'sqlite3' ) : + elif db in ( 'sqlite3' , 'sqlite' , 'sql' ) : return SqliteDict ( filename = file , flag = flag , **kwargs ) - elif 'std' == db or db.startswith ( 'dbm.' ) or db.endswith ( 'dbm' ) : + + if db_gnu and db in ( 'dbm.gnu' , 'gdbm' ) : + if kwargs : logger.warning ( 'Ignore extra %d arguments:%s' % ( len ( kwargs ) , [ k for k in kwargs ] ) ) + return db_gnu.open ( file , flag , mode ) + elif db_dbm and db in ( 'dbm.ndbm' , 'dbm' ) : + if kwargs : logger.warning ( 'Ignore extra %d arguments:%s' % ( len ( kwargs ) , [ k for k in kwargs ] ) ) + return db_dbm.open ( file , flag , mode ) + elif db_hash and db in ( 'dbhash' , ) : if kwargs : logger.warning ( 'Ignore extra %d arguments:%s' % ( len ( kwargs ) , [ k for k in kwargs ] ) ) - return std_db.open ( file , flag , mode ) + return db_hash.open ( file , flag , mode ) + elif db in ( 'dbm.dumb' , 'dumbdbm' , 'dumb' ) : + if kwargs : logger.warning ( 'Ignore extra %d arguments:%s' % ( len ( kwargs ) , [ k for k in kwargs ] ) ) + return db_dumb.open ( file , flag , mode ) + elif db in ( 'std' , 'standard' ) or not db : + if kwargs : logger.warning ( 'Ignore extra %d arguments:%s' % ( len ( kwargs ) , [ k for k in kwargs ] ) ) + return std_db.open ( file , flag , mode ) + if db_types : + logger.warning ( 'DB-type hints not used: [%s]' % ( ','.join ( db for fn in db_types ) ) ) + + if concurrent and use_berkeleydb : return berkeleydb_open ( file , flag , mode , **kwargs ) @@ -303,10 +355,12 @@ def dbopen ( file , if use_lmdb and check in ( 'lmdb' , ) : return LmdbDict ( path = file , flag = flag , **kwargs ) - if check == 'sqlite3' : + if check in ( 'sqlite3' , 'sqlite' ) : return SqliteDict ( filename = file , flag = flag , **kwargs ) if kwargs : logger.warning ( 'Ignore extra %d arguments:%s' % ( len ( kwargs ) , [ k for k in kwargs ] ) ) + + ## as a lasty resort - use the standard stuff return std_db.open ( file , flag , mode ) # ============================================================================= @@ -410,7 +464,23 @@ def clean ( self ) : from ostap.utils.docme import docme docme ( __name__ , logger = logger ) - + + logger.info ('Available DB-backends are:' ) + if use_berkeleydb : + logger.info ( ' - BerkeleyDB : %s' % str ( berkeleydb ) ) + if use_bsddb3 : + logger.info ( ' - BSDDB3 : %s' % str ( bsddb3 ) ) + if use_lmdb : + logger.info ( ' - LMDB : %s' % str ( lmdb ) ) + if db_gnu : + logger.info ( ' - GNU dbase : %s' % str ( db_gnu ) ) + if db_dbm : + logger.info ( ' - NDBM : %s' % str ( db_dbm ) ) + if db_dumb : + logger.info ( ' - DUMB : %s' % str ( db_dumb ) ) + if db_hash : + logger.info ( ' - DBHASH : %s' % str ( db_hash ) ) + # ============================================================================= ## The END # ============================================================================= diff --git a/ostap/io/tests/test_io_shelves.py b/ostap/io/tests/test_io_shelves.py index 73e23a07..34cc2738 100755 --- a/ostap/io/tests/test_io_shelves.py +++ b/ostap/io/tests/test_io_shelves.py @@ -228,22 +228,27 @@ def test_shelves2 () : if zstshelve : shelves.append ( zstshelve ) backends = [ - 'lmdb' , - 'berkleydb' , - 'berkley' , - 'bdsdb2' , - 'sqlite' , - 'sqlite3' , - '' + 'lmdb' , + 'berkeleydb' , 'berkeley' , 'berkeley-db' , + 'bsddb3' , + 'sqlite3' , 'sqlite' , 'sql' , + # + 'dbm.gnu' , 'gdbm' , + 'dbm.ndbm' , 'dbm' , + 'dbm.dumb' , 'dumbdbm' , + 'dbhash' , + 'standard' , 'std' , + # + '' , None , ] for sh in shelves : for b in backends : with sh.tmpdb ( dbtype = b ) as db : - db ['one'] = 1 - db ['two'] = 2 - db.ls() + db [ 'one' ] = 1 + db [ 'two' ] = 2 + db.ls() # ============================================================================= if '__main__' == __name__ :