Skip to content

Commit

Permalink
1. set cmake_policy to fic nuilding for virtual environemnts (conda)
Browse files Browse the repository at this point in the history 
  1. more improvements and fixes for `ostap.io.dbase` module
  • Loading branch information
@VanyaBelyaev
VanyaBelyaev committed Sep 11, 2024
1 parent fbc8b0e commit c6eeae3
Show file tree
Hide file tree
Showing 5 changed files with 116 additions and 36 deletions.
2 changes: 1 addition & 1 deletion .aux/test_with_lcg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ CMTCONFIG=$2
source /cvmfs/sft.cern.ch/lcg/views/${LCG}/${CMTCONFIG}/setup.sh
source build/INSTALL/thisostap.sh
cd build
ctest -N && cmake .. -DCMAKE_INSTALL_PREFIX=./INSTALL/ && ctest -j4 --output-on-failure --test-output-size-failed=5000000
ctest -N && cmake .. -DCMAKE_INSTALL_PREFIX=./INSTALL/ && ctest -j4 -R _io_ --output-on-failure --test-output-size-failed=5000000
11 changes: 7 additions & 4 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,12 @@ if ( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.10" )
cmake_host_system_information(RESULT OSTAP_OS_RELEASE QUERY OS_RELEASE)
endif()

if ( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.15" )
## https://cmake.org/cmake/help/latest/policy/CMP0094.html
cmake_policy ( SET CMP0094 NEW )
endif()


project(ostap LANGUAGES CXX)
include(CTest)

Expand All @@ -35,8 +41,7 @@ find_program(CMAKE_C_COMPILER NAMES $ENV{CC} gcc PATHS ENV PATH NO_DEFAULT_PA
find_program(CMAKE_CXX_COMPILER NAMES $ENV{CXX} g++ PATHS ENV PATH NO_DEFAULT_PATH)

message ( "----> C-compiler : " $ENV{CC} )
message ( x"----> C++-compiler : " $ENV{CXX} )

message ( "----> C++-compiler : " $ENV{CXX} )

# You need to tell CMake where to find the ROOT installation. This can be done in a number of ways:
# - ROOT built with classic configure/make use the provided $ROOTSYS/etc/cmake/FindROOT.cmake
Expand All @@ -54,10 +59,8 @@ add_subdirectory(examples)
add_subdirectory(scripts)
## add_subdirectory(docs)


configure_file(${CMAKE_SOURCE_DIR}/CTestCustom.cmake ${CMAKE_BINARY_DIR})


### where to put the libraries
set(LIBRARY_OUTPUT_PATH "${CMAKE_BINARY_DIR}/lib")
### where to put the executables
Expand Down
4 changes: 3 additions & 1 deletion ReleaseNotes/release_notes.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
1. ROOT warnings -> Python warning now used RuntimeWarning
1. add new test `test_math_convolution`
1. add `filter='data'` argument for `tarfile` (TMVA&Chopping)

1. set `cmake_policy` to fic nuilding for virtual environemnts (conda)
1. more improvements and fixes for `ostap.io.dbase` module

## Backward incompatible

## Bug fixes
Expand Down
110 changes: 90 additions & 20 deletions ostap/io/dbase.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,16 +43,46 @@
if sys.version_info < ( 3, 7 ) :
ordered_dict = collections.OrderedDict

# =============================================================================
try : # =======================================================================
# =========================================================================
if ( 3 , 0 ) <= sys.version_info : import dbm.gnu as db_gnu
else : import gdbm as db_gnu
# =========================================================================
except ImportError :
# =========================================================================
db_gnu = None
# =============================================================================
try : # =======================================================================
# =========================================================================
if ( 3 , 0 ) <= sys.version_info : import dbm.ndbm as db_dbm
else : import dbm as db_dbm
# =========================================================================
except ImportError :
# =========================================================================
db_dbm = None
# =============================================================================
try : # =======================================================================
# =========================================================================
if sys.version_info < ( 3,0 ) : import dbhash as db_hash
else : db_hash = None
# =========================================================================
except ImportError :
# =========================================================================
db_hash = None
# =============================================================================
if ( 3 , 0 ) <= sys.version_info : import dbm.dumb as db_dumb
else : import dumbdbm as db_dumb
# =============================================================================
## Check for Berkeley DB
# =============================================================================
use_berkeleydb = False
# =============================================================================
## make a try to use berkeleydb
if ( 3 , 6 ) <= sys.version_info :

try :

# =========================================================================
try : # ===================================================================
# =====================================================================
import berkeleydb
use_berkeleydb = True

Expand Down Expand Up @@ -81,9 +111,10 @@ def berkeleydb_open ( filename ,
db.open ( filename , dbname , filetype , berkeleydb_open_mode [ flag ] , mode )

return db

except ImportError :

# =====================================================================
except ImportError :
# =====================================================================
berkeleydb = None
use_berkeleydb = False

Expand All @@ -93,8 +124,10 @@ def berkeleydb_open ( filename ,
use_bsddb3 = False
# =============================================================================
## make a try for dbddb3
if ( 3 , 3 ) <= sys.version_info < ( 3 , 10 ) :
try :
if ( 3 , 3 ) <= sys.version_info < ( 3 , 10 ) :
# =========================================================================
try : # ===================================================================
# =====================================================================
import bsddb3
## open bsddb3 database
def bsddb3_open ( filelame ,
Expand All @@ -103,8 +136,10 @@ def bsddb3_open ( filelame ,
""" Open `bsddb3` database """
return bsddb3.hasopen ( filename , flag , mode , **kwargs )

use_bsddb3 = True
except ImportError :
use_bsddb3 = True
# =====================================================================
except ImportError :
# =====================================================================
bsddb3 = None
use_bsddb3 = False

Expand Down Expand Up @@ -140,7 +175,7 @@ def bsddb3_open ( filelame ,
# - Actually it is a bit extended form of <code>dbm.whichdb</code>
# that accounts for <code>bsddb3</code> and <code>sqlite3</code>
def whichdb ( filename ) :
"""Guess which db package to use to open a db file.
""" Guess which db package to use to open a db file.
Return values:
Expand Down Expand Up @@ -236,7 +271,7 @@ def dbopen ( file ,
flag = 'r' ,
mode = 0o666 ,
concurrent = True ,
dbtype = () , ## preferred dbtype or list of preferences
dbtype = () , ## preferred db-type as list of preferences
**kwargs ) :
"""Open or create database at path given by *file*.
Expand All @@ -262,26 +297,43 @@ def dbopen ( file ,

# 'n' flag is specified or dbase does not exist and c flag is specified
if 'n' in flag or ( check is None and 'c' in flag ) :

if isinstance ( dbtype , str ) : db_types = [ dbtype.lower() ]
elif not dbtype : db_types = ()
else : db_types = [ db.lower() for db in dbtype ]


## check the preferred database type:
for db in db_types :

if use_berkeleydb and db in ( 'berkeleydb' , 'berkeley' ) :
if use_berkeleydb and db in ( 'berkeleydb' , 'berkeley' , 'berkeley-db' ) :
return berkeleydb_open ( file , flag , mode , **kwargs )
elif use_bsddb3 and 'bdsdb3' == db :
elif use_bsddb3 and 'bsddb3' == db :
return bsddb3_open ( file , flag , mode , **kwargs )
elif use_lmdb and 'lmdb' == db :
return LmdbDict ( path = file , flag = flag , **kwargs )
elif db in ( 'sqlite' , 'sqlite3' ) :
elif db in ( 'sqlite3' , 'sqlite' , 'sql' ) :
return SqliteDict ( filename = file , flag = flag , **kwargs )
elif 'std' == db or db.startswith ( 'dbm.' ) or db.endswith ( 'dbm' ) :

if db_gnu and db in ( 'dbm.gnu' , 'gdbm' ) :
if kwargs : logger.warning ( 'Ignore extra %d arguments:%s' % ( len ( kwargs ) , [ k for k in kwargs ] ) )
return db_gnu.open ( file , flag , mode )
elif db_dbm and db in ( 'dbm.ndbm' , 'dbm' ) :
if kwargs : logger.warning ( 'Ignore extra %d arguments:%s' % ( len ( kwargs ) , [ k for k in kwargs ] ) )
return db_dbm.open ( file , flag , mode )
elif db_hash and db in ( 'dbhash' , ) :
if kwargs : logger.warning ( 'Ignore extra %d arguments:%s' % ( len ( kwargs ) , [ k for k in kwargs ] ) )
return std_db.open ( file , flag , mode )
return db_hash.open ( file , flag , mode )
elif db in ( 'dbm.dumb' , 'dumbdbm' , 'dumb' ) :
if kwargs : logger.warning ( 'Ignore extra %d arguments:%s' % ( len ( kwargs ) , [ k for k in kwargs ] ) )
return db_dumb.open ( file , flag , mode )
elif db in ( 'std' , 'standard' ) or not db :
if kwargs : logger.warning ( 'Ignore extra %d arguments:%s' % ( len ( kwargs ) , [ k for k in kwargs ] ) )
return std_db.open ( file , flag , mode )

if db_types :
logger.warning ( 'DB-type hints not used: [%s]' % ( ','.join ( db for fn in db_types ) ) )


if concurrent and use_berkeleydb :
return berkeleydb_open ( file , flag , mode , **kwargs )

Expand All @@ -303,10 +355,12 @@ def dbopen ( file ,
if use_lmdb and check in ( 'lmdb' , ) :
return LmdbDict ( path = file , flag = flag , **kwargs )

if check == 'sqlite3' :
if check in ( 'sqlite3' , 'sqlite' ) :
return SqliteDict ( filename = file , flag = flag , **kwargs )

if kwargs : logger.warning ( 'Ignore extra %d arguments:%s' % ( len ( kwargs ) , [ k for k in kwargs ] ) )

## as a lasty resort - use the standard stuff
return std_db.open ( file , flag , mode )

# =============================================================================
Expand Down Expand Up @@ -410,7 +464,23 @@ def clean ( self ) :

from ostap.utils.docme import docme
docme ( __name__ , logger = logger )


logger.info ('Available DB-backends are:' )
if use_berkeleydb :
logger.info ( ' - BerkeleyDB : %s' % str ( berkeleydb ) )
if use_bsddb3 :
logger.info ( ' - BSDDB3 : %s' % str ( bsddb3 ) )
if use_lmdb :
logger.info ( ' - LMDB : %s' % str ( lmdb ) )
if db_gnu :
logger.info ( ' - GNU dbase : %s' % str ( db_gnu ) )
if db_dbm :
logger.info ( ' - NDBM : %s' % str ( db_dbm ) )
if db_dumb :
logger.info ( ' - DUMB : %s' % str ( db_dumb ) )
if db_hash :
logger.info ( ' - DBHASH : %s' % str ( db_hash ) )

# =============================================================================
## The END
# =============================================================================
Expand Down
25 changes: 15 additions & 10 deletions ostap/io/tests/test_io_shelves.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,22 +228,27 @@ def test_shelves2 () :
if zstshelve : shelves.append ( zstshelve )

backends = [
'lmdb' ,
'berkleydb' ,
'berkley' ,
'bdsdb2' ,
'sqlite' ,
'sqlite3' ,
''
'lmdb' ,
'berkeleydb' , 'berkeley' , 'berkeley-db' ,
'bsddb3' ,
'sqlite3' , 'sqlite' , 'sql' ,
#
'dbm.gnu' , 'gdbm' ,
'dbm.ndbm' , 'dbm' ,
'dbm.dumb' , 'dumbdbm' ,
'dbhash' ,
'standard' , 'std' ,
#
'' , None ,
]

for sh in shelves :
for b in backends :
with sh.tmpdb ( dbtype = b ) as db :

db ['one'] = 1
db ['two'] = 2
db.ls()
db [ 'one' ] = 1
db [ 'two' ] = 2
db.ls()

# =============================================================================
if '__main__' == __name__ :
Expand Down

0 comments on commit c6eeae3

Please sign in to comment.