From ec574389dbb8e076e0d8fdfef8d7b5df728db5ae Mon Sep 17 00:00:00 2001 From: Vanya Belyaev Date: Wed, 21 Aug 2024 07:33:58 +0200 Subject: [PATCH] add lmdb --- ostap/io/bz2shelve.py | 16 ++-- ostap/io/compress_shelve.py | 92 ++++++++++---------- ostap/io/dbase.py | 23 +++-- ostap/io/lmdbdict.py | 136 +++++++++++++++++++++--------- ostap/io/lzshelve.py | 31 +++---- ostap/io/tests/test_io_dbases.py | 8 +- ostap/io/tests/test_io_shelves.py | 56 +++++++++--- ostap/io/zipshelve.py | 34 ++++---- ostap/io/zstshelve.py | 20 ++--- ostap/utils/basic.py | 5 +- 10 files changed, 261 insertions(+), 160 deletions(-) diff --git a/ostap/io/bz2shelve.py b/ostap/io/bz2shelve.py index ef00bb27..c209014d 100644 --- a/ostap/io/bz2shelve.py +++ b/ostap/io/bz2shelve.py @@ -185,7 +185,7 @@ def __init__( compress = 9 , writeback = False , silent = False , - keyencoding = ENCODING ) : + keyencoding = ENCODING , **kwargs ) : ## save arguments for pickling.... self.__init_args = ( filename , @@ -205,7 +205,7 @@ def __init__( compress = compress , writeback = writeback , silent = silent , - keyencoding = keyencoding ) + keyencoding = keyencoding , **kwargs ) ## needed for proper (un)pickling def __getinitargs__ ( self ) : @@ -316,7 +316,7 @@ def open ( filename , compress = 9 , writeback = False , silent = True , - keyencoding = ENCODING ) : + keyencoding = ENCODING , **kwargs ) : """Open a persistent dictionary for reading and writing. @@ -337,7 +337,7 @@ def open ( filename , compress = compress , writeback = writeback , silent = silent , - keyencoding = keyencoding ) + keyencoding = keyencoding , **kwargs ) # ============================================================================= ## @class TmpBz2Shelf @@ -355,7 +355,7 @@ def __init__( self , silent = False , keyencoding = ENCODING , remove = True , - keep = False ) : + keep = False , **kwargs ) : ## initialize the base: generate the name TmpDB.__init__ ( self , suffix = '.bz2db' , remove = remove , keep = keep ) @@ -369,7 +369,7 @@ def __init__( self , compress = compress , writeback = False , ## writeback silent = silent , - keyencoding = keyencoding ) + keyencoding = keyencoding , **kwargs ) ## close and delete the file def close ( self ) : @@ -388,7 +388,7 @@ def tmpdb ( dbtype = '' , silent = True , keyencoding = ENCODING , remove = True , ## immediate remove - keep = False ) : ## keep it + keep = False , **kwargs ) : ## keep it """Open a TEMPORARY persistent dictionary for reading and writing. The optional protocol parameter specifies the @@ -402,7 +402,7 @@ def tmpdb ( dbtype = '' , silent = silent , keyencoding = keyencoding , remove = remove , - keep = keep ) + keep = keep , **kwargs ) # ============================================================================= if '__main__' == __name__ : diff --git a/ostap/io/compress_shelve.py b/ostap/io/compress_shelve.py index f1a9f93d..30f49152 100755 --- a/ostap/io/compress_shelve.py +++ b/ostap/io/compress_shelve.py @@ -61,6 +61,8 @@ from ostap.io.pickling import ( Pickler , Unpickler, BytesIO, PROTOCOL, HIGHEST_PROTOCOL, DEFAULT_PROTOCOL ) + +import pickle # ============================================================================= from ostap.logger.logger import getLogger if '__main__' == __name__ : logger = getLogger ( 'ostap.io.compress_shelve' ) @@ -125,7 +127,7 @@ def __init__( compress = 0 , writeback = False , silent = True , - keyencoding = ENCODING ) : + keyencoding = ENCODING , **kwargs ) : ## the mode mode = _modes_.get( mode.lower() , '' ) @@ -199,34 +201,35 @@ def __init__( afiles = tuple ( [ self.dbname + suffix for suffix in ( '' , ',db' , '.dir' , '.pag' , '.dat' ) ] ) ofiles = set ( [ i for i in glob.iglob ( self.dbname + '*' ) if i in afiles ] ) + - if 3 <= python_version.major : - shelve.Shelf.__init__ ( self , - dbopen ( self.dbname , flag = mode , dbtype = dbtype ) , - protocol = protocol , - writeback = writeback , - keyencoding = keyencoding ) - else : - - shelve.Shelf.__init__ ( self , - dbopen ( self.dbname , flag = mode , dbtype = dbtype ) , - protocol = protocol , - writeback = writeback ) - self.keyencoding = keyencoding - + self.__opened = False + + ## actual database + dbase = dbopen ( self.dbname , flag = mode , dbtype = dbtype , **kwargs ) + conf = { 'protocol' : protocol , 'writeback' : writeback } + + if 3 <= python_version.major : conf [ 'keyencoding'] = keyencoding + else : self.keyencoding = keyencoding + + shelve.Shelf.__init__ ( self , dbase , **conf ) + + self.__opened = True self.__mode = mode - self.sync () + ### self.sync () - self.__dbtype = whichdb ( self.dbname ) + self.__dbtype = whichdb ( self.dbname ) + if hasattr ( self.dict , 'reopen' ) : self.dict.reopen() + nfiles = set ( [ i for i in glob.iglob ( self.dbname + '*' ) if i in afiles ] ) - ofiles if not self.__files : files = [] f = self.dbname - db = self.dbtype + db = self.dbtype if os.path.exists ( f ) and os.path.isfile ( f ) and \ db in ( 'dbm.gnu' , 'gdbm' , 'dbhash' , 'bsddb185' , 'bsddb' , 'bsddb3' , 'sqlite3' , 'berkeleydb') : files.append ( f ) @@ -259,7 +262,8 @@ def __init__( if self.__compress is True : self.__compress = self.files self.__remove = self.files - + + if 'sqlite3' == self.dbtype and self.mode in ( 'w' , 'n' ) : write = True elif 'sqlite3' != self.dbtype and self.mode in ( 'c' , 'n' ) : write = True else : write = False @@ -274,13 +278,16 @@ def __init__( dct [ 'Pickle protocol' ] = protocol dct [ 'Compress level' ] = self.__compresslevel self [ '__metainfo__' ] = dct + if not self.silent : self.ls () ff = [ os.path.basename ( f ) for f in self.files ] ff = ff [0] if 1 == len ( ff ) else ff logger.info ( 'DB files are %s|%s' % ( ff, self.dbtype ) ) - + + self.sync () + @property def dbtype ( self ) : """`dbtype' : the underlying type of database""" @@ -356,7 +363,7 @@ def __del__ ( self ) : if self.opened : self.close () # ========================================================================= - ## Iterator over avilable keys (patterns included). + ## Iterator over available keys (patterns included). # Pattern matching is performed accoriding to # fnmatch/glob/shell rules (default) or regex # @code @@ -372,8 +379,7 @@ def ikeys ( self , pattern = '' , regex = False ) : >>> for k in db.ikeys('*MC*') : print(k) """ - keys_ = self.keys() - + if not pattern : good = lambda k : True elif regex : @@ -384,9 +390,9 @@ def ikeys ( self , pattern = '' , regex = False ) : import fnmatch good = lambda s : fnmatch.fnmatchcase ( k , pattern ) - keys_ = self.keys() - for k in sorted ( keys_ ) : - if good ( k ) : yield k + + for key in self.keys () : + if good ( key ) : yield key # ========================================================================= ## Build the table of content(patterns included). @@ -412,22 +418,19 @@ def table ( self , pattern = '' , load = True , prefix = '' ) : ap = os.path.abspath ( self.nominal_dbname ) self.sync() - fs = self.disk_size() + fs = float ( self.disk_size() ) - if fs <= 0 : size = "???" - elif fs < 1024 : size = str ( fs ) - elif fs < 1024 * 1024 : - size = '%.2fkB' % ( float ( fs ) / 1024 ) - elif fs < 1024 * 1024 * 1024 : - size = '%.2fMB' % ( float ( fs ) / ( 1024 * 1024 ) ) - else : - size = '%.2fGB' % ( float ( fs ) / ( 1024 * 1024 * 1024 ) ) + if fs <= 0 : size = "???" + elif fs < 1024 : size = '%3d' % int ( fs ) + elif fs < 1024 * 1024 : size = '%.2fkB' % ( fs / 1024 ) + elif fs < 1024 * 1024 * 1024 : size = '%.2fMB' % ( fs / ( 1024 * 1024 ) ) + else : size = '%.2fGB' % ( fs / ( 1024 * 1024 * 1024 ) ) keys = [] for k in self.ikeys ( pattern ): keys.append ( k ) keys.sort() - if keys : mlen = max ( [ len(k) for k in keys] ) + 2 + if keys : mlen = max ( [ len ( k ) for k in keys ] ) + 2 else : mlen = 2 fmt = ' --> %%-%ds : %%s' % mlen @@ -531,6 +534,9 @@ def ls ( self , pattern = '' , load = True , prefix = '# ' , logger = None ) def close ( self ) : """ Close the file (and compress it if required) """ + + print ('CLOSE', self.opened ) + if not self.opened : return ## if 'sqlite3' == self.dbtype and 'c' == self.mode : write = True @@ -616,11 +622,11 @@ def __in_place_uncompress ( self , filein ) : return tuple ( ofiles ) # ========================================================================= - ## some context manager functionality : enter + ## Context manager functionality : enter def __enter__ ( self ) : return self # ========================================================================= - ## some context manager functionality : exit + ## Context manager functionality : exit def __exit__ ( self , *_ ) : self.close () # ========================================================================= @@ -637,6 +643,7 @@ def __repr__ ( self ) : return "%s('%s')|%s: %d object(s)" % ( kname , self.dbname , self.dbtype , len ( self ) ) elif self : return "%s('%s')|%s: empty" % ( kname , self.dbname , self.dbtype ) + return "Invalid/Closed %s('%s')" % ( kname , self.dbname ) __str__ = __repr__ @@ -698,13 +705,8 @@ def __setitem__ ( self , key , value ) : """ item = Item ( time.time() , value ) if self.writeback : self.cache [ key ] = item - - + ## self.dict [ key.encode ( self.keyencoding ) ] = self.compress_item ( item ) - - ## kk = key.encode ( self.keyencoding ) - ## vv = self.compress_item ( item ) - ## self.dict [ kk ] = vv # ========================================================================= ## get the disk size of the db @@ -754,7 +756,7 @@ def dbase_name ( cls , files ) : exts = set ( [ os.path.splitext ( f )[1] for f in files ] ) - if 1 == len ( files ) and whichdb ( files[0] ) : return files[0] + if 1 == len ( files ) and whichdb ( files [ 0 ] ) : return files [ 0 ] elif 1 == len ( files ) and '.db' in exts : f , _ = os.path.splitext ( files [0] ) if whichdb ( f ) : return f diff --git a/ostap/io/dbase.py b/ostap/io/dbase.py index cca521df..c0a0ed18 100644 --- a/ostap/io/dbase.py +++ b/ostap/io/dbase.py @@ -257,27 +257,32 @@ def dbopen ( file , if 'c' in flag and '' == check : check = None if os.path.exists ( file ) and os.path.isfile ( file ) : os.unlink ( file ) - + + if 'n' == flag and file and os.path.exists ( file ) : + if os.path.isfile ( file ) : + try : os.unlink ( file ) + except : pass + elif os.path.isdir ( file ) : + try : shutil.rmtree ( file ) + except : pass + # 'n' flag is specified or dbase does not exist and c flag is specified if 'n' in flag or ( check is None and 'c' in flag ) : - if isinstance ( dbtype , str ) : db_types = [ dbtype ] - else : db_types = [ db for db in dbtype ] - - print ( 'PREFERENCE1' , db_types ) + if isinstance ( dbtype , str ) : db_types = [ dbtype.lower() ] + else : db_types = [ db.lower() for db in dbtype ] for db in db_types : - if use_berkeleydb and 'berkeleydb' == db : + + if use_berkeleydb and db in ( 'berkeleydb' , 'berkeley' ) : return berkeleydb_open ( file , flag , mode , **kwargs ) if use_bsddb3 and 'bdsdb3' == db : return bsddb3_open ( file , flag , mode , **kwargs ) if use_lmdb and 'lmdb' == db : return LmdbDict ( path = file , flag = flag , **kwargs ) - if 'sqlite3' == db : + if db in ( 'sqlite' , 'sqlite3' ) : return SqliteDict ( filename = file , flag = flag , **kwargs ) - print ( 'PREFERENCE1' , db_types ) - if concurrent and use_berkeleydb : return berkeleydb_open ( file , flag , mode , **kwargs ) diff --git a/ostap/io/lmdbdict.py b/ostap/io/lmdbdict.py index dee720ff..9643317f 100644 --- a/ostap/io/lmdbdict.py +++ b/ostap/io/lmdbdict.py @@ -52,7 +52,7 @@ 'LmdbDict' ) # ============================================================================= -import sys, os +import sys, os, shutil # ============================================================================= if ( 3 , 5 ) <= sys.version_info : from collections.abc import MutableMapping else : from collections import MutableMapping @@ -71,6 +71,32 @@ logger.warning ("No `lmdb` module is available!") pass +# ============================================================================= +def _qislmdb_ ( q , path ) : + """ Is it a lmdb subdir? + >>> ok = islmdb( 'mydbase' ) + """ + if not os.path.exists ( path ) : q.put ( False ) + elif not os.path.isdir ( path ) : q.put ( False ) + else : + + data = os.path.join ( path , 'data.mdb' ) + if not os.path.exists ( data ) : q.put ( False ) + elif not os.path.isfile ( data ) : q.put ( False ) + else : + try : + if lmdb : + db = lmdb.open ( path , + readonly = True , + create = False , + subdir = True , + max_readers = 127 , + max_dbs = 127 ) + db.close() + q.put ( True ) + except : + q.put ( True ) + # ============================================================================= ## Is it a lmdb subdir? # @code @@ -80,20 +106,39 @@ def islmdb ( path ) : """ Is it a lmdb subdir? >>> ok = islmdb( 'mydbase' ) """ - if not os.path.exists ( path ) : return False - if not os.path.isdir ( path ) : return False - data = os.path.join ( path , 'data.mdb') - if not os.path.exists ( data ) : return False - if not os.path.isfile ( data ) : return False - + if not os.path.exists ( path ) : return False + if not os.path.isdir ( path ) : return False + data = os.path.join ( path , 'data.mdb' ) + if not os.path.exists ( data ) : return False + if not os.path.isfile ( data ) : return False + + try : + import multiprocess as MP + except ImportError : + import multiprocessing as MP + + q = MP.Queue() + p = MP.Process( target = _qislmdb_ , args = ( q , path ) ) + p.start() + p.join() + return q.get() + try : if lmdb : - db = lmdb.open ( path , readonly = True , create = False ) + db = lmdb.open ( path , + readonly = True , + create = False , + subdir = True , + max_readers = 127 , + max_dbs = 127 ) + print ( 'WHICH/3' , db.info() ) + db.close() + print ( 'WHICH/4' , db ) return True - except: - pass + except : + return False - return True + return False # ============================================================================= ## @class LmdbDict @@ -140,32 +185,44 @@ def __init__ ( self , ## check presence of the lmdb module assert lmdb , "`lmdb` module is not available!" - conf = { 'map_size' : 2**24 , - 'map_async' : True , - 'max_dbs' : 1 , - 'mode' : 0o755 } + assert isinstance ( autogrow , int ) and 0 <= autogrow , "Invalid `autogrow` parameter: %s" % autogrow + self.__autogrow = min ( autogrow , 32 ) + + conf = { 'map_size' : 2**24 , + 'map_async' : True , + 'max_readers' : 127 , + 'max_dbs' : 127 , + 'mode' : 0o755 } conf.update ( kwargs ) - - if 'r' == flag : - ## Open existing database for reading only (default) - env = lmdb.open ( path , readonly = True , create = False , **conf ) - elif 'w' == flag : - ## Open existing database for reading and writing - env = lmdb.open ( path , readonly = False , create = False , **conf ) - elif 'c' == flag : - # Open database for reading and writing, creating it if it doesn't exist - env = lmdb.open ( path , readonly = False , create = True , **conf ) - elif 'n' == flag : - ## remove_lmdbm(file) - env = lmdb.open ( path , readonly = False , create = True , **conf ) - else : - raise ValueError ( "Invalid flag: %s" % flag ) - assert isinstance ( autogrow , int ) and 0 <= autogrow , "Invalid `autogrow` parameter: %s" % autogrow + if 'r' == flag : cnf = { 'readonly' : True , 'create' : False } + elif 'w' == flag : cnf = { 'readonly' : False , 'create' : False } + elif 'c' == flag : cnf = { 'readonly' : False , 'create' : True } + elif 'n' == flag : cnf = { 'readonly' : False , 'create' : True } + else : raise ValueError ( "Invalid flag: %s" % flag ) - self.__autogrow = min ( autogrow , 32 ) - self.__db = env + if 'n' == flag and path and os.path.exists ( path ) : + if os.path.isfile ( path ) : + try : os.unlink ( path ) + except : pass + elif os.path.isdir ( path ) : + try : shutil.rmtree ( path ) + except : pass + if os.path.exists ( path ) : + logger.warning ( 'path exists!' ) + + env = lmdb.open ( path , **conf ) + + self.__path = path + self.__conf = conf + self.__db = env + + # ========================================================================= + ## reopen the database + def reopen ( self ) : + if self.db : self.db.close() + self.__db = lmdb.open ( self.__path , **self.__conf ) @property def db ( self ) : @@ -230,18 +287,19 @@ def __setitem__ ( self , key , value ) : >>> db = ... >>> db [ key ] = value """ - nn = max ( 1 , self.autogrow ) - vv = self.encode_value ( value ) - for i in range ( nn ): + vv = self.encode_value ( value ) + ## + while True : try: with self.db.begin ( write = True ) as txn: - txn.put ( key , vv ) + txn.put ( key = key , value = vv , overwrite = True , append = False ) return except lmdb.MapFullError : if not self.autogrow : raise new_map_size = self.map_size * 2 self.map_size = new_map_size logger.info ( 'DBASE is resized to %s' % self.map_size ) + self.__autogrow = self.autogrow - 1 raise lmdb.MapFullError ("LmdbDict: Failure to autogrow!") @@ -333,7 +391,7 @@ def pop ( self, key , default = None ) : def __delitem__( self , key ) : """ Delete certain key from the database >>> db = ... - >>> value = db.pop ( key ) + >>> del db[ key ] """ with self.db.begin ( write = True ) as txn: txn.delete ( key ) @@ -355,6 +413,8 @@ def close ( self ) : >>> db.close() """ self.db.close () + del self.__db + self.__db = None # ========================================================================= ## context manager: ENTER diff --git a/ostap/io/lzshelve.py b/ostap/io/lzshelve.py index ce8e5de4..78becba6 100644 --- a/ostap/io/lzshelve.py +++ b/ostap/io/lzshelve.py @@ -191,16 +191,17 @@ def __init__( self , compress = lzma.PRESET_DEFAULT , writeback = False , silent = False , - keyencoding = 'utf-8' ) : + keyencoding = 'utf-8' , **kwargs ) : ## save arguments for pickling.... - self.__init_args = ( filename , - mode , - dbtype , - protocol , - compress , - writeback , - silent ) + self.__init_args = ( filename , + mode , + dbtype , + protocol , + compress , + writeback , + silent , + keyencoding ) ## initialize the base class CompressShelf.__init__ ( self , @@ -211,7 +212,7 @@ def __init__( self , compress = compress , writeback = writeback , silent = silent , - keyencoding = keyencoding ) + keyencoding = keyencoding , **kwargs ) ## needed for proper (un)pickling def __getinitargs__ ( self ) : @@ -323,7 +324,7 @@ def open ( filename , compress = lzma.PRESET_DEFAULT , writeback = False , silent = True , - keyencoding = ENCODING ) : + keyencoding = ENCODING , **kwargs ) : """Open a persistent dictionary for reading and writing. @@ -344,7 +345,7 @@ def open ( filename , compress = compress , writeback = writeback , silent = silent , - keyencoding = keyencoding ) + keyencoding = keyencoding , **kwargs ) # ============================================================================= ## @class TmpLzShelf @@ -362,7 +363,7 @@ def __init__( self , silent = False , keyencoding = ENCODING , remove = True , - keep = False ) : + keep = False , **kwargs ) : ## initialize the base: generate the name TmpDB.__init__ ( self , suffix = '.lzdb' , remove = remove , keep = keep ) @@ -376,7 +377,7 @@ def __init__( self , compress = compress , writeback = False , ## writeback silent = silent , - keyencoding = keyencoding ) + keyencoding = keyencoding , **kwargs ) ## close and delete the file def close ( self ) : @@ -395,7 +396,7 @@ def tmpdb ( dbtype = '' , silent = True , keyencoding = ENCODING , remove = True , ## immediate remove - keep = False ) : ## keep it + keep = False , **kwargs ) : ## keep it """Open a TEMPORARY persistent dictionary for reading and writing. The optional protocol parameter specifies the @@ -409,7 +410,7 @@ def tmpdb ( dbtype = '' , silent = silent , keyencoding = keyencoding , remove = remove , - keep = keep ) + keep = keep , **kwargs ) # ========================================================================== __all__ = ( diff --git a/ostap/io/tests/test_io_dbases.py b/ostap/io/tests/test_io_dbases.py index 63e14832..c1e32290 100644 --- a/ostap/io/tests/test_io_dbases.py +++ b/ostap/io/tests/test_io_dbases.py @@ -163,10 +163,10 @@ def test_dbases () : value = pickle.dumps ( datum ) db [ key ] = value - ##for i in range ( 1000 ) : - ## kk = the_name () - ## key = the_key ( kk ) - ## db [ key ] = pickle.dumps ( hh ) + for i in range ( 1000 ) : + kk = the_name () + key = the_key ( kk ) + db [ key ] = pickle.dumps ( hh ) if hasattr ( db , 'sync' ) : db.sync () db.close () diff --git a/ostap/io/tests/test_io_shelves.py b/ostap/io/tests/test_io_shelves.py index c063cc1e..4bf38fcc 100755 --- a/ostap/io/tests/test_io_shelves.py +++ b/ostap/io/tests/test_io_shelves.py @@ -44,7 +44,7 @@ if '__main__' == __name__ : logger = getLogger ( 'test_io_shelves' ) else : logger = getLogger ( __name__ ) # ============================================================================= -if (3,3) <= python_version : +if ( 3 , 3 ) <= python_version : import ostap.io.lzshelve as lzshelve else : lzshelve = None @@ -80,9 +80,11 @@ hh.Fill ( random.gauss ( 50 , 10) ) data['histos'][ht] = hh - # ============================================================================= -def test_shelves(): +def test_shelves1(): + + logger = getLogger ('Test shelves') + logger.info ( 'Test variosu shelves' ) db_sql_name = CU.CleanUp.tempfile ( suffix = '.sqldb' ) db_zip_name = CU.CleanUp.tempfile ( suffix = '.zipdb' ) @@ -119,14 +121,14 @@ def test_shelves(): db_root [ k ] = data[k] - logger.info('SQLiteShelve #keys: %s' % len ( list ( db_sql .keys() ) ) ) - logger.info('ZipShelve #keys: %s' % len ( db_zip .keys() ) ) - logger.info('Bz2Shelve #keys: %s' % len ( db_bz2 .keys() ) ) - logger.info('RootShelve #keys: %s' % len ( db_root.keys() ) ) + logger.info('SQLiteShelve #keys: %s' % len ( db_sql ) ) + logger.info('ZipShelve #keys: %s' % len ( db_zip ) ) + logger.info('Bz2Shelve #keys: %s' % len ( db_bz2 ) ) + logger.info('RootShelve #keys: %s' % len ( db_root ) ) if lzshelve : - logger.info('LzShelve #keys: %s' % len ( db_lz .keys() ) ) + logger.info('LzShelve #keys: %s' % len ( db_lz ) ) if zstshelve : - logger.info('ZstShelve #keys: %s' % len ( db_zst.keys() ) ) + logger.info('ZstShelve #keys: %s' % len ( db_zst ) ) db_sql .close() db_zip .close() @@ -283,11 +285,45 @@ def test_shelves(): db [ 'data' ] = data db [ 'histos'] = data['histos'] db.ls() + + +# ============================================================================= + +def test_shelves2 () : + + shelves = [ + zipshelve , + bz2shelve , + ] + if lzshelve : shelves.append ( lzshelve ) + if zstshelve : shelves.append ( zstshelve ) + + backends = [ + 'lmdb' , + 'berkleydb' , + 'berkley' , + 'bdsdb2' , + 'sqlite' , + 'sqlite3' , + '' + ] + + for sh in shelves : + + for b in backends : + db = sh.tmpdb ( dbtype = b ) + db ['one'] = 1 + db ['two'] = 2 + print ( 'DB:' , db ) + + + # ============================================================================= if '__main__' == __name__ : - test_shelves() + test_shelves1 () + test_shelves2 () # ============================================================================= ## The END diff --git a/ostap/io/zipshelve.py b/ostap/io/zipshelve.py index ef0c4856..a96db66f 100755 --- a/ostap/io/zipshelve.py +++ b/ostap/io/zipshelve.py @@ -169,16 +169,16 @@ class ZipShelf(CompressShelf): extensions = '.zip' , '.tgz' , '.gz' ## def __init__( - self , - filename , - mode = 'c' , - dbtype = '' , - protocol = PROTOCOL , - compress = zlib.Z_BEST_COMPRESSION , - writeback = False , - silent = False , - keyencoding = ENCODING ) : - + self , + filename , + mode = 'c' , + dbtype = '' , + protocol = PROTOCOL , + compress = zlib.Z_BEST_COMPRESSION , + writeback = False , + silent = False , + keyencoding = ENCODING , **kwargs ) : + ## save arguments for pickling.... self.__init_args = ( filename , mode , @@ -197,7 +197,7 @@ def __init__( compress = compress , writeback = writeback , silent = silent , - keyencoding = keyencoding ) + keyencoding = keyencoding , **kwargs ) ## needed for proper (un)pickling def __getinitargs__ ( self ) : @@ -325,7 +325,7 @@ def open ( filename , compress = zlib.Z_BEST_COMPRESSION , writeback = False , silent = True , - keyencoding = ENCODING ) : + keyencoding = ENCODING , **kwargs ) : """Open a persistent dictionary for reading and writing. @@ -346,7 +346,7 @@ def open ( filename , compress = compress , writeback = writeback , silent = silent , - keyencoding = keyencoding ) + keyencoding = keyencoding , **kwargs ) # ============================================================================= ## @class TmpZipShelf @@ -363,7 +363,7 @@ def __init__( self , silent = False , keyencoding = ENCODING , remove = True , ## immediate remove - keep = False ) : ## keep it + keep = False , **kwargs ) : ## keep it ## initialize the base: generate the name TmpDB .__init__ ( self , suffix = '.zdb' , remove = remove , keep = keep ) @@ -375,7 +375,7 @@ def __init__( self , compress = compress , writeback = False , silent = silent , - keyencoding = keyencoding ) + keyencoding = keyencoding , **kwargs ) ## close and delete the file def close ( self ) : @@ -393,7 +393,7 @@ def tmpdb ( dbtype = '' , silent = True , keyencoding = ENCODING , remove = True , ## immediate remove - keep = False ) : ## keep it + keep = False , **kwargs ) : ## keep it """ Open a TEMPORARY persistent dictionary for reading and writing. The optional protocol parameter specifies the @@ -407,7 +407,7 @@ def tmpdb ( dbtype = '' , silent = silent , keyencoding = keyencoding , remove = remove , - keep = keep ) + keep = keep , **kwargs ) # ============================================================================= if '__main__' == __name__ : diff --git a/ostap/io/zstshelve.py b/ostap/io/zstshelve.py index 0c414e84..24f24ca4 100644 --- a/ostap/io/zstshelve.py +++ b/ostap/io/zstshelve.py @@ -193,7 +193,7 @@ def __init__( self , writeback = False , silent = False , keyencoding = ENCODING , - threads = -1 ) : + threads = -1 , **kwargs ) : ## save arguments for pickling.... self.__init_args = ( filename , @@ -212,7 +212,6 @@ def __init__( self , write_content_size = True ) self.__decompressor = zst.ZstdDecompressor ( ) - ## initialize the base class CompressShelf.__init__ ( self , filename , @@ -222,7 +221,7 @@ def __init__( self , compress = compress , writeback = writeback , silent = silent , - keyencoding = keyencoding ) + keyencoding = keyencoding , **kwargs ) @property @@ -348,7 +347,7 @@ def open ( filename , writeback = False , silent = True , keyencoding = ENCODING , - threads = -1 ) : + threads = -1 , **kwargs ) : """Open a persistent dictionary for reading and writing. @@ -360,8 +359,7 @@ def open ( filename , version of the pickle protocol (0, 1, or 2). See the module's __doc__ string for an overview of the interface. - """ - + """ return ZstShelf ( filename , mode = mode , dbtype = dbtype , @@ -370,7 +368,7 @@ def open ( filename , writeback = writeback , silent = silent , keyencoding = keyencoding , - threads = threads ) + threads = threads , **kwargs ) # ============================================================================= ## @class TmpZstShelf @@ -389,7 +387,7 @@ def __init__( self , keyencoding = ENCODING , remove = True , keep = False , - threads = -1 ) : + threads = -1 , **kwargs ) : ## initialize the base: generate the name TmpDB.__init__ ( self , suffix = '.zstdb' , remove = remove , keep = keep ) @@ -404,7 +402,7 @@ def __init__( self , writeback = False , ## writeback silent = silent , keyencoding = keyencoding , - threads = threads ) + threads = threads , **kwargs ) ## close and delete the file def close ( self ) : @@ -424,7 +422,7 @@ def tmpdb ( dbtype = '' , keyencoding = ENCODING , remove = True , ## immediate remove keep = False , ## keep it - threads = -1 ) : + threads = -1 , **kwargs ) : """Open a TEMPORARY persistent dictionary for reading and writing. The optional protocol parameter specifies the @@ -439,7 +437,7 @@ def tmpdb ( dbtype = '' , keyencoding = keyencoding , remove = remove , keep = keep , - threads = threads ) + threads = threads , **kwargs ) # ========================================================================== __all__ = ( diff --git a/ostap/utils/basic.py b/ostap/utils/basic.py index 6e0b3dc3..737ca809 100644 --- a/ostap/utils/basic.py +++ b/ostap/utils/basic.py @@ -118,14 +118,12 @@ def make_dir ( bdir ) : >>> make_dir ( path ) """ try : - if bdir : os.mkdir ( bdir ) if os.path.exists ( bdir ) and os.path.isdir ( bdir ) : return os.path.abspath ( bdir) - except OSError : - + except OSError : pass return '' @@ -217,6 +215,7 @@ def make_dirs ( name , mode = 0o777 , exist_ok = False ): if not exist_ok or not os.path.isdir ( name ) : raise + # ============================================================================= ## @class NoContext # Fake empty context manager to be used as empty placeholder