From be78617ad6d000d0635365afd9cb01c7a758dee7 Mon Sep 17 00:00:00 2001 From: Vanya Belyaev Date: Thu, 22 Aug 2024 11:33:41 +0200 Subject: [PATCH] more unification&simplification for shelves --- .aux/test_with_lcg | 3 +- ostap/io/bz2shelve.py | 117 ++------ ostap/io/compress_shelve.py | 135 ++++++--- ostap/io/lzshelve.py | 394 ++++++++++---------------- ostap/io/pickling.py | 4 +- ostap/io/tests/test_io_shelves.py | 19 +- ostap/io/zipshelve.py | 187 ++----------- ostap/io/zstshelve.py | 442 ++++++++++++------------------ 8 files changed, 488 insertions(+), 813 deletions(-) diff --git a/.aux/test_with_lcg b/.aux/test_with_lcg index f871447b..fe3921ee 100755 --- a/.aux/test_with_lcg +++ b/.aux/test_with_lcg @@ -4,5 +4,4 @@ CMTCONFIG=$2 source /cvmfs/sft.cern.ch/lcg/views/${LCG}/${CMTCONFIG}/setup.sh source build/INSTALL/thisostap.sh cd build -ctest -N && cmake .. -DCMAKE_INSTALL_PREFIX=./INSTALL/ && ctest -j4 --output-on-failure - +ctest -N && cmake .. -DCMAKE_INSTALL_PREFIX=./INSTALL/ && ctest -j4 --output-on-failure -R _io_ --test-output-size-failed=5000000 diff --git a/ostap/io/bz2shelve.py b/ostap/io/bz2shelve.py index c209014d..93d835ad 100644 --- a/ostap/io/bz2shelve.py +++ b/ostap/io/bz2shelve.py @@ -146,8 +146,8 @@ from sys import version_info as python_version # ============================================================================= import os, sys, shelve, shutil -import bz2 ## use bz2 to compress DB-content -from ostap.io.compress_shelve import CompressShelf, ENCODING, PROTOCOL, HIGHEST_PROTOCOL +import bz2 ## use bz2 to compress DB-content +from ostap.io.compress_shelve import CompressShelf, HIGHEST_PROTOCOL from ostap.io.dbase import TmpDB # ============================================================================= from ostap.logger.logger import getLogger @@ -176,36 +176,18 @@ class Bz2Shelf(CompressShelf): ## the known "standard" extensions: extensions = '.tbz' , '.tbz2' , '.bz2' ## - def __init__( - self , - filename , - mode = 'c' , - dbtype = '' , - protocol = PROTOCOL , - compress = 9 , - writeback = False , - silent = False , - keyencoding = ENCODING , **kwargs ) : - - ## save arguments for pickling.... - self.__init_args = ( filename , - mode , - dbtype , - protocol , - compress , - writeback , - silent ) + def __init__( self , + dbname , + mode = 'c' , + compress = 9 , **kwargs ) : + assert 1 <= compress <= 9 , 'Invalid `compress` for `bz2`-compression: %s' % compress + ## initialize the base class - CompressShelf.__init__ ( self , - filename , - mode = mode , - dbtype = dbtype , - protocol = protocol , - compress = compress , - writeback = writeback , - silent = silent , - keyencoding = keyencoding , **kwargs ) + CompressShelf.__init__ ( self , + dbname , + mode = mode , + compress = compress , **kwargs ) ## needed for proper (un)pickling def __getinitargs__ ( self ) : @@ -279,45 +261,12 @@ def uncompress_item ( self , value ) : """ return self.unpickle ( bz2.decompress ( value ) ) - # ========================================================================= - ## clone the database into new one - # @code - # db = ... - # ndb = db.clone ( 'new_file.db' ) - # @endcode - def clone ( self , new_name , keys = () ) : - """ Clone the database into new one - >>> old_db = ... - >>> new_db = new_db.clone ( 'new_file.db' ) - """ - new_db = Bz2Shelf ( new_name , - mode = 'c' , - dbtype = self.dbtype , - protocol = self.protocol , - compress = self.compresslevel , - writeback = self.writeback , - silent = self.silent , - keyencoding = self.keyencoding ) - - ## copy the content - copy = keys if keys else self.keys() - for key in copy : new_db [ key ] = self [ key ] - new_db.sync () - return new_db # ============================================================================= ## helper function to access Bz2Shelve data base # @author Vanya BELYAEV Ivan.Belyaev@cern.ch # @date 2010-04-30 -def open ( filename , - mode = 'c' , - dbtype = '' , - protocol = PROTOCOL , - compress = 9 , - writeback = False , - silent = True , - keyencoding = ENCODING , **kwargs ) : - +def open ( dbname , mode = 'c' , **kwargs ) : """Open a persistent dictionary for reading and writing. The filename parameter is the base filename for the underlying @@ -330,14 +279,7 @@ def open ( filename , See the module's __doc__ string for an overview of the interface. """ - return Bz2Shelf ( filename , - mode = mode , - dbtype = dbtype , - protocol = protocol , - compress = compress , - writeback = writeback , - silent = silent , - keyencoding = keyencoding , **kwargs ) + return Bz2Shelf ( dbname , mode = mode , **kwargs ) # ============================================================================= ## @class TmpBz2Shelf @@ -349,11 +291,7 @@ class TmpBz2Shelf(Bz2Shelf,TmpDB): TEMPORARY ``bzip2''-version of ``shelve''-database """ def __init__( self , - dbtype = '' , protocol = HIGHEST_PROTOCOL , - compress = 9 , - silent = False , - keyencoding = ENCODING , remove = True , keep = False , **kwargs ) : @@ -363,13 +301,12 @@ def __init__( self , ## open DB Bz2Shelf.__init__ ( self , self.tmp_name , - dbtype = dbtype , mode = 'c' , protocol = protocol , - compress = compress , - writeback = False , ## writeback - silent = silent , - keyencoding = keyencoding , **kwargs ) + writeback = False , **kwargs ) + + conf = { 'remove' : remove , 'keep' : keep } + self.kwargs.update ( conf ) ## close and delete the file def close ( self ) : @@ -382,27 +319,15 @@ def close ( self ) : ## helper function to open TEMPORARY ZipShelve data base# # @author Vanya BELYAEV Ivan.Belyaev@cern.ch # @date 2010-04-30 -def tmpdb ( dbtype = '' , - protocol = HIGHEST_PROTOCOL , - compress = 9 , - silent = True , - keyencoding = ENCODING , - remove = True , ## immediate remove - keep = False , **kwargs ) : ## keep it - """Open a TEMPORARY persistent dictionary for reading and writing. +def tmpdb ( **kwargs ) : + """ Open a TEMPORARY persistent dictionary for reading and writing. The optional protocol parameter specifies the version of the pickle protocol (0, 1, or 2). See the module's __doc__ string for an overview of the interface. """ - return TmpBz2Shelf ( dbtype = dbtype , - protocol = protocol , - compress = compress , - silent = silent , - keyencoding = keyencoding , - remove = remove , - keep = keep , **kwargs ) + return TmpBz2Shelf ( **kwargs ) # ============================================================================= if '__main__' == __name__ : diff --git a/ostap/io/compress_shelve.py b/ostap/io/compress_shelve.py index ce294687..acd88617 100755 --- a/ostap/io/compress_shelve.py +++ b/ostap/io/compress_shelve.py @@ -54,7 +54,7 @@ 'ENCODING' ) # ============================================================================= -import os, abc, shelve, shutil, glob, time, datetime +import os, abc, shelve, shutil, glob, time, datetime, zipfile, tarfile from sys import version_info as python_version from ostap.io.dbase import dbopen , whichdb, Item, ordered_dict from ostap.core.meta_info import meta_info @@ -119,9 +119,9 @@ def __init__( self , dbname , mode = 'c' , - dbtype = '' , ## preferred data base - protocol = PROTOCOL , ## pickle protocol compress = 0 , + protocol = PROTOCOL , ## pickle protocol + dbtype = '' , ## preferred data base type writeback = False , silent = True , keyencoding = ENCODING , **kwargs ) : @@ -133,7 +133,7 @@ def __init__( mode = 'c' if not 0 <= protocol <= HIGHEST_PROTOCOL : - logger.warning ("Invalid pickle protocol:%s" % protocol ) + logger.warning ("Invalid pickle protocol:%s, replace with:%s" % ( protocol , PROTOCOL ) ) protocol = PROTOCOL ## expand the actual file name @@ -142,19 +142,36 @@ def __init__( dbname = os.path.expandvars ( dbname ) dbname = os.path.expandvars ( dbname ) - self.__compresslevel = compress + self.__compresslevel = compress + self.__silent = silent + self.__protocol = protocol + self.__dbtype = dbtype + + ## all arguments for constructor + self.__kwargs = { + 'dbname' : dbname , + 'mode' : mode , + 'compress' : self.compresslevel , + 'protocol' : protocol , ## from shelve.Shelf + 'writeback' : writeback , ## from shelf.Shelf + 'dbtype' : self.dbtype , ## preferred dbtype + 'silent' : self.silent , + 'keyencoding' : keyencoding , + } + self.__kwargs.update ( kwargs ) + + self.__nominal_dbname = dbname - self.__actual_dbname = dbname + self.__actual_dbname = dbname + self.__compress = () self.__remove = () - self.__silent = silent + self.__opened = False self.__files = () - self.__dbtype = dbtype - if not self.__silent : - logger.info ( 'Open DB: %s' % dbname ) + if not self.__silent : logger.info ( 'Open DB: %s' % dbname ) ## filename without extension and the extension itself fname , ext = os.path.splitext ( dbname ) @@ -216,8 +233,8 @@ def __init__( ### self.sync () - self.__dbtype = whichdb ( self.dbname ) - if hasattr ( self.dict , 'reopen' ) : self.dict.reopen() + self.__dbtype = whichdb ( self.dbname ) ## actual dbtype + ## if hasattr ( self.dict , 'reopen' ) : self.dict.reopen() nfiles = set ( [ i for i in glob.iglob ( self.dbname + '*' ) if i in afiles ] ) - ofiles @@ -277,7 +294,8 @@ def __init__( logger.info ( 'DB files are %s|%s' % ( ff, self.dbtype ) ) self.sync () - + + @property def dbtype ( self ) : """`dbtype' : the underlying type of database""" @@ -286,7 +304,7 @@ def dbtype ( self ) : @property def protocol ( self ) : """`protocol' : pickling protocol used in the shelve""" - return self._protocol + return self.__protocol @property def compression ( self ) : @@ -326,13 +344,18 @@ def silent ( self ) : @property def protocol( self ) : "`protocol' : pickling protocol" - return self._protocol + return self.__protocol @property def files ( self ) : """`files' : the files assocated with the database""" return self.__files + @property + def kwargs ( self ) : + """`kwargs` : all constructor arguments""" + return self.__kwargs + # ========================================================================= ## valid, opened DB def __nonzero__ ( self ) : @@ -370,8 +393,7 @@ def ikeys ( self , pattern = '' , regex = False ) : """ - if not pattern : - good = lambda k : True + if not pattern : good = lambda k : True elif regex : import re re_cmp = re.compile ( pattern ) @@ -380,7 +402,6 @@ def ikeys ( self , pattern = '' , regex = False ) : import fnmatch good = lambda s : fnmatch.fnmatchcase ( k , pattern ) - for key in self.keys () : if good ( key ) : yield key @@ -525,8 +546,6 @@ def close ( self ) : """ Close the file (and compress it if required) """ - print ('CLOSE', self.opened ) - if not self.opened : return ## if 'r' != self.mode and 'n' != self.mode : @@ -541,7 +560,8 @@ def close ( self ) : if not self.silent : self.ls () shelve.Shelf.close ( self ) - self.__opened = False + self.__opened = False + ## if self.__compress : self.__in_place_compress ( self.__compress ) @@ -557,7 +577,7 @@ def close ( self ) : # ========================================================================= ## compress the files (`in-place') def __in_place_compress ( self , files ) : - """Compress the file `in-place' + """ Compress the file `in-place' - It is better to use here `os.system' or `popen'-family, but it does not work properly for multiprocessing environemnt """ @@ -581,7 +601,7 @@ def __in_place_compress ( self , files ) : # ========================================================================= ## uncompress the file (`in-place') def __in_place_uncompress ( self , filein ) : - """Uncompress the file `in-place' + """ Uncompress the file `in-place' - It is better to use here `os.system' or `popen'-family, but unfortunately it does not work properly for multithreaded environemnt """ @@ -715,7 +735,7 @@ def disk_size ( self ) : ## Create the temporary directory # The directory will be cleaned-up and deleted at-exit. @classmethod - def tempdir ( cls , suffix = '=db-dir' , prefix = 'ostap-compress-shelve-dir-' , date = True ) : + def tempdir ( cls , suffix = '-db-dir' , prefix = 'ostap-compress-shelve-dir-' , date = True ) : """ Create the temporary directory The directory will be cleaned-up and deleted at-exit. """ @@ -795,24 +815,75 @@ def compress_files ( self , files ) : # ========================================================================= ## Uncompress the file into temporary location, keep original - @abc.abstractmethod def uncompress_file ( self , filein ) : - """Uncompress the file into temporary location, keep the original""" - return NotImplemented + """ Uncompress the file into temporary location, keep the original """ + assert os.path.exists ( filein ) and os.path.isfile ( filein ) , \ + "Non existing/invalid file: %s" % filein + + items = [] + tmpdir = self.tempdir () + + ## 1) zip-archive ? + if zipfile.is_zipfile ( filein ) : + with zipfile.ZipFile ( filein , 'r' , allowZip64 = True ) as zfile : + for item in zfile.filelist : + zfile.extract ( item , path = tmpdir ) + items.append ( os.path.join ( tmpdir , item.filename ) ) + items.sort() + return tuple ( items ) + + ## 2) compressed-tar archive ? + if tarfile.is_tarfile ( filein ) : + with tarfile.open ( filein , 'r:*' ) as tfile : + for item in tfile : + tfile.extract ( item , path = tmpdir ) + items.append ( os.path.join ( tmpdir , item.name ) ) + items.sort() + return tuple ( items ) + + return None + # ========================================================================= - ## clone the database into new one + ## copy the database into new one # @code # db = ... - # ndb = db.clone ( 'new_file.db' ) + # ndb = db.copy ( 'new_file.db' , copykeys , **kwargs ) # @endcode - @abc.abstractmethod - def clone ( self , filename , keys = () ) : + def copy ( self , dbname , copykeys = () , **kwargs ) : """ Clone the database into new one >>> old_db = ... >>> new_db = new_db.clone ( 'new_file.db' ) """ - return NotImplemented + klass = type ( self ) + conf = {} + conf.update ( self.__kwargs ) ## use argument + conf.update ( kwargs ) ## redefine them + + conf [ 'mode' ] = 'c' + conf [ 'dbname' ] = dbname + + ## create newdb + newdb = klass ( **conf ) + + ## copy the required keys: + for key in copykeys : newdb [ key ] = self [ key ] + + newdb.sync() + return newdb + + # ========================================================================= + ## clone the database into new one (copy all keys) + # @code + # db = ... + # ndb = db.clone ( 'new_file.db' ) + # @endcode + def clone ( self , dbname , **kwargs ) : + """ Clone the database into new one (copy all keys) + >>> old_db = ... + >>> new_db = new_db.clone ( 'new_file.db' ) + """ + return self.copy ( dbname , copykeys = self.keys () , **kwargs ) # ============================================================================ ## a bit more decorations for shelve (optional) diff --git a/ostap/io/lzshelve.py b/ostap/io/lzshelve.py index 9cf40312..d27ebfe4 100644 --- a/ostap/io/lzshelve.py +++ b/ostap/io/lzshelve.py @@ -137,12 +137,17 @@ __date__ = "2010-04-30" __version__ = "$Revision:$" # ============================================================================= -__all__ = () +__all__ = ( + 'LzShelf' , ## database + 'TMpLzShelf' , ## database + 'open' , ## open the database + 'tmpdb' , ## open the temporary database +) # ============================================================================= from sys import version_info as python_version # ============================================================================= import os, sys, shelve, shutil -from ostap.io.compress_shelve import CompressShelf, ENCODING, PROTOCOL, HIGHEST_PROTOCOL +from ostap.io.compress_shelve import CompressShelf, HIGHEST_PROTOCOL from ostap.io.dbase import TmpDB # ============================================================================= from ostap.logger.logger import getLogger @@ -160,264 +165,165 @@ else : logger.error ( 'lzshelve is disabled for python %s' % str ( python_version ) ) lzma = None + # ============================================================================= -if lzma : - # ============================================================================= - ## @class LzShelf - # ``LZMA''-version of ``shelve''-database - # Modes: - # - 'r' Open existing database for reading only - # - 'w' Open existing database for reading and writing - # - 'c' Open database for reading and writing, creating if it does not exist (default) - # - 'n' Always create a new, empty database, open for reading and writing - # @author Vanya BELYAEV Ivan.Belyaev@cern.ch - # @date 2010-04-30 - class LzShelf(CompressShelf): - """LZMA-version of ``shelve''-database - Modes: - - 'r' Open existing database for reading only - - 'w' Open existing database for reading and writing - - 'c' Open database for reading and writing, creating if it does not exist - - 'n' Always create a new, empty database, open for reading and writing - """ - ## the known "standard" extensions: - extensions = '.txz', '.tlz' , '.xz' , '.lz' , '.lzma' - ## - def __init__( self , - filename , - mode = 'c' , - dbtype = '' , - protocol = PROTOCOL , - compress = lzma.PRESET_DEFAULT , - writeback = False , - silent = False , - keyencoding = 'utf-8' , **kwargs ) : - - ## save arguments for pickling.... - self.__init_args = ( filename , - mode , - dbtype , - protocol , - compress , - writeback , - silent , - keyencoding ) - - ## initialize the base class - CompressShelf.__init__ ( self , - filename , - mode = mode , - dbtype = dbtype , - protocol = protocol , - compress = compress , - writeback = writeback , - silent = silent , - keyencoding = keyencoding , **kwargs ) - - ## needed for proper (un)pickling - def __getinitargs__ ( self ) : - """for proper (un_pickling""" - return self.__init_args +## @class LzShelf +# ``LZMA''-version of ``shelve''-database +# Modes: +# - 'r' Open existing database for reading only +# - 'w' Open existing database for reading and writing +# - 'c' Open database for reading and writing, creating if it does not exist (default) +# - 'n' Always create a new, empty database, open for reading and writing +# @author Vanya BELYAEV Ivan.Belyaev@cern.ch +# @date 2010-04-30 +class LzShelf(CompressShelf): + """ LZMA-version of ``shelve''-database + Modes: + - 'r' Open existing database for reading only + - 'w' Open existing database for reading and writing + - 'c' Open database for reading and writing, creating if it does not exist + - 'n' Always create a new, empty database, open for reading and writing + """ + ## the known "standard" extensions: + extensions = '.txz', '.tlz' , '.xz' , '.lz' , '.lzma' + ## + def __init__( self , + dbname , + mode = 'c' , + compress = lzma.PRESET_DEFAULT , **kwargs ) : + + assert lzma, "`lzma` module is not available!" - ## needed for proper (un)pickling - def __getstate__ ( self ) : - """for proper (un)pickling""" - self.sync() - return {} + ## initialize the base class + CompressShelf.__init__ ( self , + dbname , + mode = mode , + compress = compress , **kwargs ) - ## needed for proper (un)pickling - def __setstate__ ( self , dct ) : - """for proper (un)pickling""" - pass + # ========================================================================= + ## compress (LZMA) the file into temporary location, keep original + def compress_files ( self , files ) : + """ Compress (LZMA) the file into temporary location, keep original + """ + output = self.tempfile() - # ========================================================================= - ## compress (LZMA) the file into temporary location, keep original - def compress_files ( self , files ) : - """Compress (LZMA) the file into temporary location, keep original - """ - output = self.tempfile() - - import tarfile - with tarfile.open ( output , 'x:xz' ) as tfile : - for file in files : - _ , name = os.path.split ( file ) - tfile.add ( file , name ) - return output - - # ========================================================================= - ## uncompress (LZMA) the file into temporary location, keep original - def uncompress_file ( self , filein ) : - """Uncompress (LZMA) the file into temporary location, keep original - """ + import tarfile + with tarfile.open ( output , 'x:xz' ) as tfile : + for file in files : + _ , name = os.path.split ( file ) + tfile.add ( file , name ) + return output - items = [] - tmpdir = self.tempdir () - - ## 1) try compressed-tarfile - import tarfile - if tarfile.is_tarfile ( filein ) : - with tarfile.open ( filein , 'r:*' ) as tfile : - for item in tfile : - tfile.extract ( item , path = tmpdir ) - items.append ( os.path.join ( tmpdir , item.name ) ) - items.sort() - return tuple ( items ) - - ## 2) try compressed file - import tempfile , io - fd , fileout = tempfile.mkstemp ( prefix = 'ostap-tmp-' , suffix = '-lzdb' ) - with lzma.open ( filein , 'rb' ) as fin : - with io.open ( fileout , 'wb' ) as fout : - shutil.copyfileobj ( fin , fout ) - return fileout , + # ========================================================================= + ## uncompress (LZMA) the file into temporary location, keep original + def uncompress_file ( self , filein ) : + """ Uncompress (LZMA) the file into temporary location, keep original + """ + + items = [] + tmpdir = self.tempdir () + + ## 1) try compressed-tarfile + import tarfile + if tarfile.is_tarfile ( filein ) : + with tarfile.open ( filein , 'r:*' ) as tfile : + for item in tfile : + tfile.extract ( item , path = tmpdir ) + items.append ( os.path.join ( tmpdir , item.name ) ) + items.sort() + return tuple ( items ) - # ========================================================================== - ## compress (LZMA) the item using lzma.compress - def compress_item ( self , value ) : - """Compress (LZMA) the item using ``bz2.compress'' - - see lzma.compress - """ - return lzma.compress ( self.pickle ( value ) , preset = self.compresslevel ) + ## 2) try compressed file + import tempfile , io + fd , fileout = tempfile.mkstemp ( prefix = 'ostap-tmp-' , suffix = '-lzdb' ) + with lzma.open ( filein , 'rb' ) as fin : + with io.open ( fileout , 'wb' ) as fout : + shutil.copyfileobj ( fin , fout ) + return fileout , + + # ========================================================================== + ## compress (LZMA) the item using lzma.compress + def compress_item ( self , value ) : + """ Compress (LZMA) the item using ``bz2.compress'' + - see lzma.compress + """ + return lzma.compress ( self.pickle ( value ) , preset = self.compresslevel ) - # ========================================================================= - ## uncompres (LZMA) the item using lzma.decompress - def uncompress_item ( self , value ) : - """Uncompress (LZMA) the item using ``lzma.decompress'' - - see lzma.decompress - """ - return self.unpickle ( lzma.decompress ( value ) ) + # ========================================================================= + ## uncompres (LZMA) the item using lzma.decompress + def uncompress_item ( self , value ) : + """ Uncompress (LZMA) the item using ``lzma.decompress'' + - see lzma.decompress + """ + return self.unpickle ( lzma.decompress ( value ) ) - # ========================================================================= - ## clone the database into new one - # @code - # db = ... - # ndb = db.clone ( 'new_file.db' ) - # @endcode - def clone ( self , new_name , keys = () ) : - """ Clone the database into new one - >>> old_db = ... - >>> new_db = new_db.clone ( 'new_file.db' ) - """ - new_db = LzShelf ( new_name , - mode = 'c' , - dbtype = self.dbtype , - protocol = self.protocol , - compress = self.compresslevel , - writeback = self.writeback , - silent = self.silent , - keyencoding = self.keyencoding ) - - ## copy the content - copy = keys if keys else self.keys() - for key in copy : new_db [ key ] = self [ key ] - new_db.sync () - return new_db - - # ============================================================================= - ## helper function to access LzShelve data base - # @author Vanya BELYAEV Ivan.Belyaev@cern.ch - # @date 2010-04-30 - def open ( filename , - mode = 'c' , - dbtype = '' , - protocol = PROTOCOL , - compress = lzma.PRESET_DEFAULT , - writeback = False , - silent = True , - keyencoding = ENCODING , **kwargs ) : +# ============================================================================= +## helper function to access LzShelve data base +# @author Vanya BELYAEV Ivan.Belyaev@cern.ch +# @date 2010-04-30 +def open ( dbname , mode = 'c' , **kwargs ) : + """ Open a persistent dictionary for reading and writing. - """Open a persistent dictionary for reading and writing. + The filename parameter is the base filename for the underlying + database. As a side-effect, an extension may be added to the + filename and more than one file may be created. The optional flag + parameter has the same interpretation as the flag parameter of + anydbm.open(). The optional protocol parameter specifies the + version of the pickle protocol (0, 1, or 2). + + See the module's __doc__ string for an overview of the interface. + """ + return LzShelf ( dbname , mode = mode , **kwargs ) + +# ============================================================================= +## @class TmpLzShelf +# TEMPORARY lzma-version of ``shelve''-database +# @author Vanya BELYAEV Ivan.Belyaev@cern.ch +# @date 2015-10-31 +class TmpLzShelf(LzShelf,TmpDB): + """ TEMPORARY ``LZMA''-version of ``shelve''-database + """ + def __init__( self , + protocol = HIGHEST_PROTOCOL , + remove = True , + keep = False , **kwargs ) : + + ## initialize the base: generate the name + TmpDB.__init__ ( self , suffix = '.lzdb' , remove = remove , keep = keep ) - The filename parameter is the base filename for the underlying - database. As a side-effect, an extension may be added to the - filename and more than one file may be created. The optional flag - parameter has the same interpretation as the flag parameter of - anydbm.open(). The optional protocol parameter specifies the - version of the pickle protocol (0, 1, or 2). + ## open DB + LzShelf.__init__ ( self , + self.tmp_name , + mode = 'c' , + protocol = protocol , + writeback = False , **kwargs ) - See the module's __doc__ string for an overview of the interface. - """ + conf = { 'remove' : remove , 'keep' : keep } + self.kwargs.update ( conf ) - return LzShelf ( filename , - mode = mode , - dbtype = dbtype , - protocol = protocol , - compress = compress , - writeback = writeback , - silent = silent , - keyencoding = keyencoding , **kwargs ) - - # ============================================================================= - ## @class TmpLzShelf - # TEMPORARY lzma-version of ``shelve''-database - # @author Vanya BELYAEV Ivan.Belyaev@cern.ch - # @date 2015-10-31 - class TmpLzShelf(LzShelf,TmpDB): - """ - TEMPORARY ``LZMA''-version of ``shelve''-database - """ - def __init__( self , - dbtype = '' , - protocol = HIGHEST_PROTOCOL , - compress = lzma.PRESET_DEFAULT , - silent = False , - keyencoding = ENCODING , - remove = True , - keep = False , **kwargs ) : - - ## initialize the base: generate the name - TmpDB.__init__ ( self , suffix = '.lzdb' , remove = remove , keep = keep ) - - ## open DB - LzShelf.__init__ ( self , - self.tmp_name , - mode = 'c' , - dbtype = dbtype , - protocol = protocol , - compress = compress , - writeback = False , ## writeback - silent = silent , - keyencoding = keyencoding , **kwargs ) + ## close and delete the file + def close ( self ) : + ## close the shelve file + LzShelf.close ( self ) + ## delete the file + TmpDB .clean ( self ) - ## close and delete the file - def close ( self ) : - ## close the shelve file - LzShelf.close ( self ) - ## delete the file - TmpDB .clean ( self ) - - # ============================================================================= - ## helper function to open TEMPORARY ZipShelve data base# - # @author Vanya BELYAEV Ivan.Belyaev@cern.ch - # @date 2010-04-30 - def tmpdb ( dbtype = '' , - protocol = HIGHEST_PROTOCOL , - compress = lzma.PRESET_DEFAULT , - silent = True , - keyencoding = ENCODING , - remove = True , ## immediate remove - keep = False , **kwargs ) : ## keep it - """Open a TEMPORARY persistent dictionary for reading and writing. +# ============================================================================= +## helper function to open TEMPORARY ZipShelve data base# +# @author Vanya BELYAEV Ivan.Belyaev@cern.ch +# @date 2010-04-30 +def tmpdb ( **kwargs ) : + """ Open a TEMPORARY persistent dictionary for reading and writing. - The optional protocol parameter specifies the - version of the pickle protocol (0, 1, or 2). + The optional protocol parameter specifies the + version of the pickle protocol (0, 1, or 2). - See the module's __doc__ string for an overview of the interface. - """ - return TmpLzShelf ( dbtype = dbtype , - protocol = protocol , - compress = compress , - silent = silent , - keyencoding = keyencoding , - remove = remove , - keep = keep , **kwargs ) + See the module's __doc__ string for an overview of the interface. + """ + return TmpLzShelf ( **kwargs ) - # ========================================================================== - __all__ = ( - 'LzShelf' , ## database - 'open' , ## open the database - 'tmpdb' , ## open the temporary database - ) +# ========================================================================== +if not lzma : __all__ = () # ============================================================================= if '__main__' == __name__ : diff --git a/ostap/io/pickling.py b/ostap/io/pickling.py index 0f8cb03d..3e5a9669 100644 --- a/ostap/io/pickling.py +++ b/ostap/io/pickling.py @@ -30,11 +30,11 @@ if '__main__' == __name__ : logger = getLogger ( 'ostap.io.pickling' ) else : logger = getLogger ( __name__ ) # ============================================================================= -if (3, 0 ) <= sys.version_info : +if ( 3 , 0 ) <= sys.version_info : from pickle import ( Pickler, Unpickler, DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, dumps, loads, - PicklingError, UnpicklingError ) + PicklingError, UnpicklingError ) else : DEFAULT_PROTOCOL = 2 try: diff --git a/ostap/io/tests/test_io_shelves.py b/ostap/io/tests/test_io_shelves.py index 6325890e..719d6ae2 100755 --- a/ostap/io/tests/test_io_shelves.py +++ b/ostap/io/tests/test_io_shelves.py @@ -43,11 +43,13 @@ # ============================================================================= if ( 3 , 3 ) <= python_version : import ostap.io.lzshelve as lzshelve + if not lzshelve.lzma : lzshelve = None else : lzshelve = None # ============================================================================= if ( 3 , 6 ) <= python_version : import ostap.io.zstshelve as zstshelve + if not zstshelve.zst : zstshelve = None try : import zstandard except ImportError : @@ -304,16 +306,13 @@ def test_shelves2 () : '' ] - for sh in shelves : - - for b in backends : - db = sh.tmpdb ( dbtype = b ) - db ['one'] = 1 - db ['two'] = 2 - print ( 'DB:' , db ) - - - + for sh in shelves : + for b in backends : + with sh.tmpdb ( dbtype = b ) as db : + + db ['one'] = 1 + db ['two'] = 2 + db.ls() # ============================================================================= if '__main__' == __name__ : diff --git a/ostap/io/zipshelve.py b/ostap/io/zipshelve.py index a96db66f..c252911e 100755 --- a/ostap/io/zipshelve.py +++ b/ostap/io/zipshelve.py @@ -145,7 +145,7 @@ # ============================================================================= import os, sys, shelve, shutil import zlib ## use zlib to compress DB-content -from ostap.io.compress_shelve import CompressShelf, ENCODING, PROTOCOL, HIGHEST_PROTOCOL +from ostap.io.compress_shelve import CompressShelf, HIGHEST_PROTOCOL from ostap.io.dbase import TmpDB # ============================================================================= ## @class ZipShelf @@ -169,52 +169,21 @@ class ZipShelf(CompressShelf): extensions = '.zip' , '.tgz' , '.gz' ## def __init__( - self , - filename , - mode = 'c' , - dbtype = '' , - protocol = PROTOCOL , - compress = zlib.Z_BEST_COMPRESSION , - writeback = False , - silent = False , - keyencoding = ENCODING , **kwargs ) : - - ## save arguments for pickling.... - self.__init_args = ( filename , - mode , - dbtype , - protocol , - compress , - writeback , - silent ) + self , + dbname , + mode = 'c' , + compress = zlib.Z_DEFAULT_COMPRESSION , **kwargs ) : + + assert 0 <= compress <= zlib.Z_BEST_COMPRESSION or \ + compress in ( -1 , zlib.Z_DEFAULT_COMPRESSION ) ,\ + "Invalid `compress` for `zlib`: %s" % compress ## initialize the base class - CompressShelf.__init__ ( self , - filename , - mode = mode , - dbtype = dbtype , - protocol = protocol , - compress = compress , - writeback = writeback , - silent = silent , - keyencoding = keyencoding , **kwargs ) + CompressShelf.__init__ ( self , + dbname , + mode = mode , + compress = compress , **kwargs ) - ## needed for proper (un)pickling - def __getinitargs__ ( self ) : - """for proper (un_pickling""" - return self.__init_args - - ## needed for proper (un)pickling - def __getstate__ ( self ) : - """for proper (un)pickling""" - self.sync() - return {} - - ## needed for proper (un)pickling - def __setstate__ ( self , dct ) : - """for proper (un)pickling""" - pass - # ========================================================================= ## compress the file into temporary location, keep original def compress_files ( self , files ) : @@ -223,55 +192,13 @@ def compress_files ( self , files ) : output = self.tempfile () import zipfile - with zipfile.ZipFile( output , 'w' , allowZip64 = True ) as zfile : + with zipfile.ZipFile ( output , 'w' , allowZip64 = True ) as zfile : for file in files : _ , name = os.path.split ( file ) zfile.write ( file , name ) return output - # ========================================================================= - ## uncompress (gunzip) the file into temporary location, keep original - # @code - # db = ... - # files = db.uncompress_file ( input_cmpressed_file ) - # @endcode - def uncompress_file ( self , filein ) : - """ Uncompress (gunzip) the file into temporary location, keep original - >>> db = ... - >>> files = db.uncompress_file ( input_cmpressed_file ) - """ - items = [] - tmpdir = self.tempdir () - - ## 1) try zipfile - import zipfile - if zipfile.is_zipfile ( filein ) : - with zipfile.ZipFile ( filein , 'r' , allowZip64 = True ) as zfile : - for item in zfile.filelist : - zfile.extract ( item , path = tmpdir ) - items.append ( os.path.join ( tmpdir , item.filename ) ) - items.sort() - return tuple ( items ) - - ## 2) try compressed-tarfile - import tarfile - if tarfile.is_tarfile ( filein ) : - with tarfile.open ( filein , 'r:*' ) as tfile : - for item in tfile : - tfile.extract ( item , path = tmpdir ) - items.append ( os.path.join ( tmpdir , item.name ) ) - items.sort() - return tuple ( items ) - - ## 3) try old good gzipped (single) file - import gzip , io, tempfile - fd , fileout = tempfile.mkstemp ( prefix = 'ostap-tmp-' , suffix = '-zdb' ) - with gzip.open ( filein , 'rb' ) as fin : - with io.open ( fileout , 'wb' ) as fout : - shutil.copyfileobj ( fin , fout ) - return fileout , - # ========================================================================== ## compress (zip) the item using zlib.compress def compress_item ( self , value ) : @@ -288,44 +215,12 @@ def uncompress_item ( self , value ) : """ return self.unpickle ( zlib.decompress ( value ) ) - # ========================================================================= - ## clone the database into new one - # @code - # db = ... - # ndb = db.clone ( 'new_file.db' ) - # @endcode - def clone ( self , new_name , keys = () ) : - """ Clone the database into new one - >>> old_db = ... - >>> new_db = new_db.clone ( 'new_file.db' ) - """ - new_db = ZipShelf ( new_name , - mode = 'c' , - dbtype = self.dbtype , - protocol = self.protocol , - compress = self.compresslevel , - writeback = self.writeback , - silent = self.silent , - keyencoding = self.keyencoding ) - - ## copy the content - copy = keys if keys else self.keys() - for key in copy : new_db [ key ] = self [ key ] - new_db.sync () - return new_db - # ============================================================================= ## helper function to access ZipShelve data base # @author Vanya BELYAEV Ivan.Belyaev@cern.ch # @date 2010-04-30 -def open ( filename , - mode = 'c' , ## mode/flag - dbtype = '' , ## preferred dbtype - protocol = PROTOCOL , - compress = zlib.Z_BEST_COMPRESSION , - writeback = False , - silent = True , - keyencoding = ENCODING , **kwargs ) : +def open ( dbname , + mode = 'c' , **kwargs ) : """Open a persistent dictionary for reading and writing. @@ -339,14 +234,7 @@ def open ( filename , See the module's __doc__ string for an overview of the interface. """ - return ZipShelf ( filename , - mode = mode , - dbtype = dbtype , - protocol = protocol , - compress = compress , - writeback = writeback , - silent = silent , - keyencoding = keyencoding , **kwargs ) + return ZipShelf ( dbname , mode = mode , **kwargs ) # ============================================================================= ## @class TmpZipShelf @@ -357,43 +245,33 @@ class TmpZipShelf(ZipShelf,TmpDB): """TEMPORARY Zipped-version of ``shelve''-database """ def __init__( self , - dbtype = '' , protocol = HIGHEST_PROTOCOL , compress = zlib.Z_BEST_COMPRESSION , - silent = False , - keyencoding = ENCODING , - remove = True , ## immediate remove + remove = True , ## immediate remove keep = False , **kwargs ) : ## keep it ## initialize the base: generate the name TmpDB .__init__ ( self , suffix = '.zdb' , remove = remove , keep = keep ) - ZipShelf .__init__ ( self , - self.tmp_name , - mode = 'c' , - dbtype = dbtype , - protocol = protocol , - compress = compress , - writeback = False , - silent = silent , - keyencoding = keyencoding , **kwargs ) + ZipShelf .__init__ ( self , + self.tmp_name , + mode = 'c' , + protocol = protocol , + compress = compress , + writeback = False , **kwargs ) + + conf = { 'remove' : remove , 'keep' : keep } + self.kwargs.update ( conf ) ## close and delete the file def close ( self ) : ZipShelf.close ( self ) TmpDB.clean ( self ) - # ============================================================================= ## helper function to open TEMPORARY ZipShelve data base# # @author Vanya BELYAEV Ivan.Belyaev@cern.ch # @date 2010-04-30 -def tmpdb ( dbtype = '' , - protocol = HIGHEST_PROTOCOL , - compress = zlib.Z_BEST_COMPRESSION , - silent = True , - keyencoding = ENCODING , - remove = True , ## immediate remove - keep = False , **kwargs ) : ## keep it +def tmpdb ( **kwargs ) : """ Open a TEMPORARY persistent dictionary for reading and writing. The optional protocol parameter specifies the @@ -401,13 +279,8 @@ def tmpdb ( dbtype = '' , See the module's __doc__ string for an overview of the interface. """ - return TmpZipShelf ( dbtype = dbtype , - protocol = protocol , - compress = compress , - silent = silent , - keyencoding = keyencoding , - remove = remove , - keep = keep , **kwargs ) + return TmpZipShelf ( **kwargs ) + # ============================================================================= if '__main__' == __name__ : diff --git a/ostap/io/zstshelve.py b/ostap/io/zstshelve.py index 24f24ca4..ca5ddf28 100644 --- a/ostap/io/zstshelve.py +++ b/ostap/io/zstshelve.py @@ -137,11 +137,15 @@ __date__ = "2023-02-17" __version__ = "$Revision:$" # ============================================================================= -__all__ = () +__all__ = ( + 'ZstShelf' , ## database + 'open' , ## open the database + 'tmpdb' , ## open the temporary database +) # ============================================================================= from sys import version_info as python_version # ============================================================================= -from ostap.io.compress_shelve import CompressShelf, ENCODING, PROTOCOL, HIGHEST_PROTOCOL +from ostap.io.compress_shelve import CompressShelf, HIGHEST_PROTOCOL from ostap.io.dbase import TmpDB import os, sys, shelve, shutil # ============================================================================= @@ -151,7 +155,7 @@ # ============================================================================= logger.debug ( "Simple generic (c)Pickle-based `ZST'-database" ) # ============================================================================= -if ( 3 , 6 )<= python_version : +if ( 3 , 6 ) <= python_version : try : import zstandard as zst except ImportError : @@ -160,291 +164,189 @@ else : logger.error ( 'zstshelve is disabled for python %s' % str ( python_version ) ) zst = None - + # ============================================================================= -if zst : - # ============================================================================= - ## @class ZstShelf - # `ZST'-version of `shelve'-database - # Modes: - # - 'r' Open existing database for reading only - # - 'w' Open existing database for reading and writing - # - 'c' Open database for reading and writing, creating if it does not exist (default) - # - 'n' Always create a new, empty database, open for reading and writing - # @author Vanya BELYAEV Ivan.Belyaev@cern.ch - # @date 2010-04-30 - class ZstShelf(CompressShelf): - """ZST-version of `shelve'-database - Modes: - - 'r' Open existing database for reading only - - 'w' Open existing database for reading and writing - - 'c' Open database for reading and writing, creating if it does not exist - - 'n' Always create a new, empty database, open for reading and writing - """ - ## the known "standard" extensions: - extensions = '.zst', '.zstd' - ## - def __init__( self , - filename , - mode = 'c' , - dbtype = '' , - protocol = PROTOCOL , - compress = 3 , ## level in Zstandard - writeback = False , - silent = False , - keyencoding = ENCODING , - threads = -1 , **kwargs ) : - - ## save arguments for pickling.... - self.__init_args = ( filename , - mode , - dbtype , - protocol , - compress , - writeback , - silent , - threads ) - - self.__threads = threads - self.__compressor = zst.ZstdCompressor ( level = compress , - threads = threads , - write_checksum = True , - write_content_size = True ) - self.__decompressor = zst.ZstdDecompressor ( ) - - ## initialize the base class - CompressShelf.__init__ ( self , - filename , - mode = mode , - dbtype = dbtype , - protocol = protocol , - compress = compress , - writeback = writeback , - silent = silent , - keyencoding = keyencoding , **kwargs ) - - - @property - def threads ( self ) : - """'threads' : now many (C)-thread can be used for compression/decompression?""" - return self.__threads - @property - def compressor ( self ) : - """'compressor' : get the actual compressor object""" - return self.__compressor - @property - def decompressor ( self ) : - """'decompressor' : get the actual decompressor object""" - return self.__decompressor +## @class ZstShelf +# `ZST'-version of `shelve'-database +# Modes: +# - 'r' Open existing database for reading only +# - 'w' Open existing database for reading and writing +# - 'c' Open database for reading and writing, creating if it does not exist (default) +# - 'n' Always create a new, empty database, open for reading and writing +# @author Vanya BELYAEV Ivan.Belyaev@cern.ch +# @date 2010-04-30 +class ZstShelf(CompressShelf): + """ ZST-version of `shelve'-database + Modes: + - 'r' Open existing database for reading only + - 'w' Open existing database for reading and writing + - 'c' Open database for reading and writing, creating if it does not exist + - 'n' Always create a new, empty database, open for reading and writing + """ + + extensions = '.zst', '.zstd' + ## + def __init__( self , + dbname , + mode = 'c' , + compress = 22 , ## level in Zstandard + threads = -1 , **kwargs ) : - ## needed for proper (un)pickling - def __getinitargs__ ( self ) : - """for proper (un_pickling""" - return self.__init_args + assert zst , "`zstandard` module is not available!" + assert 1 <= compress <= 22 , 'Invalid `compress` for `zstandard`-compression: %s' % compress - ## needed for proper (un)pickling - def __getstate__ ( self ) : - """for proper (un)pickling""" - self.sync() - return {} + self.__threads = threads + self.__compressor = zst.ZstdCompressor ( level = compress , + threads = threads , + write_checksum = True , + write_content_size = True ) + self.__decompressor = zst.ZstdDecompressor ( ) - ## needed for proper (un)pickling - def __setstate__ ( self , dct ) : - """for proper (un)pickling""" - pass + ## initialize the base class + CompressShelf.__init__ ( self , + dbname , + mode = mode , + compress = compress , **kwargs ) - # ========================================================================= - ## compress (zstandard) the file into temporary location, keep original - def compress_files ( self , files ) : - """Compress (zstandard) the file into temporary location, keep original - """ - output = self.tempfile() - import tarfile - with tarfile.open ( output , 'x:gz' ) as tfile : - for file in files : - _ , name = os.path.split ( file ) - tfile.add ( file , name ) + conf = { 'threads' : threads } + self.kwargs.update ( conf ) + + @property + def threads ( self ) : + """'threads' : now many (C)-thread can be used for compression/decompression?""" + return self.__threads + @property + def compressor ( self ) : + """'compressor' : get the actual compressor object""" + return self.__compressor + @property + def decompressor ( self ) : + """'decompressor' : get the actual decompressor object""" + return self.__decompressor + + # ========================================================================= + ## compress (zstandard) the file into temporary location, keep original + def compress_files ( self , files ) : + """ Compress (zstandard) the file into temporary location, keep original + """ + output = self.tempfile() + import tarfile + with tarfile.open ( output , 'x:gz' ) as tfile : + for file in files : + _ , name = os.path.split ( file ) + tfile.add ( file , name ) return output - # ========================================================================= - ## uncompress (zstandard) the file into temporary location, keep original - def uncompress_file ( self , filein ) : - """Uncompress (zstandard) the file into temporary location, keep original - """ - - items = [] - tmpdir = self.tempdir () - - ## 1) try compressed-tarfile - import tarfile - if tarfile.is_tarfile ( filein ) : - with tarfile.open ( filein , 'r:*' ) as tfile : - for item in tfile : - tfile.extract ( item , path = tmpdir ) - items.append ( os.path.join ( tmpdir , item.name ) ) - items.sort() - return tuple ( items ) + # ========================================================================= + ## uncompress (zstandard) the file into temporary location, keep original + def uncompress_file ( self , filein ) : + """ Uncompress (zstandard) the file into temporary location, keep original + """ + + items = [] + tmpdir = self.tempdir () + + ## 1) try compressed-tarfile + import tarfile + if tarfile.is_tarfile ( filein ) : + with tarfile.open ( filein , 'r:*' ) as tfile : + for item in tfile : + tfile.extract ( item , path = tmpdir ) + items.append ( os.path.join ( tmpdir , item.name ) ) + items.sort() + return tuple ( items ) - ## 2) single zst-file - import tempfile , io - fd , fileout = tempfile.mkstemp ( prefix = 'ostap-tmp-' , suffix = '-zstdb' ) - - with io.open ( filein , 'rb' ) as fin : - with io.open ( fileout , 'wb' ) as fout : - self.decompressor.copy_stream ( fin , fout ) - return fileout , + ## 2) single zst-file + import tempfile , io + fd , fileout = tempfile.mkstemp ( prefix = 'ostap-tmp-' , suffix = '-zstdb' ) + + with io.open ( filein , 'rb' ) as fin : + with io.open ( fileout , 'wb' ) as fout : + self.decompressor.copy_stream ( fin , fout ) + return fileout , - # ========================================================================== - ## compress (ZST) the item using compressor - def compress_item ( self , value ) : - """Compress (ZST) the item using compressor - """ - return self.compressor.compress ( self.pickle ( value ) ) + # ========================================================================== + ## compress (ZST) the item using compressor + def compress_item ( self , value ) : + """ Compress (ZST) the item using compressor + """ + return self.compressor.compress ( self.pickle ( value ) ) - # ========================================================================= - ## uncompres (ZST) the item using decompressor - def uncompress_item ( self , value ) : - """Uncompress (ZST) the item using decompressor - """ - return self.unpickle ( self.decompressor.decompress ( value ) ) + # ========================================================================= + ## uncompres (ZST) the item using decompressor + def uncompress_item ( self , value ) : + """ Uncompress (ZST) the item using decompressor + """ + return self.unpickle ( self.decompressor.decompress ( value ) ) + +# ============================================================================= +## helper function to access ZstShelve data base +# @author Vanya BELYAEV Ivan.Belyaev@cern.ch +# @date 2023-02-17 +def open ( dbname , mode = 'c' , **kwargs ) : + """ Open a persistent dictionary for reading and writing. + + The filename parameter is the base filename for the underlying + database. As a side-effect, an extension may be added to the + filename and more than one file may be created. The optional flag + parameter has the same interpretation as the flag parameter of + anydbm.open(). The optional protocol parameter specifies the + version of the pickle protocol (0, 1, or 2). - # ========================================================================= - ## clone the database into new one - # @code - # db = ... - # ndb = db.clone ( 'new_file.db' ) - # @endcode - def clone ( self , new_name , keys = () ) : - """ Clone the database into new one - >>> old_db = ... - >>> new_db = new_db.clone ( 'new_file.db' ) - """ - new_db = ZstShelf ( new_name , - mode = 'c' , - dbtype = self.dbtype , - protocol = self.protocol , - compress = self.compresslevel , - writeback = self.writeback , - silent = self.silent , - keyencoding = self.keyencoding , - threads = self.threads ) - - ## copy the content - copy = keys if keys else self.keys() - for key in copy : new_db [ key ] = self [ key ] - new_db.sync () - return new_db + See the module's __doc__ string for an overview of the interface. + """ + return ZstShelf ( dbname , mode = mode , **kwargs ) + +# ============================================================================= +## @class TmpZstShelf +# TEMPORARY zst-version of `shelve'-database +# @author Vanya BELYAEV Ivan.Belyaev@cern.ch +# @date 2023-02-17 +class TmpZstShelf(ZstShelf,TmpDB): + """ TEMPORARY `ZST'-version of `shelve'-database + """ + def __init__( self , + protocol = HIGHEST_PROTOCOL , + remove = True , + keep = False , **kwargs ) : - # ============================================================================= - ## helper function to access ZstShelve data base - # @author Vanya BELYAEV Ivan.Belyaev@cern.ch - # @date 2023-02-17 - def open ( filename , - mode = 'c' , - dbtype = '' , - protocol = PROTOCOL , - compress = 3 , - writeback = False , - silent = True , - keyencoding = ENCODING , - threads = -1 , **kwargs ) : + ## initialize the base: generate the name + TmpDB.__init__ ( self , suffix = '.zstdb' , remove = remove , keep = keep ) - """Open a persistent dictionary for reading and writing. + ## open DB + ZstShelf.__init__ ( self , + self.tmp_name , + mode = 'c' , + protocol = protocol , + writeback = False , **kwargs ) - The filename parameter is the base filename for the underlying - database. As a side-effect, an extension may be added to the - filename and more than one file may be created. The optional flag - parameter has the same interpretation as the flag parameter of - anydbm.open(). The optional protocol parameter specifies the - version of the pickle protocol (0, 1, or 2). + conf = { 'remove' : remove , 'keep' : keep } + self.kwargs.update ( conf ) - See the module's __doc__ string for an overview of the interface. - """ - return ZstShelf ( filename , - mode = mode , - dbtype = dbtype , - protocol = protocol , - compress = compress , - writeback = writeback , - silent = silent , - keyencoding = keyencoding , - threads = threads , **kwargs ) - # ============================================================================= - ## @class TmpZstShelf - # TEMPORARY zst-version of `shelve'-database - # @author Vanya BELYAEV Ivan.Belyaev@cern.ch - # @date 2023-02-17 - class TmpZstShelf(ZstShelf,TmpDB): - """ - TEMPORARY `ZST'-version of `shelve'-database - """ - def __init__( self , - dbtype = '' , - protocol = HIGHEST_PROTOCOL , - compress = 3 , - silent = False , - keyencoding = ENCODING , - remove = True , - keep = False , - threads = -1 , **kwargs ) : - - ## initialize the base: generate the name - TmpDB.__init__ ( self , suffix = '.zstdb' , remove = remove , keep = keep ) - - ## open DB - ZstShelf.__init__ ( self , - self.tmp_name , - mode = 'c' , - dbtype = dbtype , - protocol = protocol , - compress = compress , - writeback = False , ## writeback - silent = silent , - keyencoding = keyencoding , - threads = threads , **kwargs ) - - ## close and delete the file - def close ( self ) : - ## close the shelve file - ZstShelf.close ( self ) - ## delete the file - TmpDB .clean ( self ) - - # ============================================================================= - ## helper function to open TEMPORARY ZstShelve data base# - # @author Vanya BELYAEV Ivan.Belyaev@cern.ch - # @date 2023-02-17 - def tmpdb ( dbtype = '' , - protocol = HIGHEST_PROTOCOL , - compress = 3 , - silent = True , - keyencoding = ENCODING , - remove = True , ## immediate remove - keep = False , ## keep it - threads = -1 , **kwargs ) : - """Open a TEMPORARY persistent dictionary for reading and writing. - - The optional protocol parameter specifies the - version of the pickle protocol (0, 1, or 2). + ## close and delete the file + def close ( self ) : + ## close the shelve file + ZstShelf.close ( self ) + ## delete the file + TmpDB .clean ( self ) + +# ============================================================================= +## helper function to open TEMPORARY ZstShelve data base# +# @author Vanya BELYAEV Ivan.Belyaev@cern.ch +# @date 2023-02-17 +def tmpdb ( **kwargs ) : + """ Open a TEMPORARY persistent dictionary for reading and writing. - See the module's __doc__ string for an overview of the interface. - """ - return TmpZstShelf ( dbtype = dbtype , - protocol = protocol , - compress = compress , - silent = silent , - keyencoding = keyencoding , - remove = remove , - keep = keep , - threads = threads , **kwargs ) + The optional protocol parameter specifies the + version of the pickle protocol (0, 1, or 2). + + See the module's __doc__ string for an overview of the interface. + """ + return TmpZstShelf ( **kwargs ) - # ========================================================================== - __all__ = ( - 'ZstShelf' , ## database - 'open' , ## open the database - 'tmpdb' , ## open the temporary database - ) +# ========================================================================== +if not zst : __all__ = () # ============================================================================= if '__main__' == __name__ :