Skip to content

Commit

Permalink
Replace c-blosc extension
Browse files Browse the repository at this point in the history
  • Loading branch information
dstansby committed Jan 4, 2025
1 parent 168ab7b commit 4c56240
Show file tree
Hide file tree
Showing 6 changed files with 74 additions and 1,064 deletions.
2 changes: 0 additions & 2 deletions numcodecs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,7 @@
ncores = multiprocessing.cpu_count()
except OSError: # pragma: no cover
ncores = 1
blosc._init()
blosc.set_nthreads(min(8, ncores))
atexit.register(blosc.destroy)

from numcodecs import zstd as zstd
from numcodecs.zstd import Zstd
Expand Down
82 changes: 65 additions & 17 deletions numcodecs/blosc_v2.py → numcodecs/blosc.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,32 +18,32 @@
"""

from numcodecs.abc import Codec
import numpy as np

import blosc
from blosc import (
BITSHUFFLE,
SHUFFLE,
NOSHUFFLE,
MAX_BUFFERSIZE,
MAX_THREADS,
MAX_TYPESIZE,
VERSION_STRING,
NOSHUFFLE,
SHUFFLE,
VERSION_DATE,
VERSION_STRING,
)
from numcodecs.abc import Codec

__all__ = [
"BITSHUFFLE",
"SHUFFLE",
"NOSHUFFLE",
"MAX_BUFFERSIZE",
"MAX_THREADS",
"MAX_TYPESIZE",
"VERSION_STRING",
"NOSHUFFLE",
"SHUFFLE",
"VERSION_DATE",
"list_compressors",
"VERSION_STRING",
'get_nthreads',
"list_compressors",
]

AUTOBLOCKS = 0
Expand All @@ -52,20 +52,30 @@


def list_compressors() -> list[str]:
"""Get a list of compressors supported in blosc."""
return blosc.compressor_list()


def get_nthreads() -> int:
"""
Get the number of threads that Blosc uses internally for compression and
decompression.
"""
nthreads = blosc.set_nthreads(1)
blosc.set_nthreads(nthreads)
return nthreads

Check warning on line 66 in numcodecs/blosc.py

View check run for this annotation

Codecov / codecov/patch

numcodecs/blosc.py#L64-L66

Added lines #L64 - L66 were not covered by tests


def set_nthreads(nthreads: int) -> None:
"""
Set the number of threads that Blosc uses internally for compression and
decompression.
"""
blosc.set_nthreads(nthreads)


def cbuffer_complib(source):
def cbuffer_complib(source) -> str:
"""Return the name of the compression library used to compress `source`."""
return blosc.get_clib(source)


Expand All @@ -86,6 +96,32 @@ def _check_buffer_size(buf, max_buffer_size):


def compress(source, cname: str, clevel: int, shuffle: int = SHUFFLE, blocksize=AUTOBLOCKS):
"""
Compress data.
Parameters
----------
source : bytes-like
Data to be compressed. Can be any object supporting the buffer
protocol.
cname : bytes
Name of compression library to use.
clevel : int
Compression level.
shuffle : int
Either NOSHUFFLE (0), SHUFFLE (1), BITSHUFFLE (2) or AUTOSHUFFLE (-1). If AUTOSHUFFLE,
bit-shuffle will be used for buffers with itemsize 1, and byte-shuffle will
be used otherwise. The default is `SHUFFLE`.
blocksize : int
The requested size of the compressed blocks. If 0, an automatic blocksize will
be used.
Returns
-------
dest : bytes
Compressed data.
"""
if shuffle == AUTOSHUFFLE:
if source.itemsize == 1:
shuffle = BITSHUFFLE

Check warning on line 127 in numcodecs/blosc.py

View check run for this annotation

Codecov / codecov/patch

numcodecs/blosc.py#L126-L127

Added lines #L126 - L127 were not covered by tests
Expand All @@ -109,6 +145,23 @@ def compress(source, cname: str, clevel: int, shuffle: int = SHUFFLE, blocksize=


def decompress(source, dest: np.ndarray | bytearray | None = None):
"""
Decompress data.
Parameters
----------
source : bytes-like
Compressed data, including blosc header. Can be any object supporting the buffer
protocol.
dest : array-like, optional
Object to decompress into.
Returns
-------
dest : bytes
Object containing decompressed data.
"""
if dest is None:
return blosc.decompress(source)
elif isinstance(dest, np.ndarray):
Expand All @@ -119,7 +172,8 @@ def decompress(source, dest: np.ndarray | bytearray | None = None):


class Blosc(Codec):
"""Codec providing compression using the Blosc meta-compressor.
"""
Codec providing compression using the Blosc meta-compressor.
Parameters
----------
Expand Down Expand Up @@ -170,11 +224,5 @@ def decode(self, buf, out=None):
return decompress(buf, out)

def __repr__(self):
r = '%s(cname=%r, clevel=%r, shuffle=%s, blocksize=%s)' % (
type(self).__name__,
self.cname,
self.clevel,
_shuffle_repr[self.shuffle + 1],
self.blocksize,
)
r = f'{type(self).__name__}(cname={self.cname!r}, clevel={self.clevel!r}, shuffle={_shuffle_repr[self.shuffle + 1]}, blocksize={self.blocksize})'
return r
Loading

0 comments on commit 4c56240

Please sign in to comment.