diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bec8fb69..c31ea29e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -162,6 +162,17 @@ jobs: - name: pyright (strict type tests) run: pyright -p tests/typing + - name: stubtest (Cython stubs against the built extensions) + # Read the stubs from src/: mypy ignores the installed copies until numcodecs + # ships a py.typed marker. + env: + MYPYPATH: src + run: | + python -m mypy.stubtest --mypy-config-file pyproject.toml \ + --allowlist tests/typing/stubtest-allowlist.txt \ + numcodecs.blosc numcodecs.zstd numcodecs.lz4 numcodecs.vlen \ + numcodecs.fletcher32 numcodecs.jenkins numcodecs.compat_ext numcodecs._shuffle + test-zarr: runs-on: ubuntu-latest strategy: diff --git a/docs/abc.md b/docs/abc.md index f080766f..925757c2 100644 --- a/docs/abc.md +++ b/docs/abc.md @@ -10,4 +10,16 @@ .. automethod:: decode .. automethod:: get_config .. automethod:: from_config + +Protocols +--------- + +These describe codecs structurally, for code that accepts any object implementing the +interface whether or not it subclasses :class:`Codec`. The three kinds mirror the codec +kinds of the Zarr version 3 specification. + +.. autoclass:: CodecProtocol +.. autoclass:: BytesBytesCodec +.. autoclass:: ArrayArrayCodec +.. autoclass:: ArrayBytesCodec ``` diff --git a/docs/release.md b/docs/release.md index 51e2933b..959901dd 100644 --- a/docs/release.md +++ b/docs/release.md @@ -45,8 +45,56 @@ already written into `out`; only the returned object changes. * `JenkinsLookup3.decode` accepts any contiguous writable buffer as `out`, not only an ndarray, and raises `ValueError` when `out` has the wrong size. +* `JenkinsLookup3.get_config()` reports `prefix` as a list of ints, so a configuration + with a prefix can be serialized to JSON and passed back to `get_codec`. It held a numpy + array, which `json.dumps` rejected and `get_codec` could not compare. The constructor + accepts a list of ints as well as a bytes-like `prefix`. +* CRC32, Adler32 and CRC32C check the byte length of the input to `decode`, not its + length along the first axis, before looking for the checksum. * `Fletcher32.decode` without `out` returns a `uint8` ndarray, like the other checksum codecs, instead of a Cython memoryview slice. +* Correct type annotations that described something other than what the code does: + `numcodecs.registry.codec_registry` holds codec classes (`dict[str, type[Codec]]`) and + `entries` holds single entry points (`dict[str, EntryPoint]`). `numcodecs.compat` + functions now annotate their parameters, and `ndarray_copy` returns the type of `dst` + (or of `src` when `dst` is `None`). The `NDArrayLike`, `DType` and `FlagsObj` protocols + declare read-only properties, so `numpy.ndarray` now satisfies `NDArrayLike` for type + checkers, and `numcodecs.ndarray_like.BufferLike` names "a buffer or an ndarray-like". +* `numcodecs` defines `__all__`, so type checkers treat its re-exported codec classes as + public. `from numcodecs import *` now imports only those names. +* `register_codec` raises `ValueError` for a class without a `codec_id` when no id is + given, instead of registering it under `None`. +* `Codec.codec_id` no longer defaults to `None`. Abstract intermediate classes such as + `Checksum32` leave it unset, so reading it raises `AttributeError`; every concrete codec + sets it. + +### Enhancements + +* Annotate `numcodecs.abc.Codec`. `encode` and `decode` take and return + `numcodecs.ndarray_like.BufferLike` (an object exporting the buffer protocol, or an + ndarray-like), with `buf` positional-only, and `decode(buf, out=out)` is typed to return + the type of `out`. `get_config` returns `Mapping[str, Any]` and `from_config` returns + `Self`. Previously pyright inferred that `encode` and `decode` return `None`. +* Add structural protocols `numcodecs.abc.CodecProtocol`, `BytesBytesCodec`, + `ArrayArrayCodec` and `ArrayBytesCodec` for code that accepts any object implementing the + codec interface. +* Annotate the concrete codecs. Each `get_config` returns a TypedDict naming the codec's + parameters (`ZlibConfig`, `BloscConfig`, ...), all subclasses of + `numcodecs.abc.CodecConfig`, which carries the `id` and declares any further keys as + read-only JSON values (PEP 728 `extra_items`). `numcodecs.abc.JSON` names a + JSON-serializable value. Compressors: Zlib, GZip, BZ2, LZMA, Base64, Blosc, Zstd and LZ4. + Checksums: CRC32, Adler32, CRC32C, JenkinsLookup3 and Fletcher32. Filters: AsType, + Delta, Quantize, FixedScaleOffset, BitRound, PackBits, Shuffle and Categorize. + `typing_extensions` >= 4.13 is now required. +* Ship `.pyi` type stubs for the Cython extension modules (`blosc`, `zstd`, `lz4`, `vlen`, + `fletcher32`, `jenkins`, `compat_ext` and `_shuffle`), checked against the compiled + modules with `mypy.stubtest` in CI. Type checkers previously saw the Blosc, Zstd, LZ4, + Fletcher32 and VLen codecs as unknown. + +### Deprecations + +* Loading a codec through the `numcodecs.codecs` entry point group whose class does not + subclass `numcodecs.abc.Codec` now emits a `DeprecationWarning`. (release_0.17.1)= diff --git a/pyproject.toml b/pyproject.toml index 721d1812..654e0ee9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ description = """ A Python package providing buffer compression and transformation codecs \ for use in data storage and communication applications.""" readme = "README.md" -dependencies = ["numpy>=2.0", "packaging", "typing_extensions"] +dependencies = ["numpy>=2.0", "packaging", "typing_extensions>=4.13"] requires-python = ">=3.12" dynamic = [ "version", @@ -245,6 +245,7 @@ quote-style = "preserve" [tool.mypy] ignore_errors = false ignore_missing_imports = true +python_version = "3.12" enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] # TODO: set options below to true strict = false diff --git a/src/numcodecs/__init__.py b/src/numcodecs/__init__.py index be60bfc3..bad41090 100644 --- a/src/numcodecs/__init__.py +++ b/src/numcodecs/__init__.py @@ -25,6 +25,42 @@ from numcodecs.registry import get_codec as get_codec from numcodecs.registry import register_codec +# Names listed here are the public API re-exported from submodules. Optional codecs are +# appended below when their dependency imports. +__all__ = [ + "BZ2", + "CRC32", + "JSON", + "LZ4", + "LZMA", + "Adler32", + "AsType", + "Base64", + "BitRound", + "Blosc", + "Categorize", + "Delta", + "FixedScaleOffset", + "Fletcher32", + "GZip", + "JenkinsLookup3", + "PackBits", + "Pickle", + "Quantize", + "Shuffle", + "VLenArray", + "VLenBytes", + "VLenUTF8", + "Zlib", + "Zstd", + "blosc", + "get_codec", + "lz4", + "register_codec", + "vlen", + "zstd", +] + __version__: str = _version("numcodecs") from numcodecs.zlib import Zlib @@ -131,18 +167,22 @@ from numcodecs.zfpy import ZFPY register_codec(ZFPY) + __all__ += ["ZFPY"] with suppress(ImportError): from numcodecs.msgpacks import MsgPack register_codec(MsgPack) + __all__ += ["MsgPack"] with suppress(ImportError): from numcodecs.checksum32 import CRC32C register_codec(CRC32C) + __all__ += ["CRC32C"] with suppress(ImportError): from numcodecs.pcodec import PCodec register_codec(PCodec) + __all__ += ["PCodec"] diff --git a/src/numcodecs/_shuffle.pyi b/src/numcodecs/_shuffle.pyi new file mode 100644 index 00000000..a8041848 --- /dev/null +++ b/src/numcodecs/_shuffle.pyi @@ -0,0 +1,4 @@ +from collections.abc import Buffer + +def _doShuffle(src: Buffer, des: Buffer, element_size: int) -> None: ... +def _doUnshuffle(src: Buffer, des: Buffer, element_size: int) -> None: ... diff --git a/src/numcodecs/abc.py b/src/numcodecs/abc.py index 6865df09..0612162d 100644 --- a/src/numcodecs/abc.py +++ b/src/numcodecs/abc.py @@ -29,17 +29,55 @@ """ from abc import ABC, abstractmethod +from collections.abc import Buffer, Mapping, Sequence +from typing import Any, Protocol, Self, overload + +from typing_extensions import ReadOnly, TypedDict + +from .ndarray_like import BufferLike, NDArrayLike + +__all__ = [ + "JSON", + "ArrayArrayCodec", + "ArrayBytesCodec", + "BytesBytesCodec", + "Codec", + "CodecConfig", + "CodecProtocol", +] + +type JSON = int | float | bool | str | Sequence[JSON] | Mapping[str, JSON] | None +"""A JSON-serializable value. `Sequence` and `Mapping` rather than `list` and `dict`, so +that tuples, TypedDicts and lists of narrower types count (the same definition as +`zarr_metadata`'s `JSONValue`).""" + + +# `extra_items` (PEP 728) says that any further keys hold JSON values and are read-only. It +# lets type checkers that implement PEP 728 (pyright) treat every codec configuration as a +# `Mapping[str, JSON]`, and as Zarr v2 codec metadata. mypy does not implement PEP 728 yet, +# so it rejects the keyword and treats `CodecConfig` as a plain TypedDict. +class CodecConfig(TypedDict, extra_items=ReadOnly[JSON]): # type: ignore[call-arg] + """The configuration every codec returns from `Codec.get_config`: its `id`, plus any + further JSON-serializable parameters. + + Concrete codecs return a subclass that declares their own parameters. This is the shape + of a Zarr version 2 compressor or filter. + """ + + id: str class Codec(ABC): """Codec abstract base class.""" - # override in sub-class - codec_id: str | None = None - """Codec identifier.""" + # Declared without a value (abstract intermediate classes leave it unset) and without + # `ClassVar`, so that `Codec` instances match protocols that declare `codec_id` either + # as a plain attribute or as a read-only property. + codec_id: str + """Codec identifier. Every concrete codec class sets it as a class attribute.""" @abstractmethod - def encode(self, buf): # pragma: no cover + def encode(self, buf: BufferLike, /) -> BufferLike: # pragma: no cover """Encode data in `buf`. Parameters @@ -55,8 +93,14 @@ def encode(self, buf): # pragma: no cover protocol. """ + @overload + def decode(self, buf: BufferLike, /, out: None = None) -> BufferLike: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, /, out: OutT) -> OutT: ... @abstractmethod - def decode(self, buf, out=None): # pragma: no cover + def decode( + self, buf: BufferLike, /, out: BufferLike | None = None + ) -> BufferLike: # pragma: no cover """Decode data in `buf`. Parameters @@ -72,10 +116,10 @@ def decode(self, buf, out=None): # pragma: no cover ------- dec : buffer-like Decoded data. May be any object supporting the new-style - buffer protocol. + buffer protocol. If `out` was provided, `out` itself. """ - def get_config(self): + def get_config(self) -> Mapping[str, Any]: """Return a dictionary holding configuration parameters for this codec. Must include an 'id' field with the codec identifier. All values must be compatible with JSON encoding.""" @@ -83,7 +127,7 @@ def get_config(self): # override in sub-class if need special encoding of config values # setup config object - config = {'id': self.codec_id} + config: dict[str, Any] = {'id': self.codec_id} # by default, assume all non-private members are configuration # parameters - override this in sub-class if not the case @@ -94,8 +138,8 @@ def get_config(self): return config @classmethod - def from_config(cls, config): - """Instantiate codec from a configuration object.""" + def from_config(cls, config: Mapping[str, Any], /) -> Self: + """Instantiate codec from a configuration object, without its 'id' field.""" # N.B., assume at this point the 'id' field has been removed from # the config object @@ -105,14 +149,14 @@ def from_config(cls, config): # keyword arguments without any special decoding return cls(**config) - def __eq__(self, other): + def __eq__(self, other: object) -> bool: # override in sub-class if need special equality comparison try: - return self.get_config() == other.get_config() + return bool(self.get_config() == other.get_config()) # type: ignore[attr-defined] except AttributeError: return False - def __repr__(self): + def __repr__(self) -> str: # override in sub-class if need special representation # by default, assume all non-private members are configuration @@ -124,3 +168,44 @@ def __repr__(self): ] r += ', '.join(params) + ')' return r + + +class CodecProtocol(Protocol): + """The structural interface of a codec, for code that accepts any object implementing + it, whether or not it subclasses `Codec`.""" + + @property + def codec_id(self) -> str: ... # pragma: no cover + + def encode(self, buf: BufferLike, /) -> BufferLike: ... # pragma: no cover + + def decode(self, buf: BufferLike, /, out: Any = None) -> BufferLike: ... # pragma: no cover + + def get_config(self) -> Mapping[str, Any]: ... # pragma: no cover + + @classmethod + def from_config(cls, config: Mapping[str, Any], /) -> Self: ... # pragma: no cover + + +class BytesBytesCodec(CodecProtocol, Protocol): + """A codec that maps bytes to bytes, such as a compressor or a checksum.""" + + def encode(self, buf: BufferLike, /) -> Buffer: ... # pragma: no cover + + def decode(self, buf: BufferLike, /, out: Any = None) -> Buffer: ... # pragma: no cover + + +class ArrayArrayCodec(CodecProtocol, Protocol): + """A codec that maps an array to an array, such as a filter.""" + + def encode(self, buf: BufferLike, /) -> NDArrayLike: ... # pragma: no cover + + def decode(self, buf: BufferLike, /, out: Any = None) -> NDArrayLike: ... # pragma: no cover + + +class ArrayBytesCodec(CodecProtocol, Protocol): + """A codec that serializes an array to bytes and back.""" + + def encode(self, buf: BufferLike, /) -> Buffer: ... # pragma: no cover + + def decode(self, buf: BufferLike, /, out: Any = None) -> NDArrayLike: ... # pragma: no cover diff --git a/src/numcodecs/astype.py b/src/numcodecs/astype.py index 4947d53c..209d0843 100644 --- a/src/numcodecs/astype.py +++ b/src/numcodecs/astype.py @@ -1,7 +1,18 @@ +from typing import Any, overload + import numpy as np +import numpy.typing as npt -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_ndarray, ndarray_copy +from .ndarray_like import BufferLike + + +class AsTypeConfig(CodecConfig): + """The configuration of a `AsType` codec, as returned by `AsType.get_config`.""" + + encode_dtype: str + decode_dtype: str class AsType(Codec): @@ -40,18 +51,22 @@ class AsType(Codec): codec_id = 'astype' - def __init__(self, encode_dtype, decode_dtype): + def __init__(self, encode_dtype: npt.DTypeLike, decode_dtype: npt.DTypeLike) -> None: self.encode_dtype = np.dtype(encode_dtype) self.decode_dtype = np.dtype(decode_dtype) - def encode(self, buf): + def encode(self, buf: BufferLike) -> npt.NDArray[Any]: # normalise input arr = ensure_ndarray(buf).view(self.decode_dtype) # convert and copy return arr.astype(self.encode_dtype) - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[Any]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise input enc = ensure_ndarray(buf).view(self.encode_dtype) @@ -61,12 +76,12 @@ def decode(self, buf, out=None): # handle output return ndarray_copy(dec, out) - def get_config(self): + def get_config(self) -> AsTypeConfig: return { 'id': self.codec_id, 'encode_dtype': self.encode_dtype.str, 'decode_dtype': self.decode_dtype.str, } - def __repr__(self): + def __repr__(self) -> str: return f'{type(self).__name__}(encode_dtype={self.encode_dtype.str!r}, decode_dtype={self.decode_dtype.str!r})' diff --git a/src/numcodecs/base64.py b/src/numcodecs/base64.py index c107d5c7..2e53181d 100644 --- a/src/numcodecs/base64.py +++ b/src/numcodecs/base64.py @@ -1,7 +1,9 @@ import base64 as _base64 +from typing import cast, overload -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_contiguous_ndarray, ndarray_copy +from .ndarray_like import BufferLike class Base64(Codec): @@ -9,18 +11,25 @@ class Base64(Codec): codec_id = "base64" - def encode(self, buf): + def get_config(self) -> CodecConfig: + return cast(CodecConfig, super().get_config()) + + def encode(self, buf: BufferLike) -> bytes: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) # do compression - return _base64.standard_b64encode(buf) + return _base64.standard_b64encode(arr) - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) if out is not None: ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression - decompressed = _base64.standard_b64decode(buf) + decompressed = _base64.standard_b64decode(arr) # handle destination return ndarray_copy(decompressed, out) diff --git a/src/numcodecs/bitround.py b/src/numcodecs/bitround.py index dc8d5116..564d7ead 100644 --- a/src/numcodecs/bitround.py +++ b/src/numcodecs/bitround.py @@ -1,7 +1,17 @@ +from typing import cast, overload + import numpy as np -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_ndarray_like, ndarray_copy +from .ndarray_like import BufferLike, NDArrayLike + + +class BitRoundConfig(CodecConfig): + """The configuration of a `BitRound` codec, as returned by `BitRound.get_config`.""" + + keepbits: int + # The size in bits of the mantissa/significand for the various floating types # You cannot keep more bits of data than you have available @@ -37,12 +47,15 @@ class BitRound(Codec): codec_id = 'bitround' - def __init__(self, keepbits: int): + def __init__(self, keepbits: int) -> None: if keepbits < 0: raise ValueError("keepbits must be zero or positive") self.keepbits = keepbits - def encode(self, buf): + def get_config(self) -> BitRoundConfig: + return cast(BitRoundConfig, super().get_config()) + + def encode(self, buf: BufferLike) -> NDArrayLike: """Create int array by rounding floating-point data The itemsize will be preserved, but the output should be much more @@ -68,7 +81,11 @@ def encode(self, buf): b &= mask return b - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> NDArrayLike: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: """Remake floats from ints As with ``encode``, preserves itemsize. diff --git a/src/numcodecs/blosc.pyi b/src/numcodecs/blosc.pyi new file mode 100644 index 00000000..495be3ac --- /dev/null +++ b/src/numcodecs/blosc.pyi @@ -0,0 +1,75 @@ +from typing import Final, overload + +from .abc import Codec, CodecConfig +from .ndarray_like import BufferLike + +NOSHUFFLE: Final[int] +SHUFFLE: Final[int] +BITSHUFFLE: Final[int] +AUTOSHUFFLE: Final[int] +AUTOBLOCKS: Final[int] +MAX_OVERHEAD: Final[int] +MAX_BUFFERSIZE: Final[int] +MAX_THREADS: Final[int] +MAX_TYPESIZE: Final[int] +VERSION_STRING: Final[str] +VERSION_DATE: Final[str] +__version__: Final[str] + +use_threads: bool | None +"""Set to `True` or `False` to override whether Blosc may use threads; `None` (the +default) decides from the calling thread and process.""" + +def _init() -> None: ... +def _destroy() -> None: ... +def _get_use_threads() -> bool: ... +def _cbuffer_sizes(source: BufferLike) -> tuple[int, int, int]: ... +def _cbuffer_metainfo(source: BufferLike) -> tuple[int, bool, bool]: ... +def list_compressors() -> list[str]: ... +def get_nthreads() -> int: ... +def set_nthreads(nthreads: int) -> int: ... +def cbuffer_complib(source: BufferLike) -> str: ... +def compress( + source: BufferLike, + cname: bytes, + clevel: int, + shuffle: int = ..., + blocksize: int = ..., + typesize: int | None = None, +) -> bytes: ... +@overload +def decompress(source: BufferLike, dest: None = None) -> bytes: ... +@overload +def decompress[OutT: BufferLike](source: BufferLike, dest: OutT) -> OutT: ... + +class BloscConfig(CodecConfig): + cname: str + clevel: int + shuffle: int + blocksize: int + +class Blosc(Codec): + codec_id: str + NOSHUFFLE: int + SHUFFLE: int + BITSHUFFLE: int + AUTOSHUFFLE: int + max_buffer_size: int + cname: str + clevel: int + shuffle: int + blocksize: int + def __init__( + self, + cname: str = "lz4", + clevel: int = 5, + shuffle: int = ..., + blocksize: int = ..., + typesize: int | None = None, + ) -> None: ... + def get_config(self) -> BloscConfig: ... + def encode(self, buf: BufferLike) -> bytes: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... diff --git a/src/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx index 8c52dd95..509b9b96 100644 --- a/src/numcodecs/blosc.pyx +++ b/src/numcodecs/blosc.pyx @@ -14,10 +14,17 @@ from cpython.memoryview cimport PyMemoryView_GET_BUFFER from .compat_ext cimport PyBytes_RESIZE, ensure_contiguous_memoryview +from typing import TypedDict + from .compat import ensure_contiguous_ndarray from .abc import Codec +# The type of `Blosc.get_config()`. Declared with the functional syntax because Cython does not +# compile TypedDict class bodies; `blosc.pyi` declares the same type with the class syntax. +BloscConfig = TypedDict("BloscConfig", {"id": str, "cname": str, "clevel": int, "shuffle": int, "blocksize": int}) + + cdef extern from "blosc.h": cdef enum: BLOSC_MAX_OVERHEAD, diff --git a/src/numcodecs/bz2.py b/src/numcodecs/bz2.py index a3c8f5ec..d4fde492 100644 --- a/src/numcodecs/bz2.py +++ b/src/numcodecs/bz2.py @@ -1,7 +1,15 @@ import bz2 as _bz2 +from typing import cast, overload -from numcodecs.abc import Codec +from numcodecs.abc import Codec, CodecConfig from numcodecs.compat import ensure_contiguous_ndarray, ndarray_copy +from numcodecs.ndarray_like import BufferLike + + +class BZ2Config(CodecConfig): + """The configuration of a `BZ2` codec, as returned by `BZ2.get_config`.""" + + level: int class BZ2(Codec): @@ -16,28 +24,34 @@ class BZ2(Codec): codec_id = 'bz2' - def __init__(self, level=1): + def __init__(self, level: int = 1) -> None: self.level = level - def encode(self, buf): + def get_config(self) -> BZ2Config: + return cast(BZ2Config, super().get_config()) + + def encode(self, buf: BufferLike) -> bytes: # normalise input - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) # do compression - return _bz2.compress(buf, self.level) + return _bz2.compress(arr, self.level) - # noinspection PyMethodMayBeStatic - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) if out is not None: ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # N.B., bz2 cannot handle ndarray directly because of truth testing issues - buf = memoryview(buf) + mem = memoryview(arr) # do decompression - dec = _bz2.decompress(buf) + dec = _bz2.decompress(mem) # handle destination - Python standard library bz2 module does not # support direct decompression into buffer, so we have to copy into diff --git a/src/numcodecs/categorize.py b/src/numcodecs/categorize.py index c8199493..74739aab 100644 --- a/src/numcodecs/categorize.py +++ b/src/numcodecs/categorize.py @@ -1,7 +1,20 @@ +from collections.abc import Iterable +from typing import Any, overload + import numpy as np +import numpy.typing as npt -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_ndarray, ensure_text, ndarray_copy +from .ndarray_like import BufferLike + + +class CategorizeConfig(CodecConfig): + """The configuration of a `Categorize` codec, as returned by `Categorize.get_config`.""" + + labels: list[str] + dtype: str + astype: str class Categorize(Codec): @@ -37,7 +50,12 @@ class Categorize(Codec): codec_id = 'categorize' - def __init__(self, labels, dtype, astype='u1'): + def __init__( + self, + labels: Iterable[str | BufferLike], + dtype: npt.DTypeLike, + astype: npt.DTypeLike = 'u1', + ) -> None: self.dtype = np.dtype(dtype) if self.dtype.kind not in 'UO': raise TypeError("only unicode ('U') and object ('O') dtypes are supported") @@ -46,12 +64,13 @@ def __init__(self, labels, dtype, astype='u1'): if self.astype == np.dtype(object): raise TypeError('encoding as object array not supported') - def encode(self, buf): + def encode(self, buf: BufferLike | Iterable[str]) -> npt.NDArray[Any]: # normalise input if self.dtype == np.dtype(object): arr = np.asarray(buf, dtype=object) else: - arr = ensure_ndarray(buf).view(self.dtype) + # only an object dtype accepts a plain iterable; anything else must be a buffer + arr = ensure_ndarray(buf).view(self.dtype) # type: ignore[arg-type] # flatten to simplify implementation arr = arr.reshape(-1, order='A') @@ -65,7 +84,11 @@ def encode(self, buf): return enc - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[Any]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise input enc = ensure_ndarray(buf).view(self.astype) @@ -82,7 +105,7 @@ def decode(self, buf, out=None): # handle output return ndarray_copy(dec, out) - def get_config(self): + def get_config(self) -> CategorizeConfig: return { 'id': self.codec_id, 'labels': self.labels, @@ -90,7 +113,7 @@ def get_config(self): 'astype': self.astype.str, } - def __repr__(self): + def __repr__(self) -> str: # make sure labels part is not too long labels = repr(self.labels[:3]) if len(self.labels) > 3: diff --git a/src/numcodecs/checksum32.py b/src/numcodecs/checksum32.py index a095b171..32700ac9 100644 --- a/src/numcodecs/checksum32.py +++ b/src/numcodecs/checksum32.py @@ -4,13 +4,15 @@ import warnings import zlib from collections.abc import Buffer, Callable -from typing import Literal +from typing import Literal, NotRequired, cast, overload import numpy as np +import numpy.typing as npt -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_contiguous_ndarray, ndarray_copy from .jenkins import jenkins_lookup3 +from .ndarray_like import BufferLike crc32c_checksum: Callable[[Buffer, int], int] | None @@ -42,17 +44,39 @@ def crc32c_checksum(data: Buffer, value: int = 0) -> int: CHECKSUM_LOCATION = Literal['start', 'end'] +class Checksum32Config(CodecConfig): + """The configuration of a `Checksum32` codec (CRC32, Adler32, CRC32C), as returned by + `get_config`. `location` is present only when it was passed to the constructor; the + class default applies otherwise (`'start'` for CRC32 and Adler32, `'end'` for + CRC32C).""" + + location: NotRequired[CHECKSUM_LOCATION] + + +class JenkinsLookup3Config(Checksum32Config): + """The configuration of a `JenkinsLookup3` codec, as returned by + `JenkinsLookup3.get_config`.""" + + initval: int + prefix: list[int] | None + + class Checksum32(Codec, abc.ABC): # override in sub-class location: CHECKSUM_LOCATION = 'start' - def __init__(self, location: CHECKSUM_LOCATION | None = None): + def __init__(self, location: CHECKSUM_LOCATION | None = None) -> None: if location is not None: self.location = location if self.location not in ['start', 'end']: raise ValueError(f"Invalid checksum location: {self.location}") - def encode(self, buf): + def get_config(self) -> Checksum32Config: + return cast(Checksum32Config, super().get_config()) + + # CRC32, Adler32 and CRC32C return uint8 ndarrays; JenkinsLookup3 overrides encode and + # decode to return bytes and a memoryview. The annotations here cover both. + def encode(self, buf: BufferLike) -> npt.NDArray[np.uint8] | bytes: arr = ensure_contiguous_ndarray(buf).view('u1') checksum = self.checksum(arr) & 0xFFFFFFFF enc = np.empty(arr.nbytes + 4, dtype='u1') @@ -66,13 +90,17 @@ def encode(self, buf): ndarray_copy(arr, payload_view) return enc - def decode(self, buf, out=None): - if len(buf) < 4: + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[np.uint8] | memoryview: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: + arr = ensure_contiguous_ndarray(buf).view('u1') + if arr.nbytes < 4: raise ValueError("Input buffer is too short to contain a 32-bit checksum.") if out is not None: ensure_contiguous_ndarray(out) # check that out is a valid ndarray - arr = ensure_contiguous_ndarray(buf).view('u1') if self.location == 'start': checksum_view = arr[:4] payload_view = arr[4:] @@ -89,7 +117,9 @@ def decode(self, buf, out=None): @staticmethod @abc.abstractmethod - def checksum(data: Buffer, value: int) -> int: ... + def checksum(data: Buffer, value: int = 0, /) -> int: + """Return the checksum of `data`, continuing from `value`. Subclasses choose the + default `value`.""" class CRC32(Checksum32): @@ -147,30 +177,43 @@ class JenkinsLookup3(Checksum32): ---------- initval : int initial seed passed to the hash algorithm, default: 0 - prefix : int - bytes prepended to the buffer before evaluating the hash, default: None + prefix : bytes-like or list of int + bytes prepended to the buffer before evaluating the hash, default: None. + `get_config` reports them as a list of ints, so that the configuration is JSON. """ - checksum = jenkins_lookup3 + checksum = staticmethod(jenkins_lookup3) codec_id = "jenkins_lookup3" - def __init__(self, initval: int = 0, prefix=None): + prefix: npt.NDArray[np.uint8] | None + + def __init__(self, initval: int = 0, prefix: Buffer | list[int] | None = None) -> None: self.initval = initval if prefix is None: self.prefix = None else: - self.prefix = np.frombuffer(prefix, dtype='uint8') + self.prefix = np.frombuffer(bytes(prefix), dtype='uint8') + + def get_config(self) -> "JenkinsLookup3Config": + config = cast(JenkinsLookup3Config, dict(super().get_config())) + if self.prefix is not None: + config["prefix"] = self.prefix.tolist() + return config - def encode(self, buf): + def encode(self, buf: BufferLike) -> bytes: """Return buffer plus 4-byte Bob Jenkin's lookup3 checksum""" - buf = ensure_contiguous_ndarray(buf).ravel().view('uint8') + arr = ensure_contiguous_ndarray(buf).ravel().view('uint8') if self.prefix is None: - val = jenkins_lookup3(buf, self.initval) + val = jenkins_lookup3(arr, self.initval) else: - val = jenkins_lookup3(np.hstack((self.prefix, buf)), self.initval) - return buf.tobytes() + struct.pack(" memoryview: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: """Check Bob Jenkin's lookup3 checksum, and return buffer without it""" b = ensure_contiguous_ndarray(buf).view('uint8') if self.prefix is None: diff --git a/src/numcodecs/compat.py b/src/numcodecs/compat.py index cc4d2d13..bad12fd4 100644 --- a/src/numcodecs/compat.py +++ b/src/numcodecs/compat.py @@ -1,12 +1,19 @@ import array import codecs +from collections.abc import Buffer +from typing import Any, overload import numpy as np +import numpy.typing as npt -from .ndarray_like import NDArrayLike, is_ndarray_like +from .ndarray_like import BufferLike, NDArrayLike, is_ndarray_like -def ensure_ndarray_like(buf) -> NDArrayLike: +@overload +def ensure_ndarray_like(buf: Buffer) -> npt.NDArray[Any]: ... +@overload +def ensure_ndarray_like(buf: NDArrayLike) -> NDArrayLike: ... +def ensure_ndarray_like(buf: BufferLike) -> NDArrayLike: """Convenience function to coerce `buf` to ndarray-like array. Parameters @@ -41,7 +48,7 @@ def ensure_ndarray_like(buf) -> NDArrayLike: return buf -def ensure_ndarray(buf) -> np.ndarray: +def ensure_ndarray(buf: BufferLike) -> npt.NDArray[Any]: """Convenience function to coerce `buf` to a numpy array, if it is not already a numpy array. @@ -63,7 +70,17 @@ def ensure_ndarray(buf) -> np.ndarray: return np.array(ensure_ndarray_like(buf), copy=False) -def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> NDArrayLike: +@overload +def ensure_contiguous_ndarray_like( + buf: Buffer, max_buffer_size: int | None = None, flatten: bool = True +) -> npt.NDArray[Any]: ... +@overload +def ensure_contiguous_ndarray_like( + buf: NDArrayLike, max_buffer_size: int | None = None, flatten: bool = True +) -> NDArrayLike: ... +def ensure_contiguous_ndarray_like( + buf: BufferLike, max_buffer_size: int | None = None, flatten: bool = True +) -> NDArrayLike: """Convenience function to coerce `buf` to ndarray-like array. Also ensures that the returned value exports fully contiguous memory, and supports the new-style buffer interface. If the optional max_buffer_size is @@ -95,12 +112,12 @@ def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> N # check for object arrays, these are just memory pointers, actual memory holding # item data is scattered elsewhere - if arr.dtype == object: + if arr.dtype.kind == "O": raise TypeError("object arrays are not supported") # check for datetime or timedelta ndarray, the buffer interface doesn't support those if arr.dtype.kind in "Mm": - arr = arr.view(np.int64) # type: ignore[arg-type] + arr = arr.view(np.int64) # check memory is contiguous, if so flatten if arr.flags.c_contiguous or arr.flags.f_contiguous: @@ -117,7 +134,9 @@ def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> N return arr -def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.ndarray: +def ensure_contiguous_ndarray( + buf: BufferLike, max_buffer_size: int | None = None, flatten: bool = True +) -> npt.NDArray[Any]: """Convenience function to coerce `buf` to a numpy array, if it is not already a numpy array. Also ensures that the returned value exports fully contiguous memory, and supports the new-style buffer interface. If the optional max_buffer_size is @@ -150,7 +169,7 @@ def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.nda ) -def ensure_bytes(buf) -> bytes: +def ensure_bytes(buf: BufferLike) -> bytes: """Obtain a bytes object from memory exposed by `buf`.""" if not isinstance(buf, bytes): @@ -158,7 +177,7 @@ def ensure_bytes(buf) -> bytes: # check for object arrays, these are just memory pointers, # actual memory holding item data is scattered elsewhere - if arr.dtype == object: + if arr.dtype.kind == "O": raise TypeError("object arrays are not supported") # create bytes @@ -167,14 +186,18 @@ def ensure_bytes(buf) -> bytes: return buf -def ensure_text(s, encoding="utf-8"): - if not isinstance(s, str): - s = ensure_contiguous_ndarray(s) - s = codecs.decode(s, encoding) - return s +def ensure_text(s: str | BufferLike, encoding: str = "utf-8") -> str: + """Return `s` if it is a `str`, else decode the bytes it exposes with `encoding`.""" + if isinstance(s, str): + return s + return codecs.decode(ensure_contiguous_ndarray(s), encoding) -def ndarray_copy(src, dst) -> NDArrayLike: +@overload +def ndarray_copy[SrcT: BufferLike](src: SrcT, dst: None) -> SrcT: ... +@overload +def ndarray_copy[DstT: BufferLike](src: BufferLike, dst: DstT) -> DstT: ... +def ndarray_copy(src: BufferLike, dst: BufferLike | None) -> BufferLike: """Copy the contents of the array from `src` to `dst`, and return `dst`. If `dst` is `None`, return `src` unchanged. Otherwise `dst` itself is returned, not @@ -185,28 +208,26 @@ def ndarray_copy(src, dst) -> NDArrayLike: # no-op return src - out = dst - # ensure ndarray like - src = ensure_ndarray_like(src) - dst = ensure_ndarray_like(dst) + src_arr: NDArrayLike = ensure_ndarray_like(src) + dst_arr: NDArrayLike = ensure_ndarray_like(dst) # flatten source array - src = src.reshape(-1, order="A") + src_arr = src_arr.reshape(-1, order="A") # ensure same data type - if dst.dtype != object: - src = src.view(dst.dtype) + if dst_arr.dtype.kind != "O": + src_arr = src_arr.view(dst_arr.dtype) # reshape source to match destination - if src.shape != dst.shape: - if dst.flags.f_contiguous: + if src_arr.shape != dst_arr.shape: + if dst_arr.flags.f_contiguous: order = "F" else: order = "C" - src = src.reshape(dst.shape, order=order) + src_arr = src_arr.reshape(dst_arr.shape, order=order) # copy via numpy - np.copyto(dst, src) + np.copyto(dst_arr, src_arr) # type: ignore[arg-type] # cupy arrays support copyto too - return out + return dst diff --git a/src/numcodecs/compat_ext.pyi b/src/numcodecs/compat_ext.pyi new file mode 100644 index 00000000..f5388bfb --- /dev/null +++ b/src/numcodecs/compat_ext.pyi @@ -0,0 +1,3 @@ +from collections.abc import Buffer + +def ensure_contiguous_memoryview(obj: Buffer) -> memoryview: ... diff --git a/src/numcodecs/delta.py b/src/numcodecs/delta.py index e41f8be9..d21a1ec2 100644 --- a/src/numcodecs/delta.py +++ b/src/numcodecs/delta.py @@ -1,7 +1,18 @@ +from typing import Any, overload + import numpy as np +import numpy.typing as npt -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_ndarray, ndarray_copy +from .ndarray_like import BufferLike + + +class DeltaConfig(CodecConfig): + """The configuration of a `Delta` codec, as returned by `Delta.get_config`.""" + + dtype: str + astype: str class Delta(Codec): @@ -40,7 +51,7 @@ class Delta(Codec): codec_id = 'delta' - def __init__(self, dtype, astype=None): + def __init__(self, dtype: npt.DTypeLike, astype: npt.DTypeLike | None = None) -> None: self.dtype = np.dtype(dtype) if astype is None: self.astype = self.dtype @@ -49,7 +60,7 @@ def __init__(self, dtype, astype=None): if self.dtype == np.dtype(object) or self.astype == np.dtype(object): raise ValueError('object arrays are not supported') - def encode(self, buf): + def encode(self, buf: BufferLike) -> npt.NDArray[Any]: # normalise input arr = ensure_ndarray(buf).view(self.dtype) @@ -66,7 +77,11 @@ def encode(self, buf): enc[1:] = np.diff(arr) return enc - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[Any]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise input enc = ensure_ndarray(buf).view(self.astype) @@ -82,11 +97,11 @@ def decode(self, buf, out=None): # handle output return ndarray_copy(dec, out) - def get_config(self): + def get_config(self) -> DeltaConfig: # override to handle encoding dtypes return {'id': self.codec_id, 'dtype': self.dtype.str, 'astype': self.astype.str} - def __repr__(self): + def __repr__(self) -> str: r = f'{type(self).__name__}(dtype={self.dtype.str!r}' if self.astype != self.dtype: r += f', astype={self.astype.str!r}' diff --git a/src/numcodecs/fixedscaleoffset.py b/src/numcodecs/fixedscaleoffset.py index b1db5205..20878e2d 100644 --- a/src/numcodecs/fixedscaleoffset.py +++ b/src/numcodecs/fixedscaleoffset.py @@ -1,7 +1,20 @@ +from typing import Any, overload + import numpy as np +import numpy.typing as npt -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_ndarray, ndarray_copy +from .ndarray_like import BufferLike + + +class FixedScaleOffsetConfig(CodecConfig): + """The configuration of a `FixedScaleOffset` codec, as returned by `FixedScaleOffset.get_config`.""" + + scale: float + offset: float + dtype: str + astype: str class FixedScaleOffset(Codec): @@ -69,7 +82,13 @@ class FixedScaleOffset(Codec): codec_id = 'fixedscaleoffset' - def __init__(self, offset, scale, dtype, astype=None): + def __init__( + self, + offset: float, + scale: float, + dtype: npt.DTypeLike, + astype: npt.DTypeLike | None = None, + ) -> None: self.offset = offset self.scale = scale self.dtype = np.dtype(dtype) @@ -80,7 +99,7 @@ def __init__(self, offset, scale, dtype, astype=None): if self.dtype == np.dtype(object) or self.astype == np.dtype(object): raise ValueError('object arrays are not supported') - def encode(self, buf): + def encode(self, buf: BufferLike) -> npt.NDArray[Any]: # normalise input arr = ensure_ndarray(buf).view(self.dtype) @@ -96,7 +115,11 @@ def encode(self, buf): # convert dtype return enc.astype(self.astype, copy=False) - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[Any]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # interpret buffer as numpy array enc = ensure_ndarray(buf).view(self.astype) @@ -112,7 +135,7 @@ def decode(self, buf, out=None): # handle output return ndarray_copy(dec, out) - def get_config(self): + def get_config(self) -> FixedScaleOffsetConfig: # override to handle encoding dtypes return { 'id': self.codec_id, @@ -122,7 +145,7 @@ def get_config(self): 'astype': self.astype.str, } - def __repr__(self): + def __repr__(self) -> str: r = f'{type(self).__name__}(scale={self.scale}, offset={self.offset}, dtype={self.dtype.str!r}' if self.astype != self.dtype: r += f', astype={self.astype.str!r}' diff --git a/src/numcodecs/fletcher32.pyi b/src/numcodecs/fletcher32.pyi new file mode 100644 index 00000000..729bdb29 --- /dev/null +++ b/src/numcodecs/fletcher32.pyi @@ -0,0 +1,16 @@ +from typing import overload + +import numpy as np +import numpy.typing as npt + +from .abc import Codec, CodecConfig +from .ndarray_like import BufferLike + +class Fletcher32(Codec): + codec_id: str + def get_config(self) -> CodecConfig: ... + def encode(self, buf: BufferLike) -> bytes: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[np.uint8]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... diff --git a/src/numcodecs/gzip.py b/src/numcodecs/gzip.py index 4269bad9..3387dbf1 100644 --- a/src/numcodecs/gzip.py +++ b/src/numcodecs/gzip.py @@ -1,8 +1,16 @@ import gzip as _gzip import io +from typing import cast, overload -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_bytes, ensure_contiguous_ndarray +from .ndarray_like import BufferLike + + +class GZipConfig(CodecConfig): + """The configuration of a `GZip` codec, as returned by `GZip.get_config`.""" + + level: int class GZip(Codec): @@ -17,34 +25,39 @@ class GZip(Codec): codec_id = 'gzip' - def __init__(self, level=1): + def __init__(self, level: int = 1) -> None: self.level = level - def encode(self, buf): + def get_config(self) -> GZipConfig: + return cast(GZipConfig, super().get_config()) + + def encode(self, buf: BufferLike) -> bytes: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) # do compression compressed = io.BytesIO() with _gzip.GzipFile(fileobj=compressed, mode='wb', compresslevel=self.level) as compressor: - compressor.write(buf) + compressor.write(arr) return compressed.getvalue() - # noinspection PyMethodMayBeStatic - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise inputs # BytesIO only copies if the data is not of `bytes` type. # This allows `bytes` objects to pass through without copying. - buf = io.BytesIO(ensure_bytes(buf)) + compressed = io.BytesIO(ensure_bytes(buf)) # do decompression - with _gzip.GzipFile(fileobj=buf, mode='rb') as decompressor: - if out is not None: - out_view = ensure_contiguous_ndarray(out) - decompressor.readinto(out_view) - if decompressor.read(1) != b'': - raise ValueError("Unable to fit data into `out`") - else: - out = decompressor.read() + with _gzip.GzipFile(fileobj=compressed, mode='rb') as decompressor: + if out is None: + return decompressor.read() + out_view = ensure_contiguous_ndarray(out) + decompressor.readinto(out_view) + if decompressor.read(1) != b'': + raise ValueError("Unable to fit data into `out`") return out diff --git a/src/numcodecs/jenkins.pyi b/src/numcodecs/jenkins.pyi new file mode 100644 index 00000000..c8673b7f --- /dev/null +++ b/src/numcodecs/jenkins.pyi @@ -0,0 +1,3 @@ +from collections.abc import Buffer + +def jenkins_lookup3(_data: Buffer, initval: int = 0) -> int: ... diff --git a/src/numcodecs/lz4.pyi b/src/numcodecs/lz4.pyi new file mode 100644 index 00000000..71d941f6 --- /dev/null +++ b/src/numcodecs/lz4.pyi @@ -0,0 +1,28 @@ +from typing import Final, overload + +from .abc import Codec, CodecConfig +from .ndarray_like import BufferLike + +VERSION_STRING: Final[str] +DEFAULT_ACCELERATION: Final[int] + +def compress(source: BufferLike, acceleration: int = ...) -> bytes: ... +@overload +def decompress(source: BufferLike, dest: None = None) -> bytes: ... +@overload +def decompress[OutT: BufferLike](source: BufferLike, dest: OutT) -> OutT: ... + +class LZ4Config(CodecConfig): + acceleration: int + +class LZ4(Codec): + codec_id: str + max_buffer_size: int + acceleration: int + def __init__(self, acceleration: int = ...) -> None: ... + def get_config(self) -> LZ4Config: ... + def encode(self, buf: BufferLike) -> bytes: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... diff --git a/src/numcodecs/lz4.pyx b/src/numcodecs/lz4.pyx index 6eac246b..a0017673 100644 --- a/src/numcodecs/lz4.pyx +++ b/src/numcodecs/lz4.pyx @@ -14,10 +14,17 @@ from cpython.memoryview cimport PyMemoryView_GET_BUFFER from .compat_ext cimport PyBytes_RESIZE, ensure_contiguous_memoryview from ._utils cimport store_le32, load_le32 +from typing import TypedDict + from .compat import ensure_contiguous_ndarray from .abc import Codec +# The type of `LZ4.get_config()`. Declared with the functional syntax because Cython does not +# compile TypedDict class bodies; `lz4.pyi` declares the same type with the class syntax. +LZ4Config = TypedDict("LZ4Config", {"id": str, "acceleration": int}) + + cdef extern from "lz4.h": const char* LZ4_versionString() nogil diff --git a/src/numcodecs/lzma.py b/src/numcodecs/lzma.py index 97da0282..c5039e45 100644 --- a/src/numcodecs/lzma.py +++ b/src/numcodecs/lzma.py @@ -1,4 +1,5 @@ from types import ModuleType +from typing import Any, cast, overload _lzma: ModuleType | None = None try: @@ -11,8 +12,19 @@ if _lzma: - from .abc import Codec + from .abc import Codec, CodecConfig from .compat import ensure_contiguous_ndarray, ndarray_copy + from .ndarray_like import BufferLike + + _lzma_module: ModuleType = _lzma # narrowed from `ModuleType | None` for the methods below + + class LZMAConfig(CodecConfig): + """The configuration of an `LZMA` codec, as returned by `LZMA.get_config`.""" + + format: int + check: int + preset: int | None + filters: list[dict[str, Any]] | None # noinspection PyShadowingBuiltins class LZMA(Codec): @@ -36,36 +48,49 @@ class LZMA(Codec): codec_id = 'lzma' - def __init__(self, format=1, check=-1, preset=None, filters=None): + def __init__( + self, + format: int = 1, + check: int = -1, + preset: int | None = None, + filters: list[dict[str, Any]] | None = None, + ) -> None: self.format = format self.check = check self.preset = preset self.filters = filters - def encode(self, buf): + def get_config(self) -> LZMAConfig: + return cast(LZMAConfig, super().get_config()) + + def encode(self, buf: BufferLike) -> bytes: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) # do compression - return _lzma.compress( - buf, + return _lzma_module.compress( + arr, format=self.format, check=self.check, preset=self.preset, filters=self.filters, ) - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) if out is not None: ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression - dec = _lzma.decompress(buf, format=self.format, filters=self.filters) + dec = _lzma_module.decompress(arr, format=self.format, filters=self.filters) # handle destination return ndarray_copy(dec, out) - def __repr__(self): + def __repr__(self) -> str: return f'{type(self).__name__}(format={self.format!r}, check={self.check!r}, preset={self.preset!r}, filters={self.filters!r})' diff --git a/src/numcodecs/meson.build b/src/numcodecs/meson.build index 41a3a7a2..019b81db 100644 --- a/src/numcodecs/meson.build +++ b/src/numcodecs/meson.build @@ -286,5 +286,14 @@ py.install_sources( 'zarr3.py', 'zfpy.py', 'zlib.py', + # type stubs for the Cython extension modules + '_shuffle.pyi', + 'blosc.pyi', + 'compat_ext.pyi', + 'fletcher32.pyi', + 'jenkins.pyi', + 'lz4.pyi', + 'vlen.pyi', + 'zstd.pyi', subdir: 'numcodecs', ) diff --git a/src/numcodecs/ndarray_like.py b/src/numcodecs/ndarray_like.py index 06e15ea9..be4171bd 100644 --- a/src/numcodecs/ndarray_like.py +++ b/src/numcodecs/ndarray_like.py @@ -1,4 +1,12 @@ -from typing import Any, ClassVar, Protocol, runtime_checkable +from collections.abc import Buffer +from typing import Any, Protocol, Self, runtime_checkable + +from typing_extensions import TypeIs # in `typing` from Python 3.13 + +# Kept at module level, not as an annotated attribute of the metaclass: a protocol class +# without annotations of its own would otherwise find the metaclass's `__annotations__`, and +# `runtime_checkable` would then require instances to have the cache as an attribute. +_instancecheck_cache: dict[tuple[type, type], bool] = {} class _CachedProtocolMeta(Protocol.__class__): # type: ignore[name-defined] @@ -11,55 +19,89 @@ class _CachedProtocolMeta(Protocol.__class__): # type: ignore[name-defined] isinstance checks using the object's class as the cache key. """ - _instancecheck_cache: ClassVar[dict[tuple[type, type], bool]] = {} - - def __instancecheck__(self, instance): + def __instancecheck__(self, instance: object) -> bool: key = (self, instance.__class__) - ret = self._instancecheck_cache.get(key) + ret = _instancecheck_cache.get(key) if ret is None: ret = super().__instancecheck__(instance) - self._instancecheck_cache[key] = ret + _instancecheck_cache[key] = ret return ret +# The protocols below declare their attributes as read-only properties. numpy (and cupy) +# expose `dtype`, `shape`, `flags` and friends as read-only properties, and a protocol that +# declared them as plain (settable) attributes would not match a real ndarray. + + @runtime_checkable class DType(Protocol, metaclass=_CachedProtocolMeta): - itemsize: int - name: str - kind: str + @property + def itemsize(self) -> int: ... # pragma: no cover + @property + def name(self) -> str: ... # pragma: no cover + @property + def kind(self) -> str: ... # pragma: no cover + @property + def str(self) -> str: ... # pragma: no cover @runtime_checkable class FlagsObj(Protocol, metaclass=_CachedProtocolMeta): - c_contiguous: bool - f_contiguous: bool - owndata: bool + @property + def c_contiguous(self) -> bool: ... # pragma: no cover + @property + def f_contiguous(self) -> bool: ... # pragma: no cover + @property + def owndata(self) -> bool: ... # pragma: no cover @runtime_checkable class NDArrayLike(Protocol, metaclass=_CachedProtocolMeta): - dtype: DType - shape: tuple[int, ...] - strides: tuple[int, ...] - ndim: int - size: int - itemsize: int - nbytes: int - flags: FlagsObj + """The part of the ndarray interface numcodecs relies on. + + `numpy.ndarray` and `cupy.ndarray` both satisfy it. Methods return `Any` because numpy + overloads them heavily; convert with `numcodecs.compat.ensure_ndarray` when you need + numpy's own types. + """ + + @property + def dtype(self) -> DType: ... # pragma: no cover + @property + def shape(self) -> tuple[int, ...]: ... # pragma: no cover + @property + def strides(self) -> tuple[int, ...]: ... # pragma: no cover + @property + def ndim(self) -> int: ... # pragma: no cover + @property + def size(self) -> int: ... # pragma: no cover + @property + def itemsize(self) -> int: ... # pragma: no cover + @property + def nbytes(self) -> int: ... # pragma: no cover + @property + def flags(self) -> FlagsObj: ... # pragma: no cover def __len__(self) -> int: ... # pragma: no cover - def __getitem__(self, key) -> Any: ... # pragma: no cover + def __getitem__(self, key: Any, /) -> Any: ... # pragma: no cover + + def __setitem__(self, key: Any, value: Any, /) -> None: ... # pragma: no cover + + def tobytes(self, order: Any = ...) -> bytes: ... # pragma: no cover + + def reshape(self, *args: Any, **kwargs: Any) -> Any: ... # pragma: no cover - def __setitem__(self, key, value): ... # pragma: no cover + def view(self, *args: Any, **kwargs: Any) -> Any: ... # pragma: no cover - def tobytes(self, order: str | None = ...) -> bytes: ... # pragma: no cover + def copy(self, *args: Any, **kwargs: Any) -> Self: ... # pragma: no cover - def reshape(self, *shape: int, order: str = ...) -> "NDArrayLike": ... # pragma: no cover - def view(self, dtype: DType = ...) -> "NDArrayLike": ... # pragma: no cover +type BufferLike = Buffer | NDArrayLike +"""What codecs accept and return: an object exporting the buffer protocol (PEP 688), such +as `bytes`, `bytearray`, `memoryview` or a numpy array, or an ndarray-like such as a cupy +array.""" -def is_ndarray_like(obj: object) -> bool: +def is_ndarray_like(obj: object) -> TypeIs[NDArrayLike]: """Return True when `obj` is ndarray-like""" return isinstance(obj, NDArrayLike) diff --git a/src/numcodecs/packbits.py b/src/numcodecs/packbits.py index bcdcf866..b133628e 100644 --- a/src/numcodecs/packbits.py +++ b/src/numcodecs/packbits.py @@ -1,7 +1,11 @@ +from typing import cast, overload + import numpy as np +import numpy.typing as npt -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_ndarray, ndarray_copy +from .ndarray_like import BufferLike class PackBits(Codec): @@ -29,7 +33,10 @@ class PackBits(Codec): codec_id = 'packbits' - def encode(self, buf): + def get_config(self) -> CodecConfig: + return cast(CodecConfig, super().get_config()) + + def encode(self, buf: BufferLike) -> npt.NDArray[np.uint8]: # normalise input arr = ensure_ndarray(buf).view(bool) @@ -58,7 +65,11 @@ def encode(self, buf): return enc - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[np.bool_]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise input enc = ensure_ndarray(buf).view('u1') diff --git a/src/numcodecs/quantize.py b/src/numcodecs/quantize.py index 329a6d3d..7511155c 100644 --- a/src/numcodecs/quantize.py +++ b/src/numcodecs/quantize.py @@ -1,9 +1,20 @@ import math +from typing import Any, overload import numpy as np +import numpy.typing as npt -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_ndarray, ndarray_copy +from .ndarray_like import BufferLike + + +class QuantizeConfig(CodecConfig): + """The configuration of a `Quantize` codec, as returned by `Quantize.get_config`.""" + + digits: int + dtype: str + astype: str class Quantize(Codec): @@ -47,7 +58,9 @@ class Quantize(Codec): codec_id = 'quantize' - def __init__(self, digits, dtype, astype=None): + def __init__( + self, digits: int, dtype: npt.DTypeLike, astype: npt.DTypeLike | None = None + ) -> None: self.digits = digits self.dtype = np.dtype(dtype) if astype is None: @@ -57,7 +70,7 @@ def __init__(self, digits, dtype, astype=None): if self.dtype.kind != 'f' or self.astype.kind != 'f': raise ValueError('only floating point data types are supported') - def encode(self, buf): + def encode(self, buf: BufferLike) -> npt.NDArray[Any]: # normalise input arr = ensure_ndarray(buf).view(self.dtype) @@ -75,13 +88,17 @@ def encode(self, buf): # cast dtype return enc.astype(self.astype, copy=False) - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[Any]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # filter is lossy, decoding is no-op dec = ensure_ndarray(buf).view(self.astype) dec = dec.astype(self.dtype, copy=False) return ndarray_copy(dec, out) - def get_config(self): + def get_config(self) -> QuantizeConfig: # override to handle encoding dtypes return { 'id': self.codec_id, @@ -90,7 +107,7 @@ def get_config(self): 'astype': self.astype.str, } - def __repr__(self): + def __repr__(self) -> str: r = f'{type(self).__name__}(digits={self.digits}, dtype={self.dtype.str!r}' if self.astype != self.dtype: r += f', astype={self.astype.str!r}' diff --git a/src/numcodecs/registry.py b/src/numcodecs/registry.py index ccb5b26b..fa2a5f59 100644 --- a/src/numcodecs/registry.py +++ b/src/numcodecs/registry.py @@ -2,17 +2,22 @@ applications to dynamically register and look-up codec classes.""" import logging -from importlib.metadata import EntryPoints, entry_points +import warnings +from collections.abc import Mapping +from importlib.metadata import EntryPoint, entry_points +from typing import Any from numcodecs.abc import Codec from numcodecs.errors import UnknownCodecError logger = logging.getLogger("numcodecs") -codec_registry: dict[str, Codec] = {} -entries: dict[str, EntryPoints] = {} +codec_registry: dict[str, type[Codec]] = {} +"""Codec classes by codec id.""" +entries: dict[str, EntryPoint] = {} +"""Not-yet-loaded codec classes from the `numcodecs.codecs` entry point group, by codec id.""" -def run_entrypoints(): +def run_entrypoints() -> None: entries.clear() eps = entry_points() entries.update({e.name: e for e in eps.select(group="numcodecs.codecs")}) @@ -21,7 +26,7 @@ def run_entrypoints(): run_entrypoints() -def get_codec(config): +def get_codec(config: Mapping[str, Any]) -> Codec: """Obtain a codec for the given configuration. Parameters @@ -44,17 +49,25 @@ def get_codec(config): """ config = dict(config) codec_id = config.pop('id', None) - cls = codec_registry.get(codec_id) + cls = codec_registry.get(codec_id) if isinstance(codec_id, str) else None if cls is None and codec_id in entries: logger.debug("Auto loading codec '%s' from entrypoint", codec_id) cls = entries[codec_id].load() + if not (isinstance(cls, type) and issubclass(cls, Codec)): + warnings.warn( + f"The entry point for codec {codec_id!r} loads {cls!r}, which is not a " + "subclass of numcodecs.abc.Codec. Support for such codecs is deprecated " + "and will be removed in a future release.", + DeprecationWarning, + stacklevel=2, + ) register_codec(cls, codec_id=codec_id) if cls: return cls.from_config(config) raise UnknownCodecError(f"{codec_id!r}") -def register_codec(cls, codec_id=None): +def register_codec(cls: type[Codec], codec_id: str | None = None) -> None: """Register a codec class. Parameters @@ -69,6 +82,8 @@ def register_codec(cls, codec_id=None): """ if codec_id is None: - codec_id = cls.codec_id + codec_id = getattr(cls, "codec_id", None) + if codec_id is None: + raise ValueError(f"cannot register {cls!r}: it has no codec_id and none was given") logger.debug("Registering codec '%s'", codec_id) codec_registry[codec_id] = cls diff --git a/src/numcodecs/shuffle.py b/src/numcodecs/shuffle.py index e29e5a59..8f44315d 100644 --- a/src/numcodecs/shuffle.py +++ b/src/numcodecs/shuffle.py @@ -1,8 +1,18 @@ +from typing import cast, overload + import numpy as np +import numpy.typing as npt from ._shuffle import _doShuffle, _doUnshuffle -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_contiguous_ndarray +from .ndarray_like import BufferLike + + +class ShuffleConfig(CodecConfig): + """The configuration of a `Shuffle` codec, as returned by `Shuffle.get_config`.""" + + elementsize: int class Shuffle(Codec): @@ -17,10 +27,15 @@ class Shuffle(Codec): codec_id = 'shuffle' - def __init__(self, elementsize=4): + def __init__(self, elementsize: int = 4) -> None: self.elementsize = elementsize - def _prepare_arrays(self, buf, out): + def get_config(self) -> ShuffleConfig: + return cast(ShuffleConfig, super().get_config()) + + def _prepare_arrays( + self, buf: BufferLike, out: BufferLike | None + ) -> tuple[npt.NDArray[np.generic], npt.NDArray[np.generic]]: buf = ensure_contiguous_ndarray(buf) if out is None: @@ -37,7 +52,11 @@ def _prepare_arrays(self, buf, out): return buf, out - def encode(self, buf, out=None): + @overload + def encode(self, buf: BufferLike, out: None = None) -> npt.NDArray[np.uint8]: ... + @overload + def encode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def encode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: buf, out_arr = self._prepare_arrays(buf, out) if self.elementsize > 1: @@ -45,7 +64,11 @@ def encode(self, buf, out=None): return out_arr if out is None else out - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[np.uint8]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: buf, out_arr = self._prepare_arrays(buf, out) if self.elementsize > 1: @@ -53,5 +76,5 @@ def decode(self, buf, out=None): return out_arr if out is None else out - def __repr__(self): + def __repr__(self) -> str: return f'{type(self).__name__}(elementsize={self.elementsize})' diff --git a/src/numcodecs/vlen.pyi b/src/numcodecs/vlen.pyi new file mode 100644 index 00000000..a4443000 --- /dev/null +++ b/src/numcodecs/vlen.pyi @@ -0,0 +1,41 @@ +from collections.abc import Iterable +from typing import Any, overload + +import numpy as np +import numpy.typing as npt + +from .abc import Codec +from .ndarray_like import BufferLike + +def check_out_param(out: npt.NDArray[np.object_], n_items: int) -> npt.NDArray[np.object_]: ... + +# VLenUTF8 and VLenBytes also accept a list (or other iterable) of str or bytes. VLenArray +# needs an object array: numpy would turn a list of equal-length arrays into a 2-D array. + +class VLenUTF8(Codec): + codec_id: str + def __init__(self) -> None: ... + def encode(self, buf: BufferLike | Iterable[str]) -> bytearray: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[np.object_]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + +class VLenBytes(Codec): + codec_id: str + def __init__(self) -> None: ... + def encode(self, buf: BufferLike | Iterable[bytes]) -> bytearray: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[np.object_]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + +class VLenArray(Codec): + codec_id: str + dtype: np.dtype[Any] + def __init__(self, dtype: npt.DTypeLike) -> None: ... + def encode(self, buf: BufferLike) -> bytearray: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[np.object_]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... diff --git a/src/numcodecs/zlib.py b/src/numcodecs/zlib.py index e62658dc..262a92fa 100644 --- a/src/numcodecs/zlib.py +++ b/src/numcodecs/zlib.py @@ -1,7 +1,15 @@ import zlib as _zlib +from typing import cast, overload -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_contiguous_ndarray, ndarray_copy +from .ndarray_like import BufferLike + + +class ZlibConfig(CodecConfig): + """The configuration of a `Zlib` codec, as returned by `Zlib.get_config`.""" + + level: int class Zlib(Codec): @@ -16,25 +24,31 @@ class Zlib(Codec): codec_id = 'zlib' - def __init__(self, level=1): + def __init__(self, level: int = 1) -> None: self.level = level - def encode(self, buf): + def get_config(self) -> ZlibConfig: + return cast(ZlibConfig, super().get_config()) + + def encode(self, buf: BufferLike) -> bytes: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) # do compression - return _zlib.compress(buf, self.level) + return _zlib.compress(arr, self.level) - # noinspection PyMethodMayBeStatic - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) if out is not None: ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression - dec = _zlib.decompress(buf) + dec = _zlib.decompress(arr) # handle destination - Python standard library zlib module does not # support direct decompression into buffer, so we have to copy into diff --git a/src/numcodecs/zstd.pyi b/src/numcodecs/zstd.pyi new file mode 100644 index 00000000..50672763 --- /dev/null +++ b/src/numcodecs/zstd.pyi @@ -0,0 +1,40 @@ +from typing import Final, overload + +from .abc import Codec, CodecConfig +from .ndarray_like import BufferLike + +VERSION_NUMBER: Final[int] +MAJOR_VERSION_NUMBER: Final[int] +MINOR_VERSION_NUMBER: Final[int] +MICRO_VERSION_NUMBER: Final[int] +DEFAULT_CLEVEL: Final[int] +MAX_CLEVEL: Final[int] + +def compress(source: BufferLike, level: int = ..., checksum: bool = False) -> bytes: ... +@overload +def decompress(source: BufferLike, dest: None = None) -> bytes: ... +@overload +def decompress[OutT: BufferLike](source: BufferLike, dest: OutT) -> OutT: ... + +class ZstdConfig(CodecConfig): + level: int + checksum: bool + +class Zstd(Codec): + codec_id: str + max_buffer_size: int + level: int + checksum: bool + def __init__(self, level: int = ..., checksum: bool = False) -> None: ... + def get_config(self) -> ZstdConfig: ... + def encode(self, buf: BufferLike) -> bytes: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + @classmethod + def default_level(cls) -> int: ... + @classmethod + def min_level(cls) -> int: ... + @classmethod + def max_level(cls) -> int: ... diff --git a/src/numcodecs/zstd.pyx b/src/numcodecs/zstd.pyx index 7b6ff3c0..5037c8dc 100644 --- a/src/numcodecs/zstd.pyx +++ b/src/numcodecs/zstd.pyx @@ -11,9 +11,16 @@ from cpython.memoryview cimport PyMemoryView_GET_BUFFER from .compat_ext cimport PyBytes_RESIZE, ensure_contiguous_memoryview +from typing import TypedDict + from .compat import ensure_contiguous_ndarray from .abc import Codec + +# The type of `Zstd.get_config()`. Declared with the functional syntax because Cython does not +# compile TypedDict class bodies; `zstd.pyi` declares the same type with the class syntax. +ZstdConfig = TypedDict("ZstdConfig", {"id": str, "level": int, "checksum": bool}) + from libc.stdlib cimport malloc, realloc, free cdef extern from "stdint.h": diff --git a/tests/test_blosc.py b/tests/test_blosc.py index b920f03e..abff0592 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -7,9 +7,8 @@ import pytest try: - from numcodecs.blosc import Blosc - from numcodecs import blosc + from numcodecs.blosc import Blosc except ImportError: # pragma: no cover pytest.skip("numcodecs.blosc not available", allow_module_level=True) diff --git a/tests/test_codec_contract.py b/tests/test_codec_contract.py index bf47cc00..5d871c00 100644 --- a/tests/test_codec_contract.py +++ b/tests/test_codec_contract.py @@ -21,6 +21,7 @@ import numcodecs from numcodecs import pickles from numcodecs.abc import Codec +from numcodecs.compat import ensure_ndarray @dataclass(frozen=True) @@ -147,9 +148,9 @@ def test_decode_into_ndarray_returns_out(codec_id: str) -> None: out = np.empty_like(data) assert codec.decode(enc, out=out) is out if CASES[codec_id].is_object: - assert all(np.array_equal(a, b) for a, b in zip(out, expected, strict=True)) + assert all(np.array_equal(a, b) for a, b in zip(out, ensure_ndarray(expected), strict=True)) else: - assert np.asarray(memoryview(expected)).tobytes() == out.tobytes() + assert ensure_ndarray(expected).tobytes() == out.tobytes() @pytest.mark.parametrize( @@ -161,7 +162,7 @@ def test_decode_into_bytearray_returns_out(codec_id: str) -> None: expected = codec.decode(enc) out = bytearray(data.nbytes) assert codec.decode(enc, out=out) is out - assert bytes(out) == np.asarray(memoryview(expected)).tobytes() + assert bytes(out) == ensure_ndarray(expected).tobytes() @pytest.mark.parametrize("codec_id", _params(set())) diff --git a/tests/test_fletcher32.py b/tests/test_fletcher32.py index adfb31da..3bbefe85 100644 --- a/tests/test_fletcher32.py +++ b/tests/test_fletcher32.py @@ -1,5 +1,6 @@ import numpy as np import pytest + from numcodecs.fletcher32 import Fletcher32 diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py index 511c435c..94249117 100644 --- a/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -1,8 +1,11 @@ +import json + import numpy as np import pytest -from numcodecs.jenkins import jenkins_lookup3 from numcodecs.checksum32 import JenkinsLookup3 +from numcodecs.jenkins import jenkins_lookup3 +from numcodecs.registry import get_codec def test_jenkins_lookup3(): @@ -155,3 +158,13 @@ def test_out_wrong_size(): enc = j.encode(np.arange(8, dtype="uint8")) with pytest.raises(ValueError, match="out has 4 bytes, but the decoded data has 8"): j.decode(enc, out=bytearray(4)) + + +def test_config_with_prefix_round_trips_through_json(): + j = JenkinsLookup3(initval=7, prefix=b"FADB") + config = j.get_config() + assert config == {"id": "jenkins_lookup3", "initval": 7, "prefix": list(b"FADB")} + rebuilt = get_codec(json.loads(json.dumps(config))) + assert rebuilt == j + data = np.arange(8, dtype="uint8") + assert rebuilt.encode(data) == j.encode(data) diff --git a/tests/test_registry.py b/tests/test_registry.py index 93ed8f03..ea44ea10 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -1,4 +1,5 @@ import inspect +import types import pytest @@ -34,10 +35,43 @@ def test_all_classes_registered(): if ( inspect.isclass(obj) and issubclass(obj, numcodecs.abc.Codec) + and not inspect.isabstract(obj) and obj.codec_id not in numcodecs.registry.codec_registry - and obj.codec_id is not None # remove `None` ) } if missing: raise Exception(f"these codecs are missing: {missing}") # pragma: no cover + + +class _DuckCodec: + """Looks like a codec, but does not subclass numcodecs.abc.Codec.""" + + codec_id = "duck" + + @classmethod + def from_config(cls, config): + return cls() + + +def test_entry_point_not_a_codec_subclass_is_deprecated(monkeypatch): + monkeypatch.setitem( + numcodecs.registry.entries, "duck", types.SimpleNamespace(load=lambda: _DuckCodec) + ) + monkeypatch.delitem(numcodecs.registry.codec_registry, "duck", raising=False) + with pytest.warns(DeprecationWarning, match="not a subclass of numcodecs.abc.Codec"): + codec = get_codec({"id": "duck"}) + assert isinstance(codec, _DuckCodec) + numcodecs.registry.codec_registry.pop("duck") + + +def test_register_codec_without_id(): + class NoId(numcodecs.abc.Codec): + def encode(self, buf): # pragma: no cover + return buf + + def decode(self, buf, out=None): # pragma: no cover + return buf + + with pytest.raises(ValueError, match="has no codec_id and none was given"): + numcodecs.register_codec(NoId) diff --git a/tests/test_zstd.py b/tests/test_zstd.py index 04f82383..b47363c7 100644 --- a/tests/test_zstd.py +++ b/tests/test_zstd.py @@ -3,8 +3,8 @@ import numpy as np import pytest -from numcodecs.zstd import Zstd +from numcodecs.zstd import Zstd from tests.common import ( check_backwards_compatibility, check_config, diff --git a/tests/typing/README.md b/tests/typing/README.md index a885a096..c9f32fcf 100644 --- a/tests/typing/README.md +++ b/tests/typing/README.md @@ -21,5 +21,16 @@ configuration in `pyproject.toml` covers those. `pyrightconfig.json` turns off pyright's three `reportUnknown*` rules, because most numcodecs parameters are not annotated yet. Turn them back on once they are. +The Cython extension modules carry their types in `.pyi` stubs next to the `.pyx` sources. +`mypy.stubtest` compares those stubs with the compiled modules, against an installed +(non-editable) build: + +```bash +MYPYPATH=src python -m mypy.stubtest --mypy-config-file pyproject.toml \ + --allowlist tests/typing/stubtest-allowlist.txt \ + numcodecs.blosc numcodecs.zstd numcodecs.lz4 numcodecs.vlen \ + numcodecs.fletcher32 numcodecs.jenkins numcodecs.compat_ext numcodecs._shuffle +``` + When you change an annotation in numcodecs, add or update an `assert_type` here that shows what callers now get. diff --git a/tests/typing/mypy.ini b/tests/typing/mypy.ini index 26579ad7..0159ab2f 100644 --- a/tests/typing/mypy.ini +++ b/tests/typing/mypy.ini @@ -2,6 +2,8 @@ # Run from the repository root: `mypy --config-file tests/typing/mypy.ini tests/typing` [mypy] strict = true +# the oldest supported Python, so names that only exist in newer `typing` modules are caught +python_version = 3.12 warn_unreachable = true enable_error_code = ignore-without-code, redundant-expr, truthy-bool # Read numcodecs from the source tree. It does not ship a py.typed marker yet, so mypy diff --git a/tests/typing/pyrightconfig.json b/tests/typing/pyrightconfig.json index 42540aad..7a389763 100644 --- a/tests/typing/pyrightconfig.json +++ b/tests/typing/pyrightconfig.json @@ -10,5 +10,7 @@ "reportMissingTypeStubs": "none", "reportUnknownVariableType": "none", "reportUnknownMemberType": "none", - "reportUnknownArgumentType": "none" + "reportUnknownArgumentType": "none", + "reportMissingModuleSource": "none", + "enableTypeIgnoreComments": false } diff --git a/tests/typing/stubtest-allowlist.txt b/tests/typing/stubtest-allowlist.txt new file mode 100644 index 00000000..01daff8d --- /dev/null +++ b/tests/typing/stubtest-allowlist.txt @@ -0,0 +1,14 @@ +# Entries for `python -m mypy.stubtest`, one regular expression per line. + +# Cython adds a `__test__` dict (collected doctests) to every module. +numcodecs\..*\.__test__ + +# Cython reports `def f():` module functions as `(self, /)` at runtime. +numcodecs\.blosc\.(_init|_destroy|_get_use_threads|get_nthreads|list_compressors) + +# Cython's C-API capsule table for cimport-able functions. +numcodecs\.compat_ext\.__pyx_capi__ + +# The config TypedDicts are created at runtime with `typing.TypedDict`, which adds these +# PEP 705 attributes; stubtest does not expect them on the stubs' class-syntax TypedDicts. +numcodecs\..*Config\.__(mutable|readonly)_keys__ diff --git a/tests/typing/test_abc_types.py b/tests/typing/test_abc_types.py new file mode 100644 index 00000000..ababbebb --- /dev/null +++ b/tests/typing/test_abc_types.py @@ -0,0 +1,92 @@ +"""Static types of `numcodecs.abc`, checked against runtime behavior.""" + +from collections.abc import Mapping +from typing import Any, Self, assert_type, overload + +import numpy as np +import numpy.typing as npt + +import numcodecs +from numcodecs.abc import Codec, CodecProtocol +from numcodecs.compat import ensure_bytes, ndarray_copy +from numcodecs.ndarray_like import BufferLike + + +def _zlib() -> Codec: + return numcodecs.get_codec({"id": "zlib", "level": 1}) + + +def test_codec_methods() -> None: + codec = _zlib() + data = np.arange(16, dtype="f8") + + assert_type(codec.codec_id, str) + assert codec.codec_id == "zlib" + + enc = assert_type(codec.encode(data), BufferLike) + dec = assert_type(codec.decode(enc), BufferLike) + assert ensure_bytes(dec) == data.tobytes() + + out: npt.NDArray[np.float64] = np.empty_like(data) + assert assert_type(codec.decode(enc, out=out), npt.NDArray[np.float64]) is out + buf = bytearray(data.nbytes) + assert assert_type(codec.decode(enc, out=buf), bytearray) is buf + + config = assert_type(codec.get_config(), Mapping[str, Any]) + assert config["id"] == "zlib" + rebuilt = assert_type(type(codec).from_config({"level": 1}), Codec) + assert rebuilt == codec + + assert_type(codec == object(), bool) + + +def test_codec_satisfies_codec_protocol() -> None: + codec: CodecProtocol = _zlib() + assert codec.codec_id == "zlib" + + +class _Identity(Codec): + """A third-party codec typed as precisely as numcodecs' own.""" + + codec_id = "test-identity" + + def encode(self, buf: BufferLike, /) -> bytes: + return ensure_bytes(buf) + + @overload + def decode(self, buf: BufferLike, /, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, /, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, /, out: BufferLike | None = None) -> BufferLike: + return ndarray_copy(ensure_bytes(buf), out) + + +class _AnyTyped(Codec): + """A third-party codec annotated with `Any`, as imagecodecs does.""" + + codec_id = "test-any" + + def encode(self, buf: Any) -> Any: + return buf + + def decode(self, buf: Any, out: Any = None) -> Any: + return buf + + def get_config(self) -> dict[str, Any]: + return {"id": self.codec_id} + + @classmethod + def from_config(cls, config: Mapping[str, Any]) -> Self: + return cls() + + +def test_subclasses() -> None: + identity = _Identity() + assert assert_type(identity.encode(b"abc"), bytes) == b"abc" + assert assert_type(identity.decode(b"abc"), bytes) == b"abc" + out = bytearray(3) + assert assert_type(identity.decode(b"abc", out=out), bytearray) is out + + any_typed = _AnyTyped() + assert any_typed.decode(any_typed.encode(b"abc")) == b"abc" + assert_type(_AnyTyped.from_config({}), _AnyTyped) diff --git a/tests/typing/test_checksum_types.py b/tests/typing/test_checksum_types.py new file mode 100644 index 00000000..819a0b86 --- /dev/null +++ b/tests/typing/test_checksum_types.py @@ -0,0 +1,45 @@ +"""Static types of the checksum codecs, checked against runtime behavior.""" + +from typing import assert_type + +import numpy as np +import numpy.typing as npt + +from numcodecs.abc import CodecConfig +from numcodecs.checksum32 import ( + CRC32, + Adler32, + Checksum32Config, + JenkinsLookup3, + JenkinsLookup3Config, +) +from numcodecs.fletcher32 import Fletcher32 + +DATA = np.arange(16, dtype="f8") + + +def test_checksum32() -> None: + for codec in (CRC32(), Adler32(), CRC32(location="end")): + enc = assert_type(codec.encode(DATA), npt.NDArray[np.uint8] | bytes) + assert isinstance(enc, np.ndarray) + dec = assert_type(codec.decode(enc), npt.NDArray[np.uint8] | memoryview) + assert bytes(dec) == DATA.tobytes() + out = bytearray(DATA.nbytes) + assert assert_type(codec.decode(enc, out=out), bytearray) is out + config = assert_type(CRC32(location="end").get_config(), Checksum32Config) + assert config.get("location") == "end" + assert "location" not in Adler32().get_config() + + +def test_jenkins_lookup3() -> None: + codec = JenkinsLookup3(prefix=b"FADB") + enc = assert_type(codec.encode(DATA), bytes) + assert isinstance(enc, bytes) + dec = assert_type(codec.decode(enc), memoryview) + assert dec.tobytes() == DATA.tobytes() + config = assert_type(codec.get_config(), JenkinsLookup3Config) + assert assert_type(config["prefix"], list[int] | None) == list(b"FADB") + + +def test_fletcher32() -> None: + assert assert_type(Fletcher32().get_config(), CodecConfig) == {"id": "fletcher32"} diff --git a/tests/typing/test_compat_types.py b/tests/typing/test_compat_types.py index f656c411..e8a5bac1 100644 --- a/tests/typing/test_compat_types.py +++ b/tests/typing/test_compat_types.py @@ -10,7 +10,24 @@ import numpy as np import numpy.typing as npt -from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray, ensure_ndarray +from numcodecs.compat import ( + ensure_bytes, + ensure_contiguous_ndarray, + ensure_ndarray, + ensure_ndarray_like, + ensure_text, + ndarray_copy, +) +from numcodecs.ndarray_like import NDArrayLike, is_ndarray_like + + +def _as_ndarray_like(x: NDArrayLike) -> NDArrayLike: + """Forget the concrete type, as for an ndarray-like that is not a numpy array.""" + return x + + +def _as_object(x: object) -> object: + return x def test_ensure_bytes() -> None: @@ -29,3 +46,42 @@ def test_ensure_contiguous_ndarray() -> None: for buf in (b"abc", bytearray(b"abc"), memoryview(b"abc"), np.arange(3, dtype="u1")): result = assert_type(ensure_contiguous_ndarray(buf), npt.NDArray[Any]) assert isinstance(result, np.ndarray) + + +def test_ensure_ndarray_like() -> None: + from_bytes = assert_type(ensure_ndarray_like(b"abc"), npt.NDArray[Any]) + assert isinstance(from_bytes, np.ndarray) + # an ndarray-like that is not known to export the buffer protocol (such as a cupy array) + # comes back as the protocol type + like = _as_ndarray_like(np.arange(3)) + from_like = assert_type(ensure_ndarray_like(like), NDArrayLike) + assert from_like is like + + +def test_ensure_text() -> None: + for s in ("abc", b"abc", bytearray(b"abc"), memoryview(b"abc")): + result = assert_type(ensure_text(s), str) + assert result == "abc" + + +def test_ndarray_copy() -> None: + src = b"\x01\x02\x03" + # without a destination, the source comes back unchanged + same = assert_type(ndarray_copy(src, None), bytes) + assert same is src + # with a destination, that very object comes back, filled + dst = bytearray(3) + filled = assert_type(ndarray_copy(src, dst), bytearray) + assert filled is dst + assert bytes(dst) == src + arr: npt.NDArray[np.uint8] = np.zeros(3, dtype=np.uint8) + assert assert_type(ndarray_copy(src, arr), npt.NDArray[np.uint8]) is arr + + +def test_is_ndarray_like_narrows() -> None: + obj = _as_object(np.arange(3)) + if is_ndarray_like(obj): + assert_type(obj, NDArrayLike) + assert obj.shape == (3,) + else: # pragma: no cover + raise AssertionError diff --git a/tests/typing/test_compressor_types.py b/tests/typing/test_compressor_types.py new file mode 100644 index 00000000..0adb6fe4 --- /dev/null +++ b/tests/typing/test_compressor_types.py @@ -0,0 +1,78 @@ +"""Static types of the compressor codecs, checked against runtime behavior.""" + +import json +from collections.abc import Mapping +from typing import Any, assert_type + +import numpy as np + +from numcodecs.abc import JSON, ArrayArrayCodec, BytesBytesCodec, CodecConfig +from numcodecs.base64 import Base64 +from numcodecs.blosc import Blosc, BloscConfig +from numcodecs.bz2 import BZ2, BZ2Config +from numcodecs.gzip import GZip, GZipConfig +from numcodecs.lz4 import LZ4, LZ4Config +from numcodecs.lzma import LZMA, LZMAConfig +from numcodecs.zlib import Zlib, ZlibConfig +from numcodecs.zstd import Zstd, ZstdConfig + +DATA = np.arange(16, dtype="f8") + + +def _round_trip(codec: BytesBytesCodec) -> None: + """Every compressor: bytes in any buffer form go in, bytes come out, `out` comes back.""" + enc = codec.encode(DATA) + assert bytes(memoryview(codec.decode(enc))) == DATA.tobytes() + + +def test_compressors() -> None: + for codec in (Zlib(), GZip(), BZ2(), LZMA(), Base64(), Blosc(), Zstd(), LZ4()): + _round_trip(codec) + enc = assert_type(codec.encode(DATA), bytes) + assert isinstance(enc, bytes) + assert assert_type(codec.decode(enc), bytes) == DATA.tobytes() + out = np.empty_like(DATA) + assert codec.decode(enc, out=out) is out + buf = bytearray(DATA.nbytes) + assert assert_type(codec.decode(enc, out=buf), bytearray) is buf + + +def _json_round_trip(config: Any) -> Any: + return json.loads(json.dumps(config)) + + +def test_configs() -> None: + zlib_config = assert_type(Zlib(level=3).get_config(), ZlibConfig) + assert assert_type(zlib_config["level"], int) == 3 + assert assert_type(GZip(level=2).get_config(), GZipConfig)["level"] == 2 + assert assert_type(BZ2(level=4).get_config(), BZ2Config)["level"] == 4 + lzma_config = assert_type(LZMA(preset=5).get_config(), LZMAConfig) + assert assert_type(lzma_config["preset"], int | None) == 5 + assert assert_type(Base64().get_config(), CodecConfig) == {"id": "base64"} + blosc_config = assert_type(Blosc(cname="zstd").get_config(), BloscConfig) + assert assert_type(blosc_config["cname"], str) == "zstd" + assert set(blosc_config) == set(BloscConfig.__annotations__) + zstd_config = assert_type(Zstd(checksum=True).get_config(), ZstdConfig) + assert assert_type(zstd_config["checksum"], bool) is True + assert assert_type(LZ4().get_config(), LZ4Config)["acceleration"] == 1 + for config in (zlib_config, lzma_config, blosc_config, zstd_config): + assert _json_round_trip(config) == config + + +def test_compressors_are_not_array_codecs() -> None: + # A compressor returns bytes, which is not an ndarray-like. + codec: ArrayArrayCodec = Zlib() # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + assert not isinstance(codec.encode(DATA), np.ndarray) + + +def _json_mapping(config: Mapping[str, JSON]) -> Mapping[str, JSON]: + return config + + +def test_configs_are_json_mappings() -> None: + # pyright implements PEP 728, so CodecConfig's read-only JSON `extra_items` make every + # codec configuration a `Mapping[str, JSON]`. mypy does not implement it yet. + zlib_config = _json_mapping(Zlib().get_config()) # type: ignore[arg-type] + blosc_config = _json_mapping(Blosc().get_config()) # type: ignore[arg-type] + for config in (zlib_config, blosc_config): + assert _json_round_trip(config) == config diff --git a/tests/typing/test_cython_types.py b/tests/typing/test_cython_types.py new file mode 100644 index 00000000..ae2f3572 --- /dev/null +++ b/tests/typing/test_cython_types.py @@ -0,0 +1,59 @@ +"""Static types of the Cython codecs, as declared in their `.pyi` stubs, checked against +runtime behavior.""" + +from typing import assert_type + +import numpy as np +import numpy.typing as npt + +from numcodecs.blosc import Blosc +from numcodecs.fletcher32 import Fletcher32 +from numcodecs.jenkins import jenkins_lookup3 +from numcodecs.lz4 import LZ4 +from numcodecs.vlen import VLenArray, VLenBytes, VLenUTF8 +from numcodecs.zstd import Zstd + +DATA = np.arange(16, dtype="f8") + + +def test_compressors() -> None: + for codec in (Blosc(), Zstd(), LZ4()): + enc = assert_type(codec.encode(DATA), bytes) + assert isinstance(enc, bytes) + dec = assert_type(codec.decode(enc), bytes) + assert dec == DATA.tobytes() + out = bytearray(DATA.nbytes) + assert assert_type(codec.decode(enc, out=out), bytearray) is out + + +def test_zstd_levels() -> None: + for level in (Zstd.default_level(), Zstd.min_level(), Zstd.max_level()): + assert isinstance(assert_type(level, int), int) + + +def test_fletcher32() -> None: + codec = Fletcher32() + enc = assert_type(codec.encode(DATA), bytes) + dec = assert_type(codec.decode(enc), npt.NDArray[np.uint8]) + assert isinstance(dec, np.ndarray) + assert dec.tobytes() == DATA.tobytes() + + +def test_jenkins_lookup3() -> None: + assert isinstance(assert_type(jenkins_lookup3(b"abc"), int), int) + + +def test_vlen() -> None: + utf8 = VLenUTF8() + enc = assert_type(utf8.encode(["foo", "bar"]), bytearray) + dec = assert_type(utf8.decode(enc), npt.NDArray[np.object_]) + assert dec.tolist() == ["foo", "bar"] + + raw = VLenBytes() + assert raw.decode(assert_type(raw.encode([b"foo"]), bytearray)).tolist() == [b"foo"] + + arrays = VLenArray(dtype="i4") + values = np.empty(2, dtype=object) + values[0], values[1] = np.arange(3, dtype="i4"), np.arange(2, dtype="i4") + enc = assert_type(arrays.encode(values), bytearray) + assert assert_type(arrays.decode(enc), npt.NDArray[np.object_])[0].tolist() == [0, 1, 2] diff --git a/tests/typing/test_filter_types.py b/tests/typing/test_filter_types.py new file mode 100644 index 00000000..a1eac85d --- /dev/null +++ b/tests/typing/test_filter_types.py @@ -0,0 +1,82 @@ +"""Static types of the filter codecs, checked against runtime behavior.""" + +from typing import Any, assert_type + +import numpy as np +import numpy.typing as npt + +from numcodecs.abc import ArrayArrayCodec, CodecConfig +from numcodecs.astype import AsType, AsTypeConfig +from numcodecs.bitround import BitRound, BitRoundConfig +from numcodecs.categorize import Categorize, CategorizeConfig +from numcodecs.delta import Delta, DeltaConfig +from numcodecs.fixedscaleoffset import FixedScaleOffset, FixedScaleOffsetConfig +from numcodecs.ndarray_like import NDArrayLike +from numcodecs.packbits import PackBits +from numcodecs.quantize import Quantize, QuantizeConfig +from numcodecs.shuffle import Shuffle, ShuffleConfig + +F8 = np.linspace(0, 100, 16, dtype="f8") + + +def _as_array_codec(codec: ArrayArrayCodec) -> ArrayArrayCodec: + return codec + + +def test_numeric_filters() -> None: + codecs = ( + AsType(encode_dtype="f4", decode_dtype="f8"), + Delta(dtype="f8"), + Quantize(digits=1, dtype="f8"), + FixedScaleOffset(offset=0, scale=10, dtype="f8", astype="i4"), + ) + for codec in codecs: + _as_array_codec(codec) + enc = assert_type(codec.encode(F8), npt.NDArray[Any]) + assert isinstance(enc, np.ndarray) + dec = assert_type(codec.decode(enc), npt.NDArray[Any]) + assert isinstance(dec, np.ndarray) + out = np.empty_like(F8) + assert codec.decode(enc, out=out) is out + + +def test_bitround() -> None: + codec = BitRound(keepbits=10) + enc = assert_type(codec.encode(F8), NDArrayLike) + assert isinstance(enc, np.ndarray) + assert isinstance(assert_type(codec.decode(enc), NDArrayLike), np.ndarray) + assert assert_type(codec.get_config(), BitRoundConfig)["keepbits"] == 10 + + +def test_packbits_and_categorize() -> None: + bools = np.array([True, False] * 8) + packbits = PackBits() + enc = assert_type(packbits.encode(bools), npt.NDArray[np.uint8]) + assert enc.dtype == np.uint8 + dec = assert_type(packbits.decode(enc), npt.NDArray[np.bool_]) + assert dec.dtype == np.bool_ + assert assert_type(packbits.get_config(), CodecConfig) == {"id": "packbits"} + + categorize = Categorize(labels=["foo", "bar"], dtype=object) + codes = assert_type(categorize.encode(["foo", "bar", "baz"]), npt.NDArray[Any]) + assert codes.tolist() == [1, 2, 0] + config = assert_type(categorize.get_config(), CategorizeConfig) + assert assert_type(config["labels"], list[str]) == ["foo", "bar"] + + +def test_shuffle() -> None: + codec = Shuffle(elementsize=8) + enc = assert_type(codec.encode(F8), npt.NDArray[np.uint8]) + assert enc.dtype == np.uint8 + out = bytearray(F8.nbytes) + assert assert_type(codec.encode(F8, out=out), bytearray) is out + assert assert_type(codec.decode(enc), npt.NDArray[np.uint8]).tobytes() == F8.tobytes() + assert assert_type(codec.get_config(), ShuffleConfig)["elementsize"] == 8 + + +def test_configs() -> None: + assert assert_type(AsType("f4", "f8").get_config(), AsTypeConfig)["encode_dtype"] == " None: + codec = assert_type(get_codec({"id": "zlib", "level": 1}), Codec) + assert isinstance(codec, numcodecs.Zlib) + + +def test_codec_registry_holds_classes() -> None: + assert_type(codec_registry, dict[str, type[Codec]]) + for cls in codec_registry.values(): + assert isinstance(cls, type) + assert issubclass(cls, Codec) + + +def test_entries_hold_entry_points() -> None: + assert_type(entries, dict[str, EntryPoint]) + for entry in entries.values(): + assert isinstance(entry, EntryPoint)