diff --git a/docs/release.md b/docs/release.md index 51e2933b..bd896673 100644 --- a/docs/release.md +++ b/docs/release.md @@ -47,6 +47,22 @@ ndarray, and raises `ValueError` when `out` has the wrong size. * `Fletcher32.decode` without `out` returns a `uint8` ndarray, like the other checksum codecs, instead of a Cython memoryview slice. +* Correct type annotations that described something other than what the code does: + `numcodecs.registry.codec_registry` holds codec classes (`dict[str, type[Codec]]`) and + `entries` holds single entry points (`dict[str, EntryPoint]`). `numcodecs.compat` + functions now annotate their parameters, and `ndarray_copy` returns the type of `dst` + (or of `src` when `dst` is `None`). The `NDArrayLike`, `DType` and `FlagsObj` protocols + declare read-only properties, so `numpy.ndarray` now satisfies `NDArrayLike` for type + checkers, and `numcodecs.ndarray_like.BufferLike` names "a buffer or an ndarray-like". +* `numcodecs` defines `__all__`, so type checkers treat its re-exported codec classes as + public. `from numcodecs import *` now imports only those names. +* `register_codec` raises `ValueError` for a class without a `codec_id` when no id is + given, instead of registering it under `None`. + +### Deprecations + +* Loading a codec through the `numcodecs.codecs` entry point group whose class does not + subclass `numcodecs.abc.Codec` now emits a `DeprecationWarning`. (release_0.17.1)= diff --git a/pyproject.toml b/pyproject.toml index 721d1812..3050fdfc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -245,6 +245,7 @@ quote-style = "preserve" [tool.mypy] ignore_errors = false ignore_missing_imports = true +python_version = "3.12" enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] # TODO: set options below to true strict = false diff --git a/src/numcodecs/__init__.py b/src/numcodecs/__init__.py index be60bfc3..bad41090 100644 --- a/src/numcodecs/__init__.py +++ b/src/numcodecs/__init__.py @@ -25,6 +25,42 @@ from numcodecs.registry import get_codec as get_codec from numcodecs.registry import register_codec +# Names listed here are the public API re-exported from submodules. Optional codecs are +# appended below when their dependency imports. +__all__ = [ + "BZ2", + "CRC32", + "JSON", + "LZ4", + "LZMA", + "Adler32", + "AsType", + "Base64", + "BitRound", + "Blosc", + "Categorize", + "Delta", + "FixedScaleOffset", + "Fletcher32", + "GZip", + "JenkinsLookup3", + "PackBits", + "Pickle", + "Quantize", + "Shuffle", + "VLenArray", + "VLenBytes", + "VLenUTF8", + "Zlib", + "Zstd", + "blosc", + "get_codec", + "lz4", + "register_codec", + "vlen", + "zstd", +] + __version__: str = _version("numcodecs") from numcodecs.zlib import Zlib @@ -131,18 +167,22 @@ from numcodecs.zfpy import ZFPY register_codec(ZFPY) + __all__ += ["ZFPY"] with suppress(ImportError): from numcodecs.msgpacks import MsgPack register_codec(MsgPack) + __all__ += ["MsgPack"] with suppress(ImportError): from numcodecs.checksum32 import CRC32C register_codec(CRC32C) + __all__ += ["CRC32C"] with suppress(ImportError): from numcodecs.pcodec import PCodec register_codec(PCodec) + __all__ += ["PCodec"] diff --git a/src/numcodecs/compat.py b/src/numcodecs/compat.py index cc4d2d13..bad12fd4 100644 --- a/src/numcodecs/compat.py +++ b/src/numcodecs/compat.py @@ -1,12 +1,19 @@ import array import codecs +from collections.abc import Buffer +from typing import Any, overload import numpy as np +import numpy.typing as npt -from .ndarray_like import NDArrayLike, is_ndarray_like +from .ndarray_like import BufferLike, NDArrayLike, is_ndarray_like -def ensure_ndarray_like(buf) -> NDArrayLike: +@overload +def ensure_ndarray_like(buf: Buffer) -> npt.NDArray[Any]: ... +@overload +def ensure_ndarray_like(buf: NDArrayLike) -> NDArrayLike: ... +def ensure_ndarray_like(buf: BufferLike) -> NDArrayLike: """Convenience function to coerce `buf` to ndarray-like array. Parameters @@ -41,7 +48,7 @@ def ensure_ndarray_like(buf) -> NDArrayLike: return buf -def ensure_ndarray(buf) -> np.ndarray: +def ensure_ndarray(buf: BufferLike) -> npt.NDArray[Any]: """Convenience function to coerce `buf` to a numpy array, if it is not already a numpy array. @@ -63,7 +70,17 @@ def ensure_ndarray(buf) -> np.ndarray: return np.array(ensure_ndarray_like(buf), copy=False) -def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> NDArrayLike: +@overload +def ensure_contiguous_ndarray_like( + buf: Buffer, max_buffer_size: int | None = None, flatten: bool = True +) -> npt.NDArray[Any]: ... +@overload +def ensure_contiguous_ndarray_like( + buf: NDArrayLike, max_buffer_size: int | None = None, flatten: bool = True +) -> NDArrayLike: ... +def ensure_contiguous_ndarray_like( + buf: BufferLike, max_buffer_size: int | None = None, flatten: bool = True +) -> NDArrayLike: """Convenience function to coerce `buf` to ndarray-like array. Also ensures that the returned value exports fully contiguous memory, and supports the new-style buffer interface. If the optional max_buffer_size is @@ -95,12 +112,12 @@ def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> N # check for object arrays, these are just memory pointers, actual memory holding # item data is scattered elsewhere - if arr.dtype == object: + if arr.dtype.kind == "O": raise TypeError("object arrays are not supported") # check for datetime or timedelta ndarray, the buffer interface doesn't support those if arr.dtype.kind in "Mm": - arr = arr.view(np.int64) # type: ignore[arg-type] + arr = arr.view(np.int64) # check memory is contiguous, if so flatten if arr.flags.c_contiguous or arr.flags.f_contiguous: @@ -117,7 +134,9 @@ def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> N return arr -def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.ndarray: +def ensure_contiguous_ndarray( + buf: BufferLike, max_buffer_size: int | None = None, flatten: bool = True +) -> npt.NDArray[Any]: """Convenience function to coerce `buf` to a numpy array, if it is not already a numpy array. Also ensures that the returned value exports fully contiguous memory, and supports the new-style buffer interface. If the optional max_buffer_size is @@ -150,7 +169,7 @@ def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.nda ) -def ensure_bytes(buf) -> bytes: +def ensure_bytes(buf: BufferLike) -> bytes: """Obtain a bytes object from memory exposed by `buf`.""" if not isinstance(buf, bytes): @@ -158,7 +177,7 @@ def ensure_bytes(buf) -> bytes: # check for object arrays, these are just memory pointers, # actual memory holding item data is scattered elsewhere - if arr.dtype == object: + if arr.dtype.kind == "O": raise TypeError("object arrays are not supported") # create bytes @@ -167,14 +186,18 @@ def ensure_bytes(buf) -> bytes: return buf -def ensure_text(s, encoding="utf-8"): - if not isinstance(s, str): - s = ensure_contiguous_ndarray(s) - s = codecs.decode(s, encoding) - return s +def ensure_text(s: str | BufferLike, encoding: str = "utf-8") -> str: + """Return `s` if it is a `str`, else decode the bytes it exposes with `encoding`.""" + if isinstance(s, str): + return s + return codecs.decode(ensure_contiguous_ndarray(s), encoding) -def ndarray_copy(src, dst) -> NDArrayLike: +@overload +def ndarray_copy[SrcT: BufferLike](src: SrcT, dst: None) -> SrcT: ... +@overload +def ndarray_copy[DstT: BufferLike](src: BufferLike, dst: DstT) -> DstT: ... +def ndarray_copy(src: BufferLike, dst: BufferLike | None) -> BufferLike: """Copy the contents of the array from `src` to `dst`, and return `dst`. If `dst` is `None`, return `src` unchanged. Otherwise `dst` itself is returned, not @@ -185,28 +208,26 @@ def ndarray_copy(src, dst) -> NDArrayLike: # no-op return src - out = dst - # ensure ndarray like - src = ensure_ndarray_like(src) - dst = ensure_ndarray_like(dst) + src_arr: NDArrayLike = ensure_ndarray_like(src) + dst_arr: NDArrayLike = ensure_ndarray_like(dst) # flatten source array - src = src.reshape(-1, order="A") + src_arr = src_arr.reshape(-1, order="A") # ensure same data type - if dst.dtype != object: - src = src.view(dst.dtype) + if dst_arr.dtype.kind != "O": + src_arr = src_arr.view(dst_arr.dtype) # reshape source to match destination - if src.shape != dst.shape: - if dst.flags.f_contiguous: + if src_arr.shape != dst_arr.shape: + if dst_arr.flags.f_contiguous: order = "F" else: order = "C" - src = src.reshape(dst.shape, order=order) + src_arr = src_arr.reshape(dst_arr.shape, order=order) # copy via numpy - np.copyto(dst, src) + np.copyto(dst_arr, src_arr) # type: ignore[arg-type] # cupy arrays support copyto too - return out + return dst diff --git a/src/numcodecs/ndarray_like.py b/src/numcodecs/ndarray_like.py index 06e15ea9..be4171bd 100644 --- a/src/numcodecs/ndarray_like.py +++ b/src/numcodecs/ndarray_like.py @@ -1,4 +1,12 @@ -from typing import Any, ClassVar, Protocol, runtime_checkable +from collections.abc import Buffer +from typing import Any, Protocol, Self, runtime_checkable + +from typing_extensions import TypeIs # in `typing` from Python 3.13 + +# Kept at module level, not as an annotated attribute of the metaclass: a protocol class +# without annotations of its own would otherwise find the metaclass's `__annotations__`, and +# `runtime_checkable` would then require instances to have the cache as an attribute. +_instancecheck_cache: dict[tuple[type, type], bool] = {} class _CachedProtocolMeta(Protocol.__class__): # type: ignore[name-defined] @@ -11,55 +19,89 @@ class _CachedProtocolMeta(Protocol.__class__): # type: ignore[name-defined] isinstance checks using the object's class as the cache key. """ - _instancecheck_cache: ClassVar[dict[tuple[type, type], bool]] = {} - - def __instancecheck__(self, instance): + def __instancecheck__(self, instance: object) -> bool: key = (self, instance.__class__) - ret = self._instancecheck_cache.get(key) + ret = _instancecheck_cache.get(key) if ret is None: ret = super().__instancecheck__(instance) - self._instancecheck_cache[key] = ret + _instancecheck_cache[key] = ret return ret +# The protocols below declare their attributes as read-only properties. numpy (and cupy) +# expose `dtype`, `shape`, `flags` and friends as read-only properties, and a protocol that +# declared them as plain (settable) attributes would not match a real ndarray. + + @runtime_checkable class DType(Protocol, metaclass=_CachedProtocolMeta): - itemsize: int - name: str - kind: str + @property + def itemsize(self) -> int: ... # pragma: no cover + @property + def name(self) -> str: ... # pragma: no cover + @property + def kind(self) -> str: ... # pragma: no cover + @property + def str(self) -> str: ... # pragma: no cover @runtime_checkable class FlagsObj(Protocol, metaclass=_CachedProtocolMeta): - c_contiguous: bool - f_contiguous: bool - owndata: bool + @property + def c_contiguous(self) -> bool: ... # pragma: no cover + @property + def f_contiguous(self) -> bool: ... # pragma: no cover + @property + def owndata(self) -> bool: ... # pragma: no cover @runtime_checkable class NDArrayLike(Protocol, metaclass=_CachedProtocolMeta): - dtype: DType - shape: tuple[int, ...] - strides: tuple[int, ...] - ndim: int - size: int - itemsize: int - nbytes: int - flags: FlagsObj + """The part of the ndarray interface numcodecs relies on. + + `numpy.ndarray` and `cupy.ndarray` both satisfy it. Methods return `Any` because numpy + overloads them heavily; convert with `numcodecs.compat.ensure_ndarray` when you need + numpy's own types. + """ + + @property + def dtype(self) -> DType: ... # pragma: no cover + @property + def shape(self) -> tuple[int, ...]: ... # pragma: no cover + @property + def strides(self) -> tuple[int, ...]: ... # pragma: no cover + @property + def ndim(self) -> int: ... # pragma: no cover + @property + def size(self) -> int: ... # pragma: no cover + @property + def itemsize(self) -> int: ... # pragma: no cover + @property + def nbytes(self) -> int: ... # pragma: no cover + @property + def flags(self) -> FlagsObj: ... # pragma: no cover def __len__(self) -> int: ... # pragma: no cover - def __getitem__(self, key) -> Any: ... # pragma: no cover + def __getitem__(self, key: Any, /) -> Any: ... # pragma: no cover + + def __setitem__(self, key: Any, value: Any, /) -> None: ... # pragma: no cover + + def tobytes(self, order: Any = ...) -> bytes: ... # pragma: no cover + + def reshape(self, *args: Any, **kwargs: Any) -> Any: ... # pragma: no cover - def __setitem__(self, key, value): ... # pragma: no cover + def view(self, *args: Any, **kwargs: Any) -> Any: ... # pragma: no cover - def tobytes(self, order: str | None = ...) -> bytes: ... # pragma: no cover + def copy(self, *args: Any, **kwargs: Any) -> Self: ... # pragma: no cover - def reshape(self, *shape: int, order: str = ...) -> "NDArrayLike": ... # pragma: no cover - def view(self, dtype: DType = ...) -> "NDArrayLike": ... # pragma: no cover +type BufferLike = Buffer | NDArrayLike +"""What codecs accept and return: an object exporting the buffer protocol (PEP 688), such +as `bytes`, `bytearray`, `memoryview` or a numpy array, or an ndarray-like such as a cupy +array.""" -def is_ndarray_like(obj: object) -> bool: +def is_ndarray_like(obj: object) -> TypeIs[NDArrayLike]: """Return True when `obj` is ndarray-like""" return isinstance(obj, NDArrayLike) diff --git a/src/numcodecs/registry.py b/src/numcodecs/registry.py index ccb5b26b..758cc008 100644 --- a/src/numcodecs/registry.py +++ b/src/numcodecs/registry.py @@ -2,17 +2,22 @@ applications to dynamically register and look-up codec classes.""" import logging -from importlib.metadata import EntryPoints, entry_points +import warnings +from collections.abc import Mapping +from importlib.metadata import EntryPoint, entry_points +from typing import Any from numcodecs.abc import Codec from numcodecs.errors import UnknownCodecError logger = logging.getLogger("numcodecs") -codec_registry: dict[str, Codec] = {} -entries: dict[str, EntryPoints] = {} +codec_registry: dict[str, type[Codec]] = {} +"""Codec classes by codec id.""" +entries: dict[str, EntryPoint] = {} +"""Not-yet-loaded codec classes from the `numcodecs.codecs` entry point group, by codec id.""" -def run_entrypoints(): +def run_entrypoints() -> None: entries.clear() eps = entry_points() entries.update({e.name: e for e in eps.select(group="numcodecs.codecs")}) @@ -21,7 +26,7 @@ def run_entrypoints(): run_entrypoints() -def get_codec(config): +def get_codec(config: Mapping[str, Any]) -> Codec: """Obtain a codec for the given configuration. Parameters @@ -44,17 +49,25 @@ def get_codec(config): """ config = dict(config) codec_id = config.pop('id', None) - cls = codec_registry.get(codec_id) + cls = codec_registry.get(codec_id) if isinstance(codec_id, str) else None if cls is None and codec_id in entries: logger.debug("Auto loading codec '%s' from entrypoint", codec_id) cls = entries[codec_id].load() + if not (isinstance(cls, type) and issubclass(cls, Codec)): + warnings.warn( + f"The entry point for codec {codec_id!r} loads {cls!r}, which is not a " + "subclass of numcodecs.abc.Codec. Support for such codecs is deprecated " + "and will be removed in a future release.", + DeprecationWarning, + stacklevel=2, + ) register_codec(cls, codec_id=codec_id) if cls: return cls.from_config(config) raise UnknownCodecError(f"{codec_id!r}") -def register_codec(cls, codec_id=None): +def register_codec(cls: type[Codec], codec_id: str | None = None) -> None: """Register a codec class. Parameters @@ -70,5 +83,7 @@ def register_codec(cls, codec_id=None): """ if codec_id is None: codec_id = cls.codec_id + if codec_id is None: + raise ValueError(f"cannot register {cls!r}: it has no codec_id and none was given") logger.debug("Registering codec '%s'", codec_id) codec_registry[codec_id] = cls diff --git a/tests/test_registry.py b/tests/test_registry.py index 93ed8f03..2af73f05 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -1,4 +1,5 @@ import inspect +import types import pytest @@ -41,3 +42,36 @@ def test_all_classes_registered(): if missing: raise Exception(f"these codecs are missing: {missing}") # pragma: no cover + + +class _DuckCodec: + """Looks like a codec, but does not subclass numcodecs.abc.Codec.""" + + codec_id = "duck" + + @classmethod + def from_config(cls, config): + return cls() + + +def test_entry_point_not_a_codec_subclass_is_deprecated(monkeypatch): + monkeypatch.setitem( + numcodecs.registry.entries, "duck", types.SimpleNamespace(load=lambda: _DuckCodec) + ) + monkeypatch.delitem(numcodecs.registry.codec_registry, "duck", raising=False) + with pytest.warns(DeprecationWarning, match="not a subclass of numcodecs.abc.Codec"): + codec = get_codec({"id": "duck"}) + assert isinstance(codec, _DuckCodec) + numcodecs.registry.codec_registry.pop("duck") + + +def test_register_codec_without_id(): + class NoId(numcodecs.abc.Codec): + def encode(self, buf): # pragma: no cover + return buf + + def decode(self, buf, out=None): # pragma: no cover + return buf + + with pytest.raises(ValueError, match="has no codec_id and none was given"): + numcodecs.register_codec(NoId) diff --git a/tests/typing/mypy.ini b/tests/typing/mypy.ini index 26579ad7..0159ab2f 100644 --- a/tests/typing/mypy.ini +++ b/tests/typing/mypy.ini @@ -2,6 +2,8 @@ # Run from the repository root: `mypy --config-file tests/typing/mypy.ini tests/typing` [mypy] strict = true +# the oldest supported Python, so names that only exist in newer `typing` modules are caught +python_version = 3.12 warn_unreachable = true enable_error_code = ignore-without-code, redundant-expr, truthy-bool # Read numcodecs from the source tree. It does not ship a py.typed marker yet, so mypy diff --git a/tests/typing/test_compat_types.py b/tests/typing/test_compat_types.py index f656c411..e8a5bac1 100644 --- a/tests/typing/test_compat_types.py +++ b/tests/typing/test_compat_types.py @@ -10,7 +10,24 @@ import numpy as np import numpy.typing as npt -from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray, ensure_ndarray +from numcodecs.compat import ( + ensure_bytes, + ensure_contiguous_ndarray, + ensure_ndarray, + ensure_ndarray_like, + ensure_text, + ndarray_copy, +) +from numcodecs.ndarray_like import NDArrayLike, is_ndarray_like + + +def _as_ndarray_like(x: NDArrayLike) -> NDArrayLike: + """Forget the concrete type, as for an ndarray-like that is not a numpy array.""" + return x + + +def _as_object(x: object) -> object: + return x def test_ensure_bytes() -> None: @@ -29,3 +46,42 @@ def test_ensure_contiguous_ndarray() -> None: for buf in (b"abc", bytearray(b"abc"), memoryview(b"abc"), np.arange(3, dtype="u1")): result = assert_type(ensure_contiguous_ndarray(buf), npt.NDArray[Any]) assert isinstance(result, np.ndarray) + + +def test_ensure_ndarray_like() -> None: + from_bytes = assert_type(ensure_ndarray_like(b"abc"), npt.NDArray[Any]) + assert isinstance(from_bytes, np.ndarray) + # an ndarray-like that is not known to export the buffer protocol (such as a cupy array) + # comes back as the protocol type + like = _as_ndarray_like(np.arange(3)) + from_like = assert_type(ensure_ndarray_like(like), NDArrayLike) + assert from_like is like + + +def test_ensure_text() -> None: + for s in ("abc", b"abc", bytearray(b"abc"), memoryview(b"abc")): + result = assert_type(ensure_text(s), str) + assert result == "abc" + + +def test_ndarray_copy() -> None: + src = b"\x01\x02\x03" + # without a destination, the source comes back unchanged + same = assert_type(ndarray_copy(src, None), bytes) + assert same is src + # with a destination, that very object comes back, filled + dst = bytearray(3) + filled = assert_type(ndarray_copy(src, dst), bytearray) + assert filled is dst + assert bytes(dst) == src + arr: npt.NDArray[np.uint8] = np.zeros(3, dtype=np.uint8) + assert assert_type(ndarray_copy(src, arr), npt.NDArray[np.uint8]) is arr + + +def test_is_ndarray_like_narrows() -> None: + obj = _as_object(np.arange(3)) + if is_ndarray_like(obj): + assert_type(obj, NDArrayLike) + assert obj.shape == (3,) + else: # pragma: no cover + raise AssertionError diff --git a/tests/typing/test_registry_types.py b/tests/typing/test_registry_types.py new file mode 100644 index 00000000..47a4b9a7 --- /dev/null +++ b/tests/typing/test_registry_types.py @@ -0,0 +1,26 @@ +"""Static types of `numcodecs.registry`, checked against runtime behavior.""" + +from importlib.metadata import EntryPoint +from typing import assert_type + +import numcodecs +from numcodecs.abc import Codec +from numcodecs.registry import codec_registry, entries, get_codec + + +def test_get_codec() -> None: + codec = assert_type(get_codec({"id": "zlib", "level": 1}), Codec) + assert isinstance(codec, numcodecs.Zlib) + + +def test_codec_registry_holds_classes() -> None: + assert_type(codec_registry, dict[str, type[Codec]]) + for cls in codec_registry.values(): + assert isinstance(cls, type) + assert issubclass(cls, Codec) + + +def test_entries_hold_entry_points() -> None: + assert_type(entries, dict[str, EntryPoint]) + for entry in entries.values(): + assert isinstance(entry, EntryPoint)