Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions docs/release.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,22 @@
ndarray, and raises `ValueError` when `out` has the wrong size.
* `Fletcher32.decode` without `out` returns a `uint8` ndarray, like the other checksum
codecs, instead of a Cython memoryview slice.
* Correct type annotations that described something other than what the code does:
`numcodecs.registry.codec_registry` holds codec classes (`dict[str, type[Codec]]`) and
`entries` holds single entry points (`dict[str, EntryPoint]`). `numcodecs.compat`
functions now annotate their parameters, and `ndarray_copy` returns the type of `dst`
(or of `src` when `dst` is `None`). The `NDArrayLike`, `DType` and `FlagsObj` protocols
declare read-only properties, so `numpy.ndarray` now satisfies `NDArrayLike` for type
checkers, and `numcodecs.ndarray_like.BufferLike` names "a buffer or an ndarray-like".
* `numcodecs` defines `__all__`, so type checkers treat its re-exported codec classes as
public. `from numcodecs import *` now imports only those names.
* `register_codec` raises `ValueError` for a class without a `codec_id` when no id is
given, instead of registering it under `None`.

### Deprecations

* Loading a codec through the `numcodecs.codecs` entry point group whose class does not
subclass `numcodecs.abc.Codec` now emits a `DeprecationWarning`.

(release_0.17.1)=

Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ quote-style = "preserve"
[tool.mypy]
ignore_errors = false
ignore_missing_imports = true
python_version = "3.12"
enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
# TODO: set options below to true
strict = false
Expand Down
40 changes: 40 additions & 0 deletions src/numcodecs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,42 @@
from numcodecs.registry import get_codec as get_codec
from numcodecs.registry import register_codec

# Names listed here are the public API re-exported from submodules. Optional codecs are
# appended below when their dependency imports.
__all__ = [
"BZ2",
"CRC32",
"JSON",
"LZ4",
"LZMA",
"Adler32",
"AsType",
"Base64",
"BitRound",
"Blosc",
"Categorize",
"Delta",
"FixedScaleOffset",
"Fletcher32",
"GZip",
"JenkinsLookup3",
"PackBits",
"Pickle",
"Quantize",
"Shuffle",
"VLenArray",
"VLenBytes",
"VLenUTF8",
"Zlib",
"Zstd",
"blosc",
"get_codec",
"lz4",
"register_codec",
"vlen",
"zstd",
]

__version__: str = _version("numcodecs")
from numcodecs.zlib import Zlib

Expand Down Expand Up @@ -131,18 +167,22 @@
from numcodecs.zfpy import ZFPY

register_codec(ZFPY)
__all__ += ["ZFPY"]

with suppress(ImportError):
from numcodecs.msgpacks import MsgPack

register_codec(MsgPack)
__all__ += ["MsgPack"]

with suppress(ImportError):
from numcodecs.checksum32 import CRC32C

register_codec(CRC32C)
__all__ += ["CRC32C"]

with suppress(ImportError):
from numcodecs.pcodec import PCodec

register_codec(PCodec)
__all__ += ["PCodec"]
75 changes: 48 additions & 27 deletions src/numcodecs/compat.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
import array
import codecs
from collections.abc import Buffer
from typing import Any, overload

import numpy as np
import numpy.typing as npt

from .ndarray_like import NDArrayLike, is_ndarray_like
from .ndarray_like import BufferLike, NDArrayLike, is_ndarray_like


def ensure_ndarray_like(buf) -> NDArrayLike:
@overload
def ensure_ndarray_like(buf: Buffer) -> npt.NDArray[Any]: ...
@overload
def ensure_ndarray_like(buf: NDArrayLike) -> NDArrayLike: ...
def ensure_ndarray_like(buf: BufferLike) -> NDArrayLike:
"""Convenience function to coerce `buf` to ndarray-like array.

Parameters
Expand Down Expand Up @@ -41,7 +48,7 @@ def ensure_ndarray_like(buf) -> NDArrayLike:
return buf


def ensure_ndarray(buf) -> np.ndarray:
def ensure_ndarray(buf: BufferLike) -> npt.NDArray[Any]:
"""Convenience function to coerce `buf` to a numpy array, if it is not already a
numpy array.

Expand All @@ -63,7 +70,17 @@ def ensure_ndarray(buf) -> np.ndarray:
return np.array(ensure_ndarray_like(buf), copy=False)


def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> NDArrayLike:
@overload
def ensure_contiguous_ndarray_like(
buf: Buffer, max_buffer_size: int | None = None, flatten: bool = True
) -> npt.NDArray[Any]: ...
@overload
def ensure_contiguous_ndarray_like(
buf: NDArrayLike, max_buffer_size: int | None = None, flatten: bool = True
) -> NDArrayLike: ...
def ensure_contiguous_ndarray_like(
buf: BufferLike, max_buffer_size: int | None = None, flatten: bool = True
) -> NDArrayLike:
"""Convenience function to coerce `buf` to ndarray-like array.
Also ensures that the returned value exports fully contiguous memory,
and supports the new-style buffer interface. If the optional max_buffer_size is
Expand Down Expand Up @@ -95,12 +112,12 @@ def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> N

# check for object arrays, these are just memory pointers, actual memory holding
# item data is scattered elsewhere
if arr.dtype == object:
if arr.dtype.kind == "O":
raise TypeError("object arrays are not supported")

# check for datetime or timedelta ndarray, the buffer interface doesn't support those
if arr.dtype.kind in "Mm":
arr = arr.view(np.int64) # type: ignore[arg-type]
arr = arr.view(np.int64)

# check memory is contiguous, if so flatten
if arr.flags.c_contiguous or arr.flags.f_contiguous:
Expand All @@ -117,7 +134,9 @@ def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> N
return arr


def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.ndarray:
def ensure_contiguous_ndarray(
buf: BufferLike, max_buffer_size: int | None = None, flatten: bool = True
) -> npt.NDArray[Any]:
"""Convenience function to coerce `buf` to a numpy array, if it is not already a
numpy array. Also ensures that the returned value exports fully contiguous memory,
and supports the new-style buffer interface. If the optional max_buffer_size is
Expand Down Expand Up @@ -150,15 +169,15 @@ def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.nda
)


def ensure_bytes(buf) -> bytes:
def ensure_bytes(buf: BufferLike) -> bytes:
"""Obtain a bytes object from memory exposed by `buf`."""

if not isinstance(buf, bytes):
arr = ensure_ndarray_like(buf)

# check for object arrays, these are just memory pointers,
# actual memory holding item data is scattered elsewhere
if arr.dtype == object:
if arr.dtype.kind == "O":
raise TypeError("object arrays are not supported")

# create bytes
Expand All @@ -167,14 +186,18 @@ def ensure_bytes(buf) -> bytes:
return buf


def ensure_text(s, encoding="utf-8"):
if not isinstance(s, str):
s = ensure_contiguous_ndarray(s)
s = codecs.decode(s, encoding)
return s
def ensure_text(s: str | BufferLike, encoding: str = "utf-8") -> str:
"""Return `s` if it is a `str`, else decode the bytes it exposes with `encoding`."""
if isinstance(s, str):
return s
return codecs.decode(ensure_contiguous_ndarray(s), encoding)


def ndarray_copy(src, dst) -> NDArrayLike:
@overload
def ndarray_copy[SrcT: BufferLike](src: SrcT, dst: None) -> SrcT: ...
@overload
def ndarray_copy[DstT: BufferLike](src: BufferLike, dst: DstT) -> DstT: ...
def ndarray_copy(src: BufferLike, dst: BufferLike | None) -> BufferLike:
"""Copy the contents of the array from `src` to `dst`, and return `dst`.

If `dst` is `None`, return `src` unchanged. Otherwise `dst` itself is returned, not
Expand All @@ -185,28 +208,26 @@ def ndarray_copy(src, dst) -> NDArrayLike:
# no-op
return src

out = dst

# ensure ndarray like
src = ensure_ndarray_like(src)
dst = ensure_ndarray_like(dst)
src_arr: NDArrayLike = ensure_ndarray_like(src)
dst_arr: NDArrayLike = ensure_ndarray_like(dst)

# flatten source array
src = src.reshape(-1, order="A")
src_arr = src_arr.reshape(-1, order="A")

# ensure same data type
if dst.dtype != object:
src = src.view(dst.dtype)
if dst_arr.dtype.kind != "O":
src_arr = src_arr.view(dst_arr.dtype)

# reshape source to match destination
if src.shape != dst.shape:
if dst.flags.f_contiguous:
if src_arr.shape != dst_arr.shape:
if dst_arr.flags.f_contiguous:
order = "F"
else:
order = "C"
src = src.reshape(dst.shape, order=order)
src_arr = src_arr.reshape(dst_arr.shape, order=order)

# copy via numpy
np.copyto(dst, src)
np.copyto(dst_arr, src_arr) # type: ignore[arg-type] # cupy arrays support copyto too

return out
return dst
94 changes: 68 additions & 26 deletions src/numcodecs/ndarray_like.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,12 @@
from typing import Any, ClassVar, Protocol, runtime_checkable
from collections.abc import Buffer
from typing import Any, Protocol, Self, runtime_checkable

from typing_extensions import TypeIs # in `typing` from Python 3.13

# Kept at module level, not as an annotated attribute of the metaclass: a protocol class
# without annotations of its own would otherwise find the metaclass's `__annotations__`, and
# `runtime_checkable` would then require instances to have the cache as an attribute.
_instancecheck_cache: dict[tuple[type, type], bool] = {}


class _CachedProtocolMeta(Protocol.__class__): # type: ignore[name-defined]
Expand All @@ -11,55 +19,89 @@ class _CachedProtocolMeta(Protocol.__class__): # type: ignore[name-defined]
isinstance checks using the object's class as the cache key.
"""

_instancecheck_cache: ClassVar[dict[tuple[type, type], bool]] = {}

def __instancecheck__(self, instance):
def __instancecheck__(self, instance: object) -> bool:
key = (self, instance.__class__)
ret = self._instancecheck_cache.get(key)
ret = _instancecheck_cache.get(key)
if ret is None:
ret = super().__instancecheck__(instance)
self._instancecheck_cache[key] = ret
_instancecheck_cache[key] = ret
return ret


# The protocols below declare their attributes as read-only properties. numpy (and cupy)
# expose `dtype`, `shape`, `flags` and friends as read-only properties, and a protocol that
# declared them as plain (settable) attributes would not match a real ndarray.


@runtime_checkable
class DType(Protocol, metaclass=_CachedProtocolMeta):
itemsize: int
name: str
kind: str
@property
def itemsize(self) -> int: ... # pragma: no cover
@property
def name(self) -> str: ... # pragma: no cover
@property
def kind(self) -> str: ... # pragma: no cover
@property
def str(self) -> str: ... # pragma: no cover


@runtime_checkable
class FlagsObj(Protocol, metaclass=_CachedProtocolMeta):
c_contiguous: bool
f_contiguous: bool
owndata: bool
@property
def c_contiguous(self) -> bool: ... # pragma: no cover
@property
def f_contiguous(self) -> bool: ... # pragma: no cover
@property
def owndata(self) -> bool: ... # pragma: no cover


@runtime_checkable
class NDArrayLike(Protocol, metaclass=_CachedProtocolMeta):
dtype: DType
shape: tuple[int, ...]
strides: tuple[int, ...]
ndim: int
size: int
itemsize: int
nbytes: int
flags: FlagsObj
"""The part of the ndarray interface numcodecs relies on.

`numpy.ndarray` and `cupy.ndarray` both satisfy it. Methods return `Any` because numpy
overloads them heavily; convert with `numcodecs.compat.ensure_ndarray` when you need
numpy's own types.
"""

@property
def dtype(self) -> DType: ... # pragma: no cover
@property
def shape(self) -> tuple[int, ...]: ... # pragma: no cover
@property
def strides(self) -> tuple[int, ...]: ... # pragma: no cover
@property
def ndim(self) -> int: ... # pragma: no cover
@property
def size(self) -> int: ... # pragma: no cover
@property
def itemsize(self) -> int: ... # pragma: no cover
@property
def nbytes(self) -> int: ... # pragma: no cover
@property
def flags(self) -> FlagsObj: ... # pragma: no cover

def __len__(self) -> int: ... # pragma: no cover

def __getitem__(self, key) -> Any: ... # pragma: no cover
def __getitem__(self, key: Any, /) -> Any: ... # pragma: no cover

def __setitem__(self, key: Any, value: Any, /) -> None: ... # pragma: no cover

def tobytes(self, order: Any = ...) -> bytes: ... # pragma: no cover

def reshape(self, *args: Any, **kwargs: Any) -> Any: ... # pragma: no cover

def __setitem__(self, key, value): ... # pragma: no cover
def view(self, *args: Any, **kwargs: Any) -> Any: ... # pragma: no cover

def tobytes(self, order: str | None = ...) -> bytes: ... # pragma: no cover
def copy(self, *args: Any, **kwargs: Any) -> Self: ... # pragma: no cover

def reshape(self, *shape: int, order: str = ...) -> "NDArrayLike": ... # pragma: no cover

def view(self, dtype: DType = ...) -> "NDArrayLike": ... # pragma: no cover
type BufferLike = Buffer | NDArrayLike
"""What codecs accept and return: an object exporting the buffer protocol (PEP 688), such
as `bytes`, `bytearray`, `memoryview` or a numpy array, or an ndarray-like such as a cupy
array."""


def is_ndarray_like(obj: object) -> bool:
def is_ndarray_like(obj: object) -> TypeIs[NDArrayLike]:
"""Return True when `obj` is ndarray-like"""
return isinstance(obj, NDArrayLike)
Loading
Loading