From ff36c016cebef004d4b896609a951daa854d24b8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 21 Nov 2026 13:48:18 +0100 Subject: [PATCH 1/4] test: check the Codec contract for every codec and add static type tests Add tests/test_codec_contract.py, which runs every registered codec through the behavior the Codec base class promises: encode and decode return buffers, decode(out=...) returns the caller's out, and get_config round-trips through from_config. Known violations are strict xfails, so fixing one forces its entry to be removed. Add tests/typing, where assert_type checks (verified by mypy and pyright in strict mode) sit next to runtime assertions (verified by pytest), and a `typing` CI job that runs both checkers plus the repository's own mypy configuration. The type checkers are pinned in a new `typing` dependency group. Assisted-by: ClaudeCode:claude-opus-5-5 --- .github/workflows/ci.yaml | 37 ++++++ docs/release.md | 3 + pyproject.toml | 4 + tests/test_codec_contract.py | 203 ++++++++++++++++++++++++++++++ tests/typing/README.md | 25 ++++ tests/typing/__init__.py | 0 tests/typing/mypy.ini | 15 +++ tests/typing/pyrightconfig.json | 14 +++ tests/typing/test_compat_types.py | 31 +++++ 9 files changed, 332 insertions(+) create mode 100644 tests/test_codec_contract.py create mode 100644 tests/typing/README.md create mode 100644 tests/typing/__init__.py create mode 100644 tests/typing/mypy.ini create mode 100644 tests/typing/pyrightconfig.json create mode 100644 tests/typing/test_compat_types.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9f6583327..bec8fb69b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -125,6 +125,43 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} verbose: true + typing: + # Static types: the repository's own mypy configuration over src and tests, and the + # strict type tests in tests/typing under both mypy and pyright (see its README). + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -el {0} + + steps: + - name: Checkout source + uses: actions/checkout@v6 + with: + submodules: recursive + fetch-depth: 0 # required for version resolution + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + cache: 'pip' + + - name: Install numcodecs and the type checkers + run: | + python -m pip install --upgrade pip # --group needs pip >= 25.1 + python -m pip install -v ".[test,msgpack,crc32c]" + python -m pip install --group typing + + - name: mypy (repository configuration) + run: mypy --config-file pyproject.toml src tests + + - name: mypy (strict type tests) + run: mypy --config-file tests/typing/mypy.ini tests/typing + + - name: pyright (strict type tests) + run: pyright -p tests/typing + test-zarr: runs-on: ubuntu-latest strategy: diff --git a/docs/release.md b/docs/release.md index b54747679..6847e8ea6 100644 --- a/docs/release.md +++ b/docs/release.md @@ -29,6 +29,9 @@ wheels) rather than every branch, test the oldest supported and latest Zarr releases, use Alpine's numpy in the i386 job, and remove the unused release-notes labeler. +* Check the behavior every codec promises through `numcodecs.abc.Codec` in a new + `tests/test_codec_contract.py`, and add strict static type tests in `tests/typing`, run by + mypy and pyright in a new `typing` CI job. ### Fixes diff --git a/pyproject.toml b/pyproject.toml index f71320993..721d18125 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,6 +90,10 @@ dev = [ "pytest==9.0.3", "pytest-cov", ] +typing = [ + "mypy==2.4.1", + "pyright==1.1.414", +] test-zarr-313 = [ {include-group="dev"}, "zarr==3.1.3", diff --git a/tests/test_codec_contract.py b/tests/test_codec_contract.py new file mode 100644 index 000000000..c328a25cc --- /dev/null +++ b/tests/test_codec_contract.py @@ -0,0 +1,203 @@ +"""The behavior every codec promises through `numcodecs.abc.Codec`, checked for every codec. + +These tests pin down the contract that the type annotations on `Codec` describe: + +- `encode` and `decode` return a buffer-like object (one that exports the buffer protocol). +- `decode(buf, out=out)` writes into `out` and returns that same object. +- `get_config()` holds an `id` and JSON-serializable values, and `from_config` rebuilds an + equal codec from it once `id` is removed. +""" + +from __future__ import annotations + +import json +import warnings +from collections.abc import Callable +from dataclasses import dataclass + +import numpy as np +import pytest + +import numcodecs +from numcodecs import pickles +from numcodecs.abc import Codec + + +@dataclass(frozen=True) +class Case: + """A codec and an array it can round-trip.""" + + make: Callable[[], Codec] + data: np.ndarray + + @property + def is_object(self) -> bool: + return self.data.dtype == object + + +f8 = np.linspace(0, 100, 16, dtype="f8") +obj_str = np.array(["foo", "bar", "baz", "qux"], dtype=object) + +CASES: dict[str, Case] = { + "zlib": Case(lambda: numcodecs.Zlib(), f8), + "gzip": Case(lambda: numcodecs.GZip(), f8), + "bz2": Case(lambda: numcodecs.BZ2(), f8), + "lzma": Case(lambda: numcodecs.LZMA(), f8), + "blosc": Case(lambda: numcodecs.Blosc(), f8), + "zstd": Case(lambda: numcodecs.Zstd(), f8), + "lz4": Case(lambda: numcodecs.LZ4(), f8), + "base64": Case(lambda: numcodecs.Base64(), f8), + "shuffle": Case(lambda: numcodecs.Shuffle(elementsize=8), f8), + "crc32": Case(lambda: numcodecs.CRC32(), f8), + "adler32": Case(lambda: numcodecs.Adler32(), f8), + "jenkins_lookup3": Case(lambda: numcodecs.JenkinsLookup3(), f8), + "fletcher32": Case(lambda: numcodecs.Fletcher32(), f8), + "astype": Case(lambda: numcodecs.AsType(encode_dtype="f4", decode_dtype="f8"), f8), + "delta": Case(lambda: numcodecs.Delta(dtype="i4"), np.arange(16, dtype="i4")), + "quantize": Case(lambda: numcodecs.Quantize(digits=1, dtype="f8"), f8), + "fixedscaleoffset": Case( + lambda: numcodecs.FixedScaleOffset(offset=0, scale=10, dtype="f8", astype="i4"), f8 + ), + "bitround": Case(lambda: numcodecs.BitRound(keepbits=10), f8), + "packbits": Case(lambda: numcodecs.PackBits(), np.array([True, False] * 8)), + "categorize": Case( + lambda: numcodecs.Categorize(labels=["foo", "bar", "baz", "qux"], dtype=object), obj_str + ), + "pickle": Case(lambda: numcodecs.Pickle(), obj_str), + "json2": Case(lambda: numcodecs.JSON(), obj_str), + "vlen-utf8": Case(lambda: numcodecs.VLenUTF8(), obj_str), + "vlen-bytes": Case( + lambda: numcodecs.VLenBytes(), np.array([b"foo", b"bar", b"baz", b"qux"], dtype=object) + ), + "vlen-array": Case( + lambda: numcodecs.VLenArray(dtype="i4"), + np.array([np.arange(3, dtype="i4"), np.arange(2, dtype="i4")] * 2, dtype=object), + ), + "msgpack2": Case(lambda: numcodecs.MsgPack(), obj_str), + "zfpy": Case(lambda: numcodecs.ZFPY(), f8), + "pcodec": Case(lambda: numcodecs.PCodec(), f8), + "crc32c": Case(lambda: numcodecs.CRC32C(), f8), +} + +# Codecs whose decode(..., out=bytearray) cannot work, because the codec needs `out` to carry +# a dtype: PCodec decodes straight into a typed array. +NO_BYTEARRAY_OUT = frozenset({"pcodec"}) + +# Known violations of the `out` contract. Each entry is a bug; remove it when fixed. +# `strict=True` makes a fixed entry fail, so this list cannot go stale. +OUT_NDARRAY_XFAIL: set[str] = { + "zlib", + "bz2", + "lzma", + "base64", + "shuffle", + "zfpy", + "pcodec", + "vlen-utf8", + "vlen-bytes", + "vlen-array", +} +OUT_BYTEARRAY_XFAIL: set[str] = { + "zlib", + "bz2", + "lzma", + "base64", + "shuffle", + "crc32", + "adler32", + "crc32c", + "jenkins_lookup3", + "astype", + "delta", + "quantize", + "fixedscaleoffset", + "bitround", + "packbits", + "zfpy", +} +DECODE_RETURNS_BUFFER_XFAIL: set[str] = set() + + +def _codec(codec_id: str) -> Codec: + case = CASES[codec_id] + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return case.make() + except AttributeError: + pytest.skip(f"{codec_id} is not available (optional dependency missing)") + + +def _params( + xfail: set[str], *, skip: frozenset[str] = frozenset(), objects: bool = True +) -> list[object]: + params: list[object] = [] + for codec_id, case in CASES.items(): + if codec_id in skip or (case.is_object and not objects): + continue + marks = [pytest.mark.xfail(strict=True, reason="known bug")] if codec_id in xfail else [] + params.append(pytest.param(codec_id, marks=marks, id=codec_id)) + return params + + +@pytest.fixture(autouse=True) +def _allow_pickle(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pickles, "ALLOW_UNSAFE_PICKLE_DECODE", True) + + +def _is_buffer_like(obj: object) -> bool: + try: + memoryview(obj) # type: ignore[arg-type] + except TypeError: + return False + return True + + +def test_every_registered_codec_has_a_case() -> None: + """A new codec must be added to `CASES` so that the contract is checked for it.""" + assert set(numcodecs.registry.codec_registry) <= set(CASES) + + +@pytest.mark.parametrize("codec_id", _params(DECODE_RETURNS_BUFFER_XFAIL)) +def test_encode_decode_return_buffers(codec_id: str) -> None: + codec, data = _codec(codec_id), CASES[codec_id].data + enc = codec.encode(data) + assert _is_buffer_like(enc), type(enc) + dec = codec.decode(enc) + assert _is_buffer_like(dec), type(dec) + + +@pytest.mark.parametrize("codec_id", _params(OUT_NDARRAY_XFAIL)) +def test_decode_into_ndarray_returns_out(codec_id: str) -> None: + codec, data = _codec(codec_id), CASES[codec_id].data + enc = codec.encode(data) + expected = codec.decode(enc) + out = np.empty_like(data) + assert codec.decode(enc, out=out) is out + if CASES[codec_id].is_object: + assert all(np.array_equal(a, b) for a, b in zip(out, expected, strict=True)) + else: + assert np.asarray(memoryview(expected)).tobytes() == out.tobytes() + + +@pytest.mark.parametrize( + "codec_id", _params(OUT_BYTEARRAY_XFAIL, skip=NO_BYTEARRAY_OUT, objects=False) +) +def test_decode_into_bytearray_returns_out(codec_id: str) -> None: + codec, data = _codec(codec_id), CASES[codec_id].data + enc = codec.encode(data) + expected = codec.decode(enc) + out = bytearray(data.nbytes) + assert codec.decode(enc, out=out) is out + assert bytes(out) == np.asarray(memoryview(expected)).tobytes() + + +@pytest.mark.parametrize("codec_id", _params(set())) +def test_config_round_trip(codec_id: str) -> None: + codec = _codec(codec_id) + config = codec.get_config() + assert config["id"] == codec.codec_id + rebuilt_json = json.loads(json.dumps(config)) + assert numcodecs.get_codec(rebuilt_json) == codec + without_id = {k: v for k, v in config.items() if k != "id"} + assert type(codec).from_config(without_id) == codec diff --git a/tests/typing/README.md b/tests/typing/README.md new file mode 100644 index 000000000..a885a0967 --- /dev/null +++ b/tests/typing/README.md @@ -0,0 +1,25 @@ +# Static type tests + +The files here check numcodecs' type annotations against its runtime behavior. Each test +pairs `typing.assert_type`, which mypy and pyright check, with a runtime assertion, which +pytest checks. An annotation that type checkers accept but the runtime contradicts makes one +of the two fail. + +Run them from the repository root: + +```bash +uv pip install --group typing +pytest tests/typing +mypy --config-file tests/typing/mypy.ini tests/typing +pyright -p tests/typing +``` + +`mypy.ini` and `pyrightconfig.json` run both checkers in strict mode and read numcodecs from +`src/`. Errors inside numcodecs itself are not reported here: the repository's own mypy +configuration in `pyproject.toml` covers those. + +`pyrightconfig.json` turns off pyright's three `reportUnknown*` rules, because most numcodecs +parameters are not annotated yet. Turn them back on once they are. + +When you change an annotation in numcodecs, add or update an `assert_type` here that shows +what callers now get. diff --git a/tests/typing/__init__.py b/tests/typing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/typing/mypy.ini b/tests/typing/mypy.ini new file mode 100644 index 000000000..26579ad7e --- /dev/null +++ b/tests/typing/mypy.ini @@ -0,0 +1,15 @@ +# Strict mypy configuration for the static type tests in this directory. +# Run from the repository root: `mypy --config-file tests/typing/mypy.ini tests/typing` +[mypy] +strict = true +warn_unreachable = true +enable_error_code = ignore-without-code, redundant-expr, truthy-bool +# Read numcodecs from the source tree. It does not ship a py.typed marker yet, so mypy +# would otherwise refuse to analyze the installed copy. +mypy_path = src + +# Report errors in the tests only. numcodecs itself is checked by the repository's own +# configuration in pyproject.toml. +[mypy-numcodecs.*] +follow_imports = silent +ignore_errors = true diff --git a/tests/typing/pyrightconfig.json b/tests/typing/pyrightconfig.json new file mode 100644 index 000000000..42540aad6 --- /dev/null +++ b/tests/typing/pyrightconfig.json @@ -0,0 +1,14 @@ +{ + "include": [ + "." + ], + "typeCheckingMode": "strict", + "pythonVersion": "3.12", + "extraPaths": [ + "../../src" + ], + "reportMissingTypeStubs": "none", + "reportUnknownVariableType": "none", + "reportUnknownMemberType": "none", + "reportUnknownArgumentType": "none" +} diff --git a/tests/typing/test_compat_types.py b/tests/typing/test_compat_types.py new file mode 100644 index 000000000..f656c4111 --- /dev/null +++ b/tests/typing/test_compat_types.py @@ -0,0 +1,31 @@ +"""Static types of `numcodecs.compat`, checked against runtime behavior. + +Each `assert_type` is verified by mypy and pyright (see `tests/typing/README.md`), and the +`assert isinstance` next to it is verified by pytest. Together they keep the annotations +honest: a claim that type checkers accept but the runtime contradicts fails here. +""" + +from typing import Any, assert_type + +import numpy as np +import numpy.typing as npt + +from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray, ensure_ndarray + + +def test_ensure_bytes() -> None: + for buf in (b"abc", bytearray(b"abc"), memoryview(b"abc"), np.arange(3, dtype="u1")): + result = assert_type(ensure_bytes(buf), bytes) + assert isinstance(result, bytes) + + +def test_ensure_ndarray() -> None: + for buf in (b"abc", bytearray(b"abc"), memoryview(b"abc"), np.arange(3, dtype="u1")): + result = assert_type(ensure_ndarray(buf), npt.NDArray[Any]) + assert isinstance(result, np.ndarray) + + +def test_ensure_contiguous_ndarray() -> None: + for buf in (b"abc", bytearray(b"abc"), memoryview(b"abc"), np.arange(3, dtype="u1")): + result = assert_type(ensure_contiguous_ndarray(buf), npt.NDArray[Any]) + assert isinstance(result, np.ndarray) From 145994e1e6c902499d87214a9dca0e556a1b6eb4 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 21 Nov 2026 13:50:06 +0100 Subject: [PATCH 2/4] fix: return the caller's out from decode(buf, out=out) in every codec The Codec docstring says decode writes into `out`, and callers expect `out` back. Ten codecs (Zlib, BZ2, LZMA, Base64, Shuffle, ZFPY, PCodec and the three VLen codecs) rebound `out` to a flattened view and returned that, and numcodecs.compat.ndarray_copy did the same for any `out` that was not an ndarray, which every filter and checksum codec inherited. The decoded bytes were already in the caller's buffer; only the returned object changes. JenkinsLookup3.decode now accepts any contiguous writable buffer as `out` and checks its size, and Fletcher32.decode returns a uint8 ndarray instead of a Cython memoryview slice that no type annotation could name. The strict xfails in tests/test_codec_contract.py are removed. Assisted-by: ClaudeCode:claude-opus-5-5 --- docs/release.md | 9 +++++++++ src/numcodecs/base64.py | 2 +- src/numcodecs/bz2.py | 2 +- src/numcodecs/checksum32.py | 7 ++++++- src/numcodecs/compat.py | 10 ++++++++-- src/numcodecs/fletcher32.pyx | 2 +- src/numcodecs/lzma.py | 2 +- src/numcodecs/pcodec.py | 3 +-- src/numcodecs/shuffle.py | 20 ++++++++------------ src/numcodecs/vlen.pyx | 24 ++++++++++++------------ src/numcodecs/zfpy.py | 2 +- src/numcodecs/zlib.py | 2 +- tests/test_codec_contract.py | 32 ++------------------------------ tests/test_jenkins.py | 7 +++++++ 14 files changed, 59 insertions(+), 65 deletions(-) diff --git a/docs/release.md b/docs/release.md index 6847e8ea6..51e2933bb 100644 --- a/docs/release.md +++ b/docs/release.md @@ -38,6 +38,15 @@ * `import numcodecs` no longer fails when a zfpy post-release such as 1.0.1.post0 is installed. zfpy versions are now parsed with `packaging`, which becomes a dependency, and the `zfpy` extra now requires zfpy >= 1.0.1. {issue}`892` +* `decode(buf, out=out)` now returns `out` itself for every codec, as the `Codec` + docstring describes. Zlib, BZ2, LZMA, Base64, Shuffle, ZFPY, PCodec, the three VLen + codecs, and every codec built on `numcodecs.compat.ndarray_copy` returned a flattened view + of `out` instead, or a new ndarray when `out` was a `bytearray`. The decoded bytes were + already written into `out`; only the returned object changes. +* `JenkinsLookup3.decode` accepts any contiguous writable buffer as `out`, not only an + ndarray, and raises `ValueError` when `out` has the wrong size. +* `Fletcher32.decode` without `out` returns a `uint8` ndarray, like the other checksum + codecs, instead of a Cython memoryview slice. (release_0.17.1)= diff --git a/src/numcodecs/base64.py b/src/numcodecs/base64.py index 307c6d52b..c107d5c7b 100644 --- a/src/numcodecs/base64.py +++ b/src/numcodecs/base64.py @@ -19,7 +19,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression decompressed = _base64.standard_b64decode(buf) # handle destination diff --git a/src/numcodecs/bz2.py b/src/numcodecs/bz2.py index 2b63b147b..a3c8f5ec3 100644 --- a/src/numcodecs/bz2.py +++ b/src/numcodecs/bz2.py @@ -31,7 +31,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # N.B., bz2 cannot handle ndarray directly because of truth testing issues buf = memoryview(buf) diff --git a/src/numcodecs/checksum32.py b/src/numcodecs/checksum32.py index b02c0a6a2..a095b1718 100644 --- a/src/numcodecs/checksum32.py +++ b/src/numcodecs/checksum32.py @@ -185,7 +185,12 @@ def decode(self, buf, out=None): "This could be a sign that the data has been corrupted." ) if out is not None: - out.view("uint8")[:] = b[:-4] + out_view = ensure_contiguous_ndarray(out).view("uint8") + if out_view.nbytes != b.nbytes - 4: + raise ValueError( + f"out has {out_view.nbytes} bytes, but the decoded data has {b.nbytes - 4}" + ) + out_view[:] = b[:-4] return out return memoryview(b[:-4]) diff --git a/src/numcodecs/compat.py b/src/numcodecs/compat.py index d1844e10b..cc4d2d139 100644 --- a/src/numcodecs/compat.py +++ b/src/numcodecs/compat.py @@ -175,12 +175,18 @@ def ensure_text(s, encoding="utf-8"): def ndarray_copy(src, dst) -> NDArrayLike: - """Copy the contents of the array from `src` to `dst`.""" + """Copy the contents of the array from `src` to `dst`, and return `dst`. + + If `dst` is `None`, return `src` unchanged. Otherwise `dst` itself is returned, not + a view of it, so callers that pass a `bytearray` get that `bytearray` back. + """ if dst is None: # no-op return src + out = dst + # ensure ndarray like src = ensure_ndarray_like(src) dst = ensure_ndarray_like(dst) @@ -203,4 +209,4 @@ def ndarray_copy(src, dst) -> NDArrayLike: # copy via numpy np.copyto(dst, src) - return dst + return out diff --git a/src/numcodecs/fletcher32.pyx b/src/numcodecs/fletcher32.pyx index 1ecbb45d6..ab9ce6a8f 100644 --- a/src/numcodecs/fletcher32.pyx +++ b/src/numcodecs/fletcher32.pyx @@ -112,5 +112,5 @@ class Fletcher32(Codec): out_ptr = &out_mv[0] memcpy(out_ptr, b_ptr, b_len - FOOTER_LENGTH) else: - out = b_mv[:-FOOTER_LENGTH] + out = b[:-FOOTER_LENGTH] return out diff --git a/src/numcodecs/lzma.py b/src/numcodecs/lzma.py index 7b6b23487..97da02825 100644 --- a/src/numcodecs/lzma.py +++ b/src/numcodecs/lzma.py @@ -59,7 +59,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression dec = _lzma.decompress(buf, format=self.format, filters=self.filters) diff --git a/src/numcodecs/pcodec.py b/src/numcodecs/pcodec.py index c7108c7b8..739666b11 100644 --- a/src/numcodecs/pcodec.py +++ b/src/numcodecs/pcodec.py @@ -122,8 +122,7 @@ def encode(self, buf): def decode(self, buf, out=None): buf = ensure_bytes(buf) if out is not None: - out = ensure_contiguous_ndarray(out) - standalone.simple_decompress_into(buf, out) + standalone.simple_decompress_into(buf, ensure_contiguous_ndarray(out)) return out else: return standalone.simple_decompress(buf) diff --git a/src/numcodecs/shuffle.py b/src/numcodecs/shuffle.py index 0e7031f5e..e29e5a593 100644 --- a/src/numcodecs/shuffle.py +++ b/src/numcodecs/shuffle.py @@ -38,24 +38,20 @@ def _prepare_arrays(self, buf, out): return buf, out def encode(self, buf, out=None): - buf, out = self._prepare_arrays(buf, out) + buf, out_arr = self._prepare_arrays(buf, out) - if self.elementsize <= 1: - return out # no shuffling needed - - _doShuffle(buf.view("uint8"), out.view("uint8"), self.elementsize) + if self.elementsize > 1: + _doShuffle(buf.view("uint8"), out_arr.view("uint8"), self.elementsize) - return out + return out_arr if out is None else out def decode(self, buf, out=None): - buf, out = self._prepare_arrays(buf, out) - - if self.elementsize <= 1: - return out # no shuffling needed + buf, out_arr = self._prepare_arrays(buf, out) - _doUnshuffle(buf.view("uint8"), out.view("uint8"), self.elementsize) + if self.elementsize > 1: + _doUnshuffle(buf.view("uint8"), out_arr.view("uint8"), self.elementsize) - return out + return out_arr if out is None else out def __repr__(self): return f'{type(self).__name__}(elementsize={self.elementsize})' diff --git a/src/numcodecs/vlen.pyx b/src/numcodecs/vlen.pyx index f51e8cad7..424cc7ee6 100644 --- a/src/numcodecs/vlen.pyx +++ b/src/numcodecs/vlen.pyx @@ -154,11 +154,11 @@ class VLenUTF8(Codec): # load number of items n_items = load_le32(data) - # setup output + # setup output: fill a flat view, but return the caller's own `out` if out is not None: - out = check_out_param(out, n_items) + out_flat = check_out_param(out, n_items) else: - out = np.empty(n_items, dtype=object) + out = out_flat = np.empty(n_items, dtype=object) # iterate and decode - N.B., do not try to cast `out` as object[:] # as this causes segfaults, possibly similar to @@ -171,7 +171,7 @@ class VLenUTF8(Codec): data += HEADER_LENGTH if data + L > data_end: raise ValueError('corrupt buffer, data seem truncated') - out[i] = PyUnicode_FromStringAndSize(data, L) + out_flat[i] = PyUnicode_FromStringAndSize(data, L) data += L return out @@ -283,11 +283,11 @@ class VLenBytes(Codec): # load number of items n_items = load_le32(data) - # setup output + # setup output: fill a flat view, but return the caller's own `out` if out is not None: - out = check_out_param(out, n_items) + out_flat = check_out_param(out, n_items) else: - out = np.empty(n_items, dtype=object) + out = out_flat = np.empty(n_items, dtype=object) # iterate and decode - N.B., do not try to cast `out` as object[:] # as this causes segfaults, possibly similar to @@ -300,7 +300,7 @@ class VLenBytes(Codec): data += HEADER_LENGTH if data + L > data_end: raise ValueError('corrupt buffer, data seem truncated') - out[i] = PyBytes_FromStringAndSize(data, L) + out_flat[i] = PyBytes_FromStringAndSize(data, L) data += L return out @@ -439,11 +439,11 @@ class VLenArray(Codec): # load number of items n_items = load_le32(data) - # setup output + # setup output: fill a flat view, but return the caller's own `out` if out is not None: - out = check_out_param(out, n_items) + out_flat = check_out_param(out, n_items) else: - out = np.empty(n_items, dtype=object) + out = out_flat = np.empty(n_items, dtype=object) # iterate and decode - N.B., do not try to cast `out` as object[:] # as this causes segfaults, possibly similar to @@ -463,7 +463,7 @@ class VLenArray(Codec): v_pb = PyMemoryView_GET_BUFFER(v_mv) memcpy(v_pb.buf, data, L) - out[i] = v + out_flat[i] = v data += L return out diff --git a/src/numcodecs/zfpy.py b/src/numcodecs/zfpy.py index 41ccfe98c..86fb090a8 100644 --- a/src/numcodecs/zfpy.py +++ b/src/numcodecs/zfpy.py @@ -103,7 +103,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_bytes(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression dec = _zfpy.decompress_numpy(buf) diff --git a/src/numcodecs/zlib.py b/src/numcodecs/zlib.py index ca247af29..e62658dcb 100644 --- a/src/numcodecs/zlib.py +++ b/src/numcodecs/zlib.py @@ -31,7 +31,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression dec = _zlib.decompress(buf) diff --git a/tests/test_codec_contract.py b/tests/test_codec_contract.py index c328a25cc..bf47cc007 100644 --- a/tests/test_codec_contract.py +++ b/tests/test_codec_contract.py @@ -85,36 +85,8 @@ def is_object(self) -> bool: # Known violations of the `out` contract. Each entry is a bug; remove it when fixed. # `strict=True` makes a fixed entry fail, so this list cannot go stale. -OUT_NDARRAY_XFAIL: set[str] = { - "zlib", - "bz2", - "lzma", - "base64", - "shuffle", - "zfpy", - "pcodec", - "vlen-utf8", - "vlen-bytes", - "vlen-array", -} -OUT_BYTEARRAY_XFAIL: set[str] = { - "zlib", - "bz2", - "lzma", - "base64", - "shuffle", - "crc32", - "adler32", - "crc32c", - "jenkins_lookup3", - "astype", - "delta", - "quantize", - "fixedscaleoffset", - "bitround", - "packbits", - "zfpy", -} +OUT_NDARRAY_XFAIL: set[str] = set() +OUT_BYTEARRAY_XFAIL: set[str] = set() DECODE_RETURNS_BUFFER_XFAIL: set[str] = set() diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py index 40d972bf7..511c435ca 100644 --- a/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -148,3 +148,10 @@ def test_out(): j = JenkinsLookup3() result = j.encode(data) j.decode(result, out=data) + + +def test_out_wrong_size(): + j = JenkinsLookup3() + enc = j.encode(np.arange(8, dtype="uint8")) + with pytest.raises(ValueError, match="out has 4 bytes, but the decoded data has 8"): + j.decode(enc, out=bytearray(4)) From 52be3f56167cebaf55dca8262695106e0aeadc76 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 21 Nov 2026 13:54:40 +0100 Subject: [PATCH 3/4] fix(typing): correct registry, compat and NDArrayLike annotations and add __all__ Several annotations described something other than what the code does: - codec_registry holds classes: dict[str, type[Codec]], not dict[str, Codec]. - entries holds single entry points: dict[str, EntryPoint]. - ndarray_copy returned NDArrayLike, but hands back `src` unchanged (often bytes) when `dst` is None. It is now overloaded on `dst`. - NDArrayLike, DType and FlagsObj declared settable attributes, so no numpy.ndarray satisfied NDArrayLike under mypy or pyright. They now declare read-only properties. The isinstance cache moves to module level, because a protocol without annotations of its own would otherwise pick up the metaclass's annotation as a required member. The compat functions annotate their parameters with a new BufferLike alias (collections.abc.Buffer | NDArrayLike), is_ndarray_like returns TypeIs, and get_codec/register_codec are annotated. register_codec now refuses a class without a codec_id instead of registering it under None, and get_codec warns (DeprecationWarning) when an entry point loads a class that does not subclass Codec, so that `-> Codec` can become true. numcodecs/__init__.py defines __all__, so that type checkers treat the re-exported codec classes as public once numcodecs ships py.typed. mypy now checks against Python 3.12, the oldest supported version; doing so caught `typing.TypeIs`, which only exists from 3.13 and now comes from typing_extensions. Assisted-by: ClaudeCode:claude-opus-5-5 --- docs/release.md | 16 +++++ pyproject.toml | 1 + src/numcodecs/__init__.py | 40 ++++++++++++ src/numcodecs/compat.py | 75 ++++++++++++++--------- src/numcodecs/ndarray_like.py | 94 +++++++++++++++++++++-------- src/numcodecs/registry.py | 29 ++++++--- tests/test_registry.py | 34 +++++++++++ tests/typing/mypy.ini | 2 + tests/typing/test_compat_types.py | 58 +++++++++++++++++- tests/typing/test_registry_types.py | 26 ++++++++ 10 files changed, 314 insertions(+), 61 deletions(-) create mode 100644 tests/typing/test_registry_types.py diff --git a/docs/release.md b/docs/release.md index 51e2933bb..bd896673b 100644 --- a/docs/release.md +++ b/docs/release.md @@ -47,6 +47,22 @@ ndarray, and raises `ValueError` when `out` has the wrong size. * `Fletcher32.decode` without `out` returns a `uint8` ndarray, like the other checksum codecs, instead of a Cython memoryview slice. +* Correct type annotations that described something other than what the code does: + `numcodecs.registry.codec_registry` holds codec classes (`dict[str, type[Codec]]`) and + `entries` holds single entry points (`dict[str, EntryPoint]`). `numcodecs.compat` + functions now annotate their parameters, and `ndarray_copy` returns the type of `dst` + (or of `src` when `dst` is `None`). The `NDArrayLike`, `DType` and `FlagsObj` protocols + declare read-only properties, so `numpy.ndarray` now satisfies `NDArrayLike` for type + checkers, and `numcodecs.ndarray_like.BufferLike` names "a buffer or an ndarray-like". +* `numcodecs` defines `__all__`, so type checkers treat its re-exported codec classes as + public. `from numcodecs import *` now imports only those names. +* `register_codec` raises `ValueError` for a class without a `codec_id` when no id is + given, instead of registering it under `None`. + +### Deprecations + +* Loading a codec through the `numcodecs.codecs` entry point group whose class does not + subclass `numcodecs.abc.Codec` now emits a `DeprecationWarning`. (release_0.17.1)= diff --git a/pyproject.toml b/pyproject.toml index 721d18125..3050fdfc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -245,6 +245,7 @@ quote-style = "preserve" [tool.mypy] ignore_errors = false ignore_missing_imports = true +python_version = "3.12" enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] # TODO: set options below to true strict = false diff --git a/src/numcodecs/__init__.py b/src/numcodecs/__init__.py index be60bfc30..bad41090c 100644 --- a/src/numcodecs/__init__.py +++ b/src/numcodecs/__init__.py @@ -25,6 +25,42 @@ from numcodecs.registry import get_codec as get_codec from numcodecs.registry import register_codec +# Names listed here are the public API re-exported from submodules. Optional codecs are +# appended below when their dependency imports. +__all__ = [ + "BZ2", + "CRC32", + "JSON", + "LZ4", + "LZMA", + "Adler32", + "AsType", + "Base64", + "BitRound", + "Blosc", + "Categorize", + "Delta", + "FixedScaleOffset", + "Fletcher32", + "GZip", + "JenkinsLookup3", + "PackBits", + "Pickle", + "Quantize", + "Shuffle", + "VLenArray", + "VLenBytes", + "VLenUTF8", + "Zlib", + "Zstd", + "blosc", + "get_codec", + "lz4", + "register_codec", + "vlen", + "zstd", +] + __version__: str = _version("numcodecs") from numcodecs.zlib import Zlib @@ -131,18 +167,22 @@ from numcodecs.zfpy import ZFPY register_codec(ZFPY) + __all__ += ["ZFPY"] with suppress(ImportError): from numcodecs.msgpacks import MsgPack register_codec(MsgPack) + __all__ += ["MsgPack"] with suppress(ImportError): from numcodecs.checksum32 import CRC32C register_codec(CRC32C) + __all__ += ["CRC32C"] with suppress(ImportError): from numcodecs.pcodec import PCodec register_codec(PCodec) + __all__ += ["PCodec"] diff --git a/src/numcodecs/compat.py b/src/numcodecs/compat.py index cc4d2d139..bad12fd43 100644 --- a/src/numcodecs/compat.py +++ b/src/numcodecs/compat.py @@ -1,12 +1,19 @@ import array import codecs +from collections.abc import Buffer +from typing import Any, overload import numpy as np +import numpy.typing as npt -from .ndarray_like import NDArrayLike, is_ndarray_like +from .ndarray_like import BufferLike, NDArrayLike, is_ndarray_like -def ensure_ndarray_like(buf) -> NDArrayLike: +@overload +def ensure_ndarray_like(buf: Buffer) -> npt.NDArray[Any]: ... +@overload +def ensure_ndarray_like(buf: NDArrayLike) -> NDArrayLike: ... +def ensure_ndarray_like(buf: BufferLike) -> NDArrayLike: """Convenience function to coerce `buf` to ndarray-like array. Parameters @@ -41,7 +48,7 @@ def ensure_ndarray_like(buf) -> NDArrayLike: return buf -def ensure_ndarray(buf) -> np.ndarray: +def ensure_ndarray(buf: BufferLike) -> npt.NDArray[Any]: """Convenience function to coerce `buf` to a numpy array, if it is not already a numpy array. @@ -63,7 +70,17 @@ def ensure_ndarray(buf) -> np.ndarray: return np.array(ensure_ndarray_like(buf), copy=False) -def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> NDArrayLike: +@overload +def ensure_contiguous_ndarray_like( + buf: Buffer, max_buffer_size: int | None = None, flatten: bool = True +) -> npt.NDArray[Any]: ... +@overload +def ensure_contiguous_ndarray_like( + buf: NDArrayLike, max_buffer_size: int | None = None, flatten: bool = True +) -> NDArrayLike: ... +def ensure_contiguous_ndarray_like( + buf: BufferLike, max_buffer_size: int | None = None, flatten: bool = True +) -> NDArrayLike: """Convenience function to coerce `buf` to ndarray-like array. Also ensures that the returned value exports fully contiguous memory, and supports the new-style buffer interface. If the optional max_buffer_size is @@ -95,12 +112,12 @@ def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> N # check for object arrays, these are just memory pointers, actual memory holding # item data is scattered elsewhere - if arr.dtype == object: + if arr.dtype.kind == "O": raise TypeError("object arrays are not supported") # check for datetime or timedelta ndarray, the buffer interface doesn't support those if arr.dtype.kind in "Mm": - arr = arr.view(np.int64) # type: ignore[arg-type] + arr = arr.view(np.int64) # check memory is contiguous, if so flatten if arr.flags.c_contiguous or arr.flags.f_contiguous: @@ -117,7 +134,9 @@ def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> N return arr -def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.ndarray: +def ensure_contiguous_ndarray( + buf: BufferLike, max_buffer_size: int | None = None, flatten: bool = True +) -> npt.NDArray[Any]: """Convenience function to coerce `buf` to a numpy array, if it is not already a numpy array. Also ensures that the returned value exports fully contiguous memory, and supports the new-style buffer interface. If the optional max_buffer_size is @@ -150,7 +169,7 @@ def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.nda ) -def ensure_bytes(buf) -> bytes: +def ensure_bytes(buf: BufferLike) -> bytes: """Obtain a bytes object from memory exposed by `buf`.""" if not isinstance(buf, bytes): @@ -158,7 +177,7 @@ def ensure_bytes(buf) -> bytes: # check for object arrays, these are just memory pointers, # actual memory holding item data is scattered elsewhere - if arr.dtype == object: + if arr.dtype.kind == "O": raise TypeError("object arrays are not supported") # create bytes @@ -167,14 +186,18 @@ def ensure_bytes(buf) -> bytes: return buf -def ensure_text(s, encoding="utf-8"): - if not isinstance(s, str): - s = ensure_contiguous_ndarray(s) - s = codecs.decode(s, encoding) - return s +def ensure_text(s: str | BufferLike, encoding: str = "utf-8") -> str: + """Return `s` if it is a `str`, else decode the bytes it exposes with `encoding`.""" + if isinstance(s, str): + return s + return codecs.decode(ensure_contiguous_ndarray(s), encoding) -def ndarray_copy(src, dst) -> NDArrayLike: +@overload +def ndarray_copy[SrcT: BufferLike](src: SrcT, dst: None) -> SrcT: ... +@overload +def ndarray_copy[DstT: BufferLike](src: BufferLike, dst: DstT) -> DstT: ... +def ndarray_copy(src: BufferLike, dst: BufferLike | None) -> BufferLike: """Copy the contents of the array from `src` to `dst`, and return `dst`. If `dst` is `None`, return `src` unchanged. Otherwise `dst` itself is returned, not @@ -185,28 +208,26 @@ def ndarray_copy(src, dst) -> NDArrayLike: # no-op return src - out = dst - # ensure ndarray like - src = ensure_ndarray_like(src) - dst = ensure_ndarray_like(dst) + src_arr: NDArrayLike = ensure_ndarray_like(src) + dst_arr: NDArrayLike = ensure_ndarray_like(dst) # flatten source array - src = src.reshape(-1, order="A") + src_arr = src_arr.reshape(-1, order="A") # ensure same data type - if dst.dtype != object: - src = src.view(dst.dtype) + if dst_arr.dtype.kind != "O": + src_arr = src_arr.view(dst_arr.dtype) # reshape source to match destination - if src.shape != dst.shape: - if dst.flags.f_contiguous: + if src_arr.shape != dst_arr.shape: + if dst_arr.flags.f_contiguous: order = "F" else: order = "C" - src = src.reshape(dst.shape, order=order) + src_arr = src_arr.reshape(dst_arr.shape, order=order) # copy via numpy - np.copyto(dst, src) + np.copyto(dst_arr, src_arr) # type: ignore[arg-type] # cupy arrays support copyto too - return out + return dst diff --git a/src/numcodecs/ndarray_like.py b/src/numcodecs/ndarray_like.py index 06e15ea97..be4171bdf 100644 --- a/src/numcodecs/ndarray_like.py +++ b/src/numcodecs/ndarray_like.py @@ -1,4 +1,12 @@ -from typing import Any, ClassVar, Protocol, runtime_checkable +from collections.abc import Buffer +from typing import Any, Protocol, Self, runtime_checkable + +from typing_extensions import TypeIs # in `typing` from Python 3.13 + +# Kept at module level, not as an annotated attribute of the metaclass: a protocol class +# without annotations of its own would otherwise find the metaclass's `__annotations__`, and +# `runtime_checkable` would then require instances to have the cache as an attribute. +_instancecheck_cache: dict[tuple[type, type], bool] = {} class _CachedProtocolMeta(Protocol.__class__): # type: ignore[name-defined] @@ -11,55 +19,89 @@ class _CachedProtocolMeta(Protocol.__class__): # type: ignore[name-defined] isinstance checks using the object's class as the cache key. """ - _instancecheck_cache: ClassVar[dict[tuple[type, type], bool]] = {} - - def __instancecheck__(self, instance): + def __instancecheck__(self, instance: object) -> bool: key = (self, instance.__class__) - ret = self._instancecheck_cache.get(key) + ret = _instancecheck_cache.get(key) if ret is None: ret = super().__instancecheck__(instance) - self._instancecheck_cache[key] = ret + _instancecheck_cache[key] = ret return ret +# The protocols below declare their attributes as read-only properties. numpy (and cupy) +# expose `dtype`, `shape`, `flags` and friends as read-only properties, and a protocol that +# declared them as plain (settable) attributes would not match a real ndarray. + + @runtime_checkable class DType(Protocol, metaclass=_CachedProtocolMeta): - itemsize: int - name: str - kind: str + @property + def itemsize(self) -> int: ... # pragma: no cover + @property + def name(self) -> str: ... # pragma: no cover + @property + def kind(self) -> str: ... # pragma: no cover + @property + def str(self) -> str: ... # pragma: no cover @runtime_checkable class FlagsObj(Protocol, metaclass=_CachedProtocolMeta): - c_contiguous: bool - f_contiguous: bool - owndata: bool + @property + def c_contiguous(self) -> bool: ... # pragma: no cover + @property + def f_contiguous(self) -> bool: ... # pragma: no cover + @property + def owndata(self) -> bool: ... # pragma: no cover @runtime_checkable class NDArrayLike(Protocol, metaclass=_CachedProtocolMeta): - dtype: DType - shape: tuple[int, ...] - strides: tuple[int, ...] - ndim: int - size: int - itemsize: int - nbytes: int - flags: FlagsObj + """The part of the ndarray interface numcodecs relies on. + + `numpy.ndarray` and `cupy.ndarray` both satisfy it. Methods return `Any` because numpy + overloads them heavily; convert with `numcodecs.compat.ensure_ndarray` when you need + numpy's own types. + """ + + @property + def dtype(self) -> DType: ... # pragma: no cover + @property + def shape(self) -> tuple[int, ...]: ... # pragma: no cover + @property + def strides(self) -> tuple[int, ...]: ... # pragma: no cover + @property + def ndim(self) -> int: ... # pragma: no cover + @property + def size(self) -> int: ... # pragma: no cover + @property + def itemsize(self) -> int: ... # pragma: no cover + @property + def nbytes(self) -> int: ... # pragma: no cover + @property + def flags(self) -> FlagsObj: ... # pragma: no cover def __len__(self) -> int: ... # pragma: no cover - def __getitem__(self, key) -> Any: ... # pragma: no cover + def __getitem__(self, key: Any, /) -> Any: ... # pragma: no cover + + def __setitem__(self, key: Any, value: Any, /) -> None: ... # pragma: no cover + + def tobytes(self, order: Any = ...) -> bytes: ... # pragma: no cover + + def reshape(self, *args: Any, **kwargs: Any) -> Any: ... # pragma: no cover - def __setitem__(self, key, value): ... # pragma: no cover + def view(self, *args: Any, **kwargs: Any) -> Any: ... # pragma: no cover - def tobytes(self, order: str | None = ...) -> bytes: ... # pragma: no cover + def copy(self, *args: Any, **kwargs: Any) -> Self: ... # pragma: no cover - def reshape(self, *shape: int, order: str = ...) -> "NDArrayLike": ... # pragma: no cover - def view(self, dtype: DType = ...) -> "NDArrayLike": ... # pragma: no cover +type BufferLike = Buffer | NDArrayLike +"""What codecs accept and return: an object exporting the buffer protocol (PEP 688), such +as `bytes`, `bytearray`, `memoryview` or a numpy array, or an ndarray-like such as a cupy +array.""" -def is_ndarray_like(obj: object) -> bool: +def is_ndarray_like(obj: object) -> TypeIs[NDArrayLike]: """Return True when `obj` is ndarray-like""" return isinstance(obj, NDArrayLike) diff --git a/src/numcodecs/registry.py b/src/numcodecs/registry.py index ccb5b26bf..758cc0082 100644 --- a/src/numcodecs/registry.py +++ b/src/numcodecs/registry.py @@ -2,17 +2,22 @@ applications to dynamically register and look-up codec classes.""" import logging -from importlib.metadata import EntryPoints, entry_points +import warnings +from collections.abc import Mapping +from importlib.metadata import EntryPoint, entry_points +from typing import Any from numcodecs.abc import Codec from numcodecs.errors import UnknownCodecError logger = logging.getLogger("numcodecs") -codec_registry: dict[str, Codec] = {} -entries: dict[str, EntryPoints] = {} +codec_registry: dict[str, type[Codec]] = {} +"""Codec classes by codec id.""" +entries: dict[str, EntryPoint] = {} +"""Not-yet-loaded codec classes from the `numcodecs.codecs` entry point group, by codec id.""" -def run_entrypoints(): +def run_entrypoints() -> None: entries.clear() eps = entry_points() entries.update({e.name: e for e in eps.select(group="numcodecs.codecs")}) @@ -21,7 +26,7 @@ def run_entrypoints(): run_entrypoints() -def get_codec(config): +def get_codec(config: Mapping[str, Any]) -> Codec: """Obtain a codec for the given configuration. Parameters @@ -44,17 +49,25 @@ def get_codec(config): """ config = dict(config) codec_id = config.pop('id', None) - cls = codec_registry.get(codec_id) + cls = codec_registry.get(codec_id) if isinstance(codec_id, str) else None if cls is None and codec_id in entries: logger.debug("Auto loading codec '%s' from entrypoint", codec_id) cls = entries[codec_id].load() + if not (isinstance(cls, type) and issubclass(cls, Codec)): + warnings.warn( + f"The entry point for codec {codec_id!r} loads {cls!r}, which is not a " + "subclass of numcodecs.abc.Codec. Support for such codecs is deprecated " + "and will be removed in a future release.", + DeprecationWarning, + stacklevel=2, + ) register_codec(cls, codec_id=codec_id) if cls: return cls.from_config(config) raise UnknownCodecError(f"{codec_id!r}") -def register_codec(cls, codec_id=None): +def register_codec(cls: type[Codec], codec_id: str | None = None) -> None: """Register a codec class. Parameters @@ -70,5 +83,7 @@ def register_codec(cls, codec_id=None): """ if codec_id is None: codec_id = cls.codec_id + if codec_id is None: + raise ValueError(f"cannot register {cls!r}: it has no codec_id and none was given") logger.debug("Registering codec '%s'", codec_id) codec_registry[codec_id] = cls diff --git a/tests/test_registry.py b/tests/test_registry.py index 93ed8f032..2af73f058 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -1,4 +1,5 @@ import inspect +import types import pytest @@ -41,3 +42,36 @@ def test_all_classes_registered(): if missing: raise Exception(f"these codecs are missing: {missing}") # pragma: no cover + + +class _DuckCodec: + """Looks like a codec, but does not subclass numcodecs.abc.Codec.""" + + codec_id = "duck" + + @classmethod + def from_config(cls, config): + return cls() + + +def test_entry_point_not_a_codec_subclass_is_deprecated(monkeypatch): + monkeypatch.setitem( + numcodecs.registry.entries, "duck", types.SimpleNamespace(load=lambda: _DuckCodec) + ) + monkeypatch.delitem(numcodecs.registry.codec_registry, "duck", raising=False) + with pytest.warns(DeprecationWarning, match="not a subclass of numcodecs.abc.Codec"): + codec = get_codec({"id": "duck"}) + assert isinstance(codec, _DuckCodec) + numcodecs.registry.codec_registry.pop("duck") + + +def test_register_codec_without_id(): + class NoId(numcodecs.abc.Codec): + def encode(self, buf): # pragma: no cover + return buf + + def decode(self, buf, out=None): # pragma: no cover + return buf + + with pytest.raises(ValueError, match="has no codec_id and none was given"): + numcodecs.register_codec(NoId) diff --git a/tests/typing/mypy.ini b/tests/typing/mypy.ini index 26579ad7e..0159ab2f3 100644 --- a/tests/typing/mypy.ini +++ b/tests/typing/mypy.ini @@ -2,6 +2,8 @@ # Run from the repository root: `mypy --config-file tests/typing/mypy.ini tests/typing` [mypy] strict = true +# the oldest supported Python, so names that only exist in newer `typing` modules are caught +python_version = 3.12 warn_unreachable = true enable_error_code = ignore-without-code, redundant-expr, truthy-bool # Read numcodecs from the source tree. It does not ship a py.typed marker yet, so mypy diff --git a/tests/typing/test_compat_types.py b/tests/typing/test_compat_types.py index f656c4111..e8a5bac15 100644 --- a/tests/typing/test_compat_types.py +++ b/tests/typing/test_compat_types.py @@ -10,7 +10,24 @@ import numpy as np import numpy.typing as npt -from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray, ensure_ndarray +from numcodecs.compat import ( + ensure_bytes, + ensure_contiguous_ndarray, + ensure_ndarray, + ensure_ndarray_like, + ensure_text, + ndarray_copy, +) +from numcodecs.ndarray_like import NDArrayLike, is_ndarray_like + + +def _as_ndarray_like(x: NDArrayLike) -> NDArrayLike: + """Forget the concrete type, as for an ndarray-like that is not a numpy array.""" + return x + + +def _as_object(x: object) -> object: + return x def test_ensure_bytes() -> None: @@ -29,3 +46,42 @@ def test_ensure_contiguous_ndarray() -> None: for buf in (b"abc", bytearray(b"abc"), memoryview(b"abc"), np.arange(3, dtype="u1")): result = assert_type(ensure_contiguous_ndarray(buf), npt.NDArray[Any]) assert isinstance(result, np.ndarray) + + +def test_ensure_ndarray_like() -> None: + from_bytes = assert_type(ensure_ndarray_like(b"abc"), npt.NDArray[Any]) + assert isinstance(from_bytes, np.ndarray) + # an ndarray-like that is not known to export the buffer protocol (such as a cupy array) + # comes back as the protocol type + like = _as_ndarray_like(np.arange(3)) + from_like = assert_type(ensure_ndarray_like(like), NDArrayLike) + assert from_like is like + + +def test_ensure_text() -> None: + for s in ("abc", b"abc", bytearray(b"abc"), memoryview(b"abc")): + result = assert_type(ensure_text(s), str) + assert result == "abc" + + +def test_ndarray_copy() -> None: + src = b"\x01\x02\x03" + # without a destination, the source comes back unchanged + same = assert_type(ndarray_copy(src, None), bytes) + assert same is src + # with a destination, that very object comes back, filled + dst = bytearray(3) + filled = assert_type(ndarray_copy(src, dst), bytearray) + assert filled is dst + assert bytes(dst) == src + arr: npt.NDArray[np.uint8] = np.zeros(3, dtype=np.uint8) + assert assert_type(ndarray_copy(src, arr), npt.NDArray[np.uint8]) is arr + + +def test_is_ndarray_like_narrows() -> None: + obj = _as_object(np.arange(3)) + if is_ndarray_like(obj): + assert_type(obj, NDArrayLike) + assert obj.shape == (3,) + else: # pragma: no cover + raise AssertionError diff --git a/tests/typing/test_registry_types.py b/tests/typing/test_registry_types.py new file mode 100644 index 000000000..47a4b9a73 --- /dev/null +++ b/tests/typing/test_registry_types.py @@ -0,0 +1,26 @@ +"""Static types of `numcodecs.registry`, checked against runtime behavior.""" + +from importlib.metadata import EntryPoint +from typing import assert_type + +import numcodecs +from numcodecs.abc import Codec +from numcodecs.registry import codec_registry, entries, get_codec + + +def test_get_codec() -> None: + codec = assert_type(get_codec({"id": "zlib", "level": 1}), Codec) + assert isinstance(codec, numcodecs.Zlib) + + +def test_codec_registry_holds_classes() -> None: + assert_type(codec_registry, dict[str, type[Codec]]) + for cls in codec_registry.values(): + assert isinstance(cls, type) + assert issubclass(cls, Codec) + + +def test_entries_hold_entry_points() -> None: + assert_type(entries, dict[str, EntryPoint]) + for entry in entries.values(): + assert isinstance(entry, EntryPoint) From c26d438a50db75ac89f22184b5bbb81aff88099a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 21 Nov 2026 13:57:13 +0100 Subject: [PATCH 4/4] feat(abc): annotate the Codec base class and add structural codec protocols Codec.encode and Codec.decode were unannotated with docstring-only bodies, so mypy saw them as Any and pyright inferred that both return None, which made every typed subclass an "incompatible override" under pyright. - encode/decode take and return BufferLike (collections.abc.Buffer | NDArrayLike), with `buf` positional-only so that subclasses may rename it. - decode is overloaded so that decode(buf, out=out) returns the type of `out`, which every codec now honors. - codec_id is declared `str` without a default instead of `str | None = None`. It is not a ClassVar: with ClassVar, Codec instances do not match protocols that declare codec_id as a plain attribute (zarr-python's Numcodec) or as a property, under mypy or pyright. - get_config returns Mapping[str, Any], so that concrete codecs can later return per-codec TypedDicts, and from_config returns Self. numcodecs.abc also exports CodecProtocol and the three Zarr v3 codec kinds (BytesBytesCodec, ArrayArrayCodec, ArrayBytesCodec) as structural protocols, so consumers can type code that accepts any codec-like object. Assisted-by: ClaudeCode:claude-opus-5-5 --- docs/abc.md | 12 +++++ docs/release.md | 14 ++++++ src/numcodecs/abc.py | 87 +++++++++++++++++++++++++++----- src/numcodecs/registry.py | 2 +- tests/test_codec_contract.py | 7 +-- tests/test_registry.py | 2 +- tests/typing/test_abc_types.py | 92 ++++++++++++++++++++++++++++++++++ 7 files changed, 198 insertions(+), 18 deletions(-) create mode 100644 tests/typing/test_abc_types.py diff --git a/docs/abc.md b/docs/abc.md index f080766f2..925757c25 100644 --- a/docs/abc.md +++ b/docs/abc.md @@ -10,4 +10,16 @@ .. automethod:: decode .. automethod:: get_config .. automethod:: from_config + +Protocols +--------- + +These describe codecs structurally, for code that accepts any object implementing the +interface whether or not it subclasses :class:`Codec`. The three kinds mirror the codec +kinds of the Zarr version 3 specification. + +.. autoclass:: CodecProtocol +.. autoclass:: BytesBytesCodec +.. autoclass:: ArrayArrayCodec +.. autoclass:: ArrayBytesCodec ``` diff --git a/docs/release.md b/docs/release.md index bd896673b..11d493759 100644 --- a/docs/release.md +++ b/docs/release.md @@ -58,6 +58,20 @@ public. `from numcodecs import *` now imports only those names. * `register_codec` raises `ValueError` for a class without a `codec_id` when no id is given, instead of registering it under `None`. +* `Codec.codec_id` no longer defaults to `None`. Abstract intermediate classes such as + `Checksum32` leave it unset, so reading it raises `AttributeError`; every concrete codec + sets it. + +### Enhancements + +* Annotate `numcodecs.abc.Codec`. `encode` and `decode` take and return + `numcodecs.ndarray_like.BufferLike` (an object exporting the buffer protocol, or an + ndarray-like), with `buf` positional-only, and `decode(buf, out=out)` is typed to return + the type of `out`. `get_config` returns `Mapping[str, Any]` and `from_config` returns + `Self`. Previously pyright inferred that `encode` and `decode` return `None`. +* Add structural protocols `numcodecs.abc.CodecProtocol`, `BytesBytesCodec`, + `ArrayArrayCodec` and `ArrayBytesCodec` for code that accepts any object implementing the + codec interface. ### Deprecations diff --git a/src/numcodecs/abc.py b/src/numcodecs/abc.py index 6865df09e..fd9555690 100644 --- a/src/numcodecs/abc.py +++ b/src/numcodecs/abc.py @@ -29,17 +29,31 @@ """ from abc import ABC, abstractmethod +from collections.abc import Buffer, Mapping +from typing import Any, Protocol, Self, overload + +from .ndarray_like import BufferLike, NDArrayLike + +__all__ = [ + "ArrayArrayCodec", + "ArrayBytesCodec", + "BytesBytesCodec", + "Codec", + "CodecProtocol", +] class Codec(ABC): """Codec abstract base class.""" - # override in sub-class - codec_id: str | None = None - """Codec identifier.""" + # Declared without a value (abstract intermediate classes leave it unset) and without + # `ClassVar`, so that `Codec` instances match protocols that declare `codec_id` either + # as a plain attribute or as a read-only property. + codec_id: str + """Codec identifier. Every concrete codec class sets it as a class attribute.""" @abstractmethod - def encode(self, buf): # pragma: no cover + def encode(self, buf: BufferLike, /) -> BufferLike: # pragma: no cover """Encode data in `buf`. Parameters @@ -55,8 +69,14 @@ def encode(self, buf): # pragma: no cover protocol. """ + @overload + def decode(self, buf: BufferLike, /, out: None = None) -> BufferLike: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, /, out: OutT) -> OutT: ... @abstractmethod - def decode(self, buf, out=None): # pragma: no cover + def decode( + self, buf: BufferLike, /, out: BufferLike | None = None + ) -> BufferLike: # pragma: no cover """Decode data in `buf`. Parameters @@ -72,10 +92,10 @@ def decode(self, buf, out=None): # pragma: no cover ------- dec : buffer-like Decoded data. May be any object supporting the new-style - buffer protocol. + buffer protocol. If `out` was provided, `out` itself. """ - def get_config(self): + def get_config(self) -> Mapping[str, Any]: """Return a dictionary holding configuration parameters for this codec. Must include an 'id' field with the codec identifier. All values must be compatible with JSON encoding.""" @@ -83,7 +103,7 @@ def get_config(self): # override in sub-class if need special encoding of config values # setup config object - config = {'id': self.codec_id} + config: dict[str, Any] = {'id': self.codec_id} # by default, assume all non-private members are configuration # parameters - override this in sub-class if not the case @@ -94,8 +114,8 @@ def get_config(self): return config @classmethod - def from_config(cls, config): - """Instantiate codec from a configuration object.""" + def from_config(cls, config: Mapping[str, Any], /) -> Self: + """Instantiate codec from a configuration object, without its 'id' field.""" # N.B., assume at this point the 'id' field has been removed from # the config object @@ -105,14 +125,14 @@ def from_config(cls, config): # keyword arguments without any special decoding return cls(**config) - def __eq__(self, other): + def __eq__(self, other: object) -> bool: # override in sub-class if need special equality comparison try: - return self.get_config() == other.get_config() + return bool(self.get_config() == other.get_config()) # type: ignore[attr-defined] except AttributeError: return False - def __repr__(self): + def __repr__(self) -> str: # override in sub-class if need special representation # by default, assume all non-private members are configuration @@ -124,3 +144,44 @@ def __repr__(self): ] r += ', '.join(params) + ')' return r + + +class CodecProtocol(Protocol): + """The structural interface of a codec, for code that accepts any object implementing + it, whether or not it subclasses `Codec`.""" + + @property + def codec_id(self) -> str: ... # pragma: no cover + + def encode(self, buf: BufferLike, /) -> BufferLike: ... # pragma: no cover + + def decode(self, buf: BufferLike, /, out: Any = None) -> BufferLike: ... # pragma: no cover + + def get_config(self) -> Mapping[str, Any]: ... # pragma: no cover + + @classmethod + def from_config(cls, config: Mapping[str, Any], /) -> Self: ... # pragma: no cover + + +class BytesBytesCodec(CodecProtocol, Protocol): + """A codec that maps bytes to bytes, such as a compressor or a checksum.""" + + def encode(self, buf: BufferLike, /) -> Buffer: ... # pragma: no cover + + def decode(self, buf: BufferLike, /, out: Any = None) -> Buffer: ... # pragma: no cover + + +class ArrayArrayCodec(CodecProtocol, Protocol): + """A codec that maps an array to an array, such as a filter.""" + + def encode(self, buf: BufferLike, /) -> NDArrayLike: ... # pragma: no cover + + def decode(self, buf: BufferLike, /, out: Any = None) -> NDArrayLike: ... # pragma: no cover + + +class ArrayBytesCodec(CodecProtocol, Protocol): + """A codec that serializes an array to bytes and back.""" + + def encode(self, buf: BufferLike, /) -> Buffer: ... # pragma: no cover + + def decode(self, buf: BufferLike, /, out: Any = None) -> NDArrayLike: ... # pragma: no cover diff --git a/src/numcodecs/registry.py b/src/numcodecs/registry.py index 758cc0082..fa2a5f592 100644 --- a/src/numcodecs/registry.py +++ b/src/numcodecs/registry.py @@ -82,7 +82,7 @@ def register_codec(cls: type[Codec], codec_id: str | None = None) -> None: """ if codec_id is None: - codec_id = cls.codec_id + codec_id = getattr(cls, "codec_id", None) if codec_id is None: raise ValueError(f"cannot register {cls!r}: it has no codec_id and none was given") logger.debug("Registering codec '%s'", codec_id) diff --git a/tests/test_codec_contract.py b/tests/test_codec_contract.py index bf47cc007..5d871c002 100644 --- a/tests/test_codec_contract.py +++ b/tests/test_codec_contract.py @@ -21,6 +21,7 @@ import numcodecs from numcodecs import pickles from numcodecs.abc import Codec +from numcodecs.compat import ensure_ndarray @dataclass(frozen=True) @@ -147,9 +148,9 @@ def test_decode_into_ndarray_returns_out(codec_id: str) -> None: out = np.empty_like(data) assert codec.decode(enc, out=out) is out if CASES[codec_id].is_object: - assert all(np.array_equal(a, b) for a, b in zip(out, expected, strict=True)) + assert all(np.array_equal(a, b) for a, b in zip(out, ensure_ndarray(expected), strict=True)) else: - assert np.asarray(memoryview(expected)).tobytes() == out.tobytes() + assert ensure_ndarray(expected).tobytes() == out.tobytes() @pytest.mark.parametrize( @@ -161,7 +162,7 @@ def test_decode_into_bytearray_returns_out(codec_id: str) -> None: expected = codec.decode(enc) out = bytearray(data.nbytes) assert codec.decode(enc, out=out) is out - assert bytes(out) == np.asarray(memoryview(expected)).tobytes() + assert bytes(out) == ensure_ndarray(expected).tobytes() @pytest.mark.parametrize("codec_id", _params(set())) diff --git a/tests/test_registry.py b/tests/test_registry.py index 2af73f058..ea44ea109 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -35,8 +35,8 @@ def test_all_classes_registered(): if ( inspect.isclass(obj) and issubclass(obj, numcodecs.abc.Codec) + and not inspect.isabstract(obj) and obj.codec_id not in numcodecs.registry.codec_registry - and obj.codec_id is not None # remove `None` ) } diff --git a/tests/typing/test_abc_types.py b/tests/typing/test_abc_types.py new file mode 100644 index 000000000..ababbebb4 --- /dev/null +++ b/tests/typing/test_abc_types.py @@ -0,0 +1,92 @@ +"""Static types of `numcodecs.abc`, checked against runtime behavior.""" + +from collections.abc import Mapping +from typing import Any, Self, assert_type, overload + +import numpy as np +import numpy.typing as npt + +import numcodecs +from numcodecs.abc import Codec, CodecProtocol +from numcodecs.compat import ensure_bytes, ndarray_copy +from numcodecs.ndarray_like import BufferLike + + +def _zlib() -> Codec: + return numcodecs.get_codec({"id": "zlib", "level": 1}) + + +def test_codec_methods() -> None: + codec = _zlib() + data = np.arange(16, dtype="f8") + + assert_type(codec.codec_id, str) + assert codec.codec_id == "zlib" + + enc = assert_type(codec.encode(data), BufferLike) + dec = assert_type(codec.decode(enc), BufferLike) + assert ensure_bytes(dec) == data.tobytes() + + out: npt.NDArray[np.float64] = np.empty_like(data) + assert assert_type(codec.decode(enc, out=out), npt.NDArray[np.float64]) is out + buf = bytearray(data.nbytes) + assert assert_type(codec.decode(enc, out=buf), bytearray) is buf + + config = assert_type(codec.get_config(), Mapping[str, Any]) + assert config["id"] == "zlib" + rebuilt = assert_type(type(codec).from_config({"level": 1}), Codec) + assert rebuilt == codec + + assert_type(codec == object(), bool) + + +def test_codec_satisfies_codec_protocol() -> None: + codec: CodecProtocol = _zlib() + assert codec.codec_id == "zlib" + + +class _Identity(Codec): + """A third-party codec typed as precisely as numcodecs' own.""" + + codec_id = "test-identity" + + def encode(self, buf: BufferLike, /) -> bytes: + return ensure_bytes(buf) + + @overload + def decode(self, buf: BufferLike, /, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, /, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, /, out: BufferLike | None = None) -> BufferLike: + return ndarray_copy(ensure_bytes(buf), out) + + +class _AnyTyped(Codec): + """A third-party codec annotated with `Any`, as imagecodecs does.""" + + codec_id = "test-any" + + def encode(self, buf: Any) -> Any: + return buf + + def decode(self, buf: Any, out: Any = None) -> Any: + return buf + + def get_config(self) -> dict[str, Any]: + return {"id": self.codec_id} + + @classmethod + def from_config(cls, config: Mapping[str, Any]) -> Self: + return cls() + + +def test_subclasses() -> None: + identity = _Identity() + assert assert_type(identity.encode(b"abc"), bytes) == b"abc" + assert assert_type(identity.decode(b"abc"), bytes) == b"abc" + out = bytearray(3) + assert assert_type(identity.decode(b"abc", out=out), bytearray) is out + + any_typed = _AnyTyped() + assert any_typed.decode(any_typed.encode(b"abc")) == b"abc" + assert_type(_AnyTyped.from_config({}), _AnyTyped)