From ff36c016cebef004d4b896609a951daa854d24b8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 21 Nov 2026 13:48:18 +0100 Subject: [PATCH 1/8] test: check the Codec contract for every codec and add static type tests Add tests/test_codec_contract.py, which runs every registered codec through the behavior the Codec base class promises: encode and decode return buffers, decode(out=...) returns the caller's out, and get_config round-trips through from_config. Known violations are strict xfails, so fixing one forces its entry to be removed. Add tests/typing, where assert_type checks (verified by mypy and pyright in strict mode) sit next to runtime assertions (verified by pytest), and a `typing` CI job that runs both checkers plus the repository's own mypy configuration. The type checkers are pinned in a new `typing` dependency group. Assisted-by: ClaudeCode:claude-opus-5-5 --- .github/workflows/ci.yaml | 37 ++++++ docs/release.md | 3 + pyproject.toml | 4 + tests/test_codec_contract.py | 203 ++++++++++++++++++++++++++++++ tests/typing/README.md | 25 ++++ tests/typing/__init__.py | 0 tests/typing/mypy.ini | 15 +++ tests/typing/pyrightconfig.json | 14 +++ tests/typing/test_compat_types.py | 31 +++++ 9 files changed, 332 insertions(+) create mode 100644 tests/test_codec_contract.py create mode 100644 tests/typing/README.md create mode 100644 tests/typing/__init__.py create mode 100644 tests/typing/mypy.ini create mode 100644 tests/typing/pyrightconfig.json create mode 100644 tests/typing/test_compat_types.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9f6583327..bec8fb69b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -125,6 +125,43 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} verbose: true + typing: + # Static types: the repository's own mypy configuration over src and tests, and the + # strict type tests in tests/typing under both mypy and pyright (see its README). + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -el {0} + + steps: + - name: Checkout source + uses: actions/checkout@v6 + with: + submodules: recursive + fetch-depth: 0 # required for version resolution + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + cache: 'pip' + + - name: Install numcodecs and the type checkers + run: | + python -m pip install --upgrade pip # --group needs pip >= 25.1 + python -m pip install -v ".[test,msgpack,crc32c]" + python -m pip install --group typing + + - name: mypy (repository configuration) + run: mypy --config-file pyproject.toml src tests + + - name: mypy (strict type tests) + run: mypy --config-file tests/typing/mypy.ini tests/typing + + - name: pyright (strict type tests) + run: pyright -p tests/typing + test-zarr: runs-on: ubuntu-latest strategy: diff --git a/docs/release.md b/docs/release.md index b54747679..6847e8ea6 100644 --- a/docs/release.md +++ b/docs/release.md @@ -29,6 +29,9 @@ wheels) rather than every branch, test the oldest supported and latest Zarr releases, use Alpine's numpy in the i386 job, and remove the unused release-notes labeler. +* Check the behavior every codec promises through `numcodecs.abc.Codec` in a new + `tests/test_codec_contract.py`, and add strict static type tests in `tests/typing`, run by + mypy and pyright in a new `typing` CI job. ### Fixes diff --git a/pyproject.toml b/pyproject.toml index f71320993..721d18125 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,6 +90,10 @@ dev = [ "pytest==9.0.3", "pytest-cov", ] +typing = [ + "mypy==2.4.1", + "pyright==1.1.414", +] test-zarr-313 = [ {include-group="dev"}, "zarr==3.1.3", diff --git a/tests/test_codec_contract.py b/tests/test_codec_contract.py new file mode 100644 index 000000000..c328a25cc --- /dev/null +++ b/tests/test_codec_contract.py @@ -0,0 +1,203 @@ +"""The behavior every codec promises through `numcodecs.abc.Codec`, checked for every codec. + +These tests pin down the contract that the type annotations on `Codec` describe: + +- `encode` and `decode` return a buffer-like object (one that exports the buffer protocol). +- `decode(buf, out=out)` writes into `out` and returns that same object. +- `get_config()` holds an `id` and JSON-serializable values, and `from_config` rebuilds an + equal codec from it once `id` is removed. +""" + +from __future__ import annotations + +import json +import warnings +from collections.abc import Callable +from dataclasses import dataclass + +import numpy as np +import pytest + +import numcodecs +from numcodecs import pickles +from numcodecs.abc import Codec + + +@dataclass(frozen=True) +class Case: + """A codec and an array it can round-trip.""" + + make: Callable[[], Codec] + data: np.ndarray + + @property + def is_object(self) -> bool: + return self.data.dtype == object + + +f8 = np.linspace(0, 100, 16, dtype="f8") +obj_str = np.array(["foo", "bar", "baz", "qux"], dtype=object) + +CASES: dict[str, Case] = { + "zlib": Case(lambda: numcodecs.Zlib(), f8), + "gzip": Case(lambda: numcodecs.GZip(), f8), + "bz2": Case(lambda: numcodecs.BZ2(), f8), + "lzma": Case(lambda: numcodecs.LZMA(), f8), + "blosc": Case(lambda: numcodecs.Blosc(), f8), + "zstd": Case(lambda: numcodecs.Zstd(), f8), + "lz4": Case(lambda: numcodecs.LZ4(), f8), + "base64": Case(lambda: numcodecs.Base64(), f8), + "shuffle": Case(lambda: numcodecs.Shuffle(elementsize=8), f8), + "crc32": Case(lambda: numcodecs.CRC32(), f8), + "adler32": Case(lambda: numcodecs.Adler32(), f8), + "jenkins_lookup3": Case(lambda: numcodecs.JenkinsLookup3(), f8), + "fletcher32": Case(lambda: numcodecs.Fletcher32(), f8), + "astype": Case(lambda: numcodecs.AsType(encode_dtype="f4", decode_dtype="f8"), f8), + "delta": Case(lambda: numcodecs.Delta(dtype="i4"), np.arange(16, dtype="i4")), + "quantize": Case(lambda: numcodecs.Quantize(digits=1, dtype="f8"), f8), + "fixedscaleoffset": Case( + lambda: numcodecs.FixedScaleOffset(offset=0, scale=10, dtype="f8", astype="i4"), f8 + ), + "bitround": Case(lambda: numcodecs.BitRound(keepbits=10), f8), + "packbits": Case(lambda: numcodecs.PackBits(), np.array([True, False] * 8)), + "categorize": Case( + lambda: numcodecs.Categorize(labels=["foo", "bar", "baz", "qux"], dtype=object), obj_str + ), + "pickle": Case(lambda: numcodecs.Pickle(), obj_str), + "json2": Case(lambda: numcodecs.JSON(), obj_str), + "vlen-utf8": Case(lambda: numcodecs.VLenUTF8(), obj_str), + "vlen-bytes": Case( + lambda: numcodecs.VLenBytes(), np.array([b"foo", b"bar", b"baz", b"qux"], dtype=object) + ), + "vlen-array": Case( + lambda: numcodecs.VLenArray(dtype="i4"), + np.array([np.arange(3, dtype="i4"), np.arange(2, dtype="i4")] * 2, dtype=object), + ), + "msgpack2": Case(lambda: numcodecs.MsgPack(), obj_str), + "zfpy": Case(lambda: numcodecs.ZFPY(), f8), + "pcodec": Case(lambda: numcodecs.PCodec(), f8), + "crc32c": Case(lambda: numcodecs.CRC32C(), f8), +} + +# Codecs whose decode(..., out=bytearray) cannot work, because the codec needs `out` to carry +# a dtype: PCodec decodes straight into a typed array. +NO_BYTEARRAY_OUT = frozenset({"pcodec"}) + +# Known violations of the `out` contract. Each entry is a bug; remove it when fixed. +# `strict=True` makes a fixed entry fail, so this list cannot go stale. +OUT_NDARRAY_XFAIL: set[str] = { + "zlib", + "bz2", + "lzma", + "base64", + "shuffle", + "zfpy", + "pcodec", + "vlen-utf8", + "vlen-bytes", + "vlen-array", +} +OUT_BYTEARRAY_XFAIL: set[str] = { + "zlib", + "bz2", + "lzma", + "base64", + "shuffle", + "crc32", + "adler32", + "crc32c", + "jenkins_lookup3", + "astype", + "delta", + "quantize", + "fixedscaleoffset", + "bitround", + "packbits", + "zfpy", +} +DECODE_RETURNS_BUFFER_XFAIL: set[str] = set() + + +def _codec(codec_id: str) -> Codec: + case = CASES[codec_id] + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return case.make() + except AttributeError: + pytest.skip(f"{codec_id} is not available (optional dependency missing)") + + +def _params( + xfail: set[str], *, skip: frozenset[str] = frozenset(), objects: bool = True +) -> list[object]: + params: list[object] = [] + for codec_id, case in CASES.items(): + if codec_id in skip or (case.is_object and not objects): + continue + marks = [pytest.mark.xfail(strict=True, reason="known bug")] if codec_id in xfail else [] + params.append(pytest.param(codec_id, marks=marks, id=codec_id)) + return params + + +@pytest.fixture(autouse=True) +def _allow_pickle(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pickles, "ALLOW_UNSAFE_PICKLE_DECODE", True) + + +def _is_buffer_like(obj: object) -> bool: + try: + memoryview(obj) # type: ignore[arg-type] + except TypeError: + return False + return True + + +def test_every_registered_codec_has_a_case() -> None: + """A new codec must be added to `CASES` so that the contract is checked for it.""" + assert set(numcodecs.registry.codec_registry) <= set(CASES) + + +@pytest.mark.parametrize("codec_id", _params(DECODE_RETURNS_BUFFER_XFAIL)) +def test_encode_decode_return_buffers(codec_id: str) -> None: + codec, data = _codec(codec_id), CASES[codec_id].data + enc = codec.encode(data) + assert _is_buffer_like(enc), type(enc) + dec = codec.decode(enc) + assert _is_buffer_like(dec), type(dec) + + +@pytest.mark.parametrize("codec_id", _params(OUT_NDARRAY_XFAIL)) +def test_decode_into_ndarray_returns_out(codec_id: str) -> None: + codec, data = _codec(codec_id), CASES[codec_id].data + enc = codec.encode(data) + expected = codec.decode(enc) + out = np.empty_like(data) + assert codec.decode(enc, out=out) is out + if CASES[codec_id].is_object: + assert all(np.array_equal(a, b) for a, b in zip(out, expected, strict=True)) + else: + assert np.asarray(memoryview(expected)).tobytes() == out.tobytes() + + +@pytest.mark.parametrize( + "codec_id", _params(OUT_BYTEARRAY_XFAIL, skip=NO_BYTEARRAY_OUT, objects=False) +) +def test_decode_into_bytearray_returns_out(codec_id: str) -> None: + codec, data = _codec(codec_id), CASES[codec_id].data + enc = codec.encode(data) + expected = codec.decode(enc) + out = bytearray(data.nbytes) + assert codec.decode(enc, out=out) is out + assert bytes(out) == np.asarray(memoryview(expected)).tobytes() + + +@pytest.mark.parametrize("codec_id", _params(set())) +def test_config_round_trip(codec_id: str) -> None: + codec = _codec(codec_id) + config = codec.get_config() + assert config["id"] == codec.codec_id + rebuilt_json = json.loads(json.dumps(config)) + assert numcodecs.get_codec(rebuilt_json) == codec + without_id = {k: v for k, v in config.items() if k != "id"} + assert type(codec).from_config(without_id) == codec diff --git a/tests/typing/README.md b/tests/typing/README.md new file mode 100644 index 000000000..a885a0967 --- /dev/null +++ b/tests/typing/README.md @@ -0,0 +1,25 @@ +# Static type tests + +The files here check numcodecs' type annotations against its runtime behavior. Each test +pairs `typing.assert_type`, which mypy and pyright check, with a runtime assertion, which +pytest checks. An annotation that type checkers accept but the runtime contradicts makes one +of the two fail. + +Run them from the repository root: + +```bash +uv pip install --group typing +pytest tests/typing +mypy --config-file tests/typing/mypy.ini tests/typing +pyright -p tests/typing +``` + +`mypy.ini` and `pyrightconfig.json` run both checkers in strict mode and read numcodecs from +`src/`. Errors inside numcodecs itself are not reported here: the repository's own mypy +configuration in `pyproject.toml` covers those. + +`pyrightconfig.json` turns off pyright's three `reportUnknown*` rules, because most numcodecs +parameters are not annotated yet. Turn them back on once they are. + +When you change an annotation in numcodecs, add or update an `assert_type` here that shows +what callers now get. diff --git a/tests/typing/__init__.py b/tests/typing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/typing/mypy.ini b/tests/typing/mypy.ini new file mode 100644 index 000000000..26579ad7e --- /dev/null +++ b/tests/typing/mypy.ini @@ -0,0 +1,15 @@ +# Strict mypy configuration for the static type tests in this directory. +# Run from the repository root: `mypy --config-file tests/typing/mypy.ini tests/typing` +[mypy] +strict = true +warn_unreachable = true +enable_error_code = ignore-without-code, redundant-expr, truthy-bool +# Read numcodecs from the source tree. It does not ship a py.typed marker yet, so mypy +# would otherwise refuse to analyze the installed copy. +mypy_path = src + +# Report errors in the tests only. numcodecs itself is checked by the repository's own +# configuration in pyproject.toml. +[mypy-numcodecs.*] +follow_imports = silent +ignore_errors = true diff --git a/tests/typing/pyrightconfig.json b/tests/typing/pyrightconfig.json new file mode 100644 index 000000000..42540aad6 --- /dev/null +++ b/tests/typing/pyrightconfig.json @@ -0,0 +1,14 @@ +{ + "include": [ + "." + ], + "typeCheckingMode": "strict", + "pythonVersion": "3.12", + "extraPaths": [ + "../../src" + ], + "reportMissingTypeStubs": "none", + "reportUnknownVariableType": "none", + "reportUnknownMemberType": "none", + "reportUnknownArgumentType": "none" +} diff --git a/tests/typing/test_compat_types.py b/tests/typing/test_compat_types.py new file mode 100644 index 000000000..f656c4111 --- /dev/null +++ b/tests/typing/test_compat_types.py @@ -0,0 +1,31 @@ +"""Static types of `numcodecs.compat`, checked against runtime behavior. + +Each `assert_type` is verified by mypy and pyright (see `tests/typing/README.md`), and the +`assert isinstance` next to it is verified by pytest. Together they keep the annotations +honest: a claim that type checkers accept but the runtime contradicts fails here. +""" + +from typing import Any, assert_type + +import numpy as np +import numpy.typing as npt + +from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray, ensure_ndarray + + +def test_ensure_bytes() -> None: + for buf in (b"abc", bytearray(b"abc"), memoryview(b"abc"), np.arange(3, dtype="u1")): + result = assert_type(ensure_bytes(buf), bytes) + assert isinstance(result, bytes) + + +def test_ensure_ndarray() -> None: + for buf in (b"abc", bytearray(b"abc"), memoryview(b"abc"), np.arange(3, dtype="u1")): + result = assert_type(ensure_ndarray(buf), npt.NDArray[Any]) + assert isinstance(result, np.ndarray) + + +def test_ensure_contiguous_ndarray() -> None: + for buf in (b"abc", bytearray(b"abc"), memoryview(b"abc"), np.arange(3, dtype="u1")): + result = assert_type(ensure_contiguous_ndarray(buf), npt.NDArray[Any]) + assert isinstance(result, np.ndarray) From 145994e1e6c902499d87214a9dca0e556a1b6eb4 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 21 Nov 2026 13:50:06 +0100 Subject: [PATCH 2/8] fix: return the caller's out from decode(buf, out=out) in every codec The Codec docstring says decode writes into `out`, and callers expect `out` back. Ten codecs (Zlib, BZ2, LZMA, Base64, Shuffle, ZFPY, PCodec and the three VLen codecs) rebound `out` to a flattened view and returned that, and numcodecs.compat.ndarray_copy did the same for any `out` that was not an ndarray, which every filter and checksum codec inherited. The decoded bytes were already in the caller's buffer; only the returned object changes. JenkinsLookup3.decode now accepts any contiguous writable buffer as `out` and checks its size, and Fletcher32.decode returns a uint8 ndarray instead of a Cython memoryview slice that no type annotation could name. The strict xfails in tests/test_codec_contract.py are removed. Assisted-by: ClaudeCode:claude-opus-5-5 --- docs/release.md | 9 +++++++++ src/numcodecs/base64.py | 2 +- src/numcodecs/bz2.py | 2 +- src/numcodecs/checksum32.py | 7 ++++++- src/numcodecs/compat.py | 10 ++++++++-- src/numcodecs/fletcher32.pyx | 2 +- src/numcodecs/lzma.py | 2 +- src/numcodecs/pcodec.py | 3 +-- src/numcodecs/shuffle.py | 20 ++++++++------------ src/numcodecs/vlen.pyx | 24 ++++++++++++------------ src/numcodecs/zfpy.py | 2 +- src/numcodecs/zlib.py | 2 +- tests/test_codec_contract.py | 32 ++------------------------------ tests/test_jenkins.py | 7 +++++++ 14 files changed, 59 insertions(+), 65 deletions(-) diff --git a/docs/release.md b/docs/release.md index 6847e8ea6..51e2933bb 100644 --- a/docs/release.md +++ b/docs/release.md @@ -38,6 +38,15 @@ * `import numcodecs` no longer fails when a zfpy post-release such as 1.0.1.post0 is installed. zfpy versions are now parsed with `packaging`, which becomes a dependency, and the `zfpy` extra now requires zfpy >= 1.0.1. {issue}`892` +* `decode(buf, out=out)` now returns `out` itself for every codec, as the `Codec` + docstring describes. Zlib, BZ2, LZMA, Base64, Shuffle, ZFPY, PCodec, the three VLen + codecs, and every codec built on `numcodecs.compat.ndarray_copy` returned a flattened view + of `out` instead, or a new ndarray when `out` was a `bytearray`. The decoded bytes were + already written into `out`; only the returned object changes. +* `JenkinsLookup3.decode` accepts any contiguous writable buffer as `out`, not only an + ndarray, and raises `ValueError` when `out` has the wrong size. +* `Fletcher32.decode` without `out` returns a `uint8` ndarray, like the other checksum + codecs, instead of a Cython memoryview slice. (release_0.17.1)= diff --git a/src/numcodecs/base64.py b/src/numcodecs/base64.py index 307c6d52b..c107d5c7b 100644 --- a/src/numcodecs/base64.py +++ b/src/numcodecs/base64.py @@ -19,7 +19,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression decompressed = _base64.standard_b64decode(buf) # handle destination diff --git a/src/numcodecs/bz2.py b/src/numcodecs/bz2.py index 2b63b147b..a3c8f5ec3 100644 --- a/src/numcodecs/bz2.py +++ b/src/numcodecs/bz2.py @@ -31,7 +31,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # N.B., bz2 cannot handle ndarray directly because of truth testing issues buf = memoryview(buf) diff --git a/src/numcodecs/checksum32.py b/src/numcodecs/checksum32.py index b02c0a6a2..a095b1718 100644 --- a/src/numcodecs/checksum32.py +++ b/src/numcodecs/checksum32.py @@ -185,7 +185,12 @@ def decode(self, buf, out=None): "This could be a sign that the data has been corrupted." ) if out is not None: - out.view("uint8")[:] = b[:-4] + out_view = ensure_contiguous_ndarray(out).view("uint8") + if out_view.nbytes != b.nbytes - 4: + raise ValueError( + f"out has {out_view.nbytes} bytes, but the decoded data has {b.nbytes - 4}" + ) + out_view[:] = b[:-4] return out return memoryview(b[:-4]) diff --git a/src/numcodecs/compat.py b/src/numcodecs/compat.py index d1844e10b..cc4d2d139 100644 --- a/src/numcodecs/compat.py +++ b/src/numcodecs/compat.py @@ -175,12 +175,18 @@ def ensure_text(s, encoding="utf-8"): def ndarray_copy(src, dst) -> NDArrayLike: - """Copy the contents of the array from `src` to `dst`.""" + """Copy the contents of the array from `src` to `dst`, and return `dst`. + + If `dst` is `None`, return `src` unchanged. Otherwise `dst` itself is returned, not + a view of it, so callers that pass a `bytearray` get that `bytearray` back. + """ if dst is None: # no-op return src + out = dst + # ensure ndarray like src = ensure_ndarray_like(src) dst = ensure_ndarray_like(dst) @@ -203,4 +209,4 @@ def ndarray_copy(src, dst) -> NDArrayLike: # copy via numpy np.copyto(dst, src) - return dst + return out diff --git a/src/numcodecs/fletcher32.pyx b/src/numcodecs/fletcher32.pyx index 1ecbb45d6..ab9ce6a8f 100644 --- a/src/numcodecs/fletcher32.pyx +++ b/src/numcodecs/fletcher32.pyx @@ -112,5 +112,5 @@ class Fletcher32(Codec): out_ptr = &out_mv[0] memcpy(out_ptr, b_ptr, b_len - FOOTER_LENGTH) else: - out = b_mv[:-FOOTER_LENGTH] + out = b[:-FOOTER_LENGTH] return out diff --git a/src/numcodecs/lzma.py b/src/numcodecs/lzma.py index 7b6b23487..97da02825 100644 --- a/src/numcodecs/lzma.py +++ b/src/numcodecs/lzma.py @@ -59,7 +59,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression dec = _lzma.decompress(buf, format=self.format, filters=self.filters) diff --git a/src/numcodecs/pcodec.py b/src/numcodecs/pcodec.py index c7108c7b8..739666b11 100644 --- a/src/numcodecs/pcodec.py +++ b/src/numcodecs/pcodec.py @@ -122,8 +122,7 @@ def encode(self, buf): def decode(self, buf, out=None): buf = ensure_bytes(buf) if out is not None: - out = ensure_contiguous_ndarray(out) - standalone.simple_decompress_into(buf, out) + standalone.simple_decompress_into(buf, ensure_contiguous_ndarray(out)) return out else: return standalone.simple_decompress(buf) diff --git a/src/numcodecs/shuffle.py b/src/numcodecs/shuffle.py index 0e7031f5e..e29e5a593 100644 --- a/src/numcodecs/shuffle.py +++ b/src/numcodecs/shuffle.py @@ -38,24 +38,20 @@ def _prepare_arrays(self, buf, out): return buf, out def encode(self, buf, out=None): - buf, out = self._prepare_arrays(buf, out) + buf, out_arr = self._prepare_arrays(buf, out) - if self.elementsize <= 1: - return out # no shuffling needed - - _doShuffle(buf.view("uint8"), out.view("uint8"), self.elementsize) + if self.elementsize > 1: + _doShuffle(buf.view("uint8"), out_arr.view("uint8"), self.elementsize) - return out + return out_arr if out is None else out def decode(self, buf, out=None): - buf, out = self._prepare_arrays(buf, out) - - if self.elementsize <= 1: - return out # no shuffling needed + buf, out_arr = self._prepare_arrays(buf, out) - _doUnshuffle(buf.view("uint8"), out.view("uint8"), self.elementsize) + if self.elementsize > 1: + _doUnshuffle(buf.view("uint8"), out_arr.view("uint8"), self.elementsize) - return out + return out_arr if out is None else out def __repr__(self): return f'{type(self).__name__}(elementsize={self.elementsize})' diff --git a/src/numcodecs/vlen.pyx b/src/numcodecs/vlen.pyx index f51e8cad7..424cc7ee6 100644 --- a/src/numcodecs/vlen.pyx +++ b/src/numcodecs/vlen.pyx @@ -154,11 +154,11 @@ class VLenUTF8(Codec): # load number of items n_items = load_le32(data) - # setup output + # setup output: fill a flat view, but return the caller's own `out` if out is not None: - out = check_out_param(out, n_items) + out_flat = check_out_param(out, n_items) else: - out = np.empty(n_items, dtype=object) + out = out_flat = np.empty(n_items, dtype=object) # iterate and decode - N.B., do not try to cast `out` as object[:] # as this causes segfaults, possibly similar to @@ -171,7 +171,7 @@ class VLenUTF8(Codec): data += HEADER_LENGTH if data + L > data_end: raise ValueError('corrupt buffer, data seem truncated') - out[i] = PyUnicode_FromStringAndSize(data, L) + out_flat[i] = PyUnicode_FromStringAndSize(data, L) data += L return out @@ -283,11 +283,11 @@ class VLenBytes(Codec): # load number of items n_items = load_le32(data) - # setup output + # setup output: fill a flat view, but return the caller's own `out` if out is not None: - out = check_out_param(out, n_items) + out_flat = check_out_param(out, n_items) else: - out = np.empty(n_items, dtype=object) + out = out_flat = np.empty(n_items, dtype=object) # iterate and decode - N.B., do not try to cast `out` as object[:] # as this causes segfaults, possibly similar to @@ -300,7 +300,7 @@ class VLenBytes(Codec): data += HEADER_LENGTH if data + L > data_end: raise ValueError('corrupt buffer, data seem truncated') - out[i] = PyBytes_FromStringAndSize(data, L) + out_flat[i] = PyBytes_FromStringAndSize(data, L) data += L return out @@ -439,11 +439,11 @@ class VLenArray(Codec): # load number of items n_items = load_le32(data) - # setup output + # setup output: fill a flat view, but return the caller's own `out` if out is not None: - out = check_out_param(out, n_items) + out_flat = check_out_param(out, n_items) else: - out = np.empty(n_items, dtype=object) + out = out_flat = np.empty(n_items, dtype=object) # iterate and decode - N.B., do not try to cast `out` as object[:] # as this causes segfaults, possibly similar to @@ -463,7 +463,7 @@ class VLenArray(Codec): v_pb = PyMemoryView_GET_BUFFER(v_mv) memcpy(v_pb.buf, data, L) - out[i] = v + out_flat[i] = v data += L return out diff --git a/src/numcodecs/zfpy.py b/src/numcodecs/zfpy.py index 41ccfe98c..86fb090a8 100644 --- a/src/numcodecs/zfpy.py +++ b/src/numcodecs/zfpy.py @@ -103,7 +103,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_bytes(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression dec = _zfpy.decompress_numpy(buf) diff --git a/src/numcodecs/zlib.py b/src/numcodecs/zlib.py index ca247af29..e62658dcb 100644 --- a/src/numcodecs/zlib.py +++ b/src/numcodecs/zlib.py @@ -31,7 +31,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression dec = _zlib.decompress(buf) diff --git a/tests/test_codec_contract.py b/tests/test_codec_contract.py index c328a25cc..bf47cc007 100644 --- a/tests/test_codec_contract.py +++ b/tests/test_codec_contract.py @@ -85,36 +85,8 @@ def is_object(self) -> bool: # Known violations of the `out` contract. Each entry is a bug; remove it when fixed. # `strict=True` makes a fixed entry fail, so this list cannot go stale. -OUT_NDARRAY_XFAIL: set[str] = { - "zlib", - "bz2", - "lzma", - "base64", - "shuffle", - "zfpy", - "pcodec", - "vlen-utf8", - "vlen-bytes", - "vlen-array", -} -OUT_BYTEARRAY_XFAIL: set[str] = { - "zlib", - "bz2", - "lzma", - "base64", - "shuffle", - "crc32", - "adler32", - "crc32c", - "jenkins_lookup3", - "astype", - "delta", - "quantize", - "fixedscaleoffset", - "bitround", - "packbits", - "zfpy", -} +OUT_NDARRAY_XFAIL: set[str] = set() +OUT_BYTEARRAY_XFAIL: set[str] = set() DECODE_RETURNS_BUFFER_XFAIL: set[str] = set() diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py index 40d972bf7..511c435ca 100644 --- a/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -148,3 +148,10 @@ def test_out(): j = JenkinsLookup3() result = j.encode(data) j.decode(result, out=data) + + +def test_out_wrong_size(): + j = JenkinsLookup3() + enc = j.encode(np.arange(8, dtype="uint8")) + with pytest.raises(ValueError, match="out has 4 bytes, but the decoded data has 8"): + j.decode(enc, out=bytearray(4)) From 52be3f56167cebaf55dca8262695106e0aeadc76 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 21 Nov 2026 13:54:40 +0100 Subject: [PATCH 3/8] fix(typing): correct registry, compat and NDArrayLike annotations and add __all__ Several annotations described something other than what the code does: - codec_registry holds classes: dict[str, type[Codec]], not dict[str, Codec]. - entries holds single entry points: dict[str, EntryPoint]. - ndarray_copy returned NDArrayLike, but hands back `src` unchanged (often bytes) when `dst` is None. It is now overloaded on `dst`. - NDArrayLike, DType and FlagsObj declared settable attributes, so no numpy.ndarray satisfied NDArrayLike under mypy or pyright. They now declare read-only properties. The isinstance cache moves to module level, because a protocol without annotations of its own would otherwise pick up the metaclass's annotation as a required member. The compat functions annotate their parameters with a new BufferLike alias (collections.abc.Buffer | NDArrayLike), is_ndarray_like returns TypeIs, and get_codec/register_codec are annotated. register_codec now refuses a class without a codec_id instead of registering it under None, and get_codec warns (DeprecationWarning) when an entry point loads a class that does not subclass Codec, so that `-> Codec` can become true. numcodecs/__init__.py defines __all__, so that type checkers treat the re-exported codec classes as public once numcodecs ships py.typed. mypy now checks against Python 3.12, the oldest supported version; doing so caught `typing.TypeIs`, which only exists from 3.13 and now comes from typing_extensions. Assisted-by: ClaudeCode:claude-opus-5-5 --- docs/release.md | 16 +++++ pyproject.toml | 1 + src/numcodecs/__init__.py | 40 ++++++++++++ src/numcodecs/compat.py | 75 ++++++++++++++--------- src/numcodecs/ndarray_like.py | 94 +++++++++++++++++++++-------- src/numcodecs/registry.py | 29 ++++++--- tests/test_registry.py | 34 +++++++++++ tests/typing/mypy.ini | 2 + tests/typing/test_compat_types.py | 58 +++++++++++++++++- tests/typing/test_registry_types.py | 26 ++++++++ 10 files changed, 314 insertions(+), 61 deletions(-) create mode 100644 tests/typing/test_registry_types.py diff --git a/docs/release.md b/docs/release.md index 51e2933bb..bd896673b 100644 --- a/docs/release.md +++ b/docs/release.md @@ -47,6 +47,22 @@ ndarray, and raises `ValueError` when `out` has the wrong size. * `Fletcher32.decode` without `out` returns a `uint8` ndarray, like the other checksum codecs, instead of a Cython memoryview slice. +* Correct type annotations that described something other than what the code does: + `numcodecs.registry.codec_registry` holds codec classes (`dict[str, type[Codec]]`) and + `entries` holds single entry points (`dict[str, EntryPoint]`). `numcodecs.compat` + functions now annotate their parameters, and `ndarray_copy` returns the type of `dst` + (or of `src` when `dst` is `None`). The `NDArrayLike`, `DType` and `FlagsObj` protocols + declare read-only properties, so `numpy.ndarray` now satisfies `NDArrayLike` for type + checkers, and `numcodecs.ndarray_like.BufferLike` names "a buffer or an ndarray-like". +* `numcodecs` defines `__all__`, so type checkers treat its re-exported codec classes as + public. `from numcodecs import *` now imports only those names. +* `register_codec` raises `ValueError` for a class without a `codec_id` when no id is + given, instead of registering it under `None`. + +### Deprecations + +* Loading a codec through the `numcodecs.codecs` entry point group whose class does not + subclass `numcodecs.abc.Codec` now emits a `DeprecationWarning`. (release_0.17.1)= diff --git a/pyproject.toml b/pyproject.toml index 721d18125..3050fdfc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -245,6 +245,7 @@ quote-style = "preserve" [tool.mypy] ignore_errors = false ignore_missing_imports = true +python_version = "3.12" enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] # TODO: set options below to true strict = false diff --git a/src/numcodecs/__init__.py b/src/numcodecs/__init__.py index be60bfc30..bad41090c 100644 --- a/src/numcodecs/__init__.py +++ b/src/numcodecs/__init__.py @@ -25,6 +25,42 @@ from numcodecs.registry import get_codec as get_codec from numcodecs.registry import register_codec +# Names listed here are the public API re-exported from submodules. Optional codecs are +# appended below when their dependency imports. +__all__ = [ + "BZ2", + "CRC32", + "JSON", + "LZ4", + "LZMA", + "Adler32", + "AsType", + "Base64", + "BitRound", + "Blosc", + "Categorize", + "Delta", + "FixedScaleOffset", + "Fletcher32", + "GZip", + "JenkinsLookup3", + "PackBits", + "Pickle", + "Quantize", + "Shuffle", + "VLenArray", + "VLenBytes", + "VLenUTF8", + "Zlib", + "Zstd", + "blosc", + "get_codec", + "lz4", + "register_codec", + "vlen", + "zstd", +] + __version__: str = _version("numcodecs") from numcodecs.zlib import Zlib @@ -131,18 +167,22 @@ from numcodecs.zfpy import ZFPY register_codec(ZFPY) + __all__ += ["ZFPY"] with suppress(ImportError): from numcodecs.msgpacks import MsgPack register_codec(MsgPack) + __all__ += ["MsgPack"] with suppress(ImportError): from numcodecs.checksum32 import CRC32C register_codec(CRC32C) + __all__ += ["CRC32C"] with suppress(ImportError): from numcodecs.pcodec import PCodec register_codec(PCodec) + __all__ += ["PCodec"] diff --git a/src/numcodecs/compat.py b/src/numcodecs/compat.py index cc4d2d139..bad12fd43 100644 --- a/src/numcodecs/compat.py +++ b/src/numcodecs/compat.py @@ -1,12 +1,19 @@ import array import codecs +from collections.abc import Buffer +from typing import Any, overload import numpy as np +import numpy.typing as npt -from .ndarray_like import NDArrayLike, is_ndarray_like +from .ndarray_like import BufferLike, NDArrayLike, is_ndarray_like -def ensure_ndarray_like(buf) -> NDArrayLike: +@overload +def ensure_ndarray_like(buf: Buffer) -> npt.NDArray[Any]: ... +@overload +def ensure_ndarray_like(buf: NDArrayLike) -> NDArrayLike: ... +def ensure_ndarray_like(buf: BufferLike) -> NDArrayLike: """Convenience function to coerce `buf` to ndarray-like array. Parameters @@ -41,7 +48,7 @@ def ensure_ndarray_like(buf) -> NDArrayLike: return buf -def ensure_ndarray(buf) -> np.ndarray: +def ensure_ndarray(buf: BufferLike) -> npt.NDArray[Any]: """Convenience function to coerce `buf` to a numpy array, if it is not already a numpy array. @@ -63,7 +70,17 @@ def ensure_ndarray(buf) -> np.ndarray: return np.array(ensure_ndarray_like(buf), copy=False) -def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> NDArrayLike: +@overload +def ensure_contiguous_ndarray_like( + buf: Buffer, max_buffer_size: int | None = None, flatten: bool = True +) -> npt.NDArray[Any]: ... +@overload +def ensure_contiguous_ndarray_like( + buf: NDArrayLike, max_buffer_size: int | None = None, flatten: bool = True +) -> NDArrayLike: ... +def ensure_contiguous_ndarray_like( + buf: BufferLike, max_buffer_size: int | None = None, flatten: bool = True +) -> NDArrayLike: """Convenience function to coerce `buf` to ndarray-like array. Also ensures that the returned value exports fully contiguous memory, and supports the new-style buffer interface. If the optional max_buffer_size is @@ -95,12 +112,12 @@ def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> N # check for object arrays, these are just memory pointers, actual memory holding # item data is scattered elsewhere - if arr.dtype == object: + if arr.dtype.kind == "O": raise TypeError("object arrays are not supported") # check for datetime or timedelta ndarray, the buffer interface doesn't support those if arr.dtype.kind in "Mm": - arr = arr.view(np.int64) # type: ignore[arg-type] + arr = arr.view(np.int64) # check memory is contiguous, if so flatten if arr.flags.c_contiguous or arr.flags.f_contiguous: @@ -117,7 +134,9 @@ def ensure_contiguous_ndarray_like(buf, max_buffer_size=None, flatten=True) -> N return arr -def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.ndarray: +def ensure_contiguous_ndarray( + buf: BufferLike, max_buffer_size: int | None = None, flatten: bool = True +) -> npt.NDArray[Any]: """Convenience function to coerce `buf` to a numpy array, if it is not already a numpy array. Also ensures that the returned value exports fully contiguous memory, and supports the new-style buffer interface. If the optional max_buffer_size is @@ -150,7 +169,7 @@ def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.nda ) -def ensure_bytes(buf) -> bytes: +def ensure_bytes(buf: BufferLike) -> bytes: """Obtain a bytes object from memory exposed by `buf`.""" if not isinstance(buf, bytes): @@ -158,7 +177,7 @@ def ensure_bytes(buf) -> bytes: # check for object arrays, these are just memory pointers, # actual memory holding item data is scattered elsewhere - if arr.dtype == object: + if arr.dtype.kind == "O": raise TypeError("object arrays are not supported") # create bytes @@ -167,14 +186,18 @@ def ensure_bytes(buf) -> bytes: return buf -def ensure_text(s, encoding="utf-8"): - if not isinstance(s, str): - s = ensure_contiguous_ndarray(s) - s = codecs.decode(s, encoding) - return s +def ensure_text(s: str | BufferLike, encoding: str = "utf-8") -> str: + """Return `s` if it is a `str`, else decode the bytes it exposes with `encoding`.""" + if isinstance(s, str): + return s + return codecs.decode(ensure_contiguous_ndarray(s), encoding) -def ndarray_copy(src, dst) -> NDArrayLike: +@overload +def ndarray_copy[SrcT: BufferLike](src: SrcT, dst: None) -> SrcT: ... +@overload +def ndarray_copy[DstT: BufferLike](src: BufferLike, dst: DstT) -> DstT: ... +def ndarray_copy(src: BufferLike, dst: BufferLike | None) -> BufferLike: """Copy the contents of the array from `src` to `dst`, and return `dst`. If `dst` is `None`, return `src` unchanged. Otherwise `dst` itself is returned, not @@ -185,28 +208,26 @@ def ndarray_copy(src, dst) -> NDArrayLike: # no-op return src - out = dst - # ensure ndarray like - src = ensure_ndarray_like(src) - dst = ensure_ndarray_like(dst) + src_arr: NDArrayLike = ensure_ndarray_like(src) + dst_arr: NDArrayLike = ensure_ndarray_like(dst) # flatten source array - src = src.reshape(-1, order="A") + src_arr = src_arr.reshape(-1, order="A") # ensure same data type - if dst.dtype != object: - src = src.view(dst.dtype) + if dst_arr.dtype.kind != "O": + src_arr = src_arr.view(dst_arr.dtype) # reshape source to match destination - if src.shape != dst.shape: - if dst.flags.f_contiguous: + if src_arr.shape != dst_arr.shape: + if dst_arr.flags.f_contiguous: order = "F" else: order = "C" - src = src.reshape(dst.shape, order=order) + src_arr = src_arr.reshape(dst_arr.shape, order=order) # copy via numpy - np.copyto(dst, src) + np.copyto(dst_arr, src_arr) # type: ignore[arg-type] # cupy arrays support copyto too - return out + return dst diff --git a/src/numcodecs/ndarray_like.py b/src/numcodecs/ndarray_like.py index 06e15ea97..be4171bdf 100644 --- a/src/numcodecs/ndarray_like.py +++ b/src/numcodecs/ndarray_like.py @@ -1,4 +1,12 @@ -from typing import Any, ClassVar, Protocol, runtime_checkable +from collections.abc import Buffer +from typing import Any, Protocol, Self, runtime_checkable + +from typing_extensions import TypeIs # in `typing` from Python 3.13 + +# Kept at module level, not as an annotated attribute of the metaclass: a protocol class +# without annotations of its own would otherwise find the metaclass's `__annotations__`, and +# `runtime_checkable` would then require instances to have the cache as an attribute. +_instancecheck_cache: dict[tuple[type, type], bool] = {} class _CachedProtocolMeta(Protocol.__class__): # type: ignore[name-defined] @@ -11,55 +19,89 @@ class _CachedProtocolMeta(Protocol.__class__): # type: ignore[name-defined] isinstance checks using the object's class as the cache key. """ - _instancecheck_cache: ClassVar[dict[tuple[type, type], bool]] = {} - - def __instancecheck__(self, instance): + def __instancecheck__(self, instance: object) -> bool: key = (self, instance.__class__) - ret = self._instancecheck_cache.get(key) + ret = _instancecheck_cache.get(key) if ret is None: ret = super().__instancecheck__(instance) - self._instancecheck_cache[key] = ret + _instancecheck_cache[key] = ret return ret +# The protocols below declare their attributes as read-only properties. numpy (and cupy) +# expose `dtype`, `shape`, `flags` and friends as read-only properties, and a protocol that +# declared them as plain (settable) attributes would not match a real ndarray. + + @runtime_checkable class DType(Protocol, metaclass=_CachedProtocolMeta): - itemsize: int - name: str - kind: str + @property + def itemsize(self) -> int: ... # pragma: no cover + @property + def name(self) -> str: ... # pragma: no cover + @property + def kind(self) -> str: ... # pragma: no cover + @property + def str(self) -> str: ... # pragma: no cover @runtime_checkable class FlagsObj(Protocol, metaclass=_CachedProtocolMeta): - c_contiguous: bool - f_contiguous: bool - owndata: bool + @property + def c_contiguous(self) -> bool: ... # pragma: no cover + @property + def f_contiguous(self) -> bool: ... # pragma: no cover + @property + def owndata(self) -> bool: ... # pragma: no cover @runtime_checkable class NDArrayLike(Protocol, metaclass=_CachedProtocolMeta): - dtype: DType - shape: tuple[int, ...] - strides: tuple[int, ...] - ndim: int - size: int - itemsize: int - nbytes: int - flags: FlagsObj + """The part of the ndarray interface numcodecs relies on. + + `numpy.ndarray` and `cupy.ndarray` both satisfy it. Methods return `Any` because numpy + overloads them heavily; convert with `numcodecs.compat.ensure_ndarray` when you need + numpy's own types. + """ + + @property + def dtype(self) -> DType: ... # pragma: no cover + @property + def shape(self) -> tuple[int, ...]: ... # pragma: no cover + @property + def strides(self) -> tuple[int, ...]: ... # pragma: no cover + @property + def ndim(self) -> int: ... # pragma: no cover + @property + def size(self) -> int: ... # pragma: no cover + @property + def itemsize(self) -> int: ... # pragma: no cover + @property + def nbytes(self) -> int: ... # pragma: no cover + @property + def flags(self) -> FlagsObj: ... # pragma: no cover def __len__(self) -> int: ... # pragma: no cover - def __getitem__(self, key) -> Any: ... # pragma: no cover + def __getitem__(self, key: Any, /) -> Any: ... # pragma: no cover + + def __setitem__(self, key: Any, value: Any, /) -> None: ... # pragma: no cover + + def tobytes(self, order: Any = ...) -> bytes: ... # pragma: no cover + + def reshape(self, *args: Any, **kwargs: Any) -> Any: ... # pragma: no cover - def __setitem__(self, key, value): ... # pragma: no cover + def view(self, *args: Any, **kwargs: Any) -> Any: ... # pragma: no cover - def tobytes(self, order: str | None = ...) -> bytes: ... # pragma: no cover + def copy(self, *args: Any, **kwargs: Any) -> Self: ... # pragma: no cover - def reshape(self, *shape: int, order: str = ...) -> "NDArrayLike": ... # pragma: no cover - def view(self, dtype: DType = ...) -> "NDArrayLike": ... # pragma: no cover +type BufferLike = Buffer | NDArrayLike +"""What codecs accept and return: an object exporting the buffer protocol (PEP 688), such +as `bytes`, `bytearray`, `memoryview` or a numpy array, or an ndarray-like such as a cupy +array.""" -def is_ndarray_like(obj: object) -> bool: +def is_ndarray_like(obj: object) -> TypeIs[NDArrayLike]: """Return True when `obj` is ndarray-like""" return isinstance(obj, NDArrayLike) diff --git a/src/numcodecs/registry.py b/src/numcodecs/registry.py index ccb5b26bf..758cc0082 100644 --- a/src/numcodecs/registry.py +++ b/src/numcodecs/registry.py @@ -2,17 +2,22 @@ applications to dynamically register and look-up codec classes.""" import logging -from importlib.metadata import EntryPoints, entry_points +import warnings +from collections.abc import Mapping +from importlib.metadata import EntryPoint, entry_points +from typing import Any from numcodecs.abc import Codec from numcodecs.errors import UnknownCodecError logger = logging.getLogger("numcodecs") -codec_registry: dict[str, Codec] = {} -entries: dict[str, EntryPoints] = {} +codec_registry: dict[str, type[Codec]] = {} +"""Codec classes by codec id.""" +entries: dict[str, EntryPoint] = {} +"""Not-yet-loaded codec classes from the `numcodecs.codecs` entry point group, by codec id.""" -def run_entrypoints(): +def run_entrypoints() -> None: entries.clear() eps = entry_points() entries.update({e.name: e for e in eps.select(group="numcodecs.codecs")}) @@ -21,7 +26,7 @@ def run_entrypoints(): run_entrypoints() -def get_codec(config): +def get_codec(config: Mapping[str, Any]) -> Codec: """Obtain a codec for the given configuration. Parameters @@ -44,17 +49,25 @@ def get_codec(config): """ config = dict(config) codec_id = config.pop('id', None) - cls = codec_registry.get(codec_id) + cls = codec_registry.get(codec_id) if isinstance(codec_id, str) else None if cls is None and codec_id in entries: logger.debug("Auto loading codec '%s' from entrypoint", codec_id) cls = entries[codec_id].load() + if not (isinstance(cls, type) and issubclass(cls, Codec)): + warnings.warn( + f"The entry point for codec {codec_id!r} loads {cls!r}, which is not a " + "subclass of numcodecs.abc.Codec. Support for such codecs is deprecated " + "and will be removed in a future release.", + DeprecationWarning, + stacklevel=2, + ) register_codec(cls, codec_id=codec_id) if cls: return cls.from_config(config) raise UnknownCodecError(f"{codec_id!r}") -def register_codec(cls, codec_id=None): +def register_codec(cls: type[Codec], codec_id: str | None = None) -> None: """Register a codec class. Parameters @@ -70,5 +83,7 @@ def register_codec(cls, codec_id=None): """ if codec_id is None: codec_id = cls.codec_id + if codec_id is None: + raise ValueError(f"cannot register {cls!r}: it has no codec_id and none was given") logger.debug("Registering codec '%s'", codec_id) codec_registry[codec_id] = cls diff --git a/tests/test_registry.py b/tests/test_registry.py index 93ed8f032..2af73f058 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -1,4 +1,5 @@ import inspect +import types import pytest @@ -41,3 +42,36 @@ def test_all_classes_registered(): if missing: raise Exception(f"these codecs are missing: {missing}") # pragma: no cover + + +class _DuckCodec: + """Looks like a codec, but does not subclass numcodecs.abc.Codec.""" + + codec_id = "duck" + + @classmethod + def from_config(cls, config): + return cls() + + +def test_entry_point_not_a_codec_subclass_is_deprecated(monkeypatch): + monkeypatch.setitem( + numcodecs.registry.entries, "duck", types.SimpleNamespace(load=lambda: _DuckCodec) + ) + monkeypatch.delitem(numcodecs.registry.codec_registry, "duck", raising=False) + with pytest.warns(DeprecationWarning, match="not a subclass of numcodecs.abc.Codec"): + codec = get_codec({"id": "duck"}) + assert isinstance(codec, _DuckCodec) + numcodecs.registry.codec_registry.pop("duck") + + +def test_register_codec_without_id(): + class NoId(numcodecs.abc.Codec): + def encode(self, buf): # pragma: no cover + return buf + + def decode(self, buf, out=None): # pragma: no cover + return buf + + with pytest.raises(ValueError, match="has no codec_id and none was given"): + numcodecs.register_codec(NoId) diff --git a/tests/typing/mypy.ini b/tests/typing/mypy.ini index 26579ad7e..0159ab2f3 100644 --- a/tests/typing/mypy.ini +++ b/tests/typing/mypy.ini @@ -2,6 +2,8 @@ # Run from the repository root: `mypy --config-file tests/typing/mypy.ini tests/typing` [mypy] strict = true +# the oldest supported Python, so names that only exist in newer `typing` modules are caught +python_version = 3.12 warn_unreachable = true enable_error_code = ignore-without-code, redundant-expr, truthy-bool # Read numcodecs from the source tree. It does not ship a py.typed marker yet, so mypy diff --git a/tests/typing/test_compat_types.py b/tests/typing/test_compat_types.py index f656c4111..e8a5bac15 100644 --- a/tests/typing/test_compat_types.py +++ b/tests/typing/test_compat_types.py @@ -10,7 +10,24 @@ import numpy as np import numpy.typing as npt -from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray, ensure_ndarray +from numcodecs.compat import ( + ensure_bytes, + ensure_contiguous_ndarray, + ensure_ndarray, + ensure_ndarray_like, + ensure_text, + ndarray_copy, +) +from numcodecs.ndarray_like import NDArrayLike, is_ndarray_like + + +def _as_ndarray_like(x: NDArrayLike) -> NDArrayLike: + """Forget the concrete type, as for an ndarray-like that is not a numpy array.""" + return x + + +def _as_object(x: object) -> object: + return x def test_ensure_bytes() -> None: @@ -29,3 +46,42 @@ def test_ensure_contiguous_ndarray() -> None: for buf in (b"abc", bytearray(b"abc"), memoryview(b"abc"), np.arange(3, dtype="u1")): result = assert_type(ensure_contiguous_ndarray(buf), npt.NDArray[Any]) assert isinstance(result, np.ndarray) + + +def test_ensure_ndarray_like() -> None: + from_bytes = assert_type(ensure_ndarray_like(b"abc"), npt.NDArray[Any]) + assert isinstance(from_bytes, np.ndarray) + # an ndarray-like that is not known to export the buffer protocol (such as a cupy array) + # comes back as the protocol type + like = _as_ndarray_like(np.arange(3)) + from_like = assert_type(ensure_ndarray_like(like), NDArrayLike) + assert from_like is like + + +def test_ensure_text() -> None: + for s in ("abc", b"abc", bytearray(b"abc"), memoryview(b"abc")): + result = assert_type(ensure_text(s), str) + assert result == "abc" + + +def test_ndarray_copy() -> None: + src = b"\x01\x02\x03" + # without a destination, the source comes back unchanged + same = assert_type(ndarray_copy(src, None), bytes) + assert same is src + # with a destination, that very object comes back, filled + dst = bytearray(3) + filled = assert_type(ndarray_copy(src, dst), bytearray) + assert filled is dst + assert bytes(dst) == src + arr: npt.NDArray[np.uint8] = np.zeros(3, dtype=np.uint8) + assert assert_type(ndarray_copy(src, arr), npt.NDArray[np.uint8]) is arr + + +def test_is_ndarray_like_narrows() -> None: + obj = _as_object(np.arange(3)) + if is_ndarray_like(obj): + assert_type(obj, NDArrayLike) + assert obj.shape == (3,) + else: # pragma: no cover + raise AssertionError diff --git a/tests/typing/test_registry_types.py b/tests/typing/test_registry_types.py new file mode 100644 index 000000000..47a4b9a73 --- /dev/null +++ b/tests/typing/test_registry_types.py @@ -0,0 +1,26 @@ +"""Static types of `numcodecs.registry`, checked against runtime behavior.""" + +from importlib.metadata import EntryPoint +from typing import assert_type + +import numcodecs +from numcodecs.abc import Codec +from numcodecs.registry import codec_registry, entries, get_codec + + +def test_get_codec() -> None: + codec = assert_type(get_codec({"id": "zlib", "level": 1}), Codec) + assert isinstance(codec, numcodecs.Zlib) + + +def test_codec_registry_holds_classes() -> None: + assert_type(codec_registry, dict[str, type[Codec]]) + for cls in codec_registry.values(): + assert isinstance(cls, type) + assert issubclass(cls, Codec) + + +def test_entries_hold_entry_points() -> None: + assert_type(entries, dict[str, EntryPoint]) + for entry in entries.values(): + assert isinstance(entry, EntryPoint) From c26d438a50db75ac89f22184b5bbb81aff88099a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 21 Nov 2026 13:57:13 +0100 Subject: [PATCH 4/8] feat(abc): annotate the Codec base class and add structural codec protocols Codec.encode and Codec.decode were unannotated with docstring-only bodies, so mypy saw them as Any and pyright inferred that both return None, which made every typed subclass an "incompatible override" under pyright. - encode/decode take and return BufferLike (collections.abc.Buffer | NDArrayLike), with `buf` positional-only so that subclasses may rename it. - decode is overloaded so that decode(buf, out=out) returns the type of `out`, which every codec now honors. - codec_id is declared `str` without a default instead of `str | None = None`. It is not a ClassVar: with ClassVar, Codec instances do not match protocols that declare codec_id as a plain attribute (zarr-python's Numcodec) or as a property, under mypy or pyright. - get_config returns Mapping[str, Any], so that concrete codecs can later return per-codec TypedDicts, and from_config returns Self. numcodecs.abc also exports CodecProtocol and the three Zarr v3 codec kinds (BytesBytesCodec, ArrayArrayCodec, ArrayBytesCodec) as structural protocols, so consumers can type code that accepts any codec-like object. Assisted-by: ClaudeCode:claude-opus-5-5 --- docs/abc.md | 12 +++++ docs/release.md | 14 ++++++ src/numcodecs/abc.py | 87 +++++++++++++++++++++++++++----- src/numcodecs/registry.py | 2 +- tests/test_codec_contract.py | 7 +-- tests/test_registry.py | 2 +- tests/typing/test_abc_types.py | 92 ++++++++++++++++++++++++++++++++++ 7 files changed, 198 insertions(+), 18 deletions(-) create mode 100644 tests/typing/test_abc_types.py diff --git a/docs/abc.md b/docs/abc.md index f080766f2..925757c25 100644 --- a/docs/abc.md +++ b/docs/abc.md @@ -10,4 +10,16 @@ .. automethod:: decode .. automethod:: get_config .. automethod:: from_config + +Protocols +--------- + +These describe codecs structurally, for code that accepts any object implementing the +interface whether or not it subclasses :class:`Codec`. The three kinds mirror the codec +kinds of the Zarr version 3 specification. + +.. autoclass:: CodecProtocol +.. autoclass:: BytesBytesCodec +.. autoclass:: ArrayArrayCodec +.. autoclass:: ArrayBytesCodec ``` diff --git a/docs/release.md b/docs/release.md index bd896673b..11d493759 100644 --- a/docs/release.md +++ b/docs/release.md @@ -58,6 +58,20 @@ public. `from numcodecs import *` now imports only those names. * `register_codec` raises `ValueError` for a class without a `codec_id` when no id is given, instead of registering it under `None`. +* `Codec.codec_id` no longer defaults to `None`. Abstract intermediate classes such as + `Checksum32` leave it unset, so reading it raises `AttributeError`; every concrete codec + sets it. + +### Enhancements + +* Annotate `numcodecs.abc.Codec`. `encode` and `decode` take and return + `numcodecs.ndarray_like.BufferLike` (an object exporting the buffer protocol, or an + ndarray-like), with `buf` positional-only, and `decode(buf, out=out)` is typed to return + the type of `out`. `get_config` returns `Mapping[str, Any]` and `from_config` returns + `Self`. Previously pyright inferred that `encode` and `decode` return `None`. +* Add structural protocols `numcodecs.abc.CodecProtocol`, `BytesBytesCodec`, + `ArrayArrayCodec` and `ArrayBytesCodec` for code that accepts any object implementing the + codec interface. ### Deprecations diff --git a/src/numcodecs/abc.py b/src/numcodecs/abc.py index 6865df09e..fd9555690 100644 --- a/src/numcodecs/abc.py +++ b/src/numcodecs/abc.py @@ -29,17 +29,31 @@ """ from abc import ABC, abstractmethod +from collections.abc import Buffer, Mapping +from typing import Any, Protocol, Self, overload + +from .ndarray_like import BufferLike, NDArrayLike + +__all__ = [ + "ArrayArrayCodec", + "ArrayBytesCodec", + "BytesBytesCodec", + "Codec", + "CodecProtocol", +] class Codec(ABC): """Codec abstract base class.""" - # override in sub-class - codec_id: str | None = None - """Codec identifier.""" + # Declared without a value (abstract intermediate classes leave it unset) and without + # `ClassVar`, so that `Codec` instances match protocols that declare `codec_id` either + # as a plain attribute or as a read-only property. + codec_id: str + """Codec identifier. Every concrete codec class sets it as a class attribute.""" @abstractmethod - def encode(self, buf): # pragma: no cover + def encode(self, buf: BufferLike, /) -> BufferLike: # pragma: no cover """Encode data in `buf`. Parameters @@ -55,8 +69,14 @@ def encode(self, buf): # pragma: no cover protocol. """ + @overload + def decode(self, buf: BufferLike, /, out: None = None) -> BufferLike: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, /, out: OutT) -> OutT: ... @abstractmethod - def decode(self, buf, out=None): # pragma: no cover + def decode( + self, buf: BufferLike, /, out: BufferLike | None = None + ) -> BufferLike: # pragma: no cover """Decode data in `buf`. Parameters @@ -72,10 +92,10 @@ def decode(self, buf, out=None): # pragma: no cover ------- dec : buffer-like Decoded data. May be any object supporting the new-style - buffer protocol. + buffer protocol. If `out` was provided, `out` itself. """ - def get_config(self): + def get_config(self) -> Mapping[str, Any]: """Return a dictionary holding configuration parameters for this codec. Must include an 'id' field with the codec identifier. All values must be compatible with JSON encoding.""" @@ -83,7 +103,7 @@ def get_config(self): # override in sub-class if need special encoding of config values # setup config object - config = {'id': self.codec_id} + config: dict[str, Any] = {'id': self.codec_id} # by default, assume all non-private members are configuration # parameters - override this in sub-class if not the case @@ -94,8 +114,8 @@ def get_config(self): return config @classmethod - def from_config(cls, config): - """Instantiate codec from a configuration object.""" + def from_config(cls, config: Mapping[str, Any], /) -> Self: + """Instantiate codec from a configuration object, without its 'id' field.""" # N.B., assume at this point the 'id' field has been removed from # the config object @@ -105,14 +125,14 @@ def from_config(cls, config): # keyword arguments without any special decoding return cls(**config) - def __eq__(self, other): + def __eq__(self, other: object) -> bool: # override in sub-class if need special equality comparison try: - return self.get_config() == other.get_config() + return bool(self.get_config() == other.get_config()) # type: ignore[attr-defined] except AttributeError: return False - def __repr__(self): + def __repr__(self) -> str: # override in sub-class if need special representation # by default, assume all non-private members are configuration @@ -124,3 +144,44 @@ def __repr__(self): ] r += ', '.join(params) + ')' return r + + +class CodecProtocol(Protocol): + """The structural interface of a codec, for code that accepts any object implementing + it, whether or not it subclasses `Codec`.""" + + @property + def codec_id(self) -> str: ... # pragma: no cover + + def encode(self, buf: BufferLike, /) -> BufferLike: ... # pragma: no cover + + def decode(self, buf: BufferLike, /, out: Any = None) -> BufferLike: ... # pragma: no cover + + def get_config(self) -> Mapping[str, Any]: ... # pragma: no cover + + @classmethod + def from_config(cls, config: Mapping[str, Any], /) -> Self: ... # pragma: no cover + + +class BytesBytesCodec(CodecProtocol, Protocol): + """A codec that maps bytes to bytes, such as a compressor or a checksum.""" + + def encode(self, buf: BufferLike, /) -> Buffer: ... # pragma: no cover + + def decode(self, buf: BufferLike, /, out: Any = None) -> Buffer: ... # pragma: no cover + + +class ArrayArrayCodec(CodecProtocol, Protocol): + """A codec that maps an array to an array, such as a filter.""" + + def encode(self, buf: BufferLike, /) -> NDArrayLike: ... # pragma: no cover + + def decode(self, buf: BufferLike, /, out: Any = None) -> NDArrayLike: ... # pragma: no cover + + +class ArrayBytesCodec(CodecProtocol, Protocol): + """A codec that serializes an array to bytes and back.""" + + def encode(self, buf: BufferLike, /) -> Buffer: ... # pragma: no cover + + def decode(self, buf: BufferLike, /, out: Any = None) -> NDArrayLike: ... # pragma: no cover diff --git a/src/numcodecs/registry.py b/src/numcodecs/registry.py index 758cc0082..fa2a5f592 100644 --- a/src/numcodecs/registry.py +++ b/src/numcodecs/registry.py @@ -82,7 +82,7 @@ def register_codec(cls: type[Codec], codec_id: str | None = None) -> None: """ if codec_id is None: - codec_id = cls.codec_id + codec_id = getattr(cls, "codec_id", None) if codec_id is None: raise ValueError(f"cannot register {cls!r}: it has no codec_id and none was given") logger.debug("Registering codec '%s'", codec_id) diff --git a/tests/test_codec_contract.py b/tests/test_codec_contract.py index bf47cc007..5d871c002 100644 --- a/tests/test_codec_contract.py +++ b/tests/test_codec_contract.py @@ -21,6 +21,7 @@ import numcodecs from numcodecs import pickles from numcodecs.abc import Codec +from numcodecs.compat import ensure_ndarray @dataclass(frozen=True) @@ -147,9 +148,9 @@ def test_decode_into_ndarray_returns_out(codec_id: str) -> None: out = np.empty_like(data) assert codec.decode(enc, out=out) is out if CASES[codec_id].is_object: - assert all(np.array_equal(a, b) for a, b in zip(out, expected, strict=True)) + assert all(np.array_equal(a, b) for a, b in zip(out, ensure_ndarray(expected), strict=True)) else: - assert np.asarray(memoryview(expected)).tobytes() == out.tobytes() + assert ensure_ndarray(expected).tobytes() == out.tobytes() @pytest.mark.parametrize( @@ -161,7 +162,7 @@ def test_decode_into_bytearray_returns_out(codec_id: str) -> None: expected = codec.decode(enc) out = bytearray(data.nbytes) assert codec.decode(enc, out=out) is out - assert bytes(out) == np.asarray(memoryview(expected)).tobytes() + assert bytes(out) == ensure_ndarray(expected).tobytes() @pytest.mark.parametrize("codec_id", _params(set())) diff --git a/tests/test_registry.py b/tests/test_registry.py index 2af73f058..ea44ea109 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -35,8 +35,8 @@ def test_all_classes_registered(): if ( inspect.isclass(obj) and issubclass(obj, numcodecs.abc.Codec) + and not inspect.isabstract(obj) and obj.codec_id not in numcodecs.registry.codec_registry - and obj.codec_id is not None # remove `None` ) } diff --git a/tests/typing/test_abc_types.py b/tests/typing/test_abc_types.py new file mode 100644 index 000000000..ababbebb4 --- /dev/null +++ b/tests/typing/test_abc_types.py @@ -0,0 +1,92 @@ +"""Static types of `numcodecs.abc`, checked against runtime behavior.""" + +from collections.abc import Mapping +from typing import Any, Self, assert_type, overload + +import numpy as np +import numpy.typing as npt + +import numcodecs +from numcodecs.abc import Codec, CodecProtocol +from numcodecs.compat import ensure_bytes, ndarray_copy +from numcodecs.ndarray_like import BufferLike + + +def _zlib() -> Codec: + return numcodecs.get_codec({"id": "zlib", "level": 1}) + + +def test_codec_methods() -> None: + codec = _zlib() + data = np.arange(16, dtype="f8") + + assert_type(codec.codec_id, str) + assert codec.codec_id == "zlib" + + enc = assert_type(codec.encode(data), BufferLike) + dec = assert_type(codec.decode(enc), BufferLike) + assert ensure_bytes(dec) == data.tobytes() + + out: npt.NDArray[np.float64] = np.empty_like(data) + assert assert_type(codec.decode(enc, out=out), npt.NDArray[np.float64]) is out + buf = bytearray(data.nbytes) + assert assert_type(codec.decode(enc, out=buf), bytearray) is buf + + config = assert_type(codec.get_config(), Mapping[str, Any]) + assert config["id"] == "zlib" + rebuilt = assert_type(type(codec).from_config({"level": 1}), Codec) + assert rebuilt == codec + + assert_type(codec == object(), bool) + + +def test_codec_satisfies_codec_protocol() -> None: + codec: CodecProtocol = _zlib() + assert codec.codec_id == "zlib" + + +class _Identity(Codec): + """A third-party codec typed as precisely as numcodecs' own.""" + + codec_id = "test-identity" + + def encode(self, buf: BufferLike, /) -> bytes: + return ensure_bytes(buf) + + @overload + def decode(self, buf: BufferLike, /, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, /, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, /, out: BufferLike | None = None) -> BufferLike: + return ndarray_copy(ensure_bytes(buf), out) + + +class _AnyTyped(Codec): + """A third-party codec annotated with `Any`, as imagecodecs does.""" + + codec_id = "test-any" + + def encode(self, buf: Any) -> Any: + return buf + + def decode(self, buf: Any, out: Any = None) -> Any: + return buf + + def get_config(self) -> dict[str, Any]: + return {"id": self.codec_id} + + @classmethod + def from_config(cls, config: Mapping[str, Any]) -> Self: + return cls() + + +def test_subclasses() -> None: + identity = _Identity() + assert assert_type(identity.encode(b"abc"), bytes) == b"abc" + assert assert_type(identity.decode(b"abc"), bytes) == b"abc" + out = bytearray(3) + assert assert_type(identity.decode(b"abc", out=out), bytearray) is out + + any_typed = _AnyTyped() + assert any_typed.decode(any_typed.encode(b"abc")) == b"abc" + assert_type(_AnyTyped.from_config({}), _AnyTyped) From 32207053508624a7e6674c4ca0bd0d569fd94934 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 21 Nov 2026 14:02:22 +0100 Subject: [PATCH 5/8] feat(typing): ship .pyi stubs for the Cython extension modules blosc, zstd, lz4, vlen, fletcher32, jenkins, compat_ext and _shuffle had no stubs, so type checkers saw Blosc, Zstd, LZ4, Fletcher32 and the VLen codecs as unknown, and numcodecs' own mypy run silently treated them as Any. Each module now has a stub next to its .pyx, installed by meson. The stubs follow probed behavior: the compressors return bytes, or `out` itself when it is given; Fletcher32.decode returns a uint8 ndarray; the VLen codecs encode to bytearray and decode to object arrays. VLenArray.encode takes an object array only, since numpy turns a list of equal-length arrays into a 2-D array. The typing CI job runs mypy.stubtest against the installed extensions, with an allowlist for Cython artifacts (__test__, __pyx_capi__, and the `(self, /)` signature Cython reports for argument-less module functions). Checksum32.checksum is declared positional-only, so that JenkinsLookup3 can use jenkins_lookup3 (whose parameters are named differently) as its checksum; it is wrapped in staticmethod so type checkers do not bind it as a method. It was never bound at runtime, being a builtin function. Assisted-by: ClaudeCode:claude-opus-5-5 --- .github/workflows/ci.yaml | 11 +++++ docs/release.md | 4 ++ src/numcodecs/_shuffle.pyi | 4 ++ src/numcodecs/blosc.pyi | 68 +++++++++++++++++++++++++++++ src/numcodecs/checksum32.py | 4 +- src/numcodecs/compat_ext.pyi | 3 ++ src/numcodecs/fletcher32.pyi | 15 +++++++ src/numcodecs/jenkins.pyi | 3 ++ src/numcodecs/lz4.pyi | 24 ++++++++++ src/numcodecs/meson.build | 9 ++++ src/numcodecs/vlen.pyi | 41 +++++++++++++++++ src/numcodecs/zstd.pyi | 35 +++++++++++++++ tests/typing/README.md | 11 +++++ tests/typing/pyrightconfig.json | 3 +- tests/typing/stubtest-allowlist.txt | 10 +++++ tests/typing/test_cython_types.py | 59 +++++++++++++++++++++++++ 16 files changed, 301 insertions(+), 3 deletions(-) create mode 100644 src/numcodecs/_shuffle.pyi create mode 100644 src/numcodecs/blosc.pyi create mode 100644 src/numcodecs/compat_ext.pyi create mode 100644 src/numcodecs/fletcher32.pyi create mode 100644 src/numcodecs/jenkins.pyi create mode 100644 src/numcodecs/lz4.pyi create mode 100644 src/numcodecs/vlen.pyi create mode 100644 src/numcodecs/zstd.pyi create mode 100644 tests/typing/stubtest-allowlist.txt create mode 100644 tests/typing/test_cython_types.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bec8fb69b..c31ea29e6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -162,6 +162,17 @@ jobs: - name: pyright (strict type tests) run: pyright -p tests/typing + - name: stubtest (Cython stubs against the built extensions) + # Read the stubs from src/: mypy ignores the installed copies until numcodecs + # ships a py.typed marker. + env: + MYPYPATH: src + run: | + python -m mypy.stubtest --mypy-config-file pyproject.toml \ + --allowlist tests/typing/stubtest-allowlist.txt \ + numcodecs.blosc numcodecs.zstd numcodecs.lz4 numcodecs.vlen \ + numcodecs.fletcher32 numcodecs.jenkins numcodecs.compat_ext numcodecs._shuffle + test-zarr: runs-on: ubuntu-latest strategy: diff --git a/docs/release.md b/docs/release.md index 11d493759..abeff1b4e 100644 --- a/docs/release.md +++ b/docs/release.md @@ -72,6 +72,10 @@ * Add structural protocols `numcodecs.abc.CodecProtocol`, `BytesBytesCodec`, `ArrayArrayCodec` and `ArrayBytesCodec` for code that accepts any object implementing the codec interface. +* Ship `.pyi` type stubs for the Cython extension modules (`blosc`, `zstd`, `lz4`, `vlen`, + `fletcher32`, `jenkins`, `compat_ext` and `_shuffle`), checked against the compiled + modules with `mypy.stubtest` in CI. Type checkers previously saw the Blosc, Zstd, LZ4, + Fletcher32 and VLen codecs as unknown. ### Deprecations diff --git a/src/numcodecs/_shuffle.pyi b/src/numcodecs/_shuffle.pyi new file mode 100644 index 000000000..a80418483 --- /dev/null +++ b/src/numcodecs/_shuffle.pyi @@ -0,0 +1,4 @@ +from collections.abc import Buffer + +def _doShuffle(src: Buffer, des: Buffer, element_size: int) -> None: ... +def _doUnshuffle(src: Buffer, des: Buffer, element_size: int) -> None: ... diff --git a/src/numcodecs/blosc.pyi b/src/numcodecs/blosc.pyi new file mode 100644 index 000000000..ca11c2452 --- /dev/null +++ b/src/numcodecs/blosc.pyi @@ -0,0 +1,68 @@ +from typing import Final, overload + +from .abc import Codec +from .ndarray_like import BufferLike + +NOSHUFFLE: Final[int] +SHUFFLE: Final[int] +BITSHUFFLE: Final[int] +AUTOSHUFFLE: Final[int] +AUTOBLOCKS: Final[int] +MAX_OVERHEAD: Final[int] +MAX_BUFFERSIZE: Final[int] +MAX_THREADS: Final[int] +MAX_TYPESIZE: Final[int] +VERSION_STRING: Final[str] +VERSION_DATE: Final[str] +__version__: Final[str] + +use_threads: bool | None +"""Set to `True` or `False` to override whether Blosc may use threads; `None` (the +default) decides from the calling thread and process.""" + +def _init() -> None: ... +def _destroy() -> None: ... +def _get_use_threads() -> bool: ... +def _cbuffer_sizes(source: BufferLike) -> tuple[int, int, int]: ... +def _cbuffer_metainfo(source: BufferLike) -> tuple[int, bool, bool]: ... +def list_compressors() -> list[str]: ... +def get_nthreads() -> int: ... +def set_nthreads(nthreads: int) -> int: ... +def cbuffer_complib(source: BufferLike) -> str: ... +def compress( + source: BufferLike, + cname: bytes, + clevel: int, + shuffle: int = ..., + blocksize: int = ..., + typesize: int | None = None, +) -> bytes: ... +@overload +def decompress(source: BufferLike, dest: None = None) -> bytes: ... +@overload +def decompress[OutT: BufferLike](source: BufferLike, dest: OutT) -> OutT: ... + +class Blosc(Codec): + codec_id: str + NOSHUFFLE: int + SHUFFLE: int + BITSHUFFLE: int + AUTOSHUFFLE: int + max_buffer_size: int + cname: str + clevel: int + shuffle: int + blocksize: int + def __init__( + self, + cname: str = "lz4", + clevel: int = 5, + shuffle: int = ..., + blocksize: int = ..., + typesize: int | None = None, + ) -> None: ... + def encode(self, buf: BufferLike) -> bytes: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... diff --git a/src/numcodecs/checksum32.py b/src/numcodecs/checksum32.py index a095b1718..70094ccc7 100644 --- a/src/numcodecs/checksum32.py +++ b/src/numcodecs/checksum32.py @@ -89,7 +89,7 @@ def decode(self, buf, out=None): @staticmethod @abc.abstractmethod - def checksum(data: Buffer, value: int) -> int: ... + def checksum(data: Buffer, value: int, /) -> int: ... class CRC32(Checksum32): @@ -151,7 +151,7 @@ class JenkinsLookup3(Checksum32): bytes prepended to the buffer before evaluating the hash, default: None """ - checksum = jenkins_lookup3 + checksum = staticmethod(jenkins_lookup3) codec_id = "jenkins_lookup3" def __init__(self, initval: int = 0, prefix=None): diff --git a/src/numcodecs/compat_ext.pyi b/src/numcodecs/compat_ext.pyi new file mode 100644 index 000000000..f5388bfb9 --- /dev/null +++ b/src/numcodecs/compat_ext.pyi @@ -0,0 +1,3 @@ +from collections.abc import Buffer + +def ensure_contiguous_memoryview(obj: Buffer) -> memoryview: ... diff --git a/src/numcodecs/fletcher32.pyi b/src/numcodecs/fletcher32.pyi new file mode 100644 index 000000000..0d9e70092 --- /dev/null +++ b/src/numcodecs/fletcher32.pyi @@ -0,0 +1,15 @@ +from typing import overload + +import numpy as np +import numpy.typing as npt + +from .abc import Codec +from .ndarray_like import BufferLike + +class Fletcher32(Codec): + codec_id: str + def encode(self, buf: BufferLike) -> bytes: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[np.uint8]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... diff --git a/src/numcodecs/jenkins.pyi b/src/numcodecs/jenkins.pyi new file mode 100644 index 000000000..c8673b7f7 --- /dev/null +++ b/src/numcodecs/jenkins.pyi @@ -0,0 +1,3 @@ +from collections.abc import Buffer + +def jenkins_lookup3(_data: Buffer, initval: int = 0) -> int: ... diff --git a/src/numcodecs/lz4.pyi b/src/numcodecs/lz4.pyi new file mode 100644 index 000000000..b35f09726 --- /dev/null +++ b/src/numcodecs/lz4.pyi @@ -0,0 +1,24 @@ +from typing import Final, overload + +from .abc import Codec +from .ndarray_like import BufferLike + +VERSION_STRING: Final[str] +DEFAULT_ACCELERATION: Final[int] + +def compress(source: BufferLike, acceleration: int = ...) -> bytes: ... +@overload +def decompress(source: BufferLike, dest: None = None) -> bytes: ... +@overload +def decompress[OutT: BufferLike](source: BufferLike, dest: OutT) -> OutT: ... + +class LZ4(Codec): + codec_id: str + max_buffer_size: int + acceleration: int + def __init__(self, acceleration: int = ...) -> None: ... + def encode(self, buf: BufferLike) -> bytes: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... diff --git a/src/numcodecs/meson.build b/src/numcodecs/meson.build index 41a3a7a2f..019b81dba 100644 --- a/src/numcodecs/meson.build +++ b/src/numcodecs/meson.build @@ -286,5 +286,14 @@ py.install_sources( 'zarr3.py', 'zfpy.py', 'zlib.py', + # type stubs for the Cython extension modules + '_shuffle.pyi', + 'blosc.pyi', + 'compat_ext.pyi', + 'fletcher32.pyi', + 'jenkins.pyi', + 'lz4.pyi', + 'vlen.pyi', + 'zstd.pyi', subdir: 'numcodecs', ) diff --git a/src/numcodecs/vlen.pyi b/src/numcodecs/vlen.pyi new file mode 100644 index 000000000..a4443000f --- /dev/null +++ b/src/numcodecs/vlen.pyi @@ -0,0 +1,41 @@ +from collections.abc import Iterable +from typing import Any, overload + +import numpy as np +import numpy.typing as npt + +from .abc import Codec +from .ndarray_like import BufferLike + +def check_out_param(out: npt.NDArray[np.object_], n_items: int) -> npt.NDArray[np.object_]: ... + +# VLenUTF8 and VLenBytes also accept a list (or other iterable) of str or bytes. VLenArray +# needs an object array: numpy would turn a list of equal-length arrays into a 2-D array. + +class VLenUTF8(Codec): + codec_id: str + def __init__(self) -> None: ... + def encode(self, buf: BufferLike | Iterable[str]) -> bytearray: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[np.object_]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + +class VLenBytes(Codec): + codec_id: str + def __init__(self) -> None: ... + def encode(self, buf: BufferLike | Iterable[bytes]) -> bytearray: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[np.object_]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + +class VLenArray(Codec): + codec_id: str + dtype: np.dtype[Any] + def __init__(self, dtype: npt.DTypeLike) -> None: ... + def encode(self, buf: BufferLike) -> bytearray: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> npt.NDArray[np.object_]: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... diff --git a/src/numcodecs/zstd.pyi b/src/numcodecs/zstd.pyi new file mode 100644 index 000000000..617e31f2c --- /dev/null +++ b/src/numcodecs/zstd.pyi @@ -0,0 +1,35 @@ +from typing import Final, overload + +from .abc import Codec +from .ndarray_like import BufferLike + +VERSION_NUMBER: Final[int] +MAJOR_VERSION_NUMBER: Final[int] +MINOR_VERSION_NUMBER: Final[int] +MICRO_VERSION_NUMBER: Final[int] +DEFAULT_CLEVEL: Final[int] +MAX_CLEVEL: Final[int] + +def compress(source: BufferLike, level: int = ..., checksum: bool = False) -> bytes: ... +@overload +def decompress(source: BufferLike, dest: None = None) -> bytes: ... +@overload +def decompress[OutT: BufferLike](source: BufferLike, dest: OutT) -> OutT: ... + +class Zstd(Codec): + codec_id: str + max_buffer_size: int + level: int + checksum: bool + def __init__(self, level: int = ..., checksum: bool = False) -> None: ... + def encode(self, buf: BufferLike) -> bytes: ... + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + @classmethod + def default_level(cls) -> int: ... + @classmethod + def min_level(cls) -> int: ... + @classmethod + def max_level(cls) -> int: ... diff --git a/tests/typing/README.md b/tests/typing/README.md index a885a0967..c9f32fcfe 100644 --- a/tests/typing/README.md +++ b/tests/typing/README.md @@ -21,5 +21,16 @@ configuration in `pyproject.toml` covers those. `pyrightconfig.json` turns off pyright's three `reportUnknown*` rules, because most numcodecs parameters are not annotated yet. Turn them back on once they are. +The Cython extension modules carry their types in `.pyi` stubs next to the `.pyx` sources. +`mypy.stubtest` compares those stubs with the compiled modules, against an installed +(non-editable) build: + +```bash +MYPYPATH=src python -m mypy.stubtest --mypy-config-file pyproject.toml \ + --allowlist tests/typing/stubtest-allowlist.txt \ + numcodecs.blosc numcodecs.zstd numcodecs.lz4 numcodecs.vlen \ + numcodecs.fletcher32 numcodecs.jenkins numcodecs.compat_ext numcodecs._shuffle +``` + When you change an annotation in numcodecs, add or update an `assert_type` here that shows what callers now get. diff --git a/tests/typing/pyrightconfig.json b/tests/typing/pyrightconfig.json index 42540aad6..fac5363b8 100644 --- a/tests/typing/pyrightconfig.json +++ b/tests/typing/pyrightconfig.json @@ -10,5 +10,6 @@ "reportMissingTypeStubs": "none", "reportUnknownVariableType": "none", "reportUnknownMemberType": "none", - "reportUnknownArgumentType": "none" + "reportUnknownArgumentType": "none", + "reportMissingModuleSource": "none" } diff --git a/tests/typing/stubtest-allowlist.txt b/tests/typing/stubtest-allowlist.txt new file mode 100644 index 000000000..394073671 --- /dev/null +++ b/tests/typing/stubtest-allowlist.txt @@ -0,0 +1,10 @@ +# Entries for `python -m mypy.stubtest`, one regular expression per line. + +# Cython adds a `__test__` dict (collected doctests) to every module. +numcodecs\..*\.__test__ + +# Cython reports `def f():` module functions as `(self, /)` at runtime. +numcodecs\.blosc\.(_init|_destroy|_get_use_threads|get_nthreads|list_compressors) + +# Cython's C-API capsule table for cimport-able functions. +numcodecs\.compat_ext\.__pyx_capi__ diff --git a/tests/typing/test_cython_types.py b/tests/typing/test_cython_types.py new file mode 100644 index 000000000..ae2f35729 --- /dev/null +++ b/tests/typing/test_cython_types.py @@ -0,0 +1,59 @@ +"""Static types of the Cython codecs, as declared in their `.pyi` stubs, checked against +runtime behavior.""" + +from typing import assert_type + +import numpy as np +import numpy.typing as npt + +from numcodecs.blosc import Blosc +from numcodecs.fletcher32 import Fletcher32 +from numcodecs.jenkins import jenkins_lookup3 +from numcodecs.lz4 import LZ4 +from numcodecs.vlen import VLenArray, VLenBytes, VLenUTF8 +from numcodecs.zstd import Zstd + +DATA = np.arange(16, dtype="f8") + + +def test_compressors() -> None: + for codec in (Blosc(), Zstd(), LZ4()): + enc = assert_type(codec.encode(DATA), bytes) + assert isinstance(enc, bytes) + dec = assert_type(codec.decode(enc), bytes) + assert dec == DATA.tobytes() + out = bytearray(DATA.nbytes) + assert assert_type(codec.decode(enc, out=out), bytearray) is out + + +def test_zstd_levels() -> None: + for level in (Zstd.default_level(), Zstd.min_level(), Zstd.max_level()): + assert isinstance(assert_type(level, int), int) + + +def test_fletcher32() -> None: + codec = Fletcher32() + enc = assert_type(codec.encode(DATA), bytes) + dec = assert_type(codec.decode(enc), npt.NDArray[np.uint8]) + assert isinstance(dec, np.ndarray) + assert dec.tobytes() == DATA.tobytes() + + +def test_jenkins_lookup3() -> None: + assert isinstance(assert_type(jenkins_lookup3(b"abc"), int), int) + + +def test_vlen() -> None: + utf8 = VLenUTF8() + enc = assert_type(utf8.encode(["foo", "bar"]), bytearray) + dec = assert_type(utf8.decode(enc), npt.NDArray[np.object_]) + assert dec.tolist() == ["foo", "bar"] + + raw = VLenBytes() + assert raw.decode(assert_type(raw.encode([b"foo"]), bytearray)).tolist() == [b"foo"] + + arrays = VLenArray(dtype="i4") + values = np.empty(2, dtype=object) + values[0], values[1] = np.arange(3, dtype="i4"), np.arange(2, dtype="i4") + enc = assert_type(arrays.encode(values), bytearray) + assert assert_type(arrays.decode(enc), npt.NDArray[np.object_])[0].tolist() == [0, 1, 2] From 2dcbbcd6543a3085ba1c7aef0de71594ea7fc175 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 21 Nov 2026 14:07:41 +0100 Subject: [PATCH 6/8] feat(typing): annotate the compressor codecs with per-codec config TypedDicts Zlib, GZip, BZ2, LZMA and Base64 annotate __init__, encode (-> bytes) and decode (overloaded: bytes without `out`, the type of `out` with it), and Blosc, Zstd and LZ4 get the same through their stubs. Each get_config returns a TypedDict of the codec's parameters; for codecs that inherit the base implementation it is a cast of the inherited dict, so subclasses that add attributes keep them in their configuration as before. numcodecs.abc.CodecConfig is the common base: `id: str`, with PEP 728 `extra_items=ReadOnly[JSON]`, which lets pyright treat any codec configuration as a Mapping[str, JSON] (and as Zarr v2 codec metadata). mypy does not implement PEP 728 yet and sees a plain TypedDict. JSON uses Sequence/Mapping, matching zarr_metadata's JSONValue. `extra_items` needs typing_extensions >= 4.13 at runtime. The Cython modules define their config TypedDicts with the functional syntax, which Cython compiles; the stubs declare the same types with the class syntax. Assisted-by: ClaudeCode:claude-opus-5-5 --- docs/release.md | 6 +++ pyproject.toml | 2 +- src/numcodecs/abc.py | 26 ++++++++- src/numcodecs/base64.py | 23 +++++--- src/numcodecs/blosc.pyi | 9 +++- src/numcodecs/blosc.pyx | 7 +++ src/numcodecs/bz2.py | 34 ++++++++---- src/numcodecs/gzip.py | 45 ++++++++++------ src/numcodecs/lz4.pyi | 6 ++- src/numcodecs/lz4.pyx | 7 +++ src/numcodecs/lzma.py | 45 ++++++++++++---- src/numcodecs/zlib.py | 32 +++++++---- src/numcodecs/zstd.pyi | 7 ++- src/numcodecs/zstd.pyx | 7 +++ tests/typing/pyrightconfig.json | 3 +- tests/typing/test_compressor_types.py | 78 +++++++++++++++++++++++++++ 16 files changed, 279 insertions(+), 58 deletions(-) create mode 100644 tests/typing/test_compressor_types.py diff --git a/docs/release.md b/docs/release.md index abeff1b4e..b64ad7b26 100644 --- a/docs/release.md +++ b/docs/release.md @@ -72,6 +72,12 @@ * Add structural protocols `numcodecs.abc.CodecProtocol`, `BytesBytesCodec`, `ArrayArrayCodec` and `ArrayBytesCodec` for code that accepts any object implementing the codec interface. +* Annotate the concrete codecs. Each `get_config` returns a TypedDict naming the codec's + parameters (`ZlibConfig`, `BloscConfig`, ...), all subclasses of + `numcodecs.abc.CodecConfig`, which carries the `id` and declares any further keys as + read-only JSON values (PEP 728 `extra_items`). `numcodecs.abc.JSON` names a + JSON-serializable value. Compressors: Zlib, GZip, BZ2, LZMA, Base64, Blosc, Zstd and LZ4. + `typing_extensions` >= 4.13 is now required. * Ship `.pyi` type stubs for the Cython extension modules (`blosc`, `zstd`, `lz4`, `vlen`, `fletcher32`, `jenkins`, `compat_ext` and `_shuffle`), checked against the compiled modules with `mypy.stubtest` in CI. Type checkers previously saw the Blosc, Zstd, LZ4, diff --git a/pyproject.toml b/pyproject.toml index 3050fdfc7..654e0ee9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ description = """ A Python package providing buffer compression and transformation codecs \ for use in data storage and communication applications.""" readme = "README.md" -dependencies = ["numpy>=2.0", "packaging", "typing_extensions"] +dependencies = ["numpy>=2.0", "packaging", "typing_extensions>=4.13"] requires-python = ">=3.12" dynamic = [ "version", diff --git a/src/numcodecs/abc.py b/src/numcodecs/abc.py index fd9555690..0612162de 100644 --- a/src/numcodecs/abc.py +++ b/src/numcodecs/abc.py @@ -29,19 +29,43 @@ """ from abc import ABC, abstractmethod -from collections.abc import Buffer, Mapping +from collections.abc import Buffer, Mapping, Sequence from typing import Any, Protocol, Self, overload +from typing_extensions import ReadOnly, TypedDict + from .ndarray_like import BufferLike, NDArrayLike __all__ = [ + "JSON", "ArrayArrayCodec", "ArrayBytesCodec", "BytesBytesCodec", "Codec", + "CodecConfig", "CodecProtocol", ] +type JSON = int | float | bool | str | Sequence[JSON] | Mapping[str, JSON] | None +"""A JSON-serializable value. `Sequence` and `Mapping` rather than `list` and `dict`, so +that tuples, TypedDicts and lists of narrower types count (the same definition as +`zarr_metadata`'s `JSONValue`).""" + + +# `extra_items` (PEP 728) says that any further keys hold JSON values and are read-only. It +# lets type checkers that implement PEP 728 (pyright) treat every codec configuration as a +# `Mapping[str, JSON]`, and as Zarr v2 codec metadata. mypy does not implement PEP 728 yet, +# so it rejects the keyword and treats `CodecConfig` as a plain TypedDict. +class CodecConfig(TypedDict, extra_items=ReadOnly[JSON]): # type: ignore[call-arg] + """The configuration every codec returns from `Codec.get_config`: its `id`, plus any + further JSON-serializable parameters. + + Concrete codecs return a subclass that declares their own parameters. This is the shape + of a Zarr version 2 compressor or filter. + """ + + id: str + class Codec(ABC): """Codec abstract base class.""" diff --git a/src/numcodecs/base64.py b/src/numcodecs/base64.py index c107d5c7b..2e53181db 100644 --- a/src/numcodecs/base64.py +++ b/src/numcodecs/base64.py @@ -1,7 +1,9 @@ import base64 as _base64 +from typing import cast, overload -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_contiguous_ndarray, ndarray_copy +from .ndarray_like import BufferLike class Base64(Codec): @@ -9,18 +11,25 @@ class Base64(Codec): codec_id = "base64" - def encode(self, buf): + def get_config(self) -> CodecConfig: + return cast(CodecConfig, super().get_config()) + + def encode(self, buf: BufferLike) -> bytes: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) # do compression - return _base64.standard_b64encode(buf) + return _base64.standard_b64encode(arr) - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) if out is not None: ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression - decompressed = _base64.standard_b64decode(buf) + decompressed = _base64.standard_b64decode(arr) # handle destination return ndarray_copy(decompressed, out) diff --git a/src/numcodecs/blosc.pyi b/src/numcodecs/blosc.pyi index ca11c2452..495be3ac2 100644 --- a/src/numcodecs/blosc.pyi +++ b/src/numcodecs/blosc.pyi @@ -1,6 +1,6 @@ from typing import Final, overload -from .abc import Codec +from .abc import Codec, CodecConfig from .ndarray_like import BufferLike NOSHUFFLE: Final[int] @@ -42,6 +42,12 @@ def decompress(source: BufferLike, dest: None = None) -> bytes: ... @overload def decompress[OutT: BufferLike](source: BufferLike, dest: OutT) -> OutT: ... +class BloscConfig(CodecConfig): + cname: str + clevel: int + shuffle: int + blocksize: int + class Blosc(Codec): codec_id: str NOSHUFFLE: int @@ -61,6 +67,7 @@ class Blosc(Codec): blocksize: int = ..., typesize: int | None = None, ) -> None: ... + def get_config(self) -> BloscConfig: ... def encode(self, buf: BufferLike) -> bytes: ... @overload def decode(self, buf: BufferLike, out: None = None) -> bytes: ... diff --git a/src/numcodecs/blosc.pyx b/src/numcodecs/blosc.pyx index 8c52dd95c..509b9b96d 100644 --- a/src/numcodecs/blosc.pyx +++ b/src/numcodecs/blosc.pyx @@ -14,10 +14,17 @@ from cpython.memoryview cimport PyMemoryView_GET_BUFFER from .compat_ext cimport PyBytes_RESIZE, ensure_contiguous_memoryview +from typing import TypedDict + from .compat import ensure_contiguous_ndarray from .abc import Codec +# The type of `Blosc.get_config()`. Declared with the functional syntax because Cython does not +# compile TypedDict class bodies; `blosc.pyi` declares the same type with the class syntax. +BloscConfig = TypedDict("BloscConfig", {"id": str, "cname": str, "clevel": int, "shuffle": int, "blocksize": int}) + + cdef extern from "blosc.h": cdef enum: BLOSC_MAX_OVERHEAD, diff --git a/src/numcodecs/bz2.py b/src/numcodecs/bz2.py index a3c8f5ec3..d4fde4928 100644 --- a/src/numcodecs/bz2.py +++ b/src/numcodecs/bz2.py @@ -1,7 +1,15 @@ import bz2 as _bz2 +from typing import cast, overload -from numcodecs.abc import Codec +from numcodecs.abc import Codec, CodecConfig from numcodecs.compat import ensure_contiguous_ndarray, ndarray_copy +from numcodecs.ndarray_like import BufferLike + + +class BZ2Config(CodecConfig): + """The configuration of a `BZ2` codec, as returned by `BZ2.get_config`.""" + + level: int class BZ2(Codec): @@ -16,28 +24,34 @@ class BZ2(Codec): codec_id = 'bz2' - def __init__(self, level=1): + def __init__(self, level: int = 1) -> None: self.level = level - def encode(self, buf): + def get_config(self) -> BZ2Config: + return cast(BZ2Config, super().get_config()) + + def encode(self, buf: BufferLike) -> bytes: # normalise input - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) # do compression - return _bz2.compress(buf, self.level) + return _bz2.compress(arr, self.level) - # noinspection PyMethodMayBeStatic - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) if out is not None: ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # N.B., bz2 cannot handle ndarray directly because of truth testing issues - buf = memoryview(buf) + mem = memoryview(arr) # do decompression - dec = _bz2.decompress(buf) + dec = _bz2.decompress(mem) # handle destination - Python standard library bz2 module does not # support direct decompression into buffer, so we have to copy into diff --git a/src/numcodecs/gzip.py b/src/numcodecs/gzip.py index 4269bad93..3387dbf14 100644 --- a/src/numcodecs/gzip.py +++ b/src/numcodecs/gzip.py @@ -1,8 +1,16 @@ import gzip as _gzip import io +from typing import cast, overload -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_bytes, ensure_contiguous_ndarray +from .ndarray_like import BufferLike + + +class GZipConfig(CodecConfig): + """The configuration of a `GZip` codec, as returned by `GZip.get_config`.""" + + level: int class GZip(Codec): @@ -17,34 +25,39 @@ class GZip(Codec): codec_id = 'gzip' - def __init__(self, level=1): + def __init__(self, level: int = 1) -> None: self.level = level - def encode(self, buf): + def get_config(self) -> GZipConfig: + return cast(GZipConfig, super().get_config()) + + def encode(self, buf: BufferLike) -> bytes: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) # do compression compressed = io.BytesIO() with _gzip.GzipFile(fileobj=compressed, mode='wb', compresslevel=self.level) as compressor: - compressor.write(buf) + compressor.write(arr) return compressed.getvalue() - # noinspection PyMethodMayBeStatic - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise inputs # BytesIO only copies if the data is not of `bytes` type. # This allows `bytes` objects to pass through without copying. - buf = io.BytesIO(ensure_bytes(buf)) + compressed = io.BytesIO(ensure_bytes(buf)) # do decompression - with _gzip.GzipFile(fileobj=buf, mode='rb') as decompressor: - if out is not None: - out_view = ensure_contiguous_ndarray(out) - decompressor.readinto(out_view) - if decompressor.read(1) != b'': - raise ValueError("Unable to fit data into `out`") - else: - out = decompressor.read() + with _gzip.GzipFile(fileobj=compressed, mode='rb') as decompressor: + if out is None: + return decompressor.read() + out_view = ensure_contiguous_ndarray(out) + decompressor.readinto(out_view) + if decompressor.read(1) != b'': + raise ValueError("Unable to fit data into `out`") return out diff --git a/src/numcodecs/lz4.pyi b/src/numcodecs/lz4.pyi index b35f09726..71d941f64 100644 --- a/src/numcodecs/lz4.pyi +++ b/src/numcodecs/lz4.pyi @@ -1,6 +1,6 @@ from typing import Final, overload -from .abc import Codec +from .abc import Codec, CodecConfig from .ndarray_like import BufferLike VERSION_STRING: Final[str] @@ -12,11 +12,15 @@ def decompress(source: BufferLike, dest: None = None) -> bytes: ... @overload def decompress[OutT: BufferLike](source: BufferLike, dest: OutT) -> OutT: ... +class LZ4Config(CodecConfig): + acceleration: int + class LZ4(Codec): codec_id: str max_buffer_size: int acceleration: int def __init__(self, acceleration: int = ...) -> None: ... + def get_config(self) -> LZ4Config: ... def encode(self, buf: BufferLike) -> bytes: ... @overload def decode(self, buf: BufferLike, out: None = None) -> bytes: ... diff --git a/src/numcodecs/lz4.pyx b/src/numcodecs/lz4.pyx index 6eac246b5..a0017673a 100644 --- a/src/numcodecs/lz4.pyx +++ b/src/numcodecs/lz4.pyx @@ -14,10 +14,17 @@ from cpython.memoryview cimport PyMemoryView_GET_BUFFER from .compat_ext cimport PyBytes_RESIZE, ensure_contiguous_memoryview from ._utils cimport store_le32, load_le32 +from typing import TypedDict + from .compat import ensure_contiguous_ndarray from .abc import Codec +# The type of `LZ4.get_config()`. Declared with the functional syntax because Cython does not +# compile TypedDict class bodies; `lz4.pyi` declares the same type with the class syntax. +LZ4Config = TypedDict("LZ4Config", {"id": str, "acceleration": int}) + + cdef extern from "lz4.h": const char* LZ4_versionString() nogil diff --git a/src/numcodecs/lzma.py b/src/numcodecs/lzma.py index 97da02825..c5039e454 100644 --- a/src/numcodecs/lzma.py +++ b/src/numcodecs/lzma.py @@ -1,4 +1,5 @@ from types import ModuleType +from typing import Any, cast, overload _lzma: ModuleType | None = None try: @@ -11,8 +12,19 @@ if _lzma: - from .abc import Codec + from .abc import Codec, CodecConfig from .compat import ensure_contiguous_ndarray, ndarray_copy + from .ndarray_like import BufferLike + + _lzma_module: ModuleType = _lzma # narrowed from `ModuleType | None` for the methods below + + class LZMAConfig(CodecConfig): + """The configuration of an `LZMA` codec, as returned by `LZMA.get_config`.""" + + format: int + check: int + preset: int | None + filters: list[dict[str, Any]] | None # noinspection PyShadowingBuiltins class LZMA(Codec): @@ -36,36 +48,49 @@ class LZMA(Codec): codec_id = 'lzma' - def __init__(self, format=1, check=-1, preset=None, filters=None): + def __init__( + self, + format: int = 1, + check: int = -1, + preset: int | None = None, + filters: list[dict[str, Any]] | None = None, + ) -> None: self.format = format self.check = check self.preset = preset self.filters = filters - def encode(self, buf): + def get_config(self) -> LZMAConfig: + return cast(LZMAConfig, super().get_config()) + + def encode(self, buf: BufferLike) -> bytes: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) # do compression - return _lzma.compress( - buf, + return _lzma_module.compress( + arr, format=self.format, check=self.check, preset=self.preset, filters=self.filters, ) - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) if out is not None: ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression - dec = _lzma.decompress(buf, format=self.format, filters=self.filters) + dec = _lzma_module.decompress(arr, format=self.format, filters=self.filters) # handle destination return ndarray_copy(dec, out) - def __repr__(self): + def __repr__(self) -> str: return f'{type(self).__name__}(format={self.format!r}, check={self.check!r}, preset={self.preset!r}, filters={self.filters!r})' diff --git a/src/numcodecs/zlib.py b/src/numcodecs/zlib.py index e62658dcb..262a92fa1 100644 --- a/src/numcodecs/zlib.py +++ b/src/numcodecs/zlib.py @@ -1,7 +1,15 @@ import zlib as _zlib +from typing import cast, overload -from .abc import Codec +from .abc import Codec, CodecConfig from .compat import ensure_contiguous_ndarray, ndarray_copy +from .ndarray_like import BufferLike + + +class ZlibConfig(CodecConfig): + """The configuration of a `Zlib` codec, as returned by `Zlib.get_config`.""" + + level: int class Zlib(Codec): @@ -16,25 +24,31 @@ class Zlib(Codec): codec_id = 'zlib' - def __init__(self, level=1): + def __init__(self, level: int = 1) -> None: self.level = level - def encode(self, buf): + def get_config(self) -> ZlibConfig: + return cast(ZlibConfig, super().get_config()) + + def encode(self, buf: BufferLike) -> bytes: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) # do compression - return _zlib.compress(buf, self.level) + return _zlib.compress(arr, self.level) - # noinspection PyMethodMayBeStatic - def decode(self, buf, out=None): + @overload + def decode(self, buf: BufferLike, out: None = None) -> bytes: ... + @overload + def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ... + def decode(self, buf: BufferLike, out: BufferLike | None = None) -> BufferLike: # normalise inputs - buf = ensure_contiguous_ndarray(buf) + arr = ensure_contiguous_ndarray(buf) if out is not None: ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression - dec = _zlib.decompress(buf) + dec = _zlib.decompress(arr) # handle destination - Python standard library zlib module does not # support direct decompression into buffer, so we have to copy into diff --git a/src/numcodecs/zstd.pyi b/src/numcodecs/zstd.pyi index 617e31f2c..50672763d 100644 --- a/src/numcodecs/zstd.pyi +++ b/src/numcodecs/zstd.pyi @@ -1,6 +1,6 @@ from typing import Final, overload -from .abc import Codec +from .abc import Codec, CodecConfig from .ndarray_like import BufferLike VERSION_NUMBER: Final[int] @@ -16,12 +16,17 @@ def decompress(source: BufferLike, dest: None = None) -> bytes: ... @overload def decompress[OutT: BufferLike](source: BufferLike, dest: OutT) -> OutT: ... +class ZstdConfig(CodecConfig): + level: int + checksum: bool + class Zstd(Codec): codec_id: str max_buffer_size: int level: int checksum: bool def __init__(self, level: int = ..., checksum: bool = False) -> None: ... + def get_config(self) -> ZstdConfig: ... def encode(self, buf: BufferLike) -> bytes: ... @overload def decode(self, buf: BufferLike, out: None = None) -> bytes: ... diff --git a/src/numcodecs/zstd.pyx b/src/numcodecs/zstd.pyx index 7b6ff3c02..5037c8dce 100644 --- a/src/numcodecs/zstd.pyx +++ b/src/numcodecs/zstd.pyx @@ -11,9 +11,16 @@ from cpython.memoryview cimport PyMemoryView_GET_BUFFER from .compat_ext cimport PyBytes_RESIZE, ensure_contiguous_memoryview +from typing import TypedDict + from .compat import ensure_contiguous_ndarray from .abc import Codec + +# The type of `Zstd.get_config()`. Declared with the functional syntax because Cython does not +# compile TypedDict class bodies; `zstd.pyi` declares the same type with the class syntax. +ZstdConfig = TypedDict("ZstdConfig", {"id": str, "level": int, "checksum": bool}) + from libc.stdlib cimport malloc, realloc, free cdef extern from "stdint.h": diff --git a/tests/typing/pyrightconfig.json b/tests/typing/pyrightconfig.json index fac5363b8..7a3897631 100644 --- a/tests/typing/pyrightconfig.json +++ b/tests/typing/pyrightconfig.json @@ -11,5 +11,6 @@ "reportUnknownVariableType": "none", "reportUnknownMemberType": "none", "reportUnknownArgumentType": "none", - "reportMissingModuleSource": "none" + "reportMissingModuleSource": "none", + "enableTypeIgnoreComments": false } diff --git a/tests/typing/test_compressor_types.py b/tests/typing/test_compressor_types.py new file mode 100644 index 000000000..0adb6fe4b --- /dev/null +++ b/tests/typing/test_compressor_types.py @@ -0,0 +1,78 @@ +"""Static types of the compressor codecs, checked against runtime behavior.""" + +import json +from collections.abc import Mapping +from typing import Any, assert_type + +import numpy as np + +from numcodecs.abc import JSON, ArrayArrayCodec, BytesBytesCodec, CodecConfig +from numcodecs.base64 import Base64 +from numcodecs.blosc import Blosc, BloscConfig +from numcodecs.bz2 import BZ2, BZ2Config +from numcodecs.gzip import GZip, GZipConfig +from numcodecs.lz4 import LZ4, LZ4Config +from numcodecs.lzma import LZMA, LZMAConfig +from numcodecs.zlib import Zlib, ZlibConfig +from numcodecs.zstd import Zstd, ZstdConfig + +DATA = np.arange(16, dtype="f8") + + +def _round_trip(codec: BytesBytesCodec) -> None: + """Every compressor: bytes in any buffer form go in, bytes come out, `out` comes back.""" + enc = codec.encode(DATA) + assert bytes(memoryview(codec.decode(enc))) == DATA.tobytes() + + +def test_compressors() -> None: + for codec in (Zlib(), GZip(), BZ2(), LZMA(), Base64(), Blosc(), Zstd(), LZ4()): + _round_trip(codec) + enc = assert_type(codec.encode(DATA), bytes) + assert isinstance(enc, bytes) + assert assert_type(codec.decode(enc), bytes) == DATA.tobytes() + out = np.empty_like(DATA) + assert codec.decode(enc, out=out) is out + buf = bytearray(DATA.nbytes) + assert assert_type(codec.decode(enc, out=buf), bytearray) is buf + + +def _json_round_trip(config: Any) -> Any: + return json.loads(json.dumps(config)) + + +def test_configs() -> None: + zlib_config = assert_type(Zlib(level=3).get_config(), ZlibConfig) + assert assert_type(zlib_config["level"], int) == 3 + assert assert_type(GZip(level=2).get_config(), GZipConfig)["level"] == 2 + assert assert_type(BZ2(level=4).get_config(), BZ2Config)["level"] == 4 + lzma_config = assert_type(LZMA(preset=5).get_config(), LZMAConfig) + assert assert_type(lzma_config["preset"], int | None) == 5 + assert assert_type(Base64().get_config(), CodecConfig) == {"id": "base64"} + blosc_config = assert_type(Blosc(cname="zstd").get_config(), BloscConfig) + assert assert_type(blosc_config["cname"], str) == "zstd" + assert set(blosc_config) == set(BloscConfig.__annotations__) + zstd_config = assert_type(Zstd(checksum=True).get_config(), ZstdConfig) + assert assert_type(zstd_config["checksum"], bool) is True + assert assert_type(LZ4().get_config(), LZ4Config)["acceleration"] == 1 + for config in (zlib_config, lzma_config, blosc_config, zstd_config): + assert _json_round_trip(config) == config + + +def test_compressors_are_not_array_codecs() -> None: + # A compressor returns bytes, which is not an ndarray-like. + codec: ArrayArrayCodec = Zlib() # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + assert not isinstance(codec.encode(DATA), np.ndarray) + + +def _json_mapping(config: Mapping[str, JSON]) -> Mapping[str, JSON]: + return config + + +def test_configs_are_json_mappings() -> None: + # pyright implements PEP 728, so CodecConfig's read-only JSON `extra_items` make every + # codec configuration a `Mapping[str, JSON]`. mypy does not implement it yet. + zlib_config = _json_mapping(Zlib().get_config()) # type: ignore[arg-type] + blosc_config = _json_mapping(Blosc().get_config()) # type: ignore[arg-type] + for config in (zlib_config, blosc_config): + assert _json_round_trip(config) == config From 39f3692d5a088fb9123d51a480f386c5287b27ed Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 21 Nov 2026 17:04:34 +0100 Subject: [PATCH 7/8] style: sort test imports now that the Cython modules have stubs With .pyi stubs next to the .pyx sources, ruff resolves numcodecs.blosc, numcodecs.zstd, numcodecs.fletcher32 and numcodecs.jenkins as first-party modules and wants them grouped with the other numcodecs imports. Assisted-by: ClaudeCode:claude-opus-5-5 --- tests/test_blosc.py | 3 +-- tests/test_fletcher32.py | 1 + tests/test_jenkins.py | 2 +- tests/test_zstd.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_blosc.py b/tests/test_blosc.py index b920f03e8..abff05923 100644 --- a/tests/test_blosc.py +++ b/tests/test_blosc.py @@ -7,9 +7,8 @@ import pytest try: - from numcodecs.blosc import Blosc - from numcodecs import blosc + from numcodecs.blosc import Blosc except ImportError: # pragma: no cover pytest.skip("numcodecs.blosc not available", allow_module_level=True) diff --git a/tests/test_fletcher32.py b/tests/test_fletcher32.py index adfb31da6..3bbefe850 100644 --- a/tests/test_fletcher32.py +++ b/tests/test_fletcher32.py @@ -1,5 +1,6 @@ import numpy as np import pytest + from numcodecs.fletcher32 import Fletcher32 diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py index 511c435ca..ae4cd055a 100644 --- a/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -1,8 +1,8 @@ import numpy as np import pytest -from numcodecs.jenkins import jenkins_lookup3 from numcodecs.checksum32 import JenkinsLookup3 +from numcodecs.jenkins import jenkins_lookup3 def test_jenkins_lookup3(): diff --git a/tests/test_zstd.py b/tests/test_zstd.py index 04f823838..b47363c7d 100644 --- a/tests/test_zstd.py +++ b/tests/test_zstd.py @@ -3,8 +3,8 @@ import numpy as np import pytest -from numcodecs.zstd import Zstd +from numcodecs.zstd import Zstd from tests.common import ( check_backwards_compatibility, check_config, From 2587eb936ea5fee900f2f50a52e50a467ec1c85f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 21 Nov 2026 17:05:02 +0100 Subject: [PATCH 8/8] ci: allow the PEP 705 keys stubtest reports on the Cython config TypedDicts The Cython modules create their config TypedDicts at runtime with typing.TypedDict, which adds __mutable_keys__ and __readonly_keys__; stubtest does not expect them on the stubs' class-syntax TypedDicts. The allowlist entry was only added in the py.typed change, but it is needed from the commit that introduced the TypedDicts. Assisted-by: ClaudeCode:claude-opus-5-5 --- tests/typing/stubtest-allowlist.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/typing/stubtest-allowlist.txt b/tests/typing/stubtest-allowlist.txt index 394073671..01daff8d3 100644 --- a/tests/typing/stubtest-allowlist.txt +++ b/tests/typing/stubtest-allowlist.txt @@ -8,3 +8,7 @@ numcodecs\.blosc\.(_init|_destroy|_get_use_threads|get_nthreads|list_compressors # Cython's C-API capsule table for cimport-able functions. numcodecs\.compat_ext\.__pyx_capi__ + +# The config TypedDicts are created at runtime with `typing.TypedDict`, which adds these +# PEP 705 attributes; stubtest does not expect them on the stubs' class-syntax TypedDicts. +numcodecs\..*Config\.__(mutable|readonly)_keys__