Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
ff36c01
test: check the Codec contract for every codec and add static type tests
d-v-b Nov 21, 2026
145994e
fix: return the caller's out from decode(buf, out=out) in every codec
d-v-b Nov 21, 2026
52be3f5
fix(typing): correct registry, compat and NDArrayLike annotations and…
d-v-b Nov 21, 2026
c26d438
feat(abc): annotate the Codec base class and add structural codec pro…
d-v-b Nov 21, 2026
3220705
feat(typing): ship .pyi stubs for the Cython extension modules
d-v-b Nov 21, 2026
2dcbbcd
feat(typing): annotate the compressor codecs with per-codec config Ty…
d-v-b Nov 21, 2026
d5b3d9b
feat(typing): annotate the checksum codecs and make JenkinsLookup3 co…
d-v-b Nov 21, 2026
d8a20b3
feat(typing): annotate the filter codecs
d-v-b Nov 21, 2026
d664d8b
feat(typing): annotate the object and array-to-bytes codecs
d-v-b Nov 21, 2026
4fcbc9a
feat(typing): ship py.typed and check numcodecs with strict mypy
d-v-b Nov 21, 2026
39f3692
style: sort test imports now that the Cython modules have stubs
d-v-b Nov 21, 2026
25a5ec9
Merge branch 'typing/05-cython-stubs' into typing/06a-compressors
d-v-b Nov 21, 2026
2587eb9
ci: allow the PEP 705 keys stubtest reports on the Cython config Type…
d-v-b Nov 21, 2026
cc8a5f0
Merge branch 'typing/06d-object-codecs' into typing/07-py-typed
d-v-b Nov 21, 2026
d84c7e1
Merge branch 'typing/06a-compressors' into typing/06b-checksums
d-v-b Nov 21, 2026
accbed5
Merge branch 'typing/06b-checksums' into typing/06c-filters
d-v-b Nov 21, 2026
3dd78eb
Merge branch 'typing/06c-filters' into typing/06d-object-codecs
d-v-b Nov 21, 2026
a97e369
fix(typing): make strict mypy pass in the CI and pre-commit environments
d-v-b Nov 21, 2026
cffb3e4
Merge remote-tracking branch 'upstream/main' into typing/02-out-contract
d-v-b Nov 21, 2026
f3ff244
Merge branch 'typing/02-out-contract' into typing/03-fix-annotations
d-v-b Nov 21, 2026
8b4d3b1
Merge branch 'typing/03-fix-annotations' into typing/04-base-class
d-v-b Nov 21, 2026
50a5b27
Merge branch 'typing/04-base-class' into typing/05-cython-stubs
d-v-b Nov 21, 2026
bf3d071
Merge branch 'typing/05-cython-stubs' into typing/06a-compressors
d-v-b Nov 21, 2026
832b3a4
Merge branch 'typing/06a-compressors' into typing/06b-checksums
d-v-b Nov 21, 2026
867b068
Merge branch 'typing/06b-checksums' into typing/06c-filters
d-v-b Nov 21, 2026
cc8b7ce
Merge branch 'typing/06c-filters' into typing/06d-object-codecs
d-v-b Nov 21, 2026
fb3b333
Merge branch 'typing/06d-object-codecs' into typing/07-py-typed
d-v-b Nov 21, 2026
e3b9d2d
Merge remote-tracking branch 'upstream/main' into typing/03-fix-annot…
d-v-b Nov 21, 2026
05dabce
Merge branch 'typing/03-fix-annotations' into typing/04-base-class
d-v-b Nov 21, 2026
889670a
Merge branch 'typing/04-base-class' into typing/05-cython-stubs
d-v-b Nov 21, 2026
bc90f92
Merge branch 'typing/05-cython-stubs' into typing/06a-compressors
d-v-b Nov 21, 2026
26b856b
Merge branch 'typing/06a-compressors' into typing/06b-checksums
d-v-b Nov 21, 2026
fc88f40
Merge branch 'typing/06b-checksums' into typing/06c-filters
d-v-b Nov 21, 2026
c53d97c
Merge branch 'typing/06c-filters' into typing/06d-object-codecs
d-v-b Nov 21, 2026
18192cc
Merge branch 'typing/06d-object-codecs' into typing/07-py-typed
d-v-b Nov 21, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,15 @@ jobs:
python -m pip install -v ".[test,msgpack,crc32c]"
python -m pip install --group typing

- name: The installed package carries py.typed and the stubs
run: |
python - <<'PY'
from importlib.resources import files
pkg = files("numcodecs")
missing = [n for n in ("py.typed", "blosc.pyi", "vlen.pyi") if not (pkg / n).is_file()]
assert not missing, missing
PY

- name: mypy (repository configuration)
run: mypy --config-file pyproject.toml src tests

Expand All @@ -162,6 +171,13 @@ jobs:
- name: pyright (strict type tests)
run: pyright -p tests/typing

- name: stubtest (Cython stubs against the built extensions)
run: |
python -m mypy.stubtest --mypy-config-file pyproject.toml \
--allowlist tests/typing/stubtest-allowlist.txt \
numcodecs.blosc numcodecs.zstd numcodecs.lz4 numcodecs.vlen \
numcodecs.fletcher32 numcodecs.jenkins numcodecs.compat_ext numcodecs._shuffle

test-zarr:
runs-on: ubuntu-latest
strategy:
Expand Down
4 changes: 2 additions & 2 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ repos:
- id: sp-repo-review

- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.16.0
rev: v2.4.1
hooks:
- id: mypy
args: [--config-file, pyproject.toml]
additional_dependencies: [numpy, pytest, google-crc32c, crc32c, zfpy, 'zarr>=3.1.3']
additional_dependencies: [numpy, packaging, pytest, typing_extensions, google-crc32c, crc32c, zfpy, 'zarr>=3.1.3']
12 changes: 12 additions & 0 deletions docs/abc.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,16 @@
.. automethod:: decode
.. automethod:: get_config
.. automethod:: from_config

Protocols
---------

These describe codecs structurally, for code that accepts any object implementing the
interface whether or not it subclasses :class:`Codec`. The three kinds mirror the codec
kinds of the Zarr version 3 specification.

.. autoclass:: CodecProtocol
.. autoclass:: BytesBytesCodec
.. autoclass:: ArrayArrayCodec
.. autoclass:: ArrayBytesCodec
```
2 changes: 1 addition & 1 deletion docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

class Mock(MagicMock):
@classmethod
def __getattr__(cls, name):
def __getattr__(cls, name: str) -> "Mock":
return Mock()


Expand Down
58 changes: 58 additions & 0 deletions docs/release.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
wheels) rather than every branch, test the oldest supported and latest Zarr
releases, use Alpine's numpy in the i386 job, and remove the unused release-notes
labeler.
* Run mypy in strict mode over `src`, in CI as well as in pre-commit (now mypy 2.4.1), and
stop ignoring missing imports globally.
* Check the behavior every codec promises through `numcodecs.abc.Codec` in a new
`tests/test_codec_contract.py`, and add strict static type tests in `tests/typing`, run by
mypy and pyright in a new `typing` CI job.
Expand All @@ -45,8 +47,64 @@
already written into `out`; only the returned object changes.
* `JenkinsLookup3.decode` accepts any contiguous writable buffer as `out`, not only an
ndarray, and raises `ValueError` when `out` has the wrong size.
* `JenkinsLookup3.get_config()` reports `prefix` as a list of ints, so a configuration
with a prefix can be serialized to JSON and passed back to `get_codec`. It held a numpy
array, which `json.dumps` rejected and `get_codec` could not compare. The constructor
accepts a list of ints as well as a bytes-like `prefix`.
* CRC32, Adler32 and CRC32C check the byte length of the input to `decode`, not its
length along the first axis, before looking for the checksum.
* `Fletcher32.decode` without `out` returns a `uint8` ndarray, like the other checksum
codecs, instead of a Cython memoryview slice.
* Correct type annotations that described something other than what the code does:
`numcodecs.registry.codec_registry` holds codec classes (`dict[str, type[Codec]]`) and
`entries` holds single entry points (`dict[str, EntryPoint]`). `numcodecs.compat`
functions now annotate their parameters, and `ndarray_copy` returns the type of `dst`
(or of `src` when `dst` is `None`). The `NDArrayLike`, `DType` and `FlagsObj` protocols
declare read-only properties, so `numpy.ndarray` now satisfies `NDArrayLike` for type
checkers, and `numcodecs.ndarray_like.BufferLike` names "a buffer or an ndarray-like".
* `numcodecs` defines `__all__`, so type checkers treat its re-exported codec classes as
public. `from numcodecs import *` now imports only those names.
* `register_codec` raises `ValueError` for a class without a `codec_id` when no id is
given, instead of registering it under `None`.
* `Codec.codec_id` no longer defaults to `None`. Abstract intermediate classes such as
`Checksum32` leave it unset, so reading it raises `AttributeError`; every concrete codec
sets it.

### Enhancements

* Annotate `numcodecs.abc.Codec`. `encode` and `decode` take and return
`numcodecs.ndarray_like.BufferLike` (an object exporting the buffer protocol, or an
ndarray-like), with `buf` positional-only, and `decode(buf, out=out)` is typed to return
the type of `out`. `get_config` returns `Mapping[str, Any]` and `from_config` returns
`Self`. Previously pyright inferred that `encode` and `decode` return `None`.
* Add structural protocols `numcodecs.abc.CodecProtocol`, `BytesBytesCodec`,
`ArrayArrayCodec` and `ArrayBytesCodec` for code that accepts any object implementing the
codec interface.
* numcodecs now ships a `py.typed` marker (PEP 561), so type checkers use its annotations
and stubs instead of treating the package as untyped. Downstream projects can drop
`ignore_missing_imports` for numcodecs. Third-party codecs that subclass `Codec` are
type-checked against it for the first time, which may surface two one-line fixes: a
`# type: ignore[misc]` on `class X(Codec)` that is no longer needed, and a
`from_config(cls, config: dict[...])` override that must accept a `Mapping`.
* Annotate the concrete codecs. Each `get_config` returns a TypedDict naming the codec's
parameters (`ZlibConfig`, `BloscConfig`, ...), all subclasses of
`numcodecs.abc.CodecConfig`, which carries the `id` and declares any further keys as
read-only JSON values (PEP 728 `extra_items`). `numcodecs.abc.JSON` names a
JSON-serializable value. Compressors: Zlib, GZip, BZ2, LZMA, Base64, Blosc, Zstd and LZ4.
Checksums: CRC32, Adler32, CRC32C, JenkinsLookup3 and Fletcher32. Filters: AsType,
Delta, Quantize, FixedScaleOffset, BitRound, PackBits, Shuffle and Categorize. Object
and array-to-bytes codecs: JSON, MsgPack, Pickle (whose `decode` returns `Any`), the VLen
codecs, ZFPY and PCodec.
`typing_extensions` >= 4.13 is now required.
* Ship `.pyi` type stubs for the Cython extension modules (`blosc`, `zstd`, `lz4`, `vlen`,
`fletcher32`, `jenkins`, `compat_ext` and `_shuffle`), checked against the compiled
modules with `mypy.stubtest` in CI. Type checkers previously saw the Blosc, Zstd, LZ4,
Fletcher32 and VLen codecs as unknown.

### Deprecations

* Loading a codec through the `numcodecs.codecs` entry point group whose class does not
subclass `numcodecs.abc.Codec` now emits a `DeprecationWarning`.

(release_0.17.1)=

Expand Down
44 changes: 35 additions & 9 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ description = """
A Python package providing buffer compression and transformation codecs \
for use in data storage and communication applications."""
readme = "README.md"
dependencies = ["numpy>=2.0", "packaging", "typing_extensions"]
dependencies = ["numpy>=2.0", "packaging", "typing_extensions>=4.13"]
requires-python = ">=3.12"
dynamic = [
"version",
Expand Down Expand Up @@ -243,15 +243,41 @@ ignore = [
quote-style = "preserve"

[tool.mypy]
ignore_errors = false
ignore_missing_imports = true
python_version = "3.12"
mypy_path = "src"
strict = true
warn_unreachable = true
enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
# TODO: set options below to true
strict = false
warn_unreachable = false
warn_redundant_casts = true
warn_unused_ignores = true
warn_unused_configs = true

# Optional dependencies that ship no type information.
[[tool.mypy.overrides]]
module = [
"msgpack.*",
"pcodec.*",
"zfpy.*",
"crc32c.*",
"google_crc32c.*",
"backports.*",
# build-time and test-only dependencies
"setuptools_scm.*",
"importlib_metadata.*",
# optional: numcodecs.zarr3 forwards to zarr, which is not installed everywhere
"zarr.*",
]
ignore_missing_imports = true

# The test suite predates type annotations, and many tests pass wrongly typed values on
# purpose to check runtime errors. As before, annotated test functions are checked and
# unannotated ones are not. tests/typing is checked strictly by its own configuration
# (tests/typing/mypy.ini).
[[tool.mypy.overrides]]
module = ["tests.*"]
check_untyped_defs = false
disallow_untyped_defs = false
disallow_incomplete_defs = false
disallow_untyped_calls = false
disallow_any_generics = false
warn_return_any = false

[tool.uv]
# meson-python editable installs require build deps (ninja, meson) at runtime for
Expand Down
42 changes: 41 additions & 1 deletion src/numcodecs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,42 @@
from numcodecs.registry import get_codec as get_codec
from numcodecs.registry import register_codec

# Names listed here are the public API re-exported from submodules. Optional codecs are
# appended below when their dependency imports.
__all__ = [
"BZ2",
"CRC32",
"JSON",
"LZ4",
"LZMA",
"Adler32",
"AsType",
"Base64",
"BitRound",
"Blosc",
"Categorize",
"Delta",
"FixedScaleOffset",
"Fletcher32",
"GZip",
"JenkinsLookup3",
"PackBits",
"Pickle",
"Quantize",
"Shuffle",
"VLenArray",
"VLenBytes",
"VLenUTF8",
"Zlib",
"Zstd",
"blosc",
"get_codec",
"lz4",
"register_codec",
"vlen",
"zstd",
]

__version__: str = _version("numcodecs")
from numcodecs.zlib import Zlib

Expand All @@ -48,7 +84,7 @@
register_codec(Blosc)
# initialize blosc
try:
ncores = multiprocessing.cpu_count()
ncores: int = multiprocessing.cpu_count()
except OSError: # pragma: no cover
ncores = 1
blosc._init()
Expand Down Expand Up @@ -131,18 +167,22 @@
from numcodecs.zfpy import ZFPY

register_codec(ZFPY)
__all__ += ["ZFPY"]

with suppress(ImportError):
from numcodecs.msgpacks import MsgPack

register_codec(MsgPack)
__all__ += ["MsgPack"]

with suppress(ImportError):
from numcodecs.checksum32 import CRC32C

register_codec(CRC32C)
__all__ += ["CRC32C"]

with suppress(ImportError):
from numcodecs.pcodec import PCodec

register_codec(PCodec)
__all__ += ["PCodec"]
6 changes: 4 additions & 2 deletions src/numcodecs/_build_utils/gitversion.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,15 @@
import os


def get_version():
def get_version() -> str:
try:
from setuptools_scm import get_version

return get_version(root=os.path.join(os.path.dirname(__file__), '..', '..', '..'))
version: str = get_version(root=os.path.join(os.path.dirname(__file__), '..', '..', '..'))
except (ImportError, LookupError):
pass
else:
return version

# Fallback: read from pyproject.toml
pyproject = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'pyproject.toml')
Expand Down
4 changes: 4 additions & 0 deletions src/numcodecs/_shuffle.pyi
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
from collections.abc import Buffer

def _doShuffle(src: Buffer, des: Buffer, element_size: int) -> None: ...
def _doUnshuffle(src: Buffer, des: Buffer, element_size: int) -> None: ...
Loading
Loading