Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,17 @@ jobs:
- name: pyright (strict type tests)
run: pyright -p tests/typing

- name: stubtest (Cython stubs against the built extensions)
# Read the stubs from src/: mypy ignores the installed copies until numcodecs
# ships a py.typed marker.
env:
MYPYPATH: src
run: |
python -m mypy.stubtest --mypy-config-file pyproject.toml \
--allowlist tests/typing/stubtest-allowlist.txt \
numcodecs.blosc numcodecs.zstd numcodecs.lz4 numcodecs.vlen \
numcodecs.fletcher32 numcodecs.jenkins numcodecs.compat_ext numcodecs._shuffle

test-zarr:
runs-on: ubuntu-latest
strategy:
Expand Down
12 changes: 12 additions & 0 deletions docs/abc.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,16 @@
.. automethod:: decode
.. automethod:: get_config
.. automethod:: from_config

Protocols
---------

These describe codecs structurally, for code that accepts any object implementing the
interface whether or not it subclasses :class:`Codec`. The three kinds mirror the codec
kinds of the Zarr version 3 specification.

.. autoclass:: CodecProtocol
.. autoclass:: BytesBytesCodec
.. autoclass:: ArrayArrayCodec
.. autoclass:: ArrayBytesCodec
```
34 changes: 34 additions & 0 deletions docs/release.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,40 @@
ndarray, and raises `ValueError` when `out` has the wrong size.
* `Fletcher32.decode` without `out` returns a `uint8` ndarray, like the other checksum
codecs, instead of a Cython memoryview slice.
* Correct type annotations that described something other than what the code does:
`numcodecs.registry.codec_registry` holds codec classes (`dict[str, type[Codec]]`) and
`entries` holds single entry points (`dict[str, EntryPoint]`). `numcodecs.compat`
functions now annotate their parameters, and `ndarray_copy` returns the type of `dst`
(or of `src` when `dst` is `None`). The `NDArrayLike`, `DType` and `FlagsObj` protocols
declare read-only properties, so `numpy.ndarray` now satisfies `NDArrayLike` for type
checkers, and `numcodecs.ndarray_like.BufferLike` names "a buffer or an ndarray-like".
* `numcodecs` defines `__all__`, so type checkers treat its re-exported codec classes as
public. `from numcodecs import *` now imports only those names.
* `register_codec` raises `ValueError` for a class without a `codec_id` when no id is
given, instead of registering it under `None`.
* `Codec.codec_id` no longer defaults to `None`. Abstract intermediate classes such as
`Checksum32` leave it unset, so reading it raises `AttributeError`; every concrete codec
sets it.

### Enhancements

* Annotate `numcodecs.abc.Codec`. `encode` and `decode` take and return
`numcodecs.ndarray_like.BufferLike` (an object exporting the buffer protocol, or an
ndarray-like), with `buf` positional-only, and `decode(buf, out=out)` is typed to return
the type of `out`. `get_config` returns `Mapping[str, Any]` and `from_config` returns
`Self`. Previously pyright inferred that `encode` and `decode` return `None`.
* Add structural protocols `numcodecs.abc.CodecProtocol`, `BytesBytesCodec`,
`ArrayArrayCodec` and `ArrayBytesCodec` for code that accepts any object implementing the
codec interface.
* Ship `.pyi` type stubs for the Cython extension modules (`blosc`, `zstd`, `lz4`, `vlen`,
`fletcher32`, `jenkins`, `compat_ext` and `_shuffle`), checked against the compiled
modules with `mypy.stubtest` in CI. Type checkers previously saw the Blosc, Zstd, LZ4,
Fletcher32 and VLen codecs as unknown.

### Deprecations

* Loading a codec through the `numcodecs.codecs` entry point group whose class does not
subclass `numcodecs.abc.Codec` now emits a `DeprecationWarning`.

(release_0.17.1)=

Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ quote-style = "preserve"
[tool.mypy]
ignore_errors = false
ignore_missing_imports = true
python_version = "3.12"
enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
# TODO: set options below to true
strict = false
Expand Down
40 changes: 40 additions & 0 deletions src/numcodecs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,42 @@
from numcodecs.registry import get_codec as get_codec
from numcodecs.registry import register_codec

# Names listed here are the public API re-exported from submodules. Optional codecs are
# appended below when their dependency imports.
__all__ = [
"BZ2",
"CRC32",
"JSON",
"LZ4",
"LZMA",
"Adler32",
"AsType",
"Base64",
"BitRound",
"Blosc",
"Categorize",
"Delta",
"FixedScaleOffset",
"Fletcher32",
"GZip",
"JenkinsLookup3",
"PackBits",
"Pickle",
"Quantize",
"Shuffle",
"VLenArray",
"VLenBytes",
"VLenUTF8",
"Zlib",
"Zstd",
"blosc",
"get_codec",
"lz4",
"register_codec",
"vlen",
"zstd",
]

__version__: str = _version("numcodecs")
from numcodecs.zlib import Zlib

Expand Down Expand Up @@ -131,18 +167,22 @@
from numcodecs.zfpy import ZFPY

register_codec(ZFPY)
__all__ += ["ZFPY"]

with suppress(ImportError):
from numcodecs.msgpacks import MsgPack

register_codec(MsgPack)
__all__ += ["MsgPack"]

with suppress(ImportError):
from numcodecs.checksum32 import CRC32C

register_codec(CRC32C)
__all__ += ["CRC32C"]

with suppress(ImportError):
from numcodecs.pcodec import PCodec

register_codec(PCodec)
__all__ += ["PCodec"]
4 changes: 4 additions & 0 deletions src/numcodecs/_shuffle.pyi
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
from collections.abc import Buffer

def _doShuffle(src: Buffer, des: Buffer, element_size: int) -> None: ...
def _doUnshuffle(src: Buffer, des: Buffer, element_size: int) -> None: ...
87 changes: 74 additions & 13 deletions src/numcodecs/abc.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,31 @@
"""

from abc import ABC, abstractmethod
from collections.abc import Buffer, Mapping
from typing import Any, Protocol, Self, overload

from .ndarray_like import BufferLike, NDArrayLike

__all__ = [
"ArrayArrayCodec",
"ArrayBytesCodec",
"BytesBytesCodec",
"Codec",
"CodecProtocol",
]


class Codec(ABC):
"""Codec abstract base class."""

# override in sub-class
codec_id: str | None = None
"""Codec identifier."""
# Declared without a value (abstract intermediate classes leave it unset) and without
# `ClassVar`, so that `Codec` instances match protocols that declare `codec_id` either
# as a plain attribute or as a read-only property.
codec_id: str
"""Codec identifier. Every concrete codec class sets it as a class attribute."""

@abstractmethod
def encode(self, buf): # pragma: no cover
def encode(self, buf: BufferLike, /) -> BufferLike: # pragma: no cover
"""Encode data in `buf`.

Parameters
Expand All @@ -55,8 +69,14 @@ def encode(self, buf): # pragma: no cover
protocol.
"""

@overload
def decode(self, buf: BufferLike, /, out: None = None) -> BufferLike: ...
@overload
def decode[OutT: BufferLike](self, buf: BufferLike, /, out: OutT) -> OutT: ...
@abstractmethod
def decode(self, buf, out=None): # pragma: no cover
def decode(
self, buf: BufferLike, /, out: BufferLike | None = None
) -> BufferLike: # pragma: no cover
"""Decode data in `buf`.

Parameters
Expand All @@ -72,18 +92,18 @@ def decode(self, buf, out=None): # pragma: no cover
-------
dec : buffer-like
Decoded data. May be any object supporting the new-style
buffer protocol.
buffer protocol. If `out` was provided, `out` itself.
"""

def get_config(self):
def get_config(self) -> Mapping[str, Any]:
"""Return a dictionary holding configuration parameters for this
codec. Must include an 'id' field with the codec identifier. All
values must be compatible with JSON encoding."""

# override in sub-class if need special encoding of config values

# setup config object
config = {'id': self.codec_id}
config: dict[str, Any] = {'id': self.codec_id}

# by default, assume all non-private members are configuration
# parameters - override this in sub-class if not the case
Expand All @@ -94,8 +114,8 @@ def get_config(self):
return config

@classmethod
def from_config(cls, config):
"""Instantiate codec from a configuration object."""
def from_config(cls, config: Mapping[str, Any], /) -> Self:
"""Instantiate codec from a configuration object, without its 'id' field."""
# N.B., assume at this point the 'id' field has been removed from
# the config object

Expand All @@ -105,14 +125,14 @@ def from_config(cls, config):
# keyword arguments without any special decoding
return cls(**config)

def __eq__(self, other):
def __eq__(self, other: object) -> bool:
# override in sub-class if need special equality comparison
try:
return self.get_config() == other.get_config()
return bool(self.get_config() == other.get_config()) # type: ignore[attr-defined]
except AttributeError:
return False

def __repr__(self):
def __repr__(self) -> str:
# override in sub-class if need special representation

# by default, assume all non-private members are configuration
Expand All @@ -124,3 +144,44 @@ def __repr__(self):
]
r += ', '.join(params) + ')'
return r


class CodecProtocol(Protocol):
"""The structural interface of a codec, for code that accepts any object implementing
it, whether or not it subclasses `Codec`."""

@property
def codec_id(self) -> str: ... # pragma: no cover

def encode(self, buf: BufferLike, /) -> BufferLike: ... # pragma: no cover

def decode(self, buf: BufferLike, /, out: Any = None) -> BufferLike: ... # pragma: no cover

def get_config(self) -> Mapping[str, Any]: ... # pragma: no cover

@classmethod
def from_config(cls, config: Mapping[str, Any], /) -> Self: ... # pragma: no cover


class BytesBytesCodec(CodecProtocol, Protocol):
"""A codec that maps bytes to bytes, such as a compressor or a checksum."""

def encode(self, buf: BufferLike, /) -> Buffer: ... # pragma: no cover

def decode(self, buf: BufferLike, /, out: Any = None) -> Buffer: ... # pragma: no cover


class ArrayArrayCodec(CodecProtocol, Protocol):
"""A codec that maps an array to an array, such as a filter."""

def encode(self, buf: BufferLike, /) -> NDArrayLike: ... # pragma: no cover

def decode(self, buf: BufferLike, /, out: Any = None) -> NDArrayLike: ... # pragma: no cover


class ArrayBytesCodec(CodecProtocol, Protocol):
"""A codec that serializes an array to bytes and back."""

def encode(self, buf: BufferLike, /) -> Buffer: ... # pragma: no cover

def decode(self, buf: BufferLike, /, out: Any = None) -> NDArrayLike: ... # pragma: no cover
68 changes: 68 additions & 0 deletions src/numcodecs/blosc.pyi
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
from typing import Final, overload

from .abc import Codec
from .ndarray_like import BufferLike

NOSHUFFLE: Final[int]
SHUFFLE: Final[int]
BITSHUFFLE: Final[int]
AUTOSHUFFLE: Final[int]
AUTOBLOCKS: Final[int]
MAX_OVERHEAD: Final[int]
MAX_BUFFERSIZE: Final[int]
MAX_THREADS: Final[int]
MAX_TYPESIZE: Final[int]
VERSION_STRING: Final[str]
VERSION_DATE: Final[str]
__version__: Final[str]

use_threads: bool | None
"""Set to `True` or `False` to override whether Blosc may use threads; `None` (the
default) decides from the calling thread and process."""

def _init() -> None: ...
def _destroy() -> None: ...
def _get_use_threads() -> bool: ...
def _cbuffer_sizes(source: BufferLike) -> tuple[int, int, int]: ...
def _cbuffer_metainfo(source: BufferLike) -> tuple[int, bool, bool]: ...
def list_compressors() -> list[str]: ...
def get_nthreads() -> int: ...
def set_nthreads(nthreads: int) -> int: ...
def cbuffer_complib(source: BufferLike) -> str: ...
def compress(
source: BufferLike,
cname: bytes,
clevel: int,
shuffle: int = ...,
blocksize: int = ...,
typesize: int | None = None,
) -> bytes: ...
@overload
def decompress(source: BufferLike, dest: None = None) -> bytes: ...
@overload
def decompress[OutT: BufferLike](source: BufferLike, dest: OutT) -> OutT: ...

class Blosc(Codec):
codec_id: str
NOSHUFFLE: int
SHUFFLE: int
BITSHUFFLE: int
AUTOSHUFFLE: int
max_buffer_size: int
cname: str
clevel: int
shuffle: int
blocksize: int
def __init__(
self,
cname: str = "lz4",
clevel: int = 5,
shuffle: int = ...,
blocksize: int = ...,
typesize: int | None = None,
) -> None: ...
def encode(self, buf: BufferLike) -> bytes: ...
@overload
def decode(self, buf: BufferLike, out: None = None) -> bytes: ...
@overload
def decode[OutT: BufferLike](self, buf: BufferLike, out: OutT) -> OutT: ...
4 changes: 2 additions & 2 deletions src/numcodecs/checksum32.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ def decode(self, buf, out=None):

@staticmethod
@abc.abstractmethod
def checksum(data: Buffer, value: int) -> int: ...
def checksum(data: Buffer, value: int, /) -> int: ...


class CRC32(Checksum32):
Expand Down Expand Up @@ -151,7 +151,7 @@ class JenkinsLookup3(Checksum32):
bytes prepended to the buffer before evaluating the hash, default: None
"""

checksum = jenkins_lookup3
checksum = staticmethod(jenkins_lookup3)
codec_id = "jenkins_lookup3"

def __init__(self, initval: int = 0, prefix=None):
Expand Down
Loading
Loading