diff --git a/docs/other/pickles.md b/docs/other/pickles.md index 3e27c11d..c637cacc 100644 --- a/docs/other/pickles.md +++ b/docs/other/pickles.md @@ -1,5 +1,22 @@ # Pickle +```{admonition} Deprecated +:class: warning + +The `Pickle` codec is deprecated and will be removed in numcodecs 0.19.0. Constructing it, +directly or through `numcodecs.get_codec({"id": "pickle"})`, emits a `DeprecationWarning`. +Pickled data can only be read by Python, and decoding it can execute arbitrary code. To encode +arrays of Python objects, use one of these codecs instead: + +- [`VLenUTF8`](vlen.md) for strings +- [`VLenBytes`](vlen.md) for bytes +- [`VLenArray`](vlen.md) for variable-length numeric arrays +- [`JSON`](json.md) or [`MsgPack`](msgpacks.md) for mixed objects + +Data written with `Pickle` can still be read until the codec is removed, subject to the +decoding opt-in described below. +``` + ```{warning} Decoding pickled data can execute arbitrary code. `Pickle.decode` raises `numcodecs.errors.UnsafeDecodeError` unless the process opts in, either by setting diff --git a/docs/release.md b/docs/release.md index d4ae5425..007843b2 100644 --- a/docs/release.md +++ b/docs/release.md @@ -14,6 +14,16 @@ ## Unreleased +### Deprecations + +* The `Pickle` codec is deprecated and will be removed in numcodecs 0.19.0. Constructing + it, directly or through `numcodecs.get_codec({"id": "pickle"})`, emits a + `DeprecationWarning`. Encoding and decoding still work until removal, and decoding still + requires the opt-in described in + [GHSA-jqc8-qjfv-5m6r](https://github.com/zarr-developers/numcodecs/security/advisories/GHSA-jqc8-qjfv-5m6r). + To encode arrays of Python objects, use `VLenUTF8`, `VLenBytes`, `VLenArray`, `JSON` or + `MsgPack` instead. {issue}`706` + ### Maintenance * Build wheels for CPython 3.15 and free-threaded CPython 3.15t, using cibuildwheel 4.3.5, diff --git a/src/numcodecs/pickles.py b/src/numcodecs/pickles.py index fbc2016e..7d7292d3 100644 --- a/src/numcodecs/pickles.py +++ b/src/numcodecs/pickles.py @@ -1,5 +1,6 @@ import os import pickle +import warnings from typing import Any, overload import numpy as np @@ -25,6 +26,22 @@ def _read_opt_in_from_env() -> bool: #: because codec parameters are read from array metadata, which may be untrusted. ALLOW_UNSAFE_PICKLE_DECODE = _read_opt_in_from_env() +# The release in which the `Pickle` codec is planned to be removed. +_REMOVAL_VERSION = "0.19.0" + +_DEPRECATION_MESSAGE = ( + f"The Pickle codec is deprecated and will be removed in numcodecs {_REMOVAL_VERSION}. " + "Decoding pickled data can execute arbitrary code, and pickled data can only be read " + "by Python. To encode arrays of Python objects, use VLenUTF8 for strings, VLenBytes " + "for bytes, VLenArray for variable-length numeric arrays, or JSON or MsgPack for " + "mixed objects." +) + +# Frames in files under this directory are skipped when attributing the deprecation +# warning, so it points at the first caller outside numcodecs (user code, or the +# library that selected the codec from array metadata) rather than at numcodecs itself. +_NUMCODECS_DIR = os.path.dirname(os.path.abspath(__file__)) + os.sep + class PickleConfig(CodecConfig): """The configuration of a `Pickle` codec, as returned by `Pickle.get_config`.""" @@ -36,6 +53,12 @@ class Pickle(Codec): """Codec to encode data as as pickled bytes. Useful for encoding an array of Python string objects. + This codec is deprecated and will be removed in numcodecs 0.19.0. Constructing it, directly + or through `numcodecs.get_codec`, emits a `DeprecationWarning`. To encode arrays of Python + objects, use `numcodecs.vlen.VLenUTF8` for strings, `numcodecs.vlen.VLenBytes` for bytes, + `numcodecs.vlen.VLenArray` for variable-length numeric arrays, or `numcodecs.json.JSON` + or `numcodecs.msgpacks.MsgPack` for mixed objects. + .. warning:: Decoding pickled data can execute arbitrary code. :meth:`decode` refuses to run unless the process opts in by setting @@ -51,12 +74,21 @@ class Pickle(Codec): protocol : int, defaults to pickle.HIGHEST_PROTOCOL The protocol used to pickle data. + Warns + ----- + DeprecationWarning + Always, on construction. The warning is attributed to the first caller outside + numcodecs. + Examples -------- + >>> import warnings >>> import numcodecs as codecs >>> import numpy as np >>> x = np.array(['foo', 'bar', 'baz'], dtype='object') - >>> f = codecs.Pickle() + >>> with warnings.catch_warnings(): + ... warnings.simplefilter("ignore", DeprecationWarning) + ... f = codecs.Pickle() >>> codecs.pickles.ALLOW_UNSAFE_PICKLE_DECODE = True # only for trusted data >>> f.decode(f.encode(x)) array(['foo', 'bar', 'baz'], dtype=object) @@ -64,13 +96,17 @@ class Pickle(Codec): See Also -------- - numcodecs.msgpacks.MsgPack + numcodecs.vlen.VLenUTF8, numcodecs.vlen.VLenBytes, numcodecs.vlen.VLenArray, + numcodecs.json.JSON, numcodecs.msgpacks.MsgPack """ codec_id = 'pickle' def __init__(self, protocol: int = pickle.HIGHEST_PROTOCOL) -> None: + warnings.warn( + _DEPRECATION_MESSAGE, DeprecationWarning, skip_file_prefixes=(_NUMCODECS_DIR,) + ) self.protocol = protocol def encode(self, buf: object) -> bytes: diff --git a/tests/test_codec_contract.py b/tests/test_codec_contract.py index 5d871c00..09c9fa21 100644 --- a/tests/test_codec_contract.py +++ b/tests/test_codec_contract.py @@ -165,6 +165,8 @@ def test_decode_into_bytearray_returns_out(codec_id: str) -> None: assert bytes(out) == ensure_ndarray(expected).tobytes() +# Rebuilding the codec from its config constructs it again, outside `_codec`. +@pytest.mark.filterwarnings("ignore:The Pickle codec is deprecated:DeprecationWarning") @pytest.mark.parametrize("codec_id", _params(set())) def test_config_round_trip(codec_id: str) -> None: codec = _codec(codec_id) diff --git a/tests/test_pickles.py b/tests/test_pickles.py index 70c980eb..aa9e8671 100644 --- a/tests/test_pickles.py +++ b/tests/test_pickles.py @@ -1,12 +1,15 @@ import itertools import sys +import warnings +from functools import partial import numpy as np import pytest +import numcodecs import numcodecs.pickles from numcodecs.errors import UnsafeDecodeError -from numcodecs.pickles import _ENV_VAR, Pickle, _read_opt_in_from_env +from numcodecs.pickles import _ENV_VAR, _REMOVAL_VERSION, Pickle, _read_opt_in_from_env from numcodecs.registry import get_codec from tests.common import ( check_backwards_compatibility, @@ -16,7 +19,15 @@ greetings, ) -codecs = [Pickle(protocol=i) for i in range(5)] +# Constructing Pickle emits a DeprecationWarning. The tests below that predate the +# deprecation exercise the codec itself, so they ignore it; test_construction_warns checks it. +DEPRECATION_MATCH = "The Pickle codec is deprecated" +pytestmark = pytest.mark.filterwarnings(f"ignore:{DEPRECATION_MATCH}:DeprecationWarning") + +# Built at import time, where pytestmark does not apply. +with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message=DEPRECATION_MATCH, category=DeprecationWarning) + codecs = [Pickle(protocol=i) for i in range(5)] # object array with strings # object array with mix strings / nans @@ -70,6 +81,41 @@ def test_backwards_compatibility(): check_backwards_compatibility(Pickle.codec_id, arrays, codecs) +@pytest.mark.parametrize( + "make_codec", + [ + partial(Pickle), + partial(numcodecs.Pickle, protocol=2), + partial(Pickle.from_config, {"protocol": 3}), + partial(get_codec, {"id": "pickle"}), + partial(get_codec, {"id": "pickle", "protocol": 4}), + ], + ids=["Pickle", "numcodecs.Pickle", "Pickle.from_config", "get_codec", "get_codec-protocol"], +) +def test_construction_warns(make_codec): + """Every way of obtaining a Pickle codec warns once, attributes the warning to the + caller rather than to numcodecs internals, and leaves encode and decode working.""" + with pytest.warns(DeprecationWarning, match=DEPRECATION_MATCH) as record: + codec = make_codec() + assert len(record) == 1 + assert record[0].filename == __file__ + message = str(record[0].message) + assert _REMOVAL_VERSION in message + for alternative in ("VLenUTF8", "VLenBytes", "VLenArray", "JSON", "MsgPack"): + assert alternative in message + # encoding and decoding do not warn again + with warnings.catch_warnings(): + warnings.simplefilter("error") + check_encode_decode_array(arrays[0], codec) + + +def test_construction_raises_when_deprecation_warnings_are_errors(): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + with pytest.raises(DeprecationWarning, match=DEPRECATION_MATCH): + get_codec({"id": "pickle"}) + + def test_decode_refused_without_opt_in(monkeypatch): monkeypatch.setattr(numcodecs.pickles, "ALLOW_UNSAFE_PICKLE_DECODE", False) # get_codec is the path by which array metadata selects this codec diff --git a/tests/typing/test_object_codec_types.py b/tests/typing/test_object_codec_types.py index 05759a08..ddc01847 100644 --- a/tests/typing/test_object_codec_types.py +++ b/tests/typing/test_object_codec_types.py @@ -4,6 +4,7 @@ import numpy as np import numpy.typing as npt +import pytest from numcodecs import pickles from numcodecs.abc import ArrayBytesCodec, CodecConfig @@ -30,6 +31,7 @@ def test_json() -> None: assert assert_type(config["separators"], tuple[str, str]) == (",", ":") +@pytest.mark.filterwarnings("ignore:The Pickle codec is deprecated:DeprecationWarning") def test_pickle() -> None: codec = Pickle() enc = assert_type(codec.encode([1, 2]), bytes)