From 7478240822d23db2780c0a512c6701ef772063d3 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 21 Nov 2026 13:16:38 +0100 Subject: [PATCH] feat(pickle): deprecate the Pickle codec Constructing Pickle, directly or through get_codec({"id": "pickle"}), now emits a DeprecationWarning naming the planned removal in 0.19.0 and the codecs to use for object data instead (VLenUTF8, VLenBytes, VLenArray, JSON, MsgPack). The warning is attributed to the first caller outside numcodecs, so it points at user code or at the library that selected the codec from array metadata. Encoding and decoding are unchanged, and decoding still requires the opt-in from GHSA-jqc8-qjfv-5m6r. Refs: #706 Assisted-by: ClaudeCode:claude-opus-5.5 --- docs/other/pickles.md | 17 ++++++++++++++ docs/release.md | 10 ++++++++ src/numcodecs/pickles.py | 40 ++++++++++++++++++++++++++++++-- tests/test_pickles.py | 50 ++++++++++++++++++++++++++++++++++++++-- 4 files changed, 113 insertions(+), 4 deletions(-) diff --git a/docs/other/pickles.md b/docs/other/pickles.md index 3e27c11d..c637cacc 100644 --- a/docs/other/pickles.md +++ b/docs/other/pickles.md @@ -1,5 +1,22 @@ # Pickle +```{admonition} Deprecated +:class: warning + +The `Pickle` codec is deprecated and will be removed in numcodecs 0.19.0. Constructing it, +directly or through `numcodecs.get_codec({"id": "pickle"})`, emits a `DeprecationWarning`. +Pickled data can only be read by Python, and decoding it can execute arbitrary code. To encode +arrays of Python objects, use one of these codecs instead: + +- [`VLenUTF8`](vlen.md) for strings +- [`VLenBytes`](vlen.md) for bytes +- [`VLenArray`](vlen.md) for variable-length numeric arrays +- [`JSON`](json.md) or [`MsgPack`](msgpacks.md) for mixed objects + +Data written with `Pickle` can still be read until the codec is removed, subject to the +decoding opt-in described below. +``` + ```{warning} Decoding pickled data can execute arbitrary code. `Pickle.decode` raises `numcodecs.errors.UnsafeDecodeError` unless the process opts in, either by setting diff --git a/docs/release.md b/docs/release.md index b5474767..e6f8b109 100644 --- a/docs/release.md +++ b/docs/release.md @@ -14,6 +14,16 @@ ## Unreleased +### Deprecations + +* The `Pickle` codec is deprecated and will be removed in numcodecs 0.19.0. Constructing + it, directly or through `numcodecs.get_codec({"id": "pickle"})`, emits a + `DeprecationWarning`. Encoding and decoding still work until removal, and decoding still + requires the opt-in described in + [GHSA-jqc8-qjfv-5m6r](https://github.com/zarr-developers/numcodecs/security/advisories/GHSA-jqc8-qjfv-5m6r). + To encode arrays of Python objects, use `VLenUTF8`, `VLenBytes`, `VLenArray`, `JSON` or + `MsgPack` instead. {issue}`706` + ### Maintenance * Build wheels for CPython 3.15 and free-threaded CPython 3.15t, using cibuildwheel 4.3.5, diff --git a/src/numcodecs/pickles.py b/src/numcodecs/pickles.py index 51579f2e..60dcd38e 100644 --- a/src/numcodecs/pickles.py +++ b/src/numcodecs/pickles.py @@ -1,5 +1,6 @@ import os import pickle +import warnings import numpy as np @@ -23,11 +24,33 @@ def _read_opt_in_from_env() -> bool: #: because codec parameters are read from array metadata, which may be untrusted. ALLOW_UNSAFE_PICKLE_DECODE = _read_opt_in_from_env() +# The release in which the `Pickle` codec is planned to be removed. +_REMOVAL_VERSION = "0.19.0" + +_DEPRECATION_MESSAGE = ( + f"The Pickle codec is deprecated and will be removed in numcodecs {_REMOVAL_VERSION}. " + "Decoding pickled data can execute arbitrary code, and pickled data can only be read " + "by Python. To encode arrays of Python objects, use VLenUTF8 for strings, VLenBytes " + "for bytes, VLenArray for variable-length numeric arrays, or JSON or MsgPack for " + "mixed objects." +) + +# Frames in files under this directory are skipped when attributing the deprecation +# warning, so it points at the first caller outside numcodecs (user code, or the +# library that selected the codec from array metadata) rather than at numcodecs itself. +_NUMCODECS_DIR = os.path.dirname(os.path.abspath(__file__)) + os.sep + class Pickle(Codec): """Codec to encode data as as pickled bytes. Useful for encoding an array of Python string objects. + This codec is deprecated and will be removed in numcodecs 0.19.0. Constructing it, directly + or through `numcodecs.get_codec`, emits a `DeprecationWarning`. To encode arrays of Python + objects, use `numcodecs.vlen.VLenUTF8` for strings, `numcodecs.vlen.VLenBytes` for bytes, + `numcodecs.vlen.VLenArray` for variable-length numeric arrays, or `numcodecs.json.JSON` + or `numcodecs.msgpacks.MsgPack` for mixed objects. + .. warning:: Decoding pickled data can execute arbitrary code. :meth:`decode` refuses to run unless the process opts in by setting @@ -43,12 +66,21 @@ class Pickle(Codec): protocol : int, defaults to pickle.HIGHEST_PROTOCOL The protocol used to pickle data. + Warns + ----- + DeprecationWarning + Always, on construction. The warning is attributed to the first caller outside + numcodecs. + Examples -------- + >>> import warnings >>> import numcodecs as codecs >>> import numpy as np >>> x = np.array(['foo', 'bar', 'baz'], dtype='object') - >>> f = codecs.Pickle() + >>> with warnings.catch_warnings(): + ... warnings.simplefilter("ignore", DeprecationWarning) + ... f = codecs.Pickle() >>> codecs.pickles.ALLOW_UNSAFE_PICKLE_DECODE = True # only for trusted data >>> f.decode(f.encode(x)) array(['foo', 'bar', 'baz'], dtype=object) @@ -56,13 +88,17 @@ class Pickle(Codec): See Also -------- - numcodecs.msgpacks.MsgPack + numcodecs.vlen.VLenUTF8, numcodecs.vlen.VLenBytes, numcodecs.vlen.VLenArray, + numcodecs.json.JSON, numcodecs.msgpacks.MsgPack """ codec_id = 'pickle' def __init__(self, protocol=pickle.HIGHEST_PROTOCOL): + warnings.warn( + _DEPRECATION_MESSAGE, DeprecationWarning, skip_file_prefixes=(_NUMCODECS_DIR,) + ) self.protocol = protocol def encode(self, buf): diff --git a/tests/test_pickles.py b/tests/test_pickles.py index 70c980eb..aa9e8671 100644 --- a/tests/test_pickles.py +++ b/tests/test_pickles.py @@ -1,12 +1,15 @@ import itertools import sys +import warnings +from functools import partial import numpy as np import pytest +import numcodecs import numcodecs.pickles from numcodecs.errors import UnsafeDecodeError -from numcodecs.pickles import _ENV_VAR, Pickle, _read_opt_in_from_env +from numcodecs.pickles import _ENV_VAR, _REMOVAL_VERSION, Pickle, _read_opt_in_from_env from numcodecs.registry import get_codec from tests.common import ( check_backwards_compatibility, @@ -16,7 +19,15 @@ greetings, ) -codecs = [Pickle(protocol=i) for i in range(5)] +# Constructing Pickle emits a DeprecationWarning. The tests below that predate the +# deprecation exercise the codec itself, so they ignore it; test_construction_warns checks it. +DEPRECATION_MATCH = "The Pickle codec is deprecated" +pytestmark = pytest.mark.filterwarnings(f"ignore:{DEPRECATION_MATCH}:DeprecationWarning") + +# Built at import time, where pytestmark does not apply. +with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message=DEPRECATION_MATCH, category=DeprecationWarning) + codecs = [Pickle(protocol=i) for i in range(5)] # object array with strings # object array with mix strings / nans @@ -70,6 +81,41 @@ def test_backwards_compatibility(): check_backwards_compatibility(Pickle.codec_id, arrays, codecs) +@pytest.mark.parametrize( + "make_codec", + [ + partial(Pickle), + partial(numcodecs.Pickle, protocol=2), + partial(Pickle.from_config, {"protocol": 3}), + partial(get_codec, {"id": "pickle"}), + partial(get_codec, {"id": "pickle", "protocol": 4}), + ], + ids=["Pickle", "numcodecs.Pickle", "Pickle.from_config", "get_codec", "get_codec-protocol"], +) +def test_construction_warns(make_codec): + """Every way of obtaining a Pickle codec warns once, attributes the warning to the + caller rather than to numcodecs internals, and leaves encode and decode working.""" + with pytest.warns(DeprecationWarning, match=DEPRECATION_MATCH) as record: + codec = make_codec() + assert len(record) == 1 + assert record[0].filename == __file__ + message = str(record[0].message) + assert _REMOVAL_VERSION in message + for alternative in ("VLenUTF8", "VLenBytes", "VLenArray", "JSON", "MsgPack"): + assert alternative in message + # encoding and decoding do not warn again + with warnings.catch_warnings(): + warnings.simplefilter("error") + check_encode_decode_array(arrays[0], codec) + + +def test_construction_raises_when_deprecation_warnings_are_errors(): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + with pytest.raises(DeprecationWarning, match=DEPRECATION_MATCH): + get_codec({"id": "pickle"}) + + def test_decode_refused_without_opt_in(monkeypatch): monkeypatch.setattr(numcodecs.pickles, "ALLOW_UNSAFE_PICKLE_DECODE", False) # get_codec is the path by which array metadata selects this codec