Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions docs/other/pickles.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,22 @@
# Pickle

```{admonition} Deprecated
:class: warning

The `Pickle` codec is deprecated and will be removed in numcodecs 0.19.0. Constructing it,
directly or through `numcodecs.get_codec({"id": "pickle"})`, emits a `DeprecationWarning`.
Pickled data can only be read by Python, and decoding it can execute arbitrary code. To encode
arrays of Python objects, use one of these codecs instead:

- [`VLenUTF8`](vlen.md) for strings
- [`VLenBytes`](vlen.md) for bytes
- [`VLenArray`](vlen.md) for variable-length numeric arrays
- [`JSON`](json.md) or [`MsgPack`](msgpacks.md) for mixed objects

Data written with `Pickle` can still be read until the codec is removed, subject to the
decoding opt-in described below.
```

```{warning}
Decoding pickled data can execute arbitrary code. `Pickle.decode` raises
`numcodecs.errors.UnsafeDecodeError` unless the process opts in, either by setting
Expand Down
10 changes: 10 additions & 0 deletions docs/release.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,16 @@

## Unreleased

### Deprecations

* The `Pickle` codec is deprecated and will be removed in numcodecs 0.19.0. Constructing
it, directly or through `numcodecs.get_codec({"id": "pickle"})`, emits a
`DeprecationWarning`. Encoding and decoding still work until removal, and decoding still
requires the opt-in described in
[GHSA-jqc8-qjfv-5m6r](https://github.com/zarr-developers/numcodecs/security/advisories/GHSA-jqc8-qjfv-5m6r).
To encode arrays of Python objects, use `VLenUTF8`, `VLenBytes`, `VLenArray`, `JSON` or
`MsgPack` instead. {issue}`706`

### Maintenance

* Build wheels for CPython 3.15 and free-threaded CPython 3.15t, using cibuildwheel 4.3.5,
Expand Down
40 changes: 38 additions & 2 deletions src/numcodecs/pickles.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import os
import pickle
import warnings
from typing import Any, overload

import numpy as np
Expand All @@ -25,6 +26,22 @@ def _read_opt_in_from_env() -> bool:
#: because codec parameters are read from array metadata, which may be untrusted.
ALLOW_UNSAFE_PICKLE_DECODE = _read_opt_in_from_env()

# The release in which the `Pickle` codec is planned to be removed.
_REMOVAL_VERSION = "0.19.0"

_DEPRECATION_MESSAGE = (
f"The Pickle codec is deprecated and will be removed in numcodecs {_REMOVAL_VERSION}. "
"Decoding pickled data can execute arbitrary code, and pickled data can only be read "
"by Python. To encode arrays of Python objects, use VLenUTF8 for strings, VLenBytes "
"for bytes, VLenArray for variable-length numeric arrays, or JSON or MsgPack for "
"mixed objects."
)

# Frames in files under this directory are skipped when attributing the deprecation
# warning, so it points at the first caller outside numcodecs (user code, or the
# library that selected the codec from array metadata) rather than at numcodecs itself.
_NUMCODECS_DIR = os.path.dirname(os.path.abspath(__file__)) + os.sep


class PickleConfig(CodecConfig):
"""The configuration of a `Pickle` codec, as returned by `Pickle.get_config`."""
Expand All @@ -36,6 +53,12 @@ class Pickle(Codec):
"""Codec to encode data as as pickled bytes. Useful for encoding an array of Python string
objects.

This codec is deprecated and will be removed in numcodecs 0.19.0. Constructing it, directly
or through `numcodecs.get_codec`, emits a `DeprecationWarning`. To encode arrays of Python
objects, use `numcodecs.vlen.VLenUTF8` for strings, `numcodecs.vlen.VLenBytes` for bytes,
`numcodecs.vlen.VLenArray` for variable-length numeric arrays, or `numcodecs.json.JSON`
or `numcodecs.msgpacks.MsgPack` for mixed objects.

.. warning::
Decoding pickled data can execute arbitrary code. :meth:`decode` refuses to run
unless the process opts in by setting
Expand All @@ -51,26 +74,39 @@ class Pickle(Codec):
protocol : int, defaults to pickle.HIGHEST_PROTOCOL
The protocol used to pickle data.

Warns
-----
DeprecationWarning
Always, on construction. The warning is attributed to the first caller outside
numcodecs.

Examples
--------
>>> import warnings
>>> import numcodecs as codecs
>>> import numpy as np
>>> x = np.array(['foo', 'bar', 'baz'], dtype='object')
>>> f = codecs.Pickle()
>>> with warnings.catch_warnings():
... warnings.simplefilter("ignore", DeprecationWarning)
... f = codecs.Pickle()
>>> codecs.pickles.ALLOW_UNSAFE_PICKLE_DECODE = True # only for trusted data
>>> f.decode(f.encode(x))
array(['foo', 'bar', 'baz'], dtype=object)
>>> codecs.pickles.ALLOW_UNSAFE_PICKLE_DECODE = False

See Also
--------
numcodecs.msgpacks.MsgPack
numcodecs.vlen.VLenUTF8, numcodecs.vlen.VLenBytes, numcodecs.vlen.VLenArray,
numcodecs.json.JSON, numcodecs.msgpacks.MsgPack

"""

codec_id = 'pickle'

def __init__(self, protocol: int = pickle.HIGHEST_PROTOCOL) -> None:
warnings.warn(
_DEPRECATION_MESSAGE, DeprecationWarning, skip_file_prefixes=(_NUMCODECS_DIR,)
)
self.protocol = protocol

def encode(self, buf: object) -> bytes:
Expand Down
2 changes: 2 additions & 0 deletions tests/test_codec_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,8 @@ def test_decode_into_bytearray_returns_out(codec_id: str) -> None:
assert bytes(out) == ensure_ndarray(expected).tobytes()


# Rebuilding the codec from its config constructs it again, outside `_codec`.
@pytest.mark.filterwarnings("ignore:The Pickle codec is deprecated:DeprecationWarning")
@pytest.mark.parametrize("codec_id", _params(set()))
def test_config_round_trip(codec_id: str) -> None:
codec = _codec(codec_id)
Expand Down
50 changes: 48 additions & 2 deletions tests/test_pickles.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
import itertools
import sys
import warnings
from functools import partial

import numpy as np
import pytest

import numcodecs
import numcodecs.pickles
from numcodecs.errors import UnsafeDecodeError
from numcodecs.pickles import _ENV_VAR, Pickle, _read_opt_in_from_env
from numcodecs.pickles import _ENV_VAR, _REMOVAL_VERSION, Pickle, _read_opt_in_from_env
from numcodecs.registry import get_codec
from tests.common import (
check_backwards_compatibility,
Expand All @@ -16,7 +19,15 @@
greetings,
)

codecs = [Pickle(protocol=i) for i in range(5)]
# Constructing Pickle emits a DeprecationWarning. The tests below that predate the
# deprecation exercise the codec itself, so they ignore it; test_construction_warns checks it.
DEPRECATION_MATCH = "The Pickle codec is deprecated"
pytestmark = pytest.mark.filterwarnings(f"ignore:{DEPRECATION_MATCH}:DeprecationWarning")

# Built at import time, where pytestmark does not apply.
with warnings.catch_warnings():
warnings.filterwarnings("ignore", message=DEPRECATION_MATCH, category=DeprecationWarning)
codecs = [Pickle(protocol=i) for i in range(5)]

# object array with strings
# object array with mix strings / nans
Expand Down Expand Up @@ -70,6 +81,41 @@ def test_backwards_compatibility():
check_backwards_compatibility(Pickle.codec_id, arrays, codecs)


@pytest.mark.parametrize(
"make_codec",
[
partial(Pickle),
partial(numcodecs.Pickle, protocol=2),
partial(Pickle.from_config, {"protocol": 3}),
partial(get_codec, {"id": "pickle"}),
partial(get_codec, {"id": "pickle", "protocol": 4}),
],
ids=["Pickle", "numcodecs.Pickle", "Pickle.from_config", "get_codec", "get_codec-protocol"],
)
def test_construction_warns(make_codec):
"""Every way of obtaining a Pickle codec warns once, attributes the warning to the
caller rather than to numcodecs internals, and leaves encode and decode working."""
with pytest.warns(DeprecationWarning, match=DEPRECATION_MATCH) as record:
codec = make_codec()
assert len(record) == 1
assert record[0].filename == __file__
message = str(record[0].message)
assert _REMOVAL_VERSION in message
for alternative in ("VLenUTF8", "VLenBytes", "VLenArray", "JSON", "MsgPack"):
assert alternative in message
# encoding and decoding do not warn again
with warnings.catch_warnings():
warnings.simplefilter("error")
check_encode_decode_array(arrays[0], codec)


def test_construction_raises_when_deprecation_warnings_are_errors():
with warnings.catch_warnings():
warnings.simplefilter("error", DeprecationWarning)
with pytest.raises(DeprecationWarning, match=DEPRECATION_MATCH):
get_codec({"id": "pickle"})


def test_decode_refused_without_opt_in(monkeypatch):
monkeypatch.setattr(numcodecs.pickles, "ALLOW_UNSAFE_PICKLE_DECODE", False)
# get_codec is the path by which array metadata selects this codec
Expand Down
2 changes: 2 additions & 0 deletions tests/typing/test_object_codec_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import numpy as np
import numpy.typing as npt
import pytest

from numcodecs import pickles
from numcodecs.abc import ArrayBytesCodec, CodecConfig
Expand All @@ -30,6 +31,7 @@ def test_json() -> None:
assert assert_type(config["separators"], tuple[str, str]) == (",", ":")


@pytest.mark.filterwarnings("ignore:The Pickle codec is deprecated:DeprecationWarning")
def test_pickle() -> None:
codec = Pickle()
enc = assert_type(codec.encode([1, 2]), bytes)
Expand Down
Loading