Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/release.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,15 @@
* `import numcodecs` no longer fails when a zfpy post-release such as 1.0.1.post0 is
installed. zfpy versions are now parsed with `packaging`, which becomes a dependency,
and the `zfpy` extra now requires zfpy >= 1.0.1. {issue}`892`
* `decode(buf, out=out)` now returns `out` itself for every codec, as the `Codec`
docstring describes. Zlib, BZ2, LZMA, Base64, Shuffle, ZFPY, PCodec, the three VLen
codecs, and every codec built on `numcodecs.compat.ndarray_copy` returned a flattened view
of `out` instead, or a new ndarray when `out` was a `bytearray`. The decoded bytes were
already written into `out`; only the returned object changes.
* `JenkinsLookup3.decode` accepts any contiguous writable buffer as `out`, not only an
ndarray, and raises `ValueError` when `out` has the wrong size.
* `Fletcher32.decode` without `out` returns a `uint8` ndarray, like the other checksum
codecs, instead of a Cython memoryview slice.

(release_0.17.1)=

Expand Down
2 changes: 1 addition & 1 deletion src/numcodecs/base64.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def decode(self, buf, out=None):
# normalise inputs
buf = ensure_contiguous_ndarray(buf)
if out is not None:
out = ensure_contiguous_ndarray(out)
ensure_contiguous_ndarray(out) # validate; decode into `out` itself below
# do decompression
decompressed = _base64.standard_b64decode(buf)
# handle destination
Expand Down
2 changes: 1 addition & 1 deletion src/numcodecs/bz2.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def decode(self, buf, out=None):
# normalise inputs
buf = ensure_contiguous_ndarray(buf)
if out is not None:
out = ensure_contiguous_ndarray(out)
ensure_contiguous_ndarray(out) # validate; decode into `out` itself below

# N.B., bz2 cannot handle ndarray directly because of truth testing issues
buf = memoryview(buf)
Expand Down
7 changes: 6 additions & 1 deletion src/numcodecs/checksum32.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,12 @@ def decode(self, buf, out=None):
"This could be a sign that the data has been corrupted."
)
if out is not None:
out.view("uint8")[:] = b[:-4]
out_view = ensure_contiguous_ndarray(out).view("uint8")
if out_view.nbytes != b.nbytes - 4:
raise ValueError(
f"out has {out_view.nbytes} bytes, but the decoded data has {b.nbytes - 4}"
)
out_view[:] = b[:-4]
return out
return memoryview(b[:-4])

Expand Down
10 changes: 8 additions & 2 deletions src/numcodecs/compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,12 +175,18 @@ def ensure_text(s, encoding="utf-8"):


def ndarray_copy(src, dst) -> NDArrayLike:
"""Copy the contents of the array from `src` to `dst`."""
"""Copy the contents of the array from `src` to `dst`, and return `dst`.

If `dst` is `None`, return `src` unchanged. Otherwise `dst` itself is returned, not
a view of it, so callers that pass a `bytearray` get that `bytearray` back.
"""

if dst is None:
# no-op
return src

out = dst

# ensure ndarray like
src = ensure_ndarray_like(src)
dst = ensure_ndarray_like(dst)
Expand All @@ -203,4 +209,4 @@ def ndarray_copy(src, dst) -> NDArrayLike:
# copy via numpy
np.copyto(dst, src)

return dst
return out
2 changes: 1 addition & 1 deletion src/numcodecs/fletcher32.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -112,5 +112,5 @@ class Fletcher32(Codec):
out_ptr = &out_mv[0]
memcpy(out_ptr, b_ptr, b_len - FOOTER_LENGTH)
else:
out = b_mv[:-FOOTER_LENGTH]
out = b[:-FOOTER_LENGTH]
return out
2 changes: 1 addition & 1 deletion src/numcodecs/lzma.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def decode(self, buf, out=None):
# normalise inputs
buf = ensure_contiguous_ndarray(buf)
if out is not None:
out = ensure_contiguous_ndarray(out)
ensure_contiguous_ndarray(out) # validate; decode into `out` itself below

# do decompression
dec = _lzma.decompress(buf, format=self.format, filters=self.filters)
Expand Down
3 changes: 1 addition & 2 deletions src/numcodecs/pcodec.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,7 @@ def encode(self, buf):
def decode(self, buf, out=None):
buf = ensure_bytes(buf)
if out is not None:
out = ensure_contiguous_ndarray(out)
standalone.simple_decompress_into(buf, out)
standalone.simple_decompress_into(buf, ensure_contiguous_ndarray(out))
return out
else:
return standalone.simple_decompress(buf)
20 changes: 8 additions & 12 deletions src/numcodecs/shuffle.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,24 +38,20 @@ def _prepare_arrays(self, buf, out):
return buf, out

def encode(self, buf, out=None):
buf, out = self._prepare_arrays(buf, out)
buf, out_arr = self._prepare_arrays(buf, out)

if self.elementsize <= 1:
return out # no shuffling needed

_doShuffle(buf.view("uint8"), out.view("uint8"), self.elementsize)
if self.elementsize > 1:
_doShuffle(buf.view("uint8"), out_arr.view("uint8"), self.elementsize)

return out
return out_arr if out is None else out

def decode(self, buf, out=None):
buf, out = self._prepare_arrays(buf, out)

if self.elementsize <= 1:
return out # no shuffling needed
buf, out_arr = self._prepare_arrays(buf, out)

_doUnshuffle(buf.view("uint8"), out.view("uint8"), self.elementsize)
if self.elementsize > 1:
_doUnshuffle(buf.view("uint8"), out_arr.view("uint8"), self.elementsize)

return out
return out_arr if out is None else out

def __repr__(self):
return f'{type(self).__name__}(elementsize={self.elementsize})'
24 changes: 12 additions & 12 deletions src/numcodecs/vlen.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -154,11 +154,11 @@ class VLenUTF8(Codec):
# load number of items
n_items = load_le32(<uint8_t*>data)

# setup output
# setup output: fill a flat view, but return the caller's own `out`
if out is not None:
out = check_out_param(out, n_items)
out_flat = check_out_param(out, n_items)
else:
out = np.empty(n_items, dtype=object)
out = out_flat = np.empty(n_items, dtype=object)

# iterate and decode - N.B., do not try to cast `out` as object[:]
# as this causes segfaults, possibly similar to
Expand All @@ -171,7 +171,7 @@ class VLenUTF8(Codec):
data += HEADER_LENGTH
if data + L > data_end:
raise ValueError('corrupt buffer, data seem truncated')
out[i] = PyUnicode_FromStringAndSize(data, L)
out_flat[i] = PyUnicode_FromStringAndSize(data, L)
data += L

return out
Expand Down Expand Up @@ -283,11 +283,11 @@ class VLenBytes(Codec):
# load number of items
n_items = load_le32(<uint8_t*>data)

# setup output
# setup output: fill a flat view, but return the caller's own `out`
if out is not None:
out = check_out_param(out, n_items)
out_flat = check_out_param(out, n_items)
else:
out = np.empty(n_items, dtype=object)
out = out_flat = np.empty(n_items, dtype=object)

# iterate and decode - N.B., do not try to cast `out` as object[:]
# as this causes segfaults, possibly similar to
Expand All @@ -300,7 +300,7 @@ class VLenBytes(Codec):
data += HEADER_LENGTH
if data + L > data_end:
raise ValueError('corrupt buffer, data seem truncated')
out[i] = PyBytes_FromStringAndSize(data, L)
out_flat[i] = PyBytes_FromStringAndSize(data, L)
data += L

return out
Expand Down Expand Up @@ -439,11 +439,11 @@ class VLenArray(Codec):
# load number of items
n_items = load_le32(<uint8_t*>data)

# setup output
# setup output: fill a flat view, but return the caller's own `out`
if out is not None:
out = check_out_param(out, n_items)
out_flat = check_out_param(out, n_items)
else:
out = np.empty(n_items, dtype=object)
out = out_flat = np.empty(n_items, dtype=object)

# iterate and decode - N.B., do not try to cast `out` as object[:]
# as this causes segfaults, possibly similar to
Expand All @@ -463,7 +463,7 @@ class VLenArray(Codec):
v_pb = PyMemoryView_GET_BUFFER(v_mv)
memcpy(v_pb.buf, data, L)

out[i] = v
out_flat[i] = v
data += L

return out
2 changes: 1 addition & 1 deletion src/numcodecs/zfpy.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ def decode(self, buf, out=None):
# normalise inputs
buf = ensure_bytes(buf)
if out is not None:
out = ensure_contiguous_ndarray(out)
ensure_contiguous_ndarray(out) # validate; decode into `out` itself below

# do decompression
dec = _zfpy.decompress_numpy(buf)
Expand Down
2 changes: 1 addition & 1 deletion src/numcodecs/zlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def decode(self, buf, out=None):
# normalise inputs
buf = ensure_contiguous_ndarray(buf)
if out is not None:
out = ensure_contiguous_ndarray(out)
ensure_contiguous_ndarray(out) # validate; decode into `out` itself below

# do decompression
dec = _zlib.decompress(buf)
Expand Down
32 changes: 2 additions & 30 deletions tests/test_codec_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,36 +85,8 @@ def is_object(self) -> bool:

# Known violations of the `out` contract. Each entry is a bug; remove it when fixed.
# `strict=True` makes a fixed entry fail, so this list cannot go stale.
OUT_NDARRAY_XFAIL: set[str] = {
"zlib",
"bz2",
"lzma",
"base64",
"shuffle",
"zfpy",
"pcodec",
"vlen-utf8",
"vlen-bytes",
"vlen-array",
}
OUT_BYTEARRAY_XFAIL: set[str] = {
"zlib",
"bz2",
"lzma",
"base64",
"shuffle",
"crc32",
"adler32",
"crc32c",
"jenkins_lookup3",
"astype",
"delta",
"quantize",
"fixedscaleoffset",
"bitround",
"packbits",
"zfpy",
}
OUT_NDARRAY_XFAIL: set[str] = set()
OUT_BYTEARRAY_XFAIL: set[str] = set()
DECODE_RETURNS_BUFFER_XFAIL: set[str] = set()


Expand Down
7 changes: 7 additions & 0 deletions tests/test_jenkins.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,3 +148,10 @@ def test_out():
j = JenkinsLookup3()
result = j.encode(data)
j.decode(result, out=data)


def test_out_wrong_size():
j = JenkinsLookup3()
enc = j.encode(np.arange(8, dtype="uint8"))
with pytest.raises(ValueError, match="out has 4 bytes, but the decoded data has 8"):
j.decode(enc, out=bytearray(4))
Loading