diff --git a/docs/release.md b/docs/release.md index 6847e8ea..51e2933b 100644 --- a/docs/release.md +++ b/docs/release.md @@ -38,6 +38,15 @@ * `import numcodecs` no longer fails when a zfpy post-release such as 1.0.1.post0 is installed. zfpy versions are now parsed with `packaging`, which becomes a dependency, and the `zfpy` extra now requires zfpy >= 1.0.1. {issue}`892` +* `decode(buf, out=out)` now returns `out` itself for every codec, as the `Codec` + docstring describes. Zlib, BZ2, LZMA, Base64, Shuffle, ZFPY, PCodec, the three VLen + codecs, and every codec built on `numcodecs.compat.ndarray_copy` returned a flattened view + of `out` instead, or a new ndarray when `out` was a `bytearray`. The decoded bytes were + already written into `out`; only the returned object changes. +* `JenkinsLookup3.decode` accepts any contiguous writable buffer as `out`, not only an + ndarray, and raises `ValueError` when `out` has the wrong size. +* `Fletcher32.decode` without `out` returns a `uint8` ndarray, like the other checksum + codecs, instead of a Cython memoryview slice. (release_0.17.1)= diff --git a/src/numcodecs/base64.py b/src/numcodecs/base64.py index 307c6d52..c107d5c7 100644 --- a/src/numcodecs/base64.py +++ b/src/numcodecs/base64.py @@ -19,7 +19,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression decompressed = _base64.standard_b64decode(buf) # handle destination diff --git a/src/numcodecs/bz2.py b/src/numcodecs/bz2.py index 2b63b147..a3c8f5ec 100644 --- a/src/numcodecs/bz2.py +++ b/src/numcodecs/bz2.py @@ -31,7 +31,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # N.B., bz2 cannot handle ndarray directly because of truth testing issues buf = memoryview(buf) diff --git a/src/numcodecs/checksum32.py b/src/numcodecs/checksum32.py index b02c0a6a..a095b171 100644 --- a/src/numcodecs/checksum32.py +++ b/src/numcodecs/checksum32.py @@ -185,7 +185,12 @@ def decode(self, buf, out=None): "This could be a sign that the data has been corrupted." ) if out is not None: - out.view("uint8")[:] = b[:-4] + out_view = ensure_contiguous_ndarray(out).view("uint8") + if out_view.nbytes != b.nbytes - 4: + raise ValueError( + f"out has {out_view.nbytes} bytes, but the decoded data has {b.nbytes - 4}" + ) + out_view[:] = b[:-4] return out return memoryview(b[:-4]) diff --git a/src/numcodecs/compat.py b/src/numcodecs/compat.py index d1844e10..cc4d2d13 100644 --- a/src/numcodecs/compat.py +++ b/src/numcodecs/compat.py @@ -175,12 +175,18 @@ def ensure_text(s, encoding="utf-8"): def ndarray_copy(src, dst) -> NDArrayLike: - """Copy the contents of the array from `src` to `dst`.""" + """Copy the contents of the array from `src` to `dst`, and return `dst`. + + If `dst` is `None`, return `src` unchanged. Otherwise `dst` itself is returned, not + a view of it, so callers that pass a `bytearray` get that `bytearray` back. + """ if dst is None: # no-op return src + out = dst + # ensure ndarray like src = ensure_ndarray_like(src) dst = ensure_ndarray_like(dst) @@ -203,4 +209,4 @@ def ndarray_copy(src, dst) -> NDArrayLike: # copy via numpy np.copyto(dst, src) - return dst + return out diff --git a/src/numcodecs/fletcher32.pyx b/src/numcodecs/fletcher32.pyx index 1ecbb45d..ab9ce6a8 100644 --- a/src/numcodecs/fletcher32.pyx +++ b/src/numcodecs/fletcher32.pyx @@ -112,5 +112,5 @@ class Fletcher32(Codec): out_ptr = &out_mv[0] memcpy(out_ptr, b_ptr, b_len - FOOTER_LENGTH) else: - out = b_mv[:-FOOTER_LENGTH] + out = b[:-FOOTER_LENGTH] return out diff --git a/src/numcodecs/lzma.py b/src/numcodecs/lzma.py index 7b6b2348..97da0282 100644 --- a/src/numcodecs/lzma.py +++ b/src/numcodecs/lzma.py @@ -59,7 +59,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression dec = _lzma.decompress(buf, format=self.format, filters=self.filters) diff --git a/src/numcodecs/pcodec.py b/src/numcodecs/pcodec.py index c7108c7b..739666b1 100644 --- a/src/numcodecs/pcodec.py +++ b/src/numcodecs/pcodec.py @@ -122,8 +122,7 @@ def encode(self, buf): def decode(self, buf, out=None): buf = ensure_bytes(buf) if out is not None: - out = ensure_contiguous_ndarray(out) - standalone.simple_decompress_into(buf, out) + standalone.simple_decompress_into(buf, ensure_contiguous_ndarray(out)) return out else: return standalone.simple_decompress(buf) diff --git a/src/numcodecs/shuffle.py b/src/numcodecs/shuffle.py index 0e7031f5..e29e5a59 100644 --- a/src/numcodecs/shuffle.py +++ b/src/numcodecs/shuffle.py @@ -38,24 +38,20 @@ def _prepare_arrays(self, buf, out): return buf, out def encode(self, buf, out=None): - buf, out = self._prepare_arrays(buf, out) + buf, out_arr = self._prepare_arrays(buf, out) - if self.elementsize <= 1: - return out # no shuffling needed - - _doShuffle(buf.view("uint8"), out.view("uint8"), self.elementsize) + if self.elementsize > 1: + _doShuffle(buf.view("uint8"), out_arr.view("uint8"), self.elementsize) - return out + return out_arr if out is None else out def decode(self, buf, out=None): - buf, out = self._prepare_arrays(buf, out) - - if self.elementsize <= 1: - return out # no shuffling needed + buf, out_arr = self._prepare_arrays(buf, out) - _doUnshuffle(buf.view("uint8"), out.view("uint8"), self.elementsize) + if self.elementsize > 1: + _doUnshuffle(buf.view("uint8"), out_arr.view("uint8"), self.elementsize) - return out + return out_arr if out is None else out def __repr__(self): return f'{type(self).__name__}(elementsize={self.elementsize})' diff --git a/src/numcodecs/vlen.pyx b/src/numcodecs/vlen.pyx index f51e8cad..424cc7ee 100644 --- a/src/numcodecs/vlen.pyx +++ b/src/numcodecs/vlen.pyx @@ -154,11 +154,11 @@ class VLenUTF8(Codec): # load number of items n_items = load_le32(data) - # setup output + # setup output: fill a flat view, but return the caller's own `out` if out is not None: - out = check_out_param(out, n_items) + out_flat = check_out_param(out, n_items) else: - out = np.empty(n_items, dtype=object) + out = out_flat = np.empty(n_items, dtype=object) # iterate and decode - N.B., do not try to cast `out` as object[:] # as this causes segfaults, possibly similar to @@ -171,7 +171,7 @@ class VLenUTF8(Codec): data += HEADER_LENGTH if data + L > data_end: raise ValueError('corrupt buffer, data seem truncated') - out[i] = PyUnicode_FromStringAndSize(data, L) + out_flat[i] = PyUnicode_FromStringAndSize(data, L) data += L return out @@ -283,11 +283,11 @@ class VLenBytes(Codec): # load number of items n_items = load_le32(data) - # setup output + # setup output: fill a flat view, but return the caller's own `out` if out is not None: - out = check_out_param(out, n_items) + out_flat = check_out_param(out, n_items) else: - out = np.empty(n_items, dtype=object) + out = out_flat = np.empty(n_items, dtype=object) # iterate and decode - N.B., do not try to cast `out` as object[:] # as this causes segfaults, possibly similar to @@ -300,7 +300,7 @@ class VLenBytes(Codec): data += HEADER_LENGTH if data + L > data_end: raise ValueError('corrupt buffer, data seem truncated') - out[i] = PyBytes_FromStringAndSize(data, L) + out_flat[i] = PyBytes_FromStringAndSize(data, L) data += L return out @@ -439,11 +439,11 @@ class VLenArray(Codec): # load number of items n_items = load_le32(data) - # setup output + # setup output: fill a flat view, but return the caller's own `out` if out is not None: - out = check_out_param(out, n_items) + out_flat = check_out_param(out, n_items) else: - out = np.empty(n_items, dtype=object) + out = out_flat = np.empty(n_items, dtype=object) # iterate and decode - N.B., do not try to cast `out` as object[:] # as this causes segfaults, possibly similar to @@ -463,7 +463,7 @@ class VLenArray(Codec): v_pb = PyMemoryView_GET_BUFFER(v_mv) memcpy(v_pb.buf, data, L) - out[i] = v + out_flat[i] = v data += L return out diff --git a/src/numcodecs/zfpy.py b/src/numcodecs/zfpy.py index 41ccfe98..86fb090a 100644 --- a/src/numcodecs/zfpy.py +++ b/src/numcodecs/zfpy.py @@ -103,7 +103,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_bytes(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression dec = _zfpy.decompress_numpy(buf) diff --git a/src/numcodecs/zlib.py b/src/numcodecs/zlib.py index ca247af2..e62658dc 100644 --- a/src/numcodecs/zlib.py +++ b/src/numcodecs/zlib.py @@ -31,7 +31,7 @@ def decode(self, buf, out=None): # normalise inputs buf = ensure_contiguous_ndarray(buf) if out is not None: - out = ensure_contiguous_ndarray(out) + ensure_contiguous_ndarray(out) # validate; decode into `out` itself below # do decompression dec = _zlib.decompress(buf) diff --git a/tests/test_codec_contract.py b/tests/test_codec_contract.py index c328a25c..bf47cc00 100644 --- a/tests/test_codec_contract.py +++ b/tests/test_codec_contract.py @@ -85,36 +85,8 @@ def is_object(self) -> bool: # Known violations of the `out` contract. Each entry is a bug; remove it when fixed. # `strict=True` makes a fixed entry fail, so this list cannot go stale. -OUT_NDARRAY_XFAIL: set[str] = { - "zlib", - "bz2", - "lzma", - "base64", - "shuffle", - "zfpy", - "pcodec", - "vlen-utf8", - "vlen-bytes", - "vlen-array", -} -OUT_BYTEARRAY_XFAIL: set[str] = { - "zlib", - "bz2", - "lzma", - "base64", - "shuffle", - "crc32", - "adler32", - "crc32c", - "jenkins_lookup3", - "astype", - "delta", - "quantize", - "fixedscaleoffset", - "bitround", - "packbits", - "zfpy", -} +OUT_NDARRAY_XFAIL: set[str] = set() +OUT_BYTEARRAY_XFAIL: set[str] = set() DECODE_RETURNS_BUFFER_XFAIL: set[str] = set() diff --git a/tests/test_jenkins.py b/tests/test_jenkins.py index 40d972bf..511c435c 100644 --- a/tests/test_jenkins.py +++ b/tests/test_jenkins.py @@ -148,3 +148,10 @@ def test_out(): j = JenkinsLookup3() result = j.encode(data) j.decode(result, out=data) + + +def test_out_wrong_size(): + j = JenkinsLookup3() + enc = j.encode(np.arange(8, dtype="uint8")) + with pytest.raises(ValueError, match="out has 4 bytes, but the decoded data has 8"): + j.decode(enc, out=bytearray(4))