From fc58439ab2e336f2fff083c62f5053b07b885ede Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 17:46:10 +0100 Subject: [PATCH 01/11] fix(indexing): keep the last value when setitem gets repeated indices The indexers group selected items by chunk with an unstable argsort, so repeated indices reached the in-chunk assignment in an arbitrary order and the stored value depended on the platform, the chunk layout and the number of selected items. Sorting stably keeps repeats in the order given, and the in-chunk NumPy assignment then stores the last value. Refs zarr-developers/zarr-python#2855 Assisted-by: ClaudeCode:claude-fable-5.2 --- src/zarr/core/indexing.py | 10 +++++---- tests/test_indexing.py | 47 ++++++++++++++++++++++++++++++--------- tests/test_properties.py | 24 ++++---------------- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index 8ea59eb960..3f564023e6 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -828,8 +828,9 @@ def __init__( dim_out_sel = np.arange(nitems - 1, -1, -1) dim_sel_chunk_sorted = dim_sel_chunk[::-1] else: - # sort indices to group by chunk - dim_out_sel = np.argsort(dim_sel_chunk) + # sort indices to group by chunk; a stable sort keeps repeated indices in + # the order given, so the last value assigned to an item is the one stored + dim_out_sel = np.argsort(dim_sel_chunk, kind="stable") dim_sel = np.take(dim_sel, dim_out_sel) dim_sel_chunk_sorted = dim_sel_chunk[dim_out_sel] @@ -1343,8 +1344,9 @@ def __init__( # group points by chunk if np.any(np.diff(chunks_raveled_indices) < 0): - # optimisation, only sort if needed - sel_sort = np.argsort(chunks_raveled_indices) + # optimisation, only sort if needed; a stable sort keeps repeated points in + # the order given, so the last value assigned to a point is the one stored + sel_sort = np.argsort(chunks_raveled_indices, kind="stable") selection_broadcast = tuple(dim_sel[sel_sort] for dim_sel in selection_broadcast) chunks_raveled_indices = chunks_raveled_indices[sel_sort] else: diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 785f089513..a0931d00a8 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -507,17 +507,42 @@ def test_setitem_zarr_array_as_value() -> None: assert_array_equal(dst2[2:7], np.arange(2, 7)) -@pytest.mark.skip(reason="fails on ubuntu, windows; numpy=2.2; in CI") -def test_setitem_repeated_index(): - """oindex assignment with repeated indices writes the last value for each duplicated index position.""" - array = zarr.array(data=np.zeros((4,)), chunks=(1,)) - indexer = np.array([-1, -1, 0, 0]) - array.oindex[(indexer,)] = [0, 1, 2, 3] - np.testing.assert_array_equal(array[:], np.array([3, 0, 0, 1])) - - indexer = np.array([-1, 0, 0, -1]) - array.oindex[(indexer,)] = [0, 1, 2, 3] - np.testing.assert_array_equal(array[:], np.array([2, 0, 0, 3])) +@pytest.mark.parametrize("method", ["oindex", "vindex"]) +@pytest.mark.parametrize("layout", ["unit_chunks", "partial_chunks", "one_chunk", "sharded"]) +@pytest.mark.parametrize( + ("shape", "selection"), + [ + ((24,), ([-1, -1, 0, 0],)), + ((24,), ([-1, 0, 0, -1],)), + # long enough that an unstable sort by chunk reorders the repeats + ((24,), ([1] + [0] * 23,)), + ((6, 6), ([0, 5, 0, -1, 2] * 4, [1, 1, 4, 4, 2] * 4)), + ], + ids=["adjacent", "interleaved", "many_repeats", "two_dimensions"], +) +def test_setitem_repeated_index( + method: str, layout: str, shape: tuple[int, ...], selection: tuple[list[int], ...] +) -> None: + """Assignment with repeated indices stores the last value given for each repeated position, as NumPy does.""" + chunks = { + "unit_chunks": (1,) * len(shape), + "partial_chunks": (4,) * len(shape), + "one_chunk": shape, + "sharded": (2,) * len(shape), + }[layout] + shards = (6,) * len(shape) if layout == "sharded" else None + array = zarr.create_array(store={}, shape=shape, chunks=chunks, shards=shards, dtype="i8") + expected = np.zeros(shape, dtype="i8") + if method == "oindex": + np_selection = np.ix_(*selection) + value = np.arange(1, 1 + expected[np_selection].size).reshape(expected[np_selection].shape) + expected[np_selection] = value + array.oindex[selection] = value + else: + value = np.arange(1, 1 + len(selection[0])) + expected[selection] = value + array.vindex[selection] = value + assert_array_equal(array[:], expected) Index = list[int] | tuple[slice | int | list[int], ...] diff --git a/tests/test_properties.py b/tests/test_properties.py index b448a524aa..d5d385accf 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -17,7 +17,7 @@ import hypothesis.extra.numpy as npst import hypothesis.strategies as st -from hypothesis import assume, event, given, settings +from hypothesis import event, given, settings from tests.conftest import declared_chunk_data_sizes from zarr.abc.store import Store @@ -199,12 +199,8 @@ async def test_oindex(data: st.DataObject) -> None: actual = await async_zarray.oindex.getitem(zindexer) assert_array_equal(nparray[npindexer], actual) - # sync set - for idxr, size in zip(zindexer, nparray.shape, strict=True): - if isinstance(idxr, np.ndarray) and idxr.size != np.unique(idxr % size).size: - # behaviour of setitem with repeated indices is not guaranteed in practice - # Negative and positive spellings of the same index are duplicates too. - assume(False) + # sync set: an index that repeats a position (in either spelling) keeps the last value + # written to it, as in NumPy, so repeated indices are drawn like any other # The sharding codec sees a coordinate selection (the GH4284 path) when the # chunk selection has more than one array axis or drops an integer axis. n_array_axes = sum(isinstance(idxr, np.ndarray) for idxr in zindexer) @@ -248,19 +244,7 @@ async def test_vindex(data: st.DataObject) -> None: actual = await async_zarray.vindex.getitem(indexer) assert_array_equal(nparray[indexer], actual) - # sync set - # Reads preserve the supplied indices; normalize negative indices explicitly when - # detecting repeated points rather than relying on a read to mutate the indexer. - points = np.stack( - [ - (idxr % size).ravel() - for idxr, size in zip(np.broadcast_arrays(*indexer), nparray.shape, strict=True) - ], - axis=-1, - ) - if len(np.unique(points, axis=0)) != len(points): - # behaviour of setitem with repeated coordinates is not guaranteed in practice - assume(False) + # sync set: a repeated point keeps the last value written to it, as in NumPy new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) nparray[indexer] = new_data zarray.vindex[indexer] = new_data From aa58a965e548a30c3eefbed2a2a96ef6b62e80ce Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 17:46:41 +0100 Subject: [PATCH 02/11] docs(changes): add changelog fragment for repeated-index writes Assisted-by: ClaudeCode:claude-fable-5.2 --- changes/453.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/453.bugfix.md diff --git a/changes/453.bugfix.md b/changes/453.bugfix.md new file mode 100644 index 0000000000..1aec76d22a --- /dev/null +++ b/changes/453.bugfix.md @@ -0,0 +1 @@ +Assigning through `oindex`, `vindex` or `__setitem__` with an index array that repeats a position now stores the last value given for that position, as NumPy does. Previously the stored value depended on the platform, the chunk layout and the number of selected items. From 91b7c2c35e43a71d7ee7257c5e37e263a8ff77af Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 17:50:08 +0100 Subject: [PATCH 03/11] docs(changes): renumber the changelog fragment to the upstream PR Assisted-by: ClaudeCode:claude-fable-5.2 --- changes/{453.bugfix.md => 4643.bugfix.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{453.bugfix.md => 4643.bugfix.md} (100%) diff --git a/changes/453.bugfix.md b/changes/4643.bugfix.md similarity index 100% rename from changes/453.bugfix.md rename to changes/4643.bugfix.md From 6eb45631d909cbf915c70d70a90e9d2726ad46fa Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 18 Nov 2026 04:46:15 -0500 Subject: [PATCH 04/11] feat: narrow partial reads of uncompressed chunks within a single row (#4611) * Narrow partial reads of uncompressed chunks within a single row When a selection touches one index along the first axis, apply the same row window along the next axis, and so on. arr[t, y0:y1, x0:x1] then fetches rows y0 to y1 of frame t instead of the whole frame, and a single item is fetched for a point. The fetched block keeps the narrowed axes with length 1, so the shifted selection applies to it unchanged. Co-Authored-By: Claude Opus 5.5 * Add release note for #4611 Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Davis Bennett --- changes/4611.feature.md | 1 + src/zarr/codecs/bytes.py | 138 +++++++++++++++++++------------- tests/test_codecs/test_bytes.py | 100 +++++++++++++++++++---- 3 files changed, 171 insertions(+), 68 deletions(-) create mode 100644 changes/4611.feature.md diff --git a/changes/4611.feature.md b/changes/4611.feature.md new file mode 100644 index 0000000000..851b84bb35 --- /dev/null +++ b/changes/4611.feature.md @@ -0,0 +1 @@ +Partial reads of uncompressed chunks now narrow past the first axis. When `BytesCodec` is the only codec and a selection takes a single index along the leading axes, the byte range request covers only the rows it touches along the next axis. For example, `arr[t, y0:y1, x0:x1]` fetches rows `y0` to `y1` of frame `t` instead of the whole frame, and a single element is fetched for a point. diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 621dca78e9..9ebcaa4c02 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -1,5 +1,6 @@ from __future__ import annotations +import math import sys import warnings from dataclasses import dataclass, replace @@ -153,14 +154,16 @@ async def _decode_partial_single( The chunk is stored in C order, so each row along its first axis is a contiguous run of bytes. The rows from the first to the last one the selection touches are fetched with a single range request, and the - selection is applied to them. A selection that touches every row reads - the whole chunk, as before. + selection is applied to them. When the selection touches a single row, + the same is done within that row along the next axis, and so on, so + that `arr[t, y0:y1, x0:x1]` fetches rows `y0` to `y1` of frame `t`. A + selection that touches every row reads the whole chunk, as before. """ - window = _row_window(selection, chunk_spec.shape) + window = _block_window(selection, chunk_spec.shape) chunk_bytes = await byte_getter.get( - prototype=chunk_spec.prototype, byte_range=_row_byte_range(window, chunk_spec) + prototype=chunk_spec.prototype, byte_range=_block_byte_range(window, chunk_spec) ) - return self._decode_rows(chunk_bytes, selection, window, chunk_spec) + return self._decode_block(chunk_bytes, selection, window, chunk_spec) def _decode_partial_sync( self, @@ -169,28 +172,29 @@ def _decode_partial_sync( chunk_spec: ArraySpec, ) -> NDBuffer | None: """Sync equivalent of `_decode_partial_single`.""" - window = _row_window(selection, chunk_spec.shape) + window = _block_window(selection, chunk_spec.shape) chunk_bytes = byte_getter.get_sync( - prototype=chunk_spec.prototype, byte_range=_row_byte_range(window, chunk_spec) + prototype=chunk_spec.prototype, byte_range=_block_byte_range(window, chunk_spec) ) - return self._decode_rows(chunk_bytes, selection, window, chunk_spec) + return self._decode_block(chunk_bytes, selection, window, chunk_spec) - def _decode_rows( + def _decode_block( self, chunk_bytes: Buffer | None, selection: SelectorTuple, - window: tuple[int, int, SelectorTuple] | None, + window: tuple[int, tuple[int, ...], SelectorTuple] | None, chunk_spec: ArraySpec, ) -> NDBuffer | None: - """Decode the bytes fetched for a row window and apply the selection to them.""" + """Decode the bytes fetched for a block window and apply the selection to them.""" if chunk_bytes is None: return None if window is None: return self._decode_sync(chunk_bytes, chunk_spec)[selection] - first, stop, rows_selection = window - row_bytes = _row_bytes(chunk_spec) - start, end = first * row_bytes, stop * row_bytes - chunk_size = chunk_spec.shape[0] * row_bytes + offset, block_shape, block_selection = window + itemsize = _itemsize(chunk_spec) + start = offset * itemsize + end = start + math.prod(block_shape) * itemsize + chunk_size = math.prod(chunk_spec.shape) * itemsize # A store may not honor the byte range it was given. The length of what # it sent tells which bytes those are. if len(chunk_bytes) == end - start: @@ -206,8 +210,8 @@ def _decode_rows( f"Requested bytes {start} to {end} of a {chunk_size} byte chunk, " f"but the store returned {len(chunk_bytes)} bytes." ) - rows_spec = replace(chunk_spec, shape=(stop - first, *chunk_spec.shape[1:])) - return self._decode_sync(chunk_bytes, rows_spec)[rows_selection] + block_spec = replace(chunk_spec, shape=block_shape) + return self._decode_sync(chunk_bytes, block_spec)[block_selection] def _encode_sync( self, @@ -239,58 +243,84 @@ def compute_encoded_size(self, input_byte_length: int, _chunk_spec: ArraySpec) - return input_byte_length -def _row_window( +def _block_window( selection: SelectorTuple, shape: tuple[int, ...] -) -> tuple[int, int, SelectorTuple] | None: - """The rows along axis 0 that a selection touches, and the selection relative to them. - - Returns the first row, one past the last row, and the selection shifted so - that it indexes an array holding only those rows. Returns None when the - selection touches every row or the rows cannot be determined, in which case - the whole chunk is read. +) -> tuple[int, tuple[int, ...], SelectorTuple] | None: + """The contiguous block of a C-order chunk that a selection touches. + + Along the first axis the block spans the first to the last row the + selection touches. If that is a single row, the block is narrowed in the + same way along the next axis, and so on. The axes after that are whole. + + Returns the offset of the block from the start of the chunk in items, the + shape of the block, and the selection shifted so that it indexes the block. + The block has as many axes as the chunk; the axes it was narrowed to a + single index along have length 1. Returns None when the block is the whole + chunk or cannot be determined, in which case the whole chunk is read. """ - if len(shape) == 0 or not isinstance(selection, tuple) or len(selection) == 0: + if not isinstance(selection, tuple): + return None + offset = 0 + block_shape = list(shape) + shifted = list(selection) + for axis, (selector, size) in enumerate(zip(selection, shape, strict=False)): + window = _axis_window(selector, size) + if window is None: + break + first, stop, shifted[axis] = window + offset += first * math.prod(shape[axis + 1 :]) + block_shape[axis] = stop - first + if stop - first > 1: + break + if tuple(block_shape) == shape: return None - first_axis = selection[0] + return offset, tuple(block_shape), tuple(shifted) + + +def _axis_window(selector: Selector, size: int) -> tuple[int, int, Selector] | None: + """The indices along one axis that a selector touches, and the selector relative to them. + + Returns the first index, one past the last index, and the selector shifted + so that it indexes an axis holding only those indices. Returns None when + the indices cannot be determined. + """ shifted: Selector - if isinstance(first_axis, slice): - rows = range(*first_axis.indices(shape[0])) - if len(rows) == 0 or rows.step < 0: + if isinstance(selector, slice): + indices = range(*selector.indices(size)) + if len(indices) == 0 or indices.step < 0: return None - first, stop = rows[0], rows[-1] + 1 - shifted = slice(0, stop - first, rows.step) - elif isinstance(first_axis, int | np.integer): - first = int(first_axis) % shape[0] + first, stop = indices[0], indices[-1] + 1 + shifted = slice(0, stop - first, indices.step) + elif isinstance(selector, int | np.integer): + first = int(selector) % size stop = first + 1 shifted = 0 - elif isinstance(first_axis, np.ndarray): - if first_axis.dtype == bool: - indices = np.nonzero(first_axis)[0] + elif isinstance(selector, np.ndarray): + if selector.dtype == bool: + positions = np.nonzero(selector)[0] else: - indices = first_axis % shape[0] - if indices.size == 0: + positions = selector % size + if positions.size == 0: return None - first, stop = int(indices.min()), int(indices.max()) + 1 - shifted = first_axis[first:stop] if first_axis.dtype == bool else indices - first + first, stop = int(positions.min()), int(positions.max()) + 1 + shifted = selector[first:stop] if selector.dtype == bool else positions - first else: return None - if (first, stop) == (0, shape[0]): - return None - return first, stop, (shifted, *selection[1:]) + return first, stop, shifted -def _row_bytes(chunk_spec: ArraySpec) -> int: - """The number of bytes in one row along axis 0 of a chunk.""" - row_items = int(np.prod(chunk_spec.shape[1:])) - return chunk_spec.dtype.to_native_dtype().itemsize * row_items +def _itemsize(chunk_spec: ArraySpec) -> int: + """The number of bytes in one item of a chunk.""" + return chunk_spec.dtype.to_native_dtype().itemsize -def _row_byte_range( - window: tuple[int, int, SelectorTuple] | None, chunk_spec: ArraySpec +def _block_byte_range( + window: tuple[int, tuple[int, ...], SelectorTuple] | None, chunk_spec: ArraySpec ) -> RangeByteRequest | None: - """The byte range holding a row window, or None to read the whole chunk.""" + """The byte range holding a block window, or None to read the whole chunk.""" if window is None: return None - first, stop, _ = window - row_bytes = _row_bytes(chunk_spec) - return RangeByteRequest(first * row_bytes, stop * row_bytes) + offset, block_shape, _ = window + itemsize = _itemsize(chunk_spec) + start = offset * itemsize + return RangeByteRequest(start, start + math.prod(block_shape) * itemsize) diff --git a/tests/test_codecs/test_bytes.py b/tests/test_codecs/test_bytes.py index 8ac4df7ba1..de2cacd3c2 100644 --- a/tests/test_codecs/test_bytes.py +++ b/tests/test_codecs/test_bytes.py @@ -3,6 +3,7 @@ from __future__ import annotations import enum +import math import sys import warnings from typing import TYPE_CHECKING, Any, Literal, cast @@ -19,7 +20,7 @@ BytesCodec, Endian, EndianLiteral, - _row_window, + _block_window, ) from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import NDBuffer, default_buffer_prototype @@ -392,22 +393,52 @@ def _single_chunk_array( @pytest.mark.usefixtures("codec_pipeline") @pytest.mark.parametrize( - ("selection", "rows_read"), + ("selection", "items_read"), [ - (np.s_[500:600, 3], 100), - (np.s_[500:600], 100), - (np.s_[777, 5], 1), - (np.s_[-1], 1), - (np.s_[10:20:3, ::2], 10), # rows 10 to 19 are fetched - (np.s_[...], 10_000), + (np.s_[500:600, 3], 100 * 10), + (np.s_[500:600], 100 * 10), + (np.s_[777, 5], 1), # a single row is narrowed to the columns it needs + (np.s_[777, 2:6], 4), + (np.s_[777:778, 2:6], 4), + (np.s_[-1], 10), + (np.s_[10:20:3, ::2], 10 * 10), # rows 10 to 19 are fetched + (np.s_[...], 10_000 * 10), ], ) -def test_uncompressed_partial_read(selection: Any, rows_read: int) -> None: - """Reading part of an uncompressed chunk fetches only the rows it touches.""" +def test_uncompressed_partial_read(selection: Any, items_read: int) -> None: + """Reading part of an uncompressed chunk fetches only the block it touches.""" data = np.arange(100_000, dtype="int16").reshape(10_000, 10) arr, store = _single_chunk_array(data, compressors=None) np.testing.assert_array_equal(arr[selection], data[selection]) - assert sum(n for *_, n in store.reads) == rows_read * 10 * 2 + assert sum(n for *_, n in store.reads) == items_read * 2 + + +@pytest.mark.usefixtures("codec_pipeline") +@pytest.mark.parametrize( + ("selection", "byte_range"), + [ + ( + np.s_[7, 10:20, 5:9], + (7 * 1200 + 10 * 30, 7 * 1200 + 20 * 30), + ), # rows 10 to 19 of frame 7 + (np.s_[7, 10:20], (7 * 1200 + 10 * 30, 7 * 1200 + 20 * 30)), + (np.s_[7, 10, 5:9], (7 * 1200 + 10 * 30 + 5, 7 * 1200 + 10 * 30 + 9)), + (np.s_[7, 10, 5], (7 * 1200 + 10 * 30 + 5, 7 * 1200 + 10 * 30 + 6)), + (np.s_[7:8, 10:11, 5:9], (7 * 1200 + 10 * 30 + 5, 7 * 1200 + 10 * 30 + 9)), + (np.s_[7, :, 5:9], (7 * 1200, 8 * 1200)), # every row of frame 7 + (np.s_[7], (7 * 1200, 8 * 1200)), + (np.s_[7:9, 10:20, 5:9], (7 * 1200, 9 * 1200)), # two frames are fetched whole + ], +) +def test_uncompressed_partial_read_within_a_row( + selection: Any, byte_range: tuple[int, int] +) -> None: + """A selection that touches one index along the leading axes is narrowed along the next.""" + data = np.arange(50 * 40 * 30, dtype="uint8").reshape(50, 40, 30) + arr, store = _single_chunk_array(data, compressors=None) + np.testing.assert_array_equal(arr[selection], data[selection]) + ((_, served, _),) = store.reads + assert (served.start, served.end) == byte_range @pytest.mark.usefixtures("codec_pipeline") @@ -427,6 +458,19 @@ def test_uncompressed_partial_read_values(dtype: str, ndim: int) -> None: np.testing.assert_array_equal(arr.oindex[mask], data[mask]) coords = tuple(np.array([0, 6, 3]) % n for n in shape) np.testing.assert_array_equal(arr.vindex[coords], data[coords]) + # selections that touch a single index along the leading axes + single: list[Any] = [np.s_[4, 2:], np.s_[4, -1], np.s_[4:5, 1:3], np.s_[4, ..., 0]] + if ndim == 3: + single += [np.s_[4, 2, 1:], np.s_[4, 2:4, 1], np.s_[4, :, 1], np.s_[4, 3, ::2]] + for selection in single if ndim > 1 else []: + np.testing.assert_array_equal(arr[selection], data[selection]) + if ndim > 1: + np.testing.assert_array_equal(arr.oindex[[4], [3, 1]], data[[4]][:, [3, 1]]) + mask = np.zeros(shape[1], dtype=bool) + mask[[1, 3]] = True + np.testing.assert_array_equal(arr.oindex[4, mask], data[4][mask]) + point = tuple(np.array([n - 1, n - 1]) for n in shape) + np.testing.assert_array_equal(arr.vindex[point], data[point]) @pytest.mark.usefixtures("codec_pipeline") @@ -543,10 +587,38 @@ def test_uncompressed_partial_read_store_sends_unexpected_length() -> None: ((slice(None), 2), (10, 4)), # every row ((np.array([0, 9]), slice(None)), (10, 4)), # first and last row ((..., 2), (10, 4)), # rows not determined + ((0, slice(None)), (1, 4)), # the only row, and every column of it (slice(2, 4), (10,)), # not a tuple ((), ()), # zero-dimensional chunk ], ) -def test_row_window_reads_whole_chunk(selection: Any, shape: tuple[int, ...]) -> None: - """Selections with no narrower row window than the whole chunk give None.""" - assert _row_window(selection, shape) is None +def test_block_window_reads_whole_chunk(selection: Any, shape: tuple[int, ...]) -> None: + """Selections with no narrower block than the whole chunk give None.""" + assert _block_window(selection, shape) is None + + +@pytest.mark.parametrize( + ("selection", "shape", "offset", "block_shape"), + [ + ((slice(2, 5), slice(None)), (10, 4), 8, (3, 4)), + ((3, slice(1, 3)), (10, 4), 13, (1, 2)), + ((3, 2), (10, 4), 14, (1, 1)), + ((0, 2), (1, 4), 2, (1, 1)), # an axis of length 1 is passed through + ((3, slice(2, 2)), (10, 4), 12, (1, 4)), # no columns: stop at the row + ((3, ..., 1), (10, 4, 5), 60, (1, 4, 5)), # stop at the ellipsis + ((3,), (10, 4), 12, (1, 4)), # fewer selectors than axes + ((np.array([3, 3]), np.array([1, 2])), (10, 4), 13, (1, 2)), + ((2, 1, slice(1, 4)), (5, 4, 6), 2 * 24 + 6 + 1, (1, 1, 3)), + ], +) +def test_block_window( + selection: Any, shape: tuple[int, ...], offset: int, block_shape: tuple[int, ...] +) -> None: + """The block starts at the first item the selection touches along the narrowed axes.""" + window = _block_window(selection, shape) + assert window is not None + assert window[:2] == (offset, block_shape) + # the shifted selection picks the same items from the block as the selection does from the chunk + data = np.arange(math.prod(shape)).reshape(shape) + block = data.reshape(-1)[offset : offset + math.prod(block_shape)].reshape(block_shape) + np.testing.assert_array_equal(block[window[2]], data[selection]) From 1366d30978e6f3dd0ed398e65e10b7ffe518a161 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 18 Nov 2026 12:38:30 +0100 Subject: [PATCH 05/11] perf(indexing): radix-sort chunk indices when grouping selections by chunk The stable argsort that keeps repeated indices in order was 2-3x slower than NumPy's default sort for large unordered selections, on reads as well as writes. NumPy sorts integers of at most 16 bits with a stable radix sort, so chunk indices are narrowed to uint16 when there are at most 2**16 chunks. Building an indexer for 10M unordered points then takes about 165-200 ms, against 560-690 ms on main and 1.5-2.0 s with the plain stable sort. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- src/zarr/core/indexing.py | 25 +++++++++++++++++++------ tests/test_indexing.py | 12 ++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index 3b9f972961..0b5c9dc387 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -762,6 +762,21 @@ def sorted_run_ends( return a[run_starts].astype(np.intp, copy=False), run_ends +def stable_argsort_chunk_indices( + chunk_indices: npt.NDArray[np.intp], nchunks: int +) -> npt.NDArray[np.intp]: + """Return the indices that stably sort `chunk_indices`, each in `range(nchunks)`. + + A stable sort keeps items that share a chunk in the order given, so repeated + indices are assigned in order and the last value given is the one stored. NumPy + sorts integers of at most 16 bits with a stable radix sort, which is several times + faster than its default sort, so the indices are narrowed when `nchunks` allows it. + """ + if nchunks <= 2**16: + chunk_indices = chunk_indices.astype(np.uint16) + return np.argsort(chunk_indices, kind="stable") + + @dataclass(frozen=True) class IntArrayDimIndexer: """Integer array selection against a single dimension.""" @@ -830,9 +845,8 @@ def __init__( dim_out_sel = np.arange(nitems - 1, -1, -1) dim_sel_chunk_sorted = dim_sel_chunk[::-1] else: - # sort indices to group by chunk; a stable sort keeps repeated indices in - # the order given, so the last value assigned to an item is the one stored - dim_out_sel = np.argsort(dim_sel_chunk, kind="stable") + # sort indices to group by chunk + dim_out_sel = stable_argsort_chunk_indices(dim_sel_chunk, nchunks) dim_sel = np.take(dim_sel, dim_out_sel) dim_sel_chunk_sorted = dim_sel_chunk[dim_out_sel] @@ -1346,9 +1360,8 @@ def __init__( # group points by chunk if np.any(np.diff(chunks_raveled_indices) < 0): - # optimisation, only sort if needed; a stable sort keeps repeated points in - # the order given, so the last value assigned to a point is the one stored - sel_sort = np.argsort(chunks_raveled_indices, kind="stable") + # optimisation, only sort if needed + sel_sort = stable_argsort_chunk_indices(chunks_raveled_indices, product(cdata_shape)) selection_broadcast = tuple(dim_sel[sel_sort] for dim_sel in selection_broadcast) chunks_raveled_indices = chunks_raveled_indices[sel_sort] else: diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 53fe1be8a1..718f2883fb 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -37,6 +37,7 @@ oindex, oindex_set, replace_ellipsis, + stable_argsort_chunk_indices, ) from zarr.registry import get_ndbuffer_class from zarr.storage import MemoryStore, StorePath @@ -545,6 +546,17 @@ def test_setitem_repeated_index( assert_array_equal(array[:], expected) +@pytest.mark.parametrize("nchunks", [1, 2**8, 2**16, 2**16 + 1, 2**40]) +def test_stable_argsort_chunk_indices(nchunks: int) -> None: + """The sort matches NumPy's stable sort whether or not the chunk indices are narrowed.""" + rng = np.random.default_rng(0) + chunk_indices = np.concatenate( + [[0, nchunks - 1, nchunks - 1, 0], rng.integers(0, nchunks, 1000)] + ).astype(np.intp) + expected = np.argsort(chunk_indices, kind="stable") + assert_array_equal(stable_argsort_chunk_indices(chunk_indices, nchunks), expected) + + Index = list[int] | tuple[slice | int | list[int], ...] From 632784a0b52f101ff9263e917afa31b1da44e736 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 18 Nov 2026 14:17:48 +0100 Subject: [PATCH 06/11] fix(indexing): pick the last value for repeated indices explicitly per chunk The stable sort kept repeated indices in the order given, but the in-chunk assignment still left the stored value to the array library: NumPy does not document the order of repeated assignments and CuPy leaves it undefined. Following zarr-indexing, `_merge_chunk_array` now resolves a chunk selection that names an element more than once into one that names each element once, with the last value given for it in C order. Orthogonal selections are resolved one axis at a time; coordinate selections jointly, in linear time via `np.maximum.at` unless the chunk is much larger than the selection. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- changes/4643.bugfix.md | 2 +- src/zarr/core/chunk_utils.py | 8 +- src/zarr/core/indexing.py | 186 ++++++++++++++++++++++++++++++++++- tests/test_indexing.py | 57 ++++++++++- 4 files changed, 246 insertions(+), 7 deletions(-) diff --git a/changes/4643.bugfix.md b/changes/4643.bugfix.md index 1aec76d22a..155367cabe 100644 --- a/changes/4643.bugfix.md +++ b/changes/4643.bugfix.md @@ -1 +1 @@ -Assigning through `oindex`, `vindex` or `__setitem__` with an index array that repeats a position now stores the last value given for that position, as NumPy does. Previously the stored value depended on the platform, the chunk layout and the number of selected items. +Assigning through `oindex`, `vindex` or `__setitem__` with an index array that repeats a position now stores the last value given for that position, as NumPy does. Previously the stored value depended on the platform, the chunk layout and the number of selected items. Zarr now picks the last value itself, so the result no longer depends on the order in which the array library applies repeated assignments. diff --git a/src/zarr/core/chunk_utils.py b/src/zarr/core/chunk_utils.py index b26d5478b2..c8fe937246 100644 --- a/src/zarr/core/chunk_utils.py +++ b/src/zarr/core/chunk_utils.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, cast from zarr.abc.codec import GetResult, SupportsSyncCodec, _codec_supports_sync -from zarr.core.indexing import is_scalar +from zarr.core.indexing import is_scalar, last_assignments if TYPE_CHECKING: from collections.abc import Callable, Iterable @@ -158,6 +158,12 @@ def _merge_chunk_array( None if idx in drop_axes else slice(None) for idx in range(chunk_spec.ndim) ) chunk_value = chunk_value[item] + repeats = last_assignments(chunk_selection, chunk_spec.shape) + if repeats is not None: + # Write each element once, with the last value given for it, rather than + # leave the outcome to the array library's order of assignment. + chunk_array[repeats.selection] = repeats.pick(chunk_value.as_ndarray_like()) + return chunk_array chunk_array[chunk_selection] = chunk_value return chunk_array diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index 0b5c9dc387..cc7c8eddca 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -767,16 +767,194 @@ def stable_argsort_chunk_indices( ) -> npt.NDArray[np.intp]: """Return the indices that stably sort `chunk_indices`, each in `range(nchunks)`. - A stable sort keeps items that share a chunk in the order given, so repeated - indices are assigned in order and the last value given is the one stored. NumPy - sorts integers of at most 16 bits with a stable radix sort, which is several times - faster than its default sort, so the indices are narrowed when `nchunks` allows it. + A stable sort keeps items that share a chunk in the order given, so each chunk's + selection lists repeated indices in the order the user gave them, which is what + `last_assignments` relies on. NumPy sorts integers of at most 16 bits with a stable + radix sort, which is several times faster than its default sort, so the indices are + narrowed when `nchunks` allows it. """ if nchunks <= 2**16: chunk_indices = chunk_indices.astype(np.uint16) return np.argsort(chunk_indices, kind="stable") +class LastAssignments(NamedTuple): + """A selection that writes each element once, and how to pick the values it takes. + + `selection` names, once each, the elements an original selection wrote. Their + values are the original values broadcast to `selected_shape`, raveled when + `flatten` is set, then indexed along each axis by the matching entry of `keep`, + where `None` keeps the whole axis. + """ + + selection: tuple[Any, ...] + selected_shape: tuple[int, ...] + flatten: bool + keep: tuple[npt.NDArray[np.intp] | None, ...] + + def pick(self, values: Any) -> Any: + """Pick, from the values given for the original selection, those to write.""" + picked = np.broadcast_to(values, self.selected_shape) + if self.flatten: + picked = picked.reshape(-1) + for axis, keep in enumerate(self.keep): + if keep is not None: + picked = picked[(slice(None),) * axis + (keep,)] + return picked + + +def _last_occurrences(flat: npt.NDArray[Any], size: int) -> npt.NDArray[np.intp] | None: + """Positions to keep so that repeated indices, each in `range(size)`, take their last value. + + Returns `None` when no index repeats, so callers can use the selection as is. + """ + if flat.size < 2: + return None + if (flat[1:] >= flat[:-1]).all(): + # Non-decreasing input, the usual case: repeats are adjacent. + steps = flat[1:] != flat[:-1] + if steps.all(): + return None + return np.flatnonzero(np.append(steps, True)) + positions = np.arange(flat.size, dtype=np.intp) + if size <= 16 * flat.size: + # Few possible indices for the number given: record the last position of each + # in linear time. A maximum does not depend on the order `ufunc.at` applies + # repeated indices in. + last = np.full(size, -1, dtype=np.intp) + np.maximum.at(last, flat, positions) + keep = np.flatnonzero(last[flat] == positions) + else: + _, first_in_reversed = np.unique(flat[::-1], return_index=True) + keep = positions[-1] - first_in_reversed + if keep.size == flat.size: + return None + return keep + + +def _is_integer_array(selector: object) -> TypeGuard[npt.NDArray[np.integer[Any]]]: + return isinstance(selector, np.ndarray) and selector.dtype.kind in "iu" + + +def _orthogonal_last_assignments( + selection: tuple[Any, ...], shape: tuple[int, ...], mesh: bool +) -> LastAssignments | None: + """`last_assignments` for an orthogonal selection, one axis at a time. + + The selection is either one 1-D array among slices or, when `mesh` is set, an + `np.ix_` mesh. An element repeats only where some axis repeats an index, and the + last value for it in C order is the one at the last occurrence on each axis. + """ + new_selection = list(selection) + keep: list[npt.NDArray[np.intp] | None] = [] + selected_shape: list[int] = [] + for axis, (selector, length) in enumerate(zip(selection, shape, strict=True)): + if isinstance(selector, slice): + selected_shape.append(len(range(*selector.indices(length)))) + keep.append(None) + continue + selected_shape.append(selector.size) + kept = ( + _last_occurrences(selector.reshape(-1), length) if _is_integer_array(selector) else None + ) + keep.append(kept) + if kept is not None: + new_selection[axis] = np.take(selector, kept, axis=axis if mesh else 0) + if all(kept is None for kept in keep): + return None + return LastAssignments( + selection=tuple(new_selection), + selected_shape=tuple(selected_shape), + flatten=False, + keep=tuple(keep), + ) + + +def _selected_flat_indices( + selection: tuple[Any, ...], shape: tuple[int, ...] +) -> npt.NDArray[np.intp]: + """The flat index of each element `selection` names, in the layout of NumPy's result.""" + coords = [] + for axis, length in enumerate(shape): + axis_shape = tuple(length if d == axis else 1 for d in range(len(shape))) + # A broadcast view of each axis' coordinates, indexed by the selection, gives the + # coordinates the selection names without allocating an array of `shape`. + grid = np.broadcast_to(np.arange(length, dtype=np.intp).reshape(axis_shape), shape) + coords.append(grid[selection]) + return np.asarray(np.ravel_multi_index(tuple(coords), shape), dtype=np.intp) + + +def _selection_form( + selection: tuple[Any, ...], ndim: int +) -> Literal["array", "mesh", "coordinates", "other"] | None: + """Classify a selection by the forms zarr's indexers give chunk selections. + + `"array"` is one 1-D integer array among slices, `"mesh"` an `np.ix_` mesh of + integer arrays and `"coordinates"` one 1-D integer array per axis, all the same + length. `None` means the selection has no integer array, so cannot repeat an + element. + """ + arrays = [selector for selector in selection if isinstance(selector, np.ndarray)] + if not any(_is_integer_array(array) for array in arrays): + return None + if len(selection) != ndim: + return "other" + if len(arrays) == 1: + if arrays[0].ndim == 1 and all(isinstance(s, slice | np.ndarray) for s in selection): + return "array" + return "other" + if len(arrays) < ndim or not all(_is_integer_array(array) for array in arrays): + return "other" + if all( + array.ndim == ndim and all(array.shape[d] == 1 for d in range(ndim) if d != axis) + for axis, array in enumerate(arrays) + ): + return "mesh" + if all(array.ndim == 1 and array.size == arrays[0].size for array in arrays): + return "coordinates" + return "other" + + +def last_assignments(selection: SelectorTuple, shape: tuple[int, ...]) -> LastAssignments | None: + """Resolve the elements of an array of `shape` that `selection` assigns more than once. + + Assigning through a selection that repeats an element gives that element several + values. NumPy leaves the order of those assignments undocumented and CuPy leaves it + undefined, so the last value given in the selection's C order is chosen here, + explicitly. The choice depends only on the selection, not on the order chunks are + written in. Returns `None` when no element repeats. + """ + selectors: tuple[Any, ...] = selection if isinstance(selection, tuple) else (selection,) + form = _selection_form(selectors, len(shape)) + if form is None: + return None + if form == "array": + return _orthogonal_last_assignments(selectors, shape, mesh=False) + if form == "mesh": + return _orthogonal_last_assignments(selectors, shape, mesh=True) + if form == "coordinates": + flat = np.asarray(np.ravel_multi_index(selectors, shape), dtype=np.intp) + kept = _last_occurrences(flat, product(shape)) + if kept is None: + return None + return LastAssignments( + selection=tuple(array[kept] for array in selectors), + selected_shape=(flat.size,), + flatten=False, + keep=(kept,), + ) + flat = _selected_flat_indices(selectors, shape) + kept = _last_occurrences(flat.reshape(-1), product(shape)) + if kept is None: + return None + return LastAssignments( + selection=tuple(np.unravel_index(flat.reshape(-1)[kept], shape)), + selected_shape=flat.shape, + flatten=True, + keep=(kept,), + ) + + @dataclass(frozen=True) class IntArrayDimIndexer: """Integer array selection against a single dimension.""" diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 718f2883fb..6c340eecd6 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -20,7 +20,7 @@ from zarr import Array from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_grids import ChunkGrid -from zarr.core.common import ceildiv_int +from zarr.core.common import ceildiv_int, product from zarr.core.indexing import ( BasicSelection, CoordinateIndexer, @@ -32,6 +32,7 @@ _ArrayIndexingOrder, _iter_grid, _iter_regions, + last_assignments, make_slice_selection, normalize_integer_selection, oindex, @@ -557,6 +558,60 @@ def test_stable_argsort_chunk_indices(nchunks: int) -> None: assert_array_equal(stable_argsort_chunk_indices(chunk_indices, nchunks), expected) +_rng = np.random.default_rng(0) + + +@pytest.mark.parametrize( + "selection", + [ + (np.array([1, 2, 1, 0, 2]), slice(None)), + (slice(1, 3), np.array([2, 2, 0])), + (slice(None), _rng.integers(0, 40, 100)), + (np.array([1, 2, 2]), np.array([3, 3, 3])), + (np.array([2, 1, 2]), np.array([3, 0, 3])), + (_rng.integers(0, 30, 1000), _rng.integers(0, 40, 1000)), + (np.array([1, 2, 1]), 2), + np.ix_(np.array([2, 0, 2]), np.array([1, 1, 3])), + (np.array([0, 1, 2]), slice(None)), + (slice(None), _rng.permutation(40)), + tuple(np.unravel_index(_rng.permutation(1200), (30, 40))), + np.ix_(np.array([2, 0]), np.array([1, 3])), + (slice(None), slice(1, 3)), + ], + ids=[ + "array_then_slice", + "slice_then_array", + "long_unsorted_array", + "sorted_coordinates", + "unsorted_coordinates", + "many_coordinates", + "array_and_integer", + "mesh", + "no_repeats_orthogonal", + "no_repeats_long_unsorted_array", + "no_repeats_many_coordinates", + "no_repeats_mesh", + "no_arrays", + ], +) +def test_last_assignments(selection: tuple[Any, ...]) -> None: + """Writing through the resolved selection matches assigning each value in C order.""" + shape = (30, 40) + targets = np.arange(product(shape)).reshape(shape)[selection] + values = np.arange(targets.size).reshape(targets.shape) + expected = np.full(product(shape), -1) + for target, value in zip(targets.ravel().tolist(), values.ravel().tolist(), strict=True): + expected[target] = value + result = last_assignments(selection, shape) + if np.unique(targets).size == targets.size: + assert result is None + return + assert result is not None + actual = np.full(shape, -1) + actual[result.selection] = result.pick(values) + assert_array_equal(actual.ravel(), expected) + + Index = list[int] | tuple[slice | int | list[int], ...] From fee0b33a37238e594b94ccd017ca4179082c90c3 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 18 Nov 2026 14:45:47 +0100 Subject: [PATCH 07/11] fix(indexing): resolve repeated indices by selection position, not chunk order Repeats are now resolved before any codec sees them: `last_writes` wraps the indexer in `_set_selection` and, for each chunk projection, keeps the entry with the highest position in `out_selection` for every element named more than once. This is the last value in the selection's C order, and it does not depend on the order items are listed within a chunk, so the sort that groups items by chunk no longer needs to be stable. Above 2**16 chunks the indexers return to NumPy's default sort, recovering main's indexer speed. The per-chunk merge no longer dedupes, and the sharding codec never receives a repeated index. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- src/zarr/core/array.py | 6 +- src/zarr/core/chunk_utils.py | 8 +- src/zarr/core/indexing.py | 283 +++++++++++++++-------------------- tests/test_indexing.py | 121 +++++++++------ 4 files changed, 199 insertions(+), 219 deletions(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 3735a4da18..5da88aadf9 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -109,6 +109,7 @@ is_pure_fancy_indexing, is_pure_orthogonal_indexing, is_scalar, + last_writes, pop_fields, ) from zarr.core.metadata import ( @@ -5901,7 +5902,10 @@ async def _set_selection( out_selection, is_complete_chunk, ) - for chunk_coords, chunk_selection, out_selection, is_complete_chunk in indexer + # A scalar gives every element the same value, so repeats need no resolving. + for chunk_coords, chunk_selection, out_selection, is_complete_chunk in ( + indexer if value_buffer.shape == () else last_writes(indexer) + ) ], value_buffer, drop_axes=indexer.drop_axes, diff --git a/src/zarr/core/chunk_utils.py b/src/zarr/core/chunk_utils.py index c8fe937246..b26d5478b2 100644 --- a/src/zarr/core/chunk_utils.py +++ b/src/zarr/core/chunk_utils.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, cast from zarr.abc.codec import GetResult, SupportsSyncCodec, _codec_supports_sync -from zarr.core.indexing import is_scalar, last_assignments +from zarr.core.indexing import is_scalar if TYPE_CHECKING: from collections.abc import Callable, Iterable @@ -158,12 +158,6 @@ def _merge_chunk_array( None if idx in drop_axes else slice(None) for idx in range(chunk_spec.ndim) ) chunk_value = chunk_value[item] - repeats = last_assignments(chunk_selection, chunk_spec.shape) - if repeats is not None: - # Write each element once, with the last value given for it, rather than - # leave the outcome to the array library's order of assignment. - chunk_array[repeats.selection] = repeats.pick(chunk_value.as_ndarray_like()) - return chunk_array chunk_array[chunk_selection] = chunk_value return chunk_array diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index cc7c8eddca..7a517c7377 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -762,197 +762,150 @@ def sorted_run_ends( return a[run_starts].astype(np.intp, copy=False), run_ends -def stable_argsort_chunk_indices( +def argsort_chunk_indices( chunk_indices: npt.NDArray[np.intp], nchunks: int ) -> npt.NDArray[np.intp]: - """Return the indices that stably sort `chunk_indices`, each in `range(nchunks)`. + """Return the indices that sort `chunk_indices`, each in `range(nchunks)`. - A stable sort keeps items that share a chunk in the order given, so each chunk's - selection lists repeated indices in the order the user gave them, which is what - `last_assignments` relies on. NumPy sorts integers of at most 16 bits with a stable - radix sort, which is several times faster than its default sort, so the indices are - narrowed when `nchunks` allows it. + The order of items that share a chunk is unspecified: writes resolve repeated + indices by their position in the selection (see `last_writes`), not by their order + within a chunk. NumPy sorts integers of at most 16 bits with a radix sort, which is + several times faster than its default sort, so the indices are narrowed when + `nchunks` allows it. """ if nchunks <= 2**16: - chunk_indices = chunk_indices.astype(np.uint16) - return np.argsort(chunk_indices, kind="stable") + return np.argsort(chunk_indices.astype(np.uint16), kind="stable") + return np.argsort(chunk_indices) -class LastAssignments(NamedTuple): - """A selection that writes each element once, and how to pick the values it takes. +def _last_by_rank( + targets: npt.NDArray[Any], ranks: npt.NDArray[Any] | None +) -> npt.NDArray[np.intp] | None: + """Positions of the entries to keep so that each target keeps its highest-ranked entry. - `selection` names, once each, the elements an original selection wrote. Their - values are the original values broadcast to `selected_shape`, raveled when - `flatten` is set, then indexed along each axis by the matching entry of `keep`, - where `None` keeps the whole axis. + `targets` holds non-negative integers that may repeat. `ranks` holds distinct + integers, one per entry; `None` means each entry's rank is its position. Returns + `None` when no target repeats. """ - - selection: tuple[Any, ...] - selected_shape: tuple[int, ...] - flatten: bool - keep: tuple[npt.NDArray[np.intp] | None, ...] - - def pick(self, values: Any) -> Any: - """Pick, from the values given for the original selection, those to write.""" - picked = np.broadcast_to(values, self.selected_shape) - if self.flatten: - picked = picked.reshape(-1) - for axis, keep in enumerate(self.keep): - if keep is not None: - picked = picked[(slice(None),) * axis + (keep,)] - return picked - - -def _last_occurrences(flat: npt.NDArray[Any], size: int) -> npt.NDArray[np.intp] | None: - """Positions to keep so that repeated indices, each in `range(size)`, take their last value. - - Returns `None` when no index repeats, so callers can use the selection as is. - """ - if flat.size < 2: + n = targets.size + if n < 2: return None - if (flat[1:] >= flat[:-1]).all(): - # Non-decreasing input, the usual case: repeats are adjacent. - steps = flat[1:] != flat[:-1] + if ranks is None and (targets[1:] >= targets[:-1]).all(): + # Non-decreasing targets ranked by position: repeats are adjacent, and the last + # of each run ranks highest. + steps = targets[1:] != targets[:-1] if steps.all(): return None return np.flatnonzero(np.append(steps, True)) - positions = np.arange(flat.size, dtype=np.intp) - if size <= 16 * flat.size: - # Few possible indices for the number given: record the last position of each - # in linear time. A maximum does not depend on the order `ufunc.at` applies - # repeated indices in. - last = np.full(size, -1, dtype=np.intp) - np.maximum.at(last, flat, positions) - keep = np.flatnonzero(last[flat] == positions) + if (targets[1:] > targets[:-1]).all(): + return None + if ranks is None: + ranks = np.arange(n, dtype=np.intp) + size = int(targets.max()) + 1 + if size <= 16 * n: + # Few possible targets for the number given: record the highest rank of each in + # linear time. A maximum does not depend on the order `ufunc.at` applies + # repeated targets in. + highest = np.full(size, np.iinfo(np.intp).min, dtype=np.intp) + np.maximum.at(highest, targets, ranks) + keep = np.flatnonzero(highest[targets] == ranks) else: - _, first_in_reversed = np.unique(flat[::-1], return_index=True) - keep = positions[-1] - first_in_reversed - if keep.size == flat.size: + order = np.lexsort((ranks, targets)) + ordered = targets[order] + keep = order[np.append(ordered[1:] != ordered[:-1], True)] + if keep.size == n: return None return keep -def _is_integer_array(selector: object) -> TypeGuard[npt.NDArray[np.integer[Any]]]: - return isinstance(selector, np.ndarray) and selector.dtype.kind in "iu" +def _as_positions(out_sel: slice | npt.NDArray[Any]) -> npt.NDArray[np.intp]: + """The output positions a 1-D output selection names, as an array. + + Indexers give output slices an explicit start and stop, and no step. + """ + if isinstance(out_sel, slice): + return np.arange(out_sel.start, out_sel.stop, dtype=np.intp) + return out_sel.reshape(-1) -def _orthogonal_last_assignments( - selection: tuple[Any, ...], shape: tuple[int, ...], mesh: bool -) -> LastAssignments | None: - """`last_assignments` for an orthogonal selection, one axis at a time. +def _orthogonal_last_writes( + projection: ChunkProjection, drop_axes: tuple[int, ...] +) -> ChunkProjection: + """`last_writes` for one projection of an `OrthogonalIndexer`. - The selection is either one 1-D array among slices or, when `mesh` is set, an - `np.ix_` mesh. An element repeats only where some axis repeats an index, and the - last value for it in C order is the one at the last occurrence on each axis. + An element repeats only where some axis repeats an index, and its last value in C + order is the one at the highest output position on each axis, so each axis is + resolved on its own. """ - new_selection = list(selection) - keep: list[npt.NDArray[np.intp] | None] = [] - selected_shape: list[int] = [] - for axis, (selector, length) in enumerate(zip(selection, shape, strict=True)): - if isinstance(selector, slice): - selected_shape.append(len(range(*selector.indices(length)))) - keep.append(None) + chunk_selection = cast("tuple[Any, ...]", projection.chunk_selection) + out_selection = cast("tuple[Any, ...]", projection.out_selection) + n_arrays = sum(isinstance(sel, np.ndarray) for sel in chunk_selection) + # The indexer meshes the selections with `ix_` exactly when this holds. + mesh = n_arrays > 1 or bool(drop_axes) + out_axes = [axis for axis in range(len(chunk_selection)) if axis not in drop_axes] + chunk_sels = list(chunk_selection) + out_sels = list(out_selection) + changed = False + for out_axis, axis in enumerate(out_axes): + chunk_sel = chunk_sels[axis] + if not (isinstance(chunk_sel, np.ndarray) and chunk_sel.dtype.kind in "iu"): continue - selected_shape.append(selector.size) - kept = ( - _last_occurrences(selector.reshape(-1), length) if _is_integer_array(selector) else None - ) - keep.append(kept) - if kept is not None: - new_selection[axis] = np.take(selector, kept, axis=axis if mesh else 0) - if all(kept is None for kept in keep): - return None - return LastAssignments( - selection=tuple(new_selection), - selected_shape=tuple(selected_shape), - flatten=False, - keep=tuple(keep), + out_sel = out_sels[out_axis] + ranks = None if isinstance(out_sel, slice) else out_sel.reshape(-1) + keep = _last_by_rank(chunk_sel.reshape(-1), ranks) + if keep is None: + continue + changed = True + chunk_sels[axis] = chunk_sel.reshape(-1)[keep] + out_sels[out_axis] = _as_positions(out_sel)[keep] + if not changed: + return projection + if mesh: + chunk_selection = np.ix_(*(sel.reshape(-1) for sel in chunk_sels)) + out_selection = np.ix_(*(_as_positions(sel) for sel in out_sels)) + else: + chunk_selection = tuple(chunk_sels) + out_selection = tuple(out_sels) + return projection._replace(chunk_selection=chunk_selection, out_selection=out_selection) + + +def _coordinate_last_writes(projection: ChunkProjection) -> ChunkProjection: + """`last_writes` for one projection of a `CoordinateIndexer`.""" + chunk_selection = cast("tuple[npt.NDArray[np.intp], ...]", projection.chunk_selection) + out_selection = cast("slice | npt.NDArray[np.intp]", projection.out_selection) + if chunk_selection[0].size < 2: + return projection + extent = tuple(int(sel.max()) + 1 for sel in chunk_selection) + targets = np.ravel_multi_index(chunk_selection, extent) + ranks = None if isinstance(out_selection, slice) else out_selection + keep = _last_by_rank(targets, ranks) + if keep is None: + return projection + return projection._replace( + chunk_selection=tuple(sel[keep] for sel in chunk_selection), + out_selection=_as_positions(out_selection)[keep], ) -def _selected_flat_indices( - selection: tuple[Any, ...], shape: tuple[int, ...] -) -> npt.NDArray[np.intp]: - """The flat index of each element `selection` names, in the layout of NumPy's result.""" - coords = [] - for axis, length in enumerate(shape): - axis_shape = tuple(length if d == axis else 1 for d in range(len(shape))) - # A broadcast view of each axis' coordinates, indexed by the selection, gives the - # coordinates the selection names without allocating an array of `shape`. - grid = np.broadcast_to(np.arange(length, dtype=np.intp).reshape(axis_shape), shape) - coords.append(grid[selection]) - return np.asarray(np.ravel_multi_index(tuple(coords), shape), dtype=np.intp) - - -def _selection_form( - selection: tuple[Any, ...], ndim: int -) -> Literal["array", "mesh", "coordinates", "other"] | None: - """Classify a selection by the forms zarr's indexers give chunk selections. - - `"array"` is one 1-D integer array among slices, `"mesh"` an `np.ix_` mesh of - integer arrays and `"coordinates"` one 1-D integer array per axis, all the same - length. `None` means the selection has no integer array, so cannot repeat an - element. - """ - arrays = [selector for selector in selection if isinstance(selector, np.ndarray)] - if not any(_is_integer_array(array) for array in arrays): - return None - if len(selection) != ndim: - return "other" - if len(arrays) == 1: - if arrays[0].ndim == 1 and all(isinstance(s, slice | np.ndarray) for s in selection): - return "array" - return "other" - if len(arrays) < ndim or not all(_is_integer_array(array) for array in arrays): - return "other" - if all( - array.ndim == ndim and all(array.shape[d] == 1 for d in range(ndim) if d != axis) - for axis, array in enumerate(arrays) - ): - return "mesh" - if all(array.ndim == 1 and array.size == arrays[0].size for array in arrays): - return "coordinates" - return "other" - - -def last_assignments(selection: SelectorTuple, shape: tuple[int, ...]) -> LastAssignments | None: - """Resolve the elements of an array of `shape` that `selection` assigns more than once. +def last_writes(indexer: Indexer) -> Iterator[ChunkProjection]: + """The projections of `indexer`, each writing every element it touches once. - Assigning through a selection that repeats an element gives that element several - values. NumPy leaves the order of those assignments undocumented and CuPy leaves it - undefined, so the last value given in the selection's C order is chosen here, - explicitly. The choice depends only on the selection, not on the order chunks are - written in. Returns `None` when no element repeats. + A selection that repeats an element gives it several values. The value kept is the + last one in the C order of the selection, the result of assigning the values in + turn. It is chosen from each item's position in the selection, so it does not + depend on how items are ordered within a chunk, on the order chunks are written + in, or on how an array library orders repeated assignments, which NumPy leaves + undocumented and CuPy undefined. """ - selectors: tuple[Any, ...] = selection if isinstance(selection, tuple) else (selection,) - form = _selection_form(selectors, len(shape)) - if form is None: - return None - if form == "array": - return _orthogonal_last_assignments(selectors, shape, mesh=False) - if form == "mesh": - return _orthogonal_last_assignments(selectors, shape, mesh=True) - if form == "coordinates": - flat = np.asarray(np.ravel_multi_index(selectors, shape), dtype=np.intp) - kept = _last_occurrences(flat, product(shape)) - if kept is None: - return None - return LastAssignments( - selection=tuple(array[kept] for array in selectors), - selected_shape=(flat.size,), - flatten=False, - keep=(kept,), - ) - flat = _selected_flat_indices(selectors, shape) - kept = _last_occurrences(flat.reshape(-1), product(shape)) - if kept is None: - return None - return LastAssignments( - selection=tuple(np.unravel_index(flat.reshape(-1)[kept], shape)), - selected_shape=flat.shape, - flatten=True, - keep=(kept,), - ) + if isinstance(indexer, CoordinateIndexer): + for projection in indexer: + yield _coordinate_last_writes(projection) + elif isinstance(indexer, OrthogonalIndexer): + for projection in indexer: + yield _orthogonal_last_writes(projection, indexer.drop_axes) + else: + # Basic and block selections name each element once. + yield from indexer @dataclass(frozen=True) @@ -1024,7 +977,7 @@ def __init__( dim_sel_chunk_sorted = dim_sel_chunk[::-1] else: # sort indices to group by chunk - dim_out_sel = stable_argsort_chunk_indices(dim_sel_chunk, nchunks) + dim_out_sel = argsort_chunk_indices(dim_sel_chunk, nchunks) dim_sel = np.take(dim_sel, dim_out_sel) dim_sel_chunk_sorted = dim_sel_chunk[dim_out_sel] @@ -1539,7 +1492,7 @@ def __init__( # group points by chunk if np.any(np.diff(chunks_raveled_indices) < 0): # optimisation, only sort if needed - sel_sort = stable_argsort_chunk_indices(chunks_raveled_indices, product(cdata_shape)) + sel_sort = argsort_chunk_indices(chunks_raveled_indices, product(cdata_shape)) selection_broadcast = tuple(dim_sel[sel_sort] for dim_sel in selection_broadcast) chunks_raveled_indices = chunks_raveled_indices[sel_sort] else: diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 6c340eecd6..ed63274739 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -27,18 +27,19 @@ CoordinateSelection, IntArrayDimIndexer, MaskIndexer, + OrthogonalIndexer, OrthogonalSelection, Selection, _ArrayIndexingOrder, _iter_grid, _iter_regions, - last_assignments, + argsort_chunk_indices, + last_writes, make_slice_selection, normalize_integer_selection, oindex, oindex_set, replace_ellipsis, - stable_argsort_chunk_indices, ) from zarr.registry import get_ndbuffer_class from zarr.storage import MemoryStore, StorePath @@ -548,68 +549,96 @@ def test_setitem_repeated_index( @pytest.mark.parametrize("nchunks", [1, 2**8, 2**16, 2**16 + 1, 2**40]) -def test_stable_argsort_chunk_indices(nchunks: int) -> None: - """The sort matches NumPy's stable sort whether or not the chunk indices are narrowed.""" +def test_argsort_chunk_indices(nchunks: int) -> None: + """The sort groups chunk indices whether or not they are narrowed.""" rng = np.random.default_rng(0) chunk_indices = np.concatenate( [[0, nchunks - 1, nchunks - 1, 0], rng.integers(0, nchunks, 1000)] ).astype(np.intp) - expected = np.argsort(chunk_indices, kind="stable") - assert_array_equal(stable_argsort_chunk_indices(chunk_indices, nchunks), expected) + order = argsort_chunk_indices(chunk_indices, nchunks) + assert_array_equal(np.sort(order), np.arange(chunk_indices.size)) + assert_array_equal(chunk_indices[order], np.sort(chunk_indices)) _rng = np.random.default_rng(0) +def _reversed_within_chunk_argsort( + chunk_indices: npt.NDArray[np.intp], nchunks: int +) -> npt.NDArray[np.intp]: + """Group by chunk, listing each chunk's items in the reverse of the order given.""" + return np.lexsort((-np.arange(chunk_indices.size), chunk_indices)) + + +@pytest.mark.parametrize("within_chunk_order", ["given", "reversed"]) @pytest.mark.parametrize( - "selection", + ("method", "selection"), [ - (np.array([1, 2, 1, 0, 2]), slice(None)), - (slice(1, 3), np.array([2, 2, 0])), - (slice(None), _rng.integers(0, 40, 100)), - (np.array([1, 2, 2]), np.array([3, 3, 3])), - (np.array([2, 1, 2]), np.array([3, 0, 3])), - (_rng.integers(0, 30, 1000), _rng.integers(0, 40, 1000)), - (np.array([1, 2, 1]), 2), - np.ix_(np.array([2, 0, 2]), np.array([1, 1, 3])), - (np.array([0, 1, 2]), slice(None)), - (slice(None), _rng.permutation(40)), - tuple(np.unravel_index(_rng.permutation(1200), (30, 40))), - np.ix_(np.array([2, 0]), np.array([1, 3])), - (slice(None), slice(1, 3)), + ("oindex", ([5, 1, 5, 20, 1, 29, 5], slice(None))), + ("oindex", (slice(2, 30), [8, 8, 0, 39, 8, 3])), + ("oindex", ([1, 1, 2, 9, 9], slice(None))), + ("oindex", (_rng.integers(0, 30, 200), slice(None))), + ("oindex", ([5, 1, 5, 20], [8, 8, 0, 39, 8])), + ("oindex", (3, [8, 8, 0, 39, 8])), + ("oindex", ([20, 3, 9], [0, 39])), + ("vindex", ([5, 1, 5, 20, 5], [8, 8, 8, 0, 8])), + ("vindex", ([5, 1, 5], [8, 0, 8])), + ("vindex", ([0, 0, 1], [0, 0, 2])), + ("vindex", (_rng.integers(0, 30, 2000), _rng.integers(0, 40, 2000))), + ("vindex", ([20, 3, 9], [0, 39, 0])), ], ids=[ - "array_then_slice", - "slice_then_array", - "long_unsorted_array", - "sorted_coordinates", - "unsorted_coordinates", - "many_coordinates", - "array_and_integer", - "mesh", - "no_repeats_orthogonal", - "no_repeats_long_unsorted_array", - "no_repeats_many_coordinates", - "no_repeats_mesh", - "no_arrays", + "oindex_array_unordered", + "oindex_array_after_slice", + "oindex_array_increasing", + "oindex_array_many", + "oindex_mesh", + "oindex_mesh_with_integer", + "oindex_no_repeats", + "vindex_unordered", + "vindex_sparse", + "vindex_increasing", + "vindex_many", + "vindex_no_repeats", ], ) -def test_last_assignments(selection: tuple[Any, ...]) -> None: - """Writing through the resolved selection matches assigning each value in C order.""" - shape = (30, 40) - targets = np.arange(product(shape)).reshape(shape)[selection] - values = np.arange(targets.size).reshape(targets.shape) +def test_last_writes( + method: str, + selection: tuple[Any, ...], + within_chunk_order: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Each projection writes every element once, and the result is assigning the values in C order. + + The result must not depend on how the indexer orders the items within a chunk. + """ + if within_chunk_order == "reversed": + monkeypatch.setattr( + zarr.core.indexing, "argsort_chunk_indices", _reversed_within_chunk_argsort + ) + shape, chunks = (30, 40), (7, 9) + selection = tuple(np.asarray(s) if isinstance(s, list) else s for s in selection) + indexer: OrthogonalIndexer | CoordinateIndexer + if method == "oindex": + indexer = OrthogonalIndexer(selection, shape, ChunkGrid.from_sizes(shape, chunks)) + else: + indexer = CoordinateIndexer(selection, shape, ChunkGrid.from_sizes(shape, chunks)) + flat_ids = np.arange(product(shape)).reshape(shape) + targets = oindex(flat_ids, selection) if method == "oindex" else flat_ids[selection] + values = np.arange(targets.size).reshape(indexer.shape) expected = np.full(product(shape), -1) for target, value in zip(targets.ravel().tolist(), values.ravel().tolist(), strict=True): expected[target] = value - result = last_assignments(selection, shape) - if np.unique(targets).size == targets.size: - assert result is None - return - assert result is not None - actual = np.full(shape, -1) - actual[result.selection] = result.pick(values) - assert_array_equal(actual.ravel(), expected) + actual = np.full(product(shape), -1) + for projection in last_writes(indexer): + region = tuple( + slice(c * size, (c + 1) * size) + for c, size in zip(projection.chunk_coords, chunks, strict=True) + ) + written = flat_ids[region][projection.chunk_selection].ravel() + assert np.unique(written).size == written.size + actual[written] = values[projection.out_selection].ravel() + assert_array_equal(actual, expected) Index = list[int] | tuple[slice | int | list[int], ...] From c5ae78fec624b0d126701847a83aa5b440f185be Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 18 Nov 2026 19:29:17 +0100 Subject: [PATCH 08/11] perf(indexing): leave repeated-index writes unspecified and radix-sort up to 2**32 chunks Match zarr-indexing (#4649): the value an element receives when an index array names it more than once is unspecified, as in NumPy. This removes `last_writes` and its per-chunk resolution from the write path, and documents the rule in the user guide and the setter docstrings. Chunk indices are now radix-sorted 16 bits at a time up to 2**32 chunks, which also covers arrays with more than 2**16 chunks: building an orthogonal indexer for 10M points over 1M chunks takes 233 ms instead of 817 ms on main. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- changes/4643.bugfix.md | 1 - changes/4643.misc.md | 1 + docs/user-guide/arrays.md | 7 ++ src/zarr/core/array.py | 14 ++-- src/zarr/core/indexing.py | 144 +++----------------------------------- tests/test_indexing.py | 129 +++++++++------------------------- 6 files changed, 60 insertions(+), 236 deletions(-) delete mode 100644 changes/4643.bugfix.md create mode 100644 changes/4643.misc.md diff --git a/changes/4643.bugfix.md b/changes/4643.bugfix.md deleted file mode 100644 index 155367cabe..0000000000 --- a/changes/4643.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Assigning through `oindex`, `vindex` or `__setitem__` with an index array that repeats a position now stores the last value given for that position, as NumPy does. Previously the stored value depended on the platform, the chunk layout and the number of selected items. Zarr now picks the last value itself, so the result no longer depends on the order in which the array library applies repeated assignments. diff --git a/changes/4643.misc.md b/changes/4643.misc.md new file mode 100644 index 0000000000..d5a9a850e9 --- /dev/null +++ b/changes/4643.misc.md @@ -0,0 +1 @@ +When an `oindex`, `vindex` or `__setitem__` index array names an element more than once, the value the element receives is now documented as unspecified, as in NumPy. Remove repeated indices if the result matters. Grouping a fancy selection by chunk is also several times faster. diff --git a/docs/user-guide/arrays.md b/docs/user-guide/arrays.md index 5b025047b6..17f9b82b0d 100644 --- a/docs/user-guide/arrays.md +++ b/docs/user-guide/arrays.md @@ -549,6 +549,13 @@ z[:] = data print(np.all(z.oindex[[0, 2], :] == z[[0, 2], :])) ``` +### Repeated indices + +If an index array used for writing names an element more than once, the element is +set to one of the values given for it, and which one is unspecified, as in NumPy. +This applies to coordinate and orthogonal writes, through `vindex`, `oindex` +or the square bracket operator. Remove repeated indices if the result matters. + ### Block indexing Zarr also supports block indexing, which allows selections of whole chunks based on their diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 5da88aadf9..e895b74958 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -109,7 +109,6 @@ is_pure_fancy_indexing, is_pure_orthogonal_indexing, is_scalar, - last_writes, pop_fields, ) from zarr.core.metadata import ( @@ -3314,6 +3313,10 @@ def set_orthogonal_selection( Slices with step > 1 are supported, but slices with negative step are not. + If `selection` names an element more than once, the element is set to one of the + values given for it, and which one is unspecified, as in NumPy. Remove repeated + indices if the result matters. + Related ------- [get_basic_selection][zarr.Array.get_basic_selection], @@ -3669,6 +3672,10 @@ def set_coordinate_selection( Slices are not supported. Coordinate arrays must be provided for all dimensions of the array. + If `selection` names an element more than once, the element is set to one of the + values given for it, and which one is unspecified, as in NumPy. Remove repeated + indices if the result matters. + Related ------- [get_basic_selection][zarr.Array.get_basic_selection], @@ -5902,10 +5909,7 @@ async def _set_selection( out_selection, is_complete_chunk, ) - # A scalar gives every element the same value, so repeats need no resolving. - for chunk_coords, chunk_selection, out_selection, is_complete_chunk in ( - indexer if value_buffer.shape == () else last_writes(indexer) - ) + for chunk_coords, chunk_selection, out_selection, is_complete_chunk in indexer ], value_buffer, drop_axes=indexer.drop_axes, diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index 7a517c7377..d5d8217522 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -767,147 +767,21 @@ def argsort_chunk_indices( ) -> npt.NDArray[np.intp]: """Return the indices that sort `chunk_indices`, each in `range(nchunks)`. - The order of items that share a chunk is unspecified: writes resolve repeated - indices by their position in the selection (see `last_writes`), not by their order - within a chunk. NumPy sorts integers of at most 16 bits with a radix sort, which is - several times faster than its default sort, so the indices are narrowed when - `nchunks` allows it. + NumPy sorts integers of at most 16 bits with a radix sort, several times faster than + its default sort, so up to `2**32` chunks the indices are sorted 16 bits at a time, + low half first. Each pass is stable, so items that share a chunk keep their order. + That order is an implementation detail: the value stored for a repeated index is + unspecified. """ if nchunks <= 2**16: return np.argsort(chunk_indices.astype(np.uint16), kind="stable") + if nchunks <= 2**32: + low = np.argsort((chunk_indices & 0xFFFF).astype(np.uint16), kind="stable") + high = (chunk_indices[low] >> 16).astype(np.uint16) + return low[np.argsort(high, kind="stable")] return np.argsort(chunk_indices) -def _last_by_rank( - targets: npt.NDArray[Any], ranks: npt.NDArray[Any] | None -) -> npt.NDArray[np.intp] | None: - """Positions of the entries to keep so that each target keeps its highest-ranked entry. - - `targets` holds non-negative integers that may repeat. `ranks` holds distinct - integers, one per entry; `None` means each entry's rank is its position. Returns - `None` when no target repeats. - """ - n = targets.size - if n < 2: - return None - if ranks is None and (targets[1:] >= targets[:-1]).all(): - # Non-decreasing targets ranked by position: repeats are adjacent, and the last - # of each run ranks highest. - steps = targets[1:] != targets[:-1] - if steps.all(): - return None - return np.flatnonzero(np.append(steps, True)) - if (targets[1:] > targets[:-1]).all(): - return None - if ranks is None: - ranks = np.arange(n, dtype=np.intp) - size = int(targets.max()) + 1 - if size <= 16 * n: - # Few possible targets for the number given: record the highest rank of each in - # linear time. A maximum does not depend on the order `ufunc.at` applies - # repeated targets in. - highest = np.full(size, np.iinfo(np.intp).min, dtype=np.intp) - np.maximum.at(highest, targets, ranks) - keep = np.flatnonzero(highest[targets] == ranks) - else: - order = np.lexsort((ranks, targets)) - ordered = targets[order] - keep = order[np.append(ordered[1:] != ordered[:-1], True)] - if keep.size == n: - return None - return keep - - -def _as_positions(out_sel: slice | npt.NDArray[Any]) -> npt.NDArray[np.intp]: - """The output positions a 1-D output selection names, as an array. - - Indexers give output slices an explicit start and stop, and no step. - """ - if isinstance(out_sel, slice): - return np.arange(out_sel.start, out_sel.stop, dtype=np.intp) - return out_sel.reshape(-1) - - -def _orthogonal_last_writes( - projection: ChunkProjection, drop_axes: tuple[int, ...] -) -> ChunkProjection: - """`last_writes` for one projection of an `OrthogonalIndexer`. - - An element repeats only where some axis repeats an index, and its last value in C - order is the one at the highest output position on each axis, so each axis is - resolved on its own. - """ - chunk_selection = cast("tuple[Any, ...]", projection.chunk_selection) - out_selection = cast("tuple[Any, ...]", projection.out_selection) - n_arrays = sum(isinstance(sel, np.ndarray) for sel in chunk_selection) - # The indexer meshes the selections with `ix_` exactly when this holds. - mesh = n_arrays > 1 or bool(drop_axes) - out_axes = [axis for axis in range(len(chunk_selection)) if axis not in drop_axes] - chunk_sels = list(chunk_selection) - out_sels = list(out_selection) - changed = False - for out_axis, axis in enumerate(out_axes): - chunk_sel = chunk_sels[axis] - if not (isinstance(chunk_sel, np.ndarray) and chunk_sel.dtype.kind in "iu"): - continue - out_sel = out_sels[out_axis] - ranks = None if isinstance(out_sel, slice) else out_sel.reshape(-1) - keep = _last_by_rank(chunk_sel.reshape(-1), ranks) - if keep is None: - continue - changed = True - chunk_sels[axis] = chunk_sel.reshape(-1)[keep] - out_sels[out_axis] = _as_positions(out_sel)[keep] - if not changed: - return projection - if mesh: - chunk_selection = np.ix_(*(sel.reshape(-1) for sel in chunk_sels)) - out_selection = np.ix_(*(_as_positions(sel) for sel in out_sels)) - else: - chunk_selection = tuple(chunk_sels) - out_selection = tuple(out_sels) - return projection._replace(chunk_selection=chunk_selection, out_selection=out_selection) - - -def _coordinate_last_writes(projection: ChunkProjection) -> ChunkProjection: - """`last_writes` for one projection of a `CoordinateIndexer`.""" - chunk_selection = cast("tuple[npt.NDArray[np.intp], ...]", projection.chunk_selection) - out_selection = cast("slice | npt.NDArray[np.intp]", projection.out_selection) - if chunk_selection[0].size < 2: - return projection - extent = tuple(int(sel.max()) + 1 for sel in chunk_selection) - targets = np.ravel_multi_index(chunk_selection, extent) - ranks = None if isinstance(out_selection, slice) else out_selection - keep = _last_by_rank(targets, ranks) - if keep is None: - return projection - return projection._replace( - chunk_selection=tuple(sel[keep] for sel in chunk_selection), - out_selection=_as_positions(out_selection)[keep], - ) - - -def last_writes(indexer: Indexer) -> Iterator[ChunkProjection]: - """The projections of `indexer`, each writing every element it touches once. - - A selection that repeats an element gives it several values. The value kept is the - last one in the C order of the selection, the result of assigning the values in - turn. It is chosen from each item's position in the selection, so it does not - depend on how items are ordered within a chunk, on the order chunks are written - in, or on how an array library orders repeated assignments, which NumPy leaves - undocumented and CuPy undefined. - """ - if isinstance(indexer, CoordinateIndexer): - for projection in indexer: - yield _coordinate_last_writes(projection) - elif isinstance(indexer, OrthogonalIndexer): - for projection in indexer: - yield _orthogonal_last_writes(projection, indexer.drop_axes) - else: - # Basic and block selections name each element once. - yield from indexer - - @dataclass(frozen=True) class IntArrayDimIndexer: """Integer array selection against a single dimension.""" diff --git a/tests/test_indexing.py b/tests/test_indexing.py index ed63274739..00f4957449 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -27,14 +27,12 @@ CoordinateSelection, IntArrayDimIndexer, MaskIndexer, - OrthogonalIndexer, OrthogonalSelection, Selection, _ArrayIndexingOrder, _iter_grid, _iter_regions, argsort_chunk_indices, - last_writes, make_slice_selection, normalize_integer_selection, oindex, @@ -510,6 +508,13 @@ def test_setitem_zarr_array_as_value() -> None: assert_array_equal(dst2[2:7], np.arange(2, 7)) +def _reversed_within_chunk_argsort( + chunk_indices: npt.NDArray[np.intp], nchunks: int +) -> npt.NDArray[np.intp]: + """Group `chunk_indices` by chunk, reversing the order of items within each chunk.""" + return np.lexsort((-np.arange(chunk_indices.size), chunk_indices)) + + @pytest.mark.parametrize("method", ["oindex", "vindex"]) @pytest.mark.parametrize("layout", ["unit_chunks", "partial_chunks", "one_chunk", "sharded"]) @pytest.mark.parametrize( @@ -517,16 +522,25 @@ def test_setitem_zarr_array_as_value() -> None: [ ((24,), ([-1, -1, 0, 0],)), ((24,), ([-1, 0, 0, -1],)), - # long enough that an unstable sort by chunk reorders the repeats ((24,), ([1] + [0] * 23,)), ((6, 6), ([0, 5, 0, -1, 2] * 4, [1, 1, 4, 4, 2] * 4)), ], ids=["adjacent", "interleaved", "many_repeats", "two_dimensions"], ) +@pytest.mark.parametrize("within_chunk_order", ["default", "reversed"]) def test_setitem_repeated_index( - method: str, layout: str, shape: tuple[int, ...], selection: tuple[list[int], ...] + method: str, + layout: str, + shape: tuple[int, ...], + selection: tuple[list[int], ...], + within_chunk_order: str, + monkeypatch: pytest.MonkeyPatch, ) -> None: - """Assignment with repeated indices stores the last value given for each repeated position, as NumPy does.""" + """Assignment with repeated indices sets each repeated element to one of its values, whole, and leaves the rest as NumPy would.""" + if within_chunk_order == "reversed": + monkeypatch.setattr( + zarr.core.indexing, "argsort_chunk_indices", _reversed_within_chunk_argsort + ) chunks = { "unit_chunks": (1,) * len(shape), "partial_chunks": (4,) * len(shape), @@ -535,22 +549,26 @@ def test_setitem_repeated_index( }[layout] shards = (6,) * len(shape) if layout == "sharded" else None array = zarr.create_array(store={}, shape=shape, chunks=chunks, shards=shards, dtype="i8") - expected = np.zeros(shape, dtype="i8") + flat_ids = np.arange(product(shape)).reshape(shape) if method == "oindex": - np_selection = np.ix_(*selection) - value = np.arange(1, 1 + expected[np_selection].size).reshape(expected[np_selection].shape) - expected[np_selection] = value + targets = flat_ids[np.ix_(*selection)] + value = np.arange(1, 1 + targets.size).reshape(targets.shape) array.oindex[selection] = value else: - value = np.arange(1, 1 + len(selection[0])) - expected[selection] = value + targets = flat_ids[selection] + value = np.arange(1, 1 + targets.size) array.vindex[selection] = value - assert_array_equal(array[:], expected) + written = array[:].reshape(-1) + targets = targets.reshape(-1) + value = value.reshape(-1) + assert (np.delete(written, targets) == 0).all() + for target in np.unique(targets): + assert written[target] in value[targets == target] -@pytest.mark.parametrize("nchunks", [1, 2**8, 2**16, 2**16 + 1, 2**40]) +@pytest.mark.parametrize("nchunks", [1, 2**8, 2**16, 2**16 + 1, 2**20, 2**32, 2**32 + 1, 2**40]) def test_argsort_chunk_indices(nchunks: int) -> None: - """The sort groups chunk indices whether or not they are narrowed.""" + """The sort groups items by chunk, and up to `2**32` chunks matches NumPy's stable sort.""" rng = np.random.default_rng(0) chunk_indices = np.concatenate( [[0, nchunks - 1, nchunks - 1, 0], rng.integers(0, nchunks, 1000)] @@ -558,87 +576,8 @@ def test_argsort_chunk_indices(nchunks: int) -> None: order = argsort_chunk_indices(chunk_indices, nchunks) assert_array_equal(np.sort(order), np.arange(chunk_indices.size)) assert_array_equal(chunk_indices[order], np.sort(chunk_indices)) - - -_rng = np.random.default_rng(0) - - -def _reversed_within_chunk_argsort( - chunk_indices: npt.NDArray[np.intp], nchunks: int -) -> npt.NDArray[np.intp]: - """Group by chunk, listing each chunk's items in the reverse of the order given.""" - return np.lexsort((-np.arange(chunk_indices.size), chunk_indices)) - - -@pytest.mark.parametrize("within_chunk_order", ["given", "reversed"]) -@pytest.mark.parametrize( - ("method", "selection"), - [ - ("oindex", ([5, 1, 5, 20, 1, 29, 5], slice(None))), - ("oindex", (slice(2, 30), [8, 8, 0, 39, 8, 3])), - ("oindex", ([1, 1, 2, 9, 9], slice(None))), - ("oindex", (_rng.integers(0, 30, 200), slice(None))), - ("oindex", ([5, 1, 5, 20], [8, 8, 0, 39, 8])), - ("oindex", (3, [8, 8, 0, 39, 8])), - ("oindex", ([20, 3, 9], [0, 39])), - ("vindex", ([5, 1, 5, 20, 5], [8, 8, 8, 0, 8])), - ("vindex", ([5, 1, 5], [8, 0, 8])), - ("vindex", ([0, 0, 1], [0, 0, 2])), - ("vindex", (_rng.integers(0, 30, 2000), _rng.integers(0, 40, 2000))), - ("vindex", ([20, 3, 9], [0, 39, 0])), - ], - ids=[ - "oindex_array_unordered", - "oindex_array_after_slice", - "oindex_array_increasing", - "oindex_array_many", - "oindex_mesh", - "oindex_mesh_with_integer", - "oindex_no_repeats", - "vindex_unordered", - "vindex_sparse", - "vindex_increasing", - "vindex_many", - "vindex_no_repeats", - ], -) -def test_last_writes( - method: str, - selection: tuple[Any, ...], - within_chunk_order: str, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Each projection writes every element once, and the result is assigning the values in C order. - - The result must not depend on how the indexer orders the items within a chunk. - """ - if within_chunk_order == "reversed": - monkeypatch.setattr( - zarr.core.indexing, "argsort_chunk_indices", _reversed_within_chunk_argsort - ) - shape, chunks = (30, 40), (7, 9) - selection = tuple(np.asarray(s) if isinstance(s, list) else s for s in selection) - indexer: OrthogonalIndexer | CoordinateIndexer - if method == "oindex": - indexer = OrthogonalIndexer(selection, shape, ChunkGrid.from_sizes(shape, chunks)) - else: - indexer = CoordinateIndexer(selection, shape, ChunkGrid.from_sizes(shape, chunks)) - flat_ids = np.arange(product(shape)).reshape(shape) - targets = oindex(flat_ids, selection) if method == "oindex" else flat_ids[selection] - values = np.arange(targets.size).reshape(indexer.shape) - expected = np.full(product(shape), -1) - for target, value in zip(targets.ravel().tolist(), values.ravel().tolist(), strict=True): - expected[target] = value - actual = np.full(product(shape), -1) - for projection in last_writes(indexer): - region = tuple( - slice(c * size, (c + 1) * size) - for c, size in zip(projection.chunk_coords, chunks, strict=True) - ) - written = flat_ids[region][projection.chunk_selection].ravel() - assert np.unique(written).size == written.size - actual[written] = values[projection.out_selection].ravel() - assert_array_equal(actual, expected) + if nchunks <= 2**32: + assert_array_equal(order, np.argsort(chunk_indices, kind="stable")) Index = list[int] | tuple[slice | int | list[int], ...] From 96faa6bc772d13b08ed25410bb65c4d5d36e88f8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 18 Nov 2026 19:29:50 +0100 Subject: [PATCH 09/11] test(properties): skip repeated indices again, since their written value is unspecified Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- tests/test_properties.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/test_properties.py b/tests/test_properties.py index d5d385accf..b448a524aa 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -17,7 +17,7 @@ import hypothesis.extra.numpy as npst import hypothesis.strategies as st -from hypothesis import event, given, settings +from hypothesis import assume, event, given, settings from tests.conftest import declared_chunk_data_sizes from zarr.abc.store import Store @@ -199,8 +199,12 @@ async def test_oindex(data: st.DataObject) -> None: actual = await async_zarray.oindex.getitem(zindexer) assert_array_equal(nparray[npindexer], actual) - # sync set: an index that repeats a position (in either spelling) keeps the last value - # written to it, as in NumPy, so repeated indices are drawn like any other + # sync set + for idxr, size in zip(zindexer, nparray.shape, strict=True): + if isinstance(idxr, np.ndarray) and idxr.size != np.unique(idxr % size).size: + # behaviour of setitem with repeated indices is not guaranteed in practice + # Negative and positive spellings of the same index are duplicates too. + assume(False) # The sharding codec sees a coordinate selection (the GH4284 path) when the # chunk selection has more than one array axis or drops an integer axis. n_array_axes = sum(isinstance(idxr, np.ndarray) for idxr in zindexer) @@ -244,7 +248,19 @@ async def test_vindex(data: st.DataObject) -> None: actual = await async_zarray.vindex.getitem(indexer) assert_array_equal(nparray[indexer], actual) - # sync set: a repeated point keeps the last value written to it, as in NumPy + # sync set + # Reads preserve the supplied indices; normalize negative indices explicitly when + # detecting repeated points rather than relying on a read to mutate the indexer. + points = np.stack( + [ + (idxr % size).ravel() + for idxr, size in zip(np.broadcast_arrays(*indexer), nparray.shape, strict=True) + ], + axis=-1, + ) + if len(np.unique(points, axis=0)) != len(points): + # behaviour of setitem with repeated coordinates is not guaranteed in practice + assume(False) new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) nparray[indexer] = new_data zarray.vindex[indexer] = new_data From 31e9588645bcc3f09d784e63f95bea855aa6476f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 18 Nov 2026 20:42:55 +0100 Subject: [PATCH 10/11] perf(indexing): radix-sort chunk indices for any chunk count Sort 16 bits at a time for as many passes as the chunk count needs (up to four), instead of falling back to NumPy's default sort above 2**32 chunks. Building an orthogonal indexer for 10M points over 2**40 chunks takes 342 ms instead of 838 ms. Below 128 items per pass the plain stable sort is faster and is used instead, so the sort is stable for every input. Assisted-by: ClaudeCode:claude-fable-5-2 Co-Authored-By: Claude Fable 5.2 --- src/zarr/core/indexing.py | 29 ++++++++++++++++++----------- tests/test_indexing.py | 19 ++++++++++--------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index d5d8217522..e4384f236f 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -762,24 +762,31 @@ def sorted_run_ends( return a[run_starts].astype(np.intp, copy=False), run_ends +# Below this many items per 16-bit pass, NumPy's stable sort of the full-width indices is +# faster than radix-sorting them. +_RADIX_SORT_MIN_ITEMS_PER_PASS = 128 + + def argsort_chunk_indices( chunk_indices: npt.NDArray[np.intp], nchunks: int ) -> npt.NDArray[np.intp]: """Return the indices that sort `chunk_indices`, each in `range(nchunks)`. NumPy sorts integers of at most 16 bits with a radix sort, several times faster than - its default sort, so up to `2**32` chunks the indices are sorted 16 bits at a time, - low half first. Each pass is stable, so items that share a chunk keep their order. - That order is an implementation detail: the value stored for a repeated index is - unspecified. + its default sort, so the indices are sorted 16 bits at a time, lowest bits first. + For a few indices the plain stable sort is faster and is used instead. Either way + the sort is stable, so items that share a chunk keep their order. That order is an + implementation detail: the value stored for a repeated index is unspecified. """ - if nchunks <= 2**16: - return np.argsort(chunk_indices.astype(np.uint16), kind="stable") - if nchunks <= 2**32: - low = np.argsort((chunk_indices & 0xFFFF).astype(np.uint16), kind="stable") - high = (chunk_indices[low] >> 16).astype(np.uint16) - return low[np.argsort(high, kind="stable")] - return np.argsort(chunk_indices) + # ceil(bits / 16) passes; chunk indices are at most 64 bits wide + n_passes = min(4, max(1, -((nchunks - 1).bit_length() // -16))) + if chunk_indices.size < _RADIX_SORT_MIN_ITEMS_PER_PASS * n_passes: + return np.argsort(chunk_indices, kind="stable") + order = np.argsort(chunk_indices.astype(np.uint16), kind="stable") + for shift in range(16, 16 * n_passes, 16): + digit = (chunk_indices[order] >> shift).astype(np.uint16) + order = order[np.argsort(digit, kind="stable")] + return order @dataclass(frozen=True) diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 00f4957449..2e010cc134 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -566,18 +566,19 @@ def test_setitem_repeated_index( assert written[target] in value[targets == target] -@pytest.mark.parametrize("nchunks", [1, 2**8, 2**16, 2**16 + 1, 2**20, 2**32, 2**32 + 1, 2**40]) -def test_argsort_chunk_indices(nchunks: int) -> None: - """The sort groups items by chunk, and up to `2**32` chunks matches NumPy's stable sort.""" +@pytest.mark.parametrize("n_items", [100, 2000]) +@pytest.mark.parametrize( + "nchunks", [1, 2**8, 2**16, 2**16 + 1, 2**32, 2**32 + 1, 2**48, 2**48 + 1, 2**62, 2**70] +) +def test_argsort_chunk_indices(nchunks: int, n_items: int) -> None: + """The sort matches NumPy's stable sort for any number of chunks, and for few or many items.""" rng = np.random.default_rng(0) + highest = min(nchunks, 2**63) - 1 chunk_indices = np.concatenate( - [[0, nchunks - 1, nchunks - 1, 0], rng.integers(0, nchunks, 1000)] + [[0, highest, highest, 0], rng.integers(0, highest + 1, n_items - 4)] ).astype(np.intp) - order = argsort_chunk_indices(chunk_indices, nchunks) - assert_array_equal(np.sort(order), np.arange(chunk_indices.size)) - assert_array_equal(chunk_indices[order], np.sort(chunk_indices)) - if nchunks <= 2**32: - assert_array_equal(order, np.argsort(chunk_indices, kind="stable")) + expected = np.argsort(chunk_indices, kind="stable") + assert_array_equal(argsort_chunk_indices(chunk_indices, nchunks), expected) Index = list[int] | tuple[slice | int | list[int], ...] From caa15365f6807a37dc99f4384cbf0694c2372ab4 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Wed, 18 Nov 2026 21:51:55 +0100 Subject: [PATCH 11/11] perf(zarr-indexing): group chunk points with a radix sort and stop deduplicating writes (#4649) * perf(zarr-indexing): group chunk points with an unstable sort and stop deduplicating writes Planning grouped each chunk's points with a stable sort so that every `IndexedSet` and `JointSet` row kept request order, and the writer relied on that order to make the last repeated write win. Both promises are dropped: the order of points within a row is unspecified, and a source cell a write addresses more than once receives one of its values, whole, without saying which. Callers who need a particular value remove repeats from the selection first. Grouping now uses the default sort behind `_group_by_key`, which also skips the sort for already-grouped joint keys, and the writer no longer deduplicates targets before its single fancy assignment or `np.put`. Tests that assumed last-wins now accept any written value, and a new test runs orthogonal, vectorized and mixed repeated writes with the grouping replaced by one that reverses each row. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * docs(zarr-indexing): add changelog fragment for unspecified repeated-write order Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * docs(zarr-indexing): number the changelog fragment for upstream PR 4649 Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * perf(zarr-indexing): group chunk keys with a 16-bit radix sort NumPy's stable sort is a linear-time radix sort for integers of at most 16 bits. `_group_by_key` shifts keys to start at zero and sorts them one 16-bit digit at a time: one pass when their range fits 16 bits, two least-significant-first passes when it fits 32 bits, and the default sort beyond. Grouping 1M random keys over 65,536 chunks drops from about 46 ms to 2 ms. The order within a group stays unspecified. The approach follows zarr-python #4643's `argsort_chunk_indices`, with a shift by the minimum key because grids may have negative chunk IDs. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * perf(zarr-indexing): radix-sort chunk keys of any width `_group_by_key` fell back to the default sort above a 2**32 key range, which was slower than radix passes and, as zarr-python #4643 measured, slower at every key width. It now takes one 16-bit pass per 16 bits of the shifted key range, up to four, reading the shifted keys as uint64 so a range above 2**63 stays exact. Inputs under 128 keys per pass take one stable comparison sort, which beats the passes' fixed overhead. This mirrors #4643's `argsort_chunk_indices`; the order within a group stays unspecified. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 * docs(zarr-indexing): record why repeated write targets get no promised value Add a "Repeated write targets" design note: the library promises no value for a cell a write addresses more than once, chooses its internal order by performance (today a stable radix sort that happens to keep the last write), carries no order within a planning-table row, and does not deduplicate, because the pass would cost more than the assignment. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- packages/zarr-indexing/changes/4603.misc.md | 1 - .../zarr-indexing/changes/4649.removal.md | 1 + packages/zarr-indexing/docs/design-notes.md | 33 ++++ .../docs/guide/selection-flow.md | 6 +- .../docs/snippets/grid_partition.py | 10 +- .../src/zarr_indexing/chunk_resolution.py | 71 ++++++-- .../src/zarr_indexing/lazy_array.py | 7 +- .../zarr-indexing/src/zarr_indexing/writer.py | 33 +--- .../tests/test_chunk_resolution.py | 38 ++++ packages/zarr-indexing/tests/test_writer.py | 164 +++++++++++++----- 10 files changed, 267 insertions(+), 97 deletions(-) delete mode 100644 packages/zarr-indexing/changes/4603.misc.md create mode 100644 packages/zarr-indexing/changes/4649.removal.md diff --git a/packages/zarr-indexing/changes/4603.misc.md b/packages/zarr-indexing/changes/4603.misc.md deleted file mode 100644 index 3015c99f22..0000000000 --- a/packages/zarr-indexing/changes/4603.misc.md +++ /dev/null @@ -1 +0,0 @@ -Writes whose indices land on a chunk in sorted order no longer sort them to drop repeated indices, so deduplication is linear in the number of points; writes without repeats also skip a gather copy. diff --git a/packages/zarr-indexing/changes/4649.removal.md b/packages/zarr-indexing/changes/4649.removal.md new file mode 100644 index 0000000000..82386e9e01 --- /dev/null +++ b/packages/zarr-indexing/changes/4649.removal.md @@ -0,0 +1 @@ +`LazyArray.write` no longer promises that the last of several writes to one source cell wins: the cell receives one of the values written to it, whole, but which one is unspecified, so remove repeated indices from a selection whose result matters. `IndexedSet` and `JointSet` no longer promise an order for the points within a row either. Planning now groups points by chunk with a linear-time radix sort and writes skip deduplication, so partitioning a large unordered selection is up to 5x faster and a fancy write to a NumPy array about 10x faster. diff --git a/packages/zarr-indexing/docs/design-notes.md b/packages/zarr-indexing/docs/design-notes.md index 9efed99769..26534e6a93 100644 --- a/packages/zarr-indexing/docs/design-notes.md +++ b/packages/zarr-indexing/docs/design-notes.md @@ -193,6 +193,39 @@ rectangular-only view is a plausible next step, but it should be introduced by a consumer that needs the guarantee in its signatures rather than speculatively; `is_box` is the runtime check until then. +## Repeated write targets + +A query can address one storage cell more than once: `a.vindex[[1, 1]] = [10, 20]` +writes two values to one cell. Only one value can remain, and this library +does not promise which. `LazyArray.write` gives the cell one of the values +written to it, whole (on a masked NumPy array, the value and its mask come +from the same write), and leaves the choice unspecified. A caller whose result depends on the choice removes repeated +coordinates before writing, for example with `np.unique(..., return_index=True)`. + +Three decisions follow from that rule: + +- **Performance decides the internal order.** Planning groups each chunk's + points with whatever sort is fastest. That is currently NumPy's 16-bit radix + sort, which is stable, so a chunk's points stay in request order, and NumPy + assignment applies repeated targets in sequence in practice. Writes therefore + currently keep the last value in C order of the view. This describes the + implementation, not a contract: a faster planner or a NumPy change may pick a + different value, and tests must not depend on which one lands. +- **Planning tables carry no order within a row.** `IndexedSet` and `JointSet` + pair every point with its request position, so result placement survives + planning, but the order of points within a row is unspecified. A consumer + that needs request order sorts a row by `positions`. +- **The library does not deduplicate.** Removing repeats, or detecting them to + raise an error, needs a pass over each chunk's targets that can cost more + than the assignment itself: for a 200,000-point unordered selection written + to a NumPy array, the pass took about ten times as long as the assignment. + That cost would fall on every write to protect the few with ambiguous + input, so the caller who needs a particular value supplies unambiguous input. + +NumPy takes the same position: it documents that the final value of an element +assigned more than once through advanced indexing cannot be predicted. Array +libraries that scatter in parallel make the same choice. + ## Negative-origin domains and prependable grids Literal coordinates let a domain grow at its lower end without changing the diff --git a/packages/zarr-indexing/docs/guide/selection-flow.md b/packages/zarr-indexing/docs/guide/selection-flow.md index 749a9c36ae..662d5db223 100644 --- a/packages/zarr-indexing/docs/guide/selection-flow.md +++ b/packages/zarr-indexing/docs/guide/selection-flow.md @@ -194,8 +194,10 @@ that the supplied buffer can be copied directly into codec order. Touching every chunk alone does not prove full coverage of each chunk. Planning does not fetch bytes, decode buffers, choose concurrency, or define -write-conflict handling. Consumers must preserve request ordering where their -write API requires it, and account for memory retained by emitted operations. +write-conflict handling. The order of points within one chunk's projection is +unspecified; each point stays paired with its request position, so a consumer +whose write API depends on request order sorts by those positions. Consumers +also account for memory retained by emitted operations. ## 6. Applying the flow to sharding diff --git a/packages/zarr-indexing/docs/snippets/grid_partition.py b/packages/zarr-indexing/docs/snippets/grid_partition.py index ab7f1ee38d..2d518333e3 100644 --- a/packages/zarr-indexing/docs/snippets/grid_partition.py +++ b/packages/zarr-indexing/docs/snippets/grid_partition.py @@ -40,8 +40,9 @@ assert isinstance(rows, IndexedSet) assert rows.chunk.tolist() == [0, 1] # rows 1 and 1 land in chunk 0, row 4 in chunk 1 assert rows.pointer.tolist() == [0, 2, 3] # chunk 0 owns entries 0:2, chunk 1 owns 2:3 -assert rows.index.tolist() == [1, 1, 4] # grouped by chunk, request order within a chunk -assert rows.positions.tolist() == [1, 2, 0] # the request positions those entries fill +assert rows.index.tolist() == [1, 1, 4] # grouped by chunk, any order within one +assert sorted(rows.positions[rows.run(0)].tolist()) == [1, 2] # request positions chunk 0 fills +assert rows.positions[rows.run(1)].tolist() == [0] assert rows.local.tolist() == [1, 1, 1] # chunk-local: 1 - 0, 1 - 0, 4 - 3 points = IndexTransform.from_shape((6, 8)).vindex[np.array([0, 5, 5]), np.array([7, 0, 1])] @@ -49,8 +50,9 @@ assert joint is not None assert joint.chunk.tolist() == [[0, 1], [1, 0]] # touched chunks, lexicographic assert joint.pointer.tolist() == [0, 1, 3] # point 0 alone; points 1 and 2 share a chunk -assert joint.positions.tolist() == [0, 1, 2] -assert joint.local.tolist() == [[0, 3], [2, 0], [2, 1]] +# Each point keeps its position and chunk-local coordinates paired. +pairs = sorted(zip(joint.positions.tolist(), joint.local.tolist(), strict=True)) +assert pairs == [(0, [0, 3]), (1, [2, 0]), (2, [2, 1])] # --8<-- [end:indexed-and-joint] diff --git a/packages/zarr-indexing/src/zarr_indexing/chunk_resolution.py b/packages/zarr-indexing/src/zarr_indexing/chunk_resolution.py index 01383b8e34..0e5c62b32b 100644 --- a/packages/zarr-indexing/src/zarr_indexing/chunk_resolution.py +++ b/packages/zarr-indexing/src/zarr_indexing/chunk_resolution.py @@ -375,10 +375,11 @@ class IndexedSet: """One output dimension read through an orthogonal `ArrayMap`, one row per chunk. The map's coordinates are grouped by chunk in CSR form: row ``i`` owns - ``index[pointer[i]:pointer[i + 1]]`` (the index-array values, in request - order) and ``positions[pointer[i]:pointer[i + 1]]`` (their positions along - the request axis, ascending). `local` gives the same values as chunk-local - storage coordinates. Columns are read-only NumPy arrays. + ``index[pointer[i]:pointer[i + 1]]`` (the index-array values) and + ``positions[pointer[i]:pointer[i + 1]]`` (their positions along the request + axis), paired entry by entry. The order of entries within a row is + unspecified. `local` gives the same values as chunk-local storage + coordinates. Columns are read-only NumPy arrays. Examples -------- @@ -390,8 +391,8 @@ class IndexedSet: >>> (axis,) = plan_chunks(transform, grids).partition().sets >>> axis.chunk.tolist(), axis.pointer.tolist() ([0, 2], [0, 2, 4]) - >>> axis.local.tolist(), axis.positions.tolist() - ([1, 2, 1, 0], [1, 2, 0, 3]) + >>> sorted(zip(axis.positions.tolist(), axis.local.tolist())) + [(0, 1), (1, 1), (2, 2), (3, 0)] """ output_dimension: int @@ -422,7 +423,7 @@ class IndexedSet: """Index-array values grouped by chunk.""" positions: np.ndarray[Any, np.dtype[np.intp]] - """Positions along the request axis, grouped by chunk, ascending within a row.""" + """Positions along the request axis, grouped by chunk like `index`.""" def __post_init__(self) -> None: _own_columns(self, "chunk", "chunk_start", "chunk_extent", "pointer", "index", "positions") @@ -465,8 +466,8 @@ class JointSet: `output_dimensions`, and CSR range ``pointer[i]:pointer[i + 1]`` its points: `index` holds their index-array values per output dimension, `positions` their flat positions in this component's input block, and - `block_coordinates` those positions unravelled over the block. Columns are - read-only NumPy arrays. + `block_coordinates` those positions unravelled over the block. The order of + points within a row is unspecified. Columns are read-only NumPy arrays. Examples -------- @@ -478,8 +479,10 @@ class JointSet: ... np.array([0, 6, 6, 1]), np.array([8, 0, 1, 8]) ... ] >>> (joint,) = plan_chunks(transform, grids).partition().joint_sets - >>> joint.chunk.tolist(), joint.pointer.tolist(), joint.positions.tolist() - ([[0, 2], [2, 0]], [0, 2, 4], [0, 3, 1, 2]) + >>> joint.chunk.tolist(), joint.pointer.tolist() + ([[0, 2], [2, 0]], [0, 2, 4]) + >>> [sorted(joint.positions[joint.run(row)].tolist()) for row in range(len(joint))] + [[0, 3], [1, 2]] """ output_dimensions: tuple[int, ...] @@ -513,7 +516,7 @@ class JointSet: """Index-array values per point and output dimension, shape ``(points, k)``.""" positions: np.ndarray[Any, np.dtype[np.intp]] - """Flat block position of each point, ascending within a row.""" + """Flat block position of each point, grouped by row like `index`.""" _block_coordinates: np.ndarray[Any, np.dtype[np.intp]] | None = field( default=None, init=False, repr=False, compare=False @@ -654,13 +657,12 @@ def _indexed_set(out_dim: int, m: ArrayMap, dg: DimensionGridLike) -> IndexedSet dg.index_to_chunk(int(storage.min())) dg.index_to_chunk(int(storage.max())) chunk_ids = dg.indices_to_chunks(storage) - # Already-sorted coordinates (the common case) need no sort. - if bool((chunk_ids[1:] >= chunk_ids[:-1]).all()): + positions = _group_by_key(chunk_ids) + if positions is None: positions = np.arange(n, dtype=np.intp) sorted_ids = chunk_ids index = flat else: - positions = np.argsort(chunk_ids, kind="stable") sorted_ids = chunk_ids[positions] index = flat[positions] boundaries = np.flatnonzero(sorted_ids[1:] != sorted_ids[:-1]) + 1 @@ -681,6 +683,42 @@ def _indexed_set(out_dim: int, m: ArrayMap, dg: DimensionGridLike) -> IndexedSet ) +# Below this many keys per radix pass, one stable comparison sort is cheaper +# than the passes' fixed overhead (about 1 us each). +_RADIX_SORT_MIN_ITEMS_PER_PASS = 128 + + +def _group_by_key( + keys: np.ndarray[Any, np.dtype[np.intp]], +) -> np.ndarray[Any, np.dtype[np.intp]] | None: + """An order that groups equal keys in ascending key order, or None if `keys` already do. + + NumPy's stable sort is a linear-time radix sort for integers of at most 16 + bits, so keys shifted to start at zero are sorted one 16-bit digit at a + time, least significant first: one pass per 16 bits of their range, at + most four. Small inputs take one stable comparison sort instead. Both + branches happen to keep input order within a group, but that order is + unspecified and callers must not rely on it. + """ + if bool((keys[1:] >= keys[:-1]).all()): + return None + low = int(keys.min()) + span = int(keys.max()) - low + 1 + # ceil(bits / 16) passes; the range of int64 keys needs at most 64 bits. + n_passes = min(4, max(1, -((span - 1).bit_length() // -16))) + if keys.size < _RADIX_SORT_MIN_ITEMS_PER_PASS * n_passes: + return np.asarray(np.argsort(keys, kind="stable"), dtype=np.intp) + # A range above 2**63 wraps in int64, but the difference is exact modulo + # 2**64, so reading it as uint64 recovers the shifted keys. + shifted = (keys - low).view(np.uint64) + # `astype(np.uint16)` keeps the low 16 bits. + order = np.argsort(shifted.astype(np.uint16), kind="stable") + for shift in range(16, 16 * n_passes, 16): + digit = (shifted[order] >> np.uint64(shift)).astype(np.uint16) + order = order[np.argsort(digit, kind="stable")] + return np.asarray(order, dtype=np.intp) + + def _chunk_keys( chunk_ids: Sequence[np.ndarray[Any, np.dtype[np.intp]]], ) -> np.ndarray[Any, np.dtype[np.intp]]: @@ -724,7 +762,8 @@ def _joint_set( chunk_ids = [dim_grids[d].indices_to_chunks(block.flat_storage[d]) for d in dims] n_points = int(chunk_ids[0].size) keys = _chunk_keys(chunk_ids) - positions = np.argsort(keys, kind="stable") + grouped = _group_by_key(keys) + positions = np.arange(n_points, dtype=np.intp) if grouped is None else grouped sorted_keys = keys[positions] boundaries = np.flatnonzero(sorted_keys[1:] != sorted_keys[:-1]) + 1 pointer = np.concatenate([[0], boundaries, [n_points]]).astype(np.intp) diff --git a/packages/zarr-indexing/src/zarr_indexing/lazy_array.py b/packages/zarr-indexing/src/zarr_indexing/lazy_array.py index 43ffd5625f..be7c57a92a 100644 --- a/packages/zarr-indexing/src/zarr_indexing/lazy_array.py +++ b/packages/zarr-indexing/src/zarr_indexing/lazy_array.py @@ -1115,9 +1115,10 @@ def write(self, values: Any) -> None: """Synchronously write broadcastable values through this view. The source must support integer/slice assignment. Values are copied - before mutation, including when they alias the source. Writes follow - C-order view coordinates; the last occurrence wins for repeated source - coordinates. + before mutation, including when they alias the source. A source + coordinate the view addresses more than once receives one of the values + written to it, whole, but which one is unspecified; deduplicate the + selection first to control it. Affine selections use one basic assignment. Other selections are scattered against the source's **write grid**, discovered from diff --git a/packages/zarr-indexing/src/zarr_indexing/writer.py b/packages/zarr-indexing/src/zarr_indexing/writer.py index db69b6e691..2f56ad84e3 100644 --- a/packages/zarr-indexing/src/zarr_indexing/writer.py +++ b/packages/zarr-indexing/src/zarr_indexing/writer.py @@ -28,8 +28,10 @@ def write_into( The source must expose ``shape``, a NumPy-compatible ``dtype``, and basic integer/slice assignment. Values are snapshotted and converted before the first mutation, so source aliases and conversion failures are safe. Extra - leading singleton value axes are accepted, as in NumPy assignment. Repeated - source coordinates receive the last value in C order of the view. + leading singleton value axes are accepted, as in NumPy assignment. A source + coordinate the view addresses more than once receives one of the values + written to it, whole, but which one is unspecified; deduplicate the + selection first to control it. Independent affine maps use one basic assignment. Other selections are scattered in bulk: a NumPy source receives one fancy assignment, and a @@ -167,33 +169,11 @@ def _domain_cells(shape: tuple[int, ...]) -> np.ndarray[Any, np.dtype[np.intp]]: return np.indices(shape, dtype=np.intp).reshape(len(shape), -1).T -def _last_occurrences( - flat: np.ndarray[Any, Any], -) -> slice | np.ndarray[Any, np.dtype[np.intp]]: - """Positions to keep so that repeated flat indices take their last value. - - A full slice means every position is kept, so callers index without a copy. - """ - if flat.size < 2: - return slice(None) - if (flat[1:] >= flat[:-1]).all(): - # Non-decreasing input, the usual C-order case: repeats are adjacent, - # so no sort is needed. - steps = flat[1:] != flat[:-1] - if steps.all(): - return slice(None) - return np.flatnonzero(np.append(steps, True)) - _, first_in_reversed = np.unique(flat[::-1], return_index=True) - return np.asarray(len(flat) - 1 - first_in_reversed, dtype=np.intp) - - def _scatter_numpy(source: np.ndarray[Any, Any], transform: IndexTransform, broadcast: Any) -> None: """One fancy assignment: NumPy sources need no read-modify-write.""" positions = _domain_cells(transform.domain.shape) coords = transform.apply_many(positions + np.asarray(transform.domain.inclusive_min)) - flat = np.asarray(np.ravel_multi_index(tuple(coords.T), source.shape), dtype=np.intp) - keep = _last_occurrences(flat) - source[tuple(coords[keep].T)] = broadcast[tuple(positions[keep].T)] + source[tuple(coords.T)] = broadcast[tuple(positions.T)] def _scatter_planned( @@ -219,8 +199,7 @@ def _scatter_planned( flat = np.asarray( np.ravel_multi_index(tuple((local - hull_min).T), hull_shape), dtype=np.intp ) - keep = _last_occurrences(flat) - np.put(block, flat[keep], np.asarray(broadcast[tuple(positions[keep].T)])) + np.put(block, flat, np.asarray(broadcast[tuple(positions.T)])) source[box] = block diff --git a/packages/zarr-indexing/tests/test_chunk_resolution.py b/packages/zarr-indexing/tests/test_chunk_resolution.py index 8570fd34d2..0f899047a8 100644 --- a/packages/zarr-indexing/tests/test_chunk_resolution.py +++ b/packages/zarr-indexing/tests/test_chunk_resolution.py @@ -1100,3 +1100,41 @@ def indices_to_chunks(self, indices: Any) -> Any: assert all(p.coverage == expected for p in plan_chunks(transform, grids)) partition = plan_chunks(transform, grids).partition() assert all(bool(full) == (expected == "full") for full in partition.sets[0].full) + + +# Offsets off the 16-bit digit boundaries, and one straddling zero, so a +# missing shift to zero truncates keys out of order. +@pytest.mark.parametrize("offset", [0, -3, 2**40 + 12345, -(2**40) - 777]) +@pytest.mark.parametrize( + "span", + [1, 2, 2**8, 2**16, 2**16 + 1, 2**32, 2**32 + 1, 2**48, 2**48 + 1, 2**62, 2**64], +) +@pytest.mark.parametrize("size", [100, 2000]) +def test_group_by_key_orders_keys_ascending(size: int, span: int, offset: int) -> None: + """Every key range, from one radix pass to four or the small-input sort, groups the keys. + + Keys cover their whole range, so a pass that drops a digit or the shift to + zero misorders them. The order within a group is not checked: it is + unspecified. + """ + from zarr_indexing.chunk_resolution import _group_by_key + + if span == 2**64: + # The whole int64 range, whose span wraps when shifted to zero. + info = np.iinfo(np.int64) + low, high = int(info.min), int(info.max) + else: + low = max(int(np.iinfo(np.int64).min), offset) + high = min(int(np.iinfo(np.int64).max), low + span - 1) + rng = np.random.default_rng([size, span.bit_length(), abs(offset)]) + keys = rng.integers(low, high, size, dtype=np.int64, endpoint=True).astype(np.intp) + keys[:2] = (high, low) + assert _group_by_key(np.sort(keys)) is None + order = _group_by_key(keys) + if low == high: + assert order is None + return + assert order is not None + assert order.dtype == np.intp + np.testing.assert_array_equal(np.sort(order), np.arange(keys.size)) + assert bool((np.diff(keys[order]) >= 0).all()) diff --git a/packages/zarr-indexing/tests/test_writer.py b/packages/zarr-indexing/tests/test_writer.py index e3405d824c..0c68ba9343 100644 --- a/packages/zarr-indexing/tests/test_writer.py +++ b/packages/zarr-indexing/tests/test_writer.py @@ -29,13 +29,33 @@ def __setitem__(self, key: tuple[int | slice, ...], value: Any) -> None: self.data[key] = value +def _assert_each_cell_holds_a_written_value( + actual: Any, initial: Any, identifiers: Any, values: Any +) -> None: + """Cells the selection misses are unchanged; each addressed cell holds one value written to it. + + Which of a repeated cell's values lands is unspecified, so any of them passes. + """ + candidates: dict[int, set[Any]] = {} + for identifier, value in zip( + np.asarray(identifiers).flat, np.asarray(values).flat, strict=True + ): + candidates.setdefault(int(identifier), set()).add(value.item()) + flat_actual = np.asarray(actual).reshape(-1) + flat_initial = np.asarray(initial).reshape(-1) + for cell, (got, before) in enumerate( + zip(flat_actual.tolist(), flat_initial.tolist(), strict=True) + ): + assert got in candidates.get(cell, {before}), (cell, got, candidates.get(cell, {before})) + + @pytest.mark.parametrize( "case", ["reverse", "scalar", "newaxis", "empty", "outer", "composed", "correlated", "transpose"], ) def test_write_coordinates(case: str) -> None: data = np.arange(24).reshape(4, 6) - expected = data.copy() + initial = data.copy() # Select flat source identifiers independently of the transform evaluator. identifiers = np.arange(data.size).reshape(data.shape) transform = IndexTransform.from_shape(data.shape) @@ -74,11 +94,9 @@ def test_write_coordinates(case: str) -> None: ) identifiers = identifiers.T values = np.arange(identifiers.size).reshape(identifiers.shape) + 100 - for index, value in zip(np.asarray(identifiers).flat, values.flat, strict=True): - expected.flat[index] = value source = BasicSource(data) assert write_into(source, transform, values) is None - np.testing.assert_array_equal(data, expected) + _assert_each_cell_holds_a_written_value(data, initial, identifiers, values) if case in ("reverse", "scalar", "newaxis", "transpose"): assert len(source.calls) == 1 elif case == "empty": @@ -132,7 +150,8 @@ def test_write_constant_repeated_destination() -> None: source = BasicSource(np.zeros(3)) transform = IndexTransform(IndexDomain.from_shape((2, 3)), (ConstantMap(1),)) write_into(source, transform, [[1], [9]]) - np.testing.assert_array_equal(source.data, [0, 9, 0]) + assert source.data[[0, 2]].tolist() == [0, 0] + assert source.data[1] in (1, 9) assert len(source.calls) == 6 @@ -201,7 +220,6 @@ def test_lazy_write_and_assignment(backend: str, case: str) -> None: source[:] = initial array = LazyArray(source) identifiers = initial.copy() - expected = initial.copy() if case == "basic": view = array[::-1, ::2][1:, 1:] identifiers = identifiers[::-1, ::2][1:, 1:] @@ -222,16 +240,17 @@ def test_lazy_write_and_assignment(backend: str, case: str) -> None: np.testing.assert_array_equal(source[:], initial[::-1]) return values = np.arange(identifiers.size).reshape(identifiers.shape) + 100 - for identifier, value in zip(identifiers.flat, values.flat, strict=True): - expected.flat[identifier] = value assert view.write(values) is None - np.testing.assert_array_equal(source[:], expected) + _assert_each_cell_holds_a_written_value(source[:], initial, identifiers, values) # All selector assignment entry points write through to the original source. + written = np.array(source[:]) array.oindex[[0, 3], [1, 4]] = 77 - expected[np.ix_([0, 3], [1, 4])] = 77 + written[np.ix_([0, 3], [1, 4])] = 77 + np.testing.assert_array_equal(source[:], written) array.vindex[[0, 0, 2], [2, 2, 4]] = [10, 11, 12] - expected[[0, 0, 2], [2, 2, 4]] = [10, 11, 12] - np.testing.assert_array_equal(source[:], expected) + _assert_each_cell_holds_a_written_value( + source[:], written, initial[[0, 0, 2], [2, 2, 4]], [10, 11, 12] + ) @pytest.mark.parametrize("backend", ["numpy", "zarr"]) @@ -260,11 +279,8 @@ def test_random_selection_chain_scatter(backend: str) -> None: view = view[::step, ::-1] identifiers = identifiers[::step, ::-1] values = rng.integers(100, 200, size=identifiers.shape) - expected = initial.copy() - for identifier, value in zip(identifiers.flat, values.flat, strict=True): - expected.flat[identifier] = value view.write(values) - np.testing.assert_array_equal(source[:], expected) + _assert_each_cell_holds_a_written_value(source[:], initial, identifiers, values) class _ChunkTouches: @@ -318,11 +334,15 @@ def test_fancy_writes_touch_each_write_chunk_once(sharded: bool, case: str) -> N touched = {(r // 4, c // 4) for r, c in zip(rows, cols, strict=True)} else: view.oindex[[5, 5, 6], 2:5] = np.array([[1] * 3, [2] * 3, [3] * 3]) - expected[5, 2:5] = 2 expected[6, 2:5] = 3 touched = {(1, 0), (1, 1)} gets, sets = touches.gets, touches.sets - np.testing.assert_array_equal(source[:], expected) + actual = source[:] + if case == "duplicates": + # Row 5 is written twice; either row of values may land, but one whole value per cell. + assert set(actual[5, 2:5].tolist()) <= {1, 2} + expected[5, 2:5] = actual[5, 2:5] + np.testing.assert_array_equal(actual, expected) assert sets == len(touched), (sets, len(touched)) # At most the hull read plus zarr's own partial-chunk read per cell. assert gets <= 2 * len(touched), (gets, len(touched)) @@ -462,29 +482,85 @@ def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: np.testing.assert_array_equal(view.result(), [1, 2, 3]) -@pytest.mark.parametrize( - "flat", - [ - [], - [3], - [0, 2, 5, 7], - [0, 0, 2, 5, 5, 5, 7], - [5, 0, 5, 2, 0, 7], - [7, 5, 5, 2, 0], - ], - ids=["empty", "single", "sorted", "sorted_repeats", "unsorted_repeats", "reversed"], -) -def test_last_occurrences_matches_sequential_assignment(flat: list[int]) -> None: - from zarr_indexing.writer import _last_occurrences - - indices = np.asarray(flat, dtype=np.intp) - values = np.arange(1, indices.size + 1) - expected = np.zeros(8, dtype=values.dtype) - for index, value in zip(indices, values, strict=True): - expected[index] = value - keep = _last_occurrences(indices) - kept = indices[keep] - assert np.unique(kept).size == kept.size - actual = np.zeros(8, dtype=values.dtype) - actual[kept] = values[keep] - np.testing.assert_array_equal(actual, expected) +class GridSource: + """A readable, non-NumPy source with a write grid, so writes scatter cell by cell.""" + + def __init__(self, data: np.ndarray[Any, Any], chunks: tuple[tuple[int, ...], ...]) -> None: + self.data = data + self.shape = data.shape + self.dtype = data.dtype + self.chunks = chunks + self.reads = 0 + + def __getitem__(self, key: tuple[slice, ...]) -> np.ndarray[Any, Any]: + assert all(isinstance(item, slice) for item in key) + self.reads += 1 + return self.data[key] + + def __setitem__(self, key: tuple[slice, ...], value: Any) -> None: + assert all(isinstance(item, slice) for item in key) + self.data[key] = value + + +def _reverse_within_groups( + keys: np.ndarray[Any, np.dtype[np.intp]], +) -> np.ndarray[Any, np.dtype[np.intp]]: + """Group equal keys, listing each group's entries in reverse request order.""" + return np.asarray(np.lexsort((-np.arange(keys.size), keys)), dtype=np.intp) + + +def _repeated_selection(case: str, view: Any, identifiers: Any) -> tuple[Any, Any]: + """A selection that addresses some cells twice and lists each grid cell's points unsorted.""" + # On the 4-by-4 grid below, rows 3, 1, 3, 0, 2, 1 land in the first chunk + # row and 6, 5 in the second; the columns split 7, 5, 6 against 2, 2, 0. + rows = [3, 1, 3, 0, 6, 2, 1, 5] + cols = [7, 2, 5, 2, 0, 6] + if case == "orthogonal": + return view.oindex[rows, cols], identifiers[np.ix_(rows, cols)] + if case == "vectorized": + points = ([3, 1, 3, 6, 5, 1, 0, 6], [2, 0, 2, 7, 4, 0, 3, 5]) + return view.vindex[points], identifiers[points] + # Mixed: a vectorized pair over an orthogonal gather of the rows. + gathered = view.oindex[rows, :] + points = ([0, 2, 4, 1, 6, 0, 7, 3], [6, 6, 1, 0, 0, 6, 5, 1]) + return gathered.vindex[points], identifiers[rows, :][points] + + +@pytest.mark.parametrize("grouping", ["default", "reversed"]) +# The per-cell path copies blocks into plain arrays, so only NumPy sources carry masks. +@pytest.mark.parametrize("backend", ["planned", "numpy", "numpy-masked"]) +@pytest.mark.parametrize("case", ["orthogonal", "vectorized", "mixed"]) +def test_repeated_writes_land_one_whole_value_per_cell( + monkeypatch: pytest.MonkeyPatch, case: str, backend: str, grouping: str +) -> None: + """Repeated targets get one of their values whatever order the planner lists a cell's points in. + + Targets addressed once get exactly their value, so a writer that pairs a + point with another point's value fails whichever order grouping produces. + """ + from zarr_indexing import chunk_resolution + from zarr_indexing.lazy_array import LazyArray + + if grouping == "reversed": + monkeypatch.setattr(chunk_resolution, "_group_by_key", _reverse_within_groups) + initial = np.arange(64, dtype=np.int64).reshape(8, 8) + masked = backend == "numpy-masked" + data: Any = initial.copy() + if masked: + data = np.ma.array(data, mask=np.zeros(data.shape, dtype=bool)) + source: Any = GridSource(data, ((4, 4), (4, 4))) if backend == "planned" else data + view, identifiers = _repeated_selection(case, LazyArray(source), np.arange(64).reshape(8, 8)) + values = np.arange(identifiers.size).reshape(identifiers.shape) * 7 + 100 + if masked: + # Mask values by parity so a cell whose data and mask came from two + # different writes is caught: written value v is masked iff v is odd. + values = np.ma.array(values, mask=values % 2 == 1) + view.write(values) + if backend == "planned": + assert source.reads > 0 + _assert_each_cell_holds_a_written_value(np.ma.getdata(data), initial, identifiers, values) + if masked: + np.testing.assert_array_equal( + np.ma.getmaskarray(data), + np.isin(np.arange(64), identifiers).reshape(8, 8) & (np.ma.getdata(data) % 2 == 1), + )