diff --git a/changes/4643.misc.md b/changes/4643.misc.md new file mode 100644 index 0000000000..d5a9a850e9 --- /dev/null +++ b/changes/4643.misc.md @@ -0,0 +1 @@ +When an `oindex`, `vindex` or `__setitem__` index array names an element more than once, the value the element receives is now documented as unspecified, as in NumPy. Remove repeated indices if the result matters. Grouping a fancy selection by chunk is also several times faster. diff --git a/docs/user-guide/arrays.md b/docs/user-guide/arrays.md index 5b025047b6..17f9b82b0d 100644 --- a/docs/user-guide/arrays.md +++ b/docs/user-guide/arrays.md @@ -549,6 +549,13 @@ z[:] = data print(np.all(z.oindex[[0, 2], :] == z[[0, 2], :])) ``` +### Repeated indices + +If an index array used for writing names an element more than once, the element is +set to one of the values given for it, and which one is unspecified, as in NumPy. +This applies to coordinate and orthogonal writes, through `vindex`, `oindex` +or the square bracket operator. Remove repeated indices if the result matters. + ### Block indexing Zarr also supports block indexing, which allows selections of whole chunks based on their diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 3735a4da18..e895b74958 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -3313,6 +3313,10 @@ def set_orthogonal_selection( Slices with step > 1 are supported, but slices with negative step are not. + If `selection` names an element more than once, the element is set to one of the + values given for it, and which one is unspecified, as in NumPy. Remove repeated + indices if the result matters. + Related ------- [get_basic_selection][zarr.Array.get_basic_selection], @@ -3668,6 +3672,10 @@ def set_coordinate_selection( Slices are not supported. Coordinate arrays must be provided for all dimensions of the array. + If `selection` names an element more than once, the element is set to one of the + values given for it, and which one is unspecified, as in NumPy. Remove repeated + indices if the result matters. + Related ------- [get_basic_selection][zarr.Array.get_basic_selection], diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index ffd4035bb7..e4384f236f 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -762,6 +762,33 @@ def sorted_run_ends( return a[run_starts].astype(np.intp, copy=False), run_ends +# Below this many items per 16-bit pass, NumPy's stable sort of the full-width indices is +# faster than radix-sorting them. +_RADIX_SORT_MIN_ITEMS_PER_PASS = 128 + + +def argsort_chunk_indices( + chunk_indices: npt.NDArray[np.intp], nchunks: int +) -> npt.NDArray[np.intp]: + """Return the indices that sort `chunk_indices`, each in `range(nchunks)`. + + NumPy sorts integers of at most 16 bits with a radix sort, several times faster than + its default sort, so the indices are sorted 16 bits at a time, lowest bits first. + For a few indices the plain stable sort is faster and is used instead. Either way + the sort is stable, so items that share a chunk keep their order. That order is an + implementation detail: the value stored for a repeated index is unspecified. + """ + # ceil(bits / 16) passes; chunk indices are at most 64 bits wide + n_passes = min(4, max(1, -((nchunks - 1).bit_length() // -16))) + if chunk_indices.size < _RADIX_SORT_MIN_ITEMS_PER_PASS * n_passes: + return np.argsort(chunk_indices, kind="stable") + order = np.argsort(chunk_indices.astype(np.uint16), kind="stable") + for shift in range(16, 16 * n_passes, 16): + digit = (chunk_indices[order] >> shift).astype(np.uint16) + order = order[np.argsort(digit, kind="stable")] + return order + + @dataclass(frozen=True) class IntArrayDimIndexer: """Integer array selection against a single dimension.""" @@ -831,7 +858,7 @@ def __init__( dim_sel_chunk_sorted = dim_sel_chunk[::-1] else: # sort indices to group by chunk - dim_out_sel = np.argsort(dim_sel_chunk) + dim_out_sel = argsort_chunk_indices(dim_sel_chunk, nchunks) dim_sel = np.take(dim_sel, dim_out_sel) dim_sel_chunk_sorted = dim_sel_chunk[dim_out_sel] @@ -1346,7 +1373,7 @@ def __init__( # group points by chunk if np.any(np.diff(chunks_raveled_indices) < 0): # optimisation, only sort if needed - sel_sort = np.argsort(chunks_raveled_indices) + sel_sort = argsort_chunk_indices(chunks_raveled_indices, product(cdata_shape)) selection_broadcast = tuple(dim_sel[sel_sort] for dim_sel in selection_broadcast) chunks_raveled_indices = chunks_raveled_indices[sel_sort] else: diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 4ff35563c6..2e010cc134 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -20,7 +20,7 @@ from zarr import Array from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_grids import ChunkGrid -from zarr.core.common import ceildiv_int +from zarr.core.common import ceildiv_int, product from zarr.core.indexing import ( BasicSelection, CoordinateIndexer, @@ -32,6 +32,7 @@ _ArrayIndexingOrder, _iter_grid, _iter_regions, + argsort_chunk_indices, make_slice_selection, normalize_integer_selection, oindex, @@ -507,17 +508,77 @@ def test_setitem_zarr_array_as_value() -> None: assert_array_equal(dst2[2:7], np.arange(2, 7)) -@pytest.mark.skip(reason="fails on ubuntu, windows; numpy=2.2; in CI") -def test_setitem_repeated_index(): - """oindex assignment with repeated indices writes the last value for each duplicated index position.""" - array = zarr.array(data=np.zeros((4,)), chunks=(1,)) - indexer = np.array([-1, -1, 0, 0]) - array.oindex[(indexer,)] = [0, 1, 2, 3] - np.testing.assert_array_equal(array[:], np.array([3, 0, 0, 1])) +def _reversed_within_chunk_argsort( + chunk_indices: npt.NDArray[np.intp], nchunks: int +) -> npt.NDArray[np.intp]: + """Group `chunk_indices` by chunk, reversing the order of items within each chunk.""" + return np.lexsort((-np.arange(chunk_indices.size), chunk_indices)) - indexer = np.array([-1, 0, 0, -1]) - array.oindex[(indexer,)] = [0, 1, 2, 3] - np.testing.assert_array_equal(array[:], np.array([2, 0, 0, 3])) + +@pytest.mark.parametrize("method", ["oindex", "vindex"]) +@pytest.mark.parametrize("layout", ["unit_chunks", "partial_chunks", "one_chunk", "sharded"]) +@pytest.mark.parametrize( + ("shape", "selection"), + [ + ((24,), ([-1, -1, 0, 0],)), + ((24,), ([-1, 0, 0, -1],)), + ((24,), ([1] + [0] * 23,)), + ((6, 6), ([0, 5, 0, -1, 2] * 4, [1, 1, 4, 4, 2] * 4)), + ], + ids=["adjacent", "interleaved", "many_repeats", "two_dimensions"], +) +@pytest.mark.parametrize("within_chunk_order", ["default", "reversed"]) +def test_setitem_repeated_index( + method: str, + layout: str, + shape: tuple[int, ...], + selection: tuple[list[int], ...], + within_chunk_order: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Assignment with repeated indices sets each repeated element to one of its values, whole, and leaves the rest as NumPy would.""" + if within_chunk_order == "reversed": + monkeypatch.setattr( + zarr.core.indexing, "argsort_chunk_indices", _reversed_within_chunk_argsort + ) + chunks = { + "unit_chunks": (1,) * len(shape), + "partial_chunks": (4,) * len(shape), + "one_chunk": shape, + "sharded": (2,) * len(shape), + }[layout] + shards = (6,) * len(shape) if layout == "sharded" else None + array = zarr.create_array(store={}, shape=shape, chunks=chunks, shards=shards, dtype="i8") + flat_ids = np.arange(product(shape)).reshape(shape) + if method == "oindex": + targets = flat_ids[np.ix_(*selection)] + value = np.arange(1, 1 + targets.size).reshape(targets.shape) + array.oindex[selection] = value + else: + targets = flat_ids[selection] + value = np.arange(1, 1 + targets.size) + array.vindex[selection] = value + written = array[:].reshape(-1) + targets = targets.reshape(-1) + value = value.reshape(-1) + assert (np.delete(written, targets) == 0).all() + for target in np.unique(targets): + assert written[target] in value[targets == target] + + +@pytest.mark.parametrize("n_items", [100, 2000]) +@pytest.mark.parametrize( + "nchunks", [1, 2**8, 2**16, 2**16 + 1, 2**32, 2**32 + 1, 2**48, 2**48 + 1, 2**62, 2**70] +) +def test_argsort_chunk_indices(nchunks: int, n_items: int) -> None: + """The sort matches NumPy's stable sort for any number of chunks, and for few or many items.""" + rng = np.random.default_rng(0) + highest = min(nchunks, 2**63) - 1 + chunk_indices = np.concatenate( + [[0, highest, highest, 0], rng.integers(0, highest + 1, n_items - 4)] + ).astype(np.intp) + expected = np.argsort(chunk_indices, kind="stable") + assert_array_equal(argsort_chunk_indices(chunk_indices, nchunks), expected) Index = list[int] | tuple[slice | int | list[int], ...]