Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/4643.misc.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
When an `oindex`, `vindex` or `__setitem__` index array names an element more than once, the value the element receives is now documented as unspecified, as in NumPy. Remove repeated indices if the result matters. Grouping a fancy selection by chunk is also several times faster.
7 changes: 7 additions & 0 deletions docs/user-guide/arrays.md
Original file line number Diff line number Diff line change
Expand Up @@ -549,6 +549,13 @@ z[:] = data
print(np.all(z.oindex[[0, 2], :] == z[[0, 2], :]))
```

### Repeated indices

If an index array used for writing names an element more than once, the element is
set to one of the values given for it, and which one is unspecified, as in NumPy.
This applies to coordinate and orthogonal writes, through `vindex`, `oindex`
or the square bracket operator. Remove repeated indices if the result matters.

### Block indexing

Zarr also supports block indexing, which allows selections of whole chunks based on their
Expand Down
8 changes: 8 additions & 0 deletions src/zarr/core/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -3313,6 +3313,10 @@ def set_orthogonal_selection(

Slices with step > 1 are supported, but slices with negative step are not.

If `selection` names an element more than once, the element is set to one of the
values given for it, and which one is unspecified, as in NumPy. Remove repeated
indices if the result matters.

Related
-------
[get_basic_selection][zarr.Array.get_basic_selection],
Expand Down Expand Up @@ -3668,6 +3672,10 @@ def set_coordinate_selection(
Slices are not supported. Coordinate arrays must be provided for all dimensions
of the array.

If `selection` names an element more than once, the element is set to one of the
values given for it, and which one is unspecified, as in NumPy. Remove repeated
indices if the result matters.

Related
-------
[get_basic_selection][zarr.Array.get_basic_selection],
Expand Down
31 changes: 29 additions & 2 deletions src/zarr/core/indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -762,6 +762,33 @@ def sorted_run_ends(
return a[run_starts].astype(np.intp, copy=False), run_ends


# Below this many items per 16-bit pass, NumPy's stable sort of the full-width indices is
# faster than radix-sorting them.
_RADIX_SORT_MIN_ITEMS_PER_PASS = 128


def argsort_chunk_indices(
chunk_indices: npt.NDArray[np.intp], nchunks: int
) -> npt.NDArray[np.intp]:
"""Return the indices that sort `chunk_indices`, each in `range(nchunks)`.

NumPy sorts integers of at most 16 bits with a radix sort, several times faster than
its default sort, so the indices are sorted 16 bits at a time, lowest bits first.
For a few indices the plain stable sort is faster and is used instead. Either way
the sort is stable, so items that share a chunk keep their order. That order is an
implementation detail: the value stored for a repeated index is unspecified.
"""
# ceil(bits / 16) passes; chunk indices are at most 64 bits wide
n_passes = min(4, max(1, -((nchunks - 1).bit_length() // -16)))
if chunk_indices.size < _RADIX_SORT_MIN_ITEMS_PER_PASS * n_passes:
return np.argsort(chunk_indices, kind="stable")
order = np.argsort(chunk_indices.astype(np.uint16), kind="stable")
for shift in range(16, 16 * n_passes, 16):
digit = (chunk_indices[order] >> shift).astype(np.uint16)
order = order[np.argsort(digit, kind="stable")]
return order


@dataclass(frozen=True)
class IntArrayDimIndexer:
"""Integer array selection against a single dimension."""
Expand Down Expand Up @@ -831,7 +858,7 @@ def __init__(
dim_sel_chunk_sorted = dim_sel_chunk[::-1]
else:
# sort indices to group by chunk
dim_out_sel = np.argsort(dim_sel_chunk)
dim_out_sel = argsort_chunk_indices(dim_sel_chunk, nchunks)
dim_sel = np.take(dim_sel, dim_out_sel)
dim_sel_chunk_sorted = dim_sel_chunk[dim_out_sel]

Expand Down Expand Up @@ -1346,7 +1373,7 @@ def __init__(
# group points by chunk
if np.any(np.diff(chunks_raveled_indices) < 0):
# optimisation, only sort if needed
sel_sort = np.argsort(chunks_raveled_indices)
sel_sort = argsort_chunk_indices(chunks_raveled_indices, product(cdata_shape))
selection_broadcast = tuple(dim_sel[sel_sort] for dim_sel in selection_broadcast)
chunks_raveled_indices = chunks_raveled_indices[sel_sort]
else:
Expand Down
83 changes: 72 additions & 11 deletions tests/test_indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from zarr import Array
from zarr.core.buffer import default_buffer_prototype
from zarr.core.chunk_grids import ChunkGrid
from zarr.core.common import ceildiv_int
from zarr.core.common import ceildiv_int, product
from zarr.core.indexing import (
BasicSelection,
CoordinateIndexer,
Expand All @@ -32,6 +32,7 @@
_ArrayIndexingOrder,
_iter_grid,
_iter_regions,
argsort_chunk_indices,
make_slice_selection,
normalize_integer_selection,
oindex,
Expand Down Expand Up @@ -507,17 +508,77 @@ def test_setitem_zarr_array_as_value() -> None:
assert_array_equal(dst2[2:7], np.arange(2, 7))


@pytest.mark.skip(reason="fails on ubuntu, windows; numpy=2.2; in CI")
def test_setitem_repeated_index():
"""oindex assignment with repeated indices writes the last value for each duplicated index position."""
array = zarr.array(data=np.zeros((4,)), chunks=(1,))
indexer = np.array([-1, -1, 0, 0])
array.oindex[(indexer,)] = [0, 1, 2, 3]
np.testing.assert_array_equal(array[:], np.array([3, 0, 0, 1]))
def _reversed_within_chunk_argsort(
chunk_indices: npt.NDArray[np.intp], nchunks: int
) -> npt.NDArray[np.intp]:
"""Group `chunk_indices` by chunk, reversing the order of items within each chunk."""
return np.lexsort((-np.arange(chunk_indices.size), chunk_indices))

indexer = np.array([-1, 0, 0, -1])
array.oindex[(indexer,)] = [0, 1, 2, 3]
np.testing.assert_array_equal(array[:], np.array([2, 0, 0, 3]))

@pytest.mark.parametrize("method", ["oindex", "vindex"])
@pytest.mark.parametrize("layout", ["unit_chunks", "partial_chunks", "one_chunk", "sharded"])
@pytest.mark.parametrize(
("shape", "selection"),
[
((24,), ([-1, -1, 0, 0],)),
((24,), ([-1, 0, 0, -1],)),
((24,), ([1] + [0] * 23,)),
((6, 6), ([0, 5, 0, -1, 2] * 4, [1, 1, 4, 4, 2] * 4)),
],
ids=["adjacent", "interleaved", "many_repeats", "two_dimensions"],
)
@pytest.mark.parametrize("within_chunk_order", ["default", "reversed"])
def test_setitem_repeated_index(
method: str,
layout: str,
shape: tuple[int, ...],
selection: tuple[list[int], ...],
within_chunk_order: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Assignment with repeated indices sets each repeated element to one of its values, whole, and leaves the rest as NumPy would."""
if within_chunk_order == "reversed":
monkeypatch.setattr(
zarr.core.indexing, "argsort_chunk_indices", _reversed_within_chunk_argsort
)
chunks = {
"unit_chunks": (1,) * len(shape),
"partial_chunks": (4,) * len(shape),
"one_chunk": shape,
"sharded": (2,) * len(shape),
}[layout]
shards = (6,) * len(shape) if layout == "sharded" else None
array = zarr.create_array(store={}, shape=shape, chunks=chunks, shards=shards, dtype="i8")
flat_ids = np.arange(product(shape)).reshape(shape)
if method == "oindex":
targets = flat_ids[np.ix_(*selection)]
value = np.arange(1, 1 + targets.size).reshape(targets.shape)
array.oindex[selection] = value
else:
targets = flat_ids[selection]
value = np.arange(1, 1 + targets.size)
array.vindex[selection] = value
written = array[:].reshape(-1)
targets = targets.reshape(-1)
value = value.reshape(-1)
assert (np.delete(written, targets) == 0).all()
for target in np.unique(targets):
assert written[target] in value[targets == target]


@pytest.mark.parametrize("n_items", [100, 2000])
@pytest.mark.parametrize(
"nchunks", [1, 2**8, 2**16, 2**16 + 1, 2**32, 2**32 + 1, 2**48, 2**48 + 1, 2**62, 2**70]
)
def test_argsort_chunk_indices(nchunks: int, n_items: int) -> None:
"""The sort matches NumPy's stable sort for any number of chunks, and for few or many items."""
rng = np.random.default_rng(0)
highest = min(nchunks, 2**63) - 1
chunk_indices = np.concatenate(
[[0, highest, highest, 0], rng.integers(0, highest + 1, n_items - 4)]
).astype(np.intp)
expected = np.argsort(chunk_indices, kind="stable")
assert_array_equal(argsort_chunk_indices(chunk_indices, nchunks), expected)


Index = list[int] | tuple[slice | int | list[int], ...]
Expand Down
Loading