diff --git a/packages/zarr-indexing/changes/4603.misc.md b/packages/zarr-indexing/changes/4603.misc.md deleted file mode 100644 index 3015c99f22..0000000000 --- a/packages/zarr-indexing/changes/4603.misc.md +++ /dev/null @@ -1 +0,0 @@ -Writes whose indices land on a chunk in sorted order no longer sort them to drop repeated indices, so deduplication is linear in the number of points; writes without repeats also skip a gather copy. diff --git a/packages/zarr-indexing/changes/4649.removal.md b/packages/zarr-indexing/changes/4649.removal.md new file mode 100644 index 0000000000..82386e9e01 --- /dev/null +++ b/packages/zarr-indexing/changes/4649.removal.md @@ -0,0 +1 @@ +`LazyArray.write` no longer promises that the last of several writes to one source cell wins: the cell receives one of the values written to it, whole, but which one is unspecified, so remove repeated indices from a selection whose result matters. `IndexedSet` and `JointSet` no longer promise an order for the points within a row either. Planning now groups points by chunk with a linear-time radix sort and writes skip deduplication, so partitioning a large unordered selection is up to 5x faster and a fancy write to a NumPy array about 10x faster. diff --git a/packages/zarr-indexing/docs/design-notes.md b/packages/zarr-indexing/docs/design-notes.md index 9efed99769..26534e6a93 100644 --- a/packages/zarr-indexing/docs/design-notes.md +++ b/packages/zarr-indexing/docs/design-notes.md @@ -193,6 +193,39 @@ rectangular-only view is a plausible next step, but it should be introduced by a consumer that needs the guarantee in its signatures rather than speculatively; `is_box` is the runtime check until then. +## Repeated write targets + +A query can address one storage cell more than once: `a.vindex[[1, 1]] = [10, 20]` +writes two values to one cell. Only one value can remain, and this library +does not promise which. `LazyArray.write` gives the cell one of the values +written to it, whole (on a masked NumPy array, the value and its mask come +from the same write), and leaves the choice unspecified. A caller whose result depends on the choice removes repeated +coordinates before writing, for example with `np.unique(..., return_index=True)`. + +Three decisions follow from that rule: + +- **Performance decides the internal order.** Planning groups each chunk's + points with whatever sort is fastest. That is currently NumPy's 16-bit radix + sort, which is stable, so a chunk's points stay in request order, and NumPy + assignment applies repeated targets in sequence in practice. Writes therefore + currently keep the last value in C order of the view. This describes the + implementation, not a contract: a faster planner or a NumPy change may pick a + different value, and tests must not depend on which one lands. +- **Planning tables carry no order within a row.** `IndexedSet` and `JointSet` + pair every point with its request position, so result placement survives + planning, but the order of points within a row is unspecified. A consumer + that needs request order sorts a row by `positions`. +- **The library does not deduplicate.** Removing repeats, or detecting them to + raise an error, needs a pass over each chunk's targets that can cost more + than the assignment itself: for a 200,000-point unordered selection written + to a NumPy array, the pass took about ten times as long as the assignment. + That cost would fall on every write to protect the few with ambiguous + input, so the caller who needs a particular value supplies unambiguous input. + +NumPy takes the same position: it documents that the final value of an element +assigned more than once through advanced indexing cannot be predicted. Array +libraries that scatter in parallel make the same choice. + ## Negative-origin domains and prependable grids Literal coordinates let a domain grow at its lower end without changing the diff --git a/packages/zarr-indexing/docs/guide/selection-flow.md b/packages/zarr-indexing/docs/guide/selection-flow.md index 749a9c36ae..662d5db223 100644 --- a/packages/zarr-indexing/docs/guide/selection-flow.md +++ b/packages/zarr-indexing/docs/guide/selection-flow.md @@ -194,8 +194,10 @@ that the supplied buffer can be copied directly into codec order. Touching every chunk alone does not prove full coverage of each chunk. Planning does not fetch bytes, decode buffers, choose concurrency, or define -write-conflict handling. Consumers must preserve request ordering where their -write API requires it, and account for memory retained by emitted operations. +write-conflict handling. The order of points within one chunk's projection is +unspecified; each point stays paired with its request position, so a consumer +whose write API depends on request order sorts by those positions. Consumers +also account for memory retained by emitted operations. ## 6. Applying the flow to sharding diff --git a/packages/zarr-indexing/docs/snippets/grid_partition.py b/packages/zarr-indexing/docs/snippets/grid_partition.py index ab7f1ee38d..2d518333e3 100644 --- a/packages/zarr-indexing/docs/snippets/grid_partition.py +++ b/packages/zarr-indexing/docs/snippets/grid_partition.py @@ -40,8 +40,9 @@ assert isinstance(rows, IndexedSet) assert rows.chunk.tolist() == [0, 1] # rows 1 and 1 land in chunk 0, row 4 in chunk 1 assert rows.pointer.tolist() == [0, 2, 3] # chunk 0 owns entries 0:2, chunk 1 owns 2:3 -assert rows.index.tolist() == [1, 1, 4] # grouped by chunk, request order within a chunk -assert rows.positions.tolist() == [1, 2, 0] # the request positions those entries fill +assert rows.index.tolist() == [1, 1, 4] # grouped by chunk, any order within one +assert sorted(rows.positions[rows.run(0)].tolist()) == [1, 2] # request positions chunk 0 fills +assert rows.positions[rows.run(1)].tolist() == [0] assert rows.local.tolist() == [1, 1, 1] # chunk-local: 1 - 0, 1 - 0, 4 - 3 points = IndexTransform.from_shape((6, 8)).vindex[np.array([0, 5, 5]), np.array([7, 0, 1])] @@ -49,8 +50,9 @@ assert joint is not None assert joint.chunk.tolist() == [[0, 1], [1, 0]] # touched chunks, lexicographic assert joint.pointer.tolist() == [0, 1, 3] # point 0 alone; points 1 and 2 share a chunk -assert joint.positions.tolist() == [0, 1, 2] -assert joint.local.tolist() == [[0, 3], [2, 0], [2, 1]] +# Each point keeps its position and chunk-local coordinates paired. +pairs = sorted(zip(joint.positions.tolist(), joint.local.tolist(), strict=True)) +assert pairs == [(0, [0, 3]), (1, [2, 0]), (2, [2, 1])] # --8<-- [end:indexed-and-joint] diff --git a/packages/zarr-indexing/src/zarr_indexing/chunk_resolution.py b/packages/zarr-indexing/src/zarr_indexing/chunk_resolution.py index 01383b8e34..0e5c62b32b 100644 --- a/packages/zarr-indexing/src/zarr_indexing/chunk_resolution.py +++ b/packages/zarr-indexing/src/zarr_indexing/chunk_resolution.py @@ -375,10 +375,11 @@ class IndexedSet: """One output dimension read through an orthogonal `ArrayMap`, one row per chunk. The map's coordinates are grouped by chunk in CSR form: row ``i`` owns - ``index[pointer[i]:pointer[i + 1]]`` (the index-array values, in request - order) and ``positions[pointer[i]:pointer[i + 1]]`` (their positions along - the request axis, ascending). `local` gives the same values as chunk-local - storage coordinates. Columns are read-only NumPy arrays. + ``index[pointer[i]:pointer[i + 1]]`` (the index-array values) and + ``positions[pointer[i]:pointer[i + 1]]`` (their positions along the request + axis), paired entry by entry. The order of entries within a row is + unspecified. `local` gives the same values as chunk-local storage + coordinates. Columns are read-only NumPy arrays. Examples -------- @@ -390,8 +391,8 @@ class IndexedSet: >>> (axis,) = plan_chunks(transform, grids).partition().sets >>> axis.chunk.tolist(), axis.pointer.tolist() ([0, 2], [0, 2, 4]) - >>> axis.local.tolist(), axis.positions.tolist() - ([1, 2, 1, 0], [1, 2, 0, 3]) + >>> sorted(zip(axis.positions.tolist(), axis.local.tolist())) + [(0, 1), (1, 1), (2, 2), (3, 0)] """ output_dimension: int @@ -422,7 +423,7 @@ class IndexedSet: """Index-array values grouped by chunk.""" positions: np.ndarray[Any, np.dtype[np.intp]] - """Positions along the request axis, grouped by chunk, ascending within a row.""" + """Positions along the request axis, grouped by chunk like `index`.""" def __post_init__(self) -> None: _own_columns(self, "chunk", "chunk_start", "chunk_extent", "pointer", "index", "positions") @@ -465,8 +466,8 @@ class JointSet: `output_dimensions`, and CSR range ``pointer[i]:pointer[i + 1]`` its points: `index` holds their index-array values per output dimension, `positions` their flat positions in this component's input block, and - `block_coordinates` those positions unravelled over the block. Columns are - read-only NumPy arrays. + `block_coordinates` those positions unravelled over the block. The order of + points within a row is unspecified. Columns are read-only NumPy arrays. Examples -------- @@ -478,8 +479,10 @@ class JointSet: ... np.array([0, 6, 6, 1]), np.array([8, 0, 1, 8]) ... ] >>> (joint,) = plan_chunks(transform, grids).partition().joint_sets - >>> joint.chunk.tolist(), joint.pointer.tolist(), joint.positions.tolist() - ([[0, 2], [2, 0]], [0, 2, 4], [0, 3, 1, 2]) + >>> joint.chunk.tolist(), joint.pointer.tolist() + ([[0, 2], [2, 0]], [0, 2, 4]) + >>> [sorted(joint.positions[joint.run(row)].tolist()) for row in range(len(joint))] + [[0, 3], [1, 2]] """ output_dimensions: tuple[int, ...] @@ -513,7 +516,7 @@ class JointSet: """Index-array values per point and output dimension, shape ``(points, k)``.""" positions: np.ndarray[Any, np.dtype[np.intp]] - """Flat block position of each point, ascending within a row.""" + """Flat block position of each point, grouped by row like `index`.""" _block_coordinates: np.ndarray[Any, np.dtype[np.intp]] | None = field( default=None, init=False, repr=False, compare=False @@ -654,13 +657,12 @@ def _indexed_set(out_dim: int, m: ArrayMap, dg: DimensionGridLike) -> IndexedSet dg.index_to_chunk(int(storage.min())) dg.index_to_chunk(int(storage.max())) chunk_ids = dg.indices_to_chunks(storage) - # Already-sorted coordinates (the common case) need no sort. - if bool((chunk_ids[1:] >= chunk_ids[:-1]).all()): + positions = _group_by_key(chunk_ids) + if positions is None: positions = np.arange(n, dtype=np.intp) sorted_ids = chunk_ids index = flat else: - positions = np.argsort(chunk_ids, kind="stable") sorted_ids = chunk_ids[positions] index = flat[positions] boundaries = np.flatnonzero(sorted_ids[1:] != sorted_ids[:-1]) + 1 @@ -681,6 +683,42 @@ def _indexed_set(out_dim: int, m: ArrayMap, dg: DimensionGridLike) -> IndexedSet ) +# Below this many keys per radix pass, one stable comparison sort is cheaper +# than the passes' fixed overhead (about 1 us each). +_RADIX_SORT_MIN_ITEMS_PER_PASS = 128 + + +def _group_by_key( + keys: np.ndarray[Any, np.dtype[np.intp]], +) -> np.ndarray[Any, np.dtype[np.intp]] | None: + """An order that groups equal keys in ascending key order, or None if `keys` already do. + + NumPy's stable sort is a linear-time radix sort for integers of at most 16 + bits, so keys shifted to start at zero are sorted one 16-bit digit at a + time, least significant first: one pass per 16 bits of their range, at + most four. Small inputs take one stable comparison sort instead. Both + branches happen to keep input order within a group, but that order is + unspecified and callers must not rely on it. + """ + if bool((keys[1:] >= keys[:-1]).all()): + return None + low = int(keys.min()) + span = int(keys.max()) - low + 1 + # ceil(bits / 16) passes; the range of int64 keys needs at most 64 bits. + n_passes = min(4, max(1, -((span - 1).bit_length() // -16))) + if keys.size < _RADIX_SORT_MIN_ITEMS_PER_PASS * n_passes: + return np.asarray(np.argsort(keys, kind="stable"), dtype=np.intp) + # A range above 2**63 wraps in int64, but the difference is exact modulo + # 2**64, so reading it as uint64 recovers the shifted keys. + shifted = (keys - low).view(np.uint64) + # `astype(np.uint16)` keeps the low 16 bits. + order = np.argsort(shifted.astype(np.uint16), kind="stable") + for shift in range(16, 16 * n_passes, 16): + digit = (shifted[order] >> np.uint64(shift)).astype(np.uint16) + order = order[np.argsort(digit, kind="stable")] + return np.asarray(order, dtype=np.intp) + + def _chunk_keys( chunk_ids: Sequence[np.ndarray[Any, np.dtype[np.intp]]], ) -> np.ndarray[Any, np.dtype[np.intp]]: @@ -724,7 +762,8 @@ def _joint_set( chunk_ids = [dim_grids[d].indices_to_chunks(block.flat_storage[d]) for d in dims] n_points = int(chunk_ids[0].size) keys = _chunk_keys(chunk_ids) - positions = np.argsort(keys, kind="stable") + grouped = _group_by_key(keys) + positions = np.arange(n_points, dtype=np.intp) if grouped is None else grouped sorted_keys = keys[positions] boundaries = np.flatnonzero(sorted_keys[1:] != sorted_keys[:-1]) + 1 pointer = np.concatenate([[0], boundaries, [n_points]]).astype(np.intp) diff --git a/packages/zarr-indexing/src/zarr_indexing/lazy_array.py b/packages/zarr-indexing/src/zarr_indexing/lazy_array.py index 43ffd5625f..be7c57a92a 100644 --- a/packages/zarr-indexing/src/zarr_indexing/lazy_array.py +++ b/packages/zarr-indexing/src/zarr_indexing/lazy_array.py @@ -1115,9 +1115,10 @@ def write(self, values: Any) -> None: """Synchronously write broadcastable values through this view. The source must support integer/slice assignment. Values are copied - before mutation, including when they alias the source. Writes follow - C-order view coordinates; the last occurrence wins for repeated source - coordinates. + before mutation, including when they alias the source. A source + coordinate the view addresses more than once receives one of the values + written to it, whole, but which one is unspecified; deduplicate the + selection first to control it. Affine selections use one basic assignment. Other selections are scattered against the source's **write grid**, discovered from diff --git a/packages/zarr-indexing/src/zarr_indexing/writer.py b/packages/zarr-indexing/src/zarr_indexing/writer.py index db69b6e691..2f56ad84e3 100644 --- a/packages/zarr-indexing/src/zarr_indexing/writer.py +++ b/packages/zarr-indexing/src/zarr_indexing/writer.py @@ -28,8 +28,10 @@ def write_into( The source must expose ``shape``, a NumPy-compatible ``dtype``, and basic integer/slice assignment. Values are snapshotted and converted before the first mutation, so source aliases and conversion failures are safe. Extra - leading singleton value axes are accepted, as in NumPy assignment. Repeated - source coordinates receive the last value in C order of the view. + leading singleton value axes are accepted, as in NumPy assignment. A source + coordinate the view addresses more than once receives one of the values + written to it, whole, but which one is unspecified; deduplicate the + selection first to control it. Independent affine maps use one basic assignment. Other selections are scattered in bulk: a NumPy source receives one fancy assignment, and a @@ -167,33 +169,11 @@ def _domain_cells(shape: tuple[int, ...]) -> np.ndarray[Any, np.dtype[np.intp]]: return np.indices(shape, dtype=np.intp).reshape(len(shape), -1).T -def _last_occurrences( - flat: np.ndarray[Any, Any], -) -> slice | np.ndarray[Any, np.dtype[np.intp]]: - """Positions to keep so that repeated flat indices take their last value. - - A full slice means every position is kept, so callers index without a copy. - """ - if flat.size < 2: - return slice(None) - if (flat[1:] >= flat[:-1]).all(): - # Non-decreasing input, the usual C-order case: repeats are adjacent, - # so no sort is needed. - steps = flat[1:] != flat[:-1] - if steps.all(): - return slice(None) - return np.flatnonzero(np.append(steps, True)) - _, first_in_reversed = np.unique(flat[::-1], return_index=True) - return np.asarray(len(flat) - 1 - first_in_reversed, dtype=np.intp) - - def _scatter_numpy(source: np.ndarray[Any, Any], transform: IndexTransform, broadcast: Any) -> None: """One fancy assignment: NumPy sources need no read-modify-write.""" positions = _domain_cells(transform.domain.shape) coords = transform.apply_many(positions + np.asarray(transform.domain.inclusive_min)) - flat = np.asarray(np.ravel_multi_index(tuple(coords.T), source.shape), dtype=np.intp) - keep = _last_occurrences(flat) - source[tuple(coords[keep].T)] = broadcast[tuple(positions[keep].T)] + source[tuple(coords.T)] = broadcast[tuple(positions.T)] def _scatter_planned( @@ -219,8 +199,7 @@ def _scatter_planned( flat = np.asarray( np.ravel_multi_index(tuple((local - hull_min).T), hull_shape), dtype=np.intp ) - keep = _last_occurrences(flat) - np.put(block, flat[keep], np.asarray(broadcast[tuple(positions[keep].T)])) + np.put(block, flat, np.asarray(broadcast[tuple(positions.T)])) source[box] = block diff --git a/packages/zarr-indexing/tests/test_chunk_resolution.py b/packages/zarr-indexing/tests/test_chunk_resolution.py index 8570fd34d2..0f899047a8 100644 --- a/packages/zarr-indexing/tests/test_chunk_resolution.py +++ b/packages/zarr-indexing/tests/test_chunk_resolution.py @@ -1100,3 +1100,41 @@ def indices_to_chunks(self, indices: Any) -> Any: assert all(p.coverage == expected for p in plan_chunks(transform, grids)) partition = plan_chunks(transform, grids).partition() assert all(bool(full) == (expected == "full") for full in partition.sets[0].full) + + +# Offsets off the 16-bit digit boundaries, and one straddling zero, so a +# missing shift to zero truncates keys out of order. +@pytest.mark.parametrize("offset", [0, -3, 2**40 + 12345, -(2**40) - 777]) +@pytest.mark.parametrize( + "span", + [1, 2, 2**8, 2**16, 2**16 + 1, 2**32, 2**32 + 1, 2**48, 2**48 + 1, 2**62, 2**64], +) +@pytest.mark.parametrize("size", [100, 2000]) +def test_group_by_key_orders_keys_ascending(size: int, span: int, offset: int) -> None: + """Every key range, from one radix pass to four or the small-input sort, groups the keys. + + Keys cover their whole range, so a pass that drops a digit or the shift to + zero misorders them. The order within a group is not checked: it is + unspecified. + """ + from zarr_indexing.chunk_resolution import _group_by_key + + if span == 2**64: + # The whole int64 range, whose span wraps when shifted to zero. + info = np.iinfo(np.int64) + low, high = int(info.min), int(info.max) + else: + low = max(int(np.iinfo(np.int64).min), offset) + high = min(int(np.iinfo(np.int64).max), low + span - 1) + rng = np.random.default_rng([size, span.bit_length(), abs(offset)]) + keys = rng.integers(low, high, size, dtype=np.int64, endpoint=True).astype(np.intp) + keys[:2] = (high, low) + assert _group_by_key(np.sort(keys)) is None + order = _group_by_key(keys) + if low == high: + assert order is None + return + assert order is not None + assert order.dtype == np.intp + np.testing.assert_array_equal(np.sort(order), np.arange(keys.size)) + assert bool((np.diff(keys[order]) >= 0).all()) diff --git a/packages/zarr-indexing/tests/test_writer.py b/packages/zarr-indexing/tests/test_writer.py index e3405d824c..0c68ba9343 100644 --- a/packages/zarr-indexing/tests/test_writer.py +++ b/packages/zarr-indexing/tests/test_writer.py @@ -29,13 +29,33 @@ def __setitem__(self, key: tuple[int | slice, ...], value: Any) -> None: self.data[key] = value +def _assert_each_cell_holds_a_written_value( + actual: Any, initial: Any, identifiers: Any, values: Any +) -> None: + """Cells the selection misses are unchanged; each addressed cell holds one value written to it. + + Which of a repeated cell's values lands is unspecified, so any of them passes. + """ + candidates: dict[int, set[Any]] = {} + for identifier, value in zip( + np.asarray(identifiers).flat, np.asarray(values).flat, strict=True + ): + candidates.setdefault(int(identifier), set()).add(value.item()) + flat_actual = np.asarray(actual).reshape(-1) + flat_initial = np.asarray(initial).reshape(-1) + for cell, (got, before) in enumerate( + zip(flat_actual.tolist(), flat_initial.tolist(), strict=True) + ): + assert got in candidates.get(cell, {before}), (cell, got, candidates.get(cell, {before})) + + @pytest.mark.parametrize( "case", ["reverse", "scalar", "newaxis", "empty", "outer", "composed", "correlated", "transpose"], ) def test_write_coordinates(case: str) -> None: data = np.arange(24).reshape(4, 6) - expected = data.copy() + initial = data.copy() # Select flat source identifiers independently of the transform evaluator. identifiers = np.arange(data.size).reshape(data.shape) transform = IndexTransform.from_shape(data.shape) @@ -74,11 +94,9 @@ def test_write_coordinates(case: str) -> None: ) identifiers = identifiers.T values = np.arange(identifiers.size).reshape(identifiers.shape) + 100 - for index, value in zip(np.asarray(identifiers).flat, values.flat, strict=True): - expected.flat[index] = value source = BasicSource(data) assert write_into(source, transform, values) is None - np.testing.assert_array_equal(data, expected) + _assert_each_cell_holds_a_written_value(data, initial, identifiers, values) if case in ("reverse", "scalar", "newaxis", "transpose"): assert len(source.calls) == 1 elif case == "empty": @@ -132,7 +150,8 @@ def test_write_constant_repeated_destination() -> None: source = BasicSource(np.zeros(3)) transform = IndexTransform(IndexDomain.from_shape((2, 3)), (ConstantMap(1),)) write_into(source, transform, [[1], [9]]) - np.testing.assert_array_equal(source.data, [0, 9, 0]) + assert source.data[[0, 2]].tolist() == [0, 0] + assert source.data[1] in (1, 9) assert len(source.calls) == 6 @@ -201,7 +220,6 @@ def test_lazy_write_and_assignment(backend: str, case: str) -> None: source[:] = initial array = LazyArray(source) identifiers = initial.copy() - expected = initial.copy() if case == "basic": view = array[::-1, ::2][1:, 1:] identifiers = identifiers[::-1, ::2][1:, 1:] @@ -222,16 +240,17 @@ def test_lazy_write_and_assignment(backend: str, case: str) -> None: np.testing.assert_array_equal(source[:], initial[::-1]) return values = np.arange(identifiers.size).reshape(identifiers.shape) + 100 - for identifier, value in zip(identifiers.flat, values.flat, strict=True): - expected.flat[identifier] = value assert view.write(values) is None - np.testing.assert_array_equal(source[:], expected) + _assert_each_cell_holds_a_written_value(source[:], initial, identifiers, values) # All selector assignment entry points write through to the original source. + written = np.array(source[:]) array.oindex[[0, 3], [1, 4]] = 77 - expected[np.ix_([0, 3], [1, 4])] = 77 + written[np.ix_([0, 3], [1, 4])] = 77 + np.testing.assert_array_equal(source[:], written) array.vindex[[0, 0, 2], [2, 2, 4]] = [10, 11, 12] - expected[[0, 0, 2], [2, 2, 4]] = [10, 11, 12] - np.testing.assert_array_equal(source[:], expected) + _assert_each_cell_holds_a_written_value( + source[:], written, initial[[0, 0, 2], [2, 2, 4]], [10, 11, 12] + ) @pytest.mark.parametrize("backend", ["numpy", "zarr"]) @@ -260,11 +279,8 @@ def test_random_selection_chain_scatter(backend: str) -> None: view = view[::step, ::-1] identifiers = identifiers[::step, ::-1] values = rng.integers(100, 200, size=identifiers.shape) - expected = initial.copy() - for identifier, value in zip(identifiers.flat, values.flat, strict=True): - expected.flat[identifier] = value view.write(values) - np.testing.assert_array_equal(source[:], expected) + _assert_each_cell_holds_a_written_value(source[:], initial, identifiers, values) class _ChunkTouches: @@ -318,11 +334,15 @@ def test_fancy_writes_touch_each_write_chunk_once(sharded: bool, case: str) -> N touched = {(r // 4, c // 4) for r, c in zip(rows, cols, strict=True)} else: view.oindex[[5, 5, 6], 2:5] = np.array([[1] * 3, [2] * 3, [3] * 3]) - expected[5, 2:5] = 2 expected[6, 2:5] = 3 touched = {(1, 0), (1, 1)} gets, sets = touches.gets, touches.sets - np.testing.assert_array_equal(source[:], expected) + actual = source[:] + if case == "duplicates": + # Row 5 is written twice; either row of values may land, but one whole value per cell. + assert set(actual[5, 2:5].tolist()) <= {1, 2} + expected[5, 2:5] = actual[5, 2:5] + np.testing.assert_array_equal(actual, expected) assert sets == len(touched), (sets, len(touched)) # At most the hull read plus zarr's own partial-chunk read per cell. assert gets <= 2 * len(touched), (gets, len(touched)) @@ -462,29 +482,85 @@ def read_into(self, source: Any, context: ReadContext, out: Any, /) -> None: np.testing.assert_array_equal(view.result(), [1, 2, 3]) -@pytest.mark.parametrize( - "flat", - [ - [], - [3], - [0, 2, 5, 7], - [0, 0, 2, 5, 5, 5, 7], - [5, 0, 5, 2, 0, 7], - [7, 5, 5, 2, 0], - ], - ids=["empty", "single", "sorted", "sorted_repeats", "unsorted_repeats", "reversed"], -) -def test_last_occurrences_matches_sequential_assignment(flat: list[int]) -> None: - from zarr_indexing.writer import _last_occurrences - - indices = np.asarray(flat, dtype=np.intp) - values = np.arange(1, indices.size + 1) - expected = np.zeros(8, dtype=values.dtype) - for index, value in zip(indices, values, strict=True): - expected[index] = value - keep = _last_occurrences(indices) - kept = indices[keep] - assert np.unique(kept).size == kept.size - actual = np.zeros(8, dtype=values.dtype) - actual[kept] = values[keep] - np.testing.assert_array_equal(actual, expected) +class GridSource: + """A readable, non-NumPy source with a write grid, so writes scatter cell by cell.""" + + def __init__(self, data: np.ndarray[Any, Any], chunks: tuple[tuple[int, ...], ...]) -> None: + self.data = data + self.shape = data.shape + self.dtype = data.dtype + self.chunks = chunks + self.reads = 0 + + def __getitem__(self, key: tuple[slice, ...]) -> np.ndarray[Any, Any]: + assert all(isinstance(item, slice) for item in key) + self.reads += 1 + return self.data[key] + + def __setitem__(self, key: tuple[slice, ...], value: Any) -> None: + assert all(isinstance(item, slice) for item in key) + self.data[key] = value + + +def _reverse_within_groups( + keys: np.ndarray[Any, np.dtype[np.intp]], +) -> np.ndarray[Any, np.dtype[np.intp]]: + """Group equal keys, listing each group's entries in reverse request order.""" + return np.asarray(np.lexsort((-np.arange(keys.size), keys)), dtype=np.intp) + + +def _repeated_selection(case: str, view: Any, identifiers: Any) -> tuple[Any, Any]: + """A selection that addresses some cells twice and lists each grid cell's points unsorted.""" + # On the 4-by-4 grid below, rows 3, 1, 3, 0, 2, 1 land in the first chunk + # row and 6, 5 in the second; the columns split 7, 5, 6 against 2, 2, 0. + rows = [3, 1, 3, 0, 6, 2, 1, 5] + cols = [7, 2, 5, 2, 0, 6] + if case == "orthogonal": + return view.oindex[rows, cols], identifiers[np.ix_(rows, cols)] + if case == "vectorized": + points = ([3, 1, 3, 6, 5, 1, 0, 6], [2, 0, 2, 7, 4, 0, 3, 5]) + return view.vindex[points], identifiers[points] + # Mixed: a vectorized pair over an orthogonal gather of the rows. + gathered = view.oindex[rows, :] + points = ([0, 2, 4, 1, 6, 0, 7, 3], [6, 6, 1, 0, 0, 6, 5, 1]) + return gathered.vindex[points], identifiers[rows, :][points] + + +@pytest.mark.parametrize("grouping", ["default", "reversed"]) +# The per-cell path copies blocks into plain arrays, so only NumPy sources carry masks. +@pytest.mark.parametrize("backend", ["planned", "numpy", "numpy-masked"]) +@pytest.mark.parametrize("case", ["orthogonal", "vectorized", "mixed"]) +def test_repeated_writes_land_one_whole_value_per_cell( + monkeypatch: pytest.MonkeyPatch, case: str, backend: str, grouping: str +) -> None: + """Repeated targets get one of their values whatever order the planner lists a cell's points in. + + Targets addressed once get exactly their value, so a writer that pairs a + point with another point's value fails whichever order grouping produces. + """ + from zarr_indexing import chunk_resolution + from zarr_indexing.lazy_array import LazyArray + + if grouping == "reversed": + monkeypatch.setattr(chunk_resolution, "_group_by_key", _reverse_within_groups) + initial = np.arange(64, dtype=np.int64).reshape(8, 8) + masked = backend == "numpy-masked" + data: Any = initial.copy() + if masked: + data = np.ma.array(data, mask=np.zeros(data.shape, dtype=bool)) + source: Any = GridSource(data, ((4, 4), (4, 4))) if backend == "planned" else data + view, identifiers = _repeated_selection(case, LazyArray(source), np.arange(64).reshape(8, 8)) + values = np.arange(identifiers.size).reshape(identifiers.shape) * 7 + 100 + if masked: + # Mask values by parity so a cell whose data and mask came from two + # different writes is caught: written value v is masked iff v is odd. + values = np.ma.array(values, mask=values % 2 == 1) + view.write(values) + if backend == "planned": + assert source.reads > 0 + _assert_each_cell_holds_a_written_value(np.ma.getdata(data), initial, identifiers, values) + if masked: + np.testing.assert_array_equal( + np.ma.getmaskarray(data), + np.isin(np.arange(64), identifiers).reshape(8, 8) & (np.ma.getdata(data) % 2 == 1), + )