From e3e1c420a7d12c1eccef19ff64a994dcaff38661 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 15:28:15 +0100 Subject: [PATCH 01/17] feat(zarr-metadata): compare each codec in its simplest spelling for the chunk it receives A definition's `canonical` reduces two spellings of a configuration to one when they mean the same thing wherever the declaration is used. It receives the configuration alone, so it cannot reduce spellings that mean the same thing only for some chunks. The `endian` of a `bytes` codec is one: it means nothing when each value takes one byte, and is required when a value takes several. `ArrayCodecDefinition` gains `chunk_canonical`, a function of the configuration, the nested declarations and the chunk, and `codec_for_chunk` returns the codec of a `Stage` in that spelling, with each codec of the pipelines it contains simplified for the chunk that one receives. The result is a declaration, so `declaration_key`, `canonical_of` and `refines` work on it as on any other. Its configuration is checked again: the definition must allow it, the chunk rules must find no problem with it, and each nested declaration must stay as it is. `bytes` declares a `chunk_canonical` that drops `endian` for a data type stored in single bytes, and a v3 array model compares, hashes and refines by its codecs as `codec_for_chunk` returns them. So an `int8` array is one array with either `endian` or with none, as zarr-python 3.0 and 3.1 write it, while a `bytes` codec after a `cast_value` to a multi-byte data type is still compared by its `endian`. Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/450.feature.md | 1 + packages/zarr-metadata/docs/extensions.md | 4 +- packages/zarr-metadata/docs/models.md | 7 + .../src/zarr_metadata/definition/__init__.py | 10 +- .../zarr_metadata/definition/_definition.py | 48 ++++++ .../src/zarr_metadata/model/_array_v3.py | 19 ++- .../src/zarr_metadata/v3/_definition.py | 28 +++ .../src/zarr_metadata/v3/_pipeline.py | 107 +++++++++++- .../src/zarr_metadata/v3/codec/bytes.py | 27 +++ .../src/zarr_metadata/v3/definition.py | 11 +- .../zarr-metadata/tests/model/test_array.py | 53 ++++++ .../tests/v3/test_definitions.py | 1 + .../zarr-metadata/tests/v3/test_pipelines.py | 159 +++++++++++++++++- 13 files changed, 462 insertions(+), 13 deletions(-) create mode 100644 packages/zarr-metadata/changes/450.feature.md diff --git a/packages/zarr-metadata/changes/450.feature.md b/packages/zarr-metadata/changes/450.feature.md new file mode 100644 index 0000000000..c314478857 --- /dev/null +++ b/packages/zarr-metadata/changes/450.feature.md @@ -0,0 +1 @@ +A v3 codec that receives an array can declare its simplest spelling for the chunk it receives, with `chunk_canonical` on `ArrayCodecDefinition`. It is a function of the configuration, the nested declarations and the `Chunk`, for two spellings that mean the same thing for one chunk and not for another, which `canonical` cannot tell from the configuration alone. `codec_for_chunk` in `zarr_metadata.v3.definition` returns the codec of a `Stage` in that spelling, with each codec of the pipelines it contains simplified for the chunk that one receives. The `bytes` codec declares one: it drops an `endian` when each value takes one byte. So two `ZarrV3ArrayMetadata` models that differ only in the `endian` of a `bytes` codec that receives values of one byte (`bool`, `int8`, `uint8` or `r8`) are now equal, hash alike and refine each other. A `bytes` codec that receives values of several bytes, as it does after a `cast_value` to `float32`, is compared by its `endian` as before. diff --git a/packages/zarr-metadata/docs/extensions.md b/packages/zarr-metadata/docs/extensions.md index fc314760c8..0aa9332c56 100644 --- a/packages/zarr-metadata/docs/extensions.md +++ b/packages/zarr-metadata/docs/extensions.md @@ -122,7 +122,9 @@ Other kinds declare more: - A codec is defined by the class of its kind, `ArrayArrayCodecDefinition`, `ArrayBytesCodecDefinition` or `BytesBytesCodecDefinition`. A codec that receives an array declares what it checks about the chunk it receives - (`chunk_rules`) and any pipelines it contains (`pipelines`), and an + (`chunk_rules`), its simplest spelling for that chunk + (`chunk_canonical`, as `bytes` drops an `endian` when each value takes + one byte) and any pipelines it contains (`pipelines`), and an array -> array codec also declares what it hands on (`transition`). A bytes -> bytes codec receives bytes, so it declares none of these. diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index ea1a4ed294..ad2a700f6c 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -220,6 +220,13 @@ These are each declaration, and the fill value read against its data type. - A v3 blosc codec with `noshuffle` [ignores `typesize`](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/blosc/index.html#configuration-parameters). So it is the same codec with or without one. +- A v3 `bytes` codec's `endian` orders the bytes of each value, and the + encoding of a value that takes one byte + [does not depend on it](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/bytes/index.html#configuration-parameters). + So in a `uint8` array, a `bytes` codec is the same codec with either + `endian`, or with none. The codec is compared for the chunk it + receives: after a `cast_value` to `float32`, the two `endian` values + are two codecs. A v2 document's [dtype](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#data-type-encoding), diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index f259f3d7fa..df2f4905a7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -214,7 +214,11 @@ def acme_lz4_rules( equivalent spelling, and declarations are compared by it. Each nested declaration is written in its own simplest spelling, then the definition's `canonical` is applied (blosc drops a `typesize` that `noshuffle` ignores, and a -rectilinear grid run-length encodes its chunk shapes). The envelope +rectilinear grid run-length encodes its chunk shapes). A v3 codec that +receives an array can have a simpler spelling still for the chunk it +receives, which its `chunk_canonical` declares, and an array model +compares its codecs in that spelling (see `zarr_metadata.v3.definition`). +The envelope uses the shortest form every reader accepts: a data type with nothing to configure is its bare name, and any other declaration is an object, `{"name": ...}`, because a Zarr v3.0 reader does not accept a short-hand @@ -259,8 +263,8 @@ def acme_lz4_rules( that has a member no checker reads (the error names the TypedDict that contains it); a `name` that is not a string; an attribute declared as a function (`rules`, `canonical`, `fill_value_rules`, `storage`, -`shape_rules`, `chunk_lengths`, `chunk_rules`, `transition`, -`pipelines`) that is not a function; a data type's `fill_value` that no +`shape_rules`, `chunk_lengths`, `chunk_rules`, `chunk_canonical`, +`transition`, `pipelines`) that is not a function; a data type's `fill_value` that no checker reads, or that contains a declaration, which a value of the data type never does; a codec `kind` that is not one of the three, or a `size` that is not `"static"` or `"dynamic"`; a function that is never diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 05eefcf761..8d3a30344e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1259,6 +1259,52 @@ def written_json(declaration: AcceptedDeclaration[Any] | UnclaimedDeclaration) - return kind._envelope_json(declaration.name, declaration.configuration) +def respelled( + declaration: AcceptedDeclaration[D], + configuration: Mapping[str, JSONValue], + nested: Nested, +) -> AcceptedDeclaration[D]: + """`declaration` with `configuration` as its configuration, and each declaration in `nested` in place of the nested declaration at its location. + + This builds the declaration that another spelling of `declaration` + reads as, without reading it again. The caller answers for the + spelling: `configuration` must be one that the definition allows and + that means the same thing, and it must keep each nested declaration + where it is. `nested` holds only the nested declarations to replace, + each one accepted or unclaimed, and each is written into the + configuration as its `to_json` writes it. A location in `nested` that + does not lead into `configuration` raises `TypeError`. The result is + written as `to_json` writes it, since no document wrote it. + """ + spelled: JSONValue = dict(configuration) + for loc, inner in nested.items(): + spelled = _replaced(spelled, loc, written_json(held(inner))) + kind = declaration.read_as + name = declaration.name + carried = kind._filed_under(name)[1] is not None + return AcceptedDeclaration( + json=kind._envelope_json(name, {} if carried else cast("Mapping[str, JSONValue]", spelled)), + name=name, + definition=declaration.definition, + configuration=cast("Mapping[str, JSONValue]", spelled), + nested={**declaration.nested, **nested}, + ) + + +def item_at(value: JSONValue, path: Loc) -> JSONValue | UNSET: + """The item of `value` at `path`; `UNSET` when `path` does not lead into `value`.""" + for step in path: + if isinstance(step, str): + if not isinstance(value, Mapping) or step not in value: + return UNSET + value = value[step] + else: + if not isinstance(value, tuple) or not 0 <= step < len(value): + return UNSET + value = value[step] + return value + + Nested: TypeAlias = Mapping[Loc, ResolvedDeclaration[Any]] """The declarations in a configuration, each as the scope read it, keyed by location in the configuration.""" @@ -1902,6 +1948,7 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "held", "in_document", "is_declaration", + "item_at", "kind_of", "named_configuration", "no_rules", @@ -1909,6 +1956,7 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "read_only", "resolve", "resolve_refined", + "respelled", "ruled", "spelled_canonically", "unchanged", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 627545859c..2ea2af7ccb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -78,7 +78,7 @@ chunk_grid_lengths, default_array_bytes_codec, ) -from zarr_metadata.v3._pipeline import Stage, read_pipeline +from zarr_metadata.v3._pipeline import Stage, codec_for_chunk, read_pipeline from zarr_metadata.v3.array import ( ZARR_V3_ARRAY_METADATA_STORE_KEY, ZarrV3ArrayMetadataJSON, @@ -780,7 +780,7 @@ def refines(self, other: ZarrV3ArrayMetadata) -> bool: (self.data_type, other.data_type), (self.chunk_grid, other.chunk_grid), (self.chunk_key_encoding, other.chunk_key_encoding), - *zip(self.codecs, other.codecs, strict=True), + *zip(codecs_for_chunks(self._reading), codecs_for_chunks(other._reading), strict=True), *zip(self.storage_transformers, other.storage_transformers, strict=True), *((self.extra_fields[key], other.extra_fields[key]) for key in self.extra_fields), ) @@ -926,8 +926,10 @@ def array_key_of( The key holds each declaration's `declaration_key`, and the fill value as JSON text: in its canonical spelling when a definition in scope read the - data type, and as written otherwise. Every other member is included - as it is, with JSON members as text. + data type, and as written otherwise. A codec is compared as + `codec_for_chunk` returns it for the chunk it receives, so two + spellings that mean the same for that chunk are one codec. Every + other member is included as it is, with JSON members as text. """ data_type = held(reading.data_type) fill_value = ( @@ -940,7 +942,7 @@ def array_key_of( json_text(fill_value), declaration_key(data_type), declaration_key(held(reading.chunk_grid)), - tuple(declaration_key(held(stage.codec)) for stage in reading.pipeline), + tuple(declaration_key(codec) for codec in codecs_for_chunks(reading)), declaration_key(held(reading.chunk_key_encoding)), members.dimension_names, json_text(members.attributes), @@ -949,6 +951,13 @@ def array_key_of( ) +def codecs_for_chunks( + reading: ZarrV3ArrayMetadataReading, +) -> tuple[AcceptedDeclaration[CodecDefinition[Any]] | UnclaimedDeclaration, ...]: + """The codecs of `reading`, a read that found no problems, in pipeline order: each as `codec_for_chunk` returns it for the chunk it receives.""" + return tuple(held(codec_for_chunk(stage)) for stage in reading.pipeline) + + def extra_fields_key(extra_fields: Mapping[str, ResolvedDeclaration[Any]]) -> tuple[object, ...]: """A key for what a document's extra fields mean: each key with its declaration's `declaration_key`, sorted by key.""" return tuple( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index 0d734d81e1..dc4a3135b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -56,6 +56,7 @@ Nested, Problems, ResolvedDeclaration, + T, WithFillValue, asked, is_declaration, @@ -94,6 +95,11 @@ def no_pipelines(*_: object) -> Mapping[str, Chunk]: return {} +def unchanged_for_chunk(configuration: T, nested: object, chunk: object) -> T: + """The simplest spelling of a codec whose configuration means the same for every chunk it receives: the configuration itself.""" + return configuration + + StorageClass = Literal["single_byte", "multi_byte", "variable_length"] """How a data type's values are stored: in single bytes, in several bytes at a time, or each value in as many bytes as it needs. @@ -415,10 +421,31 @@ class ArrayCodecDefinition(CodecDefinition[C]): inner codecs receive its inner chunks, and its index codecs receive the shard index. It is called whatever the chunk rules found, and must return only what is true in either case. + + Two spellings of a configuration can mean the same thing for one + chunk and different things for another. For example, the `endian` of + a `bytes` codec changes nothing when each value takes one byte, and + is required when a value takes several. `canonical` cannot reduce + such spellings to one, because it receives the configuration alone. + `chunk_canonical` can: it receives the configuration, the nested + declarations as the scope read them, and the chunk, and returns the + simplest configuration that means the same for that chunk. It is + called only for a codec that the chunk rules found no problem with. + It simplifies only the codec's own members, and it must keep each + nested declaration as it is, because each codec of a pipeline the + codec contains is simplified for the chunk it receives in turn. What + does not depend on the chunk is left to `canonical`, which simplifies + the result as it simplifies any configuration. A chunk rule checks + only the parts of a chunk that are known, and `chunk_canonical` + simplifies only by them: a chunk with nothing known about it, + `Chunk()`, leaves every configuration as it is. `codec_for_chunk` + returns the codec of a `Stage` in that spelling. """ chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules """The spec violations for this codec with a given input chunk, located in the configuration. A problem at `()` is about the codec as a whole, such as an input it cannot encode, and is reported at the codec.""" + chunk_canonical: Callable[[C, Nested, Chunk], C] = unchanged_for_chunk + """A configuration the chunk rules allow for a given input chunk, in the simplest spelling that means the same for that chunk.""" pipelines: Callable[[C, Nested, Chunk], Mapping[str, Chunk]] = no_pipelines """The chunk each contained pipeline receives, keyed by the configuration member that holds the pipeline.""" @@ -879,6 +906,7 @@ def chunk_grid_lengths( "no_pipelines", "single_byte", "storage_of", + "unchanged_for_chunk", "unknown_chunk", "unknown_lengths", "unknown_storage", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index 592bb18a5f..9842d827da 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -33,13 +33,23 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, TypeGuard, cast -from zarr_metadata._json import ValidationProblem, is_object, is_tuple, with_input +from zarr_metadata._json import ( + ValidationProblem, + copied, + is_object, + is_tuple, + json_text, + with_input, +) +from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._definition import ( AcceptedDeclaration, ResolvedDeclaration, asked, in_document, + item_at, read_only, + respelled, ruled, ) from zarr_metadata.v3._definition import ( @@ -53,6 +63,7 @@ if TYPE_CHECKING: from collections.abc import Iterator, Mapping, Sequence + from zarr_metadata._common import JSONValue from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import ( Nested, @@ -326,4 +337,96 @@ def _handed_on( return given -__all__ = ["Stage", "read_pipeline"] +def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: + """The codec of `stage`, in the simplest spelling that means the same for the chunk it receives. + + A codec's own `canonical` reduces the spellings that mean the same + thing wherever the codec is used. Some spellings mean the same thing + only for some chunks. For example, a `bytes` codec with and without + `endian` is one codec for values that take one byte, and two for + values that take several. So a codec is compared, and written in its + simplest spelling, as the codec this function returns: first each + codec of each pipeline it contains, for the chunk that codec + receives, and then its own configuration, as its definition's + `chunk_canonical` simplifies it for the chunk in `stage`. + + A codec the scope did not read is returned as it is. A codec's own + configuration is kept as it is when the codec receives bytes, and + when its chunk rules find a problem with it, because a codec with a + problem has no simpler spelling. When nothing is simplified, the + result is the codec itself. Otherwise it is a new declaration, read + by the same definition. `canonical_of` writes it in its simplest + spelling, and `==` compares it. + + `chunk_canonical` is the extension author's code, so its result is + checked: the definition must allow it, the chunk rules must find no + problem with it, and each nested declaration must be where it was. A + result that fails a check raises `ValueError`, which indicates a bug + in the definition, not in the codec. + """ + codec = stage.codec + if not isinstance(codec, AcceptedDeclaration): + return codec + inner: dict[Loc, ResolvedDeclaration[Any]] = {} + for member, stages in stage.inner.items(): + for index, each in enumerate(stages): + simpler = codec_for_chunk(each) + if simpler is not each.codec: + inner[(member, index)] = simpler + configuration = _for_chunk(codec, stage.incoming) + if configuration is None and len(inner) == 0: + return codec + return respelled(codec, codec.configuration if configuration is None else configuration, inner) + + +def _for_chunk( + codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None +) -> Mapping[str, JSONValue] | None: + """The configuration of `codec` as its definition's `chunk_canonical` simplifies it for `chunk`, checked; None when nothing is simplified.""" + definition = codec.definition + if chunk is None or not isinstance(definition, ArrayCodecDefinition): + return None + configuration, nested = codec.configuration, codec.nested + if len(_chunk_problems(definition, configuration, nested, chunk, ())) != 0: + return None + given: object = asked( + definition, + "chunk_canonical", + lambda: cast("object", definition.chunk_canonical(read_only(configuration), nested, chunk)), + ) + if given is configuration: + return None + if not is_object(given): + msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}" + raise TypeError(msg) + simplified = cast("dict[str, JSONValue]", copied(cast("JSONValue", dict(given)))) + if json_text(simplified) == json_text(cast("JSONValue", configuration)): + return None + _, refused = definition.read_configuration(simplified) + found = ( + *refused, + *_chunk_problems(definition, simplified, nested, chunk, ()), + *( + ValidationProblem( + loc, "expected the nested declaration to stay as it is", "invalid_value" + ) + for loc in nested + if _text_at(simplified, loc) != _text_at(cast("JSONValue", configuration), loc) + ), + ) + if len(found) != 0: + msg = ( + f"{definition.name!r}: its chunk_canonical gave {simplified!r}, which does not hold: " + f"{list(found)!r}" + ) + raise ValueError(msg) + return simplified + + +def _text_at(configuration: JSONValue, loc: Loc) -> str | None: + """The JSON text of the item of `configuration` at `loc`; None when `loc` does not lead into it.""" + item = item_at(configuration, loc) + return None if item is UNSET else json_text(item) + + +__all__ = ["Stage", "codec_for_chunk", "read_pipeline"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 5b08cf3b0a..ca30fa46e2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -64,11 +64,38 @@ def _chunk_rules( ) +def _chunk_canonical( + configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk +) -> BytesCodecConfiguration: + """Remove `endian` when each value takes one byte, because it has no meaning there. + + `endian` is "Required for data types for which endianness is + applicable" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L64-L69), + and the encoding of a value that takes one byte does "not depend on + endian" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L84-L101). + So for such a data type, a `bytes` codec with either `endian`, or + with none, encodes the same bytes, and the three spellings describe + the same codec. zarr-python 3.0 wrote `endian` for these data types, + and zarr-python 3.1 does not. The storage is the one the data type + declares. If it is unknown, the configuration is kept as written. + """ + if chunk.data_type is None or "endian" not in configuration: + return configuration + if storage_of(chunk.data_type) != "single_byte": + return configuration + simpler = configuration.copy() + del simpler["endian"] + return simpler + + BYTES_CODEC: Final = ArrayBytesCodecDefinition( name=BYTES_CODEC_NAME, configuration=BytesCodecConfiguration, size="static", chunk_rules=_chunk_rules, + chunk_canonical=_chunk_canonical, ) """The `bytes` codec.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py index c98d6460c1..9c20fac4a5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -90,6 +90,14 @@ pipeline. For example, a shard's inner codecs receive its inner chunks, and its index codecs receive the shard index. Each pipeline is read the same way, and its stages are stored as the codec's `Stage.inner`. +Two spellings of a codec can mean the same thing for one chunk and not +for another, which `canonical` cannot tell, since it receives the +configuration alone. Such a codec declares its simplest spelling for a +given chunk with `chunk_canonical`. For example, `bytes` drops an +`endian` when each value takes one byte. `codec_for_chunk` returns the +codec of a `Stage` in that spelling, with each codec of the pipelines +it contains simplified for the chunk that one receives. An array model +compares its codecs this way. The reader does not guess. After a codec the scope did not read, or after a codec that does not declare its output, the next codec receives a chunk with nothing known about it, `Chunk()`, which never causes a @@ -147,7 +155,7 @@ default_array_bytes_codec, storage_of, ) -from zarr_metadata.v3._pipeline import Stage +from zarr_metadata.v3._pipeline import Stage, codec_for_chunk __all__ = [ "CORE_AND_EXTENSIONS_CONTEXT", @@ -181,6 +189,7 @@ "ZarrV3Kind", "ZarrV3StaticCodecDeclaration", "ZarrV3StorageTransformerDeclaration", + "codec_for_chunk", "default_array_bytes_codec", "storage_of", ] diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 7fd62c5442..2d2ae6cf27 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -805,6 +805,59 @@ def test_two_models_are_one_array_when_their_declarations_read_the_same( assert models[0].to_json() != models[1].to_json() +_BIG = {"name": "bytes", "configuration": {"endian": "big"}} +_TO_UINT8 = {"name": "cast_value", "configuration": {"data_type": "uint8"}} +_TO_FLOAT32 = {"name": "cast_value", "configuration": {"data_type": "float32"}} + + +def _shard_of(*codecs: object) -> dict[str, object]: + return { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [1], + "codecs": list(codecs), + "index_codecs": [_LITTLE, {"name": "crc32c"}], + }, + } + + +@pytest.mark.parametrize( + ("data_type", "left", "right", "same"), + [ + ("uint8", [_LITTLE], [{"name": "bytes"}], True), + ("uint8", [_LITTLE], [_BIG], True), + ("uint16", [_LITTLE], [_BIG], False), + ("uint8", [_TO_FLOAT32, _LITTLE], [_TO_FLOAT32, _BIG], False), + ("uint16", [_TO_UINT8, _LITTLE], [_TO_UINT8, {"name": "bytes"}], True), + ("uint8", [_shard_of(_LITTLE)], [_shard_of({"name": "bytes"})], True), + ("uint16", [_shard_of(_LITTLE)], [_shard_of(_BIG)], False), + ], + ids=[ + "one-byte-endian-or-none", + "one-byte-either-endian", + "two-bytes-either-endian", + "cast-to-four-bytes", + "cast-to-one-byte", + "shard-of-one-byte", + "shard-of-two-bytes", + ], +) +def test_two_models_are_one_array_when_each_codec_means_the_same_for_its_chunk( + data_type: str, left: object, right: object, same: bool +) -> None: + """A codec is compared for the chunk it receives, as `codec_for_chunk` returns it: the `endian` of a `bytes` codec tells two arrays apart only where a value takes several bytes. `refines` agrees with `==`, and `to_json` writes each codec as given.""" + document = ZarrV3ArrayMetadata.create_default(shape=(2,), data_type=data_type).to_json() + models = [ + ZarrV3ArrayMetadata.from_json({**document, "codecs": codecs}) for codecs in (left, right) + ] + assert (models[0] == models[1]) is same + assert models[0].refines(models[1]) is same + assert models[1].refines(models[0]) is same + assert models[0].to_json() != models[1].to_json() + if same: + assert hash(models[0]) == hash(models[1]) + + @pytest.mark.parametrize( ("left", "right", "same"), [ diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index d88dcf639e..70ac04d5eb 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -1473,6 +1473,7 @@ def test_error_a_data_type_fill_value_holding_a_declaration() -> None: (ChunkGridDefinition, "shape_rules"), (ChunkGridDefinition, "chunk_lengths"), (ArrayArrayCodecDefinition, "chunk_rules"), + (ArrayArrayCodecDefinition, "chunk_canonical"), (ArrayArrayCodecDefinition, "transition"), (ArrayArrayCodecDefinition, "pipelines"), ], diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index 7fcab1c1d2..fe477dc8f0 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -8,15 +8,19 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +import dataclasses +from typing import TYPE_CHECKING, Any, cast import pytest from typing_extensions import TypedDict +from zarr_metadata._json import json_text from zarr_metadata.definition import ( + AcceptedDeclaration, EmptyConfiguration, Nested, ResolvedDeclaration, + canonical_of, resolve, ) from zarr_metadata.model import ( @@ -26,6 +30,7 @@ from zarr_metadata.v3._pipeline import ( read_pipeline, ) +from zarr_metadata.v3.codec.bytes import BYTES_CODEC, BytesCodecConfiguration from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, ArrayArrayCodecDefinition, @@ -37,6 +42,7 @@ ZarrV3CodecDeclaration, ZarrV3Context, ZarrV3DataTypeDeclaration, + codec_for_chunk, ) if TYPE_CHECKING: @@ -422,6 +428,7 @@ def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) ("kind", "attribute"), [ (BytesBytesCodecDefinition, "chunk_rules"), + (BytesBytesCodecDefinition, "chunk_canonical"), (BytesBytesCodecDefinition, "transition"), (BytesBytesCodecDefinition, "pipelines"), (ArrayBytesCodecDefinition, "transition"), @@ -545,3 +552,153 @@ def test_error_a_stage_s_inner_pipelines_are_read_only() -> None: stage.inner["codecs"] = () # pyright: ignore[reportIndexIssue] for again in (pickle.loads(pickle.dumps(stages)), copy.deepcopy(stages)): assert again == stages + + +BIG: JSONValue = {"name": "bytes", "configuration": {"endian": "big"}} + +BARE: JSONValue = {"name": "bytes"} + +CRC32C: JSONValue = {"name": "crc32c"} + + +def _shard(*codecs: JSONValue, index_codecs: tuple[JSONValue, ...] = (LITTLE, CRC32C)) -> JSONValue: + """A shard of inner chunks of 2 by 1 that encodes them with `codecs`, and its index with `index_codecs`.""" + return { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [2, 1], + "codecs": list(codecs), + "index_codecs": list(index_codecs), + }, + } + + +def _simplest(codecs: list[JSONValue], chunk: Chunk) -> list[JSONValue]: + """Each codec of a pipeline that receives `chunk`, in its simplest spelling for the chunk it receives.""" + stages, problems = _read(codecs, chunk) + assert problems == () + return [canonical_of(codec_for_chunk(stage), ()) for stage in stages] + + +@pytest.mark.parametrize( + ("data_type", "codecs", "simplest"), + [ + # Each value takes one byte, so `endian` means nothing. + ("uint8", [LITTLE], [BARE]), + ("int8", [BIG], [BARE]), + ("bool", [LITTLE], [BARE]), + ("r8", [LITTLE], [BARE]), + ("uint8", [BARE], [BARE]), + ("uint8", [_transpose(1, 0), LITTLE, GZIP], [_transpose(1, 0), BARE, GZIP]), + # Each value takes several bytes, so `endian` says how they are ordered. + ("uint16", [LITTLE], [LITTLE]), + ("float32", [BIG], [BIG]), + ("r16", [LITTLE], [LITTLE]), + # The codec is simplified for the chunk it receives, not for the array's data type. + ("uint8", [_cast("float32"), LITTLE], [_cast("float32"), LITTLE]), + ("uint16", [_cast("uint8"), BIG], [_cast("uint8"), BARE]), + # A shard's inner codecs receive the shard's data type, and its index codecs `uint64`. + ("uint8", [_shard(LITTLE)], [_shard(BARE)]), + ("uint16", [_shard(LITTLE)], [_shard(LITTLE)]), + ("uint8", [_shard(_shard(BIG))], [_shard(_shard(BARE))]), + ("uint8", [_cast("uint16"), _shard(LITTLE)], [_cast("uint16"), _shard(LITTLE)]), + ("uint16", [_shard(_cast("uint8"), LITTLE)], [_shard(_cast("uint8"), BARE)]), + # Nothing is known of the values, so nothing is simplified. + ("acme.t", [LITTLE], [LITTLE]), + ("uint8", ["acme.shuffle", LITTLE], [{"name": "acme.shuffle"}, LITTLE]), + ("uint8", ["acme.unclaimed", LITTLE], [{"name": "acme.unclaimed"}, LITTLE]), + ], +) +def test_a_codec_is_written_in_the_simplest_spelling_for_the_chunk_it_receives( + data_type: JSONValue, codecs: list[JSONValue], simplest: list[JSONValue] +) -> None: + """`codec_for_chunk` returns each codec as its `chunk_canonical` simplifies it for the chunk it receives, and each codec of a pipeline it contains for the chunk that one receives. A codec with nothing to simplify is returned as it is.""" + found = _simplest(codecs, _of(data_type)) + assert json_text(found) == json_text(simplest) + # The simplest spelling reads without a problem, and has nothing left to simplify. + assert json_text(_simplest(found, _of(data_type))) == json_text(simplest) + stages, _ = _read(simplest, _of(data_type)) + assert all(codec_for_chunk(stage) is stage.codec for stage in stages) + + +def test_a_codec_with_a_problem_for_its_chunk_is_not_simplified() -> None: + """A codec the chunk rules find a problem with is returned as it is, as is a codec the scope refused.""" + # `bytes` does not encode values that vary in size, and `gzip` requires a level. + stages, problems = _read([LITTLE, {"name": "gzip", "configuration": {}}], _of("string")) + assert [problem.loc for problem in problems] == [(1, "configuration", "level"), (0,)] + assert all(codec_for_chunk(stage) is stage.codec for stage in stages) + + +def _bytes_with(chunk_canonical: object) -> ResolvedDeclaration[CodecDefinition[Any]]: + """A `bytes` codec with `endian`, read by a definition whose `chunk_canonical` is `chunk_canonical`.""" + lying = dataclasses.replace(BYTES_CODEC, name="acme.bytes", chunk_canonical=chunk_canonical) + declaration = {"name": "acme.bytes", "configuration": {"endian": "little"}} + return resolve(declaration, CodecDefinition, CONTEXT.union(ZarrV3Context(codecs=[lying])))[0] + + +def _stage(codec: ResolvedDeclaration[CodecDefinition[Any]], chunk: Chunk) -> Stage: + stages, problems = read_pipeline([codec], chunk) + assert problems == () + return stages[0] + + +def test_error_a_chunk_canonical_the_definition_does_not_allow() -> None: + stage = _stage(_bytes_with(lambda configuration, nested, chunk: {"endian": "middle"}), CHUNK) + with pytest.raises(ValueError, match="'acme.bytes': its chunk_canonical gave"): + codec_for_chunk(stage) + + +def test_error_a_chunk_canonical_the_chunk_rules_do_not_allow() -> None: + # A value of `uint16` takes two bytes, so the chunk rules require `endian`. + stage = _stage(_bytes_with(lambda configuration, nested, chunk: {}), _of("uint16")) + with pytest.raises(ValueError, match="'acme.bytes': its chunk_canonical gave {}, which"): + codec_for_chunk(stage) + + +def test_error_a_chunk_canonical_that_gives_something_else() -> None: + stage = _stage(_bytes_with(lambda configuration, nested, chunk: "little"), CHUNK) + with pytest.raises(TypeError, match="'acme.bytes': its chunk_canonical gives a configuration"): + codec_for_chunk(stage) + + +def test_error_a_chunk_canonical_that_raises_says_whose_it_is() -> None: + def chunk_canonical( + configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk + ) -> BytesCodecConfiguration: + raise KeyError("endian") + + with pytest.raises(KeyError) as raised: + codec_for_chunk(_stage(_bytes_with(chunk_canonical), CHUNK)) + assert raised.value.__notes__ == ["raised by the chunk_canonical of 'acme.bytes'"] + + +def test_error_a_chunk_canonical_that_writes_to_the_configuration() -> None: + def chunk_canonical( + configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk + ) -> BytesCodecConfiguration: + del configuration["endian"] + return configuration + + with pytest.raises(TypeError, match="does not support item deletion"): + codec_for_chunk(_stage(_bytes_with(chunk_canonical), CHUNK)) + + +def test_error_a_chunk_canonical_that_changes_a_nested_declaration() -> None: + # Each codec of a pipeline the codec contains is simplified for the + # chunk it receives, so the codec's own `chunk_canonical` leaves it alone. + def chunk_canonical( + configuration: AcmeHolderConfiguration, nested: Nested, chunk: Chunk + ) -> AcmeHolderConfiguration: + return {"codecs": (cast("ZarrV3CodecDeclaration", BARE),), "types": configuration["types"]} + + holder = ArrayBytesCodecDefinition( + name="acme.holder", + configuration=AcmeHolderConfiguration, + size="dynamic", + chunk_canonical=chunk_canonical, + ) + declaration = {"name": "acme.holder", "configuration": {"codecs": [LITTLE], "types": ["uint8"]}} + codec = resolve(declaration, CodecDefinition, CONTEXT.union(ZarrV3Context(codecs=[holder])))[0] + assert isinstance(codec, AcceptedDeclaration) + with pytest.raises(ValueError, match="expected the nested declaration to stay as it is"): + codec_for_chunk(_stage(codec, CHUNK)) From 1be8677c36104d59ce3d89ccc86075d4a5454006 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 15:42:36 +0100 Subject: [PATCH 02/17] feat(zarr-metadata): return each model with its document in its simplest equivalent spelling Two models are equal when their documents mean the same thing, and `to_json` writes each document as it was given, so equal models can write different documents. A consumer that needs one spelling for all of them, to compare documents field by field or to hash them, had to normalize each member itself. Each model gains `canonicalize`: the v3 and v2 array, group and consolidated metadata models. It returns the model read from its document in its simplest equivalent spelling, written from the parts that `==` compares: each declaration as `canonical_of` writes it, each v3 codec as `codec_for_chunk` returns it for the chunk it receives, and the fill value in the canonical spelling of its data type. Empty `attributes`, empty v3 `storage_transformers` and a v2 `dimension_separator` of "." are left out, since a document without them means the same. Everything else is kept as written. The result equals the model, and equal models in one scope give the same document apart from the order of the members of an object, which a `JSONWriter(sort_keys=True)` sorts. A group keeps the documents its `consolidated_metadata` lists as written, since it does not read them; the consolidated metadata models canonicalize each node they list. Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/452.feature.md | 1 + packages/zarr-metadata/docs/models.md | 75 ++++ .../src/zarr_metadata/model/__init__.py | 5 +- .../src/zarr_metadata/model/_array_v2.py | 35 ++ .../src/zarr_metadata/model/_array_v3.py | 50 +++ .../src/zarr_metadata/model/_group_v2.py | 31 ++ .../src/zarr_metadata/model/_group_v3.py | 37 ++ .../src/zarr_metadata/model/_shared.py | 46 ++ .../tests/model/test_canonicalize.py | 401 ++++++++++++++++++ 9 files changed, 680 insertions(+), 1 deletion(-) create mode 100644 packages/zarr-metadata/changes/452.feature.md create mode 100644 packages/zarr-metadata/tests/model/test_canonicalize.py diff --git a/packages/zarr-metadata/changes/452.feature.md b/packages/zarr-metadata/changes/452.feature.md new file mode 100644 index 0000000000..a79317f00b --- /dev/null +++ b/packages/zarr-metadata/changes/452.feature.md @@ -0,0 +1 @@ +Each model has a `canonicalize` method that returns the model with its document in its simplest equivalent spelling: `ZarrV3ArrayMetadata`, `ZarrV3GroupMetadata`, `ZarrV3ConsolidatedMetadata`, `ZarrV2ArrayMetadata`, `ZarrV2GroupMetadata` and `ZarrV2ConsolidatedMetadata`. The result equals the model. Every model equal to it, in the same scope, gives the same document, apart from the order of the members of an object, so `to_key_value` with `JSONWriter(sort_keys=True)` writes the same bytes for equal models. The document is written from the parts `==` compares: each declaration as `canonical_of` writes it, each v3 codec as `codec_for_chunk` returns it for the chunk it receives, and the fill value as `canonical_fill_value` writes it. Empty `attributes`, empty v3 `storage_transformers` and a v2 `dimension_separator` of `"."` are left out, and everything else is kept as written. diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index ad2a700f6c..16c4e4378a 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -130,6 +130,8 @@ To change a model, use one of these methods: - `refined_in` also reads the document in another context. The new context must claim what this context left unclaimed and contradict nothing. Otherwise it raises `ScopeConflictError`. +- `canonicalize` returns the model with its document in its simplest + equivalent spelling. See [Canonical documents](#canonical-documents). ## Writing store bytes @@ -264,3 +266,76 @@ equal to itself. Equal models have equal hashes. They may still write different documents, because `to_json` writes each document as it was given. + +## Canonical documents + +`canonicalize` returns a model with its document in its simplest +equivalent spelling. The result equals the model it came from. Every model +equal to that one, read in the same context, gives the same document. + +```python test="true" session="models" +from zarr_metadata.model import ZarrV3ArrayMetadata + +written = ZarrV3ArrayMetadata.from_json( + { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, + "fill_value": 0, + "codecs": [ + {"name": "bytes", "configuration": {"endian": "little"}}, + {"name": "zstd", "configuration": {"level": 1, "checksum": False}}, + ], + "attributes": {}, + } +) +canonical = written.canonicalize() +assert canonical == written +print(canonical.to_json()["chunk_key_encoding"]) +#> {'name': 'default'} +print(canonical.to_json()["codecs"]) +#> ({'name': 'bytes'}, {'name': 'zstd', 'configuration': {'level': 1}}) +print("attributes" in canonical.to_json()) +#> False +``` + +The canonical document is written from the same parts that `==` compares: + +- Each declaration is written as `canonical_of` writes it. +- Each v3 codec is written for the chunk it receives, as `codec_for_chunk` + returns it. Above, each `uint8` value takes one byte, so the `endian` + says nothing. +- The fill value is written as `canonical_fill_value` writes it for the + data type. +- A member that says what its absence says is left out. These are empty + `attributes`, empty v3 `storage_transformers`, and a v2 + `dimension_separator` of `"."`. +- Everything else is kept as written. This includes the attributes, and + the configuration of a declaration that nothing in the context claims, + because only its own definition could simplify it. + +A v2 codec is written with each numcodecs default it leaves out, as its +definition's `canonical` writes it. So a canonical v2 document can be +longer than the one it came from. + +The order of the members of an object is not part of the spelling, because +JSON gives it no meaning. Two equal models can give canonical documents +whose members are in different orders. A `JSONWriter` that sorts them +writes the same bytes for both, so those bytes are a key for what the +document means: + +```python test="true" session="models" +import hashlib + +stored = canonical.to_key_value({"sort_keys": True})["zarr.json"] +digest = hashlib.sha256(stored).hexdigest() +``` + +The other models work the same way. A v3 group does not read the documents +its `consolidated_metadata` lists, so its `canonicalize` keeps them as +written. `ZarrV3ConsolidatedMetadata.canonicalize` and +`ZarrV2ConsolidatedMetadata.canonicalize` write each node they list in its +simplest spelling. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 39f4ca5e58..b18d1eb092 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -42,7 +42,10 @@ scope and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as written. `replace` reads new members in the model's own scope. `with_context` and -`refined_in` read the document in another scope. `to_key_value` writes +`refined_in` read the document in another scope. `canonicalize` returns +the model with its document in its simplest equivalent spelling, the one +that every equal model gives, apart from the order of the members of an +object. `to_key_value` writes the model as it is, with a `JSONWriter`, or a mapping of its settings, deciding how the bytes are spelled (indentation, key order, ASCII escaping, trailing newline); the default is that of `json.dumps`. A v3 group reads its diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index 2e5ba2930e..dc39bdfc6f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -50,6 +50,7 @@ Derived, StoreKey, attributes_of, + canonical_members, check_literal, created, dimension_lengths, @@ -59,6 +60,7 @@ missing_keys, other_members, reading_of, + simplest, unclaimed_declarations_of, ) @@ -772,6 +774,39 @@ def __replace__(self, /, **members: Unpack[ZarrV2ArrayMetadataUpdate]) -> ZarrV2 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV2ArrayMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same document: `to_json` of each result is + the same JSON, apart from the order of the members of an object, + which `to_key_value` sorts with `JSONWriter(sort_keys=True)`. So + the bytes that writer writes are the same for equal models, and + are a key for what the document means. + + The dtype, the compressor and each filter are written as + `canonical_of` writes them, and the fill value as + `canonical_fill_value` writes it for the dtype. A + `dimension_separator` of `"."` is left out, and so are empty + `attributes`, because a document without them means the same. + Everything else is kept as written: `shape`, `chunks`, `order`, + the attributes, and whatever nothing in scope claims. + """ + compressor, filters = self.compressor, self.filters + spelled: dict[str, JSONValue] = { + "dtype": simplest(self.dtype), + "fill_value": spelled_canonically(self.dtype, self._members.fill_value), + "compressor": None if compressor is None else simplest(compressor), + "filters": None if filters is None else tuple(simplest(entry) for entry in filters), + } + document = canonical_members( + self._document, + spelled, + empty=("attributes",), + defaults={"dimension_separator": ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR}, + ) + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 2ea2af7ccb..1d15ef74fb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -52,6 +52,7 @@ Derived, StoreKey, attributes_of, + canonical_members, check_literal, created, dimension_lengths, @@ -60,6 +61,8 @@ member_changes, missing_keys, reading_of, + simplest, + simplest_members, unclaimed_declarations_of, ) from zarr_metadata.v3._common import ZarrV3DeclarationJSON, ZarrV3ExtensionField @@ -734,6 +737,29 @@ def __replace__(self, /, **members: Unpack[ZarrV3ArrayMetadataUpdate]) -> ZarrV3 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV3ArrayMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same document: `to_json` of each result is + the same JSON, apart from the order of the members of an object, + which `to_key_value` sorts with `JSONWriter(sort_keys=True)`. So + the bytes that writer writes are the same for equal models, and + are a key for what the document means. + + Each declaration is written as `canonical_of` writes it, each + codec as `codec_for_chunk` returns it for the chunk it receives, + and the fill value as `canonical_fill_value` writes it for the + data type. `attributes` and `storage_transformers` are left out + when empty, because a document without them means the same. + Everything else is kept as written: `shape`, `dimension_names`, + the attributes, and whatever nothing in scope claims, which only + its own definition could simplify. A document that is already in + this spelling is returned as it is, in a new model. + """ + document = canonical_array_document_v3(self._reading, self._members) + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. @@ -951,6 +977,30 @@ def array_key_of( ) +def canonical_array_document_v3( + reading: ZarrV3ArrayMetadataReading, members: ArrayMembersV3 +) -> dict[str, JSONValue]: + """A v3 array document in its simplest equivalent spelling, from its `reading` and refined `members`, as `ZarrV3ArrayMetadata.canonicalize` reads it. + + It is written from the parts that `array_key_of` compares, so two + documents with one key are written the same way, apart from the order + of the members of an object. + """ + data_type = held(reading.data_type) + spelled: dict[str, JSONValue] = { + "data_type": simplest(data_type), + "chunk_grid": simplest(reading.chunk_grid), + "chunk_key_encoding": simplest(reading.chunk_key_encoding), + "fill_value": spelled_canonically(data_type, members.fill_value), + "codecs": tuple(simplest(codec) for codec in codecs_for_chunks(reading)), + "storage_transformers": tuple(simplest(entry) for entry in reading.storage_transformers), + **simplest_members(reading.extra_fields), + } + return canonical_members( + members.document, spelled, empty=("attributes", "storage_transformers") + ) + + def codecs_for_chunks( reading: ZarrV3ArrayMetadataReading, ) -> tuple[AcceptedDeclaration[CodecDefinition[Any]] | UnclaimedDeclaration, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index cbd34fb85f..3f185a0f9d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -48,6 +48,7 @@ from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, + canonical_members, check_literal, load_store_json, located_conflicts, @@ -378,6 +379,11 @@ def __replace__(self, /, **members: Unpack[ZarrV2GroupMetadataUpdate]) -> ZarrV2 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV2GroupMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope: without `attributes` when they are empty, because a group with no `.zattrs` means the same. The result equals this model, and every model equal to this one gives the same document, apart from the order of the members of an object, which `to_key_value` sorts with `JSONWriter(sort_keys=True)`.""" + document = canonical_members(self._document, {}, empty=("attributes",)) + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: """Return this document read in `context`: the same group, holding that scope.""" context = scoped(context, CORE_V2_CONTEXT) @@ -586,6 +592,31 @@ def _located_declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]] def __reduce__(self) -> tuple[type[ZarrV2ConsolidatedMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) + def canonicalize(self) -> ZarrV2ConsolidatedMetadata: + """Return this document with each node in its simplest equivalent spelling, as that node's `canonicalize` writes it, read in this model's own scope. + + Each node's `.zarray` or `.zgroup` entry is written without the + attributes, and its `.zattrs` entry holds them, or is left out + when they are empty. Every other entry is kept as written. The + result equals this model. Every model equal to this one, in this + scope, gives the same document, apart from the order of the + members of an object, which a `JSONWriter(sort_keys=True)` sorts. + """ + entries = dict(object_at(self._document, "metadata")) + for path, node in self._metadata.items(): + names = self._files[path] + written = cast("dict[str, JSONValue]", node.canonicalize().to_json()) + attributes = written.pop("attributes", None) + for name, key in names.items(): + if name != ZARR_V2_ATTRIBUTES_STORE_KEY: + entries[key] = written + elif attributes is None: + del entries[key] + else: + entries[key] = attributes + document: dict[str, JSONValue] = {**self._document, "metadata": entries} + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: """Return this document with every node read in `context`, whatever that changes. Raises `MetadataValidationError` when a node has a problem in `context`.""" context = scoped(context, CORE_V2_CONTEXT) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 9c5d9dc006..33e4cd82ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -67,12 +67,14 @@ from zarr_metadata.model._shared import ( StoreKey, attributes_of, + canonical_members, check_literal, load_store_json, located_conflicts, member_changes, missing_keys, reading_of, + simplest_members, unclaimed_declarations_of, unexpected_keys, ) @@ -392,6 +394,27 @@ def __replace__(self, /, **members: Unpack[ZarrV3GroupMetadataUpdate]) -> ZarrV3 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV3GroupMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same document: `to_json` of each result is + the same JSON, apart from the order of the members of an object, + which `to_key_value` sorts with `JSONWriter(sort_keys=True)`. + + `attributes` is left out when empty, because a document without + it means the same, and each extra field a definition in scope + claims is written as `canonical_of` writes it. Everything else is + kept as written. That includes the documents that + `consolidated_metadata` lists, which a group does not read: + `consolidated().canonicalize()` returns them in their simplest + spelling, and `replace(consolidated_metadata=...)` writes them + back into the group. + """ + spelled = simplest_members(self._reading.extra_fields) + document = canonical_members(self._document, spelled, empty=("attributes",)) + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: """Return this document read in `context`, whatever that changes. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically.""" context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) @@ -595,6 +618,20 @@ def _key_of(self) -> tuple[object, ...]: def __reduce__(self) -> tuple[type[ZarrV3ConsolidatedMetadata], tuple[object, ZarrV3Context]]: return type(self), (self._document, self._context) + def canonicalize(self) -> ZarrV3ConsolidatedMetadata: + """Return this member with each document it lists in its simplest equivalent spelling, as that node's `canonicalize` writes it, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same member, apart from the order of the + members of an object, which a `JSONWriter(sort_keys=True)` sorts. + """ + metadata: dict[str, JSONValue] = { + path: cast("JSONValue", node.canonicalize().to_json()) + for path, node in self._metadata.items() + } + document: dict[str, JSONValue] = {**self._document, "metadata": metadata} + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: """Return this member with every document read in `context`, whatever that changes. Raises `MetadataValidationError` when a document has a problem in `context`. The existing readings are reused when `context` reads every claim of every document identically.""" context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py index a946c4a608..2bb83aa52e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py @@ -24,6 +24,8 @@ from zarr_metadata.definition._definition import ( RefusedDeclaration, UnclaimedDeclaration, + canonical_of, + held, is_declaration, ) from zarr_metadata.definition._scope import Conflict, claim_key @@ -204,6 +206,50 @@ def created( raise +def simplest(declaration: ResolvedDeclaration[Any] | UNSET) -> JSONValue: + """`declaration`, from a read that found no problems, in its simplest equivalent spelling, as `canonical_of` writes it. Raises `TypeError` for a declaration with none, which such a read cannot produce.""" + spelled = canonical_of(held(declaration), ()) + if spelled is None: + msg = f"expected a declaration a read found nothing wrong with, got {declaration!r}" + raise TypeError(msg) + return spelled + + +def simplest_members(extra_fields: Mapping[str, ResolvedDeclaration[Any]]) -> dict[str, JSONValue]: + """The extra fields of a document that a read found no problems with, by key, each in its simplest equivalent spelling.""" + spelled: dict[str, JSONValue] = {} + for extra in extra_fields.values(): + # An extra field is written as the member it is: its key, and its value. + spelled.update(cast("Mapping[str, JSONValue]", simplest(extra))) + return spelled + + +def canonical_members( + document: Mapping[str, JSONValue], + spelled: Mapping[str, JSONValue], + *, + empty: Iterable[str] = (), + defaults: Mapping[str, JSONValue] | None = None, +) -> dict[str, JSONValue]: + """`document`, the document a model holds, with each member in `spelled` written as `spelled` writes it, and without the members that say what their absence says. + + A member in `empty` is left out when it is empty, and a member in + `defaults` when it has the value given there. The members keep the + order `document` writes them in. A member of `spelled` that + `document` does not write is not added. + """ + canonical = {key: spelled.get(key, value) for key, value in document.items()} + for key in empty: + value = canonical.get(key) + if (is_array(value) or is_object(value)) and len(value) == 0: + del canonical[key] + if defaults is not None: + for key, default in defaults.items(): + if key in canonical and canonical[key] == default: + del canonical[key] + return canonical + + def unclaimed_declarations_of( declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], waived: Iterable[str] ) -> dict[Loc, UnclaimedDeclaration]: diff --git a/packages/zarr-metadata/tests/model/test_canonicalize.py b/packages/zarr-metadata/tests/model/test_canonicalize.py new file mode 100644 index 0000000000..ff7b92cabd --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_canonicalize.py @@ -0,0 +1,401 @@ +"""`canonicalize`: each model with its document in its simplest equivalent spelling. + +Two models are equal when their documents mean the same thing, and +`to_json` writes each document as it was given. `canonicalize` returns the +one document that every model equal to a given one writes, apart from the +order of the members of an object, which a `JSONWriter` sorts. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, cast + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from zarr_metadata._json import json_text +from zarr_metadata.model import ( + JSONWriter, + ZarrV2ArrayMetadata, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, +) +from zarr_metadata.testing.strategies import ( + Spellings, + array_documents, + consolidated_group_documents, + group_documents, +) +from zarr_metadata.v3.definition import ZARR_PYTHON_CONTEXT + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata import JSONValue + +Model = ( + ZarrV2ArrayMetadata + | ZarrV2ConsolidatedMetadata + | ZarrV2GroupMetadata + | ZarrV3ArrayMetadata + | ZarrV3ConsolidatedMetadata + | ZarrV3GroupMetadata +) + +_LITTLE: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "little"}} +_BIG: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "big"}} +_NOSHUFFLE = {"cname": "lz4", "clevel": 5, "shuffle": "noshuffle", "blocksize": 0} +_WAIVED = {"name": "acme.note", "must_understand": False, "configuration": {"b": 1, "a": 2}} + +_V3_ARRAY: dict[str, Any] = { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default"}, + "fill_value": 0, + "codecs": [{"name": "bytes"}], +} +"""A v3 array document in its simplest spelling.""" + +_V3_ARRAY_SPELLED_OUT: dict[str, Any] = { + "attributes": {}, + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": {"name": "uint8", "configuration": {}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, + "fill_value": 0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [2], + "index_location": "end", + "codecs": [_BIG, {"name": "blosc", "configuration": {**_NOSHUFFLE, "typesize": 4}}], + "index_codecs": [_LITTLE, {"name": "crc32c", "configuration": {}}], + }, + } + ], + "storage_transformers": [], + "dimension_names": [None], + "acme": _WAIVED, +} +"""A v3 array document that says in more words what `_V3_ARRAY_SIMPLEST` says.""" + +_V3_ARRAY_SIMPLEST: dict[str, Any] = { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default"}, + "fill_value": 0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [2], + "codecs": [{"name": "bytes"}, {"name": "blosc", "configuration": _NOSHUFFLE}], + "index_codecs": [_LITTLE, {"name": "crc32c"}], + }, + } + ], + "dimension_names": [None], + "acme": _WAIVED, +} + +_V3_FLOATS: dict[str, Any] = {**_V3_ARRAY, "data_type": "float32", "codecs": [_LITTLE]} + +_V3_UNCLAIMED: dict[str, Any] = { + **_V3_ARRAY, + "data_type": {"name": "acme.t", "configuration": {"bits": 8, "signed": False}}, + "fill_value": "0x00", + "codecs": [{"name": "acme.codec", "configuration": {"level": 1}}, _LITTLE], + "attributes": {"b": [1.0, {}], "a": None}, +} +"""A v3 array document that holds names nothing in scope claims. Their configurations are kept as written, since only their own definitions could simplify them, and so is the `endian` of the `bytes` codec, which receives values nothing is known of.""" + +_V3_GROUP: dict[str, Any] = {"zarr_format": 3, "node_type": "group"} + + +def _v3_listing(*documents: dict[str, Any]) -> dict[str, Any]: + return { + "kind": "inline", + "must_understand": False, + "metadata": {f"n{index}": document for index, document in enumerate(documents)}, + } + + +_V2_ARRAY: dict[str, Any] = { + "zarr_format": 2, + "shape": [4], + "chunks": [2], + "dtype": "|i1", + "compressor": {"id": "zstd", "level": 0, "checksum": False}, + "fill_value": 0, + "order": "C", + "filters": [{"id": "delta", "dtype": "|i1", "astype": "|i1"}], +} +"""A v2 array document in its canonical spelling, in which a codec writes each numcodecs default.""" + +_V2_ARRAY_SPELLED_OTHERWISE: dict[str, Any] = { + **_V2_ARRAY, + "dtype": " dict[str, Any]: + return {key: value for key, value in document.items() if key != "attributes"} + + +@pytest.mark.parametrize( + ("model", "document", "canonical"), + [ + (ZarrV3ArrayMetadata, _V3_ARRAY, _V3_ARRAY), + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _V3_ARRAY_SIMPLEST), + ( + ZarrV3ArrayMetadata, + {**_V3_FLOATS, "fill_value": "0x7fc00000"}, + {**_V3_FLOATS, "fill_value": "NaN"}, + ), + (ZarrV3ArrayMetadata, _V3_UNCLAIMED, _V3_UNCLAIMED), + (ZarrV3GroupMetadata, _V3_GROUP, _V3_GROUP), + ( + ZarrV3GroupMetadata, + {"attributes": {}, **_V3_GROUP, "acme": _WAIVED}, + {**_V3_GROUP, "acme": _WAIVED}, + ), + ( + ZarrV3GroupMetadata, + {**_V3_GROUP, "attributes": {"a": 1}}, + {**_V3_GROUP, "attributes": {"a": 1}}, + ), + # A group does not read the documents its consolidated metadata lists. + ( + ZarrV3GroupMetadata, + {**_V3_GROUP, "consolidated_metadata": _v3_listing(_V3_ARRAY_SPELLED_OUT)}, + {**_V3_GROUP, "consolidated_metadata": _v3_listing(_V3_ARRAY_SPELLED_OUT)}, + ), + ( + ZarrV3ConsolidatedMetadata, + _v3_listing(_V3_ARRAY_SPELLED_OUT, {**_V3_GROUP, "attributes": {}}), + _v3_listing(_V3_ARRAY_SIMPLEST, _V3_GROUP), + ), + (ZarrV2ArrayMetadata, _V2_ARRAY, _V2_ARRAY), + (ZarrV2ArrayMetadata, _V2_ARRAY_SPELLED_OTHERWISE, _V2_ARRAY), + (ZarrV2ArrayMetadata, _V2_FLOATS, _V2_FLOATS), + (ZarrV2GroupMetadata, _V2_GROUP, _V2_GROUP), + (ZarrV2GroupMetadata, {**_V2_GROUP, "attributes": {}}, _V2_GROUP), + ( + ZarrV2GroupMetadata, + {**_V2_GROUP, "attributes": {"a": 1}}, + {**_V2_GROUP, "attributes": {"a": 1}}, + ), + ( + ZarrV2ConsolidatedMetadata, + { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": _V2_GROUP, + ".zattrs": {}, + "a/.zarray": _v2_array_entry(_V2_ARRAY_SPELLED_OTHERWISE), + "a/.zattrs": {"x": 1}, + "b/.zarray": _v2_array_entry(_V2_ARRAY_SPELLED_OTHERWISE), + "b/.zattrs": {}, + "c/.zattrs": {}, + "notes": [1], + }, + }, + { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": _V2_GROUP, + "a/.zarray": _V2_ARRAY, + "a/.zattrs": {"x": 1}, + "b/.zarray": _V2_ARRAY, + # No node is read from these entries, so they are kept as written. + "c/.zattrs": {}, + "notes": [1], + }, + }, + ), + ], + ids=[ + "v3-array-simplest", + "v3-array-spelled-out", + "v3-array-fill-value", + "v3-array-unclaimed", + "v3-group-simplest", + "v3-group-empty-attributes", + "v3-group-attributes", + "v3-group-consolidated", + "v3-consolidated", + "v2-array-canonical", + "v2-array-spelled-otherwise", + "v2-array-nothing-to-simplify", + "v2-group-simplest", + "v2-group-empty-attributes", + "v2-group-attributes", + "v2-consolidated", + ], +) +def test_a_model_canonicalizes_to_the_simplest_spelling_of_its_document( + model: type[Model], document: dict[str, Any], canonical: dict[str, Any] +) -> None: + """`canonicalize` returns an equal model of the same class, in the same scope, whose document is in its simplest equivalent spelling, with the members that remain in the order they were written. A document already in that spelling is returned as it is.""" + read = model.from_json(document) + result = read.canonicalize() + assert json.dumps(result.to_json()) == json.dumps(canonical) + assert type(result) is type(read) + assert result.context is read.context + assert result == read + assert hash(result) == hash(read) + assert json.dumps(result.canonicalize().to_json()) == json.dumps(canonical) + + +def _reordered(document: dict[str, Any]) -> dict[str, Any]: + """`document` with its members in the reverse order, and those of each object in its `attributes` and in each member the spec does not define.""" + + def reverse(value: object) -> object: + if isinstance(value, dict): + held = cast("dict[str, object]", value) + return {key: reverse(held[key]) for key in reversed(held)} + if isinstance(value, list): + return [reverse(item) for item in cast("list[object]", value)] + return value + + return { + key: reverse(document[key]) if key in ("attributes", "acme") else document[key] + for key in reversed(document) + } + + +@pytest.mark.parametrize( + ("model", "left", "right"), + [ + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _V3_ARRAY_SIMPLEST), + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _reordered(_V3_ARRAY_SPELLED_OUT)), + (ZarrV3ArrayMetadata, _V3_UNCLAIMED, _reordered(_V3_UNCLAIMED)), + (ZarrV3ArrayMetadata, {**_V3_ARRAY, "codecs": [_LITTLE]}, {**_V3_ARRAY, "codecs": [_BIG]}), + (ZarrV3GroupMetadata, {**_V3_GROUP, "attributes": {}}, _V3_GROUP), + ( + ZarrV3ConsolidatedMetadata, + _v3_listing(_V3_ARRAY_SPELLED_OUT, _V3_GROUP), + _v3_listing(_reordered(_V3_ARRAY_SIMPLEST), {**_V3_GROUP, "attributes": {}}), + ), + (ZarrV2ArrayMetadata, _V2_ARRAY_SPELLED_OTHERWISE, _reordered(_V2_ARRAY)), + (ZarrV2GroupMetadata, {**_V2_GROUP, "attributes": {}}, _V2_GROUP), + ], +) +def test_equal_models_canonicalize_to_one_document( + model: type[Model], left: dict[str, Any], right: dict[str, Any] +) -> None: + """Models that are equal give the same canonical document, apart from the order of the members of an object: a `JSONWriter` that sorts them writes the same bytes for each.""" + models = [model.from_json(document) for document in (left, right)] + assert models[0] == models[1] + canonical = [each.canonicalize() for each in models] + assert json_text(_json_of(canonical[0])) == json_text(_json_of(canonical[1])) + assert _stored(canonical[0]) == _stored(canonical[1]) + + +@pytest.mark.parametrize( + ("model", "left", "right"), + [ + ( + ZarrV3ArrayMetadata, + {**_V3_ARRAY, "data_type": "uint16", "codecs": [_LITTLE]}, + {**_V3_ARRAY, "data_type": "uint16", "codecs": [_BIG]}, + ), + (ZarrV3ArrayMetadata, _V3_ARRAY, {**_V3_ARRAY, "dimension_names": [None]}), + ( + ZarrV3ArrayMetadata, + {**_V3_FLOATS, "fill_value": 0.0}, + {**_V3_FLOATS, "fill_value": -0.0}, + ), + (ZarrV3GroupMetadata, _V3_GROUP, {**_V3_GROUP, "attributes": {"a": 1}}), + (ZarrV2ArrayMetadata, _V2_ARRAY, {**_V2_ARRAY, "dimension_separator": "/"}), + (ZarrV2ArrayMetadata, {**_V2_ARRAY, "filters": None}, {**_V2_ARRAY, "filters": []}), + ], +) +def test_models_that_differ_canonicalize_to_different_documents( + model: type[Model], left: dict[str, Any], right: dict[str, Any] +) -> None: + """Only spellings that mean the same thing are reduced to one: models that are not equal keep documents that differ.""" + models = [model.from_json(document) for document in (left, right)] + assert models[0] != models[1] + written = [json_text(_json_of(each.canonicalize())) for each in models] + assert written[0] != written[1] + + +def _json_of(model: Model) -> JSONValue: + document: Any = model.to_json() + return document + + +def _stored(model: Model) -> object: + """The bytes a store holds for `model`, with the members of every object sorted; None for a v3 listing, which a group's document carries.""" + if isinstance(model, ZarrV3ConsolidatedMetadata): + return None + return model.to_key_value(JSONWriter(sort_keys=True)) + + +def _assert_canonicalizes( + model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata | ZarrV3ConsolidatedMetadata, +) -> None: + """`model` equals its canonical model, whose document has nothing left to simplify and reads as the same model from the JSON text a store holds.""" + canonical = model.canonicalize() + assert canonical == model + assert hash(canonical) == hash(model) + written = json.dumps(canonical.to_json()) + assert json.dumps(canonical.canonicalize().to_json()) == written + assert type(model).from_json(json.loads(written), context=ZARR_PYTHON_CONTEXT) == model + + +@pytest.mark.parametrize("spellings", ["all", "canonical"]) +@given(data=st.data()) +def test_every_array_document_canonicalizes(spellings: Spellings, data: st.DataObject) -> None: + document = data.draw(array_documents(spellings=spellings)) + _assert_canonicalizes(ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT)) + + +@given(document=group_documents()) +def test_every_group_document_canonicalizes(document: Mapping[str, Any]) -> None: + _assert_canonicalizes(ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT)) + + +@given(document=consolidated_group_documents()) +def test_every_consolidated_group_document_canonicalizes( + document: Mapping[str, Any], +) -> None: + group = ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT) + _assert_canonicalizes(group) + listing = group.consolidated() + assert listing is not None + _assert_canonicalizes(listing) + # The listing in its simplest spelling can be written back into the group. + _assert_canonicalizes(group.replace(consolidated_metadata=listing.canonicalize())) From 070840fc5f72fbcf01edefc5dc638461566771af Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 16:29:23 +0100 Subject: [PATCH 03/17] docs(zarr-metadata): name the changelog fragment after the upstream PR Assisted-by: ClaudeCode:claude-fable-5-2 --- .../zarr-metadata/changes/{450.feature.md => 4640.feature.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{450.feature.md => 4640.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/450.feature.md b/packages/zarr-metadata/changes/4640.feature.md similarity index 100% rename from packages/zarr-metadata/changes/450.feature.md rename to packages/zarr-metadata/changes/4640.feature.md From fa27e3807bed26454f3ea1825397beabe81ab623 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 16:29:55 +0100 Subject: [PATCH 04/17] docs(zarr-metadata): name the changelog fragment after the upstream PR Assisted-by: ClaudeCode:claude-fable-5-2 --- .../zarr-metadata/changes/{452.feature.md => 4642.feature.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{452.feature.md => 4642.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/452.feature.md b/packages/zarr-metadata/changes/4642.feature.md similarity index 100% rename from packages/zarr-metadata/changes/452.feature.md rename to packages/zarr-metadata/changes/4642.feature.md From 6af625bbe6cb728bf74ea83ffe88349800f7d196 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 17 Nov 2026 22:27:20 +0100 Subject: [PATCH 05/17] fix(zarr-metadata): simplify a codec for its chunk in the spelling its canonical gives chunk_canonical received the configuration as written, and canonical ran afterwards. A codec whose canonical output its chunk_canonical reduces further then had two keys for one meaning: array models that differed only in such spellings were unequal, and the simplest spelling was not a fixed point. _for_chunk now hands chunk_canonical the configuration as canonical spells it, and checks that chunk_canonical has nothing left to simplify in its own result. A definition that leaves chunk_canonical at its default is skipped without calling either. Assisted-by: ClaudeCode:claude-fable-5-2 --- .../src/zarr_metadata/v3/_definition.py | 9 ++- .../src/zarr_metadata/v3/_pipeline.py | 79 +++++++++++++++---- .../zarr-metadata/tests/v3/test_pipelines.py | 59 +++++++++++++- 3 files changed, 126 insertions(+), 21 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index dc4a3135b1..34525f27c9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -434,8 +434,11 @@ class ArrayCodecDefinition(CodecDefinition[C]): It simplifies only the codec's own members, and it must keep each nested declaration as it is, because each codec of a pipeline the codec contains is simplified for the chunk it receives in turn. What - does not depend on the chunk is left to `canonical`, which simplifies - the result as it simplifies any configuration. A chunk rule checks + does not depend on the chunk is left to `canonical`: `chunk_canonical` + receives the configuration as `canonical` spells it, and `canonical` + simplifies the result as it simplifies any configuration. The result + must be the simplest spelling, so `chunk_canonical` must change + nothing when it is asked about its own result. A chunk rule checks only the parts of a chunk that are known, and `chunk_canonical` simplifies only by them: a chunk with nothing known about it, `Chunk()`, leaves every configuration as it is. `codec_for_chunk` @@ -445,7 +448,7 @@ class ArrayCodecDefinition(CodecDefinition[C]): chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules """The spec violations for this codec with a given input chunk, located in the configuration. A problem at `()` is about the codec as a whole, such as an input it cannot encode, and is reported at the codec.""" chunk_canonical: Callable[[C, Nested, Chunk], C] = unchanged_for_chunk - """A configuration the chunk rules allow for a given input chunk, in the simplest spelling that means the same for that chunk.""" + """A configuration the chunk rules allow for a given input chunk, as `canonical` spells it, in the simplest spelling that means the same for that chunk.""" pipelines: Callable[[C, Nested, Chunk], Mapping[str, Chunk]] = no_pipelines """The chunk each contained pipeline receives, keyed by the configuration member that holds the pipeline.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index 9842d827da..6c3351a0b6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -58,6 +58,7 @@ Chunk, CodecDefinition, CodecKind, + unchanged_for_chunk, ) if TYPE_CHECKING: @@ -349,6 +350,9 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: codec of each pipeline it contains, for the chunk that codec receives, and then its own configuration, as its definition's `chunk_canonical` simplifies it for the chunk in `stage`. + `chunk_canonical` receives the configuration as the definition's + `canonical` spells it, so two spellings that `canonical` reduces to + one are simplified for the chunk in the same way. A codec the scope did not read is returned as it is. A codec's own configuration is kept as it is when the codec receives bytes, and @@ -360,9 +364,10 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: `chunk_canonical` is the extension author's code, so its result is checked: the definition must allow it, the chunk rules must find no - problem with it, and each nested declaration must be where it was. A - result that fails a check raises `ValueError`, which indicates a bug - in the definition, not in the codec. + problem with it, each nested declaration must be where it was, and + `chunk_canonical` must have nothing left to simplify in it. A result + that fails a check raises `ValueError`, which indicates a bug in the + definition, not in the codec. """ codec = stage.codec if not isinstance(codec, AcceptedDeclaration): @@ -382,25 +387,26 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: def _for_chunk( codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None ) -> Mapping[str, JSONValue] | None: - """The configuration of `codec` as its definition's `chunk_canonical` simplifies it for `chunk`, checked; None when nothing is simplified.""" + """The configuration of `codec` as its definition's `chunk_canonical` simplifies it for `chunk`, checked; None when nothing is simplified. + + `chunk_canonical` receives the configuration as the definition's + `canonical` spells it. So every spelling that `canonical` reduces to + one is simplified for the chunk in the same way, and `chunk_canonical` + does not have to know each of them. None is returned when + `chunk_canonical` changes nothing in that spelling, and the codec + then stays as written, for `canonical` to simplify as it simplifies + any configuration. + """ definition = codec.definition if chunk is None or not isinstance(definition, ArrayCodecDefinition): return None + if definition.chunk_canonical is unchanged_for_chunk: + return None configuration, nested = codec.configuration, codec.nested if len(_chunk_problems(definition, configuration, nested, chunk, ())) != 0: return None - given: object = asked( - definition, - "chunk_canonical", - lambda: cast("object", definition.chunk_canonical(read_only(configuration), nested, chunk)), - ) - if given is configuration: - return None - if not is_object(given): - msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}" - raise TypeError(msg) - simplified = cast("dict[str, JSONValue]", copied(cast("JSONValue", dict(given)))) - if json_text(simplified) == json_text(cast("JSONValue", configuration)): + simplified = _asked_for_chunk(definition, configuration, nested, chunk) + if simplified is None: return None _, refused = definition.read_configuration(simplified) found = ( @@ -420,6 +426,47 @@ def _for_chunk( f"{list(found)!r}" ) raise ValueError(msg) + # The simplest spelling has nothing left to simplify. Otherwise a + # document written from it would read as another codec. + further = _asked_for_chunk(definition, simplified, nested, chunk) + if further is not None: + msg = ( + f"{definition.name!r}: its chunk_canonical gave {simplified!r}, which is not the " + f"simplest spelling: asked again, it gave {further!r}" + ) + raise ValueError(msg) + return simplified + + +def _asked_for_chunk( + definition: ArrayCodecDefinition[Any], + configuration: Mapping[str, JSONValue], + nested: Nested, + chunk: Chunk, +) -> dict[str, JSONValue] | None: + """What `definition`'s `chunk_canonical` gives for `configuration` as its `canonical` spells it; None when it changes nothing in that spelling.""" + spelled = read_only( + asked( + definition, + "canonical", + lambda: dict( + cast("Mapping[str, JSONValue]", definition.canonical(read_only(configuration))) + ), + ) + ) + given: object = asked( + definition, + "chunk_canonical", + lambda: cast("object", definition.chunk_canonical(spelled, nested, chunk)), + ) + if given is spelled: + return None + if not is_object(given): + msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}" + raise TypeError(msg) + simplified = cast("dict[str, JSONValue]", copied(cast("JSONValue", dict(given)))) + if json_text(simplified) == json_text(cast("JSONValue", dict(spelled))): + return None return simplified diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index fe477dc8f0..920940fd92 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -9,7 +9,7 @@ from __future__ import annotations import dataclasses -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, NotRequired, cast import pytest from typing_extensions import TypedDict @@ -73,7 +73,39 @@ def _tiled(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) -> C ) """An array -> array codec that does not declare its output chunk.""" -CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(codecs=[ACME_TILE, ACME_SHUFFLE])) + +class AcmePackConfiguration(TypedDict, closed=True): + mode: NotRequired[str] + + +def _lower_mode(configuration: AcmePackConfiguration) -> AcmePackConfiguration: + """`mode` in lower case, since `acme.pack` reads it whatever its case.""" + if "mode" not in configuration: + return configuration + return {"mode": configuration["mode"].lower()} + + +def _mode_for_chunk( + configuration: AcmePackConfiguration, nested: Nested, chunk: Chunk +) -> AcmePackConfiguration: + """Without a `mode` of `"auto"`, which is what `acme.pack` does without a `mode` for a data type the scope read.""" + if isinstance(chunk.data_type, AcceptedDeclaration) and configuration.get("mode") == "auto": + return {} + return configuration + + +ACME_PACK = ArrayBytesCodecDefinition( + name="acme.pack", + configuration=AcmePackConfiguration, + size="static", + canonical=_lower_mode, + chunk_canonical=_mode_for_chunk, +) +"""An array -> bytes codec with both a `canonical` and a `chunk_canonical`: the second simplifies what the first writes.""" + +CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union( + ZarrV3Context(codecs=[ACME_TILE, ACME_SHUFFLE, ACME_PACK]) +) UINT8 = resolve("uint8", DataTypeDefinition, CONTEXT)[0] @@ -573,6 +605,10 @@ def _shard(*codecs: JSONValue, index_codecs: tuple[JSONValue, ...] = (LITTLE, CR } +def _pack(mode: str) -> JSONValue: + return {"name": "acme.pack", "configuration": {"mode": mode}} + + def _simplest(codecs: list[JSONValue], chunk: Chunk) -> list[JSONValue]: """Each codec of a pipeline that receives `chunk`, in its simplest spelling for the chunk it receives.""" stages, problems = _read(codecs, chunk) @@ -607,6 +643,11 @@ def _simplest(codecs: list[JSONValue], chunk: Chunk) -> list[JSONValue]: ("acme.t", [LITTLE], [LITTLE]), ("uint8", ["acme.shuffle", LITTLE], [{"name": "acme.shuffle"}, LITTLE]), ("uint8", ["acme.unclaimed", LITTLE], [{"name": "acme.unclaimed"}, LITTLE]), + # A codec is simplified for its chunk in the spelling its `canonical` gives it. + ("uint8", [_pack("auto")], [{"name": "acme.pack"}]), + ("uint8", [_pack("AUTO")], [{"name": "acme.pack"}]), + ("uint8", [_pack("FAST")], [_pack("fast")]), + ("acme.t", [_pack("AUTO")], [_pack("auto")]), ], ) def test_a_codec_is_written_in_the_simplest_spelling_for_the_chunk_it_receives( @@ -655,6 +696,20 @@ def test_error_a_chunk_canonical_the_chunk_rules_do_not_allow() -> None: codec_for_chunk(stage) +def test_error_a_chunk_canonical_with_something_left_to_simplify() -> None: + # Asked about `little` it gives `big`, and asked about `big` it gives `little`. + def chunk_canonical( + configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk + ) -> BytesCodecConfiguration: + return {"endian": "big" if configuration.get("endian") == "little" else "little"} + + stage = _stage(_bytes_with(chunk_canonical), CHUNK) + with pytest.raises( + ValueError, match="'acme.bytes': its chunk_canonical gave .* not the simplest" + ): + codec_for_chunk(stage) + + def test_error_a_chunk_canonical_that_gives_something_else() -> None: stage = _stage(_bytes_with(lambda configuration, nested, chunk: "little"), CHUNK) with pytest.raises(TypeError, match="'acme.bytes': its chunk_canonical gives a configuration"): From c5f7a4b9b0b004902076e1b9435bde5db474e13c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 17 Nov 2026 22:28:02 +0100 Subject: [PATCH 06/17] docs(zarr-metadata): add the changelog fragment Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/462.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/zarr-metadata/changes/462.bugfix.md diff --git a/packages/zarr-metadata/changes/462.bugfix.md b/packages/zarr-metadata/changes/462.bugfix.md new file mode 100644 index 0000000000..42ce128632 --- /dev/null +++ b/packages/zarr-metadata/changes/462.bugfix.md @@ -0,0 +1 @@ +A v3 codec's `chunk_canonical` receives the configuration as the codec's `canonical` spells it, so two spellings that `canonical` reduces to one are simplified for a chunk in the same way. Before, it received the configuration as written, and array models that differed only in such spellings were unequal. The result of `chunk_canonical` is also checked to have nothing left to simplify: a `chunk_canonical` that changes its own result raises `ValueError`. From 64c89a7648625f1fc92e91861257e6ad3885930f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 17 Nov 2026 22:33:09 +0100 Subject: [PATCH 07/17] test(zarr-metadata): check what canonicalize simplifies, in scopes that are not the default A review found that the Hypothesis tests passed with a canonicalize that returned the model's own document, that no test noticed a model losing its scope, and that four behaviours had no case: a claimed extra field, a v2 fill value, a chunk grid and a storage transformer that are respelled. The parametrized cases are now read in scopes that are not the default ones. The generated documents are compared with the same document reordered at every depth and, for arrays, with each declaration in its own simplest spelling. The reordering helper reverses every object, which the fix in #4641 allows. Assisted-by: ClaudeCode:claude-fable-5-2 --- .../tests/model/test_canonicalize.py | 206 ++++++++++++++++-- 1 file changed, 189 insertions(+), 17 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_canonicalize.py b/packages/zarr-metadata/tests/model/test_canonicalize.py index ff7b92cabd..25dda00891 100644 --- a/packages/zarr-metadata/tests/model/test_canonicalize.py +++ b/packages/zarr-metadata/tests/model/test_canonicalize.py @@ -9,13 +9,15 @@ from __future__ import annotations import json -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, NotRequired, TypeVar, cast import pytest from hypothesis import given from hypothesis import strategies as st +from typing_extensions import TypedDict, TypeIs from zarr_metadata._json import json_text +from zarr_metadata.definition._definition import canonicalize from zarr_metadata.model import ( JSONWriter, ZarrV2ArrayMetadata, @@ -31,7 +33,19 @@ consolidated_group_documents, group_documents, ) -from zarr_metadata.v3.definition import ZARR_PYTHON_CONTEXT +from zarr_metadata.v2.definition import CORE_V2_CONTEXT, ZarrV2Context +from zarr_metadata.v3.definition import ( + CORE_AND_EXTENSIONS_CONTEXT, + ZARR_PYTHON_CONTEXT, + ArrayExtraFieldDefinition, + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + DataTypeDefinition, + GroupExtraFieldDefinition, + StorageTransformerDefinition, + ZarrV3Context, +) if TYPE_CHECKING: from collections.abc import Mapping @@ -47,6 +61,50 @@ | ZarrV3GroupMetadata ) + +class AcmeLevelConfiguration(TypedDict, closed=True): + level: NotRequired[int] + + +class AcmeNoteConfiguration(TypedDict, closed=True): + must_understand: NotRequired[bool] + level: NotRequired[int] + + +C = TypeVar("C", AcmeLevelConfiguration, AcmeNoteConfiguration) + + +def _without_level_0(configuration: C) -> C: + """Without a `level` of 0, which is what both definitions here do without a `level`.""" + if configuration.get("level") != 0: + return configuration + return cast("C", {key: value for key, value in configuration.items() if key != "level"}) + + +_V3_CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union( + ZarrV3Context( + array_extra_fields=[ + ArrayExtraFieldDefinition( + name="note", configuration=AcmeNoteConfiguration, canonical=_without_level_0 + ) + ], + group_extra_fields=[ + GroupExtraFieldDefinition( + name="note", configuration=AcmeNoteConfiguration, canonical=_without_level_0 + ) + ], + storage_transformers=[ + StorageTransformerDefinition( + name="acme.st", configuration=AcmeLevelConfiguration, canonical=_without_level_0 + ) + ], + ) +) +"""The scope the v3 documents here are read in. It is not the default scope: it also claims an extra field `note` and a storage transformer `acme.st`, each with a `canonical`.""" + +_V2_CONTEXT = CORE_V2_CONTEXT.union(ZarrV2Context()) +"""The scope the v2 documents here are read in: the default scope as another object, which tells whether a model kept the scope it was read in.""" + _LITTLE: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "little"}} _BIG: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "big"}} _NOSHUFFLE = {"cname": "lz4", "clevel": 5, "shuffle": "noshuffle", "blocksize": 0} @@ -123,6 +181,39 @@ } """A v3 array document that holds names nothing in scope claims. Their configurations are kept as written, since only their own definitions could simplify them, and so is the `endian` of the `bytes` codec, which receives values nothing is known of.""" +_V3_UNCLAIMED_ENVELOPES: dict[str, Any] = { + **_V3_ARRAY, + "data_type": {"name": "acme.t", "configuration": {}}, + "fill_value": "0x00", + "codecs": [ + {"name": "acme.codec", "configuration": {}}, + "acme.other", + {"name": "acme.third", "must_understand": True, "configuration": {"b": 1, "a": {}}}, + _LITTLE, + ], +} +"""A v3 array document that holds names nothing in scope claims, each in an envelope that has a shorter form.""" + +_V3_UNCLAIMED_SHORTEST: dict[str, Any] = { + **_V3_ARRAY, + "data_type": "acme.t", + "fill_value": "0x00", + "codecs": [ + {"name": "acme.codec"}, + {"name": "acme.other"}, + {"name": "acme.third", "configuration": {"b": 1, "a": {}}}, + _LITTLE, + ], +} + + +def _rectilinear(*chunk_shapes: list[int] | list[list[int]]) -> dict[str, Any]: + return { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": list(chunk_shapes)}, + } + + _V3_GROUP: dict[str, Any] = {"zarr_format": 3, "node_type": "group"} @@ -184,6 +275,21 @@ def _v2_array_entry(document: dict[str, Any]) -> dict[str, Any]: {**_V3_FLOATS, "fill_value": "NaN"}, ), (ZarrV3ArrayMetadata, _V3_UNCLAIMED, _V3_UNCLAIMED), + (ZarrV3ArrayMetadata, _V3_UNCLAIMED_ENVELOPES, _V3_UNCLAIMED_SHORTEST), + ( + ZarrV3ArrayMetadata, + {**_V3_ARRAY, "shape": [6], "chunk_grid": _rectilinear([2, 2, 2])}, + {**_V3_ARRAY, "shape": [6], "chunk_grid": _rectilinear([[2, 3]])}, + ), + ( + ZarrV3ArrayMetadata, + { + **_V3_ARRAY, + "storage_transformers": [{"name": "acme.st", "configuration": {"level": 0}}], + }, + {**_V3_ARRAY, "storage_transformers": [{"name": "acme.st"}]}, + ), + (ZarrV3ArrayMetadata, {**_V3_ARRAY, "note": {"level": 0}}, {**_V3_ARRAY, "note": {}}), (ZarrV3GroupMetadata, _V3_GROUP, _V3_GROUP), ( ZarrV3GroupMetadata, @@ -195,6 +301,7 @@ def _v2_array_entry(document: dict[str, Any]) -> dict[str, Any]: {**_V3_GROUP, "attributes": {"a": 1}}, {**_V3_GROUP, "attributes": {"a": 1}}, ), + (ZarrV3GroupMetadata, {**_V3_GROUP, "note": {"level": 0}}, {**_V3_GROUP, "note": {}}), # A group does not read the documents its consolidated metadata lists. ( ZarrV3GroupMetadata, @@ -209,6 +316,12 @@ def _v2_array_entry(document: dict[str, Any]) -> dict[str, Any]: (ZarrV2ArrayMetadata, _V2_ARRAY, _V2_ARRAY), (ZarrV2ArrayMetadata, _V2_ARRAY_SPELLED_OTHERWISE, _V2_ARRAY), (ZarrV2ArrayMetadata, _V2_FLOATS, _V2_FLOATS), + (ZarrV2ArrayMetadata, {**_V2_FLOATS, "fill_value": 0}, {**_V2_FLOATS, "fill_value": 0.0}), + ( + ZarrV2ArrayMetadata, + {**_V2_FLOATS, "fill_value": 1e39}, + {**_V2_FLOATS, "fill_value": "Infinity"}, + ), (ZarrV2GroupMetadata, _V2_GROUP, _V2_GROUP), (ZarrV2GroupMetadata, {**_V2_GROUP, "attributes": {}}, _V2_GROUP), ( @@ -250,14 +363,21 @@ def _v2_array_entry(document: dict[str, Any]) -> dict[str, Any]: "v3-array-spelled-out", "v3-array-fill-value", "v3-array-unclaimed", + "v3-array-unclaimed-envelopes", + "v3-array-chunk-grid", + "v3-array-storage-transformer", + "v3-array-claimed-extra-field", "v3-group-simplest", "v3-group-empty-attributes", "v3-group-attributes", + "v3-group-claimed-extra-field", "v3-group-consolidated", "v3-consolidated", "v2-array-canonical", "v2-array-spelled-otherwise", "v2-array-nothing-to-simplify", + "v2-array-fill-value", + "v2-array-fill-value-out-of-range", "v2-group-simplest", "v2-group-empty-attributes", "v2-group-attributes", @@ -268,18 +388,32 @@ def test_a_model_canonicalizes_to_the_simplest_spelling_of_its_document( model: type[Model], document: dict[str, Any], canonical: dict[str, Any] ) -> None: """`canonicalize` returns an equal model of the same class, in the same scope, whose document is in its simplest equivalent spelling, with the members that remain in the order they were written. A document already in that spelling is returned as it is.""" - read = model.from_json(document) + read = _read(model, document) result = read.canonicalize() assert json.dumps(result.to_json()) == json.dumps(canonical) assert type(result) is type(read) - assert result.context is read.context + # Neither scope here is the default one, so a model that lost its scope would show. + assert result.context is (_V2_CONTEXT if _is_v2(model) else _V3_CONTEXT) assert result == read assert hash(result) == hash(read) assert json.dumps(result.canonicalize().to_json()) == json.dumps(canonical) -def _reordered(document: dict[str, Any]) -> dict[str, Any]: - """`document` with its members in the reverse order, and those of each object in its `attributes` and in each member the spec does not define.""" +def _is_v2( + model: type[Model], +) -> TypeIs[type[ZarrV2ArrayMetadata | ZarrV2ConsolidatedMetadata | ZarrV2GroupMetadata]]: + return issubclass(model, (ZarrV2ArrayMetadata, ZarrV2ConsolidatedMetadata, ZarrV2GroupMetadata)) + + +def _read(model: type[Model], document: Mapping[str, Any]) -> Model: + """The model of `document`, read in `_V2_CONTEXT` or `_V3_CONTEXT`.""" + if _is_v2(model): + return model.from_json(document, context=_V2_CONTEXT) + return model.from_json(document, context=_V3_CONTEXT) + + +def _reordered(document: Mapping[str, Any]) -> dict[str, Any]: + """`document` with the members of every object it holds, at any depth, in the reverse order. The items of an array keep their order.""" def reverse(value: object) -> object: if isinstance(value, dict): @@ -289,18 +423,20 @@ def reverse(value: object) -> object: return [reverse(item) for item in cast("list[object]", value)] return value - return { - key: reverse(document[key]) if key in ("attributes", "acme") else document[key] - for key in reversed(document) - } + return cast("dict[str, Any]", reverse(dict(document))) @pytest.mark.parametrize( ("model", "left", "right"), [ (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _V3_ARRAY_SIMPLEST), - (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _reordered(_V3_ARRAY_SPELLED_OUT)), - (ZarrV3ArrayMetadata, _V3_UNCLAIMED, _reordered(_V3_UNCLAIMED)), + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _reordered(_V3_ARRAY_SIMPLEST)), + (ZarrV3ArrayMetadata, _V3_UNCLAIMED_ENVELOPES, _reordered(_V3_UNCLAIMED_SHORTEST)), + ( + ZarrV3ArrayMetadata, + {**_V3_UNCLAIMED, "storage_transformers": []}, + _reordered(_V3_UNCLAIMED), + ), (ZarrV3ArrayMetadata, {**_V3_ARRAY, "codecs": [_LITTLE]}, {**_V3_ARRAY, "codecs": [_BIG]}), (ZarrV3GroupMetadata, {**_V3_GROUP, "attributes": {}}, _V3_GROUP), ( @@ -316,7 +452,7 @@ def test_equal_models_canonicalize_to_one_document( model: type[Model], left: dict[str, Any], right: dict[str, Any] ) -> None: """Models that are equal give the same canonical document, apart from the order of the members of an object: a `JSONWriter` that sorts them writes the same bytes for each.""" - models = [model.from_json(document) for document in (left, right)] + models = [_read(model, document) for document in (left, right)] assert models[0] == models[1] canonical = [each.canonicalize() for each in models] assert json_text(_json_of(canonical[0])) == json_text(_json_of(canonical[1])) @@ -346,7 +482,7 @@ def test_models_that_differ_canonicalize_to_different_documents( model: type[Model], left: dict[str, Any], right: dict[str, Any] ) -> None: """Only spellings that mean the same thing are reduced to one: models that are not equal keep documents that differ.""" - models = [model.from_json(document) for document in (left, right)] + models = [_read(model, document) for document in (left, right)] assert models[0] != models[1] written = [json_text(_json_of(each.canonicalize())) for each in models] assert written[0] != written[1] @@ -358,22 +494,33 @@ def _json_of(model: Model) -> JSONValue: def _stored(model: Model) -> object: - """The bytes a store holds for `model`, with the members of every object sorted; None for a v3 listing, which a group's document carries.""" + """The bytes a store holds for `model`, with the members of every object sorted. A v3 listing is stored in the document of a group.""" if isinstance(model, ZarrV3ConsolidatedMetadata): - return None + group = ZarrV3GroupMetadata.from_json(_V3_GROUP, context=model.context) + model = group.replace(consolidated_metadata=model) return model.to_key_value(JSONWriter(sort_keys=True)) def _assert_canonicalizes( model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata | ZarrV3ConsolidatedMetadata, ) -> None: - """`model` equals its canonical model, whose document has nothing left to simplify and reads as the same model from the JSON text a store holds.""" + """`model` equals its canonical model, which is in the same scope, has nothing left to simplify and reads as the same model from the JSON text a store holds. The document of `model` with the members of every object in the reverse order gives the same canonical document.""" canonical = model.canonicalize() assert canonical == model + assert canonical.context is model.context assert hash(canonical) == hash(model) written = json.dumps(canonical.to_json()) assert json.dumps(canonical.canonicalize().to_json()) == written assert type(model).from_json(json.loads(written), context=ZARR_PYTHON_CONTEXT) == model + if not isinstance(canonical, ZarrV3ConsolidatedMetadata): + document = cast("Mapping[str, Any]", _json_of(canonical)) + for member in ("attributes", "storage_transformers"): + assert member not in document or len(document[member]) != 0 + reordered = type(model).from_json( + _reordered(json.loads(json.dumps(model.to_json()))), context=ZARR_PYTHON_CONTEXT + ) + assert reordered == model + assert json_text(_json_of(reordered.canonicalize())) == json_text(_json_of(canonical)) @pytest.mark.parametrize("spellings", ["all", "canonical"]) @@ -383,6 +530,31 @@ def test_every_array_document_canonicalizes(spellings: Spellings, data: st.DataO _assert_canonicalizes(ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT)) +@given(document=array_documents(spellings="all")) +def test_every_array_document_canonicalizes_as_its_declarations_do( + document: Mapping[str, Any], +) -> None: + """A document, and the same document with each declaration in its own simplest spelling, are one array, and give one canonical document.""" + simpler = dict(document) + for member, kind in ( + ("data_type", DataTypeDefinition), + ("chunk_grid", ChunkGridDefinition), + ("chunk_key_encoding", ChunkKeyEncodingDefinition), + ): + simpler[member], problems = canonicalize(document[member], kind, ZARR_PYTHON_CONTEXT) + assert problems == () + simpler["codecs"] = [ + canonicalize(codec, CodecDefinition, ZARR_PYTHON_CONTEXT)[0] for codec in document["codecs"] + ] + models = [ + ZarrV3ArrayMetadata.from_json(each, context=ZARR_PYTHON_CONTEXT) + for each in (document, simpler) + ] + assert models[0] == models[1] + canonical = [json_text(_json_of(each.canonicalize())) for each in models] + assert canonical[0] == canonical[1] + + @given(document=group_documents()) def test_every_group_document_canonicalizes(document: Mapping[str, Any]) -> None: _assert_canonicalizes(ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT)) From 34c3378aa2365763c0c25ab84b47316fc7e58dbc Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 17 Nov 2026 22:33:10 +0100 Subject: [PATCH 08/17] docs(zarr-metadata): say how canonicalize writes a declaration nothing claims Its configuration is kept, but its envelope is written in the shortest form: an empty configuration and a must_understand of true are left out. Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/docs/models.md | 4 +++- .../zarr-metadata/src/zarr_metadata/model/_array_v3.py | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index 16c4e4378a..4e7d3ac3c8 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -315,7 +315,9 @@ The canonical document is written from the same parts that `==` compares: `dimension_separator` of `"."`. - Everything else is kept as written. This includes the attributes, and the configuration of a declaration that nothing in the context claims, - because only its own definition could simplify it. + because only its own definition could simplify it. Such a declaration + is still written in the shortest form every reader accepts, without an + empty `configuration` or a `must_understand` of `true`. A v2 codec is written with each numcodecs default it leaves out, as its definition's `canonical` writes it. So a canonical v2 document can be diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 1d15ef74fb..c0fa12a402 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -753,9 +753,12 @@ def canonicalize(self) -> ZarrV3ArrayMetadata: data type. `attributes` and `storage_transformers` are left out when empty, because a document without them means the same. Everything else is kept as written: `shape`, `dimension_names`, - the attributes, and whatever nothing in scope claims, which only - its own definition could simplify. A document that is already in - this spelling is returned as it is, in a new model. + the attributes, and the configuration of a declaration nothing in + scope claims, which only its own definition could simplify. Such + a declaration is still written in the shortest form every reader + accepts, without an empty `configuration` or a `must_understand` + of `true`. A document that is already in this spelling is + returned as it is, in a new model. """ document = canonical_array_document_v3(self._reading, self._members) return type(self)(document, context=self._context) From cb67d4b33ec75a9a40f9ce80bede7123b7c54bc2 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 17 Nov 2026 23:27:27 +0100 Subject: [PATCH 09/17] fix(zarr-metadata): check what a codec's chunk_canonical gives as the document it becomes A review of #4640 found gaps in how _for_chunk checks the result of chunk_canonical: - a result that added a nested declaration passed, and the respelled codec then held a declaration its nested mapping did not; - a result that was not JSON raised an error that did not name the definition; - an array written as a list was reported as a changed nested declaration; - a problem inside a nested declaration, which is the document's, was held against the result. The result is now refined as any JSON is, its declaration locations are compared with the codec's, and problems under a nested location are left out. respelled refuses a location that holds no nested declaration with the TypeError its docstring promises. Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/462.bugfix.md | 2 +- packages/zarr-metadata/docs/models.md | 4 +- .../zarr_metadata/definition/_definition.py | 12 +- .../src/zarr_metadata/v3/_pipeline.py | 38 +++++-- .../tests/v3/test_definitions.py | 106 +++++++++++++++++- .../zarr-metadata/tests/v3/test_pipelines.py | 76 ++++++++++++- 6 files changed, 221 insertions(+), 17 deletions(-) diff --git a/packages/zarr-metadata/changes/462.bugfix.md b/packages/zarr-metadata/changes/462.bugfix.md index 42ce128632..e75570df6e 100644 --- a/packages/zarr-metadata/changes/462.bugfix.md +++ b/packages/zarr-metadata/changes/462.bugfix.md @@ -1 +1 @@ -A v3 codec's `chunk_canonical` receives the configuration as the codec's `canonical` spells it, so two spellings that `canonical` reduces to one are simplified for a chunk in the same way. Before, it received the configuration as written, and array models that differed only in such spellings were unequal. The result of `chunk_canonical` is also checked to have nothing left to simplify: a `chunk_canonical` that changes its own result raises `ValueError`. +A v3 codec's `chunk_canonical` receives the configuration as the codec's `canonical` spells it, so two spellings that `canonical` reduces to one are simplified for a chunk in the same way. Before, it received the configuration as written, and array models that differed only in such spellings were unequal. Its result is checked more closely: one that it would simplify again, or that holds a declaration the codec did not hold, raises `ValueError`, and one that is not JSON raises `TypeError` naming the definition. A result may write an array as a list, and a problem inside a nested declaration is no longer held against it. A codec that leaves `chunk_canonical` at its default is no longer copied each time an array model is compared or hashed. diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index ad2a700f6c..6f3c7f29b8 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -226,7 +226,9 @@ These are each declaration, and the fill value read against its data type. So in a `uint8` array, a `bytes` codec is the same codec with either `endian`, or with none. The codec is compared for the chunk it receives: after a `cast_value` to `float32`, the two `endian` values - are two codecs. + are two codecs. `codec_for_chunk` returns a codec in the spelling that + is compared, which `canonical_of` cannot give, since it reads the + codec alone. A v2 document's [dtype](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#data-type-encoding), diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index f59fea98a4..a8e08e673e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1291,11 +1291,15 @@ def respelled( where it is. `nested` holds only the nested declarations to replace, each one accepted or unclaimed, and each is written into the configuration as its `to_json` writes it. A location in `nested` that - does not lead into `configuration` raises `TypeError`. The result is + is not the location of a nested declaration of `declaration`, or that + does not lead into `configuration`, raises `TypeError`. The result is written as `to_json` writes it, since no document wrote it. """ spelled: JSONValue = dict(configuration) for loc, inner in nested.items(): + if loc not in declaration.nested or item_at(spelled, loc) is UNSET: + msg = f"{loc!r} is not the location of a nested declaration in {configuration!r}" + raise TypeError(msg) spelled = _replaced(spelled, loc, written_json(held(inner))) kind = declaration.read_as name = declaration.name @@ -1309,6 +1313,11 @@ def respelled( ) +def declaration_locations(definition: Definition[Any], configuration: JSONValue) -> tuple[Loc, ...]: + """The location of each declaration that `configuration` holds, where `definition`'s TypedDict types a member with a declaration alias, in order. `configuration` is JSON with arrays as tuples.""" + return tuple(found.loc for found in _typed(definition.configuration, configuration, ())[2]) + + def item_at(value: JSONValue, path: Loc) -> JSONValue | UNSET: """The item of `value` at `path`; `UNSET` when `path` does not lead into `value`.""" for step in path: @@ -1957,6 +1966,7 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "configuration_of", "declaration_key", "declaration_kind", + "declaration_locations", "declarations_of", "document_json", "filed_under", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index 6c3351a0b6..db1bf1ff9b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -35,10 +35,10 @@ from zarr_metadata._json import ( ValidationProblem, - copied, is_object, is_tuple, json_text, + refine_user_data, with_input, ) from zarr_metadata._sentinel import UNSET @@ -46,6 +46,7 @@ AcceptedDeclaration, ResolvedDeclaration, asked, + declaration_locations, in_document, item_at, read_only, @@ -364,10 +365,12 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: `chunk_canonical` is the extension author's code, so its result is checked: the definition must allow it, the chunk rules must find no - problem with it, each nested declaration must be where it was, and - `chunk_canonical` must have nothing left to simplify in it. A result - that fails a check raises `ValueError`, which indicates a bug in the - definition, not in the codec. + problem with it, it must hold the nested declarations the codec + holds, each as it was, and `chunk_canonical` must have nothing left + to simplify in it. A result that fails a check raises `ValueError`, + which indicates a bug in the definition, not in the codec. A problem + inside a nested declaration is the document's, and is not held + against the result. """ codec = stage.codec if not isinstance(codec, AcceptedDeclaration): @@ -409,8 +412,11 @@ def _for_chunk( if simplified is None: return None _, refused = definition.read_configuration(simplified) + held_at = frozenset(declaration_locations(definition, simplified)) found = ( - *refused, + # A problem inside a nested declaration is the document's: the + # declaration is checked to be as it was written. + *(problem for problem in refused if not _inside(problem.loc, nested)), *_chunk_problems(definition, simplified, nested, chunk, ()), *( ValidationProblem( @@ -419,6 +425,12 @@ def _for_chunk( for loc in nested if _text_at(simplified, loc) != _text_at(cast("JSONValue", configuration), loc) ), + *( + ValidationProblem( + loc, "expected no declaration that the codec did not hold", "invalid_value" + ) + for loc in sorted(held_at.difference(nested), key=repr) + ), ) if len(found) != 0: msg = ( @@ -438,13 +450,18 @@ def _for_chunk( return simplified +def _inside(loc: Loc, nested: Nested) -> bool: + """Whether `loc` is at or under the location of a declaration in `nested`.""" + return any(loc[: len(held_at)] == held_at for held_at in nested) + + def _asked_for_chunk( definition: ArrayCodecDefinition[Any], configuration: Mapping[str, JSONValue], nested: Nested, chunk: Chunk, ) -> dict[str, JSONValue] | None: - """What `definition`'s `chunk_canonical` gives for `configuration` as its `canonical` spells it; None when it changes nothing in that spelling.""" + """What `definition`'s `chunk_canonical` gives for `configuration` as its `canonical` spells it, as JSON with arrays as tuples; None when it changes nothing in that spelling.""" spelled = read_only( asked( definition, @@ -461,10 +478,13 @@ def _asked_for_chunk( ) if given is spelled: return None - if not is_object(given): + # The result is the extension author's value, so it is read as any JSON + # is: an array may be a list, and what is not JSON is refused here, + # naming the definition. `read_configuration` checks the rest. + simplified, not_json = refine_user_data(dict(given)) if is_object(given) else (None, ()) + if not isinstance(simplified, dict) or len(not_json) != 0: msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}" raise TypeError(msg) - simplified = cast("dict[str, JSONValue]", copied(cast("JSONValue", dict(given)))) if json_text(simplified) == json_text(cast("JSONValue", dict(spelled))): return None return simplified diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index 359275a18f..bcc2f3c9be 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -21,7 +21,7 @@ JSONValue, ValidationProblem, ) -from zarr_metadata._json import refine_json +from zarr_metadata._json import json_text, refine_json from zarr_metadata.definition import ( AcceptedDeclaration, Context, @@ -34,7 +34,7 @@ configuration_of, resolve, ) -from zarr_metadata.definition._definition import resolve_refined +from zarr_metadata.definition._definition import item_at, resolve_refined, respelled from zarr_metadata.model import ( UNSET, ZarrV3ArrayMetadata, @@ -1824,3 +1824,105 @@ def test_error_resolve_refuses_a_kind_of_the_other_format_whatever_the_declarati """`resolve` with a kind of one format and a scope of the other raises one `TypeError`, before it reads the declaration. So the message is the same for a declaration with a name that a scope could look up and for a value with no name.""" with pytest.raises(TypeError, match=f"^{re.escape(message)}$"): resolve(declaration, kind, context) + + +def _accepted(declaration: JSONValue, kind: type[Definition[Any]]) -> AcceptedDeclaration[Any]: + read, problems = resolve(declaration, kind, CORE_AND_EXTENSIONS_CONTEXT) + assert isinstance(read, AcceptedDeclaration) + assert problems == () + return read + + +def _sharding(**members: JSONValue) -> JSONValue: + return {"name": "sharding_indexed", "configuration": {"chunk_shape": [2], **members}} + + +@pytest.mark.parametrize( + ("declaration", "kind", "configuration", "nested", "written"), + [ + # A member of its own left out, and a nested declaration replaced. + ( + _sharding(codecs=[LE], index_codecs=[LE, CRC32C], index_location="end"), + CodecDefinition, + {"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, + {("codecs", 0): {"name": "bytes"}}, + _sharding(codecs=[{"name": "bytes"}], index_codecs=[LE, CRC32C]), + ), + # Nothing nested to replace. + ( + {"name": "gzip", "configuration": {"level": 1}}, + CodecDefinition, + {"level": 5}, + {}, + {"name": "gzip", "configuration": {"level": 5}}, + ), + # A name that carries its configuration is written alone. + ("r16", DataTypeDefinition, {"bits": 16}, {}, "r16"), + ], + ids=["shard", "gzip", "name-carries-configuration"], +) +def test_respelled_is_the_declaration_that_the_other_spelling_reads_as( + declaration: JSONValue, + kind: type[Definition[Any]], + configuration: JSONValue, + nested: Mapping[tuple[str | int, ...], JSONValue], + written: JSONValue, +) -> None: + """`respelled` returns what a read of the other spelling returns: the same JSON, name, configuration, nested declarations and key. The declaration it was given is left as it was.""" + read = _accepted(declaration, kind) + before = json_text(read.to_json()) + refined = cast("Mapping[str, JSONValue]", refine_json(configuration)[0]) + replacements = {loc: _accepted(inner, CodecDefinition) for loc, inner in nested.items()} + result = respelled(read, refined, replacements) + again = _accepted(written, kind) + assert json_text(result.json) == json_text(again.json) + assert json_text(result.to_json()) == json_text(again.to_json()) + assert result.name == again.name + assert json_text(cast("JSONValue", result.configuration)) == json_text( + cast("JSONValue", again.configuration) + ) + assert dict(result.nested) == dict(again.nested) + assert result == again + assert hash(result) == hash(again) + assert json_text(read.to_json()) == before + + +@pytest.mark.parametrize( + ("configuration", "loc"), + [ + ({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ("codecs", 5)), + ({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ("codecs", -1)), + ({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ("nope", 0)), + ({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ()), + # A location of a nested declaration that the new configuration does not have. + ({"chunk_shape": [2], "codecs": [], "index_codecs": [LE, CRC32C]}, ("codecs", 0)), + ], + ids=["past-the-end", "negative", "no-such-member", "the-configuration", "left-out"], +) +def test_error_respelled_at_a_location_that_holds_no_nested_declaration( + configuration: JSONValue, loc: tuple[str | int, ...] +) -> None: + read = _accepted(_sharding(codecs=[LE], index_codecs=[LE, CRC32C]), CodecDefinition) + refined = cast("Mapping[str, JSONValue]", refine_json(configuration)[0]) + with pytest.raises(TypeError, match="is not the location of a nested declaration"): + respelled(read, refined, {loc: _accepted({"name": "bytes"}, CodecDefinition)}) + + +@pytest.mark.parametrize( + ("path", "item"), + [ + ((), {"a": ({"b": 1}, 2)}), + (("a",), ({"b": 1}, 2)), + (("a", 0, "b"), 1), + (("a", 1), 2), + (("a", 2), UNSET), + (("a", -1), UNSET), + (("b",), UNSET), + (("a", "b"), UNSET), + ((0,), UNSET), + (("a", 1, "b"), UNSET), + ], +) +def test_item_at_follows_a_location_into_json(path: tuple[str | int, ...], item: object) -> None: + """`item_at` returns the item at a location, and `UNSET` when the location does not lead into the value: a key an object lacks, an index outside an array, or a step into a scalar.""" + assert item_at({"a": ({"b": 1}, 2)}, path) == item diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index 920940fd92..0eda1926b6 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -663,11 +663,64 @@ def test_a_codec_is_written_in_the_simplest_spelling_for_the_chunk_it_receives( def test_a_codec_with_a_problem_for_its_chunk_is_not_simplified() -> None: - """A codec the chunk rules find a problem with is returned as it is, as is a codec the scope refused.""" + """A codec the chunk rules find a problem with keeps its own configuration, and its `chunk_canonical` is not asked. A codec the scope refused is returned as it is.""" # `bytes` does not encode values that vary in size, and `gzip` requires a level. stages, problems = _read([LITTLE, {"name": "gzip", "configuration": {}}], _of("string")) assert [problem.loc for problem in problems] == [(1, "configuration", "level"), (0,)] assert all(codec_for_chunk(stage) is stage.codec for stage in stages) + # A `chunk_canonical` that would drop the `endian` whatever the chunk is not asked either. + eager = _bytes_with(lambda configuration, nested, chunk: {}) + stages, problems = read_pipeline([eager], _of("string")) + assert [problem.loc for problem in problems] == [(0,)] + assert codec_for_chunk(stages[0]) is eager + + +class AcmeLeveledConfiguration(TypedDict, closed=True): + codecs: tuple[ZarrV3CodecDeclaration, ...] + level: NotRequired[int] + + +def _leveled(chunk_canonical: object, *codecs: JSONValue) -> Stage: + """The stage, for `CHUNK`, of a codec `acme.leveled` that holds `codecs` and a `level` of 3, and whose `chunk_canonical` is `chunk_canonical`. It declares no pipelines, so the codecs it holds are left as they are.""" + leveled = ArrayBytesCodecDefinition( + name="acme.leveled", + configuration=AcmeLeveledConfiguration, + size="dynamic", + chunk_canonical=chunk_canonical, # pyright: ignore[reportArgumentType] + ) + declaration = {"name": "acme.leveled", "configuration": {"codecs": list(codecs), "level": 3}} + codec = resolve(declaration, CodecDefinition, CONTEXT.union(ZarrV3Context(codecs=[leveled])))[0] + return read_pipeline([codec], CHUNK)[0][0] + + +REFUSED_GZIP: JSONValue = {"name": "gzip", "configuration": {"level": 99}, "must_understand": False} +"""A codec with problems of its own: no such level, and a codec may not waive `must_understand`. A configuration that holds it keeps it as written.""" + + +@pytest.mark.parametrize( + ("codecs", "chunk_canonical"), + [ + # An array of the result may be a list. + ( + [LITTLE, GZIP], + lambda configuration, nested, chunk: {"codecs": list(configuration["codecs"])}, + ), + # A problem inside a nested declaration is the document's, not the result's. + ( + [LITTLE, REFUSED_GZIP], + lambda configuration, nested, chunk: {"codecs": configuration["codecs"]}, + ), + ], + ids=["an-array-as-a-list", "a-nested-declaration-with-a-problem"], +) +def test_a_chunk_canonical_that_simplifies_its_own_members_is_taken_at_its_word( + codecs: list[JSONValue], chunk_canonical: object +) -> None: + """A result that drops the codec's own `level` and keeps each nested declaration as it was written passes every check, however it spells its arrays and whatever problems the nested declarations have.""" + found = codec_for_chunk(_leveled(chunk_canonical, *codecs)) + assert isinstance(found, AcceptedDeclaration) + assert json_text(cast("JSONValue", found.configuration)) == json_text({"codecs": codecs}) + assert list(found.nested) == [("codecs", 0), ("codecs", 1)] def _bytes_with(chunk_canonical: object) -> ResolvedDeclaration[CodecDefinition[Any]]: @@ -710,12 +763,29 @@ def chunk_canonical( codec_for_chunk(stage) -def test_error_a_chunk_canonical_that_gives_something_else() -> None: - stage = _stage(_bytes_with(lambda configuration, nested, chunk: "little"), CHUNK) +@pytest.mark.parametrize( + "given", ["little", {"endian": {"little"}}, {1: "little"}], ids=["a-string", "a-set", "a-key"] +) +def test_error_a_chunk_canonical_that_gives_something_else(given: object) -> None: + """A result that is not an object of JSON is refused by name, before anything reads it.""" + stage = _stage(_bytes_with(lambda configuration, nested, chunk: given), CHUNK) with pytest.raises(TypeError, match="'acme.bytes': its chunk_canonical gives a configuration"): codec_for_chunk(stage) +def test_error_a_chunk_canonical_that_adds_a_nested_declaration() -> None: + # The codec it adds is one no scope read, so the result would hold a + # declaration that its `nested` does not. + def chunk_canonical( + configuration: AcmeLeveledConfiguration, nested: Nested, chunk: Chunk + ) -> AcmeLeveledConfiguration: + added = cast("ZarrV3CodecDeclaration", {"name": "crc32c"}) + return {"codecs": (*configuration["codecs"], added)} + + with pytest.raises(ValueError, match="expected no declaration that the codec did not hold"): + codec_for_chunk(_leveled(chunk_canonical, LITTLE, GZIP)) + + def test_error_a_chunk_canonical_that_raises_says_whose_it_is() -> None: def chunk_canonical( configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk From f1e40f61d0d47513a0f1bd439e0eacf6039bff39 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 17 Nov 2026 23:32:06 +0100 Subject: [PATCH 10/17] fix(zarr-metadata): compare codecs for the chunk the richer reading knows when judging a refinement Since #4640 an array model's refines compared each side's codecs as simplified for that side's own chunks. A richer scope can know more of a chunk than a poorer one: it claims the data type, a codec that comes before, or the codec itself. It then dropped a bytes codec's endian that the poorer reading kept, and refined_in(...) no longer refined the model it came from. codec_refines compares the codec of a stage with another reading of it, simplifying both for the chunks the stage knows. A gain is judged by what the unclaimed codec reads as with the stage's claims, simplified the same way. ZarrV3ArrayMetadata.refines uses it for each codec. Assisted-by: ClaudeCode:claude-fable-5-2 --- .../src/zarr_metadata/definition/_scope.py | 10 +- .../src/zarr_metadata/model/_array_v3.py | 12 ++- .../src/zarr_metadata/v3/_pipeline.py | 85 ++++++++++++++-- .../src/zarr_metadata/v3/definition.py | 7 +- .../tests/model/test_invariants.py | 96 +++++++++++++++++++ .../zarr-metadata/tests/model/test_pair.py | 76 ++++++++++++++- .../tests/v3/test_definitions.py | 29 ++++++ .../zarr-metadata/tests/v3/test_pipelines.py | 54 +++++++++++ 8 files changed, 355 insertions(+), 14 deletions(-) create mode 100644 packages/zarr-metadata/tests/model/test_invariants.py diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py index 9a28bc2a32..34c2254759 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py @@ -171,8 +171,15 @@ def refines(declaration: ResolvedDeclaration[Any], other: ResolvedDeclaration[An def _gained(declaration: AcceptedDeclaration[Any], other: UnclaimedDeclaration) -> bool: """Whether reading `other` in the scope formed by `declaration`'s own claims produces `declaration`. This is the test for a gain.""" + return read_as_claimed(other, declaration) == declaration + + +def read_as_claimed( + other: UnclaimedDeclaration, declaration: AcceptedDeclaration[D] +) -> ResolvedDeclaration[D]: + """`other`, a declaration nothing claimed, read in the scope formed by `declaration`'s own claims: its definition and the definitions of the declarations nested in it. A gain is judged by what this returns.""" again, _ = resolve(other.json, declaration.read_as, _Claimed.of(declaration)) - return again == declaration + return cast("ResolvedDeclaration[D]", again) @dataclass(frozen=True, slots=True) @@ -240,5 +247,6 @@ def disagreements_of( "claims_of", "disagreements_of", "kind_name", + "read_as_claimed", "refines", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 2ea2af7ccb..6c8b858521 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -78,7 +78,7 @@ chunk_grid_lengths, default_array_bytes_codec, ) -from zarr_metadata.v3._pipeline import Stage, codec_for_chunk, read_pipeline +from zarr_metadata.v3._pipeline import Stage, codec_for_chunk, codec_refines, read_pipeline from zarr_metadata.v3.array import ( ZARR_V3_ARRAY_METADATA_STORE_KEY, ZarrV3ArrayMetadataJSON, @@ -767,7 +767,7 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadat return self.with_context(context) def refines(self, other: ZarrV3ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each declaration must refine its counterpart, in the order `refines` defines for declarations (including the declarations nested in each declaration). Every other member must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" + """Whether this model holds everything `other` holds. Each declaration must refine its counterpart, in the order `refines` defines for declarations (including the declarations nested in each declaration). A codec is compared as `codec_refines` compares it, for the chunk it receives in this model, so a document read in a richer scope refines the same document read in a poorer one, whatever the richer scope learns of the chunks. Every other member must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" if type(other) is not type(self): return False if len(self.codecs) != len(other.codecs) or len(self.storage_transformers) != len( @@ -780,12 +780,18 @@ def refines(self, other: ZarrV3ArrayMetadata) -> bool: (self.data_type, other.data_type), (self.chunk_grid, other.chunk_grid), (self.chunk_key_encoding, other.chunk_key_encoding), - *zip(codecs_for_chunks(self._reading), codecs_for_chunks(other._reading), strict=True), *zip(self.storage_transformers, other.storage_transformers, strict=True), *((self.extra_fields[key], other.extra_fields[key]) for key in self.extra_fields), ) if not all(refines_declaration(mine, theirs) for mine, theirs in pairs): return False + # Each codec is compared for the chunk it receives in this model, + # which knows at least as much of it as `other` does. + if not all( + codec_refines(mine, theirs.codec) + for mine, theirs in zip(self._reading.pipeline, other._reading.pipeline, strict=True) + ): + return False if len(fill_value_problems(self.data_type, other._members.fill_value)) != 0: return False return self._plain_key(self.data_type) == other._plain_key(self.data_type) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index db1bf1ff9b..3edf1df097 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -44,15 +44,19 @@ from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._definition import ( AcceptedDeclaration, + RefusedDeclaration, ResolvedDeclaration, + UnclaimedDeclaration, asked, declaration_locations, in_document, item_at, + own_key, read_only, respelled, ruled, ) +from zarr_metadata.definition._scope import claim_key, read_as_claimed, refines from zarr_metadata.v3._definition import ( ArrayArrayCodecDefinition, ArrayCodecDefinition, @@ -372,21 +376,88 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: inside a nested declaration is the document's, and is not held against the result. """ - codec = stage.codec + return _for_stage(stage.codec, stage) + + +def _for_stage( + codec: ResolvedDeclaration[CodecDefinition[Any]], stage: Stage +) -> ResolvedDeclaration[CodecDefinition[Any]]: + """`codec`, a reading of the codec at the place of `stage` in its pipeline, in the simplest spelling for the chunks that `stage` and the stages it contains receive. + + `codec` is the codec of `stage` itself, or the same codec as another + scope read it. Each codec it holds is simplified for the chunk that + the stage at its location receives. + """ if not isinstance(codec, AcceptedDeclaration): return codec inner: dict[Loc, ResolvedDeclaration[Any]] = {} - for member, stages in stage.inner.items(): - for index, each in enumerate(stages): - simpler = codec_for_chunk(each) - if simpler is not each.codec: - inner[(member, index)] = simpler + for loc, each in _inner_stages(stage).items(): + held_there = codec.nested.get(loc) + if held_there is None: + continue + simpler = _for_stage(held_there, each) + if simpler is not held_there: + inner[loc] = simpler configuration = _for_chunk(codec, stage.incoming) if configuration is None and len(inner) == 0: return codec return respelled(codec, codec.configuration if configuration is None else configuration, inner) +def _inner_stages(stage: Stage) -> dict[Loc, Stage]: + """Each stage of the pipelines the codec of `stage` contains, by the location of its codec in the configuration.""" + return { + (member, index): each + for member, stages in stage.inner.items() + for index, each in enumerate(stages) + } + + +def codec_refines(stage: Stage, other: ResolvedDeclaration[CodecDefinition[Any]]) -> bool: + """Whether the codec of `stage` refines `other`, another reading of the codec at that place in the pipeline. + + This is the order that `refines` defines for declarations, with each + codec compared as `codec_for_chunk` compares it: in its simplest + spelling for the chunk it receives. The chunk is the one `stage` + knows. A scope that read less may know less of the chunk: when it + claims no data type, or no codec that comes before, a `bytes` codec + keeps an `endian` that `stage` knows to mean nothing. So `other` is + simplified for the chunks of `stage` before the two are compared, and + a document read in a richer scope refines the same document read in + a poorer one. + """ + mine = stage.codec + if not isinstance(mine, AcceptedDeclaration) or isinstance(other, RefusedDeclaration): + # Nothing is simplified for a codec the scope did not read. + return refines(mine, other) + if isinstance(other, UnclaimedDeclaration): + # A gain, when what `other` reads as with the claims of `mine` is `mine`. + if claim_key(mine) != claim_key(other): + return False + return _for_stage(read_as_claimed(other, mine), stage) == codec_for_chunk(stage) + if mine.definition != other.definition or set(mine.nested) != set(other.nested): + return False + if own_key(_own_for_chunk(mine, stage.incoming)) != own_key( + _own_for_chunk(other, stage.incoming) + ): + return False + inner = _inner_stages(stage) + return all( + codec_refines(inner[loc], other.nested[loc]) + if loc in inner + else refines(mine.nested[loc], other.nested[loc]) + for loc in mine.nested + ) + + +def _own_for_chunk( + codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None +) -> AcceptedDeclaration[CodecDefinition[Any]]: + """`codec` with its own configuration as its `chunk_canonical` simplifies it for `chunk`, and each nested declaration as it is.""" + configuration = _for_chunk(codec, chunk) + return codec if configuration is None else respelled(codec, configuration, {}) + + def _for_chunk( codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None ) -> Mapping[str, JSONValue] | None: @@ -496,4 +567,4 @@ def _text_at(configuration: JSONValue, loc: Loc) -> str | None: return None if item is UNSET else json_text(item) -__all__ = ["Stage", "codec_for_chunk", "read_pipeline"] +__all__ = ["Stage", "codec_for_chunk", "codec_refines", "read_pipeline"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py index 9c20fac4a5..fc7e8a498e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -97,7 +97,9 @@ `endian` when each value takes one byte. `codec_for_chunk` returns the codec of a `Stage` in that spelling, with each codec of the pipelines it contains simplified for the chunk that one receives. An array model -compares its codecs this way. +compares its codecs this way. `codec_refines` tells whether the codec of +a `Stage` refines another reading of it, such as the same document read +in a scope that knew less of the chunk. The reader does not guess. After a codec the scope did not read, or after a codec that does not declare its output, the next codec receives a chunk with nothing known about it, `Chunk()`, which never causes a @@ -155,7 +157,7 @@ default_array_bytes_codec, storage_of, ) -from zarr_metadata.v3._pipeline import Stage, codec_for_chunk +from zarr_metadata.v3._pipeline import Stage, codec_for_chunk, codec_refines __all__ = [ "CORE_AND_EXTENSIONS_CONTEXT", @@ -190,6 +192,7 @@ "ZarrV3StaticCodecDeclaration", "ZarrV3StorageTransformerDeclaration", "codec_for_chunk", + "codec_refines", "default_array_bytes_codec", "storage_of", ] diff --git a/packages/zarr-metadata/tests/model/test_invariants.py b/packages/zarr-metadata/tests/model/test_invariants.py new file mode 100644 index 0000000000..84e11290ca --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_invariants.py @@ -0,0 +1,96 @@ +"""What holds for every v3 document the strategies generate: how a model compares with the same document read in another scope, or written in another order.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import pytest +from hypothesis import given + +from zarr_metadata.model import ZarrV3ArrayMetadata, ZarrV3GroupMetadata +from zarr_metadata.testing.strategies import ( + array_documents, + consolidated_group_documents, + group_documents, +) +from zarr_metadata.v3.definition import CORE_CONTEXT, ZARR_PYTHON_CONTEXT, ZarrV3Context + + +@pytest.mark.parametrize( + "poorer", + [ + ZarrV3Context(), + CORE_CONTEXT, + ZARR_PYTHON_CONTEXT.copy(update={"data_types": {}}), + ZARR_PYTHON_CONTEXT.copy(update={"codecs": {}}), + ], + ids=["empty", "core", "no-data-types", "no-codecs"], +) +@given(document=array_documents()) +def test_every_array_read_in_a_richer_scope_refines_the_one_a_poorer_scope_read( + poorer: ZarrV3Context, document: Mapping[str, Any] +) -> None: + """`refined_in` returns the model that the richer scope reads, and that model refines the one it came from, whatever the richer scope learns of the chunk each codec receives.""" + low = ZarrV3ArrayMetadata.from_json(document, context=poorer) + high = low.refined_in(ZARR_PYTHON_CONTEXT) + assert high == ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT) + assert high.refines(low) + assert low.refines(low) + + +def _reordered(value: object) -> Any: # noqa: ANN401 + """`value` with the members of every object it holds, at any depth, in the reverse order. The items of an array keep their order.""" + if isinstance(value, Mapping): + return {key: _reordered(value[key]) for key in reversed(value)} + if isinstance(value, (list, tuple)): + return [_reordered(item) for item in value] + return value + + +def _assert_one_model( + model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, + reordered: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, +) -> None: + assert reordered == model + assert hash(reordered) == hash(model) + assert reordered.refines(model) # pyright: ignore[reportArgumentType] + assert model.refines(reordered) # pyright: ignore[reportArgumentType] + + +@given(document=array_documents()) +def test_every_array_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + """The order of the members of an object means nothing, at any depth: in the document, in a configuration, and in a configuration that holds declarations.""" + _assert_one_model( + ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT), + ZarrV3ArrayMetadata.from_json(_reordered(document), context=ZARR_PYTHON_CONTEXT), + ) + + +@given(document=group_documents()) +def test_every_group_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + _assert_one_model( + ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT), + ZarrV3GroupMetadata.from_json(_reordered(document), context=ZARR_PYTHON_CONTEXT), + ) + + +@given(document=consolidated_group_documents()) +def test_every_listing_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + """A group lists the documents of its nodes. Each is read, so the listing is one model whatever order the members of those documents are in.""" + listings = [ + ZarrV3GroupMetadata.from_json(each, context=ZARR_PYTHON_CONTEXT).consolidated() + for each in (document, _reordered(document)) + ] + assert listings[0] is not None + assert listings[1] is not None + assert listings[1] == listings[0] + assert hash(listings[1]) == hash(listings[0]) + assert listings[1].refines(listings[0]) + assert listings[0].refines(listings[1]) diff --git a/packages/zarr-metadata/tests/model/test_pair.py b/packages/zarr-metadata/tests/model/test_pair.py index 998a45e3f2..0cc7df2335 100644 --- a/packages/zarr-metadata/tests/model/test_pair.py +++ b/packages/zarr-metadata/tests/model/test_pair.py @@ -265,6 +265,40 @@ def test_error_replace_refuses_a_document_with_a_problem() -> None: ZarrV3ArrayMetadata(ARRAY).replace(dimension_names=("x", "y")) +def _core_without_codecs(*names: str) -> ZarrV3Context: + """`CORE_CONTEXT` without the codecs it files under `names`.""" + kept = {name: codec for name, codec in CORE_CONTEXT.codecs.items() if name not in names} + return CORE_CONTEXT.copy(update={"codecs": kept}) + + +def _core_without_data_types(*names: str) -> ZarrV3Context: + """`CORE_CONTEXT` without the data types it files under `names`.""" + kept = {name: each for name, each in CORE_CONTEXT.data_types.items() if name not in names} + return CORE_CONTEXT.copy(update={"data_types": kept}) + + +LITTLE: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "little"}} +ENDIAN: dict[str, Any] = {**ARRAY, "codecs": [LITTLE]} +"""A `uint8` array whose `bytes` codec writes an `endian`, which means nothing for values of one byte. A scope that reads both the codec and the data type compares the codec without it.""" +TRANSPOSED: dict[str, Any] = { + **ARRAY, + "codecs": [{"name": "transpose", "configuration": {"order": [0]}}, LITTLE], +} +SHARDED_ENDIAN: dict[str, Any] = { + **ARRAY, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [1], + "codecs": [LITTLE], + "index_codecs": [LITTLE, "crc32c"], + }, + } + ], +} + + @pytest.mark.parametrize( ("model", "context", "gained"), [ @@ -272,8 +306,48 @@ def test_error_replace_refuses_a_document_with_a_problem() -> None: (ZarrV3ArrayMetadata(ARRAY, context=CORE_CONTEXT), CORE_AND_EXTENSIONS_CONTEXT, False), (ZarrV3ArrayMetadata(FLOAT, context=ZarrV3Context()), CORE_CONTEXT, True), (ZarrV3ArrayMetadata(SHARDED, context=ZarrV3Context()), CORE_CONTEXT, True), + # What the new scope claims tells it more of the chunk a codec + # receives, so it compares that codec in a simpler spelling than + # the one the document writes. + (ZarrV3ArrayMetadata(ENDIAN, context=ZarrV3Context()), CORE_CONTEXT, True), + ( + ZarrV3ArrayMetadata(ENDIAN, context=_core_without_codecs("bytes")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(ENDIAN, context=_core_without_data_types("uint8")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(TRANSPOSED, context=_core_without_codecs("transpose")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(SHARDED_ENDIAN, context=_core_without_codecs("sharding_indexed")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(SHARDED_ENDIAN, context=_core_without_codecs("bytes")), + CORE_CONTEXT, + True, + ), + ], + ids=[ + "gain", + "nothing-to-gain", + "gain-data-type-with-fill-value", + "gain-of-a-shard", + "gain-of-everything-with-an-endian-that-means-nothing", + "gain-of-a-codec-simplified-for-its-chunk", + "gain-of-the-data-type-a-codec-is-simplified-by", + "gain-of-a-codec-before-one-simplified-for-its-chunk", + "gain-of-a-shard-whose-codecs-are-simplified", + "gain-of-a-codec-in-a-shard", ], - ids=["gain", "nothing-to-gain", "gain-data-type-with-fill-value", "gain-of-a-shard"], ) def test_refined_in_moves_a_model_up_the_order( model: ZarrV3ArrayMetadata, context: ZarrV3Context, gained: bool diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index bcc2f3c9be..adfa8ecc35 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -1185,6 +1185,14 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: CRC32C: Final = {"name": "crc32c"} +def _shard_of(codec: JSONValue, *, index_first: bool = False, chunk_shape: int = 2) -> JSONValue: + """A shard that holds `codec`, with its `index_codecs` written before its `codecs` when `index_first`.""" + members: dict[str, JSONValue] = {"codecs": [codec], "index_codecs": [LE, CRC32C]} + if index_first: + members = dict(reversed(members.items())) + return {"name": "sharding_indexed", "configuration": {"chunk_shape": [chunk_shape], **members}} + + @pytest.mark.parametrize( ("one", "other", "kind", "equal"), [ @@ -1233,6 +1241,12 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: CodecDefinition, True, ), + ( + _shard_of(_shard_of(LE)), + _shard_of(_shard_of(LE, index_first=True), index_first=True), + CodecDefinition, + True, + ), ( {"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [LE, CRC32C]}}, {"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [CRC32C, LE]}}, @@ -1321,6 +1335,7 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: "unclaimed-another-configuration", "another-name", "members-holding-declarations-in-another-order", + "members-holding-declarations-in-another-order-two-deep", "declarations-in-another-order", "members-in-another-order", "blosc-typesize-noshuffle", @@ -1348,6 +1363,20 @@ def test_two_declarations_are_equal_when_they_read_the_same( assert hash(first) == hash(second) +def test_two_refused_declarations_are_equal_whatever_order_their_members_are_in() -> None: + """A refused declaration is compared by what was written and by the declarations it holds, in the order of their locations, not of the members that hold them.""" + # A shard of inner chunks with no elements is refused, and still holds its codecs. + first, _ = resolve(_shard_of(LE, chunk_shape=0), CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) + second, _ = resolve( + _shard_of(LE, chunk_shape=0, index_first=True), CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT + ) + assert isinstance(first, RefusedDeclaration) + assert isinstance(second, RefusedDeclaration) + assert list(first.nested) != list(second.nested) + assert first == second + assert hash(first) == hash(second) + + def test_a_declaration_copied_or_pickled_is_read_by_a_definition_equal_to_its_own() -> None: declaration, _ = resolve( {"name": "gzip", "configuration": {"level": 1}}, CodecDefinition, CORE_CONTEXT diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index 0eda1926b6..12ed0eacd1 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -23,11 +23,13 @@ canonical_of, resolve, ) +from zarr_metadata.definition._definition import held from zarr_metadata.model import ( ZarrV3ArrayMetadata, read_array_metadata_v3, ) from zarr_metadata.v3._pipeline import ( + codec_refines, read_pipeline, ) from zarr_metadata.v3.codec.bytes import BYTES_CODEC, BytesCodecConfiguration @@ -723,6 +725,58 @@ def test_a_chunk_canonical_that_simplifies_its_own_members_is_taken_at_its_word( assert list(found.nested) == [("codecs", 0), ("codecs", 1)] +@pytest.mark.parametrize( + ("data_type", "mine", "theirs", "without", "refines"), + [ + # The same scope: the order is equality, for the chunk each codec receives. + ("uint8", [LITTLE, GZIP], [BIG, GZIP], (), [True, True]), + ("uint16", [LITTLE, GZIP], [BIG, GZIP], (), [False, True]), + ("uint8", [_shard(BARE)], [_shard(LITTLE)], (), [True]), + # A gain: what the poorer scope left unclaimed reads as the codec, for the chunk it receives. + ("uint8", [LITTLE], [LITTLE], ("bytes",), [True]), + ("uint8", [LITTLE], [BIG], ("bytes",), [True]), + ("uint16", [LITTLE], [BIG], ("bytes",), [False]), + ("uint8", [BARE, GZIP], [LITTLE, GZIP], ("bytes", "gzip"), [True, True]), + # A gain of a codec that holds codecs, and a gain of a codec inside one. + ("uint8", [_shard(BARE)], [_shard(LITTLE)], ("sharding_indexed",), [True]), + ("uint8", [_shard(LITTLE)], [_shard(BIG)], ("bytes",), [True]), + ("uint16", [_shard(LITTLE)], [_shard(BIG)], ("bytes",), [False]), + # The poorer scope knew less of the chunk: it did not read the cast before. + ( + "uint16", + [_cast("uint8"), BARE], + [_cast("uint8"), LITTLE], + ("cast_value",), + [True, True], + ), + # Another codec under the same name is not a gain. + ("uint8", [LITTLE, GZIP], [LITTLE, {"name": "zstd"}], ("zstd",), [True, False]), + ], +) +def test_a_codec_refines_another_reading_of_it_for_the_chunk_it_receives( + data_type: JSONValue, + mine: list[JSONValue], + theirs: list[JSONValue], + without: tuple[str, ...], + refines: list[bool], +) -> None: + """`codec_refines` is the order `refines` defines for declarations, with each codec in its simplest spelling for the chunk its stage receives. `theirs` is read in a scope without the codecs named in `without`, which may leave it knowing less of the chunk.""" + kept = {name: codec for name, codec in CONTEXT.codecs.items() if name not in without} + poorer = CONTEXT.copy(update={"codecs": kept}) + stages, problems = _read(mine, _of(data_type)) + assert problems == () + others = [resolve(codec, CodecDefinition, poorer)[0] for codec in theirs] + found = [codec_refines(stage, other) for stage, other in zip(stages, others, strict=True)] + assert found == refines + # With nothing left out, each codec refines the other exactly when the two are equal. + if without == (): + back, _ = _read(theirs, _of(data_type)) + assert [ + codec_refines(stage, held(other.codec)) + for stage, other in zip(back, stages, strict=True) + ] == refines + + def _bytes_with(chunk_canonical: object) -> ResolvedDeclaration[CodecDefinition[Any]]: """A `bytes` codec with `endian`, read by a definition whose `chunk_canonical` is `chunk_canonical`.""" lying = dataclasses.replace(BYTES_CODEC, name="acme.bytes", chunk_canonical=chunk_canonical) From e1cad28f8b6ab1f4b45966ccc9297408f64194f0 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 17 Nov 2026 23:34:06 +0100 Subject: [PATCH 11/17] docs(zarr-metadata): add the changelog fragment Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/463.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/zarr-metadata/changes/463.bugfix.md diff --git a/packages/zarr-metadata/changes/463.bugfix.md b/packages/zarr-metadata/changes/463.bugfix.md new file mode 100644 index 0000000000..480b5c6762 --- /dev/null +++ b/packages/zarr-metadata/changes/463.bugfix.md @@ -0,0 +1 @@ +`ZarrV3ArrayMetadata.refines` compares each codec for the chunk it receives in the model that refines, so a document read in a richer scope refines the same document read in a poorer one, whatever the richer scope learns of the chunks. Before, it did not when the richer scope simplified a codec for its chunk: `refined_in` of a `uint8` array whose `bytes` codec writes an `endian`, read in a scope that claims no `bytes` codec, did not refine the model it came from. `codec_refines` in `zarr_metadata.v3.definition` makes the comparison for one codec. From ab8ec2e171324c7de7c56ff64ad8ed728993f294 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 18 Nov 2026 10:26:25 +0100 Subject: [PATCH 12/17] docs(zarr-metadata): name the changelog fragment after the upstream PR Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/{462.bugfix.md => 4647.bugfix.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{462.bugfix.md => 4647.bugfix.md} (100%) diff --git a/packages/zarr-metadata/changes/462.bugfix.md b/packages/zarr-metadata/changes/4647.bugfix.md similarity index 100% rename from packages/zarr-metadata/changes/462.bugfix.md rename to packages/zarr-metadata/changes/4647.bugfix.md From 4cee555099193786d1081c251992d576638ce971 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 18 Nov 2026 10:26:49 +0100 Subject: [PATCH 13/17] docs(zarr-metadata): name the changelog fragment after the upstream PR Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/{463.bugfix.md => 4648.bugfix.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{463.bugfix.md => 4648.bugfix.md} (100%) diff --git a/packages/zarr-metadata/changes/463.bugfix.md b/packages/zarr-metadata/changes/4648.bugfix.md similarity index 100% rename from packages/zarr-metadata/changes/463.bugfix.md rename to packages/zarr-metadata/changes/4648.bugfix.md From b2a1c3d0f7620411c42b2b37d2abb093d398ba2d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 18 Nov 2026 12:26:34 +0100 Subject: [PATCH 14/17] refactor(zarr-metadata): derive each model's key, canonicalize and refines from one normal form Each of the six models writes its document in its simplest equivalent spelling in one function. The key that == and hash compare is that document as JSON text with the claims the scope made, canonicalize reads it, and refines reads the other model's document in this model's scope and compares. The per-model key functions, the per-model refines traversals and the declaration-level refines are removed. A model no longer refines one whose document its scope reads with a problem, such as a configuration key that the definition does not declare under a name the other scope left unclaimed. Assisted-by: ClaudeCode:claude-fable-5-2 --- .../zarr_metadata/definition/_definition.py | 22 ++- .../src/zarr_metadata/definition/_scope.py | 79 +-------- .../src/zarr_metadata/model/_array_v2.py | 101 +++-------- .../src/zarr_metadata/model/_array_v3.py | 99 +++-------- .../src/zarr_metadata/model/_group_v2.py | 83 ++++----- .../src/zarr_metadata/model/_group_v3.py | 72 +++----- .../src/zarr_metadata/model/_keyed.py | 99 +++++++++-- packages/zarr-metadata/tests/v3/test_scope.py | 157 +++++++----------- 8 files changed, 263 insertions(+), 449 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index a8e08e673e..3bd55e8723 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1228,11 +1228,6 @@ def declaration_key(declaration: ResolvedDeclaration[Any]) -> tuple[object, ...] ) -def own_key(declaration: AcceptedDeclaration[Any]) -> tuple[object, ...]: - """The part of `declaration_key` for an accepted declaration that excludes nested declarations: the definition and the canonical spelling of its own members.""" - return declaration_key(declaration)[:3] - - def _nested_key(nested: Nested) -> tuple[tuple[Loc, tuple[object, ...]], ...]: """The key of each nested declaration, paired with its location, in the order of the locations. @@ -1916,12 +1911,16 @@ def _canonical_declaration(resolved: AcceptedDeclaration[Any]) -> JSONValue | No # result of `canonical` (the view itself when nothing is simplified) is # converted to a dict. view = read_only(cast("Mapping[str, JSONValue]", configuration)) - simplified = asked( - definition, - "canonical", - lambda: dict(cast("Mapping[str, JSONValue]", definition.canonical(view))), - ) - _, refused = definition.read_configuration(simplified) + + def spelled() -> tuple[bool, dict[str, JSONValue]]: + given = cast("Mapping[str, JSONValue]", definition.canonical(view)) + return given is view, dict(given) + + kept, simplified = asked(definition, "canonical", spelled) + # A `canonical` that returns what it received simplified nothing, so + # there is nothing of its own to check: the configuration was read, and + # each nested declaration was checked in its own simplest spelling. + refused = () if kept else definition.read_configuration(simplified)[1] if len(refused) != 0: msg = ( f"{definition.name!r}: its canonical gave {simplified!r}, which does not hold: " @@ -1980,7 +1979,6 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "kind_of", "named_configuration", "no_rules", - "own_key", "read_only", "resolve", "resolve_refined", diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py index 9a28bc2a32..f1883b51d5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py @@ -1,4 +1,4 @@ -"""Operations on scopes: which definition a reading records for each name, how one reading refines another, and where a scope disagrees with a reading. +"""Operations on scopes: which definition a reading records for each name, and where a scope disagrees with a reading. A model is a document read in a scope. The meaning of the document depends only on what the scope returned for the names the document uses @@ -6,7 +6,9 @@ document are ordered by how much they know. If a name was unclaimed and a definition later reads it, the name gains meaning and loses none. If one definition reads a name and then a different definition reads it, that is -a conflict. The internal `disagreements` function in +a conflict. A model's `refines` decides that order by reading: it reads +the other model's document in its own scope, so nothing here compares +two readings. The internal `disagreements` function in `zarr_metadata.definition._context`, which a model uses to compare a scope with its claims, is built on these operations. A scope's `union` raises the `ScopeConflictError` defined here when two scopes file @@ -17,27 +19,18 @@ from collections.abc import Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast +from typing import TYPE_CHECKING, Any, TypeAlias from zarr_metadata.definition._definition import ( - AcceptedDeclaration, Definition, - RefusedDeclaration, - UnclaimedDeclaration, - as_kind, - declaration_key, - declarations_of, filed_under, - format_of, - own_key, - resolve, ) if TYPE_CHECKING: from collections.abc import Callable, Iterable, Sequence from zarr_metadata._typed_json import Loc - from zarr_metadata.definition._definition import D, ResolvedDeclaration + from zarr_metadata.definition._definition import ResolvedDeclaration ClaimKey: TypeAlias = tuple[type[Definition[Any]], str] """A kind and the name a definition is filed under. A scope's `claimant` looks up definitions by this key.""" @@ -140,65 +133,6 @@ def claims_of( return claims -def refines(declaration: ResolvedDeclaration[Any], other: ResolvedDeclaration[Any]) -> bool: - """Whether `declaration` contains everything `other` contains: it reads the same where both read, and it may also read what `other` left unclaimed. - - This is the order in which one reading of a document refines another. - Suppose `other` is unclaimed and `declaration` is accepted. This counts as a - gain when re-reading the unclaimed declaration's JSON, with `declaration`'s - definition and the definitions of the declarations nested in `declaration`, - produces `declaration`. Two spellings of one configuration count as one gain, - because `==` treats them as one declaration, so the order is transitive - through equality. The reverse direction (accepted to unclaimed) is a - loss. One name read by two different definitions is a conflict. A - refused declaration refines only itself. Two declarations that refine each other - are equal. - """ - if isinstance(declaration, RefusedDeclaration) or isinstance(other, RefusedDeclaration): - return declaration == other - if isinstance(other, UnclaimedDeclaration): - if isinstance(declaration, UnclaimedDeclaration): - return declaration_key(declaration) == declaration_key(other) - return claim_key(declaration) == claim_key(other) and _gained(declaration, other) - if isinstance(declaration, UnclaimedDeclaration): - return False - if declaration.definition != other.definition or own_key(declaration) != own_key(other): - return False - if set(declaration.nested) != set(other.nested): - return False - return all(refines(declaration.nested[loc], other.nested[loc]) for loc in declaration.nested) - - -def _gained(declaration: AcceptedDeclaration[Any], other: UnclaimedDeclaration) -> bool: - """Whether reading `other` in the scope formed by `declaration`'s own claims produces `declaration`. This is the test for a gain.""" - again, _ = resolve(other.json, declaration.read_as, _Claimed.of(declaration)) - return again == declaration - - -@dataclass(frozen=True, slots=True) -class _Claimed: - """The scope formed by a declaration's own claims: its definition and the definitions of its nested declarations, keyed by kind and filed name. `_gained` re-reads a declaration in this scope.""" - - filed: Mapping[ClaimKey, Definition[Any]] - zarr_format: Literal[2, 3] - - @classmethod - def of(cls, declaration: AcceptedDeclaration[Any]) -> _Claimed: - claimed = claims_of(declarations_of(declaration)) - return cls( - {key: definition for key, definition in claimed.items() if definition is not None}, - format_of(declaration.read_as), - ) - - def claimant(self, kind: type[D], name: str) -> D | None: - """The definition of `kind` among the claims that reads `name`, or None if there is none. This matches a scope's `claimant`.""" - asked = as_kind(kind) - filed, _ = filed_under(asked, name) - if filed is None: - return None - return cast("D | None", self.filed.get((asked, filed))) - - @dataclass(frozen=True, slots=True) class Disagreements: """The places where a scope reads a reading's claims differently: the names that would gain a meaning, and the names that conflict (including names that would lose a meaning).""" @@ -240,5 +174,4 @@ def disagreements_of( "claims_of", "disagreements_of", "kind_name", - "refines", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index dc39bdfc6f..5353433cc6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -19,7 +19,6 @@ is_array, is_object, is_tuple, - json_text, not_an_object, outside_of, prefixed, @@ -33,7 +32,6 @@ AcceptedDeclaration, ResolvedDeclaration, UnclaimedDeclaration, - declaration_key, declarations_of, default_fill_value, fill_value_problems, @@ -43,7 +41,6 @@ spelled_canonically, ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of -from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( @@ -91,6 +88,7 @@ from collections.abc import Iterator, Sequence from zarr_metadata._typed_json import Loc + from zarr_metadata.definition._scope import Claims from zarr_metadata.v2.array import ZarrV2ArrayMetadataStoreKey from zarr_metadata.v2.attributes import ZarrV2AttributesStoreKey @@ -628,52 +626,25 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for a v2 array model: what its document means. - - The key holds each declaration's `declaration_key`, and the fill value as JSON - text, in its canonical spelling when a definition in scope read the - dtype. Every other member is included as it is, with JSON members - as text. No `.zattrs` counts as empty attributes, since both mean - that the array has none, as in v3. - """ - members = self._members - return ( - members.shape, - members.chunks, - members.order, - members.dimension_separator, - self._fill_value_key(), - declaration_key(self.dtype), - None if self.compressor is None else declaration_key(self.compressor), - None - if self.filters is None - else tuple(declaration_key(entry) for entry in self.filters), - json_text({} if members.attributes is UNSET else members.attributes), - json_text(members.extra_fields), - ) - - def _plain_key( - self, dtype: AcceptedDeclaration[ZarrV2DataTypeDefinition[Any]] | UnclaimedDeclaration - ) -> tuple[object, ...]: - """The members that hold no declaration, which `refines` compares. The fill value is in the spelling of `dtype`, the dtype of the more informed side.""" - members = self._members - return ( - members.shape, - members.chunks, - members.order, - members.dimension_separator, - json_text(spelled_canonically(dtype, members.fill_value)), - json_text({} if members.attributes is UNSET else members.attributes), - json_text(members.extra_fields), + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document in its simplest equivalent spelling. This is the one place that decides what a v2 array document means: the model's key is this document as JSON text.""" + compressor, filters = self.compressor, self.filters + spelled: dict[str, JSONValue] = { + "dtype": simplest(self.dtype), + "fill_value": spelled_canonically(self.dtype, self._members.fill_value), + "compressor": None if compressor is None else simplest(compressor), + "filters": None if filters is None else tuple(simplest(entry) for entry in filters), + } + return canonical_members( + self._document, + spelled, + empty=("attributes",), + defaults={"dimension_separator": ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR}, ) - def _fill_value_key(self) -> str: - """The fill value as `==` compares it, as JSON text: in its canonical spelling when a definition in scope read the dtype, and as written otherwise.""" - fill_value = self._members.fill_value - if isinstance(self.dtype, AcceptedDeclaration): - return json_text(spelled_canonically(self.dtype, fill_value)) - return json_text(fill_value) + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of(self._reading.declarations()) def __reduce__(self) -> tuple[type[ZarrV2ArrayMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) @@ -792,20 +763,7 @@ def canonicalize(self) -> ZarrV2ArrayMetadata: Everything else is kept as written: `shape`, `chunks`, `order`, the attributes, and whatever nothing in scope claims. """ - compressor, filters = self.compressor, self.filters - spelled: dict[str, JSONValue] = { - "dtype": simplest(self.dtype), - "fill_value": spelled_canonically(self.dtype, self._members.fill_value), - "compressor": None if compressor is None else simplest(compressor), - "filters": None if filters is None else tuple(simplest(entry) for entry in filters), - } - document = canonical_members( - self._document, - spelled, - empty=("attributes",), - defaults={"dimension_separator": ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR}, - ) - return type(self)(document, context=self._context) + return type(self)(self._normal_form, context=self._context) def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. @@ -837,25 +795,8 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadat return self.with_context(context) def refines(self, other: ZarrV2ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each declaration must refine its counterpart. A `null` compressor or `null` filters refine only a `null`. Every other member must be the same, with the fill value compared in the spelling of the more informed dtype. If that dtype rejects the other model's fill value, this is not a refinement.""" - if type(other) is not type(self): - return False - if (self.compressor is None) != (other.compressor is None): - return False - if (self.filters is None) != (other.filters is None): - return False - mine = () if self.filters is None else self.filters - theirs = () if other.filters is None else other.filters - if len(mine) != len(theirs): - return False - pairs = [(self.dtype, other.dtype), *zip(mine, theirs, strict=True)] - if self.compressor is not None and other.compressor is not None: - pairs.append((self.compressor, other.compressor)) - if not all(refines_declaration(one, another) for one, another in pairs): - return False - if len(fill_value_problems(self.dtype, other._members.fill_value)) != 0: - return False - return self._plain_key(self.dtype) == other._plain_key(self.dtype) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. So the fill value is compared in the spelling of this model's dtype. If `other`'s document has a problem in this scope, as when this model's dtype rejects its fill value, this is not a refinement. False when `other` is not a v2 array.""" + return self._refines(other, lambda: other.with_context(self._context)) # --- constructors ----------------------------------------------------- diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index c0fa12a402..d4e23eba54 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -18,7 +18,6 @@ frozen, is_array, is_object, - json_text, nested_past_the_levels, not_an_object, refine_json, @@ -34,7 +33,6 @@ Definition, ResolvedDeclaration, UnclaimedDeclaration, - declaration_key, declaration_kind, declarations_of, default_fill_value, @@ -45,9 +43,8 @@ spelled_canonically, ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of -from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike -from zarr_metadata.model._keyed import Keyed +from zarr_metadata.model._keyed import Keyed, normal_key from zarr_metadata.model._shared import ( Derived, StoreKey, @@ -91,6 +88,7 @@ from collections.abc import Iterator from zarr_metadata._typed_json import Loc + from zarr_metadata.definition._scope import Claims from zarr_metadata.v3._definition import CodecDefinition, ExtraFieldDefinition from zarr_metadata.v3.array import ZarrV3ArrayMetadataStoreKey @@ -607,21 +605,13 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for a v3 array model: what its document means, as computed by `array_key_of`.""" - return array_key_of(self._reading, self._members) - - def _plain_key( - self, data_type: AcceptedDeclaration[DataTypeDefinition[Any]] | UnclaimedDeclaration - ) -> tuple[object, ...]: - """The members that hold no declaration, which `refines` compares. The fill value is in the spelling of `data_type`, the data type of the more informed side.""" - members = self._members - return ( - members.shape, - json_text(spelled_canonically(data_type, members.fill_value)), - members.dimension_names, - json_text(members.attributes), - ) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document in its simplest equivalent spelling, as `canonical_array_document_v3` writes it.""" + return canonical_array_document_v3(self._reading, self._members) + + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of(self._reading.declarations()) def __reduce__(self) -> tuple[type[ZarrV3ArrayMetadata], tuple[object, ZarrV3Context]]: # Pickle the pair and read it again on load, so a model's reading @@ -760,8 +750,7 @@ def canonicalize(self) -> ZarrV3ArrayMetadata: of `true`. A document that is already in this spelling is returned as it is, in a new model. """ - document = canonical_array_document_v3(self._reading, self._members) - return type(self)(document, context=self._context) + return type(self)(self._normal_form, context=self._context) def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. @@ -796,28 +785,8 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadat return self.with_context(context) def refines(self, other: ZarrV3ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each declaration must refine its counterpart, in the order `refines` defines for declarations (including the declarations nested in each declaration). Every other member must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" - if type(other) is not type(self): - return False - if len(self.codecs) != len(other.codecs) or len(self.storage_transformers) != len( - other.storage_transformers - ): - return False - if self.extra_fields.keys() != other.extra_fields.keys(): - return False - pairs = ( - (self.data_type, other.data_type), - (self.chunk_grid, other.chunk_grid), - (self.chunk_key_encoding, other.chunk_key_encoding), - *zip(codecs_for_chunks(self._reading), codecs_for_chunks(other._reading), strict=True), - *zip(self.storage_transformers, other.storage_transformers, strict=True), - *((self.extra_fields[key], other.extra_fields[key]) for key in self.extra_fields), - ) - if not all(refines_declaration(mine, theirs) for mine, theirs in pairs): - return False - if len(fill_value_problems(self.data_type, other._members.fill_value)) != 0: - return False - return self._plain_key(self.data_type) == other._plain_key(self.data_type) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. So each codec is compared for the chunk it receives in this model, and the fill value in the spelling of this model's data type, whatever `other`'s scope knew of either. If `other`'s document has a problem in this scope, as when this model's data type rejects its fill value, this is not a refinement. False when `other` is not a v3 array.""" + return self._refines(other, lambda: other.with_context(self._context)) # --- constructors ----------------------------------------------------- @@ -951,32 +920,9 @@ def _derived_codecs(document: Mapping[str, object], context: ZarrV3Context) -> o def array_key_of( reading: ZarrV3ArrayMetadataReading, members: ArrayMembersV3 ) -> tuple[object, ...]: - """A key for what a v3 array document means, from its `reading` and refined `members`. `==` and `hash` compare this key for the model, and two listings of one node are compared by it. - - The key holds each declaration's `declaration_key`, and the fill value as JSON - text: in its canonical spelling when a definition in scope read the - data type, and as written otherwise. A codec is compared as - `codec_for_chunk` returns it for the chunk it receives, so two - spellings that mean the same for that chunk are one codec. Every - other member is included as it is, with JSON members as text. - """ - data_type = held(reading.data_type) - fill_value = ( - spelled_canonically(data_type, members.fill_value) - if isinstance(data_type, AcceptedDeclaration) - else members.fill_value - ) - return ( - members.shape, - json_text(fill_value), - declaration_key(data_type), - declaration_key(held(reading.chunk_grid)), - tuple(declaration_key(codec) for codec in codecs_for_chunks(reading)), - declaration_key(held(reading.chunk_key_encoding)), - members.dimension_names, - json_text(members.attributes), - tuple(declaration_key(held(entry)) for entry in reading.storage_transformers), - extra_fields_key(reading.extra_fields), + """A key for what a v3 array document means, from its `reading` and refined `members`: the key of the model, as `normal_key` computes it from the claims the reading made and from `canonical_array_document_v3`. Two listings of one node are compared by it.""" + return normal_key( + claims_of(reading.declarations()), canonical_array_document_v3(reading, members) ) @@ -985,9 +931,10 @@ def canonical_array_document_v3( ) -> dict[str, JSONValue]: """A v3 array document in its simplest equivalent spelling, from its `reading` and refined `members`, as `ZarrV3ArrayMetadata.canonicalize` reads it. - It is written from the parts that `array_key_of` compares, so two - documents with one key are written the same way, apart from the order - of the members of an object. + This is the one place that decides what a v3 array document means. + The model's key is this document as JSON text, so two documents with + one key are written the same way, apart from the order of the members + of an object. """ data_type = held(reading.data_type) spelled: dict[str, JSONValue] = { @@ -1011,14 +958,6 @@ def codecs_for_chunks( return tuple(held(codec_for_chunk(stage)) for stage in reading.pipeline) -def extra_fields_key(extra_fields: Mapping[str, ResolvedDeclaration[Any]]) -> tuple[object, ...]: - """A key for what a document's extra fields mean: each key with its declaration's `declaration_key`, sorted by key.""" - return tuple( - (key, declaration_key(extra)) - for key, extra in sorted(extra_fields.items(), key=lambda item: item[0]) - ) - - def read_array_metadata_v3( value: object, *, context: ZarrV3Context | None = None ) -> ZarrV3ArrayMetadataReading: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index 3f185a0f9d..1ae725a266 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -19,7 +19,6 @@ frozen, is_json_object, is_object, - json_text, not_an_object, object_at, prefixed, @@ -72,6 +71,7 @@ from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import ResolvedDeclaration, UnclaimedDeclaration + from zarr_metadata.definition._scope import Claims from zarr_metadata.v2.attributes import ZarrV2AttributesStoreKey from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataStoreKey from zarr_metadata.v2.group import ZarrV2GroupMetadataStoreKey @@ -358,10 +358,13 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for a v2 group model: its attributes as JSON text. No `.zattrs` counts as empty attributes, since both mean that the group has none, as in v3.""" - attributes = self._attributes - return (json_text({} if attributes is UNSET else attributes),) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document in its simplest equivalent spelling: without `attributes` when they are empty, since no `.zattrs` and an empty one both mean that the group has none.""" + return canonical_members(self._document, {}, empty=("attributes",)) + + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return {} def __reduce__(self) -> tuple[type[ZarrV2GroupMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) @@ -381,8 +384,7 @@ def __replace__(self, /, **members: Unpack[ZarrV2GroupMetadataUpdate]) -> ZarrV2 def canonicalize(self) -> ZarrV2GroupMetadata: """Return this model with its document in its simplest equivalent spelling, read in this model's own scope: without `attributes` when they are empty, because a group with no `.zattrs` means the same. The result equals this model, and every model equal to this one gives the same document, apart from the order of the members of an object, which `to_key_value` sorts with `JSONWriter(sort_keys=True)`.""" - document = canonical_members(self._document, {}, empty=("attributes",)) - return type(self)(document, context=self._context) + return type(self)(self._normal_form, context=self._context) def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: """Return this document read in `context`: the same group, holding that scope.""" @@ -394,8 +396,8 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadat return self.with_context(context) def refines(self, other: ZarrV2GroupMetadata) -> bool: - """Whether this group holds everything `other` holds, which means its attributes are written the same way, no `.zattrs` counting as empty attributes. False when `other` is not a v2 group.""" - return type(other) is type(self) and self._model_key == other._model_key + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. A group has no declarations, so this means its attributes are written the same way, no `.zattrs` counting as empty attributes. False when `other` is not a v2 group.""" + return self._refines(other, lambda: other.with_context(self._context)) @classmethod def create_default( @@ -568,18 +570,25 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for v2 consolidated metadata: each node's path and key, and every other entry as JSON text.""" - return ( - tuple(sorted((path, node._model_key) for path, node in self._metadata.items())), - self._other_entries_text(), - ) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document with each node in its simplest equivalent spelling, as that node's normal form writes it: its `.zarray` or `.zgroup` entry without the attributes, and its `.zattrs` entry holding them, or left out when they are empty.""" + entries = dict(object_at(self._document, "metadata")) + for path, node in self._metadata.items(): + names = self._files[path] + written = dict(node._normal_form) + attributes = written.pop("attributes", None) + for name, key in names.items(): + if name != ZARR_V2_ATTRIBUTES_STORE_KEY: + entries[key] = written + elif attributes is None: + del entries[key] + else: + entries[key] = attributes + return {**self._document, "metadata": entries} - def _other_entries_text(self) -> str: - """The entries that no node is read from (an orphan `.zattrs`, or any other key) as JSON text, which `==` compares.""" - entries = object_at(self._document, "metadata") - consumed = {key for names in self._files.values() for key in names.values()} - return json_text({key: value for key, value in entries.items() if key not in consumed}) + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of(self._located_declarations()) def _located_declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: """Each declaration of each array node, as read, located in the document: under the node's `.zarray` entry, at the key the document writes for it.""" @@ -602,20 +611,7 @@ def canonicalize(self) -> ZarrV2ConsolidatedMetadata: scope, gives the same document, apart from the order of the members of an object, which a `JSONWriter(sort_keys=True)` sorts. """ - entries = dict(object_at(self._document, "metadata")) - for path, node in self._metadata.items(): - names = self._files[path] - written = cast("dict[str, JSONValue]", node.canonicalize().to_json()) - attributes = written.pop("attributes", None) - for name, key in names.items(): - if name != ZARR_V2_ATTRIBUTES_STORE_KEY: - entries[key] = written - elif attributes is None: - del entries[key] - else: - entries[key] = attributes - document: dict[str, JSONValue] = {**self._document, "metadata": entries} - return type(self)(document, context=self._context) + return type(self)(self._normal_form, context=self._context) def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: """Return this document with every node read in `context`, whatever that changes. Raises `MetadataValidationError` when a node has a problem in `context`.""" @@ -648,16 +644,8 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2Consolidated return self.with_context(context) def refines(self, other: ZarrV2ConsolidatedMetadata) -> bool: - """Whether every node this holds refines the node `other` holds at the same path, both hold the same paths, and the other entries are written the same way. False when `other` is not v2 consolidated metadata.""" - if type(other) is not type(self): - return False - if self._metadata.keys() != other._metadata.keys(): - return False - if self._other_entries_text() != other._other_entries_text(): - return False - return all( - _v2_node_refines(self._metadata[path], other._metadata[path]) for path in self._metadata - ) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. So every node this holds refines the node `other` holds at the same path, both hold the same paths, and the other entries are written the same way. False when `other` is not v2 consolidated metadata.""" + return self._refines(other, lambda: other.with_context(self._context)) @classmethod def from_json( @@ -682,13 +670,6 @@ def from_key_value( ) -def _v2_node_refines(node: ZarrV2NodeMetadata, other: ZarrV2NodeMetadata) -> bool: - """Whether `node` refines `other`. Models of the same kind use their own `refines`; an array never refines a group, or the reverse.""" - if isinstance(node, ZarrV2ArrayMetadata): - return isinstance(other, ZarrV2ArrayMetadata) and node.refines(other) - return isinstance(other, ZarrV2GroupMetadata) and node.refines(other) - - _NODE_FILES: Final = ( ZARR_V2_ARRAY_METADATA_STORE_KEY, ZARR_V2_GROUP_METADATA_STORE_KEY, diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 33e4cd82ab..153fd9b5ca 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -47,13 +47,11 @@ definition_said, kind_name, ) -from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._array_v3 import ( ArrayMembersV3, ZarrV3ArrayMetadata, ZarrV3ArrayMetadataReading, array_key_of, - extra_fields_key, members_past_the_levels, must_understand_subset, no_extra_fields, @@ -102,6 +100,7 @@ from zarr_metadata._hierarchy import NodeType from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import AcceptedDeclaration, ResolvedDeclaration + from zarr_metadata.definition._scope import Claims from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataStoreKey @@ -284,12 +283,14 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for a v3 group model: its attributes as JSON text, and what each extra field means, as `extra_fields_key` computes it.""" - return ( - json_text(self._members.attributes), - extra_fields_key(self._reading.extra_fields), - ) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document in its simplest equivalent spelling. This is the one place that decides what a v3 group document means: the model's key is this document as JSON text.""" + spelled = simplest_members(self._reading.extra_fields) + return canonical_members(self._document, spelled, empty=("attributes",)) + + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of(self._reading.declarations()) def __reduce__(self) -> tuple[type[ZarrV3GroupMetadata], tuple[object, ZarrV3Context]]: # Pickle the pair, and read it again on load. @@ -411,9 +412,7 @@ def canonicalize(self) -> ZarrV3GroupMetadata: spelling, and `replace(consolidated_metadata=...)` writes them back into the group. """ - spelled = simplest_members(self._reading.extra_fields) - document = canonical_members(self._document, spelled, empty=("attributes",)) - return type(self)(document, context=self._context) + return type(self)(self._normal_form, context=self._context) def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: """Return this document read in `context`, whatever that changes. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically.""" @@ -443,15 +442,8 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadat return self.with_context(context) def refines(self, other: ZarrV3GroupMetadata) -> bool: - """Whether this group holds everything `other` holds: the same attributes, and the same extra fields, each refining its counterpart in the order `refines` defines for declarations. False when `other` is not a v3 group.""" - if type(other) is not type(self): - return False - if json_text(self._members.attributes) != json_text(other._members.attributes): - return False - mine, theirs = self.extra_fields, other.extra_fields - if mine.keys() != theirs.keys(): - return False - return all(refines_declaration(mine[key], theirs[key]) for key in mine) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. The group does not read the documents its `consolidated_metadata` lists, so they are compared as written. False when `other` is not a v3 group.""" + return self._refines(other, lambda: other.with_context(self._context)) # --- constructors ----------------------------------------------------- @@ -608,11 +600,17 @@ def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for consolidated metadata: each document's path and key, sorted by path.""" - return tuple( - (path, node._model_key) - for path, node in sorted(self.metadata.items(), key=lambda item: item[0]) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The member with each document it lists in its simplest equivalent spelling, as that node's normal form writes it.""" + metadata: dict[str, JSONValue] = { + path: dict(node._normal_form) for path, node in self._metadata.items() + } + return {**self._document, "metadata": metadata} + + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of( + located for node in self._metadata.values() for located in node.reading.declarations() ) def __reduce__(self) -> tuple[type[ZarrV3ConsolidatedMetadata], tuple[object, ZarrV3Context]]: @@ -625,12 +623,7 @@ def canonicalize(self) -> ZarrV3ConsolidatedMetadata: this scope, gives the same member, apart from the order of the members of an object, which a `JSONWriter(sort_keys=True)` sorts. """ - metadata: dict[str, JSONValue] = { - path: cast("JSONValue", node.canonicalize().to_json()) - for path, node in self._metadata.items() - } - document: dict[str, JSONValue] = {**self._document, "metadata": metadata} - return type(self)(document, context=self._context) + return type(self)(self._normal_form, context=self._context) def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: """Return this member with every document read in `context`, whatever that changes. Raises `MetadataValidationError` when a document has a problem in `context`. The existing readings are reused when `context` reads every claim of every document identically.""" @@ -666,14 +659,8 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3Consolidated return self.with_context(context) def refines(self, other: ZarrV3ConsolidatedMetadata) -> bool: - """Whether every document this holds refines the document `other` holds at the same path, and both hold the same paths. False when `other` is not consolidated metadata.""" - if type(other) is not type(self): - return False - if self._metadata.keys() != other._metadata.keys(): - return False - return all( - _node_refines(self._metadata[path], other._metadata[path]) for path in self._metadata - ) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. So every document this holds refines the document `other` holds at the same path, and both hold the same paths. False when `other` is not consolidated metadata.""" + return self._refines(other, lambda: other.with_context(self._context)) @classmethod def from_json( @@ -683,13 +670,6 @@ def from_json( return cls(data, context=context) -def _node_refines(node: ZarrV3NodeMetadata, other: ZarrV3NodeMetadata) -> bool: - """Whether `node` refines `other`. Models of the same kind use their own `refines`; an array never refines a group, or the reverse.""" - if isinstance(node, ZarrV3ArrayMetadata): - return isinstance(other, ZarrV3ArrayMetadata) and node.refines(other) - return isinstance(other, ZarrV3GroupMetadata) and node.refines(other) - - @dataclass(frozen=True, slots=True) class ZarrV3GroupMetadataReading: """A v3 group document as a scope read it: each extra field, every problem, and the model when there are no problems.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_keyed.py b/packages/zarr-metadata/src/zarr_metadata/model/_keyed.py index a26be64c02..e8171ec295 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_keyed.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_keyed.py @@ -1,25 +1,76 @@ -"""The key every model uses for comparison and hashing. The key covers everything the model shows, and is computed once, the first time the model is compared.""" +"""What every model derives from one normal form: its key, `==`, `hash`, `canonicalize` and `refines`. + +A model's normal form is its document in its simplest equivalent +spelling, as its scope reads it. Each model writes that document in one +function, `_canonical_document`. Nothing else in a model decides what +its document means: the key is the text of the normal form with the +claims the scope made, `canonicalize` reads the normal form, and +`refines` reads the other model's document in this model's scope and +compares the two normal forms. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from zarr_metadata._json import MetadataValidationError, json_text + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.definition._scope import Claims + + +def normal_key(claims: Claims, document: Mapping[str, JSONValue]) -> tuple[object, ...]: + """The key of a document in its simplest equivalent spelling, read with `claims`: the claims, and the document as JSON text. + + Two documents mean the same thing when a scope reads the names they + use with the same definitions, and they are one document in their + simplest spelling. The text sorts the members of each object, so the + order a document writes them in is not compared. + """ + return (frozenset(claims.items()), json_text(dict(document))) class Keyed: - """A model whose `==` and `hash` compare a cached key that describes what its document means. - - Each model computes its own key in `_key_of`. `_model_key` computes - the key the first time something asks for it (`==`, `hash`, or a - model that holds this one) and caches it. A model that is only read - never pays for the key. Two models are equal when they have the same - class and equal keys. A model of another class, or any other object, - is never equal to a model. + """A model whose `==`, `hash`, `canonicalize` and `refines` are derived from one normal form. + + Each model writes its normal form in `_canonical_document`, and names + the claims its scope made in `_claims`. The normal form and the key + are computed the first time something asks for them, and cached. A + model that is only read never pays for either. Two models are equal + when they have the same class and equal keys. A model of another + class, or any other object, is never equal to a model. """ - __slots__ = ("_key",) + __slots__ = ("_key", "_normal") _key: tuple[object, ...] + _normal: Mapping[str, JSONValue] - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for this model. Each model class defines its own.""" + def _canonical_document(self) -> Mapping[str, JSONValue]: + """This model's document in its simplest equivalent spelling. Each model class writes its own.""" raise NotImplementedError + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses. Each model class returns its own.""" + raise NotImplementedError + + @property + def _normal_form(self) -> Mapping[str, JSONValue]: + """The normal form. It is computed on first access and cached.""" + try: + return self._normal + except AttributeError: + normal = self._canonical_document() + self._normal = normal + return normal + + def _key_of(self) -> tuple[object, ...]: + """The key `==` and `hash` compare for this model, as `normal_key` computes it.""" + return normal_key(self._claims(), self._normal_form) + @property def _model_key(self) -> tuple[object, ...]: """The key. It is computed on first access and cached.""" @@ -37,3 +88,27 @@ def __eq__(self, other: object) -> bool: def __hash__(self) -> int: return hash(self._model_key) + + def _refines(self, other: object, read_here: Callable[[], object]) -> bool: + """Whether this model holds everything `other` holds: every claim `other` holds, and what `other`'s document reads as in this model's scope. `read_here` returns that reading, and raises `MetadataValidationError` when the document has a problem in this scope. + + A model refines another when its scope read every name the other + scope read, with the same definition, and the other model's + document, read in this model's scope, is this model. So one + reader decides what both documents mean, with everything this + scope knows: the chunk each codec receives, the data type that + spells the fill value, and the definitions of the names the + other scope left unclaimed. A document with a problem in this + scope is not one this model refines. + """ + if type(other) is not type(self): + return False + mine = self._claims() + for key, definition in other._claims().items(): + if definition is not None and mine.get(key) != definition: + return False + try: + again = read_here() + except MetadataValidationError: + return False + return again == self diff --git a/packages/zarr-metadata/tests/v3/test_scope.py b/packages/zarr-metadata/tests/v3/test_scope.py index 5ccc6b425d..678dab9805 100644 --- a/packages/zarr-metadata/tests/v3/test_scope.py +++ b/packages/zarr-metadata/tests/v3/test_scope.py @@ -2,6 +2,7 @@ from __future__ import annotations +import dataclasses import pickle from typing import Any @@ -20,7 +21,8 @@ ) from zarr_metadata.definition._context import disagreements from zarr_metadata.definition._definition import declarations_of -from zarr_metadata.definition._scope import Disagreements, claims_of, kind_name, refines +from zarr_metadata.definition._scope import Disagreements, claims_of, kind_name +from zarr_metadata.model import ZarrV3ArrayMetadata from zarr_metadata.v3.codec.bytes import BYTES_CODEC from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC from zarr_metadata.v3.codec.gzip import GZIP_CODEC @@ -166,21 +168,18 @@ def test_error_claims_of_refuses_one_name_read_two_ways() -> None: ) +ARRAY: dict[str, Any] = dict(ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json()) +"""A `uint8` array of one chunk of two values.""" GZIP_DECLARATION = {"name": "gzip", "configuration": {"level": 5}} +GZIP_WITH_AN_UNKNOWN_KEY = {"name": "gzip", "configuration": {"level": 5, "extra": 1}} +"""A gzip that the core definition reads with a problem, and that a scope without a gzip accepts as written.""" +STATIC_GZIP = dataclasses.replace(GZIP_CODEC, size="static") +"""A second definition under the core gzip's name that reads the same configurations.""" ZSTD_DECLARATION = {"name": "zstd", "configuration": {"level": 3}} BLOSC = {"cname": "zstd", "shuffle": "noshuffle", "blocksize": 0} BLOSC_LEVEL_ONE = {"name": "blosc", "configuration": {**BLOSC, "clevel": 1}} BLOSC_LEVEL_TRUE = {"name": "blosc", "configuration": {**BLOSC, "clevel": True}} """Two documents that Python's `==` treats as equal, but that `json_text` tells apart.""" -SHARD_HOLDING_REFUSED = { - "name": "sharding_indexed", - "configuration": { - "chunk_shape": [1], - "codecs": ["bytes", {"name": "gzip", "configuration": {"level": 12}}], - "index_codecs": [{"name": "bytes", "configuration": {"endian": "little"}}, "crc32c"], - }, -} -"""A shard that is read, holding a gzip that its definition refuses.""" NESTED_ZSTD = { "name": "sharding_indexed", "configuration": { @@ -191,74 +190,60 @@ def test_error_claims_of_refuses_one_name_read_two_ways() -> None: } +def _array(codec: object, context: ZarrV3Context) -> ZarrV3ArrayMetadata: + """A `uint8` array of one chunk of two values whose pipeline ends in `codec`, read in `context`. A codec that takes bytes follows a `bytes` codec.""" + shard = isinstance(codec, dict) and codec["name"] == "sharding_indexed" + codecs = [codec] if shard else [{"name": "bytes"}, codec] + return ZarrV3ArrayMetadata({**ARRAY, "codecs": codecs}, context=context) + + @pytest.mark.parametrize( - ("declaration", "other", "expected"), + ("model", "other", "expected"), [ + (_array(GZIP_DECLARATION, CORE_CONTEXT), _array(GZIP_DECLARATION, CORE_CONTEXT), True), + (_array({"name": "crc32c"}, CORE_CONTEXT), _array("crc32c", CORE_CONTEXT), True), ( - _read(GZIP_DECLARATION, CodecDefinition, CORE_CONTEXT), - _read(GZIP_DECLARATION, CodecDefinition, CORE_CONTEXT), + _array(ZSTD_DECLARATION, CORE_AND_EXTENSIONS_CONTEXT), + _array(ZSTD_DECLARATION, CORE_CONTEXT), True, ), ( - _read({"name": "crc32c"}, CodecDefinition, CORE_CONTEXT), - _read("crc32c", CodecDefinition, CORE_CONTEXT), - True, - ), - ( - _read(ZSTD_DECLARATION, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT), - _read(ZSTD_DECLARATION, CodecDefinition, CORE_CONTEXT), - True, - ), - ( - _read(ZSTD_DECLARATION, CodecDefinition, CORE_CONTEXT), - _read(ZSTD_DECLARATION, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT), + _array(ZSTD_DECLARATION, CORE_CONTEXT), + _array(ZSTD_DECLARATION, CORE_AND_EXTENSIONS_CONTEXT), False, ), ( - _read(GZIP_DECLARATION, CodecDefinition, CORE_CONTEXT), - _read(GZIP_DECLARATION, CodecDefinition, ZarrV3Context(codecs=[MY_GZIP])), + _array(GZIP_DECLARATION, CORE_CONTEXT), + _array(GZIP_DECLARATION, ZarrV3Context(codecs=[STATIC_GZIP])), False, ), ( - _read(NESTED_ZSTD, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT), - _read(NESTED_ZSTD, CodecDefinition, CORE_CONTEXT), + _array(NESTED_ZSTD, CORE_AND_EXTENSIONS_CONTEXT), + _array(NESTED_ZSTD, CORE_CONTEXT), True, ), ( - _read(NESTED_ZSTD, CodecDefinition, CORE_CONTEXT), - _read(NESTED_ZSTD, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT), + _array(NESTED_ZSTD, CORE_CONTEXT), + _array(NESTED_ZSTD, CORE_AND_EXTENSIONS_CONTEXT), False, ), + (_array("zstd", CORE_CONTEXT), _array("zstd", CORE_CONTEXT), True), ( - _read({"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_CONTEXT), - _read("gzip", CodecDefinition, CORE_CONTEXT), + _array({"name": "zstd", "configuration": {"level": 1}}, CORE_CONTEXT), + _array({"name": "zstd", "configuration": {"level": 2}}, CORE_CONTEXT), False, ), + (_array("crc32c", CORE_CONTEXT), _array({"name": "crc32c"}, ZarrV3Context()), True), ( - _read("zstd", CodecDefinition, CORE_CONTEXT), - _read("zstd", CodecDefinition, CORE_CONTEXT), - True, - ), - ( - _read({"name": "zstd", "configuration": {"level": 1}}, CodecDefinition, CORE_CONTEXT), - _read({"name": "zstd", "configuration": {"level": 2}}, CodecDefinition, CORE_CONTEXT), + _array(BLOSC_LEVEL_ONE, CORE_AND_EXTENSIONS_CONTEXT), + _array(BLOSC_LEVEL_TRUE, ZarrV3Context()), False, ), ( - _read("crc32c", CodecDefinition, CORE_CONTEXT), - _read({"name": "crc32c"}, CodecDefinition, ZarrV3Context()), - True, - ), - ( - _read(BLOSC_LEVEL_ONE, CodecDefinition, CORE_CONTEXT), - _read(BLOSC_LEVEL_TRUE, CodecDefinition, ZarrV3Context()), + _array(GZIP_DECLARATION, CORE_CONTEXT), + _array(GZIP_WITH_AN_UNKNOWN_KEY, ZarrV3Context()), False, ), - ( - _read(SHARD_HOLDING_REFUSED, CodecDefinition, CORE_CONTEXT), - _read(SHARD_HOLDING_REFUSED, CodecDefinition, CORE_CONTEXT), - True, - ), ], ids=[ "same", @@ -268,19 +253,18 @@ def test_error_claims_of_refuses_one_name_read_two_ways() -> None: "conflict", "nested-gain", "nested-loss", - "refused", "both-unclaimed", "unclaimed-differ", "gain-over-another-spelling", "no-gain-over-true-for-one", - "holding-a-refused-declaration", + "no-gain-over-a-problem", ], ) def test_refines_orders_readings_by_information( - declaration: ResolvedDeclaration[Any], other: ResolvedDeclaration[Any], expected: bool + model: ZarrV3ArrayMetadata, other: ZarrV3ArrayMetadata, expected: bool ) -> None: - """`declaration` refines `other` when it reads the same where both read, and gains where `other` left a name unclaimed, including in the declarations it holds. A loss, a conflict, a refused declaration, or two unclaimed declarations written differently do not refine.""" - assert refines(declaration, other) is expected + """A model refines another when it reads the same where both read, and gains where the other left a name unclaimed, including in the declarations a codec holds. A loss, a conflict, or two unclaimed declarations written differently do not refine. Neither does a document that the refining scope reads with a problem: what it wrote is not what the refining model holds.""" + assert model.refines(other) is expected DOCUMENTS = [ @@ -291,25 +275,22 @@ def test_refines_orders_readings_by_information( ZSTD_DECLARATION, NESTED_ZSTD, BLOSC_LEVEL_ONE, - SHARD_HOLDING_REFUSED, ] CONTEXTS = [ZarrV3Context(), CORE_CONTEXT, CORE_AND_EXTENSIONS_CONTEXT] -READINGS = [ - _read(document, CodecDefinition, context) for document in DOCUMENTS for context in CONTEXTS -] +READINGS = [_array(document, context) for document in DOCUMENTS for context in CONTEXTS] @given(st.sampled_from(READINGS), st.sampled_from(READINGS), st.sampled_from(READINGS)) def test_refines_is_a_partial_order_whose_bottom_is_equality( - a: ResolvedDeclaration[Any], b: ResolvedDeclaration[Any], c: ResolvedDeclaration[Any] + a: ZarrV3ArrayMetadata, b: ZarrV3ArrayMetadata, c: ZarrV3ArrayMetadata ) -> None: - """Over readings of documents in several scopes and spellings, `refines` is reflexive and transitive. Two declarations that refine each other are equal, and equal declarations are refined by the same declarations.""" - assert refines(a, a) - if refines(a, b) and refines(b, c): - assert refines(a, c) - assert (refines(a, b) and refines(b, a)) is (a == b) + """Over readings of documents in several scopes and spellings, `refines` is reflexive and transitive. Two models that refine each other are equal, and equal models are refined by the same models.""" + assert a.refines(a) + if a.refines(b) and b.refines(c): + assert a.refines(c) + assert (a.refines(b) and b.refines(a)) is (a == b) if a == b: - assert refines(c, a) is refines(c, b) + assert c.refines(a) is c.refines(b) @pytest.mark.parametrize( @@ -407,32 +388,18 @@ def test_a_kind_is_named_in_words(kind: type[Definition[Any]], said: str) -> Non def test_a_gain_is_judged_by_what_the_definition_reads_not_by_spelling() -> None: - """An accepted declaration refines an unclaimed one when its definition, reading the JSON of the unclaimed declaration, produces the accepted declaration. Two spellings of one configuration count as the same gain, so `refines` is transitive through `==`. A spelling that the definition reads as something else is not a gain.""" - from zarr_metadata.definition import ( - resolve, - ) - from zarr_metadata.v3.definition import ( - CORE_CONTEXT, - ChunkKeyEncodingDefinition, - ) - + """A model refines one whose scope left a name unclaimed when its own scope, reading the other document, reads this model. Two spellings of one configuration count as the same gain, so `refines` is transitive through `==`. A spelling that the definition reads as something else is not a gain.""" nothing = ZarrV3Context() - spelled_out, _ = resolve( - {"name": "default", "configuration": {"separator": "/"}}, - ChunkKeyEncodingDefinition, - CORE_CONTEXT, - ) - bare, _ = resolve({"name": "default"}, ChunkKeyEncodingDefinition, CORE_CONTEXT) - unclaimed, _ = resolve({"name": "default"}, ChunkKeyEncodingDefinition, nothing) + separated = {"name": "default", "configuration": {"separator": "/"}} + spelled_out = ZarrV3ArrayMetadata({**ARRAY, "chunk_key_encoding": separated}) + bare = ZarrV3ArrayMetadata({**ARRAY, "chunk_key_encoding": {"name": "default"}}) + unclaimed = bare.with_context(nothing) assert spelled_out == bare - assert refines(bare, unclaimed) - assert refines(spelled_out, unclaimed) - other, _ = resolve( - {"name": "default", "configuration": {"separator": "."}}, - ChunkKeyEncodingDefinition, - CORE_CONTEXT, - ) - assert not refines(other, unclaimed) + assert bare.refines(unclaimed) + assert spelled_out.refines(unclaimed) + dotted = {"name": "default", "configuration": {"separator": "."}} + other = ZarrV3ArrayMetadata({**ARRAY, "chunk_key_encoding": dotted}) + assert not other.refines(unclaimed) shard = { "name": "sharding_indexed", "configuration": { @@ -442,9 +409,9 @@ def test_a_gain_is_judged_by_what_the_definition_reads_not_by_spelling() -> None "index_location": "end", }, } - read, _ = resolve(shard, CodecDefinition, CORE_CONTEXT) - unread, _ = resolve({**shard, "must_understand": True}, CodecDefinition, nothing) - assert refines(read, unread) + read = _array(shard, CORE_CONTEXT) + unread = _array({**shard, "must_understand": True}, nothing) + assert read.refines(unread) OTHER_GZIP = BytesBytesCodecDefinition(name="gzip", configuration=EmptyConfiguration, size="static") From 37141df89106ea969041bcbb30cd0f2fd5f9fb71 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 18 Nov 2026 12:26:36 +0100 Subject: [PATCH 15/17] test(zarr-metadata): check refinement across scopes and member order for every generated document Carries the tests of zarr-developers/zarr-python#4648 that do not name codec_refines, which this branch does not have. Assisted-by: ClaudeCode:claude-fable-5-2 --- .../tests/model/test_invariants.py | 96 +++++++++++++++++++ .../zarr-metadata/tests/model/test_pair.py | 76 ++++++++++++++- .../tests/v3/test_definitions.py | 29 ++++++ 3 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 packages/zarr-metadata/tests/model/test_invariants.py diff --git a/packages/zarr-metadata/tests/model/test_invariants.py b/packages/zarr-metadata/tests/model/test_invariants.py new file mode 100644 index 0000000000..84e11290ca --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_invariants.py @@ -0,0 +1,96 @@ +"""What holds for every v3 document the strategies generate: how a model compares with the same document read in another scope, or written in another order.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import pytest +from hypothesis import given + +from zarr_metadata.model import ZarrV3ArrayMetadata, ZarrV3GroupMetadata +from zarr_metadata.testing.strategies import ( + array_documents, + consolidated_group_documents, + group_documents, +) +from zarr_metadata.v3.definition import CORE_CONTEXT, ZARR_PYTHON_CONTEXT, ZarrV3Context + + +@pytest.mark.parametrize( + "poorer", + [ + ZarrV3Context(), + CORE_CONTEXT, + ZARR_PYTHON_CONTEXT.copy(update={"data_types": {}}), + ZARR_PYTHON_CONTEXT.copy(update={"codecs": {}}), + ], + ids=["empty", "core", "no-data-types", "no-codecs"], +) +@given(document=array_documents()) +def test_every_array_read_in_a_richer_scope_refines_the_one_a_poorer_scope_read( + poorer: ZarrV3Context, document: Mapping[str, Any] +) -> None: + """`refined_in` returns the model that the richer scope reads, and that model refines the one it came from, whatever the richer scope learns of the chunk each codec receives.""" + low = ZarrV3ArrayMetadata.from_json(document, context=poorer) + high = low.refined_in(ZARR_PYTHON_CONTEXT) + assert high == ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT) + assert high.refines(low) + assert low.refines(low) + + +def _reordered(value: object) -> Any: # noqa: ANN401 + """`value` with the members of every object it holds, at any depth, in the reverse order. The items of an array keep their order.""" + if isinstance(value, Mapping): + return {key: _reordered(value[key]) for key in reversed(value)} + if isinstance(value, (list, tuple)): + return [_reordered(item) for item in value] + return value + + +def _assert_one_model( + model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, + reordered: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, +) -> None: + assert reordered == model + assert hash(reordered) == hash(model) + assert reordered.refines(model) # pyright: ignore[reportArgumentType] + assert model.refines(reordered) # pyright: ignore[reportArgumentType] + + +@given(document=array_documents()) +def test_every_array_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + """The order of the members of an object means nothing, at any depth: in the document, in a configuration, and in a configuration that holds declarations.""" + _assert_one_model( + ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT), + ZarrV3ArrayMetadata.from_json(_reordered(document), context=ZARR_PYTHON_CONTEXT), + ) + + +@given(document=group_documents()) +def test_every_group_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + _assert_one_model( + ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT), + ZarrV3GroupMetadata.from_json(_reordered(document), context=ZARR_PYTHON_CONTEXT), + ) + + +@given(document=consolidated_group_documents()) +def test_every_listing_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + """A group lists the documents of its nodes. Each is read, so the listing is one model whatever order the members of those documents are in.""" + listings = [ + ZarrV3GroupMetadata.from_json(each, context=ZARR_PYTHON_CONTEXT).consolidated() + for each in (document, _reordered(document)) + ] + assert listings[0] is not None + assert listings[1] is not None + assert listings[1] == listings[0] + assert hash(listings[1]) == hash(listings[0]) + assert listings[1].refines(listings[0]) + assert listings[0].refines(listings[1]) diff --git a/packages/zarr-metadata/tests/model/test_pair.py b/packages/zarr-metadata/tests/model/test_pair.py index 998a45e3f2..0cc7df2335 100644 --- a/packages/zarr-metadata/tests/model/test_pair.py +++ b/packages/zarr-metadata/tests/model/test_pair.py @@ -265,6 +265,40 @@ def test_error_replace_refuses_a_document_with_a_problem() -> None: ZarrV3ArrayMetadata(ARRAY).replace(dimension_names=("x", "y")) +def _core_without_codecs(*names: str) -> ZarrV3Context: + """`CORE_CONTEXT` without the codecs it files under `names`.""" + kept = {name: codec for name, codec in CORE_CONTEXT.codecs.items() if name not in names} + return CORE_CONTEXT.copy(update={"codecs": kept}) + + +def _core_without_data_types(*names: str) -> ZarrV3Context: + """`CORE_CONTEXT` without the data types it files under `names`.""" + kept = {name: each for name, each in CORE_CONTEXT.data_types.items() if name not in names} + return CORE_CONTEXT.copy(update={"data_types": kept}) + + +LITTLE: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "little"}} +ENDIAN: dict[str, Any] = {**ARRAY, "codecs": [LITTLE]} +"""A `uint8` array whose `bytes` codec writes an `endian`, which means nothing for values of one byte. A scope that reads both the codec and the data type compares the codec without it.""" +TRANSPOSED: dict[str, Any] = { + **ARRAY, + "codecs": [{"name": "transpose", "configuration": {"order": [0]}}, LITTLE], +} +SHARDED_ENDIAN: dict[str, Any] = { + **ARRAY, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [1], + "codecs": [LITTLE], + "index_codecs": [LITTLE, "crc32c"], + }, + } + ], +} + + @pytest.mark.parametrize( ("model", "context", "gained"), [ @@ -272,8 +306,48 @@ def test_error_replace_refuses_a_document_with_a_problem() -> None: (ZarrV3ArrayMetadata(ARRAY, context=CORE_CONTEXT), CORE_AND_EXTENSIONS_CONTEXT, False), (ZarrV3ArrayMetadata(FLOAT, context=ZarrV3Context()), CORE_CONTEXT, True), (ZarrV3ArrayMetadata(SHARDED, context=ZarrV3Context()), CORE_CONTEXT, True), + # What the new scope claims tells it more of the chunk a codec + # receives, so it compares that codec in a simpler spelling than + # the one the document writes. + (ZarrV3ArrayMetadata(ENDIAN, context=ZarrV3Context()), CORE_CONTEXT, True), + ( + ZarrV3ArrayMetadata(ENDIAN, context=_core_without_codecs("bytes")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(ENDIAN, context=_core_without_data_types("uint8")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(TRANSPOSED, context=_core_without_codecs("transpose")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(SHARDED_ENDIAN, context=_core_without_codecs("sharding_indexed")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(SHARDED_ENDIAN, context=_core_without_codecs("bytes")), + CORE_CONTEXT, + True, + ), + ], + ids=[ + "gain", + "nothing-to-gain", + "gain-data-type-with-fill-value", + "gain-of-a-shard", + "gain-of-everything-with-an-endian-that-means-nothing", + "gain-of-a-codec-simplified-for-its-chunk", + "gain-of-the-data-type-a-codec-is-simplified-by", + "gain-of-a-codec-before-one-simplified-for-its-chunk", + "gain-of-a-shard-whose-codecs-are-simplified", + "gain-of-a-codec-in-a-shard", ], - ids=["gain", "nothing-to-gain", "gain-data-type-with-fill-value", "gain-of-a-shard"], ) def test_refined_in_moves_a_model_up_the_order( model: ZarrV3ArrayMetadata, context: ZarrV3Context, gained: bool diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index bcc2f3c9be..adfa8ecc35 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -1185,6 +1185,14 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: CRC32C: Final = {"name": "crc32c"} +def _shard_of(codec: JSONValue, *, index_first: bool = False, chunk_shape: int = 2) -> JSONValue: + """A shard that holds `codec`, with its `index_codecs` written before its `codecs` when `index_first`.""" + members: dict[str, JSONValue] = {"codecs": [codec], "index_codecs": [LE, CRC32C]} + if index_first: + members = dict(reversed(members.items())) + return {"name": "sharding_indexed", "configuration": {"chunk_shape": [chunk_shape], **members}} + + @pytest.mark.parametrize( ("one", "other", "kind", "equal"), [ @@ -1233,6 +1241,12 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: CodecDefinition, True, ), + ( + _shard_of(_shard_of(LE)), + _shard_of(_shard_of(LE, index_first=True), index_first=True), + CodecDefinition, + True, + ), ( {"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [LE, CRC32C]}}, {"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [CRC32C, LE]}}, @@ -1321,6 +1335,7 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: "unclaimed-another-configuration", "another-name", "members-holding-declarations-in-another-order", + "members-holding-declarations-in-another-order-two-deep", "declarations-in-another-order", "members-in-another-order", "blosc-typesize-noshuffle", @@ -1348,6 +1363,20 @@ def test_two_declarations_are_equal_when_they_read_the_same( assert hash(first) == hash(second) +def test_two_refused_declarations_are_equal_whatever_order_their_members_are_in() -> None: + """A refused declaration is compared by what was written and by the declarations it holds, in the order of their locations, not of the members that hold them.""" + # A shard of inner chunks with no elements is refused, and still holds its codecs. + first, _ = resolve(_shard_of(LE, chunk_shape=0), CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) + second, _ = resolve( + _shard_of(LE, chunk_shape=0, index_first=True), CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT + ) + assert isinstance(first, RefusedDeclaration) + assert isinstance(second, RefusedDeclaration) + assert list(first.nested) != list(second.nested) + assert first == second + assert hash(first) == hash(second) + + def test_a_declaration_copied_or_pickled_is_read_by_a_definition_equal_to_its_own() -> None: declaration, _ = resolve( {"name": "gzip", "configuration": {"level": 1}}, CodecDefinition, CORE_CONTEXT From 6f86bac2eaeaa24e063ccbb941c516d1a0c765e8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 18 Nov 2026 12:32:58 +0100 Subject: [PATCH 16/17] docs(zarr-metadata): add the changelog fragment Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/465.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/zarr-metadata/changes/465.bugfix.md diff --git a/packages/zarr-metadata/changes/465.bugfix.md b/packages/zarr-metadata/changes/465.bugfix.md new file mode 100644 index 0000000000..aa3477f59f --- /dev/null +++ b/packages/zarr-metadata/changes/465.bugfix.md @@ -0,0 +1 @@ +`refines` on every model now reads the other model's document in this model's scope and compares the two, so a document read in a richer scope refines the same document read in a poorer one, whatever the richer scope learns of the chunk each codec receives. Before, `refined_in` of a `uint8` array whose `bytes` codec writes an `endian`, read in a scope that claims no `bytes` codec, did not refine the model it came from. A model no longer refines one whose document its own scope reads with a problem: a `gzip` codec with a configuration key its definition does not declare, read in a scope that claims no `gzip`, is not refined by the model of the same codec without that key. From 94fa7285c4cabe4f9132d46aa0e50b183af3ca95 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 18 Nov 2026 12:48:11 +0100 Subject: [PATCH 17/17] refactor(zarr-metadata): simplify a codec for its chunk inside the walk that writes its simplest spelling canonical_declaration writes a declaration in its simplest spelling and lets a caller that knows more take part: a pipeline gives the spelling of each codec it placed, and a step that simplifies the codec's own members for the chunk it receives. codec_for_chunk returns that spelling as JSON, so nothing builds a second declaration to read again, and respelled is removed. canonical is called from one function. Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/4648.bugfix.md | 2 +- packages/zarr-metadata/docs/models.md | 8 +- .../zarr_metadata/definition/_definition.py | 143 ++++++++------ .../src/zarr_metadata/model/_array_v3.py | 16 +- .../src/zarr_metadata/v3/_definition.py | 13 +- .../src/zarr_metadata/v3/_pipeline.py | 187 ++++++++---------- .../src/zarr_metadata/v3/definition.py | 6 +- .../zarr-metadata/tests/model/test_array.py | 2 +- .../tests/v3/test_definitions.py | 86 +------- .../zarr-metadata/tests/v3/test_pipelines.py | 70 +++++-- 10 files changed, 247 insertions(+), 286 deletions(-) diff --git a/packages/zarr-metadata/changes/4648.bugfix.md b/packages/zarr-metadata/changes/4648.bugfix.md index aa3477f59f..95256fe43d 100644 --- a/packages/zarr-metadata/changes/4648.bugfix.md +++ b/packages/zarr-metadata/changes/4648.bugfix.md @@ -1 +1 @@ -`refines` on every model now reads the other model's document in this model's scope and compares the two, so a document read in a richer scope refines the same document read in a poorer one, whatever the richer scope learns of the chunk each codec receives. Before, `refined_in` of a `uint8` array whose `bytes` codec writes an `endian`, read in a scope that claims no `bytes` codec, did not refine the model it came from. A model no longer refines one whose document its own scope reads with a problem: a `gzip` codec with a configuration key its definition does not declare, read in a scope that claims no `gzip`, is not refined by the model of the same codec without that key. +`refines` on every model now reads the other model's document in this model's scope and compares the two, so a document read in a richer scope refines the same document read in a poorer one, whatever the richer scope learns of the chunk each codec receives. Before, `refined_in` of a `uint8` array whose `bytes` codec writes an `endian`, read in a scope that claims no `bytes` codec, did not refine the model it came from. A model no longer refines one whose document its own scope reads with a problem: a `gzip` codec with a configuration key its definition does not declare, read in a scope that claims no `gzip`, is not refined by the model of the same codec without that key. `codec_for_chunk` now writes the codec as JSON in its simplest spelling for the chunk, as one step of the walk that `canonical_of` makes, and returns `None` for a codec with no simplest spelling. A codec's `chunk_canonical` receives each nested declaration in its simplest spelling. diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index e1f1d04402..0e4b631d7b 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -228,9 +228,9 @@ These are each declaration, and the fill value read against its data type. So in a `uint8` array, a `bytes` codec is the same codec with either `endian`, or with none. The codec is compared for the chunk it receives: after a `cast_value` to `float32`, the two `endian` values - are two codecs. `codec_for_chunk` returns a codec in the spelling that - is compared, which `canonical_of` cannot give, since it reads the - codec alone. + are two codecs. `codec_for_chunk` writes a codec in the spelling that + is compared, as JSON, which `canonical_of` cannot give, since it reads + the codec alone. A v2 document's [dtype](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#data-type-encoding), @@ -308,7 +308,7 @@ The canonical document is written from the same parts that `==` compares: - Each declaration is written as `canonical_of` writes it. - Each v3 codec is written for the chunk it receives, as `codec_for_chunk` - returns it. Above, each `uint8` value takes one byte, so the `endian` + writes it. Above, each `uint8` value takes one byte, so the `endian` says nothing. - The fill value is written as `canonical_fill_value` writes it for the data type. diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 3bd55e8723..d9b6c1f246 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1200,16 +1200,13 @@ def declaration_key(declaration: ResolvedDeclaration[Any]) -> tuple[object, ...] configuration: JSONValue = dict(declaration.configuration) # Each nested declaration is compared by its own key, so its location is # cleared before the definition's `canonical` sees the rest, in the - # same order as `_canonical_declaration`. `canonical` simplifies only the + # same order as `canonical_declaration`. `canonical` simplifies only the # definition's own members. for loc in declaration.nested: configuration = _replaced(configuration, loc, None) view = read_only(cast("Mapping[str, JSONValue]", configuration)) - spelled = asked( - definition, - "canonical", - lambda: json_text(cast("JSONValue", dict(definition.canonical(view)))), - ) + _, own = _spelled(definition, view) + spelled = asked(definition, "canonical", lambda: json_text(cast("JSONValue", own))) return ("read", definition, spelled, _nested_key(declaration.nested)) if isinstance(declaration, UnclaimedDeclaration): return ( @@ -1272,42 +1269,6 @@ def written_json(declaration: AcceptedDeclaration[Any] | UnclaimedDeclaration) - return kind._envelope_json(declaration.name, declaration.configuration) -def respelled( - declaration: AcceptedDeclaration[D], - configuration: Mapping[str, JSONValue], - nested: Nested, -) -> AcceptedDeclaration[D]: - """`declaration` with `configuration` as its configuration, and each declaration in `nested` in place of the nested declaration at its location. - - This builds the declaration that another spelling of `declaration` - reads as, without reading it again. The caller answers for the - spelling: `configuration` must be one that the definition allows and - that means the same thing, and it must keep each nested declaration - where it is. `nested` holds only the nested declarations to replace, - each one accepted or unclaimed, and each is written into the - configuration as its `to_json` writes it. A location in `nested` that - is not the location of a nested declaration of `declaration`, or that - does not lead into `configuration`, raises `TypeError`. The result is - written as `to_json` writes it, since no document wrote it. - """ - spelled: JSONValue = dict(configuration) - for loc, inner in nested.items(): - if loc not in declaration.nested or item_at(spelled, loc) is UNSET: - msg = f"{loc!r} is not the location of a nested declaration in {configuration!r}" - raise TypeError(msg) - spelled = _replaced(spelled, loc, written_json(held(inner))) - kind = declaration.read_as - name = declaration.name - carried = kind._filed_under(name)[1] is not None - return AcceptedDeclaration( - json=kind._envelope_json(name, {} if carried else cast("Mapping[str, JSONValue]", spelled)), - name=name, - definition=declaration.definition, - configuration=cast("Mapping[str, JSONValue]", spelled), - nested={**declaration.nested, **nested}, - ) - - def declaration_locations(definition: Definition[Any], configuration: JSONValue) -> tuple[Loc, ...]: """The location of each declaration that `configuration` holds, where `definition`'s TypedDict types a member with a declaration alias, in order. `configuration` is JSON with arrays as tuples.""" return tuple(found.loc for found in _typed(definition.configuration, configuration, ())[2]) @@ -1892,34 +1853,84 @@ def canonical_of( def _simplest(declaration: ResolvedDeclaration[Any]) -> JSONValue | None: """A declaration in its simplest spelling; None when the declaration or one of its nested declarations was refused, because a refused declaration has no simplest spelling.""" if isinstance(declaration, AcceptedDeclaration): - return _canonical_declaration(declaration) + return canonical_declaration(declaration) if isinstance(declaration, UnclaimedDeclaration): return declaration.to_json() return None -def _canonical_declaration(resolved: AcceptedDeclaration[Any]) -> JSONValue | None: - """An accepted declaration in its simplest equivalent spelling: nested declarations first, then its own members. None when a nested declaration was refused.""" +@dataclass(frozen=True, slots=True) +class Further: + """A simplification of a definition's own members by something `canonical` cannot know, such as the chunk a codec receives. + + `ask` receives the configuration in its simplest spelling so far, + read-only: its own members as `canonical` spells them, and each + nested declaration in its own simplest spelling. It returns a simpler + configuration, as JSON with arrays as tuples, or None when it + simplifies nothing. It answers for its result: it raises when the + result does not hold. `what` names the function of the definition + that `ask` calls, for the error raised when its result is not the + simplest spelling. + """ + + what: str + ask: Callable[[Mapping[str, JSONValue]], dict[str, JSONValue] | None] + + +def canonical_declaration( + resolved: AcceptedDeclaration[Any], + spelled: Mapping[Loc, JSONValue | None] | None = None, + further: Further | None = None, +) -> JSONValue | None: + """An accepted declaration in its simplest equivalent spelling: nested declarations first, then its own members. None when a nested declaration has no simplest spelling. + + This is the one walk that writes a declaration in its simplest + spelling. A caller that knows more than the declaration says, as a + pipeline knows the chunk each codec receives, takes part in it + instead of walking the declaration again. `spelled` gives the + simplest spelling of the nested declarations the caller wrote + itself, by location, with None for one that has none. Each other + nested declaration is written as this function writes it. `further` + simplifies the declaration's own members once more, after + `canonical`. What it gives is spelled by `canonical` again, and must + then be the simplest spelling: asked about it, `further` must + simplify nothing, or this raises `ValueError`, which indicates a bug + in the definition. + """ definition, name = resolved.definition, resolved.name configuration: JSONValue = dict(resolved.configuration) for loc, inner in resolved.nested.items(): - simplest = _simplest(inner) + simplest = spelled[loc] if spelled is not None and loc in spelled else _simplest(inner) if simplest is None: return None configuration = _replaced(configuration, loc, simplest) - # Every function of a definition receives this read-only view. The - # result of `canonical` (the view itself when nothing is simplified) is - # converted to a dict. - view = read_only(cast("Mapping[str, JSONValue]", configuration)) + simplified = _spelled_and_checked(definition, cast("Mapping[str, JSONValue]", configuration)) + if further is not None: + given = further.ask(read_only(simplified)) + if given is not None: + simplified = _spelled_and_checked(definition, given) + left = further.ask(read_only(simplified)) + if left is not None: + # The simplest spelling has nothing left to simplify. Otherwise + # a document written from it would read as another declaration. + msg = ( + f"{definition.name!r}: its {further.what} gave {given!r}, which is not the " + f"simplest spelling: asked again, it gave {left!r}" + ) + raise ValueError(msg) + carrying = definition._carrying_name(simplified) + if carrying is not None: + return carrying + return resolved.read_as._envelope_json(name, simplified) - def spelled() -> tuple[bool, dict[str, JSONValue]]: - given = cast("Mapping[str, JSONValue]", definition.canonical(view)) - return given is view, dict(given) - kept, simplified = asked(definition, "canonical", spelled) +def _spelled_and_checked( + definition: Definition[Any], configuration: Mapping[str, JSONValue] +) -> dict[str, JSONValue]: + """`configuration` as `definition`'s `canonical` spells it. A result that the definition does not allow raises `ValueError`, which indicates a bug in the definition.""" + kept, simplified = _spelled(definition, read_only(configuration)) # A `canonical` that returns what it received simplified nothing, so - # there is nothing of its own to check: the configuration was read, and - # each nested declaration was checked in its own simplest spelling. + # there is nothing of its own to check. refused = () if kept else definition.read_configuration(simplified)[1] if len(refused) != 0: msg = ( @@ -1927,10 +1938,19 @@ def spelled() -> tuple[bool, dict[str, JSONValue]]: f"{list(refused)!r}" ) raise ValueError(msg) - carrying = definition._carrying_name(simplified) - if carrying is not None: - return carrying - return resolved.read_as._envelope_json(name, simplified) + return simplified + + +def _spelled( + definition: Definition[Any], view: Mapping[str, JSONValue] +) -> tuple[bool, dict[str, JSONValue]]: + """Whether `definition`'s `canonical` returned `view` itself, and what it returned, as a dict. This is the one place that calls `canonical`. Every function of a definition receives a read-only view.""" + + def spelled() -> tuple[bool, dict[str, JSONValue]]: + given = cast("Mapping[str, JSONValue]", definition.canonical(view)) + return given is view, dict(given) + + return asked(definition, "canonical", spelled) def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: @@ -1951,6 +1971,7 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "C", "Context", "Definition", + "Further", "Nested", "Problems", "RefusedDeclaration", @@ -1959,6 +1980,7 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "WithFillValue", "as_kind", "asked", + "canonical_declaration", "canonical_fill_value", "canonical_of", "canonicalize", @@ -1982,7 +2004,6 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "read_only", "resolve", "resolve_refined", - "respelled", "ruled", "spelled_canonically", "unchanged", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index d4e23eba54..e257e8872c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -738,7 +738,7 @@ def canonicalize(self) -> ZarrV3ArrayMetadata: are a key for what the document means. Each declaration is written as `canonical_of` writes it, each - codec as `codec_for_chunk` returns it for the chunk it receives, + codec as `codec_for_chunk` writes it for the chunk it receives, and the fill value as `canonical_fill_value` writes it for the data type. `attributes` and `storage_transformers` are left out when empty, because a document without them means the same. @@ -942,7 +942,7 @@ def canonical_array_document_v3( "chunk_grid": simplest(reading.chunk_grid), "chunk_key_encoding": simplest(reading.chunk_key_encoding), "fill_value": spelled_canonically(data_type, members.fill_value), - "codecs": tuple(simplest(codec) for codec in codecs_for_chunks(reading)), + "codecs": tuple(_codec_spelled(stage) for stage in reading.pipeline), "storage_transformers": tuple(simplest(entry) for entry in reading.storage_transformers), **simplest_members(reading.extra_fields), } @@ -951,11 +951,13 @@ def canonical_array_document_v3( ) -def codecs_for_chunks( - reading: ZarrV3ArrayMetadataReading, -) -> tuple[AcceptedDeclaration[CodecDefinition[Any]] | UnclaimedDeclaration, ...]: - """The codecs of `reading`, a read that found no problems, in pipeline order: each as `codec_for_chunk` returns it for the chunk it receives.""" - return tuple(held(codec_for_chunk(stage)) for stage in reading.pipeline) +def _codec_spelled(stage: Stage) -> JSONValue: + """The codec of `stage`, from a read that found no problems, as `codec_for_chunk` writes it. Raises `TypeError` for a codec with no simplest spelling, which such a read cannot produce.""" + spelled = codec_for_chunk(stage) + if spelled is None: + msg = f"expected a codec a read found nothing wrong with, got {stage.codec!r}" + raise TypeError(msg) + return spelled def read_array_metadata_v3( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index 34525f27c9..715d74ab3b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -432,17 +432,18 @@ class ArrayCodecDefinition(CodecDefinition[C]): simplest configuration that means the same for that chunk. It is called only for a codec that the chunk rules found no problem with. It simplifies only the codec's own members, and it must keep each - nested declaration as it is, because each codec of a pipeline the - codec contains is simplified for the chunk it receives in turn. What - does not depend on the chunk is left to `canonical`: `chunk_canonical` - receives the configuration as `canonical` spells it, and `canonical` - simplifies the result as it simplifies any configuration. The result + nested declaration as it is given, because each codec of a pipeline + the codec contains is simplified for the chunk it receives first. + What does not depend on the chunk is left to `canonical`: + `chunk_canonical` receives the configuration as `canonical` spells + it, with each nested declaration in its simplest spelling, and + `canonical` simplifies the result as it simplifies any configuration. The result must be the simplest spelling, so `chunk_canonical` must change nothing when it is asked about its own result. A chunk rule checks only the parts of a chunk that are known, and `chunk_canonical` simplifies only by them: a chunk with nothing known about it, `Chunk()`, leaves every configuration as it is. `codec_for_chunk` - returns the codec of a `Stage` in that spelling. + writes the codec of a `Stage` in that spelling. """ chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index db1bf1ff9b..94789e1647 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -35,6 +35,7 @@ from zarr_metadata._json import ( ValidationProblem, + copied, is_object, is_tuple, json_text, @@ -44,13 +45,15 @@ from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._definition import ( AcceptedDeclaration, + Further, ResolvedDeclaration, asked, + canonical_declaration, + canonical_of, declaration_locations, in_document, item_at, read_only, - respelled, ruled, ) from zarr_metadata.v3._definition import ( @@ -339,91 +342,114 @@ def _handed_on( return given -def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: - """The codec of `stage`, in the simplest spelling that means the same for the chunk it receives. +def codec_for_chunk(stage: Stage) -> JSONValue | None: + """The codec of `stage` in its simplest equivalent spelling for the chunk it receives, as plain JSON; None when it has none. A codec's own `canonical` reduces the spellings that mean the same thing wherever the codec is used. Some spellings mean the same thing only for some chunks. For example, a `bytes` codec with and without `endian` is one codec for values that take one byte, and two for values that take several. So a codec is compared, and written in its - simplest spelling, as the codec this function returns: first each - codec of each pipeline it contains, for the chunk that codec - receives, and then its own configuration, as its definition's - `chunk_canonical` simplifies it for the chunk in `stage`. - `chunk_canonical` receives the configuration as the definition's - `canonical` spells it, so two spellings that `canonical` reduces to - one are simplified for the chunk in the same way. - - A codec the scope did not read is returned as it is. A codec's own - configuration is kept as it is when the codec receives bytes, and - when its chunk rules find a problem with it, because a codec with a - problem has no simpler spelling. When nothing is simplified, the - result is the codec itself. Otherwise it is a new declaration, read - by the same definition. `canonical_of` writes it in its simplest - spelling, and `==` compares it. + simplest spelling, as this function writes it: first each codec of + each pipeline it contains, for the chunk that codec receives, then + its own members as its definition's `canonical` spells them, and + then as its definition's `chunk_canonical` simplifies them for the + chunk in `stage`. This is the walk that `canonical_of` makes for any + declaration, with the chunks taking part in it. + + `stage` comes from a read that found no problems, as a model's + pipeline does. A codec the scope left unclaimed is written as it + was, in the shortest form every reader accepts. A codec the scope + refused has no simplest spelling, and neither has a codec that holds + one. A codec's own members are not simplified for the chunk when the + codec receives bytes, or when its chunk rules find a problem with + it. `chunk_canonical` is the extension author's code, so its result is checked: the definition must allow it, the chunk rules must find no problem with it, it must hold the nested declarations the codec - holds, each as it was, and `chunk_canonical` must have nothing left - to simplify in it. A result that fails a check raises `ValueError`, - which indicates a bug in the definition, not in the codec. A problem - inside a nested declaration is the document's, and is not held - against the result. + holds, each as it was given, and `chunk_canonical` must have nothing + left to simplify in it. A result that fails a check raises + `ValueError`, which indicates a bug in the definition, not in the + codec. """ + spelled = _spelled_for_chunk(stage) + return None if spelled is None else copied(spelled) + + +def _spelled_for_chunk(stage: Stage) -> JSONValue | None: + """What `codec_for_chunk` returns, sharing what it can with the declaration.""" codec = stage.codec if not isinstance(codec, AcceptedDeclaration): - return codec - inner: dict[Loc, ResolvedDeclaration[Any]] = {} - for member, stages in stage.inner.items(): - for index, each in enumerate(stages): - simpler = codec_for_chunk(each) - if simpler is not each.codec: - inner[(member, index)] = simpler - configuration = _for_chunk(codec, stage.incoming) - if configuration is None and len(inner) == 0: - return codec - return respelled(codec, codec.configuration if configuration is None else configuration, inner) - - -def _for_chunk( + return canonical_of(codec, ()) + inner: dict[Loc, JSONValue | None] = { + (member, index): _spelled_for_chunk(each) + for member, stages in stage.inner.items() + for index, each in enumerate(stages) + } + return canonical_declaration(codec, inner, _further(codec, stage.incoming)) + + +def _further( codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None -) -> Mapping[str, JSONValue] | None: - """The configuration of `codec` as its definition's `chunk_canonical` simplifies it for `chunk`, checked; None when nothing is simplified. - - `chunk_canonical` receives the configuration as the definition's - `canonical` spells it. So every spelling that `canonical` reduces to - one is simplified for the chunk in the same way, and `chunk_canonical` - does not have to know each of them. None is returned when - `chunk_canonical` changes nothing in that spelling, and the codec - then stays as written, for `canonical` to simplify as it simplifies - any configuration. - """ +) -> Further | None: + """How the own members of `codec` are simplified for `chunk`, after its `canonical`; None when they are not: the codec receives bytes, its `chunk_canonical` is the default, or its chunk rules find a problem with it, because a codec with a problem has no simpler spelling.""" definition = codec.definition if chunk is None or not isinstance(definition, ArrayCodecDefinition): return None if definition.chunk_canonical is unchanged_for_chunk: return None - configuration, nested = codec.configuration, codec.nested - if len(_chunk_problems(definition, configuration, nested, chunk, ())) != 0: + nested = codec.nested + if len(_chunk_problems(definition, codec.configuration, nested, chunk, ())) != 0: + return None + return Further( + "chunk_canonical", + lambda configuration: _asked_for_chunk(definition, configuration, nested, chunk), + ) + + +def _asked_for_chunk( + definition: ArrayCodecDefinition[Any], + configuration: Mapping[str, JSONValue], + nested: Nested, + chunk: Chunk, +) -> dict[str, JSONValue] | None: + """What `definition`'s `chunk_canonical` gives for `configuration`, checked, as JSON with arrays as tuples; None when it changes nothing. + + `configuration` is the read-only configuration in its simplest + spelling so far: the codec's own members as `canonical` spells them, + and each nested declaration in its simplest spelling. So every + spelling that `canonical` reduces to one is simplified for the chunk + in the same way, and `chunk_canonical` does not have to know each of + them. + """ + given: object = asked( + definition, + "chunk_canonical", + lambda: cast("object", definition.chunk_canonical(configuration, nested, chunk)), + ) + if given is configuration: return None - simplified = _asked_for_chunk(definition, configuration, nested, chunk) - if simplified is None: + # The result is the extension author's value, so it is read as any JSON + # is: an array may be a list, and what is not JSON is refused here, + # naming the definition. + simplified, not_json = refine_user_data(dict(given)) if is_object(given) else (None, ()) + if not isinstance(simplified, dict) or len(not_json) != 0: + msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}" + raise TypeError(msg) + if json_text(simplified) == json_text(cast("JSONValue", dict(configuration))): return None _, refused = definition.read_configuration(simplified) held_at = frozenset(declaration_locations(definition, simplified)) found = ( - # A problem inside a nested declaration is the document's: the - # declaration is checked to be as it was written. - *(problem for problem in refused if not _inside(problem.loc, nested)), + *refused, *_chunk_problems(definition, simplified, nested, chunk, ()), *( ValidationProblem( loc, "expected the nested declaration to stay as it is", "invalid_value" ) for loc in nested - if _text_at(simplified, loc) != _text_at(cast("JSONValue", configuration), loc) + if _text_at(simplified, loc) != _text_at(cast("JSONValue", dict(configuration)), loc) ), *( ValidationProblem( @@ -438,55 +464,6 @@ def _for_chunk( f"{list(found)!r}" ) raise ValueError(msg) - # The simplest spelling has nothing left to simplify. Otherwise a - # document written from it would read as another codec. - further = _asked_for_chunk(definition, simplified, nested, chunk) - if further is not None: - msg = ( - f"{definition.name!r}: its chunk_canonical gave {simplified!r}, which is not the " - f"simplest spelling: asked again, it gave {further!r}" - ) - raise ValueError(msg) - return simplified - - -def _inside(loc: Loc, nested: Nested) -> bool: - """Whether `loc` is at or under the location of a declaration in `nested`.""" - return any(loc[: len(held_at)] == held_at for held_at in nested) - - -def _asked_for_chunk( - definition: ArrayCodecDefinition[Any], - configuration: Mapping[str, JSONValue], - nested: Nested, - chunk: Chunk, -) -> dict[str, JSONValue] | None: - """What `definition`'s `chunk_canonical` gives for `configuration` as its `canonical` spells it, as JSON with arrays as tuples; None when it changes nothing in that spelling.""" - spelled = read_only( - asked( - definition, - "canonical", - lambda: dict( - cast("Mapping[str, JSONValue]", definition.canonical(read_only(configuration))) - ), - ) - ) - given: object = asked( - definition, - "chunk_canonical", - lambda: cast("object", definition.chunk_canonical(spelled, nested, chunk)), - ) - if given is spelled: - return None - # The result is the extension author's value, so it is read as any JSON - # is: an array may be a list, and what is not JSON is refused here, - # naming the definition. `read_configuration` checks the rest. - simplified, not_json = refine_user_data(dict(given)) if is_object(given) else (None, ()) - if not isinstance(simplified, dict) or len(not_json) != 0: - msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}" - raise TypeError(msg) - if json_text(simplified) == json_text(cast("JSONValue", dict(spelled))): - return None return simplified diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py index 9c20fac4a5..9169e23246 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -94,9 +94,9 @@ for another, which `canonical` cannot tell, since it receives the configuration alone. Such a codec declares its simplest spelling for a given chunk with `chunk_canonical`. For example, `bytes` drops an -`endian` when each value takes one byte. `codec_for_chunk` returns the -codec of a `Stage` in that spelling, with each codec of the pipelines -it contains simplified for the chunk that one receives. An array model +`endian` when each value takes one byte. `codec_for_chunk` writes the +codec of a `Stage` in that spelling, as JSON, with each codec of the +pipelines it contains simplified for the chunk that one receives. An array model compares its codecs this way. The reader does not guess. After a codec the scope did not read, or after a codec that does not declare its output, the next codec receives diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 2d2ae6cf27..406d3e2d8a 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -845,7 +845,7 @@ def _shard_of(*codecs: object) -> dict[str, object]: def test_two_models_are_one_array_when_each_codec_means_the_same_for_its_chunk( data_type: str, left: object, right: object, same: bool ) -> None: - """A codec is compared for the chunk it receives, as `codec_for_chunk` returns it: the `endian` of a `bytes` codec tells two arrays apart only where a value takes several bytes. `refines` agrees with `==`, and `to_json` writes each codec as given.""" + """A codec is compared for the chunk it receives, as `codec_for_chunk` writes it: the `endian` of a `bytes` codec tells two arrays apart only where a value takes several bytes. `refines` agrees with `==`, and `to_json` writes each codec as given.""" document = ZarrV3ArrayMetadata.create_default(shape=(2,), data_type=data_type).to_json() models = [ ZarrV3ArrayMetadata.from_json({**document, "codecs": codecs}) for codecs in (left, right) diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index adfa8ecc35..21e4aafbe6 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -21,7 +21,7 @@ JSONValue, ValidationProblem, ) -from zarr_metadata._json import json_text, refine_json +from zarr_metadata._json import refine_json from zarr_metadata.definition import ( AcceptedDeclaration, Context, @@ -34,7 +34,7 @@ configuration_of, resolve, ) -from zarr_metadata.definition._definition import item_at, resolve_refined, respelled +from zarr_metadata.definition._definition import item_at, resolve_refined from zarr_metadata.model import ( UNSET, ZarrV3ArrayMetadata, @@ -1855,88 +1855,6 @@ def test_error_resolve_refuses_a_kind_of_the_other_format_whatever_the_declarati resolve(declaration, kind, context) -def _accepted(declaration: JSONValue, kind: type[Definition[Any]]) -> AcceptedDeclaration[Any]: - read, problems = resolve(declaration, kind, CORE_AND_EXTENSIONS_CONTEXT) - assert isinstance(read, AcceptedDeclaration) - assert problems == () - return read - - -def _sharding(**members: JSONValue) -> JSONValue: - return {"name": "sharding_indexed", "configuration": {"chunk_shape": [2], **members}} - - -@pytest.mark.parametrize( - ("declaration", "kind", "configuration", "nested", "written"), - [ - # A member of its own left out, and a nested declaration replaced. - ( - _sharding(codecs=[LE], index_codecs=[LE, CRC32C], index_location="end"), - CodecDefinition, - {"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, - {("codecs", 0): {"name": "bytes"}}, - _sharding(codecs=[{"name": "bytes"}], index_codecs=[LE, CRC32C]), - ), - # Nothing nested to replace. - ( - {"name": "gzip", "configuration": {"level": 1}}, - CodecDefinition, - {"level": 5}, - {}, - {"name": "gzip", "configuration": {"level": 5}}, - ), - # A name that carries its configuration is written alone. - ("r16", DataTypeDefinition, {"bits": 16}, {}, "r16"), - ], - ids=["shard", "gzip", "name-carries-configuration"], -) -def test_respelled_is_the_declaration_that_the_other_spelling_reads_as( - declaration: JSONValue, - kind: type[Definition[Any]], - configuration: JSONValue, - nested: Mapping[tuple[str | int, ...], JSONValue], - written: JSONValue, -) -> None: - """`respelled` returns what a read of the other spelling returns: the same JSON, name, configuration, nested declarations and key. The declaration it was given is left as it was.""" - read = _accepted(declaration, kind) - before = json_text(read.to_json()) - refined = cast("Mapping[str, JSONValue]", refine_json(configuration)[0]) - replacements = {loc: _accepted(inner, CodecDefinition) for loc, inner in nested.items()} - result = respelled(read, refined, replacements) - again = _accepted(written, kind) - assert json_text(result.json) == json_text(again.json) - assert json_text(result.to_json()) == json_text(again.to_json()) - assert result.name == again.name - assert json_text(cast("JSONValue", result.configuration)) == json_text( - cast("JSONValue", again.configuration) - ) - assert dict(result.nested) == dict(again.nested) - assert result == again - assert hash(result) == hash(again) - assert json_text(read.to_json()) == before - - -@pytest.mark.parametrize( - ("configuration", "loc"), - [ - ({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ("codecs", 5)), - ({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ("codecs", -1)), - ({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ("nope", 0)), - ({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ()), - # A location of a nested declaration that the new configuration does not have. - ({"chunk_shape": [2], "codecs": [], "index_codecs": [LE, CRC32C]}, ("codecs", 0)), - ], - ids=["past-the-end", "negative", "no-such-member", "the-configuration", "left-out"], -) -def test_error_respelled_at_a_location_that_holds_no_nested_declaration( - configuration: JSONValue, loc: tuple[str | int, ...] -) -> None: - read = _accepted(_sharding(codecs=[LE], index_codecs=[LE, CRC32C]), CodecDefinition) - refined = cast("Mapping[str, JSONValue]", refine_json(configuration)[0]) - with pytest.raises(TypeError, match="is not the location of a nested declaration"): - respelled(read, refined, {loc: _accepted({"name": "bytes"}, CodecDefinition)}) - - @pytest.mark.parametrize( ("path", "item"), [ diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index 0eda1926b6..56d1358822 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -31,6 +31,10 @@ read_pipeline, ) from zarr_metadata.v3.codec.bytes import BYTES_CODEC, BytesCodecConfiguration +from zarr_metadata.v3.codec.sharding_indexed import ( + SHARDING_INDEXED_CODEC, + ShardingIndexedCodecConfiguration, +) from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, ArrayArrayCodecDefinition, @@ -613,7 +617,7 @@ def _simplest(codecs: list[JSONValue], chunk: Chunk) -> list[JSONValue]: """Each codec of a pipeline that receives `chunk`, in its simplest spelling for the chunk it receives.""" stages, problems = _read(codecs, chunk) assert problems == () - return [canonical_of(codec_for_chunk(stage), ()) for stage in stages] + return [codec_for_chunk(stage) for stage in stages] @pytest.mark.parametrize( @@ -653,26 +657,31 @@ def _simplest(codecs: list[JSONValue], chunk: Chunk) -> list[JSONValue]: def test_a_codec_is_written_in_the_simplest_spelling_for_the_chunk_it_receives( data_type: JSONValue, codecs: list[JSONValue], simplest: list[JSONValue] ) -> None: - """`codec_for_chunk` returns each codec as its `chunk_canonical` simplifies it for the chunk it receives, and each codec of a pipeline it contains for the chunk that one receives. A codec with nothing to simplify is returned as it is.""" + """`codec_for_chunk` writes each codec as its `chunk_canonical` simplifies it for the chunk it receives, and each codec of a pipeline it contains for the chunk that one receives. A codec with nothing to simplify is written as `canonical_of` writes it.""" found = _simplest(codecs, _of(data_type)) assert json_text(found) == json_text(simplest) # The simplest spelling reads without a problem, and has nothing left to simplify. assert json_text(_simplest(found, _of(data_type))) == json_text(simplest) stages, _ = _read(simplest, _of(data_type)) - assert all(codec_for_chunk(stage) is stage.codec for stage in stages) + assert [codec_for_chunk(stage) for stage in stages] == [ + canonical_of(stage.codec, ()) for stage in stages + ] def test_a_codec_with_a_problem_for_its_chunk_is_not_simplified() -> None: - """A codec the chunk rules find a problem with keeps its own configuration, and its `chunk_canonical` is not asked. A codec the scope refused is returned as it is.""" + """A codec the chunk rules find a problem with keeps its own configuration, and its `chunk_canonical` is not asked. A codec the scope refused has no simplest spelling.""" # `bytes` does not encode values that vary in size, and `gzip` requires a level. stages, problems = _read([LITTLE, {"name": "gzip", "configuration": {}}], _of("string")) assert [problem.loc for problem in problems] == [(1, "configuration", "level"), (0,)] - assert all(codec_for_chunk(stage) is stage.codec for stage in stages) + assert [codec_for_chunk(stage) for stage in stages] == [LITTLE, None] # A `chunk_canonical` that would drop the `endian` whatever the chunk is not asked either. eager = _bytes_with(lambda configuration, nested, chunk: {}) stages, problems = read_pipeline([eager], _of("string")) assert [problem.loc for problem in problems] == [(0,)] - assert codec_for_chunk(stages[0]) is eager + assert codec_for_chunk(stages[0]) == { + "name": "acme.bytes", + "configuration": {"endian": "little"}, + } class AcmeLeveledConfiguration(TypedDict, closed=True): @@ -698,29 +707,28 @@ def _leveled(chunk_canonical: object, *codecs: JSONValue) -> Stage: @pytest.mark.parametrize( - ("codecs", "chunk_canonical"), + ("codecs", "chunk_canonical", "simplest"), [ # An array of the result may be a list. ( [LITTLE, GZIP], lambda configuration, nested, chunk: {"codecs": list(configuration["codecs"])}, + {"name": "acme.leveled", "configuration": {"codecs": (LITTLE, GZIP)}}, ), - # A problem inside a nested declaration is the document's, not the result's. + # A codec that holds a refused one has no simplest spelling, so nothing is asked. ( [LITTLE, REFUSED_GZIP], lambda configuration, nested, chunk: {"codecs": configuration["codecs"]}, + None, ), ], ids=["an-array-as-a-list", "a-nested-declaration-with-a-problem"], ) def test_a_chunk_canonical_that_simplifies_its_own_members_is_taken_at_its_word( - codecs: list[JSONValue], chunk_canonical: object + codecs: list[JSONValue], chunk_canonical: object, simplest: JSONValue ) -> None: - """A result that drops the codec's own `level` and keeps each nested declaration as it was written passes every check, however it spells its arrays and whatever problems the nested declarations have.""" - found = codec_for_chunk(_leveled(chunk_canonical, *codecs)) - assert isinstance(found, AcceptedDeclaration) - assert json_text(cast("JSONValue", found.configuration)) == json_text({"codecs": codecs}) - assert list(found.nested) == [("codecs", 0), ("codecs", 1)] + """A result that drops the codec's own `level` and keeps each nested declaration as it was given passes every check, however it spells its arrays. A codec that holds a declaration its definition refused is not written at all.""" + assert codec_for_chunk(_leveled(chunk_canonical, *codecs)) == simplest def _bytes_with(chunk_canonical: object) -> ResolvedDeclaration[CodecDefinition[Any]]: @@ -827,3 +835,37 @@ def chunk_canonical( assert isinstance(codec, AcceptedDeclaration) with pytest.raises(ValueError, match="expected the nested declaration to stay as it is"): codec_for_chunk(_stage(codec, CHUNK)) + + +def test_a_chunk_canonical_receives_the_configuration_in_its_simplest_spelling() -> None: + """`chunk_canonical` is asked after everything else is simplified: the codec's own members are as `canonical` spells them, and each codec of a pipeline it contains is already written for the chunk that one receives.""" + seen: list[str] = [] + + def chunk_canonical( + configuration: ShardingIndexedCodecConfiguration, nested: Nested, chunk: Chunk + ) -> ShardingIndexedCodecConfiguration: + seen.append(json_text(cast("JSONValue", configuration))) + return configuration + + shard = dataclasses.replace( + SHARDING_INDEXED_CODEC, name="acme.shard", chunk_canonical=chunk_canonical + ) + declaration = { + "name": "acme.shard", + "configuration": { + "chunk_shape": [2, 1], + "codecs": [LITTLE], + "index_codecs": [LITTLE, "crc32c"], + "index_location": "end", + }, + } + codec = resolve(declaration, CodecDefinition, CONTEXT.union(ZarrV3Context(codecs=[shard])))[0] + written = codec_for_chunk(_stage(codec, _of("uint8"))) + # Each inner value takes one byte, an index value eight, and `end` is the default location. + simplest: JSONValue = { + "chunk_shape": [2, 1], + "codecs": [BARE], + "index_codecs": [LITTLE, CRC32C], + } + assert seen == [json_text(simplest)] + assert json_text(written) == json_text({"name": "acme.shard", "configuration": simplest})