diff --git a/packages/zarr-metadata/changes/4642.feature.md b/packages/zarr-metadata/changes/4642.feature.md new file mode 100644 index 0000000000..a79317f00b --- /dev/null +++ b/packages/zarr-metadata/changes/4642.feature.md @@ -0,0 +1 @@ +Each model has a `canonicalize` method that returns the model with its document in its simplest equivalent spelling: `ZarrV3ArrayMetadata`, `ZarrV3GroupMetadata`, `ZarrV3ConsolidatedMetadata`, `ZarrV2ArrayMetadata`, `ZarrV2GroupMetadata` and `ZarrV2ConsolidatedMetadata`. The result equals the model. Every model equal to it, in the same scope, gives the same document, apart from the order of the members of an object, so `to_key_value` with `JSONWriter(sort_keys=True)` writes the same bytes for equal models. The document is written from the parts `==` compares: each declaration as `canonical_of` writes it, each v3 codec as `codec_for_chunk` returns it for the chunk it receives, and the fill value as `canonical_fill_value` writes it. Empty `attributes`, empty v3 `storage_transformers` and a v2 `dimension_separator` of `"."` are left out, and everything else is kept as written. diff --git a/packages/zarr-metadata/changes/4647.bugfix.md b/packages/zarr-metadata/changes/4647.bugfix.md new file mode 100644 index 0000000000..e75570df6e --- /dev/null +++ b/packages/zarr-metadata/changes/4647.bugfix.md @@ -0,0 +1 @@ +A v3 codec's `chunk_canonical` receives the configuration as the codec's `canonical` spells it, so two spellings that `canonical` reduces to one are simplified for a chunk in the same way. Before, it received the configuration as written, and array models that differed only in such spellings were unequal. Its result is checked more closely: one that it would simplify again, or that holds a declaration the codec did not hold, raises `ValueError`, and one that is not JSON raises `TypeError` naming the definition. A result may write an array as a list, and a problem inside a nested declaration is no longer held against it. A codec that leaves `chunk_canonical` at its default is no longer copied each time an array model is compared or hashed. diff --git a/packages/zarr-metadata/changes/4648.bugfix.md b/packages/zarr-metadata/changes/4648.bugfix.md new file mode 100644 index 0000000000..95256fe43d --- /dev/null +++ b/packages/zarr-metadata/changes/4648.bugfix.md @@ -0,0 +1 @@ +`refines` on every model now reads the other model's document in this model's scope and compares the two, so a document read in a richer scope refines the same document read in a poorer one, whatever the richer scope learns of the chunk each codec receives. Before, `refined_in` of a `uint8` array whose `bytes` codec writes an `endian`, read in a scope that claims no `bytes` codec, did not refine the model it came from. A model no longer refines one whose document its own scope reads with a problem: a `gzip` codec with a configuration key its definition does not declare, read in a scope that claims no `gzip`, is not refined by the model of the same codec without that key. `codec_for_chunk` now writes the codec as JSON in its simplest spelling for the chunk, as one step of the walk that `canonical_of` makes, and returns `None` for a codec with no simplest spelling. A codec's `chunk_canonical` receives each nested declaration in its simplest spelling. diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index ad2a700f6c..0e4b631d7b 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -130,6 +130,8 @@ To change a model, use one of these methods: - `refined_in` also reads the document in another context. The new context must claim what this context left unclaimed and contradict nothing. Otherwise it raises `ScopeConflictError`. +- `canonicalize` returns the model with its document in its simplest + equivalent spelling. See [Canonical documents](#canonical-documents). ## Writing store bytes @@ -226,7 +228,9 @@ These are each declaration, and the fill value read against its data type. So in a `uint8` array, a `bytes` codec is the same codec with either `endian`, or with none. The codec is compared for the chunk it receives: after a `cast_value` to `float32`, the two `endian` values - are two codecs. + are two codecs. `codec_for_chunk` writes a codec in the spelling that + is compared, as JSON, which `canonical_of` cannot give, since it reads + the codec alone. A v2 document's [dtype](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#data-type-encoding), @@ -264,3 +268,78 @@ equal to itself. Equal models have equal hashes. They may still write different documents, because `to_json` writes each document as it was given. + +## Canonical documents + +`canonicalize` returns a model with its document in its simplest +equivalent spelling. The result equals the model it came from. Every model +equal to that one, read in the same context, gives the same document. + +```python test="true" session="models" +from zarr_metadata.model import ZarrV3ArrayMetadata + +written = ZarrV3ArrayMetadata.from_json( + { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, + "fill_value": 0, + "codecs": [ + {"name": "bytes", "configuration": {"endian": "little"}}, + {"name": "zstd", "configuration": {"level": 1, "checksum": False}}, + ], + "attributes": {}, + } +) +canonical = written.canonicalize() +assert canonical == written +print(canonical.to_json()["chunk_key_encoding"]) +#> {'name': 'default'} +print(canonical.to_json()["codecs"]) +#> ({'name': 'bytes'}, {'name': 'zstd', 'configuration': {'level': 1}}) +print("attributes" in canonical.to_json()) +#> False +``` + +The canonical document is written from the same parts that `==` compares: + +- Each declaration is written as `canonical_of` writes it. +- Each v3 codec is written for the chunk it receives, as `codec_for_chunk` + writes it. Above, each `uint8` value takes one byte, so the `endian` + says nothing. +- The fill value is written as `canonical_fill_value` writes it for the + data type. +- A member that says what its absence says is left out. These are empty + `attributes`, empty v3 `storage_transformers`, and a v2 + `dimension_separator` of `"."`. +- Everything else is kept as written. This includes the attributes, and + the configuration of a declaration that nothing in the context claims, + because only its own definition could simplify it. Such a declaration + is still written in the shortest form every reader accepts, without an + empty `configuration` or a `must_understand` of `true`. + +A v2 codec is written with each numcodecs default it leaves out, as its +definition's `canonical` writes it. So a canonical v2 document can be +longer than the one it came from. + +The order of the members of an object is not part of the spelling, because +JSON gives it no meaning. Two equal models can give canonical documents +whose members are in different orders. A `JSONWriter` that sorts them +writes the same bytes for both, so those bytes are a key for what the +document means: + +```python test="true" session="models" +import hashlib + +stored = canonical.to_key_value({"sort_keys": True})["zarr.json"] +digest = hashlib.sha256(stored).hexdigest() +``` + +The other models work the same way. A v3 group does not read the documents +its `consolidated_metadata` lists, so its `canonicalize` keeps them as +written. `ZarrV3ConsolidatedMetadata.canonicalize` and +`ZarrV2ConsolidatedMetadata.canonicalize` write each node they list in its +simplest spelling. diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index f59fea98a4..d9b6c1f246 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1200,16 +1200,13 @@ def declaration_key(declaration: ResolvedDeclaration[Any]) -> tuple[object, ...] configuration: JSONValue = dict(declaration.configuration) # Each nested declaration is compared by its own key, so its location is # cleared before the definition's `canonical` sees the rest, in the - # same order as `_canonical_declaration`. `canonical` simplifies only the + # same order as `canonical_declaration`. `canonical` simplifies only the # definition's own members. for loc in declaration.nested: configuration = _replaced(configuration, loc, None) view = read_only(cast("Mapping[str, JSONValue]", configuration)) - spelled = asked( - definition, - "canonical", - lambda: json_text(cast("JSONValue", dict(definition.canonical(view)))), - ) + _, own = _spelled(definition, view) + spelled = asked(definition, "canonical", lambda: json_text(cast("JSONValue", own))) return ("read", definition, spelled, _nested_key(declaration.nested)) if isinstance(declaration, UnclaimedDeclaration): return ( @@ -1228,11 +1225,6 @@ def declaration_key(declaration: ResolvedDeclaration[Any]) -> tuple[object, ...] ) -def own_key(declaration: AcceptedDeclaration[Any]) -> tuple[object, ...]: - """The part of `declaration_key` for an accepted declaration that excludes nested declarations: the definition and the canonical spelling of its own members.""" - return declaration_key(declaration)[:3] - - def _nested_key(nested: Nested) -> tuple[tuple[Loc, tuple[object, ...]], ...]: """The key of each nested declaration, paired with its location, in the order of the locations. @@ -1277,36 +1269,9 @@ def written_json(declaration: AcceptedDeclaration[Any] | UnclaimedDeclaration) - return kind._envelope_json(declaration.name, declaration.configuration) -def respelled( - declaration: AcceptedDeclaration[D], - configuration: Mapping[str, JSONValue], - nested: Nested, -) -> AcceptedDeclaration[D]: - """`declaration` with `configuration` as its configuration, and each declaration in `nested` in place of the nested declaration at its location. - - This builds the declaration that another spelling of `declaration` - reads as, without reading it again. The caller answers for the - spelling: `configuration` must be one that the definition allows and - that means the same thing, and it must keep each nested declaration - where it is. `nested` holds only the nested declarations to replace, - each one accepted or unclaimed, and each is written into the - configuration as its `to_json` writes it. A location in `nested` that - does not lead into `configuration` raises `TypeError`. The result is - written as `to_json` writes it, since no document wrote it. - """ - spelled: JSONValue = dict(configuration) - for loc, inner in nested.items(): - spelled = _replaced(spelled, loc, written_json(held(inner))) - kind = declaration.read_as - name = declaration.name - carried = kind._filed_under(name)[1] is not None - return AcceptedDeclaration( - json=kind._envelope_json(name, {} if carried else cast("Mapping[str, JSONValue]", spelled)), - name=name, - definition=declaration.definition, - configuration=cast("Mapping[str, JSONValue]", spelled), - nested={**declaration.nested, **nested}, - ) +def declaration_locations(definition: Definition[Any], configuration: JSONValue) -> tuple[Loc, ...]: + """The location of each declaration that `configuration` holds, where `definition`'s TypedDict types a member with a declaration alias, in order. `configuration` is JSON with arrays as tuples.""" + return tuple(found.loc for found in _typed(definition.configuration, configuration, ())[2]) def item_at(value: JSONValue, path: Loc) -> JSONValue | UNSET: @@ -1888,41 +1853,104 @@ def canonical_of( def _simplest(declaration: ResolvedDeclaration[Any]) -> JSONValue | None: """A declaration in its simplest spelling; None when the declaration or one of its nested declarations was refused, because a refused declaration has no simplest spelling.""" if isinstance(declaration, AcceptedDeclaration): - return _canonical_declaration(declaration) + return canonical_declaration(declaration) if isinstance(declaration, UnclaimedDeclaration): return declaration.to_json() return None -def _canonical_declaration(resolved: AcceptedDeclaration[Any]) -> JSONValue | None: - """An accepted declaration in its simplest equivalent spelling: nested declarations first, then its own members. None when a nested declaration was refused.""" +@dataclass(frozen=True, slots=True) +class Further: + """A simplification of a definition's own members by something `canonical` cannot know, such as the chunk a codec receives. + + `ask` receives the configuration in its simplest spelling so far, + read-only: its own members as `canonical` spells them, and each + nested declaration in its own simplest spelling. It returns a simpler + configuration, as JSON with arrays as tuples, or None when it + simplifies nothing. It answers for its result: it raises when the + result does not hold. `what` names the function of the definition + that `ask` calls, for the error raised when its result is not the + simplest spelling. + """ + + what: str + ask: Callable[[Mapping[str, JSONValue]], dict[str, JSONValue] | None] + + +def canonical_declaration( + resolved: AcceptedDeclaration[Any], + spelled: Mapping[Loc, JSONValue | None] | None = None, + further: Further | None = None, +) -> JSONValue | None: + """An accepted declaration in its simplest equivalent spelling: nested declarations first, then its own members. None when a nested declaration has no simplest spelling. + + This is the one walk that writes a declaration in its simplest + spelling. A caller that knows more than the declaration says, as a + pipeline knows the chunk each codec receives, takes part in it + instead of walking the declaration again. `spelled` gives the + simplest spelling of the nested declarations the caller wrote + itself, by location, with None for one that has none. Each other + nested declaration is written as this function writes it. `further` + simplifies the declaration's own members once more, after + `canonical`. What it gives is spelled by `canonical` again, and must + then be the simplest spelling: asked about it, `further` must + simplify nothing, or this raises `ValueError`, which indicates a bug + in the definition. + """ definition, name = resolved.definition, resolved.name configuration: JSONValue = dict(resolved.configuration) for loc, inner in resolved.nested.items(): - simplest = _simplest(inner) + simplest = spelled[loc] if spelled is not None and loc in spelled else _simplest(inner) if simplest is None: return None configuration = _replaced(configuration, loc, simplest) - # Every function of a definition receives this read-only view. The - # result of `canonical` (the view itself when nothing is simplified) is - # converted to a dict. - view = read_only(cast("Mapping[str, JSONValue]", configuration)) - simplified = asked( - definition, - "canonical", - lambda: dict(cast("Mapping[str, JSONValue]", definition.canonical(view))), - ) - _, refused = definition.read_configuration(simplified) + simplified = _spelled_and_checked(definition, cast("Mapping[str, JSONValue]", configuration)) + if further is not None: + given = further.ask(read_only(simplified)) + if given is not None: + simplified = _spelled_and_checked(definition, given) + left = further.ask(read_only(simplified)) + if left is not None: + # The simplest spelling has nothing left to simplify. Otherwise + # a document written from it would read as another declaration. + msg = ( + f"{definition.name!r}: its {further.what} gave {given!r}, which is not the " + f"simplest spelling: asked again, it gave {left!r}" + ) + raise ValueError(msg) + carrying = definition._carrying_name(simplified) + if carrying is not None: + return carrying + return resolved.read_as._envelope_json(name, simplified) + + +def _spelled_and_checked( + definition: Definition[Any], configuration: Mapping[str, JSONValue] +) -> dict[str, JSONValue]: + """`configuration` as `definition`'s `canonical` spells it. A result that the definition does not allow raises `ValueError`, which indicates a bug in the definition.""" + kept, simplified = _spelled(definition, read_only(configuration)) + # A `canonical` that returns what it received simplified nothing, so + # there is nothing of its own to check. + refused = () if kept else definition.read_configuration(simplified)[1] if len(refused) != 0: msg = ( f"{definition.name!r}: its canonical gave {simplified!r}, which does not hold: " f"{list(refused)!r}" ) raise ValueError(msg) - carrying = definition._carrying_name(simplified) - if carrying is not None: - return carrying - return resolved.read_as._envelope_json(name, simplified) + return simplified + + +def _spelled( + definition: Definition[Any], view: Mapping[str, JSONValue] +) -> tuple[bool, dict[str, JSONValue]]: + """Whether `definition`'s `canonical` returned `view` itself, and what it returned, as a dict. This is the one place that calls `canonical`. Every function of a definition receives a read-only view.""" + + def spelled() -> tuple[bool, dict[str, JSONValue]]: + given = cast("Mapping[str, JSONValue]", definition.canonical(view)) + return given is view, dict(given) + + return asked(definition, "canonical", spelled) def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: @@ -1943,6 +1971,7 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "C", "Context", "Definition", + "Further", "Nested", "Problems", "RefusedDeclaration", @@ -1951,12 +1980,14 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "WithFillValue", "as_kind", "asked", + "canonical_declaration", "canonical_fill_value", "canonical_of", "canonicalize", "configuration_of", "declaration_key", "declaration_kind", + "declaration_locations", "declarations_of", "document_json", "filed_under", @@ -1970,11 +2001,9 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "kind_of", "named_configuration", "no_rules", - "own_key", "read_only", "resolve", "resolve_refined", - "respelled", "ruled", "spelled_canonically", "unchanged", diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py index 9a28bc2a32..f1883b51d5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py @@ -1,4 +1,4 @@ -"""Operations on scopes: which definition a reading records for each name, how one reading refines another, and where a scope disagrees with a reading. +"""Operations on scopes: which definition a reading records for each name, and where a scope disagrees with a reading. A model is a document read in a scope. The meaning of the document depends only on what the scope returned for the names the document uses @@ -6,7 +6,9 @@ document are ordered by how much they know. If a name was unclaimed and a definition later reads it, the name gains meaning and loses none. If one definition reads a name and then a different definition reads it, that is -a conflict. The internal `disagreements` function in +a conflict. A model's `refines` decides that order by reading: it reads +the other model's document in its own scope, so nothing here compares +two readings. The internal `disagreements` function in `zarr_metadata.definition._context`, which a model uses to compare a scope with its claims, is built on these operations. A scope's `union` raises the `ScopeConflictError` defined here when two scopes file @@ -17,27 +19,18 @@ from collections.abc import Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast +from typing import TYPE_CHECKING, Any, TypeAlias from zarr_metadata.definition._definition import ( - AcceptedDeclaration, Definition, - RefusedDeclaration, - UnclaimedDeclaration, - as_kind, - declaration_key, - declarations_of, filed_under, - format_of, - own_key, - resolve, ) if TYPE_CHECKING: from collections.abc import Callable, Iterable, Sequence from zarr_metadata._typed_json import Loc - from zarr_metadata.definition._definition import D, ResolvedDeclaration + from zarr_metadata.definition._definition import ResolvedDeclaration ClaimKey: TypeAlias = tuple[type[Definition[Any]], str] """A kind and the name a definition is filed under. A scope's `claimant` looks up definitions by this key.""" @@ -140,65 +133,6 @@ def claims_of( return claims -def refines(declaration: ResolvedDeclaration[Any], other: ResolvedDeclaration[Any]) -> bool: - """Whether `declaration` contains everything `other` contains: it reads the same where both read, and it may also read what `other` left unclaimed. - - This is the order in which one reading of a document refines another. - Suppose `other` is unclaimed and `declaration` is accepted. This counts as a - gain when re-reading the unclaimed declaration's JSON, with `declaration`'s - definition and the definitions of the declarations nested in `declaration`, - produces `declaration`. Two spellings of one configuration count as one gain, - because `==` treats them as one declaration, so the order is transitive - through equality. The reverse direction (accepted to unclaimed) is a - loss. One name read by two different definitions is a conflict. A - refused declaration refines only itself. Two declarations that refine each other - are equal. - """ - if isinstance(declaration, RefusedDeclaration) or isinstance(other, RefusedDeclaration): - return declaration == other - if isinstance(other, UnclaimedDeclaration): - if isinstance(declaration, UnclaimedDeclaration): - return declaration_key(declaration) == declaration_key(other) - return claim_key(declaration) == claim_key(other) and _gained(declaration, other) - if isinstance(declaration, UnclaimedDeclaration): - return False - if declaration.definition != other.definition or own_key(declaration) != own_key(other): - return False - if set(declaration.nested) != set(other.nested): - return False - return all(refines(declaration.nested[loc], other.nested[loc]) for loc in declaration.nested) - - -def _gained(declaration: AcceptedDeclaration[Any], other: UnclaimedDeclaration) -> bool: - """Whether reading `other` in the scope formed by `declaration`'s own claims produces `declaration`. This is the test for a gain.""" - again, _ = resolve(other.json, declaration.read_as, _Claimed.of(declaration)) - return again == declaration - - -@dataclass(frozen=True, slots=True) -class _Claimed: - """The scope formed by a declaration's own claims: its definition and the definitions of its nested declarations, keyed by kind and filed name. `_gained` re-reads a declaration in this scope.""" - - filed: Mapping[ClaimKey, Definition[Any]] - zarr_format: Literal[2, 3] - - @classmethod - def of(cls, declaration: AcceptedDeclaration[Any]) -> _Claimed: - claimed = claims_of(declarations_of(declaration)) - return cls( - {key: definition for key, definition in claimed.items() if definition is not None}, - format_of(declaration.read_as), - ) - - def claimant(self, kind: type[D], name: str) -> D | None: - """The definition of `kind` among the claims that reads `name`, or None if there is none. This matches a scope's `claimant`.""" - asked = as_kind(kind) - filed, _ = filed_under(asked, name) - if filed is None: - return None - return cast("D | None", self.filed.get((asked, filed))) - - @dataclass(frozen=True, slots=True) class Disagreements: """The places where a scope reads a reading's claims differently: the names that would gain a meaning, and the names that conflict (including names that would lose a meaning).""" @@ -240,5 +174,4 @@ def disagreements_of( "claims_of", "disagreements_of", "kind_name", - "refines", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 39f4ca5e58..b18d1eb092 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -42,7 +42,10 @@ scope and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as written. `replace` reads new members in the model's own scope. `with_context` and -`refined_in` read the document in another scope. `to_key_value` writes +`refined_in` read the document in another scope. `canonicalize` returns +the model with its document in its simplest equivalent spelling, the one +that every equal model gives, apart from the order of the members of an +object. `to_key_value` writes the model as it is, with a `JSONWriter`, or a mapping of its settings, deciding how the bytes are spelled (indentation, key order, ASCII escaping, trailing newline); the default is that of `json.dumps`. A v3 group reads its diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index 2e5ba2930e..5353433cc6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -19,7 +19,6 @@ is_array, is_object, is_tuple, - json_text, not_an_object, outside_of, prefixed, @@ -33,7 +32,6 @@ AcceptedDeclaration, ResolvedDeclaration, UnclaimedDeclaration, - declaration_key, declarations_of, default_fill_value, fill_value_problems, @@ -43,13 +41,13 @@ spelled_canonically, ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of -from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( Derived, StoreKey, attributes_of, + canonical_members, check_literal, created, dimension_lengths, @@ -59,6 +57,7 @@ missing_keys, other_members, reading_of, + simplest, unclaimed_declarations_of, ) @@ -89,6 +88,7 @@ from collections.abc import Iterator, Sequence from zarr_metadata._typed_json import Loc + from zarr_metadata.definition._scope import Claims from zarr_metadata.v2.array import ZarrV2ArrayMetadataStoreKey from zarr_metadata.v2.attributes import ZarrV2AttributesStoreKey @@ -626,52 +626,25 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for a v2 array model: what its document means. - - The key holds each declaration's `declaration_key`, and the fill value as JSON - text, in its canonical spelling when a definition in scope read the - dtype. Every other member is included as it is, with JSON members - as text. No `.zattrs` counts as empty attributes, since both mean - that the array has none, as in v3. - """ - members = self._members - return ( - members.shape, - members.chunks, - members.order, - members.dimension_separator, - self._fill_value_key(), - declaration_key(self.dtype), - None if self.compressor is None else declaration_key(self.compressor), - None - if self.filters is None - else tuple(declaration_key(entry) for entry in self.filters), - json_text({} if members.attributes is UNSET else members.attributes), - json_text(members.extra_fields), - ) - - def _plain_key( - self, dtype: AcceptedDeclaration[ZarrV2DataTypeDefinition[Any]] | UnclaimedDeclaration - ) -> tuple[object, ...]: - """The members that hold no declaration, which `refines` compares. The fill value is in the spelling of `dtype`, the dtype of the more informed side.""" - members = self._members - return ( - members.shape, - members.chunks, - members.order, - members.dimension_separator, - json_text(spelled_canonically(dtype, members.fill_value)), - json_text({} if members.attributes is UNSET else members.attributes), - json_text(members.extra_fields), + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document in its simplest equivalent spelling. This is the one place that decides what a v2 array document means: the model's key is this document as JSON text.""" + compressor, filters = self.compressor, self.filters + spelled: dict[str, JSONValue] = { + "dtype": simplest(self.dtype), + "fill_value": spelled_canonically(self.dtype, self._members.fill_value), + "compressor": None if compressor is None else simplest(compressor), + "filters": None if filters is None else tuple(simplest(entry) for entry in filters), + } + return canonical_members( + self._document, + spelled, + empty=("attributes",), + defaults={"dimension_separator": ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR}, ) - def _fill_value_key(self) -> str: - """The fill value as `==` compares it, as JSON text: in its canonical spelling when a definition in scope read the dtype, and as written otherwise.""" - fill_value = self._members.fill_value - if isinstance(self.dtype, AcceptedDeclaration): - return json_text(spelled_canonically(self.dtype, fill_value)) - return json_text(fill_value) + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of(self._reading.declarations()) def __reduce__(self) -> tuple[type[ZarrV2ArrayMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) @@ -772,6 +745,26 @@ def __replace__(self, /, **members: Unpack[ZarrV2ArrayMetadataUpdate]) -> ZarrV2 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV2ArrayMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same document: `to_json` of each result is + the same JSON, apart from the order of the members of an object, + which `to_key_value` sorts with `JSONWriter(sort_keys=True)`. So + the bytes that writer writes are the same for equal models, and + are a key for what the document means. + + The dtype, the compressor and each filter are written as + `canonical_of` writes them, and the fill value as + `canonical_fill_value` writes it for the dtype. A + `dimension_separator` of `"."` is left out, and so are empty + `attributes`, because a document without them means the same. + Everything else is kept as written: `shape`, `chunks`, `order`, + the attributes, and whatever nothing in scope claims. + """ + return type(self)(self._normal_form, context=self._context) + def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. @@ -802,25 +795,8 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadat return self.with_context(context) def refines(self, other: ZarrV2ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each declaration must refine its counterpart. A `null` compressor or `null` filters refine only a `null`. Every other member must be the same, with the fill value compared in the spelling of the more informed dtype. If that dtype rejects the other model's fill value, this is not a refinement.""" - if type(other) is not type(self): - return False - if (self.compressor is None) != (other.compressor is None): - return False - if (self.filters is None) != (other.filters is None): - return False - mine = () if self.filters is None else self.filters - theirs = () if other.filters is None else other.filters - if len(mine) != len(theirs): - return False - pairs = [(self.dtype, other.dtype), *zip(mine, theirs, strict=True)] - if self.compressor is not None and other.compressor is not None: - pairs.append((self.compressor, other.compressor)) - if not all(refines_declaration(one, another) for one, another in pairs): - return False - if len(fill_value_problems(self.dtype, other._members.fill_value)) != 0: - return False - return self._plain_key(self.dtype) == other._plain_key(self.dtype) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. So the fill value is compared in the spelling of this model's dtype. If `other`'s document has a problem in this scope, as when this model's dtype rejects its fill value, this is not a refinement. False when `other` is not a v2 array.""" + return self._refines(other, lambda: other.with_context(self._context)) # --- constructors ----------------------------------------------------- diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 2ea2af7ccb..e257e8872c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -18,7 +18,6 @@ frozen, is_array, is_object, - json_text, nested_past_the_levels, not_an_object, refine_json, @@ -34,7 +33,6 @@ Definition, ResolvedDeclaration, UnclaimedDeclaration, - declaration_key, declaration_kind, declarations_of, default_fill_value, @@ -45,13 +43,13 @@ spelled_canonically, ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of -from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike -from zarr_metadata.model._keyed import Keyed +from zarr_metadata.model._keyed import Keyed, normal_key from zarr_metadata.model._shared import ( Derived, StoreKey, attributes_of, + canonical_members, check_literal, created, dimension_lengths, @@ -60,6 +58,8 @@ member_changes, missing_keys, reading_of, + simplest, + simplest_members, unclaimed_declarations_of, ) from zarr_metadata.v3._common import ZarrV3DeclarationJSON, ZarrV3ExtensionField @@ -88,6 +88,7 @@ from collections.abc import Iterator from zarr_metadata._typed_json import Loc + from zarr_metadata.definition._scope import Claims from zarr_metadata.v3._definition import CodecDefinition, ExtraFieldDefinition from zarr_metadata.v3.array import ZarrV3ArrayMetadataStoreKey @@ -604,21 +605,13 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for a v3 array model: what its document means, as computed by `array_key_of`.""" - return array_key_of(self._reading, self._members) - - def _plain_key( - self, data_type: AcceptedDeclaration[DataTypeDefinition[Any]] | UnclaimedDeclaration - ) -> tuple[object, ...]: - """The members that hold no declaration, which `refines` compares. The fill value is in the spelling of `data_type`, the data type of the more informed side.""" - members = self._members - return ( - members.shape, - json_text(spelled_canonically(data_type, members.fill_value)), - members.dimension_names, - json_text(members.attributes), - ) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document in its simplest equivalent spelling, as `canonical_array_document_v3` writes it.""" + return canonical_array_document_v3(self._reading, self._members) + + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of(self._reading.declarations()) def __reduce__(self) -> tuple[type[ZarrV3ArrayMetadata], tuple[object, ZarrV3Context]]: # Pickle the pair and read it again on load, so a model's reading @@ -734,6 +727,31 @@ def __replace__(self, /, **members: Unpack[ZarrV3ArrayMetadataUpdate]) -> ZarrV3 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV3ArrayMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same document: `to_json` of each result is + the same JSON, apart from the order of the members of an object, + which `to_key_value` sorts with `JSONWriter(sort_keys=True)`. So + the bytes that writer writes are the same for equal models, and + are a key for what the document means. + + Each declaration is written as `canonical_of` writes it, each + codec as `codec_for_chunk` writes it for the chunk it receives, + and the fill value as `canonical_fill_value` writes it for the + data type. `attributes` and `storage_transformers` are left out + when empty, because a document without them means the same. + Everything else is kept as written: `shape`, `dimension_names`, + the attributes, and the configuration of a declaration nothing in + scope claims, which only its own definition could simplify. Such + a declaration is still written in the shortest form every reader + accepts, without an empty `configuration` or a `must_understand` + of `true`. A document that is already in this spelling is + returned as it is, in a new model. + """ + return type(self)(self._normal_form, context=self._context) + def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. @@ -767,28 +785,8 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadat return self.with_context(context) def refines(self, other: ZarrV3ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each declaration must refine its counterpart, in the order `refines` defines for declarations (including the declarations nested in each declaration). Every other member must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" - if type(other) is not type(self): - return False - if len(self.codecs) != len(other.codecs) or len(self.storage_transformers) != len( - other.storage_transformers - ): - return False - if self.extra_fields.keys() != other.extra_fields.keys(): - return False - pairs = ( - (self.data_type, other.data_type), - (self.chunk_grid, other.chunk_grid), - (self.chunk_key_encoding, other.chunk_key_encoding), - *zip(codecs_for_chunks(self._reading), codecs_for_chunks(other._reading), strict=True), - *zip(self.storage_transformers, other.storage_transformers, strict=True), - *((self.extra_fields[key], other.extra_fields[key]) for key in self.extra_fields), - ) - if not all(refines_declaration(mine, theirs) for mine, theirs in pairs): - return False - if len(fill_value_problems(self.data_type, other._members.fill_value)) != 0: - return False - return self._plain_key(self.data_type) == other._plain_key(self.data_type) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. So each codec is compared for the chunk it receives in this model, and the fill value in the spelling of this model's data type, whatever `other`'s scope knew of either. If `other`'s document has a problem in this scope, as when this model's data type rejects its fill value, this is not a refinement. False when `other` is not a v3 array.""" + return self._refines(other, lambda: other.with_context(self._context)) # --- constructors ----------------------------------------------------- @@ -922,50 +920,46 @@ def _derived_codecs(document: Mapping[str, object], context: ZarrV3Context) -> o def array_key_of( reading: ZarrV3ArrayMetadataReading, members: ArrayMembersV3 ) -> tuple[object, ...]: - """A key for what a v3 array document means, from its `reading` and refined `members`. `==` and `hash` compare this key for the model, and two listings of one node are compared by it. - - The key holds each declaration's `declaration_key`, and the fill value as JSON - text: in its canonical spelling when a definition in scope read the - data type, and as written otherwise. A codec is compared as - `codec_for_chunk` returns it for the chunk it receives, so two - spellings that mean the same for that chunk are one codec. Every - other member is included as it is, with JSON members as text. - """ - data_type = held(reading.data_type) - fill_value = ( - spelled_canonically(data_type, members.fill_value) - if isinstance(data_type, AcceptedDeclaration) - else members.fill_value - ) - return ( - members.shape, - json_text(fill_value), - declaration_key(data_type), - declaration_key(held(reading.chunk_grid)), - tuple(declaration_key(codec) for codec in codecs_for_chunks(reading)), - declaration_key(held(reading.chunk_key_encoding)), - members.dimension_names, - json_text(members.attributes), - tuple(declaration_key(held(entry)) for entry in reading.storage_transformers), - extra_fields_key(reading.extra_fields), + """A key for what a v3 array document means, from its `reading` and refined `members`: the key of the model, as `normal_key` computes it from the claims the reading made and from `canonical_array_document_v3`. Two listings of one node are compared by it.""" + return normal_key( + claims_of(reading.declarations()), canonical_array_document_v3(reading, members) ) -def codecs_for_chunks( - reading: ZarrV3ArrayMetadataReading, -) -> tuple[AcceptedDeclaration[CodecDefinition[Any]] | UnclaimedDeclaration, ...]: - """The codecs of `reading`, a read that found no problems, in pipeline order: each as `codec_for_chunk` returns it for the chunk it receives.""" - return tuple(held(codec_for_chunk(stage)) for stage in reading.pipeline) - +def canonical_array_document_v3( + reading: ZarrV3ArrayMetadataReading, members: ArrayMembersV3 +) -> dict[str, JSONValue]: + """A v3 array document in its simplest equivalent spelling, from its `reading` and refined `members`, as `ZarrV3ArrayMetadata.canonicalize` reads it. -def extra_fields_key(extra_fields: Mapping[str, ResolvedDeclaration[Any]]) -> tuple[object, ...]: - """A key for what a document's extra fields mean: each key with its declaration's `declaration_key`, sorted by key.""" - return tuple( - (key, declaration_key(extra)) - for key, extra in sorted(extra_fields.items(), key=lambda item: item[0]) + This is the one place that decides what a v3 array document means. + The model's key is this document as JSON text, so two documents with + one key are written the same way, apart from the order of the members + of an object. + """ + data_type = held(reading.data_type) + spelled: dict[str, JSONValue] = { + "data_type": simplest(data_type), + "chunk_grid": simplest(reading.chunk_grid), + "chunk_key_encoding": simplest(reading.chunk_key_encoding), + "fill_value": spelled_canonically(data_type, members.fill_value), + "codecs": tuple(_codec_spelled(stage) for stage in reading.pipeline), + "storage_transformers": tuple(simplest(entry) for entry in reading.storage_transformers), + **simplest_members(reading.extra_fields), + } + return canonical_members( + members.document, spelled, empty=("attributes", "storage_transformers") ) +def _codec_spelled(stage: Stage) -> JSONValue: + """The codec of `stage`, from a read that found no problems, as `codec_for_chunk` writes it. Raises `TypeError` for a codec with no simplest spelling, which such a read cannot produce.""" + spelled = codec_for_chunk(stage) + if spelled is None: + msg = f"expected a codec a read found nothing wrong with, got {stage.codec!r}" + raise TypeError(msg) + return spelled + + def read_array_metadata_v3( value: object, *, context: ZarrV3Context | None = None ) -> ZarrV3ArrayMetadataReading: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index cbd34fb85f..1ae725a266 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -19,7 +19,6 @@ frozen, is_json_object, is_object, - json_text, not_an_object, object_at, prefixed, @@ -48,6 +47,7 @@ from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, + canonical_members, check_literal, load_store_json, located_conflicts, @@ -71,6 +71,7 @@ from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import ResolvedDeclaration, UnclaimedDeclaration + from zarr_metadata.definition._scope import Claims from zarr_metadata.v2.attributes import ZarrV2AttributesStoreKey from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataStoreKey from zarr_metadata.v2.group import ZarrV2GroupMetadataStoreKey @@ -357,10 +358,13 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for a v2 group model: its attributes as JSON text. No `.zattrs` counts as empty attributes, since both mean that the group has none, as in v3.""" - attributes = self._attributes - return (json_text({} if attributes is UNSET else attributes),) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document in its simplest equivalent spelling: without `attributes` when they are empty, since no `.zattrs` and an empty one both mean that the group has none.""" + return canonical_members(self._document, {}, empty=("attributes",)) + + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return {} def __reduce__(self) -> tuple[type[ZarrV2GroupMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) @@ -378,6 +382,10 @@ def __replace__(self, /, **members: Unpack[ZarrV2GroupMetadataUpdate]) -> ZarrV2 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV2GroupMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope: without `attributes` when they are empty, because a group with no `.zattrs` means the same. The result equals this model, and every model equal to this one gives the same document, apart from the order of the members of an object, which `to_key_value` sorts with `JSONWriter(sort_keys=True)`.""" + return type(self)(self._normal_form, context=self._context) + def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: """Return this document read in `context`: the same group, holding that scope.""" context = scoped(context, CORE_V2_CONTEXT) @@ -388,8 +396,8 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadat return self.with_context(context) def refines(self, other: ZarrV2GroupMetadata) -> bool: - """Whether this group holds everything `other` holds, which means its attributes are written the same way, no `.zattrs` counting as empty attributes. False when `other` is not a v2 group.""" - return type(other) is type(self) and self._model_key == other._model_key + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. A group has no declarations, so this means its attributes are written the same way, no `.zattrs` counting as empty attributes. False when `other` is not a v2 group.""" + return self._refines(other, lambda: other.with_context(self._context)) @classmethod def create_default( @@ -562,18 +570,25 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for v2 consolidated metadata: each node's path and key, and every other entry as JSON text.""" - return ( - tuple(sorted((path, node._model_key) for path, node in self._metadata.items())), - self._other_entries_text(), - ) - - def _other_entries_text(self) -> str: - """The entries that no node is read from (an orphan `.zattrs`, or any other key) as JSON text, which `==` compares.""" - entries = object_at(self._document, "metadata") - consumed = {key for names in self._files.values() for key in names.values()} - return json_text({key: value for key, value in entries.items() if key not in consumed}) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document with each node in its simplest equivalent spelling, as that node's normal form writes it: its `.zarray` or `.zgroup` entry without the attributes, and its `.zattrs` entry holding them, or left out when they are empty.""" + entries = dict(object_at(self._document, "metadata")) + for path, node in self._metadata.items(): + names = self._files[path] + written = dict(node._normal_form) + attributes = written.pop("attributes", None) + for name, key in names.items(): + if name != ZARR_V2_ATTRIBUTES_STORE_KEY: + entries[key] = written + elif attributes is None: + del entries[key] + else: + entries[key] = attributes + return {**self._document, "metadata": entries} + + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of(self._located_declarations()) def _located_declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: """Each declaration of each array node, as read, located in the document: under the node's `.zarray` entry, at the key the document writes for it.""" @@ -586,6 +601,18 @@ def _located_declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]] def __reduce__(self) -> tuple[type[ZarrV2ConsolidatedMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) + def canonicalize(self) -> ZarrV2ConsolidatedMetadata: + """Return this document with each node in its simplest equivalent spelling, as that node's `canonicalize` writes it, read in this model's own scope. + + Each node's `.zarray` or `.zgroup` entry is written without the + attributes, and its `.zattrs` entry holds them, or is left out + when they are empty. Every other entry is kept as written. The + result equals this model. Every model equal to this one, in this + scope, gives the same document, apart from the order of the + members of an object, which a `JSONWriter(sort_keys=True)` sorts. + """ + return type(self)(self._normal_form, context=self._context) + def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: """Return this document with every node read in `context`, whatever that changes. Raises `MetadataValidationError` when a node has a problem in `context`.""" context = scoped(context, CORE_V2_CONTEXT) @@ -617,16 +644,8 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2Consolidated return self.with_context(context) def refines(self, other: ZarrV2ConsolidatedMetadata) -> bool: - """Whether every node this holds refines the node `other` holds at the same path, both hold the same paths, and the other entries are written the same way. False when `other` is not v2 consolidated metadata.""" - if type(other) is not type(self): - return False - if self._metadata.keys() != other._metadata.keys(): - return False - if self._other_entries_text() != other._other_entries_text(): - return False - return all( - _v2_node_refines(self._metadata[path], other._metadata[path]) for path in self._metadata - ) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. So every node this holds refines the node `other` holds at the same path, both hold the same paths, and the other entries are written the same way. False when `other` is not v2 consolidated metadata.""" + return self._refines(other, lambda: other.with_context(self._context)) @classmethod def from_json( @@ -651,13 +670,6 @@ def from_key_value( ) -def _v2_node_refines(node: ZarrV2NodeMetadata, other: ZarrV2NodeMetadata) -> bool: - """Whether `node` refines `other`. Models of the same kind use their own `refines`; an array never refines a group, or the reverse.""" - if isinstance(node, ZarrV2ArrayMetadata): - return isinstance(other, ZarrV2ArrayMetadata) and node.refines(other) - return isinstance(other, ZarrV2GroupMetadata) and node.refines(other) - - _NODE_FILES: Final = ( ZARR_V2_ARRAY_METADATA_STORE_KEY, ZARR_V2_GROUP_METADATA_STORE_KEY, diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 9c5d9dc006..153fd9b5ca 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -47,13 +47,11 @@ definition_said, kind_name, ) -from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._array_v3 import ( ArrayMembersV3, ZarrV3ArrayMetadata, ZarrV3ArrayMetadataReading, array_key_of, - extra_fields_key, members_past_the_levels, must_understand_subset, no_extra_fields, @@ -67,12 +65,14 @@ from zarr_metadata.model._shared import ( StoreKey, attributes_of, + canonical_members, check_literal, load_store_json, located_conflicts, member_changes, missing_keys, reading_of, + simplest_members, unclaimed_declarations_of, unexpected_keys, ) @@ -100,6 +100,7 @@ from zarr_metadata._hierarchy import NodeType from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import AcceptedDeclaration, ResolvedDeclaration + from zarr_metadata.definition._scope import Claims from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataStoreKey @@ -282,12 +283,14 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for a v3 group model: its attributes as JSON text, and what each extra field means, as `extra_fields_key` computes it.""" - return ( - json_text(self._members.attributes), - extra_fields_key(self._reading.extra_fields), - ) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document in its simplest equivalent spelling. This is the one place that decides what a v3 group document means: the model's key is this document as JSON text.""" + spelled = simplest_members(self._reading.extra_fields) + return canonical_members(self._document, spelled, empty=("attributes",)) + + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of(self._reading.declarations()) def __reduce__(self) -> tuple[type[ZarrV3GroupMetadata], tuple[object, ZarrV3Context]]: # Pickle the pair, and read it again on load. @@ -392,6 +395,25 @@ def __replace__(self, /, **members: Unpack[ZarrV3GroupMetadataUpdate]) -> ZarrV3 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV3GroupMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same document: `to_json` of each result is + the same JSON, apart from the order of the members of an object, + which `to_key_value` sorts with `JSONWriter(sort_keys=True)`. + + `attributes` is left out when empty, because a document without + it means the same, and each extra field a definition in scope + claims is written as `canonical_of` writes it. Everything else is + kept as written. That includes the documents that + `consolidated_metadata` lists, which a group does not read: + `consolidated().canonicalize()` returns them in their simplest + spelling, and `replace(consolidated_metadata=...)` writes them + back into the group. + """ + return type(self)(self._normal_form, context=self._context) + def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: """Return this document read in `context`, whatever that changes. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically.""" context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) @@ -420,15 +442,8 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadat return self.with_context(context) def refines(self, other: ZarrV3GroupMetadata) -> bool: - """Whether this group holds everything `other` holds: the same attributes, and the same extra fields, each refining its counterpart in the order `refines` defines for declarations. False when `other` is not a v3 group.""" - if type(other) is not type(self): - return False - if json_text(self._members.attributes) != json_text(other._members.attributes): - return False - mine, theirs = self.extra_fields, other.extra_fields - if mine.keys() != theirs.keys(): - return False - return all(refines_declaration(mine[key], theirs[key]) for key in mine) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. The group does not read the documents its `consolidated_metadata` lists, so they are compared as written. False when `other` is not a v3 group.""" + return self._refines(other, lambda: other.with_context(self._context)) # --- constructors ----------------------------------------------------- @@ -585,16 +600,31 @@ def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for consolidated metadata: each document's path and key, sorted by path.""" - return tuple( - (path, node._model_key) - for path, node in sorted(self.metadata.items(), key=lambda item: item[0]) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The member with each document it lists in its simplest equivalent spelling, as that node's normal form writes it.""" + metadata: dict[str, JSONValue] = { + path: dict(node._normal_form) for path, node in self._metadata.items() + } + return {**self._document, "metadata": metadata} + + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of( + located for node in self._metadata.values() for located in node.reading.declarations() ) def __reduce__(self) -> tuple[type[ZarrV3ConsolidatedMetadata], tuple[object, ZarrV3Context]]: return type(self), (self._document, self._context) + def canonicalize(self) -> ZarrV3ConsolidatedMetadata: + """Return this member with each document it lists in its simplest equivalent spelling, as that node's `canonicalize` writes it, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same member, apart from the order of the + members of an object, which a `JSONWriter(sort_keys=True)` sorts. + """ + return type(self)(self._normal_form, context=self._context) + def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: """Return this member with every document read in `context`, whatever that changes. Raises `MetadataValidationError` when a document has a problem in `context`. The existing readings are reused when `context` reads every claim of every document identically.""" context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) @@ -629,14 +659,8 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3Consolidated return self.with_context(context) def refines(self, other: ZarrV3ConsolidatedMetadata) -> bool: - """Whether every document this holds refines the document `other` holds at the same path, and both hold the same paths. False when `other` is not consolidated metadata.""" - if type(other) is not type(self): - return False - if self._metadata.keys() != other._metadata.keys(): - return False - return all( - _node_refines(self._metadata[path], other._metadata[path]) for path in self._metadata - ) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. So every document this holds refines the document `other` holds at the same path, and both hold the same paths. False when `other` is not consolidated metadata.""" + return self._refines(other, lambda: other.with_context(self._context)) @classmethod def from_json( @@ -646,13 +670,6 @@ def from_json( return cls(data, context=context) -def _node_refines(node: ZarrV3NodeMetadata, other: ZarrV3NodeMetadata) -> bool: - """Whether `node` refines `other`. Models of the same kind use their own `refines`; an array never refines a group, or the reverse.""" - if isinstance(node, ZarrV3ArrayMetadata): - return isinstance(other, ZarrV3ArrayMetadata) and node.refines(other) - return isinstance(other, ZarrV3GroupMetadata) and node.refines(other) - - @dataclass(frozen=True, slots=True) class ZarrV3GroupMetadataReading: """A v3 group document as a scope read it: each extra field, every problem, and the model when there are no problems.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_keyed.py b/packages/zarr-metadata/src/zarr_metadata/model/_keyed.py index a26be64c02..e8171ec295 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_keyed.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_keyed.py @@ -1,25 +1,76 @@ -"""The key every model uses for comparison and hashing. The key covers everything the model shows, and is computed once, the first time the model is compared.""" +"""What every model derives from one normal form: its key, `==`, `hash`, `canonicalize` and `refines`. + +A model's normal form is its document in its simplest equivalent +spelling, as its scope reads it. Each model writes that document in one +function, `_canonical_document`. Nothing else in a model decides what +its document means: the key is the text of the normal form with the +claims the scope made, `canonicalize` reads the normal form, and +`refines` reads the other model's document in this model's scope and +compares the two normal forms. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from zarr_metadata._json import MetadataValidationError, json_text + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.definition._scope import Claims + + +def normal_key(claims: Claims, document: Mapping[str, JSONValue]) -> tuple[object, ...]: + """The key of a document in its simplest equivalent spelling, read with `claims`: the claims, and the document as JSON text. + + Two documents mean the same thing when a scope reads the names they + use with the same definitions, and they are one document in their + simplest spelling. The text sorts the members of each object, so the + order a document writes them in is not compared. + """ + return (frozenset(claims.items()), json_text(dict(document))) class Keyed: - """A model whose `==` and `hash` compare a cached key that describes what its document means. - - Each model computes its own key in `_key_of`. `_model_key` computes - the key the first time something asks for it (`==`, `hash`, or a - model that holds this one) and caches it. A model that is only read - never pays for the key. Two models are equal when they have the same - class and equal keys. A model of another class, or any other object, - is never equal to a model. + """A model whose `==`, `hash`, `canonicalize` and `refines` are derived from one normal form. + + Each model writes its normal form in `_canonical_document`, and names + the claims its scope made in `_claims`. The normal form and the key + are computed the first time something asks for them, and cached. A + model that is only read never pays for either. Two models are equal + when they have the same class and equal keys. A model of another + class, or any other object, is never equal to a model. """ - __slots__ = ("_key",) + __slots__ = ("_key", "_normal") _key: tuple[object, ...] + _normal: Mapping[str, JSONValue] - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for this model. Each model class defines its own.""" + def _canonical_document(self) -> Mapping[str, JSONValue]: + """This model's document in its simplest equivalent spelling. Each model class writes its own.""" raise NotImplementedError + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses. Each model class returns its own.""" + raise NotImplementedError + + @property + def _normal_form(self) -> Mapping[str, JSONValue]: + """The normal form. It is computed on first access and cached.""" + try: + return self._normal + except AttributeError: + normal = self._canonical_document() + self._normal = normal + return normal + + def _key_of(self) -> tuple[object, ...]: + """The key `==` and `hash` compare for this model, as `normal_key` computes it.""" + return normal_key(self._claims(), self._normal_form) + @property def _model_key(self) -> tuple[object, ...]: """The key. It is computed on first access and cached.""" @@ -37,3 +88,27 @@ def __eq__(self, other: object) -> bool: def __hash__(self) -> int: return hash(self._model_key) + + def _refines(self, other: object, read_here: Callable[[], object]) -> bool: + """Whether this model holds everything `other` holds: every claim `other` holds, and what `other`'s document reads as in this model's scope. `read_here` returns that reading, and raises `MetadataValidationError` when the document has a problem in this scope. + + A model refines another when its scope read every name the other + scope read, with the same definition, and the other model's + document, read in this model's scope, is this model. So one + reader decides what both documents mean, with everything this + scope knows: the chunk each codec receives, the data type that + spells the fill value, and the definitions of the names the + other scope left unclaimed. A document with a problem in this + scope is not one this model refines. + """ + if type(other) is not type(self): + return False + mine = self._claims() + for key, definition in other._claims().items(): + if definition is not None and mine.get(key) != definition: + return False + try: + again = read_here() + except MetadataValidationError: + return False + return again == self diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py index a946c4a608..2bb83aa52e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py @@ -24,6 +24,8 @@ from zarr_metadata.definition._definition import ( RefusedDeclaration, UnclaimedDeclaration, + canonical_of, + held, is_declaration, ) from zarr_metadata.definition._scope import Conflict, claim_key @@ -204,6 +206,50 @@ def created( raise +def simplest(declaration: ResolvedDeclaration[Any] | UNSET) -> JSONValue: + """`declaration`, from a read that found no problems, in its simplest equivalent spelling, as `canonical_of` writes it. Raises `TypeError` for a declaration with none, which such a read cannot produce.""" + spelled = canonical_of(held(declaration), ()) + if spelled is None: + msg = f"expected a declaration a read found nothing wrong with, got {declaration!r}" + raise TypeError(msg) + return spelled + + +def simplest_members(extra_fields: Mapping[str, ResolvedDeclaration[Any]]) -> dict[str, JSONValue]: + """The extra fields of a document that a read found no problems with, by key, each in its simplest equivalent spelling.""" + spelled: dict[str, JSONValue] = {} + for extra in extra_fields.values(): + # An extra field is written as the member it is: its key, and its value. + spelled.update(cast("Mapping[str, JSONValue]", simplest(extra))) + return spelled + + +def canonical_members( + document: Mapping[str, JSONValue], + spelled: Mapping[str, JSONValue], + *, + empty: Iterable[str] = (), + defaults: Mapping[str, JSONValue] | None = None, +) -> dict[str, JSONValue]: + """`document`, the document a model holds, with each member in `spelled` written as `spelled` writes it, and without the members that say what their absence says. + + A member in `empty` is left out when it is empty, and a member in + `defaults` when it has the value given there. The members keep the + order `document` writes them in. A member of `spelled` that + `document` does not write is not added. + """ + canonical = {key: spelled.get(key, value) for key, value in document.items()} + for key in empty: + value = canonical.get(key) + if (is_array(value) or is_object(value)) and len(value) == 0: + del canonical[key] + if defaults is not None: + for key, default in defaults.items(): + if key in canonical and canonical[key] == default: + del canonical[key] + return canonical + + def unclaimed_declarations_of( declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], waived: Iterable[str] ) -> dict[Loc, UnclaimedDeclaration]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index dc4a3135b1..715d74ab3b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -432,20 +432,24 @@ class ArrayCodecDefinition(CodecDefinition[C]): simplest configuration that means the same for that chunk. It is called only for a codec that the chunk rules found no problem with. It simplifies only the codec's own members, and it must keep each - nested declaration as it is, because each codec of a pipeline the - codec contains is simplified for the chunk it receives in turn. What - does not depend on the chunk is left to `canonical`, which simplifies - the result as it simplifies any configuration. A chunk rule checks + nested declaration as it is given, because each codec of a pipeline + the codec contains is simplified for the chunk it receives first. + What does not depend on the chunk is left to `canonical`: + `chunk_canonical` receives the configuration as `canonical` spells + it, with each nested declaration in its simplest spelling, and + `canonical` simplifies the result as it simplifies any configuration. The result + must be the simplest spelling, so `chunk_canonical` must change + nothing when it is asked about its own result. A chunk rule checks only the parts of a chunk that are known, and `chunk_canonical` simplifies only by them: a chunk with nothing known about it, `Chunk()`, leaves every configuration as it is. `codec_for_chunk` - returns the codec of a `Stage` in that spelling. + writes the codec of a `Stage` in that spelling. """ chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules """The spec violations for this codec with a given input chunk, located in the configuration. A problem at `()` is about the codec as a whole, such as an input it cannot encode, and is reported at the codec.""" chunk_canonical: Callable[[C, Nested, Chunk], C] = unchanged_for_chunk - """A configuration the chunk rules allow for a given input chunk, in the simplest spelling that means the same for that chunk.""" + """A configuration the chunk rules allow for a given input chunk, as `canonical` spells it, in the simplest spelling that means the same for that chunk.""" pipelines: Callable[[C, Nested, Chunk], Mapping[str, Chunk]] = no_pipelines """The chunk each contained pipeline receives, keyed by the configuration member that holds the pipeline.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index 9842d827da..94789e1647 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -39,17 +39,21 @@ is_object, is_tuple, json_text, + refine_user_data, with_input, ) from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._definition import ( AcceptedDeclaration, + Further, ResolvedDeclaration, asked, + canonical_declaration, + canonical_of, + declaration_locations, in_document, item_at, read_only, - respelled, ruled, ) from zarr_metadata.v3._definition import ( @@ -58,6 +62,7 @@ Chunk, CodecDefinition, CodecKind, + unchanged_for_chunk, ) if TYPE_CHECKING: @@ -337,72 +342,105 @@ def _handed_on( return given -def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: - """The codec of `stage`, in the simplest spelling that means the same for the chunk it receives. +def codec_for_chunk(stage: Stage) -> JSONValue | None: + """The codec of `stage` in its simplest equivalent spelling for the chunk it receives, as plain JSON; None when it has none. A codec's own `canonical` reduces the spellings that mean the same thing wherever the codec is used. Some spellings mean the same thing only for some chunks. For example, a `bytes` codec with and without `endian` is one codec for values that take one byte, and two for values that take several. So a codec is compared, and written in its - simplest spelling, as the codec this function returns: first each - codec of each pipeline it contains, for the chunk that codec - receives, and then its own configuration, as its definition's - `chunk_canonical` simplifies it for the chunk in `stage`. - - A codec the scope did not read is returned as it is. A codec's own - configuration is kept as it is when the codec receives bytes, and - when its chunk rules find a problem with it, because a codec with a - problem has no simpler spelling. When nothing is simplified, the - result is the codec itself. Otherwise it is a new declaration, read - by the same definition. `canonical_of` writes it in its simplest - spelling, and `==` compares it. + simplest spelling, as this function writes it: first each codec of + each pipeline it contains, for the chunk that codec receives, then + its own members as its definition's `canonical` spells them, and + then as its definition's `chunk_canonical` simplifies them for the + chunk in `stage`. This is the walk that `canonical_of` makes for any + declaration, with the chunks taking part in it. + + `stage` comes from a read that found no problems, as a model's + pipeline does. A codec the scope left unclaimed is written as it + was, in the shortest form every reader accepts. A codec the scope + refused has no simplest spelling, and neither has a codec that holds + one. A codec's own members are not simplified for the chunk when the + codec receives bytes, or when its chunk rules find a problem with + it. `chunk_canonical` is the extension author's code, so its result is checked: the definition must allow it, the chunk rules must find no - problem with it, and each nested declaration must be where it was. A - result that fails a check raises `ValueError`, which indicates a bug - in the definition, not in the codec. + problem with it, it must hold the nested declarations the codec + holds, each as it was given, and `chunk_canonical` must have nothing + left to simplify in it. A result that fails a check raises + `ValueError`, which indicates a bug in the definition, not in the + codec. """ + spelled = _spelled_for_chunk(stage) + return None if spelled is None else copied(spelled) + + +def _spelled_for_chunk(stage: Stage) -> JSONValue | None: + """What `codec_for_chunk` returns, sharing what it can with the declaration.""" codec = stage.codec if not isinstance(codec, AcceptedDeclaration): - return codec - inner: dict[Loc, ResolvedDeclaration[Any]] = {} - for member, stages in stage.inner.items(): - for index, each in enumerate(stages): - simpler = codec_for_chunk(each) - if simpler is not each.codec: - inner[(member, index)] = simpler - configuration = _for_chunk(codec, stage.incoming) - if configuration is None and len(inner) == 0: - return codec - return respelled(codec, codec.configuration if configuration is None else configuration, inner) - - -def _for_chunk( + return canonical_of(codec, ()) + inner: dict[Loc, JSONValue | None] = { + (member, index): _spelled_for_chunk(each) + for member, stages in stage.inner.items() + for index, each in enumerate(stages) + } + return canonical_declaration(codec, inner, _further(codec, stage.incoming)) + + +def _further( codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None -) -> Mapping[str, JSONValue] | None: - """The configuration of `codec` as its definition's `chunk_canonical` simplifies it for `chunk`, checked; None when nothing is simplified.""" +) -> Further | None: + """How the own members of `codec` are simplified for `chunk`, after its `canonical`; None when they are not: the codec receives bytes, its `chunk_canonical` is the default, or its chunk rules find a problem with it, because a codec with a problem has no simpler spelling.""" definition = codec.definition if chunk is None or not isinstance(definition, ArrayCodecDefinition): return None - configuration, nested = codec.configuration, codec.nested - if len(_chunk_problems(definition, configuration, nested, chunk, ())) != 0: + if definition.chunk_canonical is unchanged_for_chunk: + return None + nested = codec.nested + if len(_chunk_problems(definition, codec.configuration, nested, chunk, ())) != 0: return None + return Further( + "chunk_canonical", + lambda configuration: _asked_for_chunk(definition, configuration, nested, chunk), + ) + + +def _asked_for_chunk( + definition: ArrayCodecDefinition[Any], + configuration: Mapping[str, JSONValue], + nested: Nested, + chunk: Chunk, +) -> dict[str, JSONValue] | None: + """What `definition`'s `chunk_canonical` gives for `configuration`, checked, as JSON with arrays as tuples; None when it changes nothing. + + `configuration` is the read-only configuration in its simplest + spelling so far: the codec's own members as `canonical` spells them, + and each nested declaration in its simplest spelling. So every + spelling that `canonical` reduces to one is simplified for the chunk + in the same way, and `chunk_canonical` does not have to know each of + them. + """ given: object = asked( definition, "chunk_canonical", - lambda: cast("object", definition.chunk_canonical(read_only(configuration), nested, chunk)), + lambda: cast("object", definition.chunk_canonical(configuration, nested, chunk)), ) if given is configuration: return None - if not is_object(given): + # The result is the extension author's value, so it is read as any JSON + # is: an array may be a list, and what is not JSON is refused here, + # naming the definition. + simplified, not_json = refine_user_data(dict(given)) if is_object(given) else (None, ()) + if not isinstance(simplified, dict) or len(not_json) != 0: msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}" raise TypeError(msg) - simplified = cast("dict[str, JSONValue]", copied(cast("JSONValue", dict(given)))) - if json_text(simplified) == json_text(cast("JSONValue", configuration)): + if json_text(simplified) == json_text(cast("JSONValue", dict(configuration))): return None _, refused = definition.read_configuration(simplified) + held_at = frozenset(declaration_locations(definition, simplified)) found = ( *refused, *_chunk_problems(definition, simplified, nested, chunk, ()), @@ -411,7 +449,13 @@ def _for_chunk( loc, "expected the nested declaration to stay as it is", "invalid_value" ) for loc in nested - if _text_at(simplified, loc) != _text_at(cast("JSONValue", configuration), loc) + if _text_at(simplified, loc) != _text_at(cast("JSONValue", dict(configuration)), loc) + ), + *( + ValidationProblem( + loc, "expected no declaration that the codec did not hold", "invalid_value" + ) + for loc in sorted(held_at.difference(nested), key=repr) ), ) if len(found) != 0: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py index 9c20fac4a5..9169e23246 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -94,9 +94,9 @@ for another, which `canonical` cannot tell, since it receives the configuration alone. Such a codec declares its simplest spelling for a given chunk with `chunk_canonical`. For example, `bytes` drops an -`endian` when each value takes one byte. `codec_for_chunk` returns the -codec of a `Stage` in that spelling, with each codec of the pipelines -it contains simplified for the chunk that one receives. An array model +`endian` when each value takes one byte. `codec_for_chunk` writes the +codec of a `Stage` in that spelling, as JSON, with each codec of the +pipelines it contains simplified for the chunk that one receives. An array model compares its codecs this way. The reader does not guess. After a codec the scope did not read, or after a codec that does not declare its output, the next codec receives diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 2d2ae6cf27..406d3e2d8a 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -845,7 +845,7 @@ def _shard_of(*codecs: object) -> dict[str, object]: def test_two_models_are_one_array_when_each_codec_means_the_same_for_its_chunk( data_type: str, left: object, right: object, same: bool ) -> None: - """A codec is compared for the chunk it receives, as `codec_for_chunk` returns it: the `endian` of a `bytes` codec tells two arrays apart only where a value takes several bytes. `refines` agrees with `==`, and `to_json` writes each codec as given.""" + """A codec is compared for the chunk it receives, as `codec_for_chunk` writes it: the `endian` of a `bytes` codec tells two arrays apart only where a value takes several bytes. `refines` agrees with `==`, and `to_json` writes each codec as given.""" document = ZarrV3ArrayMetadata.create_default(shape=(2,), data_type=data_type).to_json() models = [ ZarrV3ArrayMetadata.from_json({**document, "codecs": codecs}) for codecs in (left, right) diff --git a/packages/zarr-metadata/tests/model/test_canonicalize.py b/packages/zarr-metadata/tests/model/test_canonicalize.py new file mode 100644 index 0000000000..25dda00891 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_canonicalize.py @@ -0,0 +1,573 @@ +"""`canonicalize`: each model with its document in its simplest equivalent spelling. + +Two models are equal when their documents mean the same thing, and +`to_json` writes each document as it was given. `canonicalize` returns the +one document that every model equal to a given one writes, apart from the +order of the members of an object, which a `JSONWriter` sorts. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, NotRequired, TypeVar, cast + +import pytest +from hypothesis import given +from hypothesis import strategies as st +from typing_extensions import TypedDict, TypeIs + +from zarr_metadata._json import json_text +from zarr_metadata.definition._definition import canonicalize +from zarr_metadata.model import ( + JSONWriter, + ZarrV2ArrayMetadata, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, +) +from zarr_metadata.testing.strategies import ( + Spellings, + array_documents, + consolidated_group_documents, + group_documents, +) +from zarr_metadata.v2.definition import CORE_V2_CONTEXT, ZarrV2Context +from zarr_metadata.v3.definition import ( + CORE_AND_EXTENSIONS_CONTEXT, + ZARR_PYTHON_CONTEXT, + ArrayExtraFieldDefinition, + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + DataTypeDefinition, + GroupExtraFieldDefinition, + StorageTransformerDefinition, + ZarrV3Context, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata import JSONValue + +Model = ( + ZarrV2ArrayMetadata + | ZarrV2ConsolidatedMetadata + | ZarrV2GroupMetadata + | ZarrV3ArrayMetadata + | ZarrV3ConsolidatedMetadata + | ZarrV3GroupMetadata +) + + +class AcmeLevelConfiguration(TypedDict, closed=True): + level: NotRequired[int] + + +class AcmeNoteConfiguration(TypedDict, closed=True): + must_understand: NotRequired[bool] + level: NotRequired[int] + + +C = TypeVar("C", AcmeLevelConfiguration, AcmeNoteConfiguration) + + +def _without_level_0(configuration: C) -> C: + """Without a `level` of 0, which is what both definitions here do without a `level`.""" + if configuration.get("level") != 0: + return configuration + return cast("C", {key: value for key, value in configuration.items() if key != "level"}) + + +_V3_CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union( + ZarrV3Context( + array_extra_fields=[ + ArrayExtraFieldDefinition( + name="note", configuration=AcmeNoteConfiguration, canonical=_without_level_0 + ) + ], + group_extra_fields=[ + GroupExtraFieldDefinition( + name="note", configuration=AcmeNoteConfiguration, canonical=_without_level_0 + ) + ], + storage_transformers=[ + StorageTransformerDefinition( + name="acme.st", configuration=AcmeLevelConfiguration, canonical=_without_level_0 + ) + ], + ) +) +"""The scope the v3 documents here are read in. It is not the default scope: it also claims an extra field `note` and a storage transformer `acme.st`, each with a `canonical`.""" + +_V2_CONTEXT = CORE_V2_CONTEXT.union(ZarrV2Context()) +"""The scope the v2 documents here are read in: the default scope as another object, which tells whether a model kept the scope it was read in.""" + +_LITTLE: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "little"}} +_BIG: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "big"}} +_NOSHUFFLE = {"cname": "lz4", "clevel": 5, "shuffle": "noshuffle", "blocksize": 0} +_WAIVED = {"name": "acme.note", "must_understand": False, "configuration": {"b": 1, "a": 2}} + +_V3_ARRAY: dict[str, Any] = { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default"}, + "fill_value": 0, + "codecs": [{"name": "bytes"}], +} +"""A v3 array document in its simplest spelling.""" + +_V3_ARRAY_SPELLED_OUT: dict[str, Any] = { + "attributes": {}, + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": {"name": "uint8", "configuration": {}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, + "fill_value": 0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [2], + "index_location": "end", + "codecs": [_BIG, {"name": "blosc", "configuration": {**_NOSHUFFLE, "typesize": 4}}], + "index_codecs": [_LITTLE, {"name": "crc32c", "configuration": {}}], + }, + } + ], + "storage_transformers": [], + "dimension_names": [None], + "acme": _WAIVED, +} +"""A v3 array document that says in more words what `_V3_ARRAY_SIMPLEST` says.""" + +_V3_ARRAY_SIMPLEST: dict[str, Any] = { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default"}, + "fill_value": 0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [2], + "codecs": [{"name": "bytes"}, {"name": "blosc", "configuration": _NOSHUFFLE}], + "index_codecs": [_LITTLE, {"name": "crc32c"}], + }, + } + ], + "dimension_names": [None], + "acme": _WAIVED, +} + +_V3_FLOATS: dict[str, Any] = {**_V3_ARRAY, "data_type": "float32", "codecs": [_LITTLE]} + +_V3_UNCLAIMED: dict[str, Any] = { + **_V3_ARRAY, + "data_type": {"name": "acme.t", "configuration": {"bits": 8, "signed": False}}, + "fill_value": "0x00", + "codecs": [{"name": "acme.codec", "configuration": {"level": 1}}, _LITTLE], + "attributes": {"b": [1.0, {}], "a": None}, +} +"""A v3 array document that holds names nothing in scope claims. Their configurations are kept as written, since only their own definitions could simplify them, and so is the `endian` of the `bytes` codec, which receives values nothing is known of.""" + +_V3_UNCLAIMED_ENVELOPES: dict[str, Any] = { + **_V3_ARRAY, + "data_type": {"name": "acme.t", "configuration": {}}, + "fill_value": "0x00", + "codecs": [ + {"name": "acme.codec", "configuration": {}}, + "acme.other", + {"name": "acme.third", "must_understand": True, "configuration": {"b": 1, "a": {}}}, + _LITTLE, + ], +} +"""A v3 array document that holds names nothing in scope claims, each in an envelope that has a shorter form.""" + +_V3_UNCLAIMED_SHORTEST: dict[str, Any] = { + **_V3_ARRAY, + "data_type": "acme.t", + "fill_value": "0x00", + "codecs": [ + {"name": "acme.codec"}, + {"name": "acme.other"}, + {"name": "acme.third", "configuration": {"b": 1, "a": {}}}, + _LITTLE, + ], +} + + +def _rectilinear(*chunk_shapes: list[int] | list[list[int]]) -> dict[str, Any]: + return { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": list(chunk_shapes)}, + } + + +_V3_GROUP: dict[str, Any] = {"zarr_format": 3, "node_type": "group"} + + +def _v3_listing(*documents: dict[str, Any]) -> dict[str, Any]: + return { + "kind": "inline", + "must_understand": False, + "metadata": {f"n{index}": document for index, document in enumerate(documents)}, + } + + +_V2_ARRAY: dict[str, Any] = { + "zarr_format": 2, + "shape": [4], + "chunks": [2], + "dtype": "|i1", + "compressor": {"id": "zstd", "level": 0, "checksum": False}, + "fill_value": 0, + "order": "C", + "filters": [{"id": "delta", "dtype": "|i1", "astype": "|i1"}], +} +"""A v2 array document in its canonical spelling, in which a codec writes each numcodecs default.""" + +_V2_ARRAY_SPELLED_OTHERWISE: dict[str, Any] = { + **_V2_ARRAY, + "dtype": " dict[str, Any]: + return {key: value for key, value in document.items() if key != "attributes"} + + +@pytest.mark.parametrize( + ("model", "document", "canonical"), + [ + (ZarrV3ArrayMetadata, _V3_ARRAY, _V3_ARRAY), + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _V3_ARRAY_SIMPLEST), + ( + ZarrV3ArrayMetadata, + {**_V3_FLOATS, "fill_value": "0x7fc00000"}, + {**_V3_FLOATS, "fill_value": "NaN"}, + ), + (ZarrV3ArrayMetadata, _V3_UNCLAIMED, _V3_UNCLAIMED), + (ZarrV3ArrayMetadata, _V3_UNCLAIMED_ENVELOPES, _V3_UNCLAIMED_SHORTEST), + ( + ZarrV3ArrayMetadata, + {**_V3_ARRAY, "shape": [6], "chunk_grid": _rectilinear([2, 2, 2])}, + {**_V3_ARRAY, "shape": [6], "chunk_grid": _rectilinear([[2, 3]])}, + ), + ( + ZarrV3ArrayMetadata, + { + **_V3_ARRAY, + "storage_transformers": [{"name": "acme.st", "configuration": {"level": 0}}], + }, + {**_V3_ARRAY, "storage_transformers": [{"name": "acme.st"}]}, + ), + (ZarrV3ArrayMetadata, {**_V3_ARRAY, "note": {"level": 0}}, {**_V3_ARRAY, "note": {}}), + (ZarrV3GroupMetadata, _V3_GROUP, _V3_GROUP), + ( + ZarrV3GroupMetadata, + {"attributes": {}, **_V3_GROUP, "acme": _WAIVED}, + {**_V3_GROUP, "acme": _WAIVED}, + ), + ( + ZarrV3GroupMetadata, + {**_V3_GROUP, "attributes": {"a": 1}}, + {**_V3_GROUP, "attributes": {"a": 1}}, + ), + (ZarrV3GroupMetadata, {**_V3_GROUP, "note": {"level": 0}}, {**_V3_GROUP, "note": {}}), + # A group does not read the documents its consolidated metadata lists. + ( + ZarrV3GroupMetadata, + {**_V3_GROUP, "consolidated_metadata": _v3_listing(_V3_ARRAY_SPELLED_OUT)}, + {**_V3_GROUP, "consolidated_metadata": _v3_listing(_V3_ARRAY_SPELLED_OUT)}, + ), + ( + ZarrV3ConsolidatedMetadata, + _v3_listing(_V3_ARRAY_SPELLED_OUT, {**_V3_GROUP, "attributes": {}}), + _v3_listing(_V3_ARRAY_SIMPLEST, _V3_GROUP), + ), + (ZarrV2ArrayMetadata, _V2_ARRAY, _V2_ARRAY), + (ZarrV2ArrayMetadata, _V2_ARRAY_SPELLED_OTHERWISE, _V2_ARRAY), + (ZarrV2ArrayMetadata, _V2_FLOATS, _V2_FLOATS), + (ZarrV2ArrayMetadata, {**_V2_FLOATS, "fill_value": 0}, {**_V2_FLOATS, "fill_value": 0.0}), + ( + ZarrV2ArrayMetadata, + {**_V2_FLOATS, "fill_value": 1e39}, + {**_V2_FLOATS, "fill_value": "Infinity"}, + ), + (ZarrV2GroupMetadata, _V2_GROUP, _V2_GROUP), + (ZarrV2GroupMetadata, {**_V2_GROUP, "attributes": {}}, _V2_GROUP), + ( + ZarrV2GroupMetadata, + {**_V2_GROUP, "attributes": {"a": 1}}, + {**_V2_GROUP, "attributes": {"a": 1}}, + ), + ( + ZarrV2ConsolidatedMetadata, + { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": _V2_GROUP, + ".zattrs": {}, + "a/.zarray": _v2_array_entry(_V2_ARRAY_SPELLED_OTHERWISE), + "a/.zattrs": {"x": 1}, + "b/.zarray": _v2_array_entry(_V2_ARRAY_SPELLED_OTHERWISE), + "b/.zattrs": {}, + "c/.zattrs": {}, + "notes": [1], + }, + }, + { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": _V2_GROUP, + "a/.zarray": _V2_ARRAY, + "a/.zattrs": {"x": 1}, + "b/.zarray": _V2_ARRAY, + # No node is read from these entries, so they are kept as written. + "c/.zattrs": {}, + "notes": [1], + }, + }, + ), + ], + ids=[ + "v3-array-simplest", + "v3-array-spelled-out", + "v3-array-fill-value", + "v3-array-unclaimed", + "v3-array-unclaimed-envelopes", + "v3-array-chunk-grid", + "v3-array-storage-transformer", + "v3-array-claimed-extra-field", + "v3-group-simplest", + "v3-group-empty-attributes", + "v3-group-attributes", + "v3-group-claimed-extra-field", + "v3-group-consolidated", + "v3-consolidated", + "v2-array-canonical", + "v2-array-spelled-otherwise", + "v2-array-nothing-to-simplify", + "v2-array-fill-value", + "v2-array-fill-value-out-of-range", + "v2-group-simplest", + "v2-group-empty-attributes", + "v2-group-attributes", + "v2-consolidated", + ], +) +def test_a_model_canonicalizes_to_the_simplest_spelling_of_its_document( + model: type[Model], document: dict[str, Any], canonical: dict[str, Any] +) -> None: + """`canonicalize` returns an equal model of the same class, in the same scope, whose document is in its simplest equivalent spelling, with the members that remain in the order they were written. A document already in that spelling is returned as it is.""" + read = _read(model, document) + result = read.canonicalize() + assert json.dumps(result.to_json()) == json.dumps(canonical) + assert type(result) is type(read) + # Neither scope here is the default one, so a model that lost its scope would show. + assert result.context is (_V2_CONTEXT if _is_v2(model) else _V3_CONTEXT) + assert result == read + assert hash(result) == hash(read) + assert json.dumps(result.canonicalize().to_json()) == json.dumps(canonical) + + +def _is_v2( + model: type[Model], +) -> TypeIs[type[ZarrV2ArrayMetadata | ZarrV2ConsolidatedMetadata | ZarrV2GroupMetadata]]: + return issubclass(model, (ZarrV2ArrayMetadata, ZarrV2ConsolidatedMetadata, ZarrV2GroupMetadata)) + + +def _read(model: type[Model], document: Mapping[str, Any]) -> Model: + """The model of `document`, read in `_V2_CONTEXT` or `_V3_CONTEXT`.""" + if _is_v2(model): + return model.from_json(document, context=_V2_CONTEXT) + return model.from_json(document, context=_V3_CONTEXT) + + +def _reordered(document: Mapping[str, Any]) -> dict[str, Any]: + """`document` with the members of every object it holds, at any depth, in the reverse order. The items of an array keep their order.""" + + def reverse(value: object) -> object: + if isinstance(value, dict): + held = cast("dict[str, object]", value) + return {key: reverse(held[key]) for key in reversed(held)} + if isinstance(value, list): + return [reverse(item) for item in cast("list[object]", value)] + return value + + return cast("dict[str, Any]", reverse(dict(document))) + + +@pytest.mark.parametrize( + ("model", "left", "right"), + [ + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _V3_ARRAY_SIMPLEST), + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _reordered(_V3_ARRAY_SIMPLEST)), + (ZarrV3ArrayMetadata, _V3_UNCLAIMED_ENVELOPES, _reordered(_V3_UNCLAIMED_SHORTEST)), + ( + ZarrV3ArrayMetadata, + {**_V3_UNCLAIMED, "storage_transformers": []}, + _reordered(_V3_UNCLAIMED), + ), + (ZarrV3ArrayMetadata, {**_V3_ARRAY, "codecs": [_LITTLE]}, {**_V3_ARRAY, "codecs": [_BIG]}), + (ZarrV3GroupMetadata, {**_V3_GROUP, "attributes": {}}, _V3_GROUP), + ( + ZarrV3ConsolidatedMetadata, + _v3_listing(_V3_ARRAY_SPELLED_OUT, _V3_GROUP), + _v3_listing(_reordered(_V3_ARRAY_SIMPLEST), {**_V3_GROUP, "attributes": {}}), + ), + (ZarrV2ArrayMetadata, _V2_ARRAY_SPELLED_OTHERWISE, _reordered(_V2_ARRAY)), + (ZarrV2GroupMetadata, {**_V2_GROUP, "attributes": {}}, _V2_GROUP), + ], +) +def test_equal_models_canonicalize_to_one_document( + model: type[Model], left: dict[str, Any], right: dict[str, Any] +) -> None: + """Models that are equal give the same canonical document, apart from the order of the members of an object: a `JSONWriter` that sorts them writes the same bytes for each.""" + models = [_read(model, document) for document in (left, right)] + assert models[0] == models[1] + canonical = [each.canonicalize() for each in models] + assert json_text(_json_of(canonical[0])) == json_text(_json_of(canonical[1])) + assert _stored(canonical[0]) == _stored(canonical[1]) + + +@pytest.mark.parametrize( + ("model", "left", "right"), + [ + ( + ZarrV3ArrayMetadata, + {**_V3_ARRAY, "data_type": "uint16", "codecs": [_LITTLE]}, + {**_V3_ARRAY, "data_type": "uint16", "codecs": [_BIG]}, + ), + (ZarrV3ArrayMetadata, _V3_ARRAY, {**_V3_ARRAY, "dimension_names": [None]}), + ( + ZarrV3ArrayMetadata, + {**_V3_FLOATS, "fill_value": 0.0}, + {**_V3_FLOATS, "fill_value": -0.0}, + ), + (ZarrV3GroupMetadata, _V3_GROUP, {**_V3_GROUP, "attributes": {"a": 1}}), + (ZarrV2ArrayMetadata, _V2_ARRAY, {**_V2_ARRAY, "dimension_separator": "/"}), + (ZarrV2ArrayMetadata, {**_V2_ARRAY, "filters": None}, {**_V2_ARRAY, "filters": []}), + ], +) +def test_models_that_differ_canonicalize_to_different_documents( + model: type[Model], left: dict[str, Any], right: dict[str, Any] +) -> None: + """Only spellings that mean the same thing are reduced to one: models that are not equal keep documents that differ.""" + models = [_read(model, document) for document in (left, right)] + assert models[0] != models[1] + written = [json_text(_json_of(each.canonicalize())) for each in models] + assert written[0] != written[1] + + +def _json_of(model: Model) -> JSONValue: + document: Any = model.to_json() + return document + + +def _stored(model: Model) -> object: + """The bytes a store holds for `model`, with the members of every object sorted. A v3 listing is stored in the document of a group.""" + if isinstance(model, ZarrV3ConsolidatedMetadata): + group = ZarrV3GroupMetadata.from_json(_V3_GROUP, context=model.context) + model = group.replace(consolidated_metadata=model) + return model.to_key_value(JSONWriter(sort_keys=True)) + + +def _assert_canonicalizes( + model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata | ZarrV3ConsolidatedMetadata, +) -> None: + """`model` equals its canonical model, which is in the same scope, has nothing left to simplify and reads as the same model from the JSON text a store holds. The document of `model` with the members of every object in the reverse order gives the same canonical document.""" + canonical = model.canonicalize() + assert canonical == model + assert canonical.context is model.context + assert hash(canonical) == hash(model) + written = json.dumps(canonical.to_json()) + assert json.dumps(canonical.canonicalize().to_json()) == written + assert type(model).from_json(json.loads(written), context=ZARR_PYTHON_CONTEXT) == model + if not isinstance(canonical, ZarrV3ConsolidatedMetadata): + document = cast("Mapping[str, Any]", _json_of(canonical)) + for member in ("attributes", "storage_transformers"): + assert member not in document or len(document[member]) != 0 + reordered = type(model).from_json( + _reordered(json.loads(json.dumps(model.to_json()))), context=ZARR_PYTHON_CONTEXT + ) + assert reordered == model + assert json_text(_json_of(reordered.canonicalize())) == json_text(_json_of(canonical)) + + +@pytest.mark.parametrize("spellings", ["all", "canonical"]) +@given(data=st.data()) +def test_every_array_document_canonicalizes(spellings: Spellings, data: st.DataObject) -> None: + document = data.draw(array_documents(spellings=spellings)) + _assert_canonicalizes(ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT)) + + +@given(document=array_documents(spellings="all")) +def test_every_array_document_canonicalizes_as_its_declarations_do( + document: Mapping[str, Any], +) -> None: + """A document, and the same document with each declaration in its own simplest spelling, are one array, and give one canonical document.""" + simpler = dict(document) + for member, kind in ( + ("data_type", DataTypeDefinition), + ("chunk_grid", ChunkGridDefinition), + ("chunk_key_encoding", ChunkKeyEncodingDefinition), + ): + simpler[member], problems = canonicalize(document[member], kind, ZARR_PYTHON_CONTEXT) + assert problems == () + simpler["codecs"] = [ + canonicalize(codec, CodecDefinition, ZARR_PYTHON_CONTEXT)[0] for codec in document["codecs"] + ] + models = [ + ZarrV3ArrayMetadata.from_json(each, context=ZARR_PYTHON_CONTEXT) + for each in (document, simpler) + ] + assert models[0] == models[1] + canonical = [json_text(_json_of(each.canonicalize())) for each in models] + assert canonical[0] == canonical[1] + + +@given(document=group_documents()) +def test_every_group_document_canonicalizes(document: Mapping[str, Any]) -> None: + _assert_canonicalizes(ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT)) + + +@given(document=consolidated_group_documents()) +def test_every_consolidated_group_document_canonicalizes( + document: Mapping[str, Any], +) -> None: + group = ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT) + _assert_canonicalizes(group) + listing = group.consolidated() + assert listing is not None + _assert_canonicalizes(listing) + # The listing in its simplest spelling can be written back into the group. + _assert_canonicalizes(group.replace(consolidated_metadata=listing.canonicalize())) diff --git a/packages/zarr-metadata/tests/model/test_invariants.py b/packages/zarr-metadata/tests/model/test_invariants.py new file mode 100644 index 0000000000..84e11290ca --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_invariants.py @@ -0,0 +1,96 @@ +"""What holds for every v3 document the strategies generate: how a model compares with the same document read in another scope, or written in another order.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import pytest +from hypothesis import given + +from zarr_metadata.model import ZarrV3ArrayMetadata, ZarrV3GroupMetadata +from zarr_metadata.testing.strategies import ( + array_documents, + consolidated_group_documents, + group_documents, +) +from zarr_metadata.v3.definition import CORE_CONTEXT, ZARR_PYTHON_CONTEXT, ZarrV3Context + + +@pytest.mark.parametrize( + "poorer", + [ + ZarrV3Context(), + CORE_CONTEXT, + ZARR_PYTHON_CONTEXT.copy(update={"data_types": {}}), + ZARR_PYTHON_CONTEXT.copy(update={"codecs": {}}), + ], + ids=["empty", "core", "no-data-types", "no-codecs"], +) +@given(document=array_documents()) +def test_every_array_read_in_a_richer_scope_refines_the_one_a_poorer_scope_read( + poorer: ZarrV3Context, document: Mapping[str, Any] +) -> None: + """`refined_in` returns the model that the richer scope reads, and that model refines the one it came from, whatever the richer scope learns of the chunk each codec receives.""" + low = ZarrV3ArrayMetadata.from_json(document, context=poorer) + high = low.refined_in(ZARR_PYTHON_CONTEXT) + assert high == ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT) + assert high.refines(low) + assert low.refines(low) + + +def _reordered(value: object) -> Any: # noqa: ANN401 + """`value` with the members of every object it holds, at any depth, in the reverse order. The items of an array keep their order.""" + if isinstance(value, Mapping): + return {key: _reordered(value[key]) for key in reversed(value)} + if isinstance(value, (list, tuple)): + return [_reordered(item) for item in value] + return value + + +def _assert_one_model( + model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, + reordered: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, +) -> None: + assert reordered == model + assert hash(reordered) == hash(model) + assert reordered.refines(model) # pyright: ignore[reportArgumentType] + assert model.refines(reordered) # pyright: ignore[reportArgumentType] + + +@given(document=array_documents()) +def test_every_array_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + """The order of the members of an object means nothing, at any depth: in the document, in a configuration, and in a configuration that holds declarations.""" + _assert_one_model( + ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT), + ZarrV3ArrayMetadata.from_json(_reordered(document), context=ZARR_PYTHON_CONTEXT), + ) + + +@given(document=group_documents()) +def test_every_group_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + _assert_one_model( + ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT), + ZarrV3GroupMetadata.from_json(_reordered(document), context=ZARR_PYTHON_CONTEXT), + ) + + +@given(document=consolidated_group_documents()) +def test_every_listing_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + """A group lists the documents of its nodes. Each is read, so the listing is one model whatever order the members of those documents are in.""" + listings = [ + ZarrV3GroupMetadata.from_json(each, context=ZARR_PYTHON_CONTEXT).consolidated() + for each in (document, _reordered(document)) + ] + assert listings[0] is not None + assert listings[1] is not None + assert listings[1] == listings[0] + assert hash(listings[1]) == hash(listings[0]) + assert listings[1].refines(listings[0]) + assert listings[0].refines(listings[1]) diff --git a/packages/zarr-metadata/tests/model/test_pair.py b/packages/zarr-metadata/tests/model/test_pair.py index 998a45e3f2..0cc7df2335 100644 --- a/packages/zarr-metadata/tests/model/test_pair.py +++ b/packages/zarr-metadata/tests/model/test_pair.py @@ -265,6 +265,40 @@ def test_error_replace_refuses_a_document_with_a_problem() -> None: ZarrV3ArrayMetadata(ARRAY).replace(dimension_names=("x", "y")) +def _core_without_codecs(*names: str) -> ZarrV3Context: + """`CORE_CONTEXT` without the codecs it files under `names`.""" + kept = {name: codec for name, codec in CORE_CONTEXT.codecs.items() if name not in names} + return CORE_CONTEXT.copy(update={"codecs": kept}) + + +def _core_without_data_types(*names: str) -> ZarrV3Context: + """`CORE_CONTEXT` without the data types it files under `names`.""" + kept = {name: each for name, each in CORE_CONTEXT.data_types.items() if name not in names} + return CORE_CONTEXT.copy(update={"data_types": kept}) + + +LITTLE: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "little"}} +ENDIAN: dict[str, Any] = {**ARRAY, "codecs": [LITTLE]} +"""A `uint8` array whose `bytes` codec writes an `endian`, which means nothing for values of one byte. A scope that reads both the codec and the data type compares the codec without it.""" +TRANSPOSED: dict[str, Any] = { + **ARRAY, + "codecs": [{"name": "transpose", "configuration": {"order": [0]}}, LITTLE], +} +SHARDED_ENDIAN: dict[str, Any] = { + **ARRAY, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [1], + "codecs": [LITTLE], + "index_codecs": [LITTLE, "crc32c"], + }, + } + ], +} + + @pytest.mark.parametrize( ("model", "context", "gained"), [ @@ -272,8 +306,48 @@ def test_error_replace_refuses_a_document_with_a_problem() -> None: (ZarrV3ArrayMetadata(ARRAY, context=CORE_CONTEXT), CORE_AND_EXTENSIONS_CONTEXT, False), (ZarrV3ArrayMetadata(FLOAT, context=ZarrV3Context()), CORE_CONTEXT, True), (ZarrV3ArrayMetadata(SHARDED, context=ZarrV3Context()), CORE_CONTEXT, True), + # What the new scope claims tells it more of the chunk a codec + # receives, so it compares that codec in a simpler spelling than + # the one the document writes. + (ZarrV3ArrayMetadata(ENDIAN, context=ZarrV3Context()), CORE_CONTEXT, True), + ( + ZarrV3ArrayMetadata(ENDIAN, context=_core_without_codecs("bytes")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(ENDIAN, context=_core_without_data_types("uint8")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(TRANSPOSED, context=_core_without_codecs("transpose")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(SHARDED_ENDIAN, context=_core_without_codecs("sharding_indexed")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(SHARDED_ENDIAN, context=_core_without_codecs("bytes")), + CORE_CONTEXT, + True, + ), + ], + ids=[ + "gain", + "nothing-to-gain", + "gain-data-type-with-fill-value", + "gain-of-a-shard", + "gain-of-everything-with-an-endian-that-means-nothing", + "gain-of-a-codec-simplified-for-its-chunk", + "gain-of-the-data-type-a-codec-is-simplified-by", + "gain-of-a-codec-before-one-simplified-for-its-chunk", + "gain-of-a-shard-whose-codecs-are-simplified", + "gain-of-a-codec-in-a-shard", ], - ids=["gain", "nothing-to-gain", "gain-data-type-with-fill-value", "gain-of-a-shard"], ) def test_refined_in_moves_a_model_up_the_order( model: ZarrV3ArrayMetadata, context: ZarrV3Context, gained: bool diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index 359275a18f..21e4aafbe6 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -34,7 +34,7 @@ configuration_of, resolve, ) -from zarr_metadata.definition._definition import resolve_refined +from zarr_metadata.definition._definition import item_at, resolve_refined from zarr_metadata.model import ( UNSET, ZarrV3ArrayMetadata, @@ -1185,6 +1185,14 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: CRC32C: Final = {"name": "crc32c"} +def _shard_of(codec: JSONValue, *, index_first: bool = False, chunk_shape: int = 2) -> JSONValue: + """A shard that holds `codec`, with its `index_codecs` written before its `codecs` when `index_first`.""" + members: dict[str, JSONValue] = {"codecs": [codec], "index_codecs": [LE, CRC32C]} + if index_first: + members = dict(reversed(members.items())) + return {"name": "sharding_indexed", "configuration": {"chunk_shape": [chunk_shape], **members}} + + @pytest.mark.parametrize( ("one", "other", "kind", "equal"), [ @@ -1233,6 +1241,12 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: CodecDefinition, True, ), + ( + _shard_of(_shard_of(LE)), + _shard_of(_shard_of(LE, index_first=True), index_first=True), + CodecDefinition, + True, + ), ( {"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [LE, CRC32C]}}, {"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [CRC32C, LE]}}, @@ -1321,6 +1335,7 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: "unclaimed-another-configuration", "another-name", "members-holding-declarations-in-another-order", + "members-holding-declarations-in-another-order-two-deep", "declarations-in-another-order", "members-in-another-order", "blosc-typesize-noshuffle", @@ -1348,6 +1363,20 @@ def test_two_declarations_are_equal_when_they_read_the_same( assert hash(first) == hash(second) +def test_two_refused_declarations_are_equal_whatever_order_their_members_are_in() -> None: + """A refused declaration is compared by what was written and by the declarations it holds, in the order of their locations, not of the members that hold them.""" + # A shard of inner chunks with no elements is refused, and still holds its codecs. + first, _ = resolve(_shard_of(LE, chunk_shape=0), CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) + second, _ = resolve( + _shard_of(LE, chunk_shape=0, index_first=True), CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT + ) + assert isinstance(first, RefusedDeclaration) + assert isinstance(second, RefusedDeclaration) + assert list(first.nested) != list(second.nested) + assert first == second + assert hash(first) == hash(second) + + def test_a_declaration_copied_or_pickled_is_read_by_a_definition_equal_to_its_own() -> None: declaration, _ = resolve( {"name": "gzip", "configuration": {"level": 1}}, CodecDefinition, CORE_CONTEXT @@ -1824,3 +1853,23 @@ def test_error_resolve_refuses_a_kind_of_the_other_format_whatever_the_declarati """`resolve` with a kind of one format and a scope of the other raises one `TypeError`, before it reads the declaration. So the message is the same for a declaration with a name that a scope could look up and for a value with no name.""" with pytest.raises(TypeError, match=f"^{re.escape(message)}$"): resolve(declaration, kind, context) + + +@pytest.mark.parametrize( + ("path", "item"), + [ + ((), {"a": ({"b": 1}, 2)}), + (("a",), ({"b": 1}, 2)), + (("a", 0, "b"), 1), + (("a", 1), 2), + (("a", 2), UNSET), + (("a", -1), UNSET), + (("b",), UNSET), + (("a", "b"), UNSET), + ((0,), UNSET), + (("a", 1, "b"), UNSET), + ], +) +def test_item_at_follows_a_location_into_json(path: tuple[str | int, ...], item: object) -> None: + """`item_at` returns the item at a location, and `UNSET` when the location does not lead into the value: a key an object lacks, an index outside an array, or a step into a scalar.""" + assert item_at({"a": ({"b": 1}, 2)}, path) == item diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index fe477dc8f0..56d1358822 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -9,7 +9,7 @@ from __future__ import annotations import dataclasses -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, NotRequired, cast import pytest from typing_extensions import TypedDict @@ -31,6 +31,10 @@ read_pipeline, ) from zarr_metadata.v3.codec.bytes import BYTES_CODEC, BytesCodecConfiguration +from zarr_metadata.v3.codec.sharding_indexed import ( + SHARDING_INDEXED_CODEC, + ShardingIndexedCodecConfiguration, +) from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, ArrayArrayCodecDefinition, @@ -73,7 +77,39 @@ def _tiled(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) -> C ) """An array -> array codec that does not declare its output chunk.""" -CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(codecs=[ACME_TILE, ACME_SHUFFLE])) + +class AcmePackConfiguration(TypedDict, closed=True): + mode: NotRequired[str] + + +def _lower_mode(configuration: AcmePackConfiguration) -> AcmePackConfiguration: + """`mode` in lower case, since `acme.pack` reads it whatever its case.""" + if "mode" not in configuration: + return configuration + return {"mode": configuration["mode"].lower()} + + +def _mode_for_chunk( + configuration: AcmePackConfiguration, nested: Nested, chunk: Chunk +) -> AcmePackConfiguration: + """Without a `mode` of `"auto"`, which is what `acme.pack` does without a `mode` for a data type the scope read.""" + if isinstance(chunk.data_type, AcceptedDeclaration) and configuration.get("mode") == "auto": + return {} + return configuration + + +ACME_PACK = ArrayBytesCodecDefinition( + name="acme.pack", + configuration=AcmePackConfiguration, + size="static", + canonical=_lower_mode, + chunk_canonical=_mode_for_chunk, +) +"""An array -> bytes codec with both a `canonical` and a `chunk_canonical`: the second simplifies what the first writes.""" + +CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union( + ZarrV3Context(codecs=[ACME_TILE, ACME_SHUFFLE, ACME_PACK]) +) UINT8 = resolve("uint8", DataTypeDefinition, CONTEXT)[0] @@ -573,11 +609,15 @@ def _shard(*codecs: JSONValue, index_codecs: tuple[JSONValue, ...] = (LITTLE, CR } +def _pack(mode: str) -> JSONValue: + return {"name": "acme.pack", "configuration": {"mode": mode}} + + def _simplest(codecs: list[JSONValue], chunk: Chunk) -> list[JSONValue]: """Each codec of a pipeline that receives `chunk`, in its simplest spelling for the chunk it receives.""" stages, problems = _read(codecs, chunk) assert problems == () - return [canonical_of(codec_for_chunk(stage), ()) for stage in stages] + return [codec_for_chunk(stage) for stage in stages] @pytest.mark.parametrize( @@ -607,26 +647,88 @@ def _simplest(codecs: list[JSONValue], chunk: Chunk) -> list[JSONValue]: ("acme.t", [LITTLE], [LITTLE]), ("uint8", ["acme.shuffle", LITTLE], [{"name": "acme.shuffle"}, LITTLE]), ("uint8", ["acme.unclaimed", LITTLE], [{"name": "acme.unclaimed"}, LITTLE]), + # A codec is simplified for its chunk in the spelling its `canonical` gives it. + ("uint8", [_pack("auto")], [{"name": "acme.pack"}]), + ("uint8", [_pack("AUTO")], [{"name": "acme.pack"}]), + ("uint8", [_pack("FAST")], [_pack("fast")]), + ("acme.t", [_pack("AUTO")], [_pack("auto")]), ], ) def test_a_codec_is_written_in_the_simplest_spelling_for_the_chunk_it_receives( data_type: JSONValue, codecs: list[JSONValue], simplest: list[JSONValue] ) -> None: - """`codec_for_chunk` returns each codec as its `chunk_canonical` simplifies it for the chunk it receives, and each codec of a pipeline it contains for the chunk that one receives. A codec with nothing to simplify is returned as it is.""" + """`codec_for_chunk` writes each codec as its `chunk_canonical` simplifies it for the chunk it receives, and each codec of a pipeline it contains for the chunk that one receives. A codec with nothing to simplify is written as `canonical_of` writes it.""" found = _simplest(codecs, _of(data_type)) assert json_text(found) == json_text(simplest) # The simplest spelling reads without a problem, and has nothing left to simplify. assert json_text(_simplest(found, _of(data_type))) == json_text(simplest) stages, _ = _read(simplest, _of(data_type)) - assert all(codec_for_chunk(stage) is stage.codec for stage in stages) + assert [codec_for_chunk(stage) for stage in stages] == [ + canonical_of(stage.codec, ()) for stage in stages + ] def test_a_codec_with_a_problem_for_its_chunk_is_not_simplified() -> None: - """A codec the chunk rules find a problem with is returned as it is, as is a codec the scope refused.""" + """A codec the chunk rules find a problem with keeps its own configuration, and its `chunk_canonical` is not asked. A codec the scope refused has no simplest spelling.""" # `bytes` does not encode values that vary in size, and `gzip` requires a level. stages, problems = _read([LITTLE, {"name": "gzip", "configuration": {}}], _of("string")) assert [problem.loc for problem in problems] == [(1, "configuration", "level"), (0,)] - assert all(codec_for_chunk(stage) is stage.codec for stage in stages) + assert [codec_for_chunk(stage) for stage in stages] == [LITTLE, None] + # A `chunk_canonical` that would drop the `endian` whatever the chunk is not asked either. + eager = _bytes_with(lambda configuration, nested, chunk: {}) + stages, problems = read_pipeline([eager], _of("string")) + assert [problem.loc for problem in problems] == [(0,)] + assert codec_for_chunk(stages[0]) == { + "name": "acme.bytes", + "configuration": {"endian": "little"}, + } + + +class AcmeLeveledConfiguration(TypedDict, closed=True): + codecs: tuple[ZarrV3CodecDeclaration, ...] + level: NotRequired[int] + + +def _leveled(chunk_canonical: object, *codecs: JSONValue) -> Stage: + """The stage, for `CHUNK`, of a codec `acme.leveled` that holds `codecs` and a `level` of 3, and whose `chunk_canonical` is `chunk_canonical`. It declares no pipelines, so the codecs it holds are left as they are.""" + leveled = ArrayBytesCodecDefinition( + name="acme.leveled", + configuration=AcmeLeveledConfiguration, + size="dynamic", + chunk_canonical=chunk_canonical, # pyright: ignore[reportArgumentType] + ) + declaration = {"name": "acme.leveled", "configuration": {"codecs": list(codecs), "level": 3}} + codec = resolve(declaration, CodecDefinition, CONTEXT.union(ZarrV3Context(codecs=[leveled])))[0] + return read_pipeline([codec], CHUNK)[0][0] + + +REFUSED_GZIP: JSONValue = {"name": "gzip", "configuration": {"level": 99}, "must_understand": False} +"""A codec with problems of its own: no such level, and a codec may not waive `must_understand`. A configuration that holds it keeps it as written.""" + + +@pytest.mark.parametrize( + ("codecs", "chunk_canonical", "simplest"), + [ + # An array of the result may be a list. + ( + [LITTLE, GZIP], + lambda configuration, nested, chunk: {"codecs": list(configuration["codecs"])}, + {"name": "acme.leveled", "configuration": {"codecs": (LITTLE, GZIP)}}, + ), + # A codec that holds a refused one has no simplest spelling, so nothing is asked. + ( + [LITTLE, REFUSED_GZIP], + lambda configuration, nested, chunk: {"codecs": configuration["codecs"]}, + None, + ), + ], + ids=["an-array-as-a-list", "a-nested-declaration-with-a-problem"], +) +def test_a_chunk_canonical_that_simplifies_its_own_members_is_taken_at_its_word( + codecs: list[JSONValue], chunk_canonical: object, simplest: JSONValue +) -> None: + """A result that drops the codec's own `level` and keeps each nested declaration as it was given passes every check, however it spells its arrays. A codec that holds a declaration its definition refused is not written at all.""" + assert codec_for_chunk(_leveled(chunk_canonical, *codecs)) == simplest def _bytes_with(chunk_canonical: object) -> ResolvedDeclaration[CodecDefinition[Any]]: @@ -655,12 +757,43 @@ def test_error_a_chunk_canonical_the_chunk_rules_do_not_allow() -> None: codec_for_chunk(stage) -def test_error_a_chunk_canonical_that_gives_something_else() -> None: - stage = _stage(_bytes_with(lambda configuration, nested, chunk: "little"), CHUNK) +def test_error_a_chunk_canonical_with_something_left_to_simplify() -> None: + # Asked about `little` it gives `big`, and asked about `big` it gives `little`. + def chunk_canonical( + configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk + ) -> BytesCodecConfiguration: + return {"endian": "big" if configuration.get("endian") == "little" else "little"} + + stage = _stage(_bytes_with(chunk_canonical), CHUNK) + with pytest.raises( + ValueError, match="'acme.bytes': its chunk_canonical gave .* not the simplest" + ): + codec_for_chunk(stage) + + +@pytest.mark.parametrize( + "given", ["little", {"endian": {"little"}}, {1: "little"}], ids=["a-string", "a-set", "a-key"] +) +def test_error_a_chunk_canonical_that_gives_something_else(given: object) -> None: + """A result that is not an object of JSON is refused by name, before anything reads it.""" + stage = _stage(_bytes_with(lambda configuration, nested, chunk: given), CHUNK) with pytest.raises(TypeError, match="'acme.bytes': its chunk_canonical gives a configuration"): codec_for_chunk(stage) +def test_error_a_chunk_canonical_that_adds_a_nested_declaration() -> None: + # The codec it adds is one no scope read, so the result would hold a + # declaration that its `nested` does not. + def chunk_canonical( + configuration: AcmeLeveledConfiguration, nested: Nested, chunk: Chunk + ) -> AcmeLeveledConfiguration: + added = cast("ZarrV3CodecDeclaration", {"name": "crc32c"}) + return {"codecs": (*configuration["codecs"], added)} + + with pytest.raises(ValueError, match="expected no declaration that the codec did not hold"): + codec_for_chunk(_leveled(chunk_canonical, LITTLE, GZIP)) + + def test_error_a_chunk_canonical_that_raises_says_whose_it_is() -> None: def chunk_canonical( configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk @@ -702,3 +835,37 @@ def chunk_canonical( assert isinstance(codec, AcceptedDeclaration) with pytest.raises(ValueError, match="expected the nested declaration to stay as it is"): codec_for_chunk(_stage(codec, CHUNK)) + + +def test_a_chunk_canonical_receives_the_configuration_in_its_simplest_spelling() -> None: + """`chunk_canonical` is asked after everything else is simplified: the codec's own members are as `canonical` spells them, and each codec of a pipeline it contains is already written for the chunk that one receives.""" + seen: list[str] = [] + + def chunk_canonical( + configuration: ShardingIndexedCodecConfiguration, nested: Nested, chunk: Chunk + ) -> ShardingIndexedCodecConfiguration: + seen.append(json_text(cast("JSONValue", configuration))) + return configuration + + shard = dataclasses.replace( + SHARDING_INDEXED_CODEC, name="acme.shard", chunk_canonical=chunk_canonical + ) + declaration = { + "name": "acme.shard", + "configuration": { + "chunk_shape": [2, 1], + "codecs": [LITTLE], + "index_codecs": [LITTLE, "crc32c"], + "index_location": "end", + }, + } + codec = resolve(declaration, CodecDefinition, CONTEXT.union(ZarrV3Context(codecs=[shard])))[0] + written = codec_for_chunk(_stage(codec, _of("uint8"))) + # Each inner value takes one byte, an index value eight, and `end` is the default location. + simplest: JSONValue = { + "chunk_shape": [2, 1], + "codecs": [BARE], + "index_codecs": [LITTLE, CRC32C], + } + assert seen == [json_text(simplest)] + assert json_text(written) == json_text({"name": "acme.shard", "configuration": simplest}) diff --git a/packages/zarr-metadata/tests/v3/test_scope.py b/packages/zarr-metadata/tests/v3/test_scope.py index 5ccc6b425d..678dab9805 100644 --- a/packages/zarr-metadata/tests/v3/test_scope.py +++ b/packages/zarr-metadata/tests/v3/test_scope.py @@ -2,6 +2,7 @@ from __future__ import annotations +import dataclasses import pickle from typing import Any @@ -20,7 +21,8 @@ ) from zarr_metadata.definition._context import disagreements from zarr_metadata.definition._definition import declarations_of -from zarr_metadata.definition._scope import Disagreements, claims_of, kind_name, refines +from zarr_metadata.definition._scope import Disagreements, claims_of, kind_name +from zarr_metadata.model import ZarrV3ArrayMetadata from zarr_metadata.v3.codec.bytes import BYTES_CODEC from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC from zarr_metadata.v3.codec.gzip import GZIP_CODEC @@ -166,21 +168,18 @@ def test_error_claims_of_refuses_one_name_read_two_ways() -> None: ) +ARRAY: dict[str, Any] = dict(ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json()) +"""A `uint8` array of one chunk of two values.""" GZIP_DECLARATION = {"name": "gzip", "configuration": {"level": 5}} +GZIP_WITH_AN_UNKNOWN_KEY = {"name": "gzip", "configuration": {"level": 5, "extra": 1}} +"""A gzip that the core definition reads with a problem, and that a scope without a gzip accepts as written.""" +STATIC_GZIP = dataclasses.replace(GZIP_CODEC, size="static") +"""A second definition under the core gzip's name that reads the same configurations.""" ZSTD_DECLARATION = {"name": "zstd", "configuration": {"level": 3}} BLOSC = {"cname": "zstd", "shuffle": "noshuffle", "blocksize": 0} BLOSC_LEVEL_ONE = {"name": "blosc", "configuration": {**BLOSC, "clevel": 1}} BLOSC_LEVEL_TRUE = {"name": "blosc", "configuration": {**BLOSC, "clevel": True}} """Two documents that Python's `==` treats as equal, but that `json_text` tells apart.""" -SHARD_HOLDING_REFUSED = { - "name": "sharding_indexed", - "configuration": { - "chunk_shape": [1], - "codecs": ["bytes", {"name": "gzip", "configuration": {"level": 12}}], - "index_codecs": [{"name": "bytes", "configuration": {"endian": "little"}}, "crc32c"], - }, -} -"""A shard that is read, holding a gzip that its definition refuses.""" NESTED_ZSTD = { "name": "sharding_indexed", "configuration": { @@ -191,74 +190,60 @@ def test_error_claims_of_refuses_one_name_read_two_ways() -> None: } +def _array(codec: object, context: ZarrV3Context) -> ZarrV3ArrayMetadata: + """A `uint8` array of one chunk of two values whose pipeline ends in `codec`, read in `context`. A codec that takes bytes follows a `bytes` codec.""" + shard = isinstance(codec, dict) and codec["name"] == "sharding_indexed" + codecs = [codec] if shard else [{"name": "bytes"}, codec] + return ZarrV3ArrayMetadata({**ARRAY, "codecs": codecs}, context=context) + + @pytest.mark.parametrize( - ("declaration", "other", "expected"), + ("model", "other", "expected"), [ + (_array(GZIP_DECLARATION, CORE_CONTEXT), _array(GZIP_DECLARATION, CORE_CONTEXT), True), + (_array({"name": "crc32c"}, CORE_CONTEXT), _array("crc32c", CORE_CONTEXT), True), ( - _read(GZIP_DECLARATION, CodecDefinition, CORE_CONTEXT), - _read(GZIP_DECLARATION, CodecDefinition, CORE_CONTEXT), + _array(ZSTD_DECLARATION, CORE_AND_EXTENSIONS_CONTEXT), + _array(ZSTD_DECLARATION, CORE_CONTEXT), True, ), ( - _read({"name": "crc32c"}, CodecDefinition, CORE_CONTEXT), - _read("crc32c", CodecDefinition, CORE_CONTEXT), - True, - ), - ( - _read(ZSTD_DECLARATION, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT), - _read(ZSTD_DECLARATION, CodecDefinition, CORE_CONTEXT), - True, - ), - ( - _read(ZSTD_DECLARATION, CodecDefinition, CORE_CONTEXT), - _read(ZSTD_DECLARATION, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT), + _array(ZSTD_DECLARATION, CORE_CONTEXT), + _array(ZSTD_DECLARATION, CORE_AND_EXTENSIONS_CONTEXT), False, ), ( - _read(GZIP_DECLARATION, CodecDefinition, CORE_CONTEXT), - _read(GZIP_DECLARATION, CodecDefinition, ZarrV3Context(codecs=[MY_GZIP])), + _array(GZIP_DECLARATION, CORE_CONTEXT), + _array(GZIP_DECLARATION, ZarrV3Context(codecs=[STATIC_GZIP])), False, ), ( - _read(NESTED_ZSTD, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT), - _read(NESTED_ZSTD, CodecDefinition, CORE_CONTEXT), + _array(NESTED_ZSTD, CORE_AND_EXTENSIONS_CONTEXT), + _array(NESTED_ZSTD, CORE_CONTEXT), True, ), ( - _read(NESTED_ZSTD, CodecDefinition, CORE_CONTEXT), - _read(NESTED_ZSTD, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT), + _array(NESTED_ZSTD, CORE_CONTEXT), + _array(NESTED_ZSTD, CORE_AND_EXTENSIONS_CONTEXT), False, ), + (_array("zstd", CORE_CONTEXT), _array("zstd", CORE_CONTEXT), True), ( - _read({"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_CONTEXT), - _read("gzip", CodecDefinition, CORE_CONTEXT), + _array({"name": "zstd", "configuration": {"level": 1}}, CORE_CONTEXT), + _array({"name": "zstd", "configuration": {"level": 2}}, CORE_CONTEXT), False, ), + (_array("crc32c", CORE_CONTEXT), _array({"name": "crc32c"}, ZarrV3Context()), True), ( - _read("zstd", CodecDefinition, CORE_CONTEXT), - _read("zstd", CodecDefinition, CORE_CONTEXT), - True, - ), - ( - _read({"name": "zstd", "configuration": {"level": 1}}, CodecDefinition, CORE_CONTEXT), - _read({"name": "zstd", "configuration": {"level": 2}}, CodecDefinition, CORE_CONTEXT), + _array(BLOSC_LEVEL_ONE, CORE_AND_EXTENSIONS_CONTEXT), + _array(BLOSC_LEVEL_TRUE, ZarrV3Context()), False, ), ( - _read("crc32c", CodecDefinition, CORE_CONTEXT), - _read({"name": "crc32c"}, CodecDefinition, ZarrV3Context()), - True, - ), - ( - _read(BLOSC_LEVEL_ONE, CodecDefinition, CORE_CONTEXT), - _read(BLOSC_LEVEL_TRUE, CodecDefinition, ZarrV3Context()), + _array(GZIP_DECLARATION, CORE_CONTEXT), + _array(GZIP_WITH_AN_UNKNOWN_KEY, ZarrV3Context()), False, ), - ( - _read(SHARD_HOLDING_REFUSED, CodecDefinition, CORE_CONTEXT), - _read(SHARD_HOLDING_REFUSED, CodecDefinition, CORE_CONTEXT), - True, - ), ], ids=[ "same", @@ -268,19 +253,18 @@ def test_error_claims_of_refuses_one_name_read_two_ways() -> None: "conflict", "nested-gain", "nested-loss", - "refused", "both-unclaimed", "unclaimed-differ", "gain-over-another-spelling", "no-gain-over-true-for-one", - "holding-a-refused-declaration", + "no-gain-over-a-problem", ], ) def test_refines_orders_readings_by_information( - declaration: ResolvedDeclaration[Any], other: ResolvedDeclaration[Any], expected: bool + model: ZarrV3ArrayMetadata, other: ZarrV3ArrayMetadata, expected: bool ) -> None: - """`declaration` refines `other` when it reads the same where both read, and gains where `other` left a name unclaimed, including in the declarations it holds. A loss, a conflict, a refused declaration, or two unclaimed declarations written differently do not refine.""" - assert refines(declaration, other) is expected + """A model refines another when it reads the same where both read, and gains where the other left a name unclaimed, including in the declarations a codec holds. A loss, a conflict, or two unclaimed declarations written differently do not refine. Neither does a document that the refining scope reads with a problem: what it wrote is not what the refining model holds.""" + assert model.refines(other) is expected DOCUMENTS = [ @@ -291,25 +275,22 @@ def test_refines_orders_readings_by_information( ZSTD_DECLARATION, NESTED_ZSTD, BLOSC_LEVEL_ONE, - SHARD_HOLDING_REFUSED, ] CONTEXTS = [ZarrV3Context(), CORE_CONTEXT, CORE_AND_EXTENSIONS_CONTEXT] -READINGS = [ - _read(document, CodecDefinition, context) for document in DOCUMENTS for context in CONTEXTS -] +READINGS = [_array(document, context) for document in DOCUMENTS for context in CONTEXTS] @given(st.sampled_from(READINGS), st.sampled_from(READINGS), st.sampled_from(READINGS)) def test_refines_is_a_partial_order_whose_bottom_is_equality( - a: ResolvedDeclaration[Any], b: ResolvedDeclaration[Any], c: ResolvedDeclaration[Any] + a: ZarrV3ArrayMetadata, b: ZarrV3ArrayMetadata, c: ZarrV3ArrayMetadata ) -> None: - """Over readings of documents in several scopes and spellings, `refines` is reflexive and transitive. Two declarations that refine each other are equal, and equal declarations are refined by the same declarations.""" - assert refines(a, a) - if refines(a, b) and refines(b, c): - assert refines(a, c) - assert (refines(a, b) and refines(b, a)) is (a == b) + """Over readings of documents in several scopes and spellings, `refines` is reflexive and transitive. Two models that refine each other are equal, and equal models are refined by the same models.""" + assert a.refines(a) + if a.refines(b) and b.refines(c): + assert a.refines(c) + assert (a.refines(b) and b.refines(a)) is (a == b) if a == b: - assert refines(c, a) is refines(c, b) + assert c.refines(a) is c.refines(b) @pytest.mark.parametrize( @@ -407,32 +388,18 @@ def test_a_kind_is_named_in_words(kind: type[Definition[Any]], said: str) -> Non def test_a_gain_is_judged_by_what_the_definition_reads_not_by_spelling() -> None: - """An accepted declaration refines an unclaimed one when its definition, reading the JSON of the unclaimed declaration, produces the accepted declaration. Two spellings of one configuration count as the same gain, so `refines` is transitive through `==`. A spelling that the definition reads as something else is not a gain.""" - from zarr_metadata.definition import ( - resolve, - ) - from zarr_metadata.v3.definition import ( - CORE_CONTEXT, - ChunkKeyEncodingDefinition, - ) - + """A model refines one whose scope left a name unclaimed when its own scope, reading the other document, reads this model. Two spellings of one configuration count as the same gain, so `refines` is transitive through `==`. A spelling that the definition reads as something else is not a gain.""" nothing = ZarrV3Context() - spelled_out, _ = resolve( - {"name": "default", "configuration": {"separator": "/"}}, - ChunkKeyEncodingDefinition, - CORE_CONTEXT, - ) - bare, _ = resolve({"name": "default"}, ChunkKeyEncodingDefinition, CORE_CONTEXT) - unclaimed, _ = resolve({"name": "default"}, ChunkKeyEncodingDefinition, nothing) + separated = {"name": "default", "configuration": {"separator": "/"}} + spelled_out = ZarrV3ArrayMetadata({**ARRAY, "chunk_key_encoding": separated}) + bare = ZarrV3ArrayMetadata({**ARRAY, "chunk_key_encoding": {"name": "default"}}) + unclaimed = bare.with_context(nothing) assert spelled_out == bare - assert refines(bare, unclaimed) - assert refines(spelled_out, unclaimed) - other, _ = resolve( - {"name": "default", "configuration": {"separator": "."}}, - ChunkKeyEncodingDefinition, - CORE_CONTEXT, - ) - assert not refines(other, unclaimed) + assert bare.refines(unclaimed) + assert spelled_out.refines(unclaimed) + dotted = {"name": "default", "configuration": {"separator": "."}} + other = ZarrV3ArrayMetadata({**ARRAY, "chunk_key_encoding": dotted}) + assert not other.refines(unclaimed) shard = { "name": "sharding_indexed", "configuration": { @@ -442,9 +409,9 @@ def test_a_gain_is_judged_by_what_the_definition_reads_not_by_spelling() -> None "index_location": "end", }, } - read, _ = resolve(shard, CodecDefinition, CORE_CONTEXT) - unread, _ = resolve({**shard, "must_understand": True}, CodecDefinition, nothing) - assert refines(read, unread) + read = _array(shard, CORE_CONTEXT) + unread = _array({**shard, "must_understand": True}, nothing) + assert read.refines(unread) OTHER_GZIP = BytesBytesCodecDefinition(name="gzip", configuration=EmptyConfiguration, size="static")