diff --git a/packages/zarr-metadata/changes/465.bugfix.md b/packages/zarr-metadata/changes/465.bugfix.md new file mode 100644 index 0000000000..aa3477f59f --- /dev/null +++ b/packages/zarr-metadata/changes/465.bugfix.md @@ -0,0 +1 @@ +`refines` on every model now reads the other model's document in this model's scope and compares the two, so a document read in a richer scope refines the same document read in a poorer one, whatever the richer scope learns of the chunk each codec receives. Before, `refined_in` of a `uint8` array whose `bytes` codec writes an `endian`, read in a scope that claims no `bytes` codec, did not refine the model it came from. A model no longer refines one whose document its own scope reads with a problem: a `gzip` codec with a configuration key its definition does not declare, read in a scope that claims no `gzip`, is not refined by the model of the same codec without that key. diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index a8e08e673e..3bd55e8723 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1228,11 +1228,6 @@ def declaration_key(declaration: ResolvedDeclaration[Any]) -> tuple[object, ...] ) -def own_key(declaration: AcceptedDeclaration[Any]) -> tuple[object, ...]: - """The part of `declaration_key` for an accepted declaration that excludes nested declarations: the definition and the canonical spelling of its own members.""" - return declaration_key(declaration)[:3] - - def _nested_key(nested: Nested) -> tuple[tuple[Loc, tuple[object, ...]], ...]: """The key of each nested declaration, paired with its location, in the order of the locations. @@ -1916,12 +1911,16 @@ def _canonical_declaration(resolved: AcceptedDeclaration[Any]) -> JSONValue | No # result of `canonical` (the view itself when nothing is simplified) is # converted to a dict. view = read_only(cast("Mapping[str, JSONValue]", configuration)) - simplified = asked( - definition, - "canonical", - lambda: dict(cast("Mapping[str, JSONValue]", definition.canonical(view))), - ) - _, refused = definition.read_configuration(simplified) + + def spelled() -> tuple[bool, dict[str, JSONValue]]: + given = cast("Mapping[str, JSONValue]", definition.canonical(view)) + return given is view, dict(given) + + kept, simplified = asked(definition, "canonical", spelled) + # A `canonical` that returns what it received simplified nothing, so + # there is nothing of its own to check: the configuration was read, and + # each nested declaration was checked in its own simplest spelling. + refused = () if kept else definition.read_configuration(simplified)[1] if len(refused) != 0: msg = ( f"{definition.name!r}: its canonical gave {simplified!r}, which does not hold: " @@ -1980,7 +1979,6 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "kind_of", "named_configuration", "no_rules", - "own_key", "read_only", "resolve", "resolve_refined", diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py index 9a28bc2a32..f1883b51d5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py @@ -1,4 +1,4 @@ -"""Operations on scopes: which definition a reading records for each name, how one reading refines another, and where a scope disagrees with a reading. +"""Operations on scopes: which definition a reading records for each name, and where a scope disagrees with a reading. A model is a document read in a scope. The meaning of the document depends only on what the scope returned for the names the document uses @@ -6,7 +6,9 @@ document are ordered by how much they know. If a name was unclaimed and a definition later reads it, the name gains meaning and loses none. If one definition reads a name and then a different definition reads it, that is -a conflict. The internal `disagreements` function in +a conflict. A model's `refines` decides that order by reading: it reads +the other model's document in its own scope, so nothing here compares +two readings. The internal `disagreements` function in `zarr_metadata.definition._context`, which a model uses to compare a scope with its claims, is built on these operations. A scope's `union` raises the `ScopeConflictError` defined here when two scopes file @@ -17,27 +19,18 @@ from collections.abc import Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast +from typing import TYPE_CHECKING, Any, TypeAlias from zarr_metadata.definition._definition import ( - AcceptedDeclaration, Definition, - RefusedDeclaration, - UnclaimedDeclaration, - as_kind, - declaration_key, - declarations_of, filed_under, - format_of, - own_key, - resolve, ) if TYPE_CHECKING: from collections.abc import Callable, Iterable, Sequence from zarr_metadata._typed_json import Loc - from zarr_metadata.definition._definition import D, ResolvedDeclaration + from zarr_metadata.definition._definition import ResolvedDeclaration ClaimKey: TypeAlias = tuple[type[Definition[Any]], str] """A kind and the name a definition is filed under. A scope's `claimant` looks up definitions by this key.""" @@ -140,65 +133,6 @@ def claims_of( return claims -def refines(declaration: ResolvedDeclaration[Any], other: ResolvedDeclaration[Any]) -> bool: - """Whether `declaration` contains everything `other` contains: it reads the same where both read, and it may also read what `other` left unclaimed. - - This is the order in which one reading of a document refines another. - Suppose `other` is unclaimed and `declaration` is accepted. This counts as a - gain when re-reading the unclaimed declaration's JSON, with `declaration`'s - definition and the definitions of the declarations nested in `declaration`, - produces `declaration`. Two spellings of one configuration count as one gain, - because `==` treats them as one declaration, so the order is transitive - through equality. The reverse direction (accepted to unclaimed) is a - loss. One name read by two different definitions is a conflict. A - refused declaration refines only itself. Two declarations that refine each other - are equal. - """ - if isinstance(declaration, RefusedDeclaration) or isinstance(other, RefusedDeclaration): - return declaration == other - if isinstance(other, UnclaimedDeclaration): - if isinstance(declaration, UnclaimedDeclaration): - return declaration_key(declaration) == declaration_key(other) - return claim_key(declaration) == claim_key(other) and _gained(declaration, other) - if isinstance(declaration, UnclaimedDeclaration): - return False - if declaration.definition != other.definition or own_key(declaration) != own_key(other): - return False - if set(declaration.nested) != set(other.nested): - return False - return all(refines(declaration.nested[loc], other.nested[loc]) for loc in declaration.nested) - - -def _gained(declaration: AcceptedDeclaration[Any], other: UnclaimedDeclaration) -> bool: - """Whether reading `other` in the scope formed by `declaration`'s own claims produces `declaration`. This is the test for a gain.""" - again, _ = resolve(other.json, declaration.read_as, _Claimed.of(declaration)) - return again == declaration - - -@dataclass(frozen=True, slots=True) -class _Claimed: - """The scope formed by a declaration's own claims: its definition and the definitions of its nested declarations, keyed by kind and filed name. `_gained` re-reads a declaration in this scope.""" - - filed: Mapping[ClaimKey, Definition[Any]] - zarr_format: Literal[2, 3] - - @classmethod - def of(cls, declaration: AcceptedDeclaration[Any]) -> _Claimed: - claimed = claims_of(declarations_of(declaration)) - return cls( - {key: definition for key, definition in claimed.items() if definition is not None}, - format_of(declaration.read_as), - ) - - def claimant(self, kind: type[D], name: str) -> D | None: - """The definition of `kind` among the claims that reads `name`, or None if there is none. This matches a scope's `claimant`.""" - asked = as_kind(kind) - filed, _ = filed_under(asked, name) - if filed is None: - return None - return cast("D | None", self.filed.get((asked, filed))) - - @dataclass(frozen=True, slots=True) class Disagreements: """The places where a scope reads a reading's claims differently: the names that would gain a meaning, and the names that conflict (including names that would lose a meaning).""" @@ -240,5 +174,4 @@ def disagreements_of( "claims_of", "disagreements_of", "kind_name", - "refines", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index dc39bdfc6f..5353433cc6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -19,7 +19,6 @@ is_array, is_object, is_tuple, - json_text, not_an_object, outside_of, prefixed, @@ -33,7 +32,6 @@ AcceptedDeclaration, ResolvedDeclaration, UnclaimedDeclaration, - declaration_key, declarations_of, default_fill_value, fill_value_problems, @@ -43,7 +41,6 @@ spelled_canonically, ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of -from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( @@ -91,6 +88,7 @@ from collections.abc import Iterator, Sequence from zarr_metadata._typed_json import Loc + from zarr_metadata.definition._scope import Claims from zarr_metadata.v2.array import ZarrV2ArrayMetadataStoreKey from zarr_metadata.v2.attributes import ZarrV2AttributesStoreKey @@ -628,52 +626,25 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for a v2 array model: what its document means. - - The key holds each declaration's `declaration_key`, and the fill value as JSON - text, in its canonical spelling when a definition in scope read the - dtype. Every other member is included as it is, with JSON members - as text. No `.zattrs` counts as empty attributes, since both mean - that the array has none, as in v3. - """ - members = self._members - return ( - members.shape, - members.chunks, - members.order, - members.dimension_separator, - self._fill_value_key(), - declaration_key(self.dtype), - None if self.compressor is None else declaration_key(self.compressor), - None - if self.filters is None - else tuple(declaration_key(entry) for entry in self.filters), - json_text({} if members.attributes is UNSET else members.attributes), - json_text(members.extra_fields), - ) - - def _plain_key( - self, dtype: AcceptedDeclaration[ZarrV2DataTypeDefinition[Any]] | UnclaimedDeclaration - ) -> tuple[object, ...]: - """The members that hold no declaration, which `refines` compares. The fill value is in the spelling of `dtype`, the dtype of the more informed side.""" - members = self._members - return ( - members.shape, - members.chunks, - members.order, - members.dimension_separator, - json_text(spelled_canonically(dtype, members.fill_value)), - json_text({} if members.attributes is UNSET else members.attributes), - json_text(members.extra_fields), + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document in its simplest equivalent spelling. This is the one place that decides what a v2 array document means: the model's key is this document as JSON text.""" + compressor, filters = self.compressor, self.filters + spelled: dict[str, JSONValue] = { + "dtype": simplest(self.dtype), + "fill_value": spelled_canonically(self.dtype, self._members.fill_value), + "compressor": None if compressor is None else simplest(compressor), + "filters": None if filters is None else tuple(simplest(entry) for entry in filters), + } + return canonical_members( + self._document, + spelled, + empty=("attributes",), + defaults={"dimension_separator": ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR}, ) - def _fill_value_key(self) -> str: - """The fill value as `==` compares it, as JSON text: in its canonical spelling when a definition in scope read the dtype, and as written otherwise.""" - fill_value = self._members.fill_value - if isinstance(self.dtype, AcceptedDeclaration): - return json_text(spelled_canonically(self.dtype, fill_value)) - return json_text(fill_value) + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of(self._reading.declarations()) def __reduce__(self) -> tuple[type[ZarrV2ArrayMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) @@ -792,20 +763,7 @@ def canonicalize(self) -> ZarrV2ArrayMetadata: Everything else is kept as written: `shape`, `chunks`, `order`, the attributes, and whatever nothing in scope claims. """ - compressor, filters = self.compressor, self.filters - spelled: dict[str, JSONValue] = { - "dtype": simplest(self.dtype), - "fill_value": spelled_canonically(self.dtype, self._members.fill_value), - "compressor": None if compressor is None else simplest(compressor), - "filters": None if filters is None else tuple(simplest(entry) for entry in filters), - } - document = canonical_members( - self._document, - spelled, - empty=("attributes",), - defaults={"dimension_separator": ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR}, - ) - return type(self)(document, context=self._context) + return type(self)(self._normal_form, context=self._context) def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. @@ -837,25 +795,8 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadat return self.with_context(context) def refines(self, other: ZarrV2ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each declaration must refine its counterpart. A `null` compressor or `null` filters refine only a `null`. Every other member must be the same, with the fill value compared in the spelling of the more informed dtype. If that dtype rejects the other model's fill value, this is not a refinement.""" - if type(other) is not type(self): - return False - if (self.compressor is None) != (other.compressor is None): - return False - if (self.filters is None) != (other.filters is None): - return False - mine = () if self.filters is None else self.filters - theirs = () if other.filters is None else other.filters - if len(mine) != len(theirs): - return False - pairs = [(self.dtype, other.dtype), *zip(mine, theirs, strict=True)] - if self.compressor is not None and other.compressor is not None: - pairs.append((self.compressor, other.compressor)) - if not all(refines_declaration(one, another) for one, another in pairs): - return False - if len(fill_value_problems(self.dtype, other._members.fill_value)) != 0: - return False - return self._plain_key(self.dtype) == other._plain_key(self.dtype) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. So the fill value is compared in the spelling of this model's dtype. If `other`'s document has a problem in this scope, as when this model's dtype rejects its fill value, this is not a refinement. False when `other` is not a v2 array.""" + return self._refines(other, lambda: other.with_context(self._context)) # --- constructors ----------------------------------------------------- diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index c0fa12a402..d4e23eba54 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -18,7 +18,6 @@ frozen, is_array, is_object, - json_text, nested_past_the_levels, not_an_object, refine_json, @@ -34,7 +33,6 @@ Definition, ResolvedDeclaration, UnclaimedDeclaration, - declaration_key, declaration_kind, declarations_of, default_fill_value, @@ -45,9 +43,8 @@ spelled_canonically, ) from zarr_metadata.definition._scope import ScopeConflictError, claims_of -from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike -from zarr_metadata.model._keyed import Keyed +from zarr_metadata.model._keyed import Keyed, normal_key from zarr_metadata.model._shared import ( Derived, StoreKey, @@ -91,6 +88,7 @@ from collections.abc import Iterator from zarr_metadata._typed_json import Loc + from zarr_metadata.definition._scope import Claims from zarr_metadata.v3._definition import CodecDefinition, ExtraFieldDefinition from zarr_metadata.v3.array import ZarrV3ArrayMetadataStoreKey @@ -607,21 +605,13 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for a v3 array model: what its document means, as computed by `array_key_of`.""" - return array_key_of(self._reading, self._members) - - def _plain_key( - self, data_type: AcceptedDeclaration[DataTypeDefinition[Any]] | UnclaimedDeclaration - ) -> tuple[object, ...]: - """The members that hold no declaration, which `refines` compares. The fill value is in the spelling of `data_type`, the data type of the more informed side.""" - members = self._members - return ( - members.shape, - json_text(spelled_canonically(data_type, members.fill_value)), - members.dimension_names, - json_text(members.attributes), - ) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document in its simplest equivalent spelling, as `canonical_array_document_v3` writes it.""" + return canonical_array_document_v3(self._reading, self._members) + + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of(self._reading.declarations()) def __reduce__(self) -> tuple[type[ZarrV3ArrayMetadata], tuple[object, ZarrV3Context]]: # Pickle the pair and read it again on load, so a model's reading @@ -760,8 +750,7 @@ def canonicalize(self) -> ZarrV3ArrayMetadata: of `true`. A document that is already in this spelling is returned as it is, in a new model. """ - document = canonical_array_document_v3(self._reading, self._members) - return type(self)(document, context=self._context) + return type(self)(self._normal_form, context=self._context) def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. @@ -796,28 +785,8 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadat return self.with_context(context) def refines(self, other: ZarrV3ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each declaration must refine its counterpart, in the order `refines` defines for declarations (including the declarations nested in each declaration). Every other member must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" - if type(other) is not type(self): - return False - if len(self.codecs) != len(other.codecs) or len(self.storage_transformers) != len( - other.storage_transformers - ): - return False - if self.extra_fields.keys() != other.extra_fields.keys(): - return False - pairs = ( - (self.data_type, other.data_type), - (self.chunk_grid, other.chunk_grid), - (self.chunk_key_encoding, other.chunk_key_encoding), - *zip(codecs_for_chunks(self._reading), codecs_for_chunks(other._reading), strict=True), - *zip(self.storage_transformers, other.storage_transformers, strict=True), - *((self.extra_fields[key], other.extra_fields[key]) for key in self.extra_fields), - ) - if not all(refines_declaration(mine, theirs) for mine, theirs in pairs): - return False - if len(fill_value_problems(self.data_type, other._members.fill_value)) != 0: - return False - return self._plain_key(self.data_type) == other._plain_key(self.data_type) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. So each codec is compared for the chunk it receives in this model, and the fill value in the spelling of this model's data type, whatever `other`'s scope knew of either. If `other`'s document has a problem in this scope, as when this model's data type rejects its fill value, this is not a refinement. False when `other` is not a v3 array.""" + return self._refines(other, lambda: other.with_context(self._context)) # --- constructors ----------------------------------------------------- @@ -951,32 +920,9 @@ def _derived_codecs(document: Mapping[str, object], context: ZarrV3Context) -> o def array_key_of( reading: ZarrV3ArrayMetadataReading, members: ArrayMembersV3 ) -> tuple[object, ...]: - """A key for what a v3 array document means, from its `reading` and refined `members`. `==` and `hash` compare this key for the model, and two listings of one node are compared by it. - - The key holds each declaration's `declaration_key`, and the fill value as JSON - text: in its canonical spelling when a definition in scope read the - data type, and as written otherwise. A codec is compared as - `codec_for_chunk` returns it for the chunk it receives, so two - spellings that mean the same for that chunk are one codec. Every - other member is included as it is, with JSON members as text. - """ - data_type = held(reading.data_type) - fill_value = ( - spelled_canonically(data_type, members.fill_value) - if isinstance(data_type, AcceptedDeclaration) - else members.fill_value - ) - return ( - members.shape, - json_text(fill_value), - declaration_key(data_type), - declaration_key(held(reading.chunk_grid)), - tuple(declaration_key(codec) for codec in codecs_for_chunks(reading)), - declaration_key(held(reading.chunk_key_encoding)), - members.dimension_names, - json_text(members.attributes), - tuple(declaration_key(held(entry)) for entry in reading.storage_transformers), - extra_fields_key(reading.extra_fields), + """A key for what a v3 array document means, from its `reading` and refined `members`: the key of the model, as `normal_key` computes it from the claims the reading made and from `canonical_array_document_v3`. Two listings of one node are compared by it.""" + return normal_key( + claims_of(reading.declarations()), canonical_array_document_v3(reading, members) ) @@ -985,9 +931,10 @@ def canonical_array_document_v3( ) -> dict[str, JSONValue]: """A v3 array document in its simplest equivalent spelling, from its `reading` and refined `members`, as `ZarrV3ArrayMetadata.canonicalize` reads it. - It is written from the parts that `array_key_of` compares, so two - documents with one key are written the same way, apart from the order - of the members of an object. + This is the one place that decides what a v3 array document means. + The model's key is this document as JSON text, so two documents with + one key are written the same way, apart from the order of the members + of an object. """ data_type = held(reading.data_type) spelled: dict[str, JSONValue] = { @@ -1011,14 +958,6 @@ def codecs_for_chunks( return tuple(held(codec_for_chunk(stage)) for stage in reading.pipeline) -def extra_fields_key(extra_fields: Mapping[str, ResolvedDeclaration[Any]]) -> tuple[object, ...]: - """A key for what a document's extra fields mean: each key with its declaration's `declaration_key`, sorted by key.""" - return tuple( - (key, declaration_key(extra)) - for key, extra in sorted(extra_fields.items(), key=lambda item: item[0]) - ) - - def read_array_metadata_v3( value: object, *, context: ZarrV3Context | None = None ) -> ZarrV3ArrayMetadataReading: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index 3f185a0f9d..1ae725a266 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -19,7 +19,6 @@ frozen, is_json_object, is_object, - json_text, not_an_object, object_at, prefixed, @@ -72,6 +71,7 @@ from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import ResolvedDeclaration, UnclaimedDeclaration + from zarr_metadata.definition._scope import Claims from zarr_metadata.v2.attributes import ZarrV2AttributesStoreKey from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataStoreKey from zarr_metadata.v2.group import ZarrV2GroupMetadataStoreKey @@ -358,10 +358,13 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for a v2 group model: its attributes as JSON text. No `.zattrs` counts as empty attributes, since both mean that the group has none, as in v3.""" - attributes = self._attributes - return (json_text({} if attributes is UNSET else attributes),) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document in its simplest equivalent spelling: without `attributes` when they are empty, since no `.zattrs` and an empty one both mean that the group has none.""" + return canonical_members(self._document, {}, empty=("attributes",)) + + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return {} def __reduce__(self) -> tuple[type[ZarrV2GroupMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) @@ -381,8 +384,7 @@ def __replace__(self, /, **members: Unpack[ZarrV2GroupMetadataUpdate]) -> ZarrV2 def canonicalize(self) -> ZarrV2GroupMetadata: """Return this model with its document in its simplest equivalent spelling, read in this model's own scope: without `attributes` when they are empty, because a group with no `.zattrs` means the same. The result equals this model, and every model equal to this one gives the same document, apart from the order of the members of an object, which `to_key_value` sorts with `JSONWriter(sort_keys=True)`.""" - document = canonical_members(self._document, {}, empty=("attributes",)) - return type(self)(document, context=self._context) + return type(self)(self._normal_form, context=self._context) def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: """Return this document read in `context`: the same group, holding that scope.""" @@ -394,8 +396,8 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadat return self.with_context(context) def refines(self, other: ZarrV2GroupMetadata) -> bool: - """Whether this group holds everything `other` holds, which means its attributes are written the same way, no `.zattrs` counting as empty attributes. False when `other` is not a v2 group.""" - return type(other) is type(self) and self._model_key == other._model_key + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. A group has no declarations, so this means its attributes are written the same way, no `.zattrs` counting as empty attributes. False when `other` is not a v2 group.""" + return self._refines(other, lambda: other.with_context(self._context)) @classmethod def create_default( @@ -568,18 +570,25 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for v2 consolidated metadata: each node's path and key, and every other entry as JSON text.""" - return ( - tuple(sorted((path, node._model_key) for path, node in self._metadata.items())), - self._other_entries_text(), - ) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document with each node in its simplest equivalent spelling, as that node's normal form writes it: its `.zarray` or `.zgroup` entry without the attributes, and its `.zattrs` entry holding them, or left out when they are empty.""" + entries = dict(object_at(self._document, "metadata")) + for path, node in self._metadata.items(): + names = self._files[path] + written = dict(node._normal_form) + attributes = written.pop("attributes", None) + for name, key in names.items(): + if name != ZARR_V2_ATTRIBUTES_STORE_KEY: + entries[key] = written + elif attributes is None: + del entries[key] + else: + entries[key] = attributes + return {**self._document, "metadata": entries} - def _other_entries_text(self) -> str: - """The entries that no node is read from (an orphan `.zattrs`, or any other key) as JSON text, which `==` compares.""" - entries = object_at(self._document, "metadata") - consumed = {key for names in self._files.values() for key in names.values()} - return json_text({key: value for key, value in entries.items() if key not in consumed}) + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of(self._located_declarations()) def _located_declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]]]: """Each declaration of each array node, as read, located in the document: under the node's `.zarray` entry, at the key the document writes for it.""" @@ -602,20 +611,7 @@ def canonicalize(self) -> ZarrV2ConsolidatedMetadata: scope, gives the same document, apart from the order of the members of an object, which a `JSONWriter(sort_keys=True)` sorts. """ - entries = dict(object_at(self._document, "metadata")) - for path, node in self._metadata.items(): - names = self._files[path] - written = cast("dict[str, JSONValue]", node.canonicalize().to_json()) - attributes = written.pop("attributes", None) - for name, key in names.items(): - if name != ZARR_V2_ATTRIBUTES_STORE_KEY: - entries[key] = written - elif attributes is None: - del entries[key] - else: - entries[key] = attributes - document: dict[str, JSONValue] = {**self._document, "metadata": entries} - return type(self)(document, context=self._context) + return type(self)(self._normal_form, context=self._context) def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: """Return this document with every node read in `context`, whatever that changes. Raises `MetadataValidationError` when a node has a problem in `context`.""" @@ -648,16 +644,8 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2Consolidated return self.with_context(context) def refines(self, other: ZarrV2ConsolidatedMetadata) -> bool: - """Whether every node this holds refines the node `other` holds at the same path, both hold the same paths, and the other entries are written the same way. False when `other` is not v2 consolidated metadata.""" - if type(other) is not type(self): - return False - if self._metadata.keys() != other._metadata.keys(): - return False - if self._other_entries_text() != other._other_entries_text(): - return False - return all( - _v2_node_refines(self._metadata[path], other._metadata[path]) for path in self._metadata - ) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. So every node this holds refines the node `other` holds at the same path, both hold the same paths, and the other entries are written the same way. False when `other` is not v2 consolidated metadata.""" + return self._refines(other, lambda: other.with_context(self._context)) @classmethod def from_json( @@ -682,13 +670,6 @@ def from_key_value( ) -def _v2_node_refines(node: ZarrV2NodeMetadata, other: ZarrV2NodeMetadata) -> bool: - """Whether `node` refines `other`. Models of the same kind use their own `refines`; an array never refines a group, or the reverse.""" - if isinstance(node, ZarrV2ArrayMetadata): - return isinstance(other, ZarrV2ArrayMetadata) and node.refines(other) - return isinstance(other, ZarrV2GroupMetadata) and node.refines(other) - - _NODE_FILES: Final = ( ZARR_V2_ARRAY_METADATA_STORE_KEY, ZARR_V2_GROUP_METADATA_STORE_KEY, diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 33e4cd82ab..153fd9b5ca 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -47,13 +47,11 @@ definition_said, kind_name, ) -from zarr_metadata.definition._scope import refines as refines_declaration from zarr_metadata.model._array_v3 import ( ArrayMembersV3, ZarrV3ArrayMetadata, ZarrV3ArrayMetadataReading, array_key_of, - extra_fields_key, members_past_the_levels, must_understand_subset, no_extra_fields, @@ -102,6 +100,7 @@ from zarr_metadata._hierarchy import NodeType from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import AcceptedDeclaration, ResolvedDeclaration + from zarr_metadata.definition._scope import Claims from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataStoreKey @@ -284,12 +283,14 @@ def to_key_value( def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for a v3 group model: its attributes as JSON text, and what each extra field means, as `extra_fields_key` computes it.""" - return ( - json_text(self._members.attributes), - extra_fields_key(self._reading.extra_fields), - ) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The document in its simplest equivalent spelling. This is the one place that decides what a v3 group document means: the model's key is this document as JSON text.""" + spelled = simplest_members(self._reading.extra_fields) + return canonical_members(self._document, spelled, empty=("attributes",)) + + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of(self._reading.declarations()) def __reduce__(self) -> tuple[type[ZarrV3GroupMetadata], tuple[object, ZarrV3Context]]: # Pickle the pair, and read it again on load. @@ -411,9 +412,7 @@ def canonicalize(self) -> ZarrV3GroupMetadata: spelling, and `replace(consolidated_metadata=...)` writes them back into the group. """ - spelled = simplest_members(self._reading.extra_fields) - document = canonical_members(self._document, spelled, empty=("attributes",)) - return type(self)(document, context=self._context) + return type(self)(self._normal_form, context=self._context) def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: """Return this document read in `context`, whatever that changes. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically.""" @@ -443,15 +442,8 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadat return self.with_context(context) def refines(self, other: ZarrV3GroupMetadata) -> bool: - """Whether this group holds everything `other` holds: the same attributes, and the same extra fields, each refining its counterpart in the order `refines` defines for declarations. False when `other` is not a v3 group.""" - if type(other) is not type(self): - return False - if json_text(self._members.attributes) != json_text(other._members.attributes): - return False - mine, theirs = self.extra_fields, other.extra_fields - if mine.keys() != theirs.keys(): - return False - return all(refines_declaration(mine[key], theirs[key]) for key in mine) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. The group does not read the documents its `consolidated_metadata` lists, so they are compared as written. False when `other` is not a v3 group.""" + return self._refines(other, lambda: other.with_context(self._context)) # --- constructors ----------------------------------------------------- @@ -608,11 +600,17 @@ def to_json(self) -> ZarrV3ConsolidatedMetadataJSON: def __repr__(self) -> str: return f"{type(self).__name__}({self._document!r}, context={self._context!r})" - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for consolidated metadata: each document's path and key, sorted by path.""" - return tuple( - (path, node._model_key) - for path, node in sorted(self.metadata.items(), key=lambda item: item[0]) + def _canonical_document(self) -> Mapping[str, JSONValue]: + """The member with each document it lists in its simplest equivalent spelling, as that node's normal form writes it.""" + metadata: dict[str, JSONValue] = { + path: dict(node._normal_form) for path, node in self._metadata.items() + } + return {**self._document, "metadata": metadata} + + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses.""" + return claims_of( + located for node in self._metadata.values() for located in node.reading.declarations() ) def __reduce__(self) -> tuple[type[ZarrV3ConsolidatedMetadata], tuple[object, ZarrV3Context]]: @@ -625,12 +623,7 @@ def canonicalize(self) -> ZarrV3ConsolidatedMetadata: this scope, gives the same member, apart from the order of the members of an object, which a `JSONWriter(sort_keys=True)` sorts. """ - metadata: dict[str, JSONValue] = { - path: cast("JSONValue", node.canonicalize().to_json()) - for path, node in self._metadata.items() - } - document: dict[str, JSONValue] = {**self._document, "metadata": metadata} - return type(self)(document, context=self._context) + return type(self)(self._normal_form, context=self._context) def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: """Return this member with every document read in `context`, whatever that changes. Raises `MetadataValidationError` when a document has a problem in `context`. The existing readings are reused when `context` reads every claim of every document identically.""" @@ -666,14 +659,8 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3Consolidated return self.with_context(context) def refines(self, other: ZarrV3ConsolidatedMetadata) -> bool: - """Whether every document this holds refines the document `other` holds at the same path, and both hold the same paths. False when `other` is not consolidated metadata.""" - if type(other) is not type(self): - return False - if self._metadata.keys() != other._metadata.keys(): - return False - return all( - _node_refines(self._metadata[path], other._metadata[path]) for path in self._metadata - ) + """Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. So every document this holds refines the document `other` holds at the same path, and both hold the same paths. False when `other` is not consolidated metadata.""" + return self._refines(other, lambda: other.with_context(self._context)) @classmethod def from_json( @@ -683,13 +670,6 @@ def from_json( return cls(data, context=context) -def _node_refines(node: ZarrV3NodeMetadata, other: ZarrV3NodeMetadata) -> bool: - """Whether `node` refines `other`. Models of the same kind use their own `refines`; an array never refines a group, or the reverse.""" - if isinstance(node, ZarrV3ArrayMetadata): - return isinstance(other, ZarrV3ArrayMetadata) and node.refines(other) - return isinstance(other, ZarrV3GroupMetadata) and node.refines(other) - - @dataclass(frozen=True, slots=True) class ZarrV3GroupMetadataReading: """A v3 group document as a scope read it: each extra field, every problem, and the model when there are no problems.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_keyed.py b/packages/zarr-metadata/src/zarr_metadata/model/_keyed.py index a26be64c02..e8171ec295 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_keyed.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_keyed.py @@ -1,25 +1,76 @@ -"""The key every model uses for comparison and hashing. The key covers everything the model shows, and is computed once, the first time the model is compared.""" +"""What every model derives from one normal form: its key, `==`, `hash`, `canonicalize` and `refines`. + +A model's normal form is its document in its simplest equivalent +spelling, as its scope reads it. Each model writes that document in one +function, `_canonical_document`. Nothing else in a model decides what +its document means: the key is the text of the normal form with the +claims the scope made, `canonicalize` reads the normal form, and +`refines` reads the other model's document in this model's scope and +compares the two normal forms. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from zarr_metadata._json import MetadataValidationError, json_text + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.definition._scope import Claims + + +def normal_key(claims: Claims, document: Mapping[str, JSONValue]) -> tuple[object, ...]: + """The key of a document in its simplest equivalent spelling, read with `claims`: the claims, and the document as JSON text. + + Two documents mean the same thing when a scope reads the names they + use with the same definitions, and they are one document in their + simplest spelling. The text sorts the members of each object, so the + order a document writes them in is not compared. + """ + return (frozenset(claims.items()), json_text(dict(document))) class Keyed: - """A model whose `==` and `hash` compare a cached key that describes what its document means. - - Each model computes its own key in `_key_of`. `_model_key` computes - the key the first time something asks for it (`==`, `hash`, or a - model that holds this one) and caches it. A model that is only read - never pays for the key. Two models are equal when they have the same - class and equal keys. A model of another class, or any other object, - is never equal to a model. + """A model whose `==`, `hash`, `canonicalize` and `refines` are derived from one normal form. + + Each model writes its normal form in `_canonical_document`, and names + the claims its scope made in `_claims`. The normal form and the key + are computed the first time something asks for them, and cached. A + model that is only read never pays for either. Two models are equal + when they have the same class and equal keys. A model of another + class, or any other object, is never equal to a model. """ - __slots__ = ("_key",) + __slots__ = ("_key", "_normal") _key: tuple[object, ...] + _normal: Mapping[str, JSONValue] - def _key_of(self) -> tuple[object, ...]: - """The key `==` and `hash` compare for this model. Each model class defines its own.""" + def _canonical_document(self) -> Mapping[str, JSONValue]: + """This model's document in its simplest equivalent spelling. Each model class writes its own.""" raise NotImplementedError + def _claims(self) -> Claims: + """The claims the scope made for each name the document uses. Each model class returns its own.""" + raise NotImplementedError + + @property + def _normal_form(self) -> Mapping[str, JSONValue]: + """The normal form. It is computed on first access and cached.""" + try: + return self._normal + except AttributeError: + normal = self._canonical_document() + self._normal = normal + return normal + + def _key_of(self) -> tuple[object, ...]: + """The key `==` and `hash` compare for this model, as `normal_key` computes it.""" + return normal_key(self._claims(), self._normal_form) + @property def _model_key(self) -> tuple[object, ...]: """The key. It is computed on first access and cached.""" @@ -37,3 +88,27 @@ def __eq__(self, other: object) -> bool: def __hash__(self) -> int: return hash(self._model_key) + + def _refines(self, other: object, read_here: Callable[[], object]) -> bool: + """Whether this model holds everything `other` holds: every claim `other` holds, and what `other`'s document reads as in this model's scope. `read_here` returns that reading, and raises `MetadataValidationError` when the document has a problem in this scope. + + A model refines another when its scope read every name the other + scope read, with the same definition, and the other model's + document, read in this model's scope, is this model. So one + reader decides what both documents mean, with everything this + scope knows: the chunk each codec receives, the data type that + spells the fill value, and the definitions of the names the + other scope left unclaimed. A document with a problem in this + scope is not one this model refines. + """ + if type(other) is not type(self): + return False + mine = self._claims() + for key, definition in other._claims().items(): + if definition is not None and mine.get(key) != definition: + return False + try: + again = read_here() + except MetadataValidationError: + return False + return again == self diff --git a/packages/zarr-metadata/tests/model/test_invariants.py b/packages/zarr-metadata/tests/model/test_invariants.py new file mode 100644 index 0000000000..84e11290ca --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_invariants.py @@ -0,0 +1,96 @@ +"""What holds for every v3 document the strategies generate: how a model compares with the same document read in another scope, or written in another order.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import pytest +from hypothesis import given + +from zarr_metadata.model import ZarrV3ArrayMetadata, ZarrV3GroupMetadata +from zarr_metadata.testing.strategies import ( + array_documents, + consolidated_group_documents, + group_documents, +) +from zarr_metadata.v3.definition import CORE_CONTEXT, ZARR_PYTHON_CONTEXT, ZarrV3Context + + +@pytest.mark.parametrize( + "poorer", + [ + ZarrV3Context(), + CORE_CONTEXT, + ZARR_PYTHON_CONTEXT.copy(update={"data_types": {}}), + ZARR_PYTHON_CONTEXT.copy(update={"codecs": {}}), + ], + ids=["empty", "core", "no-data-types", "no-codecs"], +) +@given(document=array_documents()) +def test_every_array_read_in_a_richer_scope_refines_the_one_a_poorer_scope_read( + poorer: ZarrV3Context, document: Mapping[str, Any] +) -> None: + """`refined_in` returns the model that the richer scope reads, and that model refines the one it came from, whatever the richer scope learns of the chunk each codec receives.""" + low = ZarrV3ArrayMetadata.from_json(document, context=poorer) + high = low.refined_in(ZARR_PYTHON_CONTEXT) + assert high == ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT) + assert high.refines(low) + assert low.refines(low) + + +def _reordered(value: object) -> Any: # noqa: ANN401 + """`value` with the members of every object it holds, at any depth, in the reverse order. The items of an array keep their order.""" + if isinstance(value, Mapping): + return {key: _reordered(value[key]) for key in reversed(value)} + if isinstance(value, (list, tuple)): + return [_reordered(item) for item in value] + return value + + +def _assert_one_model( + model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, + reordered: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, +) -> None: + assert reordered == model + assert hash(reordered) == hash(model) + assert reordered.refines(model) # pyright: ignore[reportArgumentType] + assert model.refines(reordered) # pyright: ignore[reportArgumentType] + + +@given(document=array_documents()) +def test_every_array_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + """The order of the members of an object means nothing, at any depth: in the document, in a configuration, and in a configuration that holds declarations.""" + _assert_one_model( + ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT), + ZarrV3ArrayMetadata.from_json(_reordered(document), context=ZARR_PYTHON_CONTEXT), + ) + + +@given(document=group_documents()) +def test_every_group_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + _assert_one_model( + ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT), + ZarrV3GroupMetadata.from_json(_reordered(document), context=ZARR_PYTHON_CONTEXT), + ) + + +@given(document=consolidated_group_documents()) +def test_every_listing_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + """A group lists the documents of its nodes. Each is read, so the listing is one model whatever order the members of those documents are in.""" + listings = [ + ZarrV3GroupMetadata.from_json(each, context=ZARR_PYTHON_CONTEXT).consolidated() + for each in (document, _reordered(document)) + ] + assert listings[0] is not None + assert listings[1] is not None + assert listings[1] == listings[0] + assert hash(listings[1]) == hash(listings[0]) + assert listings[1].refines(listings[0]) + assert listings[0].refines(listings[1]) diff --git a/packages/zarr-metadata/tests/model/test_pair.py b/packages/zarr-metadata/tests/model/test_pair.py index 998a45e3f2..0cc7df2335 100644 --- a/packages/zarr-metadata/tests/model/test_pair.py +++ b/packages/zarr-metadata/tests/model/test_pair.py @@ -265,6 +265,40 @@ def test_error_replace_refuses_a_document_with_a_problem() -> None: ZarrV3ArrayMetadata(ARRAY).replace(dimension_names=("x", "y")) +def _core_without_codecs(*names: str) -> ZarrV3Context: + """`CORE_CONTEXT` without the codecs it files under `names`.""" + kept = {name: codec for name, codec in CORE_CONTEXT.codecs.items() if name not in names} + return CORE_CONTEXT.copy(update={"codecs": kept}) + + +def _core_without_data_types(*names: str) -> ZarrV3Context: + """`CORE_CONTEXT` without the data types it files under `names`.""" + kept = {name: each for name, each in CORE_CONTEXT.data_types.items() if name not in names} + return CORE_CONTEXT.copy(update={"data_types": kept}) + + +LITTLE: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "little"}} +ENDIAN: dict[str, Any] = {**ARRAY, "codecs": [LITTLE]} +"""A `uint8` array whose `bytes` codec writes an `endian`, which means nothing for values of one byte. A scope that reads both the codec and the data type compares the codec without it.""" +TRANSPOSED: dict[str, Any] = { + **ARRAY, + "codecs": [{"name": "transpose", "configuration": {"order": [0]}}, LITTLE], +} +SHARDED_ENDIAN: dict[str, Any] = { + **ARRAY, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [1], + "codecs": [LITTLE], + "index_codecs": [LITTLE, "crc32c"], + }, + } + ], +} + + @pytest.mark.parametrize( ("model", "context", "gained"), [ @@ -272,8 +306,48 @@ def test_error_replace_refuses_a_document_with_a_problem() -> None: (ZarrV3ArrayMetadata(ARRAY, context=CORE_CONTEXT), CORE_AND_EXTENSIONS_CONTEXT, False), (ZarrV3ArrayMetadata(FLOAT, context=ZarrV3Context()), CORE_CONTEXT, True), (ZarrV3ArrayMetadata(SHARDED, context=ZarrV3Context()), CORE_CONTEXT, True), + # What the new scope claims tells it more of the chunk a codec + # receives, so it compares that codec in a simpler spelling than + # the one the document writes. + (ZarrV3ArrayMetadata(ENDIAN, context=ZarrV3Context()), CORE_CONTEXT, True), + ( + ZarrV3ArrayMetadata(ENDIAN, context=_core_without_codecs("bytes")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(ENDIAN, context=_core_without_data_types("uint8")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(TRANSPOSED, context=_core_without_codecs("transpose")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(SHARDED_ENDIAN, context=_core_without_codecs("sharding_indexed")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(SHARDED_ENDIAN, context=_core_without_codecs("bytes")), + CORE_CONTEXT, + True, + ), + ], + ids=[ + "gain", + "nothing-to-gain", + "gain-data-type-with-fill-value", + "gain-of-a-shard", + "gain-of-everything-with-an-endian-that-means-nothing", + "gain-of-a-codec-simplified-for-its-chunk", + "gain-of-the-data-type-a-codec-is-simplified-by", + "gain-of-a-codec-before-one-simplified-for-its-chunk", + "gain-of-a-shard-whose-codecs-are-simplified", + "gain-of-a-codec-in-a-shard", ], - ids=["gain", "nothing-to-gain", "gain-data-type-with-fill-value", "gain-of-a-shard"], ) def test_refined_in_moves_a_model_up_the_order( model: ZarrV3ArrayMetadata, context: ZarrV3Context, gained: bool diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index bcc2f3c9be..adfa8ecc35 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -1185,6 +1185,14 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: CRC32C: Final = {"name": "crc32c"} +def _shard_of(codec: JSONValue, *, index_first: bool = False, chunk_shape: int = 2) -> JSONValue: + """A shard that holds `codec`, with its `index_codecs` written before its `codecs` when `index_first`.""" + members: dict[str, JSONValue] = {"codecs": [codec], "index_codecs": [LE, CRC32C]} + if index_first: + members = dict(reversed(members.items())) + return {"name": "sharding_indexed", "configuration": {"chunk_shape": [chunk_shape], **members}} + + @pytest.mark.parametrize( ("one", "other", "kind", "equal"), [ @@ -1233,6 +1241,12 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: CodecDefinition, True, ), + ( + _shard_of(_shard_of(LE)), + _shard_of(_shard_of(LE, index_first=True), index_first=True), + CodecDefinition, + True, + ), ( {"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [LE, CRC32C]}}, {"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [CRC32C, LE]}}, @@ -1321,6 +1335,7 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: "unclaimed-another-configuration", "another-name", "members-holding-declarations-in-another-order", + "members-holding-declarations-in-another-order-two-deep", "declarations-in-another-order", "members-in-another-order", "blosc-typesize-noshuffle", @@ -1348,6 +1363,20 @@ def test_two_declarations_are_equal_when_they_read_the_same( assert hash(first) == hash(second) +def test_two_refused_declarations_are_equal_whatever_order_their_members_are_in() -> None: + """A refused declaration is compared by what was written and by the declarations it holds, in the order of their locations, not of the members that hold them.""" + # A shard of inner chunks with no elements is refused, and still holds its codecs. + first, _ = resolve(_shard_of(LE, chunk_shape=0), CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) + second, _ = resolve( + _shard_of(LE, chunk_shape=0, index_first=True), CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT + ) + assert isinstance(first, RefusedDeclaration) + assert isinstance(second, RefusedDeclaration) + assert list(first.nested) != list(second.nested) + assert first == second + assert hash(first) == hash(second) + + def test_a_declaration_copied_or_pickled_is_read_by_a_definition_equal_to_its_own() -> None: declaration, _ = resolve( {"name": "gzip", "configuration": {"level": 1}}, CodecDefinition, CORE_CONTEXT diff --git a/packages/zarr-metadata/tests/v3/test_scope.py b/packages/zarr-metadata/tests/v3/test_scope.py index 5ccc6b425d..678dab9805 100644 --- a/packages/zarr-metadata/tests/v3/test_scope.py +++ b/packages/zarr-metadata/tests/v3/test_scope.py @@ -2,6 +2,7 @@ from __future__ import annotations +import dataclasses import pickle from typing import Any @@ -20,7 +21,8 @@ ) from zarr_metadata.definition._context import disagreements from zarr_metadata.definition._definition import declarations_of -from zarr_metadata.definition._scope import Disagreements, claims_of, kind_name, refines +from zarr_metadata.definition._scope import Disagreements, claims_of, kind_name +from zarr_metadata.model import ZarrV3ArrayMetadata from zarr_metadata.v3.codec.bytes import BYTES_CODEC from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC from zarr_metadata.v3.codec.gzip import GZIP_CODEC @@ -166,21 +168,18 @@ def test_error_claims_of_refuses_one_name_read_two_ways() -> None: ) +ARRAY: dict[str, Any] = dict(ZarrV3ArrayMetadata.create_default(shape=(2,)).to_json()) +"""A `uint8` array of one chunk of two values.""" GZIP_DECLARATION = {"name": "gzip", "configuration": {"level": 5}} +GZIP_WITH_AN_UNKNOWN_KEY = {"name": "gzip", "configuration": {"level": 5, "extra": 1}} +"""A gzip that the core definition reads with a problem, and that a scope without a gzip accepts as written.""" +STATIC_GZIP = dataclasses.replace(GZIP_CODEC, size="static") +"""A second definition under the core gzip's name that reads the same configurations.""" ZSTD_DECLARATION = {"name": "zstd", "configuration": {"level": 3}} BLOSC = {"cname": "zstd", "shuffle": "noshuffle", "blocksize": 0} BLOSC_LEVEL_ONE = {"name": "blosc", "configuration": {**BLOSC, "clevel": 1}} BLOSC_LEVEL_TRUE = {"name": "blosc", "configuration": {**BLOSC, "clevel": True}} """Two documents that Python's `==` treats as equal, but that `json_text` tells apart.""" -SHARD_HOLDING_REFUSED = { - "name": "sharding_indexed", - "configuration": { - "chunk_shape": [1], - "codecs": ["bytes", {"name": "gzip", "configuration": {"level": 12}}], - "index_codecs": [{"name": "bytes", "configuration": {"endian": "little"}}, "crc32c"], - }, -} -"""A shard that is read, holding a gzip that its definition refuses.""" NESTED_ZSTD = { "name": "sharding_indexed", "configuration": { @@ -191,74 +190,60 @@ def test_error_claims_of_refuses_one_name_read_two_ways() -> None: } +def _array(codec: object, context: ZarrV3Context) -> ZarrV3ArrayMetadata: + """A `uint8` array of one chunk of two values whose pipeline ends in `codec`, read in `context`. A codec that takes bytes follows a `bytes` codec.""" + shard = isinstance(codec, dict) and codec["name"] == "sharding_indexed" + codecs = [codec] if shard else [{"name": "bytes"}, codec] + return ZarrV3ArrayMetadata({**ARRAY, "codecs": codecs}, context=context) + + @pytest.mark.parametrize( - ("declaration", "other", "expected"), + ("model", "other", "expected"), [ + (_array(GZIP_DECLARATION, CORE_CONTEXT), _array(GZIP_DECLARATION, CORE_CONTEXT), True), + (_array({"name": "crc32c"}, CORE_CONTEXT), _array("crc32c", CORE_CONTEXT), True), ( - _read(GZIP_DECLARATION, CodecDefinition, CORE_CONTEXT), - _read(GZIP_DECLARATION, CodecDefinition, CORE_CONTEXT), + _array(ZSTD_DECLARATION, CORE_AND_EXTENSIONS_CONTEXT), + _array(ZSTD_DECLARATION, CORE_CONTEXT), True, ), ( - _read({"name": "crc32c"}, CodecDefinition, CORE_CONTEXT), - _read("crc32c", CodecDefinition, CORE_CONTEXT), - True, - ), - ( - _read(ZSTD_DECLARATION, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT), - _read(ZSTD_DECLARATION, CodecDefinition, CORE_CONTEXT), - True, - ), - ( - _read(ZSTD_DECLARATION, CodecDefinition, CORE_CONTEXT), - _read(ZSTD_DECLARATION, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT), + _array(ZSTD_DECLARATION, CORE_CONTEXT), + _array(ZSTD_DECLARATION, CORE_AND_EXTENSIONS_CONTEXT), False, ), ( - _read(GZIP_DECLARATION, CodecDefinition, CORE_CONTEXT), - _read(GZIP_DECLARATION, CodecDefinition, ZarrV3Context(codecs=[MY_GZIP])), + _array(GZIP_DECLARATION, CORE_CONTEXT), + _array(GZIP_DECLARATION, ZarrV3Context(codecs=[STATIC_GZIP])), False, ), ( - _read(NESTED_ZSTD, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT), - _read(NESTED_ZSTD, CodecDefinition, CORE_CONTEXT), + _array(NESTED_ZSTD, CORE_AND_EXTENSIONS_CONTEXT), + _array(NESTED_ZSTD, CORE_CONTEXT), True, ), ( - _read(NESTED_ZSTD, CodecDefinition, CORE_CONTEXT), - _read(NESTED_ZSTD, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT), + _array(NESTED_ZSTD, CORE_CONTEXT), + _array(NESTED_ZSTD, CORE_AND_EXTENSIONS_CONTEXT), False, ), + (_array("zstd", CORE_CONTEXT), _array("zstd", CORE_CONTEXT), True), ( - _read({"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_CONTEXT), - _read("gzip", CodecDefinition, CORE_CONTEXT), + _array({"name": "zstd", "configuration": {"level": 1}}, CORE_CONTEXT), + _array({"name": "zstd", "configuration": {"level": 2}}, CORE_CONTEXT), False, ), + (_array("crc32c", CORE_CONTEXT), _array({"name": "crc32c"}, ZarrV3Context()), True), ( - _read("zstd", CodecDefinition, CORE_CONTEXT), - _read("zstd", CodecDefinition, CORE_CONTEXT), - True, - ), - ( - _read({"name": "zstd", "configuration": {"level": 1}}, CodecDefinition, CORE_CONTEXT), - _read({"name": "zstd", "configuration": {"level": 2}}, CodecDefinition, CORE_CONTEXT), + _array(BLOSC_LEVEL_ONE, CORE_AND_EXTENSIONS_CONTEXT), + _array(BLOSC_LEVEL_TRUE, ZarrV3Context()), False, ), ( - _read("crc32c", CodecDefinition, CORE_CONTEXT), - _read({"name": "crc32c"}, CodecDefinition, ZarrV3Context()), - True, - ), - ( - _read(BLOSC_LEVEL_ONE, CodecDefinition, CORE_CONTEXT), - _read(BLOSC_LEVEL_TRUE, CodecDefinition, ZarrV3Context()), + _array(GZIP_DECLARATION, CORE_CONTEXT), + _array(GZIP_WITH_AN_UNKNOWN_KEY, ZarrV3Context()), False, ), - ( - _read(SHARD_HOLDING_REFUSED, CodecDefinition, CORE_CONTEXT), - _read(SHARD_HOLDING_REFUSED, CodecDefinition, CORE_CONTEXT), - True, - ), ], ids=[ "same", @@ -268,19 +253,18 @@ def test_error_claims_of_refuses_one_name_read_two_ways() -> None: "conflict", "nested-gain", "nested-loss", - "refused", "both-unclaimed", "unclaimed-differ", "gain-over-another-spelling", "no-gain-over-true-for-one", - "holding-a-refused-declaration", + "no-gain-over-a-problem", ], ) def test_refines_orders_readings_by_information( - declaration: ResolvedDeclaration[Any], other: ResolvedDeclaration[Any], expected: bool + model: ZarrV3ArrayMetadata, other: ZarrV3ArrayMetadata, expected: bool ) -> None: - """`declaration` refines `other` when it reads the same where both read, and gains where `other` left a name unclaimed, including in the declarations it holds. A loss, a conflict, a refused declaration, or two unclaimed declarations written differently do not refine.""" - assert refines(declaration, other) is expected + """A model refines another when it reads the same where both read, and gains where the other left a name unclaimed, including in the declarations a codec holds. A loss, a conflict, or two unclaimed declarations written differently do not refine. Neither does a document that the refining scope reads with a problem: what it wrote is not what the refining model holds.""" + assert model.refines(other) is expected DOCUMENTS = [ @@ -291,25 +275,22 @@ def test_refines_orders_readings_by_information( ZSTD_DECLARATION, NESTED_ZSTD, BLOSC_LEVEL_ONE, - SHARD_HOLDING_REFUSED, ] CONTEXTS = [ZarrV3Context(), CORE_CONTEXT, CORE_AND_EXTENSIONS_CONTEXT] -READINGS = [ - _read(document, CodecDefinition, context) for document in DOCUMENTS for context in CONTEXTS -] +READINGS = [_array(document, context) for document in DOCUMENTS for context in CONTEXTS] @given(st.sampled_from(READINGS), st.sampled_from(READINGS), st.sampled_from(READINGS)) def test_refines_is_a_partial_order_whose_bottom_is_equality( - a: ResolvedDeclaration[Any], b: ResolvedDeclaration[Any], c: ResolvedDeclaration[Any] + a: ZarrV3ArrayMetadata, b: ZarrV3ArrayMetadata, c: ZarrV3ArrayMetadata ) -> None: - """Over readings of documents in several scopes and spellings, `refines` is reflexive and transitive. Two declarations that refine each other are equal, and equal declarations are refined by the same declarations.""" - assert refines(a, a) - if refines(a, b) and refines(b, c): - assert refines(a, c) - assert (refines(a, b) and refines(b, a)) is (a == b) + """Over readings of documents in several scopes and spellings, `refines` is reflexive and transitive. Two models that refine each other are equal, and equal models are refined by the same models.""" + assert a.refines(a) + if a.refines(b) and b.refines(c): + assert a.refines(c) + assert (a.refines(b) and b.refines(a)) is (a == b) if a == b: - assert refines(c, a) is refines(c, b) + assert c.refines(a) is c.refines(b) @pytest.mark.parametrize( @@ -407,32 +388,18 @@ def test_a_kind_is_named_in_words(kind: type[Definition[Any]], said: str) -> Non def test_a_gain_is_judged_by_what_the_definition_reads_not_by_spelling() -> None: - """An accepted declaration refines an unclaimed one when its definition, reading the JSON of the unclaimed declaration, produces the accepted declaration. Two spellings of one configuration count as the same gain, so `refines` is transitive through `==`. A spelling that the definition reads as something else is not a gain.""" - from zarr_metadata.definition import ( - resolve, - ) - from zarr_metadata.v3.definition import ( - CORE_CONTEXT, - ChunkKeyEncodingDefinition, - ) - + """A model refines one whose scope left a name unclaimed when its own scope, reading the other document, reads this model. Two spellings of one configuration count as the same gain, so `refines` is transitive through `==`. A spelling that the definition reads as something else is not a gain.""" nothing = ZarrV3Context() - spelled_out, _ = resolve( - {"name": "default", "configuration": {"separator": "/"}}, - ChunkKeyEncodingDefinition, - CORE_CONTEXT, - ) - bare, _ = resolve({"name": "default"}, ChunkKeyEncodingDefinition, CORE_CONTEXT) - unclaimed, _ = resolve({"name": "default"}, ChunkKeyEncodingDefinition, nothing) + separated = {"name": "default", "configuration": {"separator": "/"}} + spelled_out = ZarrV3ArrayMetadata({**ARRAY, "chunk_key_encoding": separated}) + bare = ZarrV3ArrayMetadata({**ARRAY, "chunk_key_encoding": {"name": "default"}}) + unclaimed = bare.with_context(nothing) assert spelled_out == bare - assert refines(bare, unclaimed) - assert refines(spelled_out, unclaimed) - other, _ = resolve( - {"name": "default", "configuration": {"separator": "."}}, - ChunkKeyEncodingDefinition, - CORE_CONTEXT, - ) - assert not refines(other, unclaimed) + assert bare.refines(unclaimed) + assert spelled_out.refines(unclaimed) + dotted = {"name": "default", "configuration": {"separator": "."}} + other = ZarrV3ArrayMetadata({**ARRAY, "chunk_key_encoding": dotted}) + assert not other.refines(unclaimed) shard = { "name": "sharding_indexed", "configuration": { @@ -442,9 +409,9 @@ def test_a_gain_is_judged_by_what_the_definition_reads_not_by_spelling() -> None "index_location": "end", }, } - read, _ = resolve(shard, CodecDefinition, CORE_CONTEXT) - unread, _ = resolve({**shard, "must_understand": True}, CodecDefinition, nothing) - assert refines(read, unread) + read = _array(shard, CORE_CONTEXT) + unread = _array({**shard, "must_understand": True}, nothing) + assert read.refines(unread) OTHER_GZIP = BytesBytesCodecDefinition(name="gzip", configuration=EmptyConfiguration, size="static")