Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/zarr-metadata/changes/465.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`refines` on every model now reads the other model's document in this model's scope and compares the two, so a document read in a richer scope refines the same document read in a poorer one, whatever the richer scope learns of the chunk each codec receives. Before, `refined_in` of a `uint8` array whose `bytes` codec writes an `endian`, read in a scope that claims no `bytes` codec, did not refine the model it came from. A model no longer refines one whose document its own scope reads with a problem: a `gzip` codec with a configuration key its definition does not declare, read in a scope that claims no `gzip`, is not refined by the model of the same codec without that key.
22 changes: 10 additions & 12 deletions packages/zarr-metadata/src/zarr_metadata/definition/_definition.py
Original file line number Diff line number Diff line change
Expand Up @@ -1228,11 +1228,6 @@ def declaration_key(declaration: ResolvedDeclaration[Any]) -> tuple[object, ...]
)


def own_key(declaration: AcceptedDeclaration[Any]) -> tuple[object, ...]:
"""The part of `declaration_key` for an accepted declaration that excludes nested declarations: the definition and the canonical spelling of its own members."""
return declaration_key(declaration)[:3]


def _nested_key(nested: Nested) -> tuple[tuple[Loc, tuple[object, ...]], ...]:
"""The key of each nested declaration, paired with its location, in the order of the locations.

Expand Down Expand Up @@ -1916,12 +1911,16 @@ def _canonical_declaration(resolved: AcceptedDeclaration[Any]) -> JSONValue | No
# result of `canonical` (the view itself when nothing is simplified) is
# converted to a dict.
view = read_only(cast("Mapping[str, JSONValue]", configuration))
simplified = asked(
definition,
"canonical",
lambda: dict(cast("Mapping[str, JSONValue]", definition.canonical(view))),
)
_, refused = definition.read_configuration(simplified)

def spelled() -> tuple[bool, dict[str, JSONValue]]:
given = cast("Mapping[str, JSONValue]", definition.canonical(view))
return given is view, dict(given)

kept, simplified = asked(definition, "canonical", spelled)
# A `canonical` that returns what it received simplified nothing, so
# there is nothing of its own to check: the configuration was read, and
# each nested declaration was checked in its own simplest spelling.
refused = () if kept else definition.read_configuration(simplified)[1]
if len(refused) != 0:
msg = (
f"{definition.name!r}: its canonical gave {simplified!r}, which does not hold: "
Expand Down Expand Up @@ -1980,7 +1979,6 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue:
"kind_of",
"named_configuration",
"no_rules",
"own_key",
"read_only",
"resolve",
"resolve_refined",
Expand Down
79 changes: 6 additions & 73 deletions packages/zarr-metadata/src/zarr_metadata/definition/_scope.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
"""Operations on scopes: which definition a reading records for each name, how one reading refines another, and where a scope disagrees with a reading.
"""Operations on scopes: which definition a reading records for each name, and where a scope disagrees with a reading.

A model is a document read in a scope. The meaning of the document
depends only on what the scope returned for the names the document uses
(its *claims*), not on everything the scope contains. Two readings of one
document are ordered by how much they know. If a name was unclaimed and a
definition later reads it, the name gains meaning and loses none. If one
definition reads a name and then a different definition reads it, that is
a conflict. The internal `disagreements` function in
a conflict. A model's `refines` decides that order by reading: it reads
the other model's document in its own scope, so nothing here compares
two readings. The internal `disagreements` function in
`zarr_metadata.definition._context`, which a model uses to compare a
scope with its claims, is built on these operations. A scope's `union`
raises the `ScopeConflictError` defined here when two scopes file
Expand All @@ -17,27 +19,18 @@

from collections.abc import Mapping
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast
from typing import TYPE_CHECKING, Any, TypeAlias

from zarr_metadata.definition._definition import (
AcceptedDeclaration,
Definition,
RefusedDeclaration,
UnclaimedDeclaration,
as_kind,
declaration_key,
declarations_of,
filed_under,
format_of,
own_key,
resolve,
)

if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Sequence

from zarr_metadata._typed_json import Loc
from zarr_metadata.definition._definition import D, ResolvedDeclaration
from zarr_metadata.definition._definition import ResolvedDeclaration

ClaimKey: TypeAlias = tuple[type[Definition[Any]], str]
"""A kind and the name a definition is filed under. A scope's `claimant` looks up definitions by this key."""
Expand Down Expand Up @@ -140,65 +133,6 @@ def claims_of(
return claims


def refines(declaration: ResolvedDeclaration[Any], other: ResolvedDeclaration[Any]) -> bool:
"""Whether `declaration` contains everything `other` contains: it reads the same where both read, and it may also read what `other` left unclaimed.

This is the order in which one reading of a document refines another.
Suppose `other` is unclaimed and `declaration` is accepted. This counts as a
gain when re-reading the unclaimed declaration's JSON, with `declaration`'s
definition and the definitions of the declarations nested in `declaration`,
produces `declaration`. Two spellings of one configuration count as one gain,
because `==` treats them as one declaration, so the order is transitive
through equality. The reverse direction (accepted to unclaimed) is a
loss. One name read by two different definitions is a conflict. A
refused declaration refines only itself. Two declarations that refine each other
are equal.
"""
if isinstance(declaration, RefusedDeclaration) or isinstance(other, RefusedDeclaration):
return declaration == other
if isinstance(other, UnclaimedDeclaration):
if isinstance(declaration, UnclaimedDeclaration):
return declaration_key(declaration) == declaration_key(other)
return claim_key(declaration) == claim_key(other) and _gained(declaration, other)
if isinstance(declaration, UnclaimedDeclaration):
return False
if declaration.definition != other.definition or own_key(declaration) != own_key(other):
return False
if set(declaration.nested) != set(other.nested):
return False
return all(refines(declaration.nested[loc], other.nested[loc]) for loc in declaration.nested)


def _gained(declaration: AcceptedDeclaration[Any], other: UnclaimedDeclaration) -> bool:
"""Whether reading `other` in the scope formed by `declaration`'s own claims produces `declaration`. This is the test for a gain."""
again, _ = resolve(other.json, declaration.read_as, _Claimed.of(declaration))
return again == declaration


@dataclass(frozen=True, slots=True)
class _Claimed:
"""The scope formed by a declaration's own claims: its definition and the definitions of its nested declarations, keyed by kind and filed name. `_gained` re-reads a declaration in this scope."""

filed: Mapping[ClaimKey, Definition[Any]]
zarr_format: Literal[2, 3]

@classmethod
def of(cls, declaration: AcceptedDeclaration[Any]) -> _Claimed:
claimed = claims_of(declarations_of(declaration))
return cls(
{key: definition for key, definition in claimed.items() if definition is not None},
format_of(declaration.read_as),
)

def claimant(self, kind: type[D], name: str) -> D | None:
"""The definition of `kind` among the claims that reads `name`, or None if there is none. This matches a scope's `claimant`."""
asked = as_kind(kind)
filed, _ = filed_under(asked, name)
if filed is None:
return None
return cast("D | None", self.filed.get((asked, filed)))


@dataclass(frozen=True, slots=True)
class Disagreements:
"""The places where a scope reads a reading's claims differently: the names that would gain a meaning, and the names that conflict (including names that would lose a meaning)."""
Expand Down Expand Up @@ -240,5 +174,4 @@ def disagreements_of(
"claims_of",
"disagreements_of",
"kind_name",
"refines",
]
101 changes: 21 additions & 80 deletions packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
is_array,
is_object,
is_tuple,
json_text,
not_an_object,
outside_of,
prefixed,
Expand All @@ -33,7 +32,6 @@
AcceptedDeclaration,
ResolvedDeclaration,
UnclaimedDeclaration,
declaration_key,
declarations_of,
default_fill_value,
fill_value_problems,
Expand All @@ -43,7 +41,6 @@
spelled_canonically,
)
from zarr_metadata.definition._scope import ScopeConflictError, claims_of
from zarr_metadata.definition._scope import refines as refines_declaration
from zarr_metadata.model._json_writer import DEFAULT_JSON_WRITER, JSONWriter, JSONWriterLike
from zarr_metadata.model._keyed import Keyed
from zarr_metadata.model._shared import (
Expand Down Expand Up @@ -91,6 +88,7 @@
from collections.abc import Iterator, Sequence

from zarr_metadata._typed_json import Loc
from zarr_metadata.definition._scope import Claims
from zarr_metadata.v2.array import ZarrV2ArrayMetadataStoreKey
from zarr_metadata.v2.attributes import ZarrV2AttributesStoreKey

Expand Down Expand Up @@ -628,52 +626,25 @@ def to_key_value(
def __repr__(self) -> str:
return f"{type(self).__name__}({self._document!r}, context={self._context!r})"

def _key_of(self) -> tuple[object, ...]:
"""The key `==` and `hash` compare for a v2 array model: what its document means.

The key holds each declaration's `declaration_key`, and the fill value as JSON
text, in its canonical spelling when a definition in scope read the
dtype. Every other member is included as it is, with JSON members
as text. No `.zattrs` counts as empty attributes, since both mean
that the array has none, as in v3.
"""
members = self._members
return (
members.shape,
members.chunks,
members.order,
members.dimension_separator,
self._fill_value_key(),
declaration_key(self.dtype),
None if self.compressor is None else declaration_key(self.compressor),
None
if self.filters is None
else tuple(declaration_key(entry) for entry in self.filters),
json_text({} if members.attributes is UNSET else members.attributes),
json_text(members.extra_fields),
)

def _plain_key(
self, dtype: AcceptedDeclaration[ZarrV2DataTypeDefinition[Any]] | UnclaimedDeclaration
) -> tuple[object, ...]:
"""The members that hold no declaration, which `refines` compares. The fill value is in the spelling of `dtype`, the dtype of the more informed side."""
members = self._members
return (
members.shape,
members.chunks,
members.order,
members.dimension_separator,
json_text(spelled_canonically(dtype, members.fill_value)),
json_text({} if members.attributes is UNSET else members.attributes),
json_text(members.extra_fields),
def _canonical_document(self) -> Mapping[str, JSONValue]:
"""The document in its simplest equivalent spelling. This is the one place that decides what a v2 array document means: the model's key is this document as JSON text."""
compressor, filters = self.compressor, self.filters
spelled: dict[str, JSONValue] = {
"dtype": simplest(self.dtype),
"fill_value": spelled_canonically(self.dtype, self._members.fill_value),
"compressor": None if compressor is None else simplest(compressor),
"filters": None if filters is None else tuple(simplest(entry) for entry in filters),
}
return canonical_members(
self._document,
spelled,
empty=("attributes",),
defaults={"dimension_separator": ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR},
)

def _fill_value_key(self) -> str:
"""The fill value as `==` compares it, as JSON text: in its canonical spelling when a definition in scope read the dtype, and as written otherwise."""
fill_value = self._members.fill_value
if isinstance(self.dtype, AcceptedDeclaration):
return json_text(spelled_canonically(self.dtype, fill_value))
return json_text(fill_value)
def _claims(self) -> Claims:
"""The claims the scope made for each name the document uses."""
return claims_of(self._reading.declarations())

def __reduce__(self) -> tuple[type[ZarrV2ArrayMetadata], tuple[object, ZarrV2Context]]:
return type(self), (self._document, self._context)
Expand Down Expand Up @@ -792,20 +763,7 @@ def canonicalize(self) -> ZarrV2ArrayMetadata:
Everything else is kept as written: `shape`, `chunks`, `order`,
the attributes, and whatever nothing in scope claims.
"""
compressor, filters = self.compressor, self.filters
spelled: dict[str, JSONValue] = {
"dtype": simplest(self.dtype),
"fill_value": spelled_canonically(self.dtype, self._members.fill_value),
"compressor": None if compressor is None else simplest(compressor),
"filters": None if filters is None else tuple(simplest(entry) for entry in filters),
}
document = canonical_members(
self._document,
spelled,
empty=("attributes",),
defaults={"dimension_separator": ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR},
)
return type(self)(document, context=self._context)
return type(self)(self._normal_form, context=self._context)

def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadata:
"""Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict.
Expand Down Expand Up @@ -837,25 +795,8 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadat
return self.with_context(context)

def refines(self, other: ZarrV2ArrayMetadata) -> bool:
"""Whether this model holds everything `other` holds. Each declaration must refine its counterpart. A `null` compressor or `null` filters refine only a `null`. Every other member must be the same, with the fill value compared in the spelling of the more informed dtype. If that dtype rejects the other model's fill value, this is not a refinement."""
if type(other) is not type(self):
return False
if (self.compressor is None) != (other.compressor is None):
return False
if (self.filters is None) != (other.filters is None):
return False
mine = () if self.filters is None else self.filters
theirs = () if other.filters is None else other.filters
if len(mine) != len(theirs):
return False
pairs = [(self.dtype, other.dtype), *zip(mine, theirs, strict=True)]
if self.compressor is not None and other.compressor is not None:
pairs.append((self.compressor, other.compressor))
if not all(refines_declaration(one, another) for one, another in pairs):
return False
if len(fill_value_problems(self.dtype, other._members.fill_value)) != 0:
return False
return self._plain_key(self.dtype) == other._plain_key(self.dtype)
"""Whether this model holds everything `other` holds: its scope read every name that `other`'s scope read, with the same definition, and `other`'s document, read in this model's scope, is this model. So the fill value is compared in the spelling of this model's dtype. If `other`'s document has a problem in this scope, as when this model's dtype rejects its fill value, this is not a refinement. False when `other` is not a v2 array."""
return self._refines(other, lambda: other.with_context(self._context))

# --- constructors -----------------------------------------------------

Expand Down
Loading
Loading