From 20977fdac92ba7a66b1ee3d3505029a5a46844ff Mon Sep 17 00:00:00 2001 From: "Thomas G. Close" Date: Tue, 13 Oct 2026 11:51:04 +1100 Subject: [PATCH] adds ConvertibleTo[X] type (which is mime round-tripable) to mean any type can be converted to X using the registered converters --- docs/source/api.rst | 12 +++ docs/source/read_write.rst | 50 ++++++++++ .../extras/core/tests/test_converter.py | 8 +- .../extras/core/tests/test_convertible.py | 97 +++++++++++++++++++ fileformats/core/__init__.py | 44 ++++----- fileformats/core/convertible.py | 76 +++++++++++++++ fileformats/core/fileset.py | 39 +++++--- fileformats/core/identification.py | 24 ++++- fileformats/core/loaded.py | 29 ++++-- 9 files changed, 333 insertions(+), 46 deletions(-) create mode 100644 extras/fileformats/extras/core/tests/test_convertible.py create mode 100644 fileformats/core/convertible.py diff --git a/docs/source/api.rst b/docs/source/api.rst index 14174e86..0b674ab0 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -49,6 +49,18 @@ Annotations for data loaded from a file format (see :ref:`Loaded data`) .. autofunction:: fileformats.core.check_loaded +Convertible-format annotations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Annotation for the formats that can be converted to a file format (see +:ref:`Convertible formats`) + +.. autoclass:: fileformats.core.ConvertibleTo + +.. autoclass:: fileformats.core.ConvertibleToMarker + :members: from_hint, resolve, hint + + Extra hooks ~~~~~~~~~~~ diff --git a/docs/source/read_write.rst b/docs/source/read_write.rst index 399b7543..fdece7df 100644 --- a/docs/source/read_write.rst +++ b/docs/source/read_write.rst @@ -201,6 +201,56 @@ classmethod ) ... +Convertible formats +~~~~~~~~~~~~~~~~~~~ + +The formats that can be converted to a format, with the converters that are installed +in the current environment, are returned by the ``convertible_from`` classmethod as a +union that includes the format itself, annotated with a marker recording the format it +was created from + +.. code-block:: python + + >>> from fileformats.image import Png + >>> Png.convertible_from() + typing.Annotated[fileformats.image.raster.Png | fileformats.image.raster.RasterImage, ConvertibleToMarker(format=)] + +Functions that accept any format that can be converted to the one they need can be +annotated with ``ConvertibleTo[]``. At runtime this evaluates to +``.convertible_from()``, so the format it was created from can be recovered in +the same way as for :ref:`Loaded data` + +.. code-block:: python + + import inspect + from fileformats.core import ConvertibleTo, ConvertibleToMarker + from fileformats.image import Png + + def thumbnail(image: ConvertibleTo[Png]) -> Png: + ... + + hint = inspect.signature(thumbnail, eval_str=True).parameters["image"].annotation + marker = ConvertibleToMarker.from_hint(hint) + marker.format # -> Png + png = marker.format.convert(image) + +The union is resolved when the annotation is evaluated, so it doesn't include converters +that are registered afterwards. ``marker.hint()`` returns the annotated union, and ``marker.resolve()`` the plain +union, for the converters that are registered at the time they are called. + +In "MIME-like" strings (see :ref:`MIME types`) it is written as the format it was +created from with a "+convertible-to" suffix instead of as the union, so the formats +are resolved again from the converters that are installed in the environment that the +string is read in + +.. code-block:: python + + >>> from fileformats.core import to_mime, from_mime + >>> to_mime(ConvertibleTo[Png], official=False) + "image/png+convertible-to" + >>> from_mime("image/png+convertible-to") == ConvertibleTo[Png] + True + .. _Pydra: https://pydra.readthedocs.io diff --git a/extras/fileformats/extras/core/tests/test_converter.py b/extras/fileformats/extras/core/tests/test_converter.py index caf5b3bc..24143492 100644 --- a/extras/fileformats/extras/core/tests/test_converter.py +++ b/extras/fileformats/extras/core/tests/test_converter.py @@ -1,3 +1,4 @@ +import typing as ty from pathlib import Path import attrs @@ -5,7 +6,7 @@ from pydra.compose import python, shell from conftest import write_test_file -from fileformats.core import converter +from fileformats.core import ConvertibleToMarker, converter from fileformats.core.exceptions import FormatConversionError from fileformats.generic import File, FileSet from fileformats.testing import ( @@ -114,4 +115,7 @@ def test_convert_mapped_conversion(work_dir): ], ) def test_convertible_from(klass: type[FileSet], convertible_from: type[FileSet]): - assert klass.convertible_from() == convertible_from + assert ( + klass.convertible_from() + == ty.Annotated[convertible_from, ConvertibleToMarker(klass)] + ) diff --git a/extras/fileformats/extras/core/tests/test_convertible.py b/extras/fileformats/extras/core/tests/test_convertible.py new file mode 100644 index 00000000..5f954cc9 --- /dev/null +++ b/extras/fileformats/extras/core/tests/test_convertible.py @@ -0,0 +1,97 @@ +import inspect +import typing as ty + +import pytest + +# Registers the converters between the testing formats +import test_converter # noqa: F401 + +from fileformats.core import ( + ConvertibleTo, + ConvertibleToMarker, + LoadedMarker, + from_mime, + to_mime, +) +from fileformats.core.exceptions import FormatRecognitionError +from fileformats.image import Png, RasterImage +from fileformats.testing import Bar, Baz, Foo, Qux + + +def test_convertible_to_hint(): + hint = ConvertibleTo[Bar] + assert ty.get_origin(hint) is ty.Annotated + assert ty.get_args(hint)[0] == Bar | Baz | Foo + marker = ConvertibleToMarker.from_hint(hint) + assert marker == ConvertibleToMarker(Bar) + assert marker.format is Bar + assert hint == Bar.convertible_from() + assert marker.hint() == hint + assert marker.resolve() == Bar | Baz | Foo + assert LoadedMarker.from_hint(hint) is None + + +def test_convertible_no_converters(): + hint = ConvertibleTo[Baz] + assert ty.get_args(hint)[0] is Baz + assert ConvertibleToMarker.from_hint(hint).format is Baz + + +def test_convertible_in_signature(): + def func(in_file: ConvertibleTo[Bar] | None = None) -> Foo: + raise NotImplementedError + + sig = inspect.signature(func, eval_str=True) + in_marker = ConvertibleToMarker.from_hint(sig.parameters["in_file"].annotation) + assert in_marker.format is Bar + assert ConvertibleToMarker.from_hint(sig.return_annotation) is None + + +def test_convertible_requires_format(): + with pytest.raises(TypeError, match="requires a FileSet subclass"): + ConvertibleTo[int] + with pytest.raises(TypeError, match="requires a FileSet subclass"): + ConvertibleTo[Foo | Bar] + with pytest.raises(TypeError, match="only for annotations"): + ConvertibleTo() # type: ignore[operator] + + +@pytest.mark.parametrize( + ["hint", "mime"], + [ + [ConvertibleTo[Bar], "testing/bar+convertible-to"], + [ConvertibleTo[Foo], "testing/foo+convertible-to"], + [ConvertibleTo[Baz], "testing/baz+convertible-to"], + [ConvertibleTo[Bar] | None, "testing/bar+convertible-to?"], + [list[ConvertibleTo[Bar]], "testing/bar+convertible-to+list-of"], + [ConvertibleTo[Bar] | Qux, "testing/bar+convertible-to|testing/qux"], + ], +) +def test_convertible_mime_roundtrip(hint: ty.Any, mime: str): + assert to_mime(hint, official=False) == mime + assert from_mime(mime) == hint + + +def test_convertible_mime_official(): + with pytest.raises(TypeError, match="official"): + to_mime(ConvertibleTo[Bar]) + + +def test_convertible_mime_not_format(): + with pytest.raises(FormatRecognitionError): + from_mime("testing/not-a-format+convertible-to") + + +def test_convertible_sorted_by_specificity(): + """Subclasses come before their base classes so that the first type in the union + that matches a file-set is the most specific one""" + for hint in (Bar.convertible_from(), Png.convertible_from()): + args = ty.get_args(ty.get_args(hint)[0]) + assert len(args) > 1 + for i, tp in enumerate(args): + assert not any(issubclass(later, tp) for later in args[i + 1 :]) + assert ty.get_args(ty.get_args(Png.convertible_from())[0]) == (Png, RasterImage) + by_name = ty.get_args( + ty.get_args(Bar.convertible_from(union_sort_key=lambda t: t.__name__))[0] + ) + assert by_name == (Bar, Baz, Foo) diff --git a/fileformats/core/__init__.py b/fileformats/core/__init__.py index a54d13fb..1431ee0e 100644 --- a/fileformats/core/__init__.py +++ b/fileformats/core/__init__.py @@ -1,42 +1,40 @@ from ._version import __version__ from .classifier import Classifier +from .convertible import ConvertibleTo, ConvertibleToMarker from .datatype import DataType, FieldPrimitive -from .mock import MockMixin -from .fileset import FileSet, FileSetMetadata, FileSetPrimitive +from .decorators import mtime_cached_property, validated_property +from .extras import converter, extra, extra_implementation, find_extra_implementation from .field import Field -from .identification import ( - to_mime, - from_mime, - find_matching, - from_paths, -) -from .sampling import SampleFileGenerator -from .extras import extra, extra_implementation, find_extra_implementation, converter +from .fileset import FileSet, FileSetMetadata, FileSetPrimitive +from .identification import find_matching, from_mime, from_paths, to_mime from .loaded import Loaded, LoadedMarker, check_loaded -from .decorators import validated_property, mtime_cached_property +from .mock import MockMixin +from .sampling import SampleFileGenerator __all__ = [ "__version__", "Classifier", + "converter", + "ConvertibleTo", + "ConvertibleToMarker", "DataType", + "Field", + "FieldPrimitive", "FileSet", "FileSetMetadata", - "FieldPrimitive", "FileSetPrimitive", - "MockMixin", - "Field", - "to_mime", - "from_mime", + "find_extra_implementation", "find_matching", + "from_mime", "from_paths", - "SampleFileGenerator", - "extra", - "extra_implementation", - "find_extra_implementation", - "converter", "Loaded", "LoadedMarker", - "check_loaded", - "validated_property", + "MockMixin", "mtime_cached_property", + "SampleFileGenerator", + "to_mime", + "validated_property", + "check_loaded", + "extra", + "extra_implementation", ] diff --git a/fileformats/core/convertible.py b/fileformats/core/convertible.py new file mode 100644 index 00000000..097dfc6f --- /dev/null +++ b/fileformats/core/convertible.py @@ -0,0 +1,76 @@ +"""`ConvertibleTo[Format]` annotation for the formats that can be converted to a file +format. + +``ConvertibleTo[Png]`` evaluates at runtime to ``Png.convertible_from()``, which is:: + + Annotated[, ConvertibleToMarker(Png)] + +i.e. the union of the formats that can be converted to ``Png`` by the converters +installed in the current environment, while anything that reads signatures can recover +the target format with ``ConvertibleToMarker.from_hint(hint).format``. Static type +checkers see it as ``Any``. + +It is written to MIME-like strings as "/+convertible-to" by +`to_mime`, so the union is resolved again from the converters that are installed in the +environment the string is read in. +""" + +import inspect +import typing as ty +from dataclasses import dataclass + +from .loaded import FormatMarker + +if ty.TYPE_CHECKING: + from typing_extensions import TypeAliasType + +__all__ = ["ConvertibleTo", "ConvertibleToMarker"] + + +@dataclass(frozen=True) +class ConvertibleToMarker(FormatMarker): + """Metadata attached to an ``Annotated`` hint recording the format that the + annotated types can be converted to""" + + # Appended to the MIME-like string of the format by `to_mime` + mime_suffix: ty.ClassVar[str] = "+convertible-to" + + def hint(self) -> ty.Any: + """The hint for the marker, resolved from the converters that are currently + registered, which may have changed since the hint was created""" + return self.format.convertible_from() + + def resolve(self) -> ty.Any: + """The format, or union of formats, that can be converted to the format with + the converters that are currently registered, i.e. `hint` without the + annotation""" + return ty.get_args(self.hint())[0] + + +if ty.TYPE_CHECKING: + # Checkers can't evaluate __class_getitem__, so give them a generic alias to Any, + # see `Loaded` + FormatT = ty.TypeVar("FormatT") + ConvertibleTo = TypeAliasType("ConvertibleTo", ty.Any, type_params=(FormatT,)) +else: + + class ConvertibleTo: + """``ConvertibleTo[Format]`` -> ``Format.convertible_from()``, i.e. + ``Annotated[, ConvertibleToMarker(Format)]``""" + + def __new__(cls, *args: ty.Any, **kwargs: ty.Any) -> ty.NoReturn: + raise TypeError( + "ConvertibleTo is only for annotations, use Format.convert()" + ) + + def __class_getitem__(cls, fmt: ty.Any) -> ty.Any: + if isinstance(fmt, ty.TypeVar): + return ty.Any + + from .fileset import FileSet + + if not (inspect.isclass(fmt) and issubclass(fmt, FileSet)): + raise TypeError( + f"ConvertibleTo[...] requires a FileSet subclass, not {fmt!r}" + ) + return ConvertibleToMarker(fmt).hint() diff --git a/fileformats/core/fileset.py b/fileformats/core/fileset.py index b45f6c0c..166c1acc 100644 --- a/fileformats/core/fileset.py +++ b/fileformats/core/fileset.py @@ -14,7 +14,7 @@ from collections import Counter from copy import copy from enum import Enum, IntEnum -from operator import attrgetter, itemgetter +from operator import itemgetter from pathlib import Path from warnings import warn @@ -28,6 +28,7 @@ from fileformats.core.utils import _excluded_subpackages from .classifier import Classifier +from .convertible import ConvertibleToMarker from .datatype import DataType from .decorators import VALIDATED_PROPERTY_FLAG, classproperty, mtime_cached_property from .exceptions import ( @@ -62,6 +63,13 @@ class SupportsDunderGT(ty.Protocol): def __gt__(self, other: ty.Any) -> bool: ... +def specificity_sort_key(datatype: ty.Type[DataType]) -> ty.Tuple[int, str]: + """Key to sort types from the most to the least specific, and then by their name. + A class always has more classes in its MRO than any of its base classes, so + subclasses are always sorted before their base classes""" + return (-len(inspect.getmro(datatype)), datatype.__name__) + + FileSetPrimitive: TypeAlias = ty.Union[os.PathLike[str], ty.Sequence[os.PathLike[str]]] @@ -884,20 +892,27 @@ def convertible_from( union_sort_key: ty.Callable[ [ty.Type[DataType]], ty.Union[SupportsDunderLT, SupportsDunderGT], - ] = attrgetter("__name__"), - ) -> ty.Type["DataType"]: - """Union of types that can be converted to this type, including the current type. - If there are no other types that can be converted to this type, return the current type + ] = specificity_sort_key, + ) -> ty.Any: + """Union of types that can be converted to this type, including the current type + (or just the current type if there are no other types that can be converted to + it), annotated with the type it was created from so that it can be written to + a MIME-like string as "/+convertible-to" instead of the + union, i.e. ``Annotated[, ConvertibleToMarker(cls)]``, which is what + ``ConvertibleTo[cls]`` evaluates to Parameters ---------- union_sort_key : callable[[DataType], Any], optional - A function used to sort the union of types. Defaults to sorting by the type name. + A function used to sort the union of types. Defaults to sorting from the + most to the least specific type (i.e. subclasses before their base classes), + so that the first type in the union that matches a file-set is the most + specific one, and then by the type name. Returns ------- - Type[DataType] - The type or union of types that can be converted to this type. + Annotated[Type[DataType], ConvertibleToMarker] + The annotated type or union of types that can be converted to this type. """ datatypes: ty.List[ty.Type[DataType]] = [cls] @@ -912,8 +927,9 @@ def convertible_from( if issubclass(subcls, cls): subcls._import_extras_module() datatypes.extend(subcls.get_converters_dict().keys()) + marker = ConvertibleToMarker(cls) if len(datatypes) == 1: - return cls + return ty.Annotated[cls, marker] concrete_datatypes = set() def add_concrete(datatype: ty.Type[DataType]) -> None: @@ -930,9 +946,10 @@ def add_concrete(datatype: ty.Type[DataType]) -> None: for datatype in datatypes: add_concrete(datatype) # TODO: Might want to sort datatypes in a more specific order - return functools.reduce( + union = functools.reduce( operator.or_, sorted(concrete_datatypes, key=union_sort_key) - ) # type: ignore[return-value] + ) + return ty.Annotated[union, marker] @classmethod def get_converter_defs(cls, source_format: type) -> ty.List["Converter"]: diff --git a/fileformats/core/identification.py b/fileformats/core/identification.py index 7b1dcd62..d2a16379 100644 --- a/fileformats/core/identification.py +++ b/fileformats/core/identification.py @@ -9,6 +9,7 @@ import fileformats.core from fileformats.core.exceptions import FormatDefinitionError, FormatRecognitionError +from .convertible import ConvertibleToMarker from .utils import ( add_exc_note, fspaths_converter, @@ -130,6 +131,15 @@ def from_mime( return tuple[from_mime(item_mime)] # type: ignore if "|" in mime_str: return functools.reduce(operator.or_, (from_mime(t) for t in mime_str.split("|"))) # type: ignore + if mime_str.endswith(ConvertibleToMarker.mime_suffix): + fmt = from_mime(mime_str[: -len(ConvertibleToMarker.mime_suffix)]) + if not (inspect.isclass(fmt) and issubclass(fmt, fileformats.core.FileSet)): + raise FormatRecognitionError( + f"'{ConvertibleToMarker.mime_suffix}' in '{mime_str}' can only be " + f"applied to file-set formats, not {fmt}" + ) + # Resolved from the converters installed in the current environment + return ConvertibleToMarker(fmt).hint() # type: ignore[no-any-return] return fileformats.core.DataType.from_mime(mime_str) @@ -155,11 +165,23 @@ def to_mime( registry if not """ if is_optional(datatype): - return to_mime(get_optional_type(datatype)) + "?" + return to_mime(get_optional_type(datatype), official=official) + "?" # Handle forward references if isinstance(datatype, ty.ForwardRef): datatype = datatype.__forward_arg__ origin = ty.get_origin(datatype) + if origin is ty.Annotated: + marker = ConvertibleToMarker.from_hint(datatype) + if marker is not None: + if official: + raise TypeError( + f"Cannot convert {datatype} to official mime-type as it " + "depends on the converters that are installed, please use " + 'official=False to convert to "mime-like" string instead' + ) + # Only the format is written, not the formats that are convertible to + # it, so they are resolved again when the string is read + return to_mime(marker.format, official=False) + marker.mime_suffix if ( origin is None and inspect.isclass(datatype) diff --git a/fileformats/core/loaded.py b/fileformats/core/loaded.py index b3d22eb5..70b70bd7 100644 --- a/fileformats/core/loaded.py +++ b/fileformats/core/loaded.py @@ -29,18 +29,18 @@ __all__ = ["Loaded", "LoadedMarker", "check_loaded"] +MarkerT = ty.TypeVar("MarkerT", bound="FormatMarker") + + @dataclass(frozen=True) -class LoadedMarker: - """Metadata attached to an ``Annotated`` hint recording the source format""" +class FormatMarker: + """Base for metadata attached to an ``Annotated`` hint recording the format it was + created from, e.g. ``Loaded[X]`` or ``ConvertibleTo[X]``""" format: ty.Type["FileSet"] - def accepts(self, fmt: ty.Type["FileSet"]) -> bool: - """Whether data loaded from `fmt` satisfies this marker (covariant)""" - return issubclass(fmt, self.format) - @classmethod - def from_hint(cls, tp: ty.Any) -> ty.Optional["LoadedMarker"]: + def from_hint(cls: ty.Type[MarkerT], tp: ty.Any) -> ty.Optional[MarkerT]: """Return the marker of a ``Loaded[X]`` or ``Loaded[X] | None`` hint, or None for any other hint. @@ -55,7 +55,9 @@ def from_hint(cls, tp: ty.Any) -> ty.Optional["LoadedMarker"]: return cls._from_annotated(tp) @classmethod - def all_from_hint(cls, tp: ty.Any) -> ty.Optional[ty.List["LoadedMarker"]]: + def all_from_hint( + cls: ty.Type[MarkerT], tp: ty.Any + ) -> ty.Optional[ty.List[MarkerT]]: """Return the markers of a union of ``Loaded[X]`` hints, optionally including None (e.g. ``Loaded[X] | Loaded[Y] | None``), in the order they appear, which callers can use to find the formats that data can be loaded from. A single @@ -77,7 +79,7 @@ def all_from_hint(cls, tp: ty.Any) -> ty.Optional[ty.List["LoadedMarker"]]: return markers # type: ignore[return-value] @classmethod - def _from_annotated(cls, tp: ty.Any) -> ty.Optional["LoadedMarker"]: + def _from_annotated(cls: ty.Type[MarkerT], tp: ty.Any) -> ty.Optional[MarkerT]: if ty.get_origin(tp) is Annotated: for meta in tp.__metadata__: if isinstance(meta, cls): @@ -85,6 +87,15 @@ def _from_annotated(cls, tp: ty.Any) -> ty.Optional["LoadedMarker"]: return None +@dataclass(frozen=True) +class LoadedMarker(FormatMarker): + """Metadata attached to an ``Annotated`` hint recording the source format""" + + def accepts(self, fmt: ty.Type["FileSet"]) -> bool: + """Whether data loaded from `fmt` satisfies this marker (covariant)""" + return issubclass(fmt, self.format) + + def resolve_loaded_type(tp: ty.Any) -> ty.Any: """Resolve a dotted-path string such as "pydicom.FileDataset" to the object it names, or ``Any`` if its package isn't installed. Non-strings pass through."""