From 3927589c74330a63393cc143cc9fc995186b0a74 Mon Sep 17 00:00:00 2001 From: Franco Pestilli Date: Tue, 10 Nov 2026 19:00:24 +0000 Subject: [PATCH] Move zip/folder I/O behind a storage-backend seam Introduce trx/backends/ with a small TrxBackend protocol and a registry, and move the code that walks a .trx zip or folder, memory-maps each array and assembles a TrxFile (load_from_zip, load_from_directory, _create_trx_from_pointer, zip_from_folder, and the bodies of load and save) into trx.backends.memmap.MemmapBackend. The logic is unchanged; it has only moved. trx.trx_file_memmap keeps every public name: load() and save() now pick the backend from the path through the registry, and load_from_zip, load_from_directory, zip_from_folder and TrxFile._create_trx_from_pointer delegate to the memmap module, so DIPY, scilpy and existing scripts import exactly what they did before. This is the first step toward letting a TrxFile open and save other containers (a Zarr Vectors store is the motivating case) without touching the TRX specification or the TrxFile API. Adds trx/tests/test_backends.py covering the registry, extension dispatch, a save/load round trip, the legacy names, and registering a custom backend. Existing suite: 205 passed before and after. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01AfVKrdLrUh3AEnK2A5Ez2L --- trx/backends/__init__.py | 27 +++ trx/backends/base.py | 97 +++++++++ trx/backends/memmap.py | 400 +++++++++++++++++++++++++++++++++++++ trx/tests/test_backends.py | 120 +++++++++++ trx/trx_file_memmap.py | 332 +++--------------------------- 5 files changed, 674 insertions(+), 302 deletions(-) create mode 100644 trx/backends/__init__.py create mode 100644 trx/backends/base.py create mode 100644 trx/backends/memmap.py create mode 100644 trx/tests/test_backends.py diff --git a/trx/backends/__init__.py b/trx/backends/__init__.py new file mode 100644 index 0000000..d840f59 --- /dev/null +++ b/trx/backends/__init__.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +"""Storage backends for TRX data. + +The TRX data model is container-independent. A backend maps that model onto +one on-disk container. :class:`MemmapBackend` implements the TRX +specification's zip-or-folder layout and is registered by default. + +Importing this package registers the default backend. +""" + +from trx.backends.base import ( + TrxBackend, + backend_for_loading, + backend_for_saving, + register_backend, + registered_backends, +) +from trx.backends.memmap import MemmapBackend + +__all__ = [ + "MemmapBackend", + "TrxBackend", + "backend_for_loading", + "backend_for_saving", + "register_backend", + "registered_backends", +] diff --git a/trx/backends/base.py b/trx/backends/base.py new file mode 100644 index 0000000..a58a46f --- /dev/null +++ b/trx/backends/base.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +"""Storage backend protocol and registry. + +A *backend* knows how to read a :class:`~trx.trx_file_memmap.TrxFile` from, +and write one to, a particular on-disk container. The TRX data model +(header, positions, offsets, dpv, dps, groups, dpg) is the same for every +backend; only the container differs. + +Today there is one backend, :class:`~trx.backends.memmap.MemmapBackend`, +which implements the TRX specification's zip-or-folder layout with +memory-mapped arrays. The registry exists so that other containers can be +added without touching :mod:`trx.trx_file_memmap`. +""" + +from typing import TYPE_CHECKING, Any, List, Protocol, Type, runtime_checkable + +if TYPE_CHECKING: # pragma: no cover - typing only + from trx.trx_file_memmap import TrxFile + + +@runtime_checkable +class TrxBackend(Protocol): + """What a storage backend must provide. + + All methods are class-level: a backend is stateless and is selected per + call from the path it is asked to handle. + """ + + #: Short identifier, e.g. ``"memmap"``. + name: str + + @classmethod + def can_load(cls, path: str) -> bool: + """Return True if ``path`` is a container this backend can read.""" + ... + + @classmethod + def can_save(cls, path: str) -> bool: + """Return True if ``path`` names a container this backend can write.""" + ... + + @classmethod + def load(cls, path: str) -> "TrxFile": + """Read ``path`` and return a :class:`TrxFile`.""" + ... + + @classmethod + def save(cls, tgm: "TrxFile", path: str, **options: Any) -> None: + """Write ``tgm`` to ``path``.""" + ... + + +_REGISTRY: List[Type[TrxBackend]] = [] + + +def register_backend(backend: Type[TrxBackend]) -> Type[TrxBackend]: + """Register a backend class; usable as a class decorator. + + Backends are consulted in registration order, so the default memmap + backend, registered first, keeps precedence for ``.trx`` files. + """ + if backend not in _REGISTRY: + _REGISTRY.append(backend) + return backend + + +def registered_backends() -> List[Type[TrxBackend]]: + """Return the registered backends in precedence order.""" + return list(_REGISTRY) + + +def backend_for_loading(path: str) -> Type[TrxBackend]: + """Return the first registered backend whose :meth:`can_load` accepts ``path``. + + Raises + ------ + ValueError + If no backend recognises ``path``. + """ + for backend in _REGISTRY: + if backend.can_load(path): + return backend + raise ValueError(f"No TRX storage backend can read {path!r}.") + + +def backend_for_saving(path: str) -> Type[TrxBackend]: + """Return the first registered backend whose :meth:`can_save` accepts ``path``. + + Raises + ------ + ValueError + If no backend recognises the destination. + """ + for backend in _REGISTRY: + if backend.can_save(path): + return backend + raise ValueError("Unsupported extension.") diff --git a/trx/backends/memmap.py b/trx/backends/memmap.py new file mode 100644 index 0000000..25b2524 --- /dev/null +++ b/trx/backends/memmap.py @@ -0,0 +1,400 @@ +# -*- coding: utf-8 -*- +"""The TRX specification's native container: a zip file or folder of memmaps. + +This module holds the code that used to live in :mod:`trx.trx_file_memmap` +for walking a ``.trx`` archive or directory, memory-mapping each array and +assembling a :class:`~trx.trx_file_memmap.TrxFile`, and for writing one back +out. The behaviour is unchanged; it has only moved behind the +:class:`~trx.backends.base.TrxBackend` protocol so that other containers can +sit beside it. + +The functions ``load_from_zip``, ``load_from_directory`` and +``zip_from_folder`` keep their names and are re-exported from +:mod:`trx.trx_file_memmap`. +""" + +import json +import logging +import os +import shutil +import struct +from typing import Any, Dict, Optional, Tuple +import zipfile + +from nibabel.streamlines.array_sequence import ArraySequence +import numpy as np + +from trx.backends.base import register_backend +from trx.io import get_trx_tmp_dir + +_ZIP_LOCAL_HEADER_SIZE = 30 +_ZIP_LOCAL_HEADER_SIGNATURE = b"PK\x03\x04" + + +def _core(): + """Import :mod:`trx.trx_file_memmap` lazily to avoid a circular import.""" + from trx import trx_file_memmap + + return trx_file_memmap + + +def _read_header(fobj) -> dict: + """Parse ``header.json`` into the dict shape ``TrxFile`` expects.""" + header = json.load(fobj) + header["VOXEL_TO_RASMM"] = np.reshape(header["VOXEL_TO_RASMM"], (4, 4)).astype( + np.float32 + ) + header["DIMENSIONS"] = np.array(header["DIMENSIONS"], dtype=np.uint16) + return header + + +def load_from_zip(filename: str): + """Load a TrxFile from a single zipfile. + + Note: Does not work with compressed zipfiles. + + Parameters + ---------- + filename : str + Path of the zipped TrxFile. + + Returns + ------- + TrxFile + TrxFile representing the read data. + """ + tmm = _core() + with zipfile.ZipFile(filename, mode="r") as zf: + with zf.open("header.json") as zf_header: + header = _read_header(zf_header) + + files_pointer_size = {} + for zip_info in zf.filelist: + elem_filename = zip_info.filename + _, ext = os.path.splitext(elem_filename) + if ext == ".json" or zip_info.is_dir(): + continue + + if not tmm._is_dtype_valid(ext): + continue + + if ext == ".bit": + ext = ".bool" + + # Read actual local file header to get correct data offset. + # We can't use zip_info.FileHeader() because ZIP spec allows local + # headers to differ from central directory entries. + # See: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT + zf.fp.seek(zip_info.header_offset) + local_header = zf.fp.read(_ZIP_LOCAL_HEADER_SIZE) + if len(local_header) < _ZIP_LOCAL_HEADER_SIZE: + raise ValueError(f"Truncated local file header for {elem_filename}") + if local_header[:4] != _ZIP_LOCAL_HEADER_SIGNATURE: + raise ValueError( + f"Invalid local file header signature for {elem_filename}" + ) + fname_len, extra_len = struct.unpack(" None: + """Zip on-disk memmaps into a single file. + + Parameters + ---------- + directory : str + The path to the on-disk memmap directory. + filename : str + The path where the zip file should be created. + compression_standard : int, optional + The compression standard to use, as defined by the ZipFile library. + Default is ZIP_STORED. + """ + with zipfile.ZipFile(filename, mode="w", compression=compression_standard) as zf: + for root, _, files in os.walk(directory): + for name in files: + curr_filename = os.path.join(root, name) + tmp_filename = curr_filename.replace(directory, "")[1:] + zf.write(curr_filename, tmp_filename) + + +@register_backend +class MemmapBackend: + """Zip-or-folder TRX container with memory-mapped arrays (the TRX spec).""" + + name = "memmap" + + @classmethod + def can_load(cls, path: str) -> bool: + """A regular file (zip) or a directory holding ``header.json``.""" + if os.path.isfile(path): + return zipfile.is_zipfile(path) + if os.path.isdir(path): + return os.path.isfile(os.path.join(path, "header.json")) + return False + + @classmethod + def can_save(cls, path: str) -> bool: + """``.trx``, ``.zip`` or an extension-less folder path.""" + _, ext = os.path.splitext(path) + return ext.lower() in [".zip", ".trx", ""] + + @classmethod + def load(cls, path: str): + """Load a TrxFile (compressed or not) from a zip file or a folder. + + A deflated zip is extracted to a temporary directory first; the + returned object keeps a handle to it, so call ``close()`` when done. + """ + if os.path.isfile(path): + was_compressed = False + with zipfile.ZipFile(path, "r") as zf: + for info in zf.infolist(): + if info.compress_type != 0: + was_compressed = True + break + if was_compressed: + with zipfile.ZipFile(path, "r") as zf: + tmp_dir = get_trx_tmp_dir() + zf.extractall(tmp_dir.name) + tgm = load_from_directory(tmp_dir.name) + tgm._uncompressed_folder_handle = tmp_dir + logging.info( + "File was compressed, call the close() function before exiting." + ) + else: + tgm = load_from_zip(path) + elif os.path.isdir(path): + tgm = load_from_directory(path) + else: + raise ValueError("File/Folder does not exist") + return tgm + + @classmethod + def save( + cls, tgm, path: str, compression_standard: Any = zipfile.ZIP_STORED, **_: Any + ) -> None: + """Write ``tgm`` as a zip (``.trx``/``.zip``) or as a plain folder.""" + _, ext = os.path.splitext(path) + if ext.lower() not in [".zip", ".trx", ""]: + raise ValueError("Unsupported extension.") + + copy_trx = tgm.deepcopy() + copy_trx.resize() + tmp_dir_name = copy_trx._uncompressed_folder_handle.name + if ext.lower() in [".zip", ".trx"]: + zip_from_folder(tmp_dir_name, path, compression_standard) + else: + if os.path.isdir(path): + shutil.rmtree(path) + shutil.copytree(tmp_dir_name, path) + copy_trx.close() diff --git a/trx/tests/test_backends.py b/trx/tests/test_backends.py new file mode 100644 index 0000000..0bc5630 --- /dev/null +++ b/trx/tests/test_backends.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +"""Tests for the storage-backend registry and the default memmap backend.""" + +import os +import zipfile + +import numpy as np +import pytest + +import trx.backends as backends +from trx.backends.memmap import MemmapBackend +import trx.trx_file_memmap as tmm + + +def _small_trx(): + """A tiny in-memory TrxFile: two streamlines, one dps and one dpv key.""" + trx = tmm.TrxFile(nb_vertices=5, nb_streamlines=2) + trx.streamlines._data[:] = np.arange(15, dtype=np.float32).reshape(5, 3) + trx.streamlines._offsets[:] = np.array([0, 2], dtype=np.uint32) + trx.streamlines._lengths[:] = np.array([2, 3], dtype=np.uint32) + trx.data_per_streamline["w"] = np.array([[1.0], [2.0]], dtype=np.float32) + return trx + + +def test_memmap_backend_is_registered_first(): + registered = backends.registered_backends() + assert registered[0] is MemmapBackend + assert isinstance(MemmapBackend, type) + assert MemmapBackend.name == "memmap" + + +def test_backend_for_saving_by_extension(tmp_path): + assert backends.backend_for_saving(str(tmp_path / "a.trx")) is MemmapBackend + assert backends.backend_for_saving(str(tmp_path / "a.zip")) is MemmapBackend + assert backends.backend_for_saving(str(tmp_path / "folder")) is MemmapBackend + with pytest.raises(ValueError, match="Unsupported extension"): + backends.backend_for_saving(str(tmp_path / "a.trk")) + + +def test_backend_for_loading_zip_and_folder(tmp_path): + trx = _small_trx() + zip_path = str(tmp_path / "small.trx") + dir_path = str(tmp_path / "small_dir") + tmm.save(trx, zip_path) + tmm.save(trx, dir_path) + + assert zipfile.is_zipfile(zip_path) + assert os.path.isfile(os.path.join(dir_path, "header.json")) + assert backends.backend_for_loading(zip_path) is MemmapBackend + assert backends.backend_for_loading(dir_path) is MemmapBackend + + not_trx = tmp_path / "plain.txt" + not_trx.write_text("hello") + with pytest.raises(ValueError, match="No TRX storage backend"): + backends.backend_for_loading(str(not_trx)) + + +def test_load_round_trip_through_registry(tmp_path): + trx = _small_trx() + zip_path = str(tmp_path / "rt.trx") + tmm.save(trx, zip_path) + loaded = tmm.load(zip_path) + try: + assert loaded.header["NB_STREAMLINES"] == 2 + assert loaded.header["NB_VERTICES"] == 5 + np.testing.assert_array_equal(loaded.streamlines._data, trx.streamlines._data) + np.testing.assert_array_equal( + np.asarray(loaded.data_per_streamline["w"]).ravel(), [1.0, 2.0] + ) + finally: + loaded.close() + + +def test_legacy_names_still_exported(): + # Downstream code imports these from trx.trx_file_memmap. + for name in ( + "load", + "save", + "load_from_zip", + "load_from_directory", + "zip_from_folder", + "concatenate", + "TrxFile", + ): + assert hasattr(tmm, name), name + assert hasattr(tmm.TrxFile, "_create_trx_from_pointer") + + +def test_custom_backend_can_be_registered(tmp_path): + class DummyBackend: + name = "dummy" + saved = [] + + @classmethod + def can_load(cls, path): + return path.endswith(".dummy") + + @classmethod + def can_save(cls, path): + return path.endswith(".dummy") + + @classmethod + def load(cls, path): + return tmm.TrxFile() + + @classmethod + def save(cls, tgm, path, **options): + cls.saved.append((path, options)) + + backends.register_backend(DummyBackend) + try: + assert isinstance(DummyBackend, backends.TrxBackend) + target = str(tmp_path / "x.dummy") + assert backends.backend_for_saving(target) is DummyBackend + tmm.save(_small_trx(), target) + assert DummyBackend.saved and DummyBackend.saved[0][0] == target + # The memmap backend still wins for its own extensions. + assert backends.backend_for_saving(str(tmp_path / "y.trx")) is MemmapBackend + finally: + backends.base._REGISTRY.remove(DummyBackend) diff --git a/trx/trx_file_memmap.py b/trx/trx_file_memmap.py index 5ce575e..69f3bae 100644 --- a/trx/trx_file_memmap.py +++ b/trx/trx_file_memmap.py @@ -5,8 +5,6 @@ import json import logging import os -import shutil -import struct from typing import Any, List, Optional, Tuple, Type, Union import zipfile @@ -19,6 +17,8 @@ from nibabel.streamlines.trk import TrkFile import numpy as np +import trx.backends as _backends +import trx.backends.memmap as _memmap_backend from trx.io import get_trx_tmp_dir from trx.utils import ( append_generator_to_dict, @@ -321,7 +321,10 @@ def _create_memmap( def load(input_obj: str, check_dpg: bool = True) -> Type["TrxFile"]: - """Load a TrxFile (compressed or not). + """Load a TrxFile from any registered storage backend. + + The default backend reads the TRX specification's zip-or-folder layout + (compressed or not); see :mod:`trx.backends`. Parameters ---------- @@ -338,29 +341,12 @@ def load(input_obj: str, check_dpg: bool = True) -> Type["TrxFile"]: # TODO Check if 0 streamlines, then 0 vertices is expected (vice-versa) # TODO 4x4 affine matrices should contains values (no all-zeros) # TODO 3x1 dimensions array should contains values at each position (int) - if os.path.isfile(input_obj): - was_compressed = False - with zipfile.ZipFile(input_obj, "r") as zf: - for info in zf.infolist(): - if info.compress_type != 0: - was_compressed = True - break - if was_compressed: - with zipfile.ZipFile(input_obj, "r") as zf: - tmp_dir = get_trx_tmp_dir() - zf.extractall(tmp_dir.name) - tgm = load_from_directory(tmp_dir.name) - tgm._uncompressed_folder_handle = tmp_dir - logging.info( - "File was compressed, call the close() function before exiting." - ) - else: - tgm = load_from_zip(input_obj) - elif os.path.isdir(input_obj): - tgm = load_from_directory(input_obj) - else: + if not (os.path.isfile(input_obj) or os.path.isdir(input_obj)): raise ValueError("File/Folder does not exist") + backend = _backends.backend_for_loading(input_obj) + tgm = backend.load(input_obj) + # Example of robust check for metadata if check_dpg: for dpg in tgm.data_per_group.keys(): @@ -370,121 +356,19 @@ def load(input_obj: str, check_dpg: bool = True) -> Type["TrxFile"]: def load_from_zip(filename: str) -> Type["TrxFile"]: - """Load a TrxFile from a single zipfile. - - Note: Does not work with compressed zipfiles. + """Load a TrxFile from a single (uncompressed) zipfile. - Parameters - ---------- - filename : str - Path of the zipped TrxFile. - - Returns - ------- - TrxFile - TrxFile representing the read data. + Kept for backward compatibility; see :mod:`trx.backends.memmap`. """ - with zipfile.ZipFile(filename, mode="r") as zf: - with zf.open("header.json") as zf_header: - header = json.load(zf_header) - header["VOXEL_TO_RASMM"] = np.reshape( - header["VOXEL_TO_RASMM"], (4, 4) - ).astype(np.float32) - header["DIMENSIONS"] = np.array(header["DIMENSIONS"], dtype=np.uint16) - - files_pointer_size = {} - for zip_info in zf.filelist: - elem_filename = zip_info.filename - _, ext = os.path.splitext(elem_filename) - if ext == ".json" or zip_info.is_dir(): - continue - - if not _is_dtype_valid(ext): - continue - raise ValueError(f"The dtype {elem_filename} is not supported") - - if ext == ".bit": - ext = ".bool" - - # Read actual local file header to get correct data offset. - # We can't use zip_info.FileHeader() because ZIP spec allows local - # headers to differ from central directory entries. - # See: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT - _ZIP_LOCAL_HEADER_SIZE = 30 - _ZIP_LOCAL_HEADER_SIGNATURE = b"PK\x03\x04" - - zf.fp.seek(zip_info.header_offset) - local_header = zf.fp.read(_ZIP_LOCAL_HEADER_SIZE) - if len(local_header) < _ZIP_LOCAL_HEADER_SIZE: - raise ValueError(f"Truncated local file header for {elem_filename}") - if local_header[:4] != _ZIP_LOCAL_HEADER_SIGNATURE: - raise ValueError( - f"Invalid local file header signature for {elem_filename}" - ) - fname_len, extra_len = struct.unpack(" Type["TrxFile"]: """Load a TrxFile from a folder containing memmaps. - Parameters - ---------- - directory : str - Path of the directory containing TRX data. - - Returns - ------- - TrxFile - TrxFile representing the read data. + Kept for backward compatibility; see :mod:`trx.backends.memmap`. """ - - directory = os.path.abspath(directory) - with open(os.path.join(directory, "header.json")) as header: - header = json.load(header) - header["VOXEL_TO_RASMM"] = np.reshape(header["VOXEL_TO_RASMM"], (4, 4)).astype( - np.float32 - ) - header["DIMENSIONS"] = np.array(header["DIMENSIONS"], dtype=np.uint16) - files_pointer_size = {} - for root, _dirs, files in os.walk(directory): - for name in files: - elem_filename = os.path.join(root, name) - _, ext = os.path.splitext(elem_filename) - if ext == ".json": - continue - - if not _is_dtype_valid(ext): - raise ValueError(f"The dtype of {elem_filename} is not supported") - - if ext == ".bit": - ext = ".bool" - - dtype_size = np.dtype(ext[1:]).itemsize - size = os.path.getsize(elem_filename) / dtype_size - if size.is_integer(): - files_pointer_size[elem_filename] = 0, int(size) - elif os.path.getsize(elem_filename) == 1: - files_pointer_size[elem_filename] = 0, 0 - else: - raise ValueError("Wrong size or datatype") - - return TrxFile._create_trx_from_pointer(header, files_pointer_size, root=directory) + return _memmap_backend.load_from_directory(directory) def _filter_empty_trx_files(trx_list: List["TrxFile"]) -> List["TrxFile"]: @@ -835,7 +719,10 @@ def concatenate( def save( tgm: "TrxFile", filename: str, compression_standard: Any = zipfile.ZIP_STORED ) -> None: - """Save a TrxFile (compressed or not). + """Save a TrxFile through the storage backend that handles ``filename``. + + With the default backend, ``.trx``/``.zip`` write a zip archive and an + extension-less path writes a plain folder. Parameters ---------- @@ -845,22 +732,10 @@ def save( The path to save the TrxFile to. compression_standard : int, optional The compression standard to use, as defined by the ZipFile library. - Default is zipfile.ZIP_STORED. + Default is zipfile.ZIP_STORED. Only meaningful for zip containers. """ - _, ext = os.path.splitext(filename) - if ext.lower() not in [".zip", ".trx", ""]: - raise ValueError("Unsupported extension.") - - copy_trx = tgm.deepcopy() - copy_trx.resize() - tmp_dir_name = copy_trx._uncompressed_folder_handle.name - if ext.lower() in [".zip", ".trx"]: - zip_from_folder(tmp_dir_name, filename, compression_standard) - else: - if os.path.isdir(filename): - shutil.rmtree(filename) - shutil.copytree(tmp_dir_name, filename) - copy_trx.close() + backend = _backends.backend_for_saving(filename) + backend.save(tgm, filename, compression_standard=compression_standard) def zip_from_folder( @@ -868,22 +743,9 @@ def zip_from_folder( ) -> None: """Zip on-disk memmaps into a single file. - Parameters - ---------- - directory : str - The path to the on-disk memmap directory. - filename : str - The path where the zip file should be created. - compression_standard : int, optional - The compression standard to use, as defined by the ZipFile library. - Default is zipfile.ZIP_STORED. + Kept for backward compatibility; see :mod:`trx.backends.memmap`. """ - with zipfile.ZipFile(filename, mode="w", compression=compression_standard) as zf: - for root, _, files in os.walk(directory): - for name in files: - curr_filename = os.path.join(root, name) - tmp_filename = curr_filename.replace(directory, "")[1:] - zf.write(curr_filename, tmp_filename) + _memmap_backend.zip_from_folder(directory, filename, compression_standard) class TrxFile: @@ -1377,7 +1239,7 @@ def _initialize_empty_trx( # noqa: C901 return tgm - def _create_trx_from_pointer( # noqa: C901 + def _create_trx_from_pointer( header: dict, dict_pointer_size: dict, root_zip: Optional[str] = None, @@ -1385,146 +1247,12 @@ def _create_trx_from_pointer( # noqa: C901 ) -> Type["TrxFile"]: """Create a TrxFile after reading the structure of a zip/folder. - Parameters - ---------- - header : dict - A TrxFile header dictionary which will be used for the new TrxFile. - dict_pointer_size : dict - A dictionary containing the filenames of all the files within the - TrxFile disk file/folder. - root_zip : str, optional - The path of the ZipFile pointer. - root : str, optional - The dirname of the ZipFile pointer. - - Returns - ------- - TrxFile - A TrxFile constructed from the pointer provided. + Kept for backward compatibility; the implementation lives in + :func:`trx.backends.memmap.create_trx_from_pointer`. """ - tgm = TrxFile() - tgm.header = header - - # Handle empty TRX files early - no positions/offsets to load - if header["NB_STREAMLINES"] == 0 or header["NB_VERTICES"] == 0: - return tgm - - positions, offsets = None, None - for elem_filename in dict_pointer_size.keys(): - if root_zip: - filename = root_zip - else: - filename = elem_filename - - folder = os.path.dirname(elem_filename) - base, dim, ext = _split_ext_with_dimensionality(elem_filename) - if ext == ".bit": - ext = ".bool" - mem_adress, size = dict_pointer_size[elem_filename] - - if root is not None: - # This is for Unix - if os.name != "nt" and folder.startswith(root.rstrip("/")): - folder = folder.replace(root, "").lstrip("/") - # These three are for Windows - elif os.path.isdir(folder) and os.path.basename(folder) in [ - "dpv", - "dps", - "groups", - ]: - folder = os.path.basename(folder) - elif os.path.basename(os.path.dirname(folder)) == "dpg": - folder = os.path.join("dpg", os.path.basename(folder)) - else: - folder = "" - - # Parse/walk the directory tree - if base == "positions" and folder == "": - if size != tgm.header["NB_VERTICES"] * 3 or dim != 3: - raise ValueError("Wrong data size/dimensionality.") - positions = _create_memmap( - filename, - mode="r+", - offset=mem_adress, - shape=(tgm.header["NB_VERTICES"], 3), - dtype=ext[1:], - ) - elif base == "offsets" and folder == "": - if size != tgm.header["NB_STREAMLINES"] + 1 or dim != 1: - raise ValueError("Wrong offsets size/dimensionality.") - offsets = _create_memmap( - filename, - mode="r+", - offset=mem_adress, - shape=(tgm.header["NB_STREAMLINES"] + 1,), - dtype=ext[1:], - ) - if offsets[-1] != 0: - lengths = _compute_lengths(offsets) - else: - lengths = [0] - elif folder == "dps": - nb_scalar = size / tgm.header["NB_STREAMLINES"] - if not nb_scalar.is_integer() or nb_scalar != dim: - raise ValueError("Wrong dps size/dimensionality.") - else: - shape = (tgm.header["NB_STREAMLINES"], int(nb_scalar)) - - tgm.data_per_streamline[base] = _create_memmap( - filename, mode="r+", offset=mem_adress, shape=shape, dtype=ext[1:] - ) - elif folder == "dpv": - nb_scalar = size / tgm.header["NB_VERTICES"] - if not nb_scalar.is_integer() or nb_scalar != dim: - raise ValueError("Wrong dpv size/dimensionality.") - else: - shape = (tgm.header["NB_VERTICES"], int(nb_scalar)) - - tgm.data_per_vertex[base] = _create_memmap( - filename, mode="r+", offset=mem_adress, shape=shape, dtype=ext[1:] - ) - elif folder.startswith("dpg"): - if int(size) != dim: - raise ValueError("Wrong dpg size/dimensionality.") - else: - shape = (1, int(size)) - - # Handle the two-layers architecture - data_name = os.path.basename(base) - sub_folder = os.path.basename(folder) - if sub_folder not in tgm.data_per_group: - tgm.data_per_group[sub_folder] = {} - tgm.data_per_group[sub_folder][data_name] = _create_memmap( - filename, mode="r+", offset=mem_adress, shape=shape, dtype=ext[1:] - ) - elif folder == "groups": - # Groups are simply indices, nothing else - # TODO Crash if not uint? - if dim != 1: - raise ValueError("Wrong group dimensionality.") - else: - shape = (int(size),) - tgm.groups[base] = _create_memmap( - filename, mode="r+", offset=mem_adress, shape=shape, dtype=ext[1:] - ) - else: - logging.error(f"{elem_filename} is not part of a valid structure.") - - # All essential array must be declared - if positions is not None and offsets is not None: - tgm.streamlines._data = positions - tgm.streamlines._offsets = offsets[:-1] - tgm.streamlines._lengths = lengths - else: - raise ValueError("Missing essential data.") - - for dpv_key in tgm.data_per_vertex: - tmp = tgm.data_per_vertex[dpv_key] - tgm.data_per_vertex[dpv_key] = ArraySequence() - tgm.data_per_vertex[dpv_key]._data = tmp - tgm.data_per_vertex[dpv_key]._offsets = offsets[:-1] - tgm.data_per_vertex[dpv_key]._lengths = lengths - return tgm + return _memmap_backend.create_trx_from_pointer( + header, dict_pointer_size, root_zip=root_zip, root=root + ) def resize( # noqa: C901 self,