diff --git a/benchmark/pipelines/cellprofiler_nuclei.py b/benchmark/pipelines/cellprofiler_nuclei.py index f7b27bfe97..a9fb404a73 100644 --- a/benchmark/pipelines/cellprofiler_nuclei.py +++ b/benchmark/pipelines/cellprofiler_nuclei.py @@ -26,7 +26,7 @@ from openhcs.core.memory.decorators import numpy as numpy_func from openhcs.core.pipeline.function_contracts import artifact_outputs -from openhcs.processing.materialization import csv_materializer +from openhcs.core.materialization import csv_materializer from openhcs.processing.backends.analysis.cell_counting_cpu import ( materialize_segmentation_masks, ) diff --git a/benchmark/pipelines/cellprofiler_nuclei_cupy.py b/benchmark/pipelines/cellprofiler_nuclei_cupy.py index 1c19a60413..329dfe737e 100644 --- a/benchmark/pipelines/cellprofiler_nuclei_cupy.py +++ b/benchmark/pipelines/cellprofiler_nuclei_cupy.py @@ -22,7 +22,7 @@ from openhcs.core.memory.decorators import cupy as cupy_func from openhcs.core.pipeline.function_contracts import artifact_outputs - from openhcs.processing.materialization import csv_materializer + from openhcs.core.materialization import csv_materializer from openhcs.processing.backends.analysis.cell_counting_cpu import ( materialize_segmentation_masks, ) diff --git a/benchmark/pipelines/cellprofiler_nuclei_gpu.py b/benchmark/pipelines/cellprofiler_nuclei_gpu.py index 189f6242d9..2cfab676ca 100644 --- a/benchmark/pipelines/cellprofiler_nuclei_gpu.py +++ b/benchmark/pipelines/cellprofiler_nuclei_gpu.py @@ -18,7 +18,7 @@ from openhcs.core.memory.decorators import pyclesperanto as pyclesperanto_func from openhcs.core.pipeline.function_contracts import artifact_outputs -from openhcs.processing.materialization import csv_materializer +from openhcs.core.materialization import csv_materializer from openhcs.processing.backends.analysis.cell_counting_cpu import ( materialize_segmentation_masks, ) diff --git a/benchmark/validation/corpus.py b/benchmark/validation/corpus.py index 0e498a2990..90bbd66d90 100644 --- a/benchmark/validation/corpus.py +++ b/benchmark/validation/corpus.py @@ -9,11 +9,11 @@ import shutil import zipfile import zlib -from dataclasses import asdict, dataclass, is_dataclass +from dataclasses import dataclass from datetime import UTC, datetime -from enum import Enum from pathlib import Path -from typing import Any + +from python_introspect import to_jsonable from benchmark.contracts.dataset import DatasetSpec from benchmark.contracts.validation import ( @@ -185,10 +185,10 @@ def prepare( "source_root": str(source_root), "raw_root": str(raw_root), "artifacts": [ - _json_value(artifact) for artifact in validation.artifacts + to_jsonable(artifact) for artifact in validation.artifacts ], "repository_sources": [ - _json_value(source) for source in validation.repository_sources + to_jsonable(source) for source in validation.repository_sources ], "reference_decoder": { "url": validation.reference_decoder_url, @@ -197,7 +197,7 @@ def prepare( "partition_counts": _partition_counts(records), "input_plane_count": len(records), "reference_count": len(references), - "trial_split": _json_value(validation.trial_split), + "trial_split": to_jsonable(validation.trial_split), "development": { "source_set_count": len( {record.source_set_id for record in development_records} @@ -243,13 +243,13 @@ def prepare( }, }, "authoring_tracks": [ - _json_value(track) for track in validation.authoring_tracks + to_jsonable(track) for track in validation.authoring_tracks ], "published_assay_references": [ - _json_value(reference) + to_jsonable(reference) for reference in validation.published_assay_references ], - "dsl_contract": _json_value(dsl_contract), + "dsl_contract": to_jsonable(dsl_contract), "blindness_boundary": ( "Only authoring/ is supplied before pipeline freeze. " "frozen_execution/ is disclosed after freeze for unchanged " @@ -758,7 +758,7 @@ def freeze_pipeline( f"Pipeline is already frozen for this corpus: {receipt_path}" ) receipt_path.write_text( - json.dumps(_json_value(receipt), indent=2, sort_keys=True) + "\n", + json.dumps(to_jsonable(receipt), indent=2, sort_keys=True) + "\n", encoding="utf-8", ) return receipt @@ -876,17 +876,3 @@ def _partition_counts( for record in records: counts[record.partition.value] = counts.get(record.partition.value, 0) + 1 return counts - - -def _json_value(value: Any) -> Any: - if is_dataclass(value) and not isinstance(value, type): - return {key: _json_value(item) for key, item in asdict(value).items()} - if isinstance(value, Enum): - return value.value - if isinstance(value, Path): - return str(value) - if isinstance(value, dict): - return {str(key): _json_value(item) for key, item in value.items()} - if isinstance(value, (tuple, list)): - return [_json_value(item) for item in value] - return value diff --git a/docs/audits/architecture_a_m.json b/docs/audits/architecture_a_m.json index a4e273211a..40697ef178 100644 --- a/docs/audits/architecture_a_m.json +++ b/docs/audits/architecture_a_m.json @@ -187,7 +187,7 @@ "role": "Executes a native plate with separate analysis destinations and verifies per-directory and merged output summaries without writing a summary into the input plate." }, { - "path": "openhcs/processing/materialization/core.py", + "path": "openhcs/core/materialization/core.py", "sha256": "1f5a89a1eb50766ef093df3fece16d6bb03422b92d062fe8ae6471656b1ebd45", "role": "Owns writer runtime-path selection before rendering and the shared ROI target request." } @@ -209,7 +209,7 @@ "On 2026-09-08, re-read the full explanation and documented per-directory storage ownership and output-root global consolidation. The native multi-directory regression passes, and a GUI/MCP POST pilot consolidated all three analysis directories and their merged summary successfully.", "2026-09-09: Re-read this page against the changed source authorities for release 0.8.3; retained its Diataxis purpose and updated affected workflow claims.", "2026-09-16: Re-read the complete page against the projected-ROI filename-identity change. Consolidation still selects paths through the shared target request and does not own source-identity projection; 249 focused materialization and adjacent runtime tests passed.", - "2026-09-22 PR #126 targeted authority-delta review: re-read the complete page against `openhcs/core/config.py`, `tests/unit/test_orchestrator_lane_planning.py`, `openhcs/processing/materialization/core.py`; retained the supported claims, findings, and disposition, then refreshed only these compared evidence hashes. Review scope and the separate blocking runtime evidence are recorded in `docs/audits/pr126_authority_delta_review_20260922.md`.", + "2026-09-22 PR #126 targeted authority-delta review: re-read the complete page against `openhcs/core/config.py`, `tests/unit/test_orchestrator_lane_planning.py`, `openhcs/core/materialization/core.py`; retained the supported claims, findings, and disposition, then refreshed only these compared evidence hashes. Review scope and the separate blocking runtime evidence are recorded in `docs/audits/pr126_authority_delta_review_20260922.md`.", "2026-09-22 PR #126 integrated-repair follow-up: re-read the complete page and the final owner/test changes in `tests/unit/test_function_artifact_materialization.py`. The runtime-observation materialization path now applies the same nominal image reducer as ordinary materialization, preserving exact source-address ownership for consolidation and rejecting conflicting duplicate payloads; the documented no-rescan and typed-record boundary remains accurate. Refreshed only these compared evidence hashes; details are retained in `docs/audits/pr126_authority_delta_review_20260922.md`." ] }, diff --git a/docs/audits/architecture_n_z_development.json b/docs/audits/architecture_n_z_development.json index 3d583b9137..c4219b8e03 100644 --- a/docs/audits/architecture_n_z_development.json +++ b/docs/audits/architecture_n_z_development.json @@ -1019,7 +1019,7 @@ "role": "Native Napari Shapes-layer ROI manager projection and selection behaviour." }, { - "path": "openhcs/processing/materialization/core.py", + "path": "openhcs/core/materialization/core.py", "sha256": "1f5a89a1eb50766ef093df3fece16d6bb03422b92d062fe8ae6471656b1ebd45", "role": "Nominal artifact materialisation and ROI projection strategies." }, @@ -1043,7 +1043,7 @@ "On 2026-08-24, rechecked the explanation after every text-emitting writer adopted the nominal text output declaration; ROI ownership remains unchanged.", "2026-09-09: Re-read this page against the changed source authorities for release 0.8.3; retained its Diataxis purpose and updated affected workflow claims.", "2026-09-16: Re-read the complete explanation against projected ROI filename resolution. Documented that complete record identity fills only missing filename coordinates while collapsed semantic axes remain absent; 249 focused materialization and adjacent runtime tests passed.", - "2026-09-22 PR #126 targeted authority-delta review: re-read the complete page against `openhcs/processing/materialization/core.py`; retained the supported claims, findings, and disposition, then refreshed only these compared evidence hashes. Review scope and the separate blocking runtime evidence are recorded in `docs/audits/pr126_authority_delta_review_20260922.md`." + "2026-09-22 PR #126 targeted authority-delta review: re-read the complete page against `openhcs/core/materialization/core.py`; retained the supported claims, findings, and disposition, then refreshed only these compared evidence hashes. Review scope and the separate blocking runtime evidence are recorded in `docs/audits/pr126_authority_delta_review_20260922.md`." ] }, { diff --git a/docs/refactor/consolidation-20261009/K8-contract-materialization.md b/docs/refactor/consolidation-20261009/K8-contract-materialization.md new file mode 100644 index 0000000000..424a1ce468 --- /dev/null +++ b/docs/refactor/consolidation-20261009/K8-contract-materialization.md @@ -0,0 +1,105 @@ +# K8: Contract and materialization ownership + +**Head audited:** `openhcs` `main` at `d597aa224` (#1205), finished on `44980dfe2` (E1 #1203 merged). **Rules:** [00-RULES.md](00-RULES.md). The contract half is a closure note; the rest follows the receipt template. + +## Already done (G5, verified at this head) + +- `ProcessingContract` is an `AutoRegisterMeta` family in `openhcs/core/processing_contracts.py`; members `Pure2DContract`, `Pure3DContract`, `FlexibleContract`, `VolumetricToSliceContract` own `execute`. +- The decorator takes the class (`core/memory/decorators.py:22-46`); no `declared_processing_contract`, `from_declared_name` or `ProcessingContract[...]` remains in `openhcs/`. The string spelling exists only at the function-cache boundary (`unified_registry.py` reads `ProcessingContract.for_key`, writes `.key`) and in error messages. +- Guards: `tests/unit/test_g5_processing_contract_guards.py`. + +Nothing was left of the contract half. + +## What was wrong + +**Kernel code lived in `processing`, one value-to-JSON answer was written three times, and result file kinds were an enum whose meaning was re-decided at three call sites (IDEN-4, IMPL-2, MEMB-2).** + +1. **Materialization lived in `processing` but is kernel code.** `openhcs/processing/materialization/` (5,834 lines) imports only `openhcs.core`, `polystore`, `zmqruntime` and `metaclass_registry`. Seven core modules imported it (`runtime_stores.py`, `runtime_execution_validation.py`, `pipeline/artifact_planning.py`, `runtime_exports.py`, `post_execute.py`, `pipeline/function_contracts.py`, `steps/function_artifact_materialization.py`); 97 files in `openhcs/`, `tests/`, `benchmark/` and `scripts/` named it. +2. **A third value-to-JSON switch.** `json_payload_value` (`materialization/core.py`, a `singledispatch` over dict, `ColumnarRows`, list, tuple, `DataFrame`, `Series`, NumPy scalars and arrays) and its forwarding wrapper `JsonPayloadAuthority`. +3. **`benchmark/validation/corpus.py` `_json_value`**, a fourth. +4. **Dead computation.** `identify_secondary_objects` built `SecondaryObjectStats` (`processing/backends/cellprofiler/secondary.py`) only to time it under the profiler key `iso_stats`, and never read it. The class had no other user. +5. **`FileFormat(Enum)`** (`constants/constants.py`): ten members, four never referenced (`TORCH`, `JAX`, `CUPY`, `TENSORFLOW`). Its meaning was re-decided in `plate_image_inventory.py` (suffix matching by a hand-listed member tuple, ROI by `value[0]`; preview by `is ROI` / `in {CSV, JSON, TEXT}` / `is not CSV`; `streamable_roi_path` by `is ROI`) and in `pyqt_gui/widgets/image_browser.py` (a `RESULT_FILE_ACTIONS` side table of `ResultFileAction` records keyed by member, and `is ROI`). + +## As built + +- **Package move.** `openhcs/core/materialization/`; every importer updated; no re-export. Live docs (`docs/source/development/callable_artifact_authoring.rst`, the two `docs/audits` indexes) follow; archived plans and recorded paper supplements are left as written. +- **One encoder.** `render_json` calls `python_introspect.to_jsonable`. Joined by registration: `ColumnarRows` (in `core/runtime_tabular_values.py`, as row mappings), `pandas.DataFrame` and `Series` (in `materialization/core.py`); NumPy values through the existing `core/numpy_jsonable.py`. `corpus.py` calls `to_jsonable` directly. **Deleted:** `json_payload_value` and its eight registrations, `JsonPayloadAuthority`, `_json_value`. +- **`ResultFileFormat` family** (`core/result_file_formats.py`, registry key `name`, the spelling in metadata tables and agent DTOs): `TiffResultFile`, `NumpyResultFile`, `TextResultFile`, `JsonResultFile(TextResultFile)`, `CsvResultFile(TextResultFile)`, `RoiArchiveResultFile`. Each declares `suffixes`, and owns `names_path`, `preview(reader, record, …)` (none / text / table / ROI), `open_with(opener, path)` (default application / stream ROIs) and the capability `streams_rois`. `ResultFileFormat.for_suffix_of(path)` iterates the registry. The image browser supplies `ResultFileBrowserOpener(ResultFileOpener)`; the inventory's reader subclasses `ResultFilePreviewReader`. Both are ABCs, not protocols. **Deleted:** the enum, its four unreferenced members, `RESULT_FILE_ACTIONS`, `ResultFileAction`, the two handler functions, the member tuple and membership set in the inventory, and two rows of E1's enum-dispatch table. +- **Secondary stats.** The computation, `SecondaryObjectStats` and the two imports only it used are deleted. + +## Deletion certificates + +| Deleted | Certificate | +|---|---| +| `SecondaryObjectStats` and its computation, profiler key `iso_stats` | **unused**: no reader in `openhcs/`, `tests/`, `benchmark/`, `scripts/`; its base has no registry | +| `FileFormat.TORCH/JAX/CUPY/TENSORFLOW` | **unused**: no reference in `openhcs/`, `tests/`, `scripts/` | +| `FileFormat` enum, `RESULT_FILE_ACTIONS`, `ResultFileAction`, inventory member tuples | **derived**: the `ResultFileFormat` registry; names, suffixes and per-kind behaviour are equal by `test_result_file_kinds_are_a_family` and the inventory tests | +| `openhcs.processing.materialization` (public import path) | **contract**: rule 1, no re-exports of moved names | +| `json_payload_value`, `JsonPayloadAuthority`, `_json_value` | **tests-only** for three inputs the old code admitted and tests do not exercise (below); otherwise **derived** from `to_jsonable` | + +**Tests-only differences of `to_jsonable` against `json_payload_value`:** +- a mapping key that is `True`/`False`/`None` is now spelled `"True"`/`"False"`/`"None"` (`json.dumps` spelled it `true`/`false`/`null`); +- a value with no JSON form that is not a dataclass, enum, path, callable or registered class now raises in `to_jsonable` instead of in `json.dumps` (both raise `TypeError`); +- enums, paths, sets and family classes inside a JSON payload, which used to raise, now encode. + +`PlateResultFileRecord.file_format` passed through `to_jsonable` is now the kind's name (`"CSV"`); as an enum it was the suffix list. I found no reader of that spelling. + +## Persisted state + +| Store | Class | At cutover | +|---|---|---| +| Function registry cache, compiled plans | runtime | reset | +| Saved pipelines | durable | unchanged: they name functions through the catalog | +| **User custom functions that import `openhcs.processing.materialization`** | durable | **K8-Q1 (owner): no tool; such files fail loudly on import and are edited to `openhcs.core.materialization`.** Same default as G7-Q1 and K4-Q1 | + +## Template-crop file naming (coordinator-assigned; accepted: tests were stale) + +`tests/unit/test_template_crop_pipeline_execution.py` (16 cases) expected `images/A01_s001_w1_z001_t001_cropped_stack.tif`. Bisected (multi-plane case, 1,435 revisions from `bb76b5e44`): the first failing commit is `0dd77da46` (2026-10-03, "separate ordinary checkpoint and named artifact identities"), a deliberate rule in `core/steps/function_runtime.py` pinned by `test_function_runtime_source_projection.py::test_save_outputs_positional_lowering_preserves_explicit_payload_identity[unwrapped]`: a named image returned unwrapped keeps the source filename for its main-flow checkpoint, and the named artifact is materialized separately. `bb76b5e44` (#395, B5's bisect) changed only the singleton selection. The test now expects source names in `images/` and one `cropped_stack` file per channel in `results/`; the per-channel pixel assertion is unchanged. No production change. + +With the owner (not changed here): `results/` slices are named by position (`…_slice_000.tif` is channel 2 when `target_channels=[1]`); the crop is written twice. + +## Handoff + +| Item | Evidence | Belongs to | +|---|---|---| +| `polystore.formats.FileFormat` is also an enum (`get_format_from_extension`, `is_pixel_payload`), a second roster of file kinds with different suffix sets | `external/PolyStore/src/polystore/formats.py`; read by `core/source_bindings.py:362` | a PolyStore library surface (L-series); `ResultFileFormat` could then derive its suffixes from it | +| Function registry imports from core (`function_reference.py`, `function_step_transport.py`) | the third concept core reads from `processing` | P (layout) | +| `test_official30_fiji_variants_project_registered_viewer_configs` fails with `NapariViewer has no attribute 'family'` | seen at `d597aa224` without the parity environment; not touched here | G6 | + +## Guards + +`tests/unit/test_k8_materialization_guards.py`: +- the old package directory is gone and no `.py` under `openhcs/`, `tests/`, `benchmark/`, `scripts/` names it; +- no `singledispatch` other than `to_jsonable` registers containers together with NumPy arrays and scalars; the three deleted converter names are not defined; +- no `FileFormat` class in `openhcs/`; the `ResultFileFormat` registry is exactly the six kinds, each found by its own suffixes. + +## New-case experiment + +A new result file kind (say `.parquet`): before, an enum member, an entry in the inventory's member tuple, a branch in the preview reader and a `RESULT_FILE_ACTIONS` record. After: one `ResultFileFormat` subclass. + +## Done when + +The package is in `openhcs/core/`, nothing names the old path, the three converters are gone, `FileFormat` is a family, the guards pass, parity is 30/30, and the ratchet and nra-r1 show no regression. + +## Debt removed with the move + +The ratchet and nra-r1 key debt by file path, so a moved file must arrive clean. Before the move `materialization/core.py` and `path_scopes.py` carried 35 foreign absence probes, 3 long boolean chains (12 terms), 1 string dispatch (3 arms), 2 type switches (6 arms) and 3 redundant type checks. All are zero now; each answer went to its owner: + +- **`Output`** owns `stored_content()` (an image carries its metadata), `describes_image()`, `numbers_samples()` and its owner comparison. +- **Options** (`options.py`): `TabularExtractionOptions.declares_fields/has_fields/extracts_rows/requests_extraction`, `ImageFileOptions.names_files_by_template`. `TabularRowsAuthority.requested` is deleted. +- **`MaterializationSpec.disallowed_backends`**, **`MaterializationContext.sample_numbers_for_export`**, **`MaterializationInput.is_sequence`**, **`SourceStemAuthoritySelection.has_parser`**, one `runtime_planes()` for three copies of the same filter. +- **Type switches are dispatch:** `_mapping_column_length` and `_file_bundle_entry_output` are `singledispatch` functions. +- **String dispatch is a table:** `NONFINITE_FLOAT_CODES` is the one declaration the CSV writer and reader both iterate (was 1/2/3 written on both sides). +- **Archive identity** compares facet by facet (`_identity_facets`). +- **Other owners, one small method each:** `SourceSpatialDomain.with_undeclared_shape_from`, `SourceImageIdentity.has_path/has_component_metadata`, `SourceImageProvenancePlanes.names_every_path`, `MeasurementTable.unowned_feature_names`, `ProcessingContext.execution_axis_count`. +- **Redundant type checks deleted:** the `__post_init__` `isinstance` checks of `ColumnarCsvOutput.content/options` and `RenderedColumnarCsvOutput.options`. Certificate **tests-only**: the three constructors in the tree pass the declared types; a caller passing another type now fails later instead of at construction. + +Left for the surfaces that own the type: +- **K1:** `MaterializationInputItem.has_leading_plane_axis()` still reads `metadata.plane_axis` against `None`; it becomes the plane-axis family's answer when `NoPlaneAxis` lands. +- **K7 (#1209):** materialization annotates `ProcessingContext` where it reads execution state (`path_scopes.py`, `MaterializationContext.context`); these become `ExecutingContext` when K7 is on main (not yet at `625150118`). `ProcessingContext.execution_axis_count` moves with it. + +## Outcome + +- Guards 3/3. Structural ratchet (`openhcs`, `scripts`, `benchmark`) and nra-r1: no key increased. +- Importing unit files: the failures are the same nine as on `origin/main`; the others pass alone. +- Official30 parity passed on the final tree. diff --git a/docs/refactor/receipts/public-native-receiving-134-20261004/controller.py b/docs/refactor/receipts/public-native-receiving-134-20261004/controller.py index 61709a072b..65db9a8523 100644 --- a/docs/refactor/receipts/public-native-receiving-134-20261004/controller.py +++ b/docs/refactor/receipts/public-native-receiving-134-20261004/controller.py @@ -47,7 +47,7 @@ def environment(): from openhcs.core.source_metadata import SourceVoxelSpacing from openhcs.core.runtime_spatial_graph import SpatialGraph,SpatialGraphNode,SpatialGraphEdge from openhcs.core.steps.function_runtime import FunctionOutputContextStrategy -from openhcs.processing.materialization import MaterializationSpec,SpatialGraphROIOptions,materialize +from openhcs.core.materialization import MaterializationSpec,SpatialGraphROIOptions,materialize from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata from openhcs.agent.dto.execution import ExecutionConnectionSpec from openhcs.agent.dto.plate import PlateFileStreamRequest diff --git a/docs/source/development/callable_artifact_authoring.rst b/docs/source/development/callable_artifact_authoring.rst index 727cdcac90..9cbdca02ea 100644 --- a/docs/source/development/callable_artifact_authoring.rst +++ b/docs/source/development/callable_artifact_authoring.rst @@ -143,7 +143,7 @@ The dataclass owns the row schema, including the empty-row case. RuntimeMeasurementFeature, RuntimeMeasurementFeatureOwner, ) from openhcs.core.processing_contracts import Pure2DContract - from openhcs.processing.materialization import ( + from openhcs.core.materialization import ( CsvOptions, MaterializationSpec, ROIOptions, ) @@ -297,7 +297,7 @@ This declaration block is for an operation that already produces labels and from openhcs.core.runtime_measurements import ( ObjectCoreMeasurementFeature, RuntimeMeasurementFeatureOwner, ) - from openhcs.processing.materialization import ( + from openhcs.core.materialization import ( CsvOptions, MaterializationSpec, PointROIOptions, ) @@ -514,7 +514,7 @@ runtime axis, not biological object counts or an inferred well identity: ) from openhcs.core.runtime_measurements import MeasurementTable from openhcs.core.runtime_stores import RuntimeArtifactBatch - from openhcs.processing.materialization import CsvOptions, MaterializationSpec + from openhcs.core.materialization import CsvOptions, MaterializationSpec @dataclass(frozen=True) class PlateSummaryRow: diff --git a/openhcs/constants/constants.py b/openhcs/constants/constants.py index f326a35fd9..cc9e87281a 100644 --- a/openhcs/constants/constants.py +++ b/openhcs/constants/constants.py @@ -5,7 +5,6 @@ These constants are governed by various doctrinal clauses. """ -from enum import Enum from abc import ABC from typing import ClassVar, Set @@ -153,19 +152,6 @@ class OrchestratorExecFailed(OrchestratorState): LOADABLE_IMAGE_EXTENSIONS: Set[str] = _TIFF_IMAGE_EXTENSIONS | _RASTER_IMAGE_EXTENSIONS -class FileFormat(Enum): - TIFF = list(DEFAULT_IMAGE_EXTENSIONS) - NUMPY = [".npy"] - TORCH = [".pt", ".torch", ".pth"] - JAX = [".jax"] - CUPY = [".cupy", ".craw"] - TENSORFLOW = [".tf"] - JSON = [".json"] - CSV = [".csv"] - TEXT = [".txt", ".py", ".md", ".swc"] - ROI = [".roi.zip"] - - DEFAULT_BACKEND = Backend.MEMORY REQUIRES_DISK_READ = "requires_disk_read" REQUIRES_DISK_WRITE = "requires_disk_write" diff --git a/openhcs/core/context/processing_context.py b/openhcs/core/context/processing_context.py index 1a20391c5e..5815fed6ac 100644 --- a/openhcs/core/context/processing_context.py +++ b/openhcs/core/context/processing_context.py @@ -172,6 +172,12 @@ def bind_execution_runtime( self.execution_runtime = runtime self.reset_completed_step_outputs() + def execution_axis_count(self) -> int: + """Return how many axis values the bound execution runs; 0 before binding.""" + if self.execution_runtime is None: + return 0 + return len(self.execution_runtime.execution_axis_values) + def reset_completed_step_outputs(self) -> None: """Start an execution's saved-output facts without retaining old payloads.""" from openhcs.core.steps.abstract import StepExecutionObservation diff --git a/openhcs/core/image_file_serialization.py b/openhcs/core/image_file_serialization.py index 17a897c95b..78fa510f4a 100644 --- a/openhcs/core/image_file_serialization.py +++ b/openhcs/core/image_file_serialization.py @@ -15,7 +15,11 @@ from metaclass_registry import AutoRegisterMeta from polystore.config import TiffConfig, TiffPhotometric, TiffPlanarConfig -from openhcs.constants.constants import FileFormat +from openhcs.core.result_file_formats import ( + NumpyResultFile, + ResultFileFormat, + TiffResultFile, +) from openhcs.core.callable_contract import CompilerPreparedAutoRegisterFamily from openhcs.core.image_quantization_numba import quantize_image_uint8 from metaclass_registry.strategies import NominalTypeStrategyFamilyMixin @@ -251,7 +255,7 @@ class ImageFileFormat(CompilerPreparedAutoRegisterFamily, metaclass=AutoRegister __skip_if_no_key__ = True format_key: ClassVar[str | None] = None suffixes: ClassVar[tuple[str, ...]] = () - browser_file_format: ClassVar[FileFormat] = FileFormat.TIFF + browser_file_format: ClassVar[type[ResultFileFormat]] = TiffResultFile @classmethod def cache_preparation_operations(cls) -> tuple[PreparationOperation, ...]: @@ -461,7 +465,7 @@ class NumpyImageFileFormat(ImageFileFormat): format_key = "numpy" suffixes = (".npy",) - browser_file_format = FileFormat.NUMPY + browser_file_format = NumpyResultFile def preserves_pixel_values(self, source_dtype: Any) -> bool: del source_dtype diff --git a/openhcs/processing/materialization/__init__.py b/openhcs/core/materialization/__init__.py similarity index 85% rename from openhcs/processing/materialization/__init__.py rename to openhcs/core/materialization/__init__.py index 2e41f35105..8849528cc4 100644 --- a/openhcs/processing/materialization/__init__.py +++ b/openhcs/core/materialization/__init__.py @@ -1,7 +1,7 @@ """Materialization public API (writer-based).""" -from openhcs.processing.materialization.constants import WriteMode -from openhcs.processing.materialization.core import ( +from openhcs.core.materialization.constants import WriteMode +from openhcs.core.materialization.core import ( BackendSaver, MaterializationBatch, MaterializationContext, @@ -15,7 +15,7 @@ registered_materialization_option_types, tabular_field_names_from_materialization, ) -from openhcs.processing.materialization.options import ( +from openhcs.core.materialization.options import ( CsvOptions, FileBundleOptions, FileOutputOptions, @@ -30,16 +30,16 @@ TextOptions, TiffStackOptions, ) -from openhcs.processing.materialization.path_scopes import ( +from openhcs.core.materialization.path_scopes import ( ExecutionAxisMaterializationRelativePathScope, MaterializationRelativePathScope, SharedMaterializationRelativePathScope, ) -from openhcs.processing.materialization.persistence import ( +from openhcs.core.materialization.persistence import ( StreamingOnlyMaterializationSpec, TerminalMaterializationSpec, ) -from openhcs.processing.materialization.presets import ( +from openhcs.core.materialization.presets import ( csv_dataclass_materializer, csv_materializer, csv_only, diff --git a/openhcs/processing/materialization/constants.py b/openhcs/core/materialization/constants.py similarity index 100% rename from openhcs/processing/materialization/constants.py rename to openhcs/core/materialization/constants.py diff --git a/openhcs/processing/materialization/core.py b/openhcs/core/materialization/core.py similarity index 95% rename from openhcs/processing/materialization/core.py rename to openhcs/core/materialization/core.py index d1aa093a28..84c7e142d1 100644 --- a/openhcs/processing/materialization/core.py +++ b/openhcs/core/materialization/core.py @@ -24,6 +24,7 @@ import numpy as np import pandas as pd from metaclass_registry import AutoRegisterMeta +from python_introspect import JsonValue, to_jsonable from polystore.config import TiffConfig from polystore.streaming.viewer_transport import ( PathMappedViewerStreamSourceMetadata, @@ -32,6 +33,7 @@ ) from zmqruntime.viewer_protocol import ViewerWireValue +import openhcs.core.numpy_jsonable # noqa: F401 (numpy values in to_jsonable) from openhcs.core.source_path_identity import source_path_identity from openhcs.core.artifacts.payloads import ArtifactMaterializationPayload from openhcs.core.component_set import ComponentSet @@ -84,8 +86,8 @@ StreamViewerComponentMetadataProjector, StreamScopedDisplayConfig, ) -from openhcs.processing.materialization.constants import WriteMode -from openhcs.processing.materialization.options import ( +from openhcs.core.materialization.constants import WriteMode +from openhcs.core.materialization.options import ( CsvOptions, FileBundleOptions, FileOutputOptions, @@ -205,12 +207,7 @@ def filemanager_batches( ) for indices, config in ImageFileFormat.storage_write_batches( tuple( - ( - ImagePayload.of(output.content).with_metadata(output.metadata) - if output.metadata is not None - and ImageFileFormat.is_image_path(output.path) - else output.content - ) + output.stored_content() for output in outputs ), tuple(output.path for output in outputs), @@ -282,6 +279,13 @@ class BackendKwargsAbsent: NormalizedBackendKwargs: TypeAlias = dict[str, "BackendCallKwargs"] BackendKwargsInput = BackendKwargs | BackendKwargsAbsent PrimaryPathSelector = Callable[[list["Output"]], str] + +NONFINITE_FLOAT_CODES: tuple[tuple[int, Callable[[np.ndarray], np.ndarray], float], ...] = ( + (1, np.isnan, np.nan), + (2, np.isposinf, np.inf), + (3, np.isneginf, -np.inf), +) +"""Code stored for each non-finite float a CSV cell cannot spell, its test and value.""" CandidatePathSelector = Callable[[FileOutputOptions, str], tuple[str, ...]] @@ -299,13 +303,30 @@ def table_shape(self) -> tuple[tuple[str, ...], int] | None: def same_materialization_owner(self, other: Output) -> bool: """Compare current destination/source declarations, independent of content.""" + return self._declared_owner() == other._declared_owner() + + def _declared_owner(self) -> tuple[object, ...]: return ( - self.path == other.path - and self.metadata == other.metadata - and self.variable_components == other.variable_components - and self.sample_numbers_by_axis == other.sample_numbers_by_axis + self.path, + self.metadata, + self.variable_components, + self.sample_numbers_by_axis, ) + def stored_content(self) -> MaterializationValue: + """Return the content as a backend stores it: an image carries its metadata.""" + if self.metadata is None or not ImageFileFormat.is_image_path(self.path): + return self.content + return ImagePayload.of(self.content).with_metadata(self.metadata) + + def describes_image(self) -> bool: + """Whether this output carries image metadata to project for viewers.""" + return self.metadata is not None + + def numbers_samples(self) -> bool: + """Whether this output already carries its per-axis sample numbers.""" + return self.sample_numbers_by_axis is not None + @classmethod def from_metadata( cls, @@ -525,12 +546,6 @@ class ColumnarCsvOutput(Output): content: ColumnarRows options: CsvOptions = field(default_factory=CsvOptions) - def __post_init__(self) -> None: - if not isinstance(self.content, ColumnarRows): - raise TypeError("Columnar CSV content must be ColumnarRows.") - if not isinstance(self.options, CsvOptions): - raise TypeError("Columnar CSV output requires CSV writer options.") - def rendered(self) -> Utf8TextOutput: return Utf8TextOutput( path=self.path, @@ -576,7 +591,7 @@ def realized_for_composition( for name in self.content.columns: values = ColumnarRows.column_array(self.content.column_values(name)) selected = name in rendered_columns or name in partition_fields - if not selected and self.options.fields is not None: + if not selected and self.options.declares_fields(): continue mask = MeasurementSparseColumnarRows.present_mask(values) if not selected and bool(np.any(mask)): @@ -618,9 +633,8 @@ def realized_for_composition( elif values.dtype.kind == "f": if bool(np.any(~np.isfinite(values))): codes = np.zeros(len(values), dtype=np.uint8) - codes[np.isnan(values)] = 1 - codes[np.isposinf(values)] = 2 - codes[np.isneginf(values)] = 3 + for code, is_member, _value in NONFINITE_FLOAT_CODES: + codes[is_member(values)] = code nonfinite_codes[name] = ( RenderedColumnarCsvOutput.pack_nonfinite_codes(codes) ) @@ -689,7 +703,7 @@ def compose( raise ValueError( "CSV partitions declare incompatible paths/schema/policy." ) - if output.sample_numbers_by_axis is not None: + if output.numbers_samples(): raise ValueError( "CSV source numbering must be composed by its existing owner." ) @@ -749,8 +763,6 @@ class RenderedColumnarCsvOutput( ) def __post_init__(self) -> None: - if not isinstance(self.options, CsvOptions): - raise TypeError("Realized CSV requires its writer options.") if not self.content.startswith(self.header_content): raise ValueError("Realized CSV does not contain its rendered header.") if self.data_row_count < 0 or tuple(self.key_columns) != self.partition_fields: @@ -919,9 +931,8 @@ def __getitem__(self, name: str) -> Sequence[object]: values = np.asarray(texts, dtype=object) if dtype.kind == "f" and codes is not None: original = codes[offset : offset + count] - values[original == 1] = np.nan - values[original == 2] = np.inf - values[original == 3] = -np.inf + for code, _is_member, value in NONFINITE_FLOAT_CODES: + values[original == code] = value segments.append(values) offset += count values = np.concatenate(segments) if segments else np.empty(0, dtype=object) @@ -982,7 +993,7 @@ def compose( raise ValueError( "Realized CSV partitions declare incompatible output ownership." ) - if any(output.sample_numbers_by_axis is not None for output in values): + if any(output.numbers_samples() for output in values): raise ValueError( "CSV source numbering must be composed by its existing owner." ) @@ -1220,6 +1231,11 @@ class MaterializationInputItem(RuntimeProjectedPayloadItem): def data(self) -> MaterializationValue: return runtime_array_operand(array_data_of(self.value)) + def has_leading_plane_axis(self) -> bool: + """Whether the payload's first dimension is a declared plane axis.""" + plane_axis = self.metadata.plane_axis + return plane_axis is not None + @property def runtime_plane(self) -> RuntimeProjectionPlaneMetadata | None: """Return runtime display-plane identity for this materialized item.""" @@ -1233,20 +1249,8 @@ def domain_for_item(self, item: MaterializationInputItem) -> SourceSpatialDomain metadata_domain = item.metadata.source_spatial_domain.with_value_name( "ROI materialization source image", ) - if metadata_domain.source_shape_yx is not None: - return metadata_domain - local_shape_yx = item.metadata.spatial_shape_yx(item.data) - if local_shape_yx is None: - return metadata_domain - return SourceSpatialDomain( - origin_yx=( - metadata_domain.origin_yx - if metadata_domain.origin_yx is not None - else (0, 0) - ), - source_shape_yx=local_shape_yx, - fill_value=metadata_domain.fill_value, - value_name=metadata_domain.value_name, + return metadata_domain.with_undeclared_shape_from( + lambda: item.metadata.spatial_shape_yx(item.data) ) def domain_for_target( @@ -1347,6 +1351,9 @@ def from_processing_context( return cls(parser=None) return cls(parser=microscope_handler.parser) + def has_parser(self) -> bool: + return self.parser is not None + def required_parser(self) -> FilenameParser: if self.parser is None: raise ValueError("Parser-backed source-stem authority requires a parser.") @@ -1479,10 +1486,7 @@ class ComponentMetadataSourceStemResolutionPolicy(MissingSourceStemResolutionPol strategy_key = "component_metadata" def matches(self, context: ImagePayloadMetadata) -> bool: - return ( - context.source_provenance.scalar_source_identity.component_metadata - is not None - ) + return context.source_provenance.scalar_source_identity.has_component_metadata() def source_stem( self, @@ -1557,7 +1561,7 @@ def matches_context( cls, context: SourceStemAuthoritySelection, ) -> bool: - return context.parser is not None + return context.has_parser() @classmethod def build_for_context( @@ -1720,20 +1724,18 @@ def compatible_with( ) -> bool: if self.path != other.path: return False - if self.source_identity is None or other.source_identity is None: - return True - if ( - self.component_address_identity is not None - and other.component_address_identity is not None - ): - return self.component_address_identity == other.component_address_identity - if ( - self.source_stem_identity is not None - and other.source_stem_identity is not None + # The first facet both identities declare decides; an identity that + # declares none is compatible with any. + for mine, theirs in zip( + self._identity_facets(), other._identity_facets(), strict=True ): - return self.source_stem_identity == other.source_stem_identity + if mine is not None and theirs is not None: + return mine == theirs return True + def _identity_facets(self) -> tuple[object | None, object | None]: + return (self.component_address_identity, self.source_stem_identity) + def merge_with( self, other: ROIMaterializationArchiveIdentity, @@ -2033,6 +2035,10 @@ def from_image_value( ), ) + def is_sequence(self) -> bool: + """Whether the writer received an explicit sequence of payloads.""" + return self.sequence_type is not None + @property def data(self) -> MaterializationValue: data_items = [item.data for item in self.items] @@ -2348,7 +2354,7 @@ def projected_outputs(self, outputs: Sequence[Output]) -> tuple[Output, ...]: display_semantics = self.values.stream_request.display_semantics parser = self.values.stream_request.source.identity.microscope_handler.parser for output in outputs: - if output.metadata is None: + if not output.describes_image(): result.append(output) continue items = StreamImagePayloadMetadataProjector.payload_items_for_display( @@ -2399,12 +2405,12 @@ def _output_fields( source_identity.component_metadata if source_identity is not None else None ) if component_metadata is None: - if not output.viewer_stream_requires_source_metadata: - return None - raise ValueError( - "Viewer stream materialization requires output metadata with " - "source_component_metadata." - ) + if output.viewer_stream_requires_source_metadata: + raise ValueError( + "Viewer stream materialization requires output metadata with " + "source_component_metadata." + ) + return None item_fields = output.viewer_item_fields() display_config = output.viewer_display_config( self.values.stream_request.display_config @@ -2484,9 +2490,10 @@ def save_all( return saved_outputs def _prepare_path(self, backend: str, backend_instance, path: str) -> None: - if not backend_instance.requires_filesystem_validation: - return + if backend_instance.requires_filesystem_validation: + self._prepare_filesystem_path(backend, path) + def _prepare_filesystem_path(self, backend: str, path: str) -> None: self.filemanager.ensure_directory( str(source_path_identity(path).parent), backend ) @@ -2518,6 +2525,17 @@ class MaterializationContext: output_plan: ArtifactOutputPlan = field(kw_only=True) materialization_spec: MaterializationSpec | None = None + def sample_numbers_for_export( + self, relative_path: Path + ) -> Mapping[str, tuple[int, ...]] | None: + """Return the sample numbers the running step recorded for an exported file.""" + if self.context is None: + return None + step_outputs = self.context.runtime_step_outputs + if step_outputs is None: + return None + return step_outputs.sample_numbers_by_export_path.get(relative_path) + def named_source_filename( self, metadata: ImagePayloadMetadata, extension: str, ) -> str | None: @@ -2699,23 +2717,27 @@ def columnar_rows_tabular_rows( return TabularRows(TabularRow(dict(row)) for row in data.row_mappings()) +@singledispatch def _mapping_column_length(value: MaterializationValue) -> int | None: """Return row count for a mapping value that behaves like a table column.""" - if isinstance(value, Mapping): - return None - if isinstance(value, (str, bytes, bytearray)): - return None shape = getattr(value, "shape", None) if shape: return int(shape[0]) - if isinstance(value, Sequence): - return len(value) try: return len(value) # type: ignore[arg-type] except TypeError: return None +@_mapping_column_length.register(Mapping) +@_mapping_column_length.register(str) +@_mapping_column_length.register(bytes) +@_mapping_column_length.register(bytearray) +def _one_cell_column_length(value: MaterializationValue) -> None: + """A mapping or a text value is one cell repeated on every row.""" + return None + + def _mapping_column_value( value: MaterializationValue, row_index: int, @@ -2765,15 +2787,6 @@ class TabularRowsAuthority: SLICE_INDEX_FIELD: ClassVar[str] = "slice_index" - @staticmethod - def requested(options: TabularExtractionOptions) -> bool: - return bool( - options.fields - or options.row_field - or options.row_unpacker - or options.row_columns - ) - @classmethod def build( cls, @@ -2872,7 +2885,7 @@ def _direct_csv_mapping_rows( options: CsvOptions, ) -> tuple[Sequence[Mapping], tuple[str, ...]] | None: """Return existing mapping rows when generic extraction would be a no-op.""" - if options.row_unpacker is not None or options.row_field is not None: + if options.extracts_rows(): return None if not isinstance(data, (list, tuple)): return None @@ -2899,9 +2912,8 @@ def _direct_csv_object_rows( ) -> tuple[Sequence, tuple[str, ...]] | None: """Return object rows when declared fields let us avoid dict materialization.""" if ( - options.row_unpacker is not None - or options.row_field is not None - or not options.fields + options.extracts_rows() + or not options.has_fields() or not isinstance(data, (list, tuple)) ): return None @@ -2993,7 +3005,7 @@ def fieldnames( def _render_json(data: MaterializationValue, options: JsonOptions) -> str: payload: MaterializationValue - if TabularRowsAuthority.requested(options): + if options.requests_extraction(): payload = TabularRowsAuthority.build(data, options) else: payload = data @@ -3001,68 +3013,17 @@ def _render_json(data: MaterializationValue, options: JsonOptions) -> str: if options.wrap_list and isinstance(payload, list): payload = {"total_items": len(payload), "results": payload} - return json.dumps(JsonPayloadAuthority.jsonable(payload), indent=options.indent) - - -class JsonPayloadAuthority: - """Convert materialized payloads into JSON-native values before encoding.""" - - @classmethod - def jsonable(cls, value: MaterializationValue) -> MaterializationValue: - return json_payload_value(value) - + return json.dumps(to_jsonable(payload), indent=options.indent) -@singledispatch -def json_payload_value(value: MaterializationValue) -> MaterializationValue: - if supports_measurement_row_mapping(value): - return { - key: JsonPayloadAuthority.jsonable(item) - for key, item in measurement_row_mapping(value).items() - } - return value +@to_jsonable.register(pd.DataFrame) +def _jsonable_dataframe(value: pd.DataFrame) -> JsonValue: + return to_jsonable(value.to_dict(orient="records")) -@json_payload_value.register(dict) -def json_payload_mapping(value: dict) -> dict: - return {key: JsonPayloadAuthority.jsonable(item) for key, item in value.items()} - -@json_payload_value.register(ColumnarRows) -def json_payload_columnar_rows(value: ColumnarRows) -> list[dict]: - return [ - {key: JsonPayloadAuthority.jsonable(item) for key, item in row.items()} - for row in value.iter_row_mappings() - ] - - -@json_payload_value.register(list) -def json_payload_list(value: list) -> list: - return [JsonPayloadAuthority.jsonable(item) for item in value] - - -@json_payload_value.register(tuple) -def json_payload_tuple(value: tuple) -> tuple: - return tuple(JsonPayloadAuthority.jsonable(item) for item in value) - - -@json_payload_value.register(pd.DataFrame) -def json_payload_dataframe(value: pd.DataFrame) -> TabularRows: - return TabularRows(TabularRow(dict(row)) for row in value.to_dict(orient="records")) - - -@json_payload_value.register(pd.Series) -def json_payload_series(value: pd.Series) -> dict: - return value.to_dict() - - -@json_payload_value.register(np.generic) -def json_payload_numpy_scalar(value: np.generic) -> MaterializationValue: - return value.item() - - -@json_payload_value.register(np.ndarray) -def json_payload_numpy_array(value: np.ndarray) -> list: - return value.tolist() +@to_jsonable.register(pd.Series) +def _jsonable_series(value: pd.Series) -> JsonValue: + return to_jsonable(value.to_dict()) class SingleFileWriterAuthority: @@ -3214,7 +3175,7 @@ def _image_relative_output_path( *, sequence_index: int, ) -> str: - if options.relative_path_template is None: + if not options.names_files_by_template(): return context.paths(options).primary_output_path(options) metadata = data.metadata @@ -3261,7 +3222,7 @@ def paths(options: FileOutputOptions, base_path: str) -> tuple[str, ...]: raise TypeError( "ImageFileCandidatePathAuthority requires ImageFileOptions." ) - if options.relative_path_template is None: + if not options.names_files_by_template(): candidate = PathHelper(base_path, options).primary_output_path(options) else: candidate = _relative_materialization_output_path( @@ -3315,7 +3276,7 @@ def write_image_file( ) outputs = [] for sequence_index, item in enumerate(projected_items, start=1): - if options.relative_path_template is not None: + if options.names_files_by_template(): path = _image_relative_output_path( item.value, options, context, sequence_index=sequence_index, ) @@ -3357,9 +3318,6 @@ def _file_bundle_outputs( outputs: list[Output] = [] normalized_paths: set[PurePosixPath] = set() - step_outputs = ( - context.context.runtime_step_outputs if context.context is not None else None - ) for relative_value, content in data.items(): relative_path = _normalized_relative_materialization_path(relative_value) if relative_path in normalized_paths: @@ -3371,35 +3329,45 @@ def _file_bundle_outputs( context.base_path, relative_path, ) - sample_numbers = ( - step_outputs.sample_numbers_by_export_path.get(Path(relative_value)) - if step_outputs is not None - else None - ) - if isinstance(content, Output): - output = replace( + outputs.append( + _file_bundle_entry_output( content, - path=output_path, - sample_numbers_by_axis=sample_numbers, - ).rendered() - elif isinstance(content, str): - output = Utf8TextOutput.from_text( - path=output_path, - content=content, - sample_numbers_by_axis=sample_numbers, - ) - elif isinstance(content, bytes): - output = Output( - path=output_path, - content=content, - sample_numbers_by_axis=sample_numbers, + output_path, + context.sample_numbers_for_export(Path(relative_value)), ) - else: - raise TypeError("File bundle values must be text, bytes or typed outputs.") - outputs.append(output) + ) return outputs +@singledispatch +def _file_bundle_entry_output( + content: MaterializationValue, + path: str, + sample_numbers: Mapping[str, tuple[int, ...]] | None, +) -> Output: + """Return the output one file-bundle entry declares.""" + raise TypeError("File bundle values must be text, bytes or typed outputs.") + + +@_file_bundle_entry_output.register(Output) +def _typed_file_bundle_entry_output(content, path, sample_numbers) -> Output: + return replace( + content, path=path, sample_numbers_by_axis=sample_numbers + ).rendered() + + +@_file_bundle_entry_output.register(str) +def _text_file_bundle_entry_output(content, path, sample_numbers) -> Output: + return Utf8TextOutput.from_text( + path=path, content=content, sample_numbers_by_axis=sample_numbers + ) + + +@_file_bundle_entry_output.register(bytes) +def _bytes_file_bundle_entry_output(content, path, sample_numbers) -> Output: + return Output(path=path, content=content, sample_numbers_by_axis=sample_numbers) + + @writer_for(FileBundleOptions) def write_file_bundle( data: MaterializationValue, @@ -3469,12 +3437,8 @@ def has_addressable_identity(self) -> bool: def has_single_component_address(self) -> bool: if len(self.items) != 1: return False - return ( - self.items[ - 0 - ].metadata.source_provenance.scalar_source_identity.component_metadata - is not None - ) + metadata = self.items[0].metadata + return metadata.source_provenance.scalar_source_identity.has_component_metadata() @property def requires_source_identity(self) -> bool: @@ -4014,18 +3978,11 @@ def _write_point_roi_zip( if id_field is None: raise ValueError("PointROIOptions requires an object-measurement subject.") features = (options.z_feature, options.y_feature, options.x_feature) - if payload.measurement_feature_owner is not None: - foreign = [ - feature.value - for feature in features - if not payload.measurement_feature_owner.owns_measurement_feature_name( - feature.value - ) - ] - if foreign: - raise ValueError( - f"Point coordinate features are not owned by the table: {foreign!r}." - ) + foreign = payload.unowned_feature_names(feature.value for feature in features) + if foreign: + raise ValueError( + f"Point coordinate features are not owned by the table: {foreign!r}." + ) coordinate_fields = tuple( feature.measurement_row_field_name for feature in features ) @@ -4034,10 +3991,7 @@ def _write_point_roi_zip( raise ValueError(f"Point measurement fields are missing: {sorted(missing)!r}.") provenance = payload.source_provenance.with_common_scalar_identity_from_planes() source_planes = provenance.source_image_provenance_planes - if provenance.source_path is None and ( - not source_planes.has_values - or any(path is None for path in source_planes.paths) - ): + if not (provenance.source_identity.has_path() or source_planes.names_every_path()): raise ValueError( "Point ROI ZIP requires a source-image path or exact paths for every " "source plane." @@ -4167,7 +4121,7 @@ class TiffStackSlicePayloadAuthority: def slices( materialization_input: MaterializationInput, ) -> tuple[MaterializationValue, ...]: - if materialization_input.sequence_type is not None: + if materialization_input.is_sequence(): return tuple(item.data for item in materialization_input.items) if len(materialization_input.items) != 1: raise ValueError( @@ -4175,7 +4129,7 @@ def slices( "explicit payload sequence." ) item = materialization_input.items[0] - if item.metadata.plane_axis is None: + if not item.has_leading_plane_axis(): return (item.data,) data = np.asarray(item.data) if data.ndim < 3 or data.shape[0] <= 0: @@ -4344,8 +4298,12 @@ def output_array( value: MaterializationValue, options: TiffStackOptions, ) -> MaterializationValue: - if not options.normalize_uint8: - return value + if options.normalize_uint8: + return cls.as_uint8(value) + return value + + @classmethod + def as_uint8(cls, value: MaterializationValue) -> MaterializationValue: array = np.asarray(value) if array.dtype == cls.UINT8_DTYPE: return value @@ -4471,12 +4429,13 @@ def axes(self) -> frozenset[str]: ) return frozenset(ordered_axes) + def runtime_planes(self) -> tuple[RuntimeProjectionPlaneMetadata, ...]: + """Return the runtime plane identities the items carry.""" + planes = (item.runtime_plane for item in self.items) + return tuple(plane for plane in planes if plane is not None) + def runtime_plane_rank(self) -> int: - ranks = tuple( - len(item.runtime_plane.plane_indices) - for item in self.items - if item.runtime_plane is not None - ) + ranks = tuple(len(plane.plane_indices) for plane in self.runtime_planes()) if not ranks: return 0 first = ranks[0] @@ -4488,11 +4447,7 @@ def runtime_plane_rank(self) -> int: return first def runtime_plane_shape(self) -> tuple[int, ...]: - shapes = tuple( - item.runtime_plane.plane_shape - for item in self.items - if item.runtime_plane is not None - ) + shapes = tuple(plane.plane_shape for plane in self.runtime_planes()) if not shapes: return () first = shapes[0] @@ -4505,9 +4460,7 @@ def runtime_plane_shape(self) -> tuple[int, ...]: def runtime_planes_carry_source_identity(self) -> bool: states = tuple( - item.runtime_plane.carries_source_plane_identity - for item in self.items - if item.runtime_plane is not None + plane.carries_source_plane_identity for plane in self.runtime_planes() ) if not states: return False @@ -4808,6 +4761,12 @@ def __objectstate_rebuild__( write_mode=write_mode, ) + def disallowed_backends(self, backends: Sequence[str]) -> list[str]: + """Return the backends outside this spec's allow-list; none without a list.""" + if not self.allowed_backends: + return [] + return [backend for backend in backends if backend not in self.allowed_backends] + def participates_in_runtime_export_observation(self) -> bool: """Explicit materialization specs are externally observed exports.""" return True @@ -4940,9 +4899,7 @@ class AllowedBackendsAuthority: @staticmethod def validate(spec: MaterializationSpec, backends: list[str]) -> None: - if not spec.allowed_backends: - return - invalid = [b for b in backends if b not in spec.allowed_backends] + invalid = spec.disallowed_backends(backends) if invalid: raise ValueError( f"Backend(s) {invalid} not in allowed backends for this spec: " diff --git a/openhcs/processing/materialization/options.py b/openhcs/core/materialization/options.py similarity index 86% rename from openhcs/processing/materialization/options.py rename to openhcs/core/materialization/options.py index d0a737c4a2..7e645a0ebe 100644 --- a/openhcs/processing/materialization/options.py +++ b/openhcs/core/materialization/options.py @@ -18,7 +18,7 @@ from openhcs.core.runtime_tabular_values import ColumnarRows from openhcs.core.runtime_measurements import RuntimeMeasurementFeature -from openhcs.processing.materialization.path_scopes import ( +from openhcs.core.materialization.path_scopes import ( MaterializationRelativePathScope, SharedMaterializationRelativePathScope, ) @@ -81,6 +81,22 @@ class TabularExtractionOptions: row_columns: Dict[str, str] = field(default_factory=dict) row_unpacker: Optional[Callable[[Any], List[Dict[str, Any]]]] = None + def declares_fields(self) -> bool: + """Whether the columns are restricted to a declared list (possibly empty).""" + return self.fields is not None + + def has_fields(self) -> bool: + """Whether at least one column is declared.""" + return bool(self.fields) + + def extracts_rows(self) -> bool: + """Whether rows come from a row field or an unpacker, not the value itself.""" + return self.row_unpacker is not None or self.row_field is not None + + def requests_extraction(self) -> bool: + """Whether any extraction option is set.""" + return any((self.fields, self.row_field, self.row_unpacker, self.row_columns)) + @dataclass(frozen=True) class CsvOptions(FileOutputOptions, SourceOptions, TabularExtractionOptions): @@ -110,14 +126,14 @@ def render_parts( schema: tuple[tuple[str, ...], tuple[tuple[str, ...], ...]] | None = None, ) -> tuple[str, str]: """Derive header and complete CSV through this writer's format owner.""" - from openhcs.processing.materialization.core import _render_csv_rows + from openhcs.core.materialization.core import _render_csv_rows columns, _headers = self.csv_schema(rows) if schema is None else schema return _render_csv_rows((), columns), self.render(rows) def render(self, data: Any) -> str: """Render through the existing CSV format owner.""" - from openhcs.processing.materialization.core import _render_csv + from openhcs.core.materialization.core import _render_csv return _render_csv(data, self) @@ -220,6 +236,10 @@ class ImageFileOptions(FileOutputOptions, SourceOptions): default_factory=SharedMaterializationRelativePathScope ) + def names_files_by_template(self) -> bool: + """Whether each file is named from ``relative_path_template``.""" + return self.relative_path_template is not None + @dataclass(frozen=True) class FileBundleOptions(FileOutputOptions): diff --git a/openhcs/processing/materialization/path_scopes.py b/openhcs/core/materialization/path_scopes.py similarity index 90% rename from openhcs/processing/materialization/path_scopes.py rename to openhcs/core/materialization/path_scopes.py index 7610da4b77..c42aaaf5ef 100644 --- a/openhcs/processing/materialization/path_scopes.py +++ b/openhcs/core/materialization/path_scopes.py @@ -45,9 +45,7 @@ def scoped_path( relative_path: PurePosixPath, context: ProcessingContext | None, ) -> PurePosixPath: - if context is None or context.execution_runtime is None: - return relative_path - if len(context.execution_runtime.execution_axis_values) <= 1: + if context is None or context.execution_axis_count() <= 1: return relative_path from openhcs.core.source_projection import OpenHCSPlaneAddress diff --git a/openhcs/processing/materialization/persistence.py b/openhcs/core/materialization/persistence.py similarity index 100% rename from openhcs/processing/materialization/persistence.py rename to openhcs/core/materialization/persistence.py diff --git a/openhcs/processing/materialization/presets.py b/openhcs/core/materialization/presets.py similarity index 98% rename from openhcs/processing/materialization/presets.py rename to openhcs/core/materialization/presets.py index c15bc98a07..caacb6aae7 100644 --- a/openhcs/processing/materialization/presets.py +++ b/openhcs/core/materialization/presets.py @@ -9,8 +9,8 @@ from dataclasses import dataclass, fields as dataclass_fields, is_dataclass from typing import Any, Callable, Dict, List, Optional -from openhcs.processing.materialization.core import MaterializationSpec -from openhcs.processing.materialization.options import ( +from openhcs.core.materialization.core import MaterializationSpec +from openhcs.core.materialization.options import ( CsvOptions, JsonOptions, ROIOptions, diff --git a/openhcs/core/pipeline/artifact_planning.py b/openhcs/core/pipeline/artifact_planning.py index faeb7c6407..464ce1d78a 100644 --- a/openhcs/core/pipeline/artifact_planning.py +++ b/openhcs/core/pipeline/artifact_planning.py @@ -42,7 +42,7 @@ callable_contract_artifact_declarations, ) from metaclass_registry.strategies import MostDerivedContextStrategyMixin -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, ImageFileOptions, NameByArtifactName, diff --git a/openhcs/core/pipeline/function_contracts.py b/openhcs/core/pipeline/function_contracts.py index 22edfc875d..5668ea78e6 100644 --- a/openhcs/core/pipeline/function_contracts.py +++ b/openhcs/core/pipeline/function_contracts.py @@ -37,7 +37,7 @@ AlwaysRequiresVariableComponentStack, VariableComponentStackRequirement, ) -from openhcs.processing.materialization import MaterializationSpec +from openhcs.core.materialization import MaterializationSpec from openhcs.core.axes import AxisRole F = TypeVar("F", bound=Callable) diff --git a/openhcs/core/plate_image_inventory.py b/openhcs/core/plate_image_inventory.py index bf91a91c85..043a8450d2 100644 --- a/openhcs/core/plate_image_inventory.py +++ b/openhcs/core/plate_image_inventory.py @@ -14,9 +14,13 @@ from polystore.base import ImageSamplingRequest, ImageSamplingResult from polystore.virtual_workspace import SourcePixelRef -from openhcs.constants.constants import Backend, FileFormat +from openhcs.constants.constants import Backend from openhcs.core.image_file_serialization import ImageFileFormat from openhcs.core.plate_file_inventory import PlateFileInventoryQuery, PlateFileKind +from openhcs.core.result_file_formats import ( + ResultFileFormat, + ResultFilePreviewReader, +) from openhcs.core.pipeline.path_planner import PathPlannerPathAuthority from openhcs.core.source_workspace_projection import ( VirtualWorkspacePathLookup, @@ -379,7 +383,7 @@ class PlateResultFileRecord: relative_path: str full_path: str - file_format: FileFormat + file_format: type[ResultFileFormat] metadata: Mapping[str, JsonValue] = field(default_factory=dict) source_ref: SourcePixelRef | None = None source_projection: "SourceProjection | None" = None @@ -401,7 +405,7 @@ class PlateFileRecord: source_path: str | None = None relative_path: str | None = None full_path: str | None = None - file_format: FileFormat | None = None + file_format: type[ResultFileFormat] | None = None source_ref: SourcePixelRef | None = None source_projection: "SourceProjection | None" = None @@ -425,7 +429,7 @@ def streamable_image_path(self) -> str | None: @property def streamable_roi_path(self) -> str | None: """Expose ROI artifacts through their declared result format.""" - if self.kind is PlateFileKind.RESULT and self.file_format is FileFormat.ROI: + if self.kind is PlateFileKind.RESULT and self.file_format.streams_rois: return self.full_path or self.key return None @@ -511,7 +515,7 @@ class PlateResultFilePreview: omitted_reason: str | None = None -class PlateResultFilePreviewReader: +class PlateResultFilePreviewReader(ResultFilePreviewReader): """Read bounded previews for analysis result files.""" DEFAULT_MAX_LINES = 8 @@ -525,28 +529,49 @@ def preview( max_lines: int = DEFAULT_MAX_LINES, max_bytes: int = DEFAULT_MAX_BYTES, ) -> PlateResultFilePreview | None: - if record.file_format is FileFormat.ROI: - return cls._roi_preview( - record, - max_examples=max_lines, - max_bytes=max_bytes, - ) - if record.file_format not in { - FileFormat.CSV, - FileFormat.JSON, - FileFormat.TEXT, - }: - return None - path = record.full_path_obj + return record.file_format.preview( + cls, record, max_lines=max_lines, max_bytes=max_bytes + ) + + @staticmethod + def _unreadable_preview(path: Path, max_bytes: int) -> PlateResultFilePreview | None: + """Return the preview that says why the file is not read, if it is not.""" if not path.exists(): return PlateResultFilePreview(omitted_reason="file does not exist") if path.stat().st_size > max_bytes: return PlateResultFilePreview( omitted_reason=f"file exceeds max preview bytes ({max_bytes})" ) + return None + + @classmethod + def text_preview( + cls, + record: PlateResultFileRecord, + *, + max_lines: int, + max_bytes: int, + ) -> PlateResultFilePreview: + path = record.full_path_obj + unreadable = cls._unreadable_preview(path, max_bytes) + if unreadable is not None: + return unreadable + lines, truncated = cls._text_lines(path, max_lines=max_lines) + return PlateResultFilePreview(text_lines=lines, truncated=truncated) + + @classmethod + def csv_preview( + cls, + record: PlateResultFileRecord, + *, + max_lines: int, + max_bytes: int, + ) -> PlateResultFilePreview: + path = record.full_path_obj + unreadable = cls._unreadable_preview(path, max_bytes) + if unreadable is not None: + return unreadable lines, truncated = cls._text_lines(path, max_lines=max_lines) - if record.file_format is not FileFormat.CSV: - return PlateResultFilePreview(text_lines=lines, truncated=truncated) columns, rows = cls._csv_table_from_path(path, max_rows=max_lines) return PlateResultFilePreview( text_lines=lines, @@ -556,7 +581,7 @@ def preview( ) @classmethod - def _roi_preview( + def roi_preview( cls, record: PlateResultFileRecord, *, @@ -998,16 +1023,10 @@ def from_directory_files( ) @staticmethod - def _result_file_format(file_path: Path) -> FileFormat | None: + def _result_file_format(file_path: Path) -> type[ResultFileFormat] | None: if ImageFileFormat.is_image_path(file_path): return ImageFileFormat.require_path(file_path).browser_file_format - if file_path.name.endswith(FileFormat.ROI.value[0]): - return FileFormat.ROI - suffix = file_path.suffix.lower() - for file_format in (FileFormat.CSV, FileFormat.JSON, FileFormat.TEXT): - if suffix in file_format.value: - return file_format - return None + return ResultFileFormat.for_suffix_of(file_path) @dataclass(frozen=True, slots=True) diff --git a/openhcs/core/post_execute.py b/openhcs/core/post_execute.py index 340025b598..9e20f9cdc8 100644 --- a/openhcs/core/post_execute.py +++ b/openhcs/core/post_execute.py @@ -30,7 +30,7 @@ MaterializedRuntimeArtifact, RuntimeArtifactMaterialization, ) - from openhcs.processing.materialization.core import Output + from openhcs.core.materialization.core import Output HookObservations = Mapping[str, object] diff --git a/openhcs/core/result_file_formats.py b/openhcs/core/result_file_formats.py new file mode 100644 index 0000000000..ab560abb05 --- /dev/null +++ b/openhcs/core/result_file_formats.py @@ -0,0 +1,144 @@ +"""Kinds of files a plate's result directories hold. + +Each kind is a class: it declares the suffixes that identify it, picks the +preview a reader builds for it, and picks what a browser does when the file is +opened. The registry key (``name``) is the spelling used in metadata tables and +agent DTOs. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from pathlib import Path +from typing import ClassVar + +from metaclass_registry import AutoRegisterMeta + +class ResultFilePreviewReader(ABC): + """The previews a reader can build; each kind picks one.""" + + @classmethod + @abstractmethod + def text_preview(cls, record, *, max_lines: int, max_bytes: int): + """Preview the file as text lines.""" + + @classmethod + @abstractmethod + def csv_preview(cls, record, *, max_lines: int, max_bytes: int): + """Preview the file as text lines plus parsed columns and rows.""" + + @classmethod + @abstractmethod + def roi_preview(cls, record, *, max_examples: int, max_bytes: int): + """Preview the ROIs the archive holds.""" + + +class ResultFileOpener(ABC): + """What a file browser can do with a result file.""" + + @abstractmethod + def open_in_default_app(self, file_path: Path) -> None: + """Hand the file to the desktop's default application.""" + + @abstractmethod + def stream_rois(self, file_path: Path) -> None: + """Send the file's ROIs to the enabled viewers.""" + + +class ResultFileFormat(ABC, metaclass=AutoRegisterMeta): + """One kind of result file.""" + + __registry_key__ = "name" + __skip_if_no_key__ = True + name: ClassVar[str | None] = None + suffixes: ClassVar[tuple[str, ...]] = () + streams_rois: ClassVar[bool] = False + + @classmethod + def names_path(cls, file_path: Path) -> bool: + """Whether this kind's suffixes identify the path.""" + return file_path.suffix.lower() in cls.suffixes + + @classmethod + def for_suffix_of(cls, file_path: Path) -> type[ResultFileFormat] | None: + """Return the kind whose suffixes identify the path, if any.""" + for member in cls.__registry__.values(): + if member.names_path(file_path): + return member + return None + + @classmethod + def preview( + cls, + reader: type[ResultFilePreviewReader], + record, + *, + max_lines: int, + max_bytes: int, + ): + """Build this kind's bounded preview; kinds without one return None.""" + return None + + @classmethod + def open_with(cls, opener: ResultFileOpener, file_path: Path) -> None: + """Open the file the way this kind is opened.""" + opener.open_in_default_app(file_path) + + +class TiffResultFile(ResultFileFormat): + """An image file; identified by its image format, not by suffix here.""" + + name = "TIFF" + + +class NumpyResultFile(ResultFileFormat): + """A NumPy array file; identified by its image format, not by suffix here.""" + + name = "NUMPY" + + +class TextResultFile(ResultFileFormat): + """A file previewed as text lines.""" + + name = "TEXT" + suffixes = (".txt", ".py", ".md", ".swc") + + @classmethod + def preview(cls, reader, record, *, max_lines, max_bytes): + return reader.text_preview(record, max_lines=max_lines, max_bytes=max_bytes) + + +class JsonResultFile(TextResultFile): + name = "JSON" + suffixes = (".json",) + + +class CsvResultFile(TextResultFile): + """A table: previewed as text lines plus parsed columns and rows.""" + + name = "CSV" + suffixes = (".csv",) + + @classmethod + def preview(cls, reader, record, *, max_lines, max_bytes): + return reader.csv_preview(record, max_lines=max_lines, max_bytes=max_bytes) + + +class RoiArchiveResultFile(ResultFileFormat): + """A ``.roi.zip`` archive of ROIs.""" + + name = "ROI" + suffixes = (".roi.zip",) + streams_rois = True + + @classmethod + def names_path(cls, file_path: Path) -> bool: + return file_path.name.endswith(cls.suffixes) + + @classmethod + def preview(cls, reader, record, *, max_lines, max_bytes): + return reader.roi_preview(record, max_examples=max_lines, max_bytes=max_bytes) + + @classmethod + def open_with(cls, opener: ResultFileOpener, file_path: Path) -> None: + opener.stream_rois(file_path) diff --git a/openhcs/core/runtime_execution_validation.py b/openhcs/core/runtime_execution_validation.py index 5ecde8aed8..71fcaaaf62 100644 --- a/openhcs/core/runtime_execution_validation.py +++ b/openhcs/core/runtime_execution_validation.py @@ -27,7 +27,7 @@ source_component_metadata_items, ) from openhcs.core.source_spatial_domain import SourceSpatialDomain -from openhcs.processing.materialization import Output +from openhcs.core.materialization import Output RuntimeArtifactViewerComponentIdentity = tuple[tuple[str, str], ...] diff --git a/openhcs/core/runtime_exports.py b/openhcs/core/runtime_exports.py index a9d35b6922..dce6aa3257 100644 --- a/openhcs/core/runtime_exports.py +++ b/openhcs/core/runtime_exports.py @@ -17,14 +17,14 @@ from openhcs.core.runtime_measurements import MeasurementTable from openhcs.core.runtime_stores import StoredRuntimeValue from openhcs.core.steps.abstract import StepExecutionObservation -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, FileBundleOptions, FileOutputOptions, ImageFileOptions, MaterializationSpec, ) -from openhcs.processing.materialization.core import materialization_is_empty +from openhcs.core.materialization.core import materialization_is_empty if TYPE_CHECKING: from openhcs.core.orchestrator.execution_result import RuntimeExecutionObservation diff --git a/openhcs/core/runtime_measurements.py b/openhcs/core/runtime_measurements.py index 68b6396cc2..e9c1687e7c 100644 --- a/openhcs/core/runtime_measurements.py +++ b/openhcs/core/runtime_measurements.py @@ -101,6 +101,16 @@ def __post_init__(self, *source_provenance_values: object) -> None: self.rows.validate_fields() self.validate_runtime_slice_axis() + def unowned_feature_names(self, names: Iterable[str]) -> list[str]: + """Return the names this table's feature owner does not own; none without an owner.""" + if self.measurement_feature_owner is None: + return [] + return [ + name + for name in names + if not self.measurement_feature_owner.owns_measurement_feature_name(name) + ] + def runtime_slice_row_domain(self) -> tuple[int, ...] | None: """Return the runtime-slice indices these rows carry, if they carry the axis.""" return MeasurementTable.shared_row_axis_domain( diff --git a/openhcs/core/runtime_stores.py b/openhcs/core/runtime_stores.py index edcf387eec..ad053b6662 100644 --- a/openhcs/core/runtime_stores.py +++ b/openhcs/core/runtime_stores.py @@ -47,7 +47,7 @@ from openhcs.core.axes import Axis, PartitionAxis if TYPE_CHECKING: - from openhcs.processing.materialization.core import Output + from openhcs.core.materialization.core import Output @dataclass(frozen=True, slots=True) diff --git a/openhcs/core/runtime_tabular_values.py b/openhcs/core/runtime_tabular_values.py index c603c29feb..c4c971fbc1 100644 --- a/openhcs/core/runtime_tabular_values.py +++ b/openhcs/core/runtime_tabular_values.py @@ -14,6 +14,9 @@ from typing import Annotated, Any, Union, get_args, get_origin, get_type_hints import numpy as np +from python_introspect import JsonValue, to_jsonable + +import openhcs.core.numpy_jsonable # noqa: F401 (numpy values in to_jsonable) from openhcs.core.runtime_plane_projection import ( RuntimePlaneAxis, @@ -303,6 +306,11 @@ def merge_exact( return tuple(merged) +@to_jsonable.register(ColumnarRows) +def _jsonable_columnar_rows(value: ColumnarRows) -> JsonValue: + return [to_jsonable(row) for row in value.iter_row_mappings()] + + def supports_measurement_row_mapping(row: object) -> bool: """Return whether ``measurement_row_mapping`` can project this row.""" return isinstance(row, Mapping) or is_dataclass(row) diff --git a/openhcs/core/source_image_provenance.py b/openhcs/core/source_image_provenance.py index 7a4d6a499c..8efcca2271 100644 --- a/openhcs/core/source_image_provenance.py +++ b/openhcs/core/source_image_provenance.py @@ -93,6 +93,14 @@ class SourceImageIdentity: component_metadata: SourceComponentMetadata | None = None _identity: SourceProvenanceIdentity = field(init=False, repr=False) + def has_path(self) -> bool: + """Whether this identity names the image's source path.""" + return self.path is not None + + def has_component_metadata(self) -> bool: + """Whether this identity addresses the image by parsed components.""" + return self.component_metadata is not None + def __post_init__(self) -> None: self.path = normalize_source_path(self.path) if self.component_metadata is not None: @@ -655,6 +663,10 @@ def plane_count_sources(self) -> tuple[int, ...]: count for count in (path_count, component_metadata_count) if count > 0 ) + def names_every_path(self) -> bool: + """Whether there are source planes and each has a known path.""" + return self.has_values and all(path is not None for path in self.paths) + @property def has_values(self) -> bool: return bool(self.planes) diff --git a/openhcs/core/source_spatial_domain.py b/openhcs/core/source_spatial_domain.py index 38c0d15276..c5b89af313 100644 --- a/openhcs/core/source_spatial_domain.py +++ b/openhcs/core/source_spatial_domain.py @@ -92,6 +92,25 @@ class SourceSpatialDomain(metaclass=AutoRegisterMeta): spatial_rank: ClassVar[int] = 2 axis_names: ClassVar[tuple[str, ...]] = ("y", "x") + def with_undeclared_shape_from( + self, local_shape_yx: Callable[[], tuple[int, int] | None] + ) -> "SourceSpatialDomain": + """Declare the source shape from the array when none is declared. + + ``local_shape_yx`` is called only when this domain declares no shape. + """ + if self.source_shape_yx is not None: + return self + shape_yx = local_shape_yx() + if shape_yx is None: + return self + return SourceSpatialDomain( + origin_yx=self.origin_yx if self.origin_yx is not None else (0, 0), + source_shape_yx=shape_yx, + fill_value=self.fill_value, + value_name=self.value_name, + ) + @classmethod def for_rank(cls, spatial_rank: int) -> type["SourceSpatialDomain"]: """Return the declared spatial domain of one rank.""" diff --git a/openhcs/core/steps/function_artifact_materialization.py b/openhcs/core/steps/function_artifact_materialization.py index ae912d7b9a..b701b305ff 100644 --- a/openhcs/core/steps/function_artifact_materialization.py +++ b/openhcs/core/steps/function_artifact_materialization.py @@ -50,7 +50,7 @@ StreamSourceComponentMetadataItems, ) from openhcs.core.streaming_config_factory import StreamingViewerSurface -from openhcs.processing.materialization.core import ( +from openhcs.core.materialization.core import ( BackendKwargs, BackendKwargsAuthority, MaterializationSpec, diff --git a/openhcs/domains/microscopy/analysis_consolidation.py b/openhcs/domains/microscopy/analysis_consolidation.py index 4d4a5c4e0a..b3751e2696 100644 --- a/openhcs/domains/microscopy/analysis_consolidation.py +++ b/openhcs/domains/microscopy/analysis_consolidation.py @@ -34,7 +34,7 @@ MaterializedRuntimeArtifact, RuntimeArtifactMaterialization, ) - from openhcs.processing.materialization.core import Output + from openhcs.core.materialization.core import Output @dataclass(frozen=True, slots=True) diff --git a/openhcs/processing/backends/analysis/cell_counting_cpu.py b/openhcs/processing/backends/analysis/cell_counting_cpu.py index db68dd9025..0436c24bf0 100644 --- a/openhcs/processing/backends/analysis/cell_counting_cpu.py +++ b/openhcs/processing/backends/analysis/cell_counting_cpu.py @@ -31,7 +31,7 @@ DataclassMeasurementColumnarRows, ) from openhcs.core.pipeline.function_contracts import artifact_outputs -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( MaterializationSpec, CsvOptions, JsonOptions, diff --git a/openhcs/processing/backends/analysis/cell_counting_cupy.py b/openhcs/processing/backends/analysis/cell_counting_cupy.py index 31427ece41..8db16e6c4e 100644 --- a/openhcs/processing/backends/analysis/cell_counting_cupy.py +++ b/openhcs/processing/backends/analysis/cell_counting_cupy.py @@ -27,7 +27,7 @@ colocalization_analyzer_catalog, detection_method_catalog, ) -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, JsonOptions, MaterializationSpec, diff --git a/openhcs/processing/backends/analysis/cell_counting_pyclesperanto.py b/openhcs/processing/backends/analysis/cell_counting_pyclesperanto.py index c639fe8c01..774fff2ca8 100644 --- a/openhcs/processing/backends/analysis/cell_counting_pyclesperanto.py +++ b/openhcs/processing/backends/analysis/cell_counting_pyclesperanto.py @@ -19,7 +19,7 @@ # OpenHCS imports from openhcs.core.memory import pyclesperanto as pyclesperanto_func from openhcs.core.pipeline.function_contracts import artifact_outputs -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( MaterializationSpec, CsvOptions, JsonOptions, diff --git a/openhcs/processing/backends/analysis/consolidate_analysis_results.py b/openhcs/processing/backends/analysis/consolidate_analysis_results.py index b38c45a415..550197e0f2 100644 --- a/openhcs/processing/backends/analysis/consolidate_analysis_results.py +++ b/openhcs/processing/backends/analysis/consolidate_analysis_results.py @@ -34,7 +34,7 @@ from openhcs.core.memory import numpy as numpy_func from openhcs.core.pipeline.function_contracts import artifact_outputs from openhcs.core.vfs_protocol import PlateInputDirectory -from openhcs.processing.materialization import CsvOptions, MaterializationSpec +from openhcs.core.materialization import CsvOptions, MaterializationSpec if TYPE_CHECKING: from openhcs.core.dataset_sources.interfaces import FilenameParser diff --git a/openhcs/processing/backends/analysis/consolidate_special_outputs.py b/openhcs/processing/backends/analysis/consolidate_special_outputs.py index 4be8f85149..93be0fd822 100644 --- a/openhcs/processing/backends/analysis/consolidate_special_outputs.py +++ b/openhcs/processing/backends/analysis/consolidate_special_outputs.py @@ -24,7 +24,7 @@ from openhcs.core.memory import numpy as numpy_func from openhcs.core.pipeline.function_contracts import artifact_outputs from openhcs.core.vfs_protocol import PlateInputDirectory -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, JsonOptions, MaterializationSpec, diff --git a/openhcs/processing/backends/analysis/count_cells_simple.py b/openhcs/processing/backends/analysis/count_cells_simple.py index e43231676a..1f2c8989b2 100644 --- a/openhcs/processing/backends/analysis/count_cells_simple.py +++ b/openhcs/processing/backends/analysis/count_cells_simple.py @@ -21,7 +21,7 @@ DataclassMeasurementColumnarRows, ) from openhcs.core.pipeline.function_contracts import artifact_inputs, artifact_outputs -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, MaterializationSpec, ROIOptions, diff --git a/openhcs/processing/backends/analysis/hmm_axon.py b/openhcs/processing/backends/analysis/hmm_axon.py index cc5cbefc33..dcbf0d5782 100644 --- a/openhcs/processing/backends/analysis/hmm_axon.py +++ b/openhcs/processing/backends/analysis/hmm_axon.py @@ -21,7 +21,7 @@ from openhcs.constants.constants import Backend from openhcs.core.memory import numpy from openhcs.core.pipeline.function_contracts import artifact_outputs -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, JsonOptions, MaterializationSpec, diff --git a/openhcs/processing/backends/analysis/hmm_axon_torbi.py b/openhcs/processing/backends/analysis/hmm_axon_torbi.py index 3edd6598fa..af7dec897c 100644 --- a/openhcs/processing/backends/analysis/hmm_axon_torbi.py +++ b/openhcs/processing/backends/analysis/hmm_axon_torbi.py @@ -22,7 +22,7 @@ from openhcs.core.lazy_gpu_imports import torch from openhcs.core.memory import torch as torch_func from openhcs.core.pipeline.function_contracts import artifact_outputs -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, JsonOptions, MaterializationSpec, diff --git a/openhcs/processing/backends/analysis/multi_template_matching.py b/openhcs/processing/backends/analysis/multi_template_matching.py index 31ef8344d8..206d677e85 100644 --- a/openhcs/processing/backends/analysis/multi_template_matching.py +++ b/openhcs/processing/backends/analysis/multi_template_matching.py @@ -23,12 +23,12 @@ from openhcs.core.pipeline.function_contracts import artifact_outputs from openhcs.core.projected_image_output import SelectedPlaneImageOutput from openhcs.core.vfs_protocol import PlateInputFile -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, MaterializationSpec, TiffStackOptions, ) -from openhcs.processing.materialization.options import NameByArtifactName +from openhcs.core.materialization.options import NameByArtifactName logger = logging.getLogger(__name__) diff --git a/openhcs/processing/backends/analysis/neurite_outgrowth.py b/openhcs/processing/backends/analysis/neurite_outgrowth.py index 8c15cd8014..c0f3d945c8 100644 --- a/openhcs/processing/backends/analysis/neurite_outgrowth.py +++ b/openhcs/processing/backends/analysis/neurite_outgrowth.py @@ -62,7 +62,7 @@ SpatialGraphNode, ) from openhcs.core.source_metadata import SourceVoxelSpacing, SourceVoxelSpacingUnit -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, ImageFileOptions, MaterializationSpec, diff --git a/openhcs/processing/backends/analysis/skan_axon_analysis.py b/openhcs/processing/backends/analysis/skan_axon_analysis.py index 17e2c5a851..bd79a9e983 100644 --- a/openhcs/processing/backends/analysis/skan_axon_analysis.py +++ b/openhcs/processing/backends/analysis/skan_axon_analysis.py @@ -32,7 +32,7 @@ DataclassMeasurementColumnarRows, ) from openhcs.core.pipeline.function_contracts import artifact_outputs -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, MaterializationSpec, ROIOptions, diff --git a/openhcs/processing/backends/analysis/skeletonize_and_save.py b/openhcs/processing/backends/analysis/skeletonize_and_save.py index 59a702d82e..718678fb31 100644 --- a/openhcs/processing/backends/analysis/skeletonize_and_save.py +++ b/openhcs/processing/backends/analysis/skeletonize_and_save.py @@ -19,7 +19,7 @@ DataclassMeasurementColumnarRows, ) from openhcs.core.pipeline.function_contracts import artifact_outputs -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, MaterializationSpec, ROIOptions, diff --git a/openhcs/processing/backends/cellprofiler/export_to_database.py b/openhcs/processing/backends/cellprofiler/export_to_database.py index 3dd9458538..0e06a4528a 100644 --- a/openhcs/processing/backends/cellprofiler/export_to_database.py +++ b/openhcs/processing/backends/cellprofiler/export_to_database.py @@ -66,7 +66,7 @@ parse_cellprofiler_bool, parse_cellprofiler_int, ) -from openhcs.processing.materialization import FileBundleOptions, MaterializationSpec +from openhcs.core.materialization import FileBundleOptions, MaterializationSpec if TYPE_CHECKING: from openhcs.core.function_patterns import ( diff --git a/openhcs/processing/backends/cellprofiler/primary_object_diagnostics.py b/openhcs/processing/backends/cellprofiler/primary_object_diagnostics.py index 247dfb8ad6..ba9ed89d97 100644 --- a/openhcs/processing/backends/cellprofiler/primary_object_diagnostics.py +++ b/openhcs/processing/backends/cellprofiler/primary_object_diagnostics.py @@ -30,7 +30,7 @@ SmallRemovedLabels, UneditedLabels, ) -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( ImageFileOptions, TerminalMaterializationSpec, NameByArtifactName, diff --git a/openhcs/processing/backends/cellprofiler/save_images.py b/openhcs/processing/backends/cellprofiler/save_images.py index 80317fe21d..369d52db79 100644 --- a/openhcs/processing/backends/cellprofiler/save_images.py +++ b/openhcs/processing/backends/cellprofiler/save_images.py @@ -60,7 +60,7 @@ from openhcs.core.processing_contracts import ( Pure3DContract, ) -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( ExecutionAxisMaterializationRelativePathScope, ImageFileOptions, MaterializationSpec, diff --git a/openhcs/processing/backends/cellprofiler/secondary.py b/openhcs/processing/backends/cellprofiler/secondary.py index 92efdf75ee..f397bcf3b1 100644 --- a/openhcs/processing/backends/cellprofiler/secondary.py +++ b/openhcs/processing/backends/cellprofiler/secondary.py @@ -301,12 +301,10 @@ def artifact_contract_outputs( CellProfilerThresholdAssignment, CellProfilerThresholdMethod, CellProfilerThresholdRequest, - CellProfilerThresholdResult, CellProfilerThresholdScope, CellProfilerThresholdSettings, CellProfilerVarianceMethod, OutputObjectThresholdMeasurementRecordRowsMixin, - ThresholdMeasurementFeatureRecord, ThresholdPrimitiveBackendStrategy, ThresholdSettingsModule, normalize_cellprofiler_image, @@ -786,61 +784,6 @@ class ThresholdMethod(Enum): TRIANGLE = "triangle" -@dataclass -class SecondaryObjectStats(ThresholdMeasurementFeatureRecord): - slice_index: Annotated[int, MeasurementRowAxisField.SLICE_INDEX] - method: SecondaryMethod - object_count: int - mean_area: float - median_area: float - total_area: int - area_coverage_percent: float - final_threshold: float - original_threshold: float = 0.0 - weighted_variance: float = 0.0 - sum_of_entropies: float = 0.0 - - @classmethod - def from_labels( - cls, - labels: np.ndarray, - *, - image_shape: tuple[int, int], - threshold: CellProfilerThresholdResult, - method: SecondaryMethod, - ) -> "SecondaryObjectStats": - areas = np.bincount(np.asarray(labels).ravel())[1:] - positive_areas = areas[areas > 0] - object_count = int(positive_areas.size) - if object_count == 0: - mean_area = 0.0 - median_area = 0.0 - total_area = 0 - else: - mean_area = float(np.mean(positive_areas)) - median_area = float(np.median(positive_areas)) - total_area = int(np.sum(positive_areas)) - height, width = image_shape - pixel_count = height * width - if pixel_count: - area_coverage = 100.0 * total_area / pixel_count - else: - area_coverage = 0.0 - return cls( - slice_index=0, - method=method, - object_count=object_count, - mean_area=mean_area, - median_area=median_area, - total_area=total_area, - area_coverage_percent=area_coverage, - final_threshold=float(threshold.final_threshold), - original_threshold=float(threshold.original_threshold), - weighted_variance=float(threshold.weighted_variance), - sum_of_entropies=float(threshold.sum_of_entropies), - ) - - @dataclass(frozen=True) class SecondaryObjectLabels: """CellProfiler object-container label variants for secondary objects.""" @@ -1274,19 +1217,6 @@ def _execute_identify_secondary_objects( method=method.value, ) phase_started_at = time.perf_counter() - stats = SecondaryObjectStats.from_labels( - object_labels.segmented, - image_shape=img.shape, - threshold=threshold, - method=method, - ) - runtime_profiler.log( - "iso_stats", - time.perf_counter() - phase_started_at, - function="identify_secondary_objects", - method=method.value, - ) - phase_started_at = time.perf_counter() secondary_output = SourceImageObjectLabelBuildRequest( image=image, labels=object_labels.segmented, @@ -2077,7 +2007,6 @@ def _prepare_identify_tertiary_objects() -> None: SecondaryDistanceTransformBackendStrategy, SecondaryMethod, SecondaryObjectLabels, - SecondaryObjectStats, SecondaryPropagationBackendStrategy, SecondarySegmentationRequest, SecondarySegmentationStrategy, diff --git a/openhcs/processing/backends/cellprofiler/spreadsheet_export.py b/openhcs/processing/backends/cellprofiler/spreadsheet_export.py index b3b917548c..786e9bcc2b 100644 --- a/openhcs/processing/backends/cellprofiler/spreadsheet_export.py +++ b/openhcs/processing/backends/cellprofiler/spreadsheet_export.py @@ -100,14 +100,14 @@ SettingToKeywordBinding, parse_cellprofiler_bool, ) -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, FileBundleOptions, MaterializationSpec, WriteMode, ) -from openhcs.processing.materialization.core import ColumnarCsvOutput, RenderedColumnarCsvOutput +from openhcs.core.materialization.core import ColumnarCsvOutput, RenderedColumnarCsvOutput if TYPE_CHECKING: from openhcs.core.function_patterns import FunctionInvocationKey diff --git a/openhcs/processing/backends/enhance/basic_processor_jax.py b/openhcs/processing/backends/enhance/basic_processor_jax.py index 3ec9de0485..42ff7f7ebf 100644 --- a/openhcs/processing/backends/enhance/basic_processor_jax.py +++ b/openhcs/processing/backends/enhance/basic_processor_jax.py @@ -26,7 +26,7 @@ from openhcs.core.processing_contracts import ( Pure3DContract, ) -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( ImageFileOptions, MaterializationSpec, NameByArtifactName, diff --git a/openhcs/processing/backends/pos_gen/tile_position_artifacts.py b/openhcs/processing/backends/pos_gen/tile_position_artifacts.py index c90ea74297..68a9a41b92 100644 --- a/openhcs/processing/backends/pos_gen/tile_position_artifacts.py +++ b/openhcs/processing/backends/pos_gen/tile_position_artifacts.py @@ -7,7 +7,7 @@ from openhcs.core.artifacts.specs import ArtifactSpec from openhcs.core.artifacts.kinds import SpecialArtifactType -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( JsonOptions, MaterializationSpec, NameByArtifactName, diff --git a/openhcs/processing/custom_functions/templates.py b/openhcs/processing/custom_functions/templates.py index 860359dd1f..eea3d5fee6 100644 --- a/openhcs/processing/custom_functions/templates.py +++ b/openhcs/processing/custom_functions/templates.py @@ -11,7 +11,7 @@ For analysis functions that produce structured outputs (cell counts, measurements, etc.): - Use @artifact_outputs decorator to declare outputs - - Use materialization functions from openhcs.processing.materialization + - Use materialization functions from openhcs.core.materialization - Return tuple: (processed_image, analysis_result_1, analysis_result_2, ...) """ @@ -112,7 +112,7 @@ def my_custom_function(image, scale: float = 1.0, offset: float = 0.0): # from openhcs.core.artifacts.specs import ArtifactSpec # from openhcs.core.measurement_row_materialization import DataclassMeasurementColumnarRows # from openhcs.core.pipeline.function_contracts import artifact_outputs -# from openhcs.processing.materialization import csv_only +# from openhcs.core.materialization import csv_only # # 2. Declare the semantic output and derive its columns from the row dataclass: # @@ -323,7 +323,7 @@ def my_custom_function(image, radius: float = 2.0): NUMPY_ANALYSIS_TEMPLATE = """from openhcs.core.memory import numpy from openhcs.core.pipeline.function_contracts import artifact_outputs -from openhcs.processing.materialization import CsvOptions, MaterializationSpec +from openhcs.core.materialization import CsvOptions, MaterializationSpec from dataclasses import dataclass from typing import List, Tuple import numpy as np @@ -384,7 +384,7 @@ def my_analysis_function(image, threshold: float = 0.5) -> Tuple[np.ndarray, Lis NUMPY_DUAL_OUTPUT_TEMPLATE = """from openhcs.core.memory import numpy from openhcs.core.pipeline.function_contracts import artifact_outputs -from openhcs.processing.materialization import CsvOptions, JsonOptions, MaterializationSpec +from openhcs.core.materialization import CsvOptions, JsonOptions, MaterializationSpec from dataclasses import dataclass from typing import List, Tuple, Dict, Any import numpy as np diff --git a/openhcs/pyqt_gui/widgets/image_browser.py b/openhcs/pyqt_gui/widgets/image_browser.py index 6feb1765c6..c25daee751 100644 --- a/openhcs/pyqt_gui/widgets/image_browser.py +++ b/openhcs/pyqt_gui/widgets/image_browser.py @@ -39,8 +39,8 @@ ImageTableValue, ) -from openhcs.constants.constants import FileFormat from openhcs.core.config import StreamingConfig +from openhcs.core.result_file_formats import ResultFileFormat, ResultFileOpener from openhcs.core.plate_image_inventory import ( PlateFileInventory, PlateFileKind, @@ -96,18 +96,6 @@ def _streaming_config_field_names() -> tuple[str, ...]: return StreamingConfig.supported_config_keys() -@dataclass(frozen=True) -class ResultFileAction: - """Double-click behavior for one result-file type.""" - - file_type: FileFormat - display_name: str - handle: Callable[["ImageBrowserWidget", Path], None] - - def run(self, browser: "ImageBrowserWidget", file_path: Path) -> None: - self.handle(browser, file_path) - - @dataclass(slots=True) class ImageBrowserItem(Mapping[str, ImageTableValue]): """One image/result row plus optional result-file action metadata.""" @@ -116,7 +104,7 @@ class ImageBrowserItem(Mapping[str, ImageTableValue]): metadata: dict[str, ImageTableValue] file_kind: PlateFileKind = PlateFileKind.IMAGE streamable_image_path: str | None = None - result_file_type: FileFormat | None = None + result_file_type: type[ResultFileFormat] | None = None full_path: Path | None = None @property @@ -142,36 +130,17 @@ def __len__(self) -> int: return len(self.metadata) -def _stream_roi_result(browser: "ImageBrowserWidget", file_path: Path) -> None: - browser._stream_roi_file(file_path) - +@dataclass(frozen=True) +class ResultFileBrowserOpener(ResultFileOpener): + """What the image browser does when a result file is opened.""" -def _open_result_in_default_app(browser: "ImageBrowserWidget", file_path: Path) -> None: - subprocess.run(["xdg-open", str(file_path)], check=False) + browser: "ImageBrowserWidget" + def open_in_default_app(self, file_path: Path) -> None: + subprocess.run(["xdg-open", str(file_path)], check=False) -RESULT_FILE_ACTIONS = { - FileFormat.ROI: ResultFileAction( - file_type=FileFormat.ROI, - display_name="ROI", - handle=_stream_roi_result, - ), - FileFormat.CSV: ResultFileAction( - file_type=FileFormat.CSV, - display_name="CSV", - handle=_open_result_in_default_app, - ), - FileFormat.JSON: ResultFileAction( - file_type=FileFormat.JSON, - display_name="JSON", - handle=_open_result_in_default_app, - ), - FileFormat.TEXT: ResultFileAction( - file_type=FileFormat.TEXT, - display_name="TEXT", - handle=_open_result_in_default_app, - ), -} + def stream_rois(self, file_path: Path) -> None: + self.browser._stream_roi_file(file_path) @dataclass(frozen=True) @@ -1018,10 +987,9 @@ def _items_from_file_records( and record.file_format is not None and record.full_path is not None ): - action = RESULT_FILE_ACTIONS[record.file_format] logger.info( "IMAGE BROWSER RESULTS: matched as %s: %s", - action.display_name, + record.file_format.name, record.key, ) result_items[record.key] = ImageBrowserItem( @@ -1253,7 +1221,7 @@ def _handle_result_double_click(self, item: ImageBrowserItem): """Handle double-click on a result file - stream ROIs or display CSV.""" if item.result_file_type is None: raise RuntimeError(f"Image browser item {item.key!r} has no result type.") - RESULT_FILE_ACTIONS[item.result_file_type].run(self, item.result_path()) + item.result_file_type.open_with(ResultFileBrowserOpener(self), item.result_path()) def _view_selected_in_viewer(self, config_key: str): """View all selected images in the specified viewer as a batch (builds hyperstack).""" @@ -1270,7 +1238,7 @@ def _view_selected_in_viewer(self, config_key: str): roi_filenames = [ item.key for item in selected_items - if item.result_file_type is FileFormat.ROI + if item.is_result and item.result_file_type.streams_rois ] logger.info( diff --git a/openhcs/serialization/pycodify_formatters.py b/openhcs/serialization/pycodify_formatters.py index 4aa795791b..7eae46b939 100644 --- a/openhcs/serialization/pycodify_formatters.py +++ b/openhcs/serialization/pycodify_formatters.py @@ -42,7 +42,7 @@ class MaterializationSpecFormatter(SourceFormatter): priority = 110 def can_format(self, value) -> bool: - from openhcs.processing.materialization.core import MaterializationSpec + from openhcs.core.materialization.core import MaterializationSpec return isinstance(value, MaterializationSpec) @@ -63,7 +63,7 @@ def format(self, value, context: FormatContext) -> SourceFragment: args.append(f"allowed_backends={allowed_backends_frag.code}") if value.primary != 0: args.append(f"primary={value.primary}") - from openhcs.processing.materialization.constants import WriteMode + from openhcs.core.materialization.constants import WriteMode if value.write_mode is not WriteMode.OVERWRITE: write_mode_frag = to_source(value.write_mode, item_ctx) diff --git a/paper/figures/build_slas_outputs.py b/paper/figures/build_slas_outputs.py index 780ec22332..bdf6b8a50d 100644 --- a/paper/figures/build_slas_outputs.py +++ b/paper/figures/build_slas_outputs.py @@ -138,8 +138,8 @@ def build(): ROOT / "openhcs/core/measurement_row_materialization.py", ROOT / "openhcs/core/pipeline/artifact_planning.py", ROOT / "openhcs/core/steps/function_artifact_materialization.py", - ROOT / "openhcs/processing/materialization/options.py", - ROOT / "openhcs/processing/materialization/core.py", + ROOT / "openhcs/core/materialization/options.py", + ROOT / "openhcs/core/materialization/core.py", ROOT / "openhcs/processing/backends/analysis/neurite_outgrowth.py", ROOT / "openhcs/runtime/napari_streaming_handlers.py", ROOT / "openhcs/runtime/napari_viewer_server.py", diff --git a/tests/diagnostics/create_native_result_reopening_fixture.py b/tests/diagnostics/create_native_result_reopening_fixture.py index 98a9641c5a..41f4cf8a16 100644 --- a/tests/diagnostics/create_native_result_reopening_fixture.py +++ b/tests/diagnostics/create_native_result_reopening_fixture.py @@ -18,7 +18,7 @@ ) from openhcs.core.source_metadata import SourceVoxelSpacing from openhcs.core.source_spatial_domain import SourceSpatialDomain -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( MaterializationSpec, ROIOptions, materialize, diff --git a/tests/diagnostics/volume_projection_fixture.py b/tests/diagnostics/volume_projection_fixture.py index c6977d06ae..12ec045359 100644 --- a/tests/diagnostics/volume_projection_fixture.py +++ b/tests/diagnostics/volume_projection_fixture.py @@ -28,7 +28,7 @@ from openhcs.core.processing_contracts import ( Pure3DContract, ) -from openhcs.processing.materialization import CsvOptions, MaterializationSpec, ROIOptions +from openhcs.core.materialization import CsvOptions, MaterializationSpec, ROIOptions class VolumeProjectionFixtureFeature(RuntimeMeasurementFeature): diff --git a/tests/integration/test_count_cells_simple_dual_channel.py b/tests/integration/test_count_cells_simple_dual_channel.py index 79f9c6ba38..8804075b1c 100644 --- a/tests/integration/test_count_cells_simple_dual_channel.py +++ b/tests/integration/test_count_cells_simple_dual_channel.py @@ -183,7 +183,7 @@ def test_dual_channel_count_runs_on_synthetic_plate_with_channel_stack( """ from openhcs.core.memory import numpy from openhcs.core.pipeline.function_contracts import artifact_outputs -from openhcs.processing.materialization import CsvOptions, MaterializationSpec, ROIOptions +from openhcs.core.materialization import CsvOptions, MaterializationSpec, ROIOptions import numpy as np from openhcs.microscopes.imagexpress import ImageXpressHandler diff --git a/tests/integration/test_measurement_declaration_journey.py b/tests/integration/test_measurement_declaration_journey.py index 69d86c9430..355b5ad4e3 100644 --- a/tests/integration/test_measurement_declaration_journey.py +++ b/tests/integration/test_measurement_declaration_journey.py @@ -88,7 +88,7 @@ CustomFunctionRuntimeRegistry, register_custom_function, ) -from openhcs.processing.materialization import CsvOptions, MaterializationSpec +from openhcs.core.materialization import CsvOptions, MaterializationSpec from openhcs.core.axes import Ungrouped from openhcs.domains.microscopy.axes import Microscopy from openhcs.core.dataset_sources.source_bindings_source import SourceBindingsSource diff --git a/tests/integration/test_neurite_outgrowth_synthetic_plate.py b/tests/integration/test_neurite_outgrowth_synthetic_plate.py index 4192e66170..8cea2dba2f 100644 --- a/tests/integration/test_neurite_outgrowth_synthetic_plate.py +++ b/tests/integration/test_neurite_outgrowth_synthetic_plate.py @@ -62,7 +62,7 @@ VirtualWorkspacePathLookup, VirtualWorkspaceSourceProjection, ) -from openhcs.processing.materialization.core import ( +from openhcs.core.materialization.core import ( Output, ViewerStreamBackendCallKwargs, ) diff --git a/tests/integration/test_primary_segmentation_diagnostics_journey.py b/tests/integration/test_primary_segmentation_diagnostics_journey.py index 43b879ec68..2df5096e55 100644 --- a/tests/integration/test_primary_segmentation_diagnostics_journey.py +++ b/tests/integration/test_primary_segmentation_diagnostics_journey.py @@ -159,7 +159,7 @@ def test_normal_compiled_runtime_persists_diagnostics_and_preserves_secondary_bi from openhcs.processing.backends.cellprofiler.thresholding import ( CellProfilerThresholdMethod, ) - from openhcs.processing.materialization.core import MaterializationInput + from openhcs.core.materialization.core import MaterializationInput from python_introspect import to_jsonable def binding(module, plan_type, artifact_type): diff --git a/tests/integration/test_source_schema_aggregate_publication.py b/tests/integration/test_source_schema_aggregate_publication.py index a993ef7531..c6e37dfe87 100644 --- a/tests/integration/test_source_schema_aggregate_publication.py +++ b/tests/integration/test_source_schema_aggregate_publication.py @@ -48,7 +48,7 @@ CustomFunctionRuntimeRegistry, register_custom_function, ) -from openhcs.processing.materialization import CsvOptions, MaterializationSpec +from openhcs.core.materialization import CsvOptions, MaterializationSpec from openhcs.core.axes import Ungrouped from openhcs.domains.microscopy.axes import Microscopy from openhcs.core.dataset_sources.source_bindings_source import SourceBindingsSource diff --git a/tests/pyqt_gui/test_image_browser_filter_controller.py b/tests/pyqt_gui/test_image_browser_filter_controller.py index fb5d82e26b..6c2256f128 100644 --- a/tests/pyqt_gui/test_image_browser_filter_controller.py +++ b/tests/pyqt_gui/test_image_browser_filter_controller.py @@ -4,7 +4,11 @@ from types import SimpleNamespace from polystore.disk import DiskStorageBackend from polystore.filemanager import FileManager -from openhcs.constants.constants import FileFormat +from openhcs.core.result_file_formats import ( + RoiArchiveResultFile, + TextResultFile, + TiffResultFile, +) from openhcs.core.component_filters import ComponentFilters from openhcs.core.components.parser_metaprogramming import FilenameParseResult from openhcs.core.config import GlobalPipelineConfig @@ -140,6 +144,9 @@ def source_workspace_metadata_document(self, _plate_path: Path): def source_dataset(self, _plate_path: Path): return None + def source_workspace_root(self, plate_path: Path) -> Path: + return plate_path + def analysis_result_directories(self, _plate_path: Path): return ( AnalysisResultDirectory( @@ -245,7 +252,7 @@ def test_image_browser_projects_shared_result_inventory_records(tmp_path: Path): PlateResultFileRecord( relative_path="results/A01.roi.zip", full_path=str(roi_path), - file_format=FileFormat.ROI, + file_format=RoiArchiveResultFile, metadata={ "filename": "results/A01.roi.zip", "type": "ROI", @@ -255,7 +262,7 @@ def test_image_browser_projects_shared_result_inventory_records(tmp_path: Path): PlateResultFileRecord( relative_path="results/summary.txt", full_path=str(text_path), - file_format=FileFormat.TEXT, + file_format=TextResultFile, metadata={ "filename": "results/summary.txt", "type": "TEXT", @@ -268,11 +275,11 @@ def test_image_browser_projects_shared_result_inventory_records(tmp_path: Path): result_items = ImageBrowserWidget._result_items_from_inventory(inventory) item = result_items["results/A01.roi.zip"] - assert item.result_file_type is FileFormat.ROI + assert item.result_file_type is RoiArchiveResultFile assert item.full_path == roi_path assert item.metadata["type"] == "ROI" text_item = result_items["results/summary.txt"] - assert text_item.result_file_type is FileFormat.TEXT + assert text_item.result_file_type is TextResultFile assert text_item.full_path == text_path assert text_item.metadata["type"] == "TEXT" @@ -288,7 +295,7 @@ def test_image_browser_projects_result_images_as_streamable_images(tmp_path: Pat PlateResultFileRecord( relative_path="results/A01_labels.tif", full_path=str(image_path), - file_format=FileFormat.TIFF, + file_format=TiffResultFile, metadata={ "filename": "results/A01_labels.tif", "type": "TIFF", @@ -449,4 +456,4 @@ def test_plate_file_inventory_query_returns_unified_browser_records( assert query_result.total_count == 1 assert query_result.records[0].key == "images/A01_w1.tif" assert image_items["images/A01_w1.tif"].metadata["well"] == "A01" - assert result_items["results/A01_summary.txt"].result_file_type is FileFormat.TEXT + assert result_items["results/A01_summary.txt"].result_file_type is TextResultFile diff --git a/tests/unit/agent/test_callable_artifact_reference.py b/tests/unit/agent/test_callable_artifact_reference.py index d06cf13ced..f9a5a29d67 100644 --- a/tests/unit/agent/test_callable_artifact_reference.py +++ b/tests/unit/agent/test_callable_artifact_reference.py @@ -45,7 +45,7 @@ ) from openhcs.processing.custom_functions import manager as custom_manager -from openhcs.processing.materialization import materialize +from openhcs.core.materialization import materialize from polystore.disk import DiskStorageBackend from polystore.filemanager import FileManager from polystore.roi import load_rois_from_zip diff --git a/tests/unit/agent/test_mcp_self_description_contracts.py b/tests/unit/agent/test_mcp_self_description_contracts.py index d76e4caf52..483e8a16b8 100644 --- a/tests/unit/agent/test_mcp_self_description_contracts.py +++ b/tests/unit/agent/test_mcp_self_description_contracts.py @@ -11,7 +11,9 @@ from openhcs.agent.path_policy import AgentPathPolicy, AgentPathPolicyError from openhcs.agent.services.config_service import ConfigService -from openhcs.constants.constants import FileFormat +from openhcs.core.result_file_formats import ( + RoiArchiveResultFile, +) from openhcs.core.plate_file_inventory import ( PlateFileInventoryQuery, PlateFileKind, @@ -89,7 +91,7 @@ def test_exact_plate_file_path_reports_kind_exclusion() -> None: PlateResultFileRecord( relative_path="results/A01_rois.roi.zip", full_path=result_path, - file_format=FileFormat.ROI, + file_format=RoiArchiveResultFile, ), ), ) diff --git a/tests/unit/agent/test_plate_streaming_service.py b/tests/unit/agent/test_plate_streaming_service.py index a0c4aba2d6..0a4347af7e 100644 --- a/tests/unit/agent/test_plate_streaming_service.py +++ b/tests/unit/agent/test_plate_streaming_service.py @@ -16,7 +16,13 @@ from openhcs.agent.services.plate_inspection_service import PlateInspectionContext from openhcs.agent.services.plate_inspection_service import PlateInspectionService from openhcs.agent.services.plate_streaming_service import PlateStreamingService -from openhcs.constants.constants import Backend, FileFormat +from openhcs.constants.constants import Backend +from openhcs.core.result_file_formats import ( + CsvResultFile, + JsonResultFile, + RoiArchiveResultFile, + TiffResultFile, +) from openhcs.core.plate_image_inventory import ( PlateFileInventory, PlateFileRecord, @@ -36,7 +42,7 @@ from openhcs.core.runtime_tabular_values import FieldSpec from openhcs.core.source_image_provenance import SourceImageProvenancePlanes from openhcs.core.viewer_streaming_service import ViewerStreamingSource -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( MaterializationSpec, PointROIOptions, materialize, @@ -627,7 +633,7 @@ def test_plate_streaming_service_rejects_non_roi_result_files(monkeypatch): PlateResultFileRecord( relative_path="images_results/A01_counts.csv", full_path="/plate/images_results/A01_counts.csv", - file_format=FileFormat.CSV, + file_format=CsvResultFile, metadata={"well": "A01"}, ), ), @@ -662,7 +668,7 @@ def test_plate_streaming_service_streams_roi_result_full_path_and_metadata(monke PlateResultFileRecord( relative_path="images_results/A01_w1_rois.roi.zip", full_path=roi_full_path, - file_format=FileFormat.ROI, + file_format=RoiArchiveResultFile, metadata={ "well": "A01", "site": 1, @@ -728,7 +734,7 @@ def test_plate_streaming_service_reports_explicit_result_excluded_by_default_kin PlateResultFileRecord( relative_path="images_results/A01_w1_rois.roi.zip", full_path=roi_full_path, - file_format=FileFormat.ROI, + file_format=RoiArchiveResultFile, metadata={"well": "A01", "channel": 1}, ), ), @@ -770,7 +776,7 @@ def test_explicit_stream_path_selects_its_declared_dual_projection(): PlateResultFileRecord( relative_path=relative_path, full_path=full_path, - file_format=FileFormat.TIFF, + file_format=TiffResultFile, ), ), ) @@ -800,25 +806,25 @@ def test_plate_streaming_service_query_limit_counts_streamable_roi_results(monke PlateResultFileRecord( relative_path="checkpoints_step7_results/A01_measurements.json", full_path="/plate_openhcs/checkpoints_step7_results/A01_measurements.json", - file_format=FileFormat.JSON, + file_format=JsonResultFile, metadata={"well": "A01"}, ), PlateResultFileRecord( relative_path="checkpoints_step7_results/A01_summary.csv", full_path="/plate_openhcs/checkpoints_step7_results/A01_summary.csv", - file_format=FileFormat.CSV, + file_format=CsvResultFile, metadata={"well": "A01"}, ), PlateResultFileRecord( relative_path="checkpoints_step7_results/A01_w1_rois.roi.zip", full_path=roi_one_path, - file_format=FileFormat.ROI, + file_format=RoiArchiveResultFile, metadata={"well": "A01", "channel": 1}, ), PlateResultFileRecord( relative_path="checkpoints_step7_results/A01_w2_rois.roi.zip", full_path=roi_two_path, - file_format=FileFormat.ROI, + file_format=RoiArchiveResultFile, metadata={"well": "A01", "channel": 2}, ), ), @@ -885,7 +891,7 @@ def test_plate_streaming_service_uses_context_plate_for_output_roi_stream(monkey PlateResultFileRecord( relative_path="images_results/A01_w1_rois.roi.zip", full_path=roi_full_path, - file_format=FileFormat.ROI, + file_format=RoiArchiveResultFile, metadata={}, ), ), @@ -943,7 +949,7 @@ def test_explicit_result_route_keeps_source_context_and_requires_native_binding( PlateResultFileRecord( relative_path="misleading_A01_w2.roi.zip", full_path=path, - file_format=FileFormat.ROI, + file_format=RoiArchiveResultFile, metadata={}, ), ), diff --git a/tests/unit/test_aggregate_layer_materialization_515.py b/tests/unit/test_aggregate_layer_materialization_515.py index fe916d15da..1d62431c7c 100644 --- a/tests/unit/test_aggregate_layer_materialization_515.py +++ b/tests/unit/test_aggregate_layer_materialization_515.py @@ -49,7 +49,7 @@ from polystore.streaming.viewer_transport import ViewerStreamKwarg from polystore.streaming.viewer_transport import ViewerStreamBackendKwargs from polystore.streaming_constants import StreamingDataType -from openhcs.processing.materialization.core import ( +from openhcs.core.materialization.core import ( BackendSaver, Output, ViewerStreamBackendCallKwargs, diff --git a/tests/unit/test_aggregate_source_scope.py b/tests/unit/test_aggregate_source_scope.py index e17ba4f9d2..7855206b26 100644 --- a/tests/unit/test_aggregate_source_scope.py +++ b/tests/unit/test_aggregate_source_scope.py @@ -31,8 +31,8 @@ ) from openhcs.core.source_metadata import SourceVoxelSpacing from openhcs.core.steps.abstract import StepExecutionObservation -from openhcs.processing.materialization import CsvOptions, MaterializationSpec -from openhcs.processing.materialization.core import Output, materialization_outputs +from openhcs.core.materialization import CsvOptions, MaterializationSpec +from openhcs.core.materialization.core import Output, materialization_outputs from polystore.filemanager import FileManager from polystore.memory import MemoryStorageBackend diff --git a/tests/unit/test_artifact_output_materialization_planner.py b/tests/unit/test_artifact_output_materialization_planner.py index 3e5b391dee..e405af5271 100644 --- a/tests/unit/test_artifact_output_materialization_planner.py +++ b/tests/unit/test_artifact_output_materialization_planner.py @@ -1,5 +1,5 @@ import numpy as np -from openhcs.processing.materialization.persistence import TerminalMaterializationSpec +from openhcs.core.materialization.persistence import TerminalMaterializationSpec from polystore.filemanager import FileManager from polystore.memory import MemoryStorageBackend @@ -15,7 +15,7 @@ ArtifactOutputMaterializationPlanner, ) from openhcs.core.runtime_exports import RuntimeExportExpectation -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, ImageFileOptions, NameByArtifactName, diff --git a/tests/unit/test_artifact_output_ownership.py b/tests/unit/test_artifact_output_ownership.py index 471c2a7f4f..28a2a9218d 100644 --- a/tests/unit/test_artifact_output_ownership.py +++ b/tests/unit/test_artifact_output_ownership.py @@ -35,7 +35,7 @@ from openhcs.interop.cellprofiler.runtime.adapter import ( CellProfilerRecordedArtifactOutputPolicy, ) -from openhcs.processing.materialization import CsvOptions, MaterializationSpec +from openhcs.core.materialization import CsvOptions, MaterializationSpec @pytest.mark.parametrize("invalid_policy", [None, False, ArtifactOutputPolicy]) diff --git a/tests/unit/test_automatic_image_role_materialization_435.py b/tests/unit/test_automatic_image_role_materialization_435.py index 25720be57c..97f84d7bfc 100644 --- a/tests/unit/test_automatic_image_role_materialization_435.py +++ b/tests/unit/test_automatic_image_role_materialization_435.py @@ -1,6 +1,6 @@ """Default retained image artifacts keep their declared output role identity.""" -from openhcs.processing.materialization.options import NameByArtifactName, NameBySourceIdentity +from openhcs.core.materialization.options import NameByArtifactName, NameBySourceIdentity from dataclasses import replace from pathlib import Path import json @@ -151,7 +151,7 @@ def test_automatic_two_role_images_publish_their_actual_saved_occurrences(tmp_pa "explicit_source_declared_terminal", "terminal_source_declared_explicit", )) def test_retained_role_policy_preserves_authored_and_unnamed_image_paths(tmp_path, purpose): - from openhcs.processing.materialization import ( + from openhcs.core.materialization import ( ImageFileOptions, MaterializationSpec, TerminalMaterializationSpec, materialization_outputs, ) @@ -211,7 +211,7 @@ def test_retained_role_policy_preserves_authored_and_unnamed_image_paths(tmp_pat def test_authored_duplicate_template_still_fails_before_any_save(tmp_path): - from openhcs.processing.materialization import ImageFileOptions, TerminalMaterializationSpec, materialization_outputs + from openhcs.core.materialization import ImageFileOptions, TerminalMaterializationSpec, materialization_outputs spec = TerminalMaterializationSpec(ImageFileOptions( filename_suffix=".tif", relative_path_template="same.tif", @@ -235,7 +235,7 @@ def test_authored_duplicate_template_still_fails_before_any_save(tmp_path): @pytest.mark.parametrize("plane_count", (1, 2)) def test_retained_image_qualifier_respects_complete_writer_suffix(tmp_path, suffix, plane_count): from openhcs.core.steps.function_artifact_materialization import RuntimeArtifactMaterialization - from openhcs.processing.materialization import ImageFileOptions, TerminalMaterializationSpec + from openhcs.core.materialization import ImageFileOptions, TerminalMaterializationSpec manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) context = context_stub(manager, parser=SourceSchemaFilenameParser()) @@ -274,7 +274,7 @@ def test_retained_image_qualifier_respects_complete_writer_suffix(tmp_path, suff def test_distinct_aliases_that_normalize_to_one_filename_still_conflict(tmp_path): from openhcs.core.steps.function_artifact_materialization import RuntimeArtifactMaterialization - from openhcs.processing.materialization import ImageFileOptions, TerminalMaterializationSpec + from openhcs.core.materialization import ImageFileOptions, TerminalMaterializationSpec manager = FileManager({Backend.MEMORY.value: MemoryStorageBackend()}) context = context_stub(manager, parser=SourceSchemaFilenameParser()) @@ -301,8 +301,8 @@ def test_distinct_aliases_that_normalize_to_one_filename_still_conflict(tmp_path def test_batch_binds_actual_purpose_once_without_rewriting_plan_or_source(monkeypatch, tmp_path): - from openhcs.processing.materialization import ImageFileOptions, MaterializationSpec, TerminalMaterializationSpec - from openhcs.processing.materialization.core import MaterializationBatch, MaterializationContext, Output, WriterSpec, _WRITERS_BY_OPTIONS + from openhcs.core.materialization import ImageFileOptions, MaterializationSpec, TerminalMaterializationSpec + from openhcs.core.materialization.core import MaterializationBatch, MaterializationContext, Output, WriterSpec, _WRITERS_BY_OPTIONS options = ImageFileOptions(filename_suffix=".tif") declared = TerminalMaterializationSpec(options) @@ -340,8 +340,8 @@ def custom_writer(data, actual_options, actual_context): def test_direct_image_writer_without_purpose_binding_retains_legacy_source_path(tmp_path): - from openhcs.processing.materialization import ImageFileOptions, TerminalMaterializationSpec - from openhcs.processing.materialization.core import MaterializationContext, write_image_file + from openhcs.core.materialization import ImageFileOptions, TerminalMaterializationSpec + from openhcs.core.materialization.core import MaterializationContext, write_image_file options = ImageFileOptions(filename_suffix=".tif") plan = ArtifactOutputPlan(name="Role", path="/memory/role", artifact_type=ImageArtifactType, @@ -362,7 +362,7 @@ def test_direct_image_writer_without_purpose_binding_retains_legacy_source_path( @pytest.mark.parametrize("purpose", ("named_terminal", "explicit", "unnamed_terminal")) def test_parser_free_render_requires_parser_only_for_automatic_named_role(tmp_path, purpose): - from openhcs.processing.materialization import ImageFileOptions, MaterializationSpec, TerminalMaterializationSpec, materialization_outputs + from openhcs.core.materialization import ImageFileOptions, MaterializationSpec, TerminalMaterializationSpec, materialization_outputs options = ImageFileOptions(filename_suffix=".tif") spec = MaterializationSpec(options) if purpose == "explicit" else TerminalMaterializationSpec(options) diff --git a/tests/unit/test_bioformats_saved_calibration_440.py b/tests/unit/test_bioformats_saved_calibration_440.py index 605400786e..f9590293ad 100644 --- a/tests/unit/test_bioformats_saved_calibration_440.py +++ b/tests/unit/test_bioformats_saved_calibration_440.py @@ -32,7 +32,7 @@ from openhcs.microscopes.bioformats import BioFormatsHandler from openhcs.core.dataset_sources.plane_stores import PlaneStoreUnavailableError from openhcs.core.dataset_sources.openhcs_format import OpenHCSMetadataHandler -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( MaterializationSpec, ROIOptions, materialize, diff --git a/tests/unit/test_cellprofiler_analyst_export.py b/tests/unit/test_cellprofiler_analyst_export.py index 2750f799db..df518775e0 100644 --- a/tests/unit/test_cellprofiler_analyst_export.py +++ b/tests/unit/test_cellprofiler_analyst_export.py @@ -97,7 +97,7 @@ ExportToDatabaseModule, export_to_database, ) -from openhcs.processing.materialization import FileBundleOptions +from openhcs.core.materialization import FileBundleOptions from openhcs.core.runtime_artifact_values import RuntimeValue from openhcs.domains.microscopy.axes import Microscopy diff --git a/tests/unit/test_cellprofiler_save_images.py b/tests/unit/test_cellprofiler_save_images.py index f2025d5982..d298fe9c8d 100644 --- a/tests/unit/test_cellprofiler_save_images.py +++ b/tests/unit/test_cellprofiler_save_images.py @@ -68,7 +68,7 @@ from openhcs.core.processing_contracts import ( Pure3DContract, ) -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( ExecutionAxisMaterializationRelativePathScope, ImageFileOptions, MaterializationSpec, diff --git a/tests/unit/test_cellprofiler_spreadsheet_export.py b/tests/unit/test_cellprofiler_spreadsheet_export.py index 9879ae68c8..92ff158895 100644 --- a/tests/unit/test_cellprofiler_spreadsheet_export.py +++ b/tests/unit/test_cellprofiler_spreadsheet_export.py @@ -94,7 +94,7 @@ ) from openhcs.processing.backends.cellprofiler.crop import CropModule from openhcs.processing.backends.cellprofiler.tracking import TrackObjectsModule -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( FileBundleOptions, MaterializationSpec, WriteMode, @@ -578,7 +578,7 @@ def test_export_to_spreadsheet_bundle_uses_generic_file_materialization() -> Non artifact_batch=batch, context=context, ) - from openhcs.processing.materialization.core import ColumnarCsvOutput + from openhcs.core.materialization.core import ColumnarCsvOutput assert all(isinstance(output, ColumnarCsvOutput) for output in bundle.values()) outputs = materialization_outputs( MaterializationSpec(FileBundleOptions()), @@ -2163,7 +2163,7 @@ def test_partitioned_export_admits_only_consumed_relationship_subject(selection_ from openhcs.processing.backends.cellprofiler.spreadsheet_export import ( _partitioned_spreadsheet_export, ) - from openhcs.processing.materialization.core import ColumnarCsvOutput + from openhcs.core.materialization.core import ColumnarCsvOutput records = {} for ordinal in range(1, 13): @@ -2256,7 +2256,7 @@ def test_partitioned_relationship_export_preserves_producer_then_axis_order( overlapping_declarations: bool, ) -> None: from dataclasses import replace - from openhcs.processing.materialization.core import ColumnarCsvOutput + from openhcs.core.materialization.core import ColumnarCsvOutput from openhcs.core.runtime_batch_contracts import ( RuntimeArtifactPartitionBatchRequest, ) @@ -2359,7 +2359,7 @@ def test_rendered_partitions_keep_sparse_csv_without_raw_value_transport( ): import pickle import numpy as np - from openhcs.processing.materialization.core import ( + from openhcs.core.materialization.core import ( ColumnarCsvOutput, RenderedColumnarCsvOutput, ) @@ -2429,7 +2429,7 @@ def test_rendered_partitions_keep_sparse_csv_without_raw_value_transport( def test_rendered_matching_headers_compose_without_decoding_value_columns(): import numpy as np - from openhcs.processing.materialization.core import ( + from openhcs.core.materialization.core import ( ColumnarCsvOutput, RenderedColumnarCsvOutput, ) @@ -2464,8 +2464,8 @@ def test_rendered_matching_headers_compose_without_decoding_value_columns(): def test_csv_writer_shape_preserves_contextual_headers_and_empty_dialects(): - from openhcs.processing.materialization.core import ColumnarCsvOutput - from openhcs.processing.materialization.options import CsvOptions + from openhcs.core.materialization.core import ColumnarCsvOutput + from openhcs.core.materialization.options import CsvOptions from openhcs.processing.backends.cellprofiler.spreadsheet_export import ( CellProfilerSpreadsheetCsvOptions, ) @@ -2510,7 +2510,7 @@ def cp_options(subjects): assert generic_empty.table_shape() == ((), 0) # The nominal field owner rejects empty names; the generic mapping writer # still preserves the one-column blank CSV grammar. - from openhcs.processing.materialization.core import _render_csv_rows + from openhcs.core.materialization.core import _render_csv_rows with pytest.raises(ValueError, match="cannot be empty"): FieldSpec("") @@ -2519,7 +2519,7 @@ def cp_options(subjects): def test_rendered_fixed_fields_snapshot_mutable_cells_and_keep_unrendered_identity(): import numpy as np - from openhcs.processing.materialization.core import ( + from openhcs.core.materialization.core import ( ColumnarCsvOutput, RenderedColumnarCsvOutput, ) @@ -2567,7 +2567,7 @@ def test_rendered_fixed_fields_snapshot_mutable_cells_and_keep_unrendered_identi def test_compact_csv_fallback_preserves_physical_dtype_promotion(nan_representation): import pickle import numpy as np - from openhcs.processing.materialization.core import ( + from openhcs.core.materialization.core import ( ColumnarCsvOutput, RenderedColumnarCsvOutput, ) @@ -2657,7 +2657,7 @@ def test_compact_csv_fallback_preserves_physical_dtype_promotion(nan_representat def test_compact_csv_nested_fast_compose_keeps_original_dtype_segments(): import numpy as np - from openhcs.processing.materialization.core import ( + from openhcs.core.materialization.core import ( ColumnarCsvOutput, RenderedColumnarCsvOutput, ) @@ -2717,11 +2717,11 @@ def test_compact_csv_nested_fast_compose_keeps_original_dtype_segments(): def test_compact_csv_preserves_original_semantic_schema_conflicts(): import numpy as np - from openhcs.processing.materialization.core import ( + from openhcs.core.materialization.core import ( ColumnarCsvOutput, RenderedColumnarCsvOutput, ) - from openhcs.processing.materialization.options import CsvOptions + from openhcs.core.materialization.options import CsvOptions outputs = tuple( ColumnarCsvOutput( @@ -2740,7 +2740,7 @@ def test_compact_csv_preserves_original_semantic_schema_conflicts(): def test_compact_csv_requested_absent_field_uses_original_merged_declaration(): import numpy as np - from openhcs.processing.materialization.core import ( + from openhcs.core.materialization.core import ( ColumnarCsvOutput, RenderedColumnarCsvOutput, ) diff --git a/tests/unit/test_collapsed_persisted_occurrence_435.py b/tests/unit/test_collapsed_persisted_occurrence_435.py index 8f4b823f66..aeea7e02a3 100644 --- a/tests/unit/test_collapsed_persisted_occurrence_435.py +++ b/tests/unit/test_collapsed_persisted_occurrence_435.py @@ -39,7 +39,7 @@ PrimaryImageMetadataTarget, ) from openhcs.core.dataset_sources.source_schema import SourceSchemaFilenameParser -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( ImageFileOptions, MaterializationSpec, NameByArtifactName, diff --git a/tests/unit/test_compiled_execution.py b/tests/unit/test_compiled_execution.py index a0d4e6dc84..0cbc06324c 100644 --- a/tests/unit/test_compiled_execution.py +++ b/tests/unit/test_compiled_execution.py @@ -47,8 +47,8 @@ ) from openhcs.core.runtime_stores import RuntimeValueStore from openhcs.core.runtime_tabular_values import FieldSpec -from openhcs.processing.materialization import CsvOptions, MaterializationSpec -from openhcs.processing.materialization.persistence import TerminalMaterializationSpec +from openhcs.core.materialization import CsvOptions, MaterializationSpec +from openhcs.core.materialization.persistence import TerminalMaterializationSpec from openhcs.domains.microscopy.axes import Microscopy diff --git a/tests/unit/test_compiler_auto_materialization_policy.py b/tests/unit/test_compiler_auto_materialization_policy.py index dac756a8a0..464ba2c6c8 100644 --- a/tests/unit/test_compiler_auto_materialization_policy.py +++ b/tests/unit/test_compiler_auto_materialization_policy.py @@ -11,12 +11,12 @@ from openhcs.core.pipeline.materialization_flag_planner import ( MaterializationFlagPlanner, ) -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( ImageFileOptions, MaterializationSpec, ROIOptions, ) -from openhcs.processing.materialization.persistence import TerminalMaterializationSpec +from openhcs.core.materialization.persistence import TerminalMaterializationSpec def test_disabling_automatic_materialization_keeps_only_declared_exports( diff --git a/tests/unit/test_cppipe_execution_validation.py b/tests/unit/test_cppipe_execution_validation.py index 3bf2eb9c17..37e5ad019a 100644 --- a/tests/unit/test_cppipe_execution_validation.py +++ b/tests/unit/test_cppipe_execution_validation.py @@ -42,7 +42,7 @@ RuntimeValueStore, ) from openhcs.core.steps.abstract import StepExecutionObservation -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, FileBundleOptions, ImageFileOptions, diff --git a/tests/unit/test_derived_role_publication_435.py b/tests/unit/test_derived_role_publication_435.py index e7d7a202be..1e136e8780 100644 --- a/tests/unit/test_derived_role_publication_435.py +++ b/tests/unit/test_derived_role_publication_435.py @@ -1,6 +1,6 @@ """Issue435: real saved image occurrences projected through two existing owners.""" -from openhcs.processing.materialization.options import NameByArtifactName, NameBySourceIdentity +from openhcs.core.materialization.options import NameByArtifactName, NameBySourceIdentity from dataclasses import asdict, replace import json @@ -41,7 +41,7 @@ from openhcs.core.runtime_plane_projection import RuntimePlaneAxis, RuntimePlaneAxisValueProjection from python_introspect import to_jsonable -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( ImageFileOptions, MaterializationSpec, ) from openhcs.domains.microscopy.axes import Microscopy diff --git a/tests/unit/test_e1_enum_dispatch_guards.py b/tests/unit/test_e1_enum_dispatch_guards.py index 6dd14222cb..4b9b06f7a9 100644 --- a/tests/unit/test_e1_enum_dispatch_guards.py +++ b/tests/unit/test_e1_enum_dispatch_guards.py @@ -37,8 +37,6 @@ ('CellProfilerMeasurementTargetScope', 'openhcs/interop/cellprofiler/module_artifact_declarations.py'): CELLPROFILER_SPELLING, ('CellProfilerObjectTableMode', 'openhcs/interop/cellprofiler/analyst_export.py'): CELLPROFILER_SPELLING, ('CellProfilerSourceImageProjectionField', 'openhcs/interop/cellprofiler/analyst_export.py'): CELLPROFILER_SPELLING, - ('FileFormat', 'openhcs/core/plate_image_inventory.py'): "E2: restates polystore's format family (get_format_from_extension); key result actions by that family", - ('FileFormat', 'openhcs/pyqt_gui/widgets/image_browser.py'): "E2: restates polystore's format family (get_format_from_extension); key result actions by that family", ('FunctionStepExecutionScope', 'openhcs/core/artifacts/input_projection.py'): 'K5 callable contract (#1202 merged, still an enum); needs an owner', ('FunctionStepExecutionScope', 'openhcs/core/callable_contract.py'): 'K5 callable contract (#1202 merged, still an enum); needs an owner', ('FunctionStepExecutionScope', 'openhcs/core/function_patterns.py'): 'K5 callable contract (#1202 merged, still an enum); needs an owner', diff --git a/tests/unit/test_file_materialization_options.py b/tests/unit/test_file_materialization_options.py index 37fa1d9d51..487e498313 100644 --- a/tests/unit/test_file_materialization_options.py +++ b/tests/unit/test_file_materialization_options.py @@ -13,14 +13,14 @@ from openhcs.core.runtime_image_values import ImagePayloadMetadata from openhcs.core.runtime_plane_projection import RuntimePlaneAxis from openhcs.core.source_image_provenance import SourceImageProvenancePlanes -from openhcs.processing.materialization import options as materialization_options -from openhcs.processing.materialization.constants import WriteMode -from openhcs.processing.materialization.core import ( +from openhcs.core.materialization import options as materialization_options +from openhcs.core.materialization.constants import WriteMode +from openhcs.core.materialization.core import ( MaterializationSpec, materialization_outputs, materialize, ) -from openhcs.processing.materialization.options import ( +from openhcs.core.materialization.options import ( FileOutputOptions, NameByArtifactName, NameBySourceIdentity, diff --git a/tests/unit/test_fragmented_roi_materialization.py b/tests/unit/test_fragmented_roi_materialization.py index b53b6eb336..5a421f91cc 100644 --- a/tests/unit/test_fragmented_roi_materialization.py +++ b/tests/unit/test_fragmented_roi_materialization.py @@ -17,7 +17,7 @@ ObjectLabelVariantData, ) from openhcs.core.source_spatial_domain import SourceSpatialDomain -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( ImageFileOptions, MaterializationSpec, ROIOptions, diff --git a/tests/unit/test_function_artifact_materialization.py b/tests/unit/test_function_artifact_materialization.py index 72e29f12e5..e323e6667e 100644 --- a/tests/unit/test_function_artifact_materialization.py +++ b/tests/unit/test_function_artifact_materialization.py @@ -8,7 +8,7 @@ import numpy as np import pytest -from openhcs.processing.materialization.persistence import TerminalMaterializationSpec +from openhcs.core.materialization.persistence import TerminalMaterializationSpec from polystore.base import DataSink, ensure_storage_registry, storage_registry from polystore.filemanager import FileManager from polystore.napari_stream import NapariStreamingBackend @@ -148,7 +148,7 @@ from openhcs.processing.backends.pos_gen.tile_position_artifacts import ( TILE_POSITIONS_OUTPUT, ) -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, FileBundleOptions, JsonOptions, @@ -160,11 +160,11 @@ roi_zip, tiff_stack, ) -from openhcs.processing.materialization.core import ( +from openhcs.core.materialization.core import ( MaterializationSpec, materialization_outputs, ) -from openhcs.processing.materialization.options import ( +from openhcs.core.materialization.options import ( ImageFileOptions, NameByArtifactName, NameBySourceIdentity, diff --git a/tests/unit/test_function_artifact_materialization_io_hotpath.py b/tests/unit/test_function_artifact_materialization_io_hotpath.py index c012c6b36e..ced1d417e7 100644 --- a/tests/unit/test_function_artifact_materialization_io_hotpath.py +++ b/tests/unit/test_function_artifact_materialization_io_hotpath.py @@ -20,12 +20,12 @@ from openhcs.core.artifacts.kinds import ImageArtifactType from openhcs.core.runtime_artifact_values import RuntimeValue from openhcs.core.runtime_stores import RuntimeValueStore -from openhcs.processing.materialization.core import ( +from openhcs.core.materialization.core import ( MaterializationSpec, Output, RawBackendKwargs, ) -from openhcs.processing.materialization.options import ImageFileOptions +from openhcs.core.materialization.options import ImageFileOptions class _PersistentBackend(DataSink): diff --git a/tests/unit/test_function_artifact_outputs.py b/tests/unit/test_function_artifact_outputs.py index a194b21732..2e3b441312 100644 --- a/tests/unit/test_function_artifact_outputs.py +++ b/tests/unit/test_function_artifact_outputs.py @@ -114,7 +114,7 @@ Pure3DContract, VolumetricToSliceContract, ) -from openhcs.processing.materialization import CsvOptions, MaterializationSpec +from openhcs.core.materialization import CsvOptions, MaterializationSpec from openhcs.processing.backends.analysis.multi_template_matching import ( TemplateMatchResult, ) diff --git a/tests/unit/test_function_outputs.py b/tests/unit/test_function_outputs.py index d1cadc1851..20f35e2b54 100644 --- a/tests/unit/test_function_outputs.py +++ b/tests/unit/test_function_outputs.py @@ -99,8 +99,8 @@ from openhcs.core.dataset_sources.interfaces import MetadataHandler from openhcs.core.dataset_sources.openhcs_format import OpenHCSMetadataHandler from openhcs.core.dataset_sources.source_schema import SourceSchemaFilenameParser -from openhcs.processing.materialization.core import Output -from openhcs.processing.materialization import ( +from openhcs.core.materialization.core import Output +from openhcs.core.materialization import ( ImageFileOptions, MaterializationSpec, NameBySourceIdentity, diff --git a/tests/unit/test_function_patterns.py b/tests/unit/test_function_patterns.py index 09b99e894f..a9b8be83df 100644 --- a/tests/unit/test_function_patterns.py +++ b/tests/unit/test_function_patterns.py @@ -54,7 +54,7 @@ from openhcs.core.runtime_object_labels import ObjectLabelValue from openhcs.core.runtime_relationships import ObjectRelationshipDeclaration from openhcs.core.source_bindings import CompiledSourceBindingPlan -from openhcs.processing.materialization import csv_only +from openhcs.core.materialization import csv_only from openhcs.domains.microscopy.axes import Microscopy diff --git a/tests/unit/test_function_step_execution_scope.py b/tests/unit/test_function_step_execution_scope.py index fecfdf01fd..ce5ea535ed 100644 --- a/tests/unit/test_function_step_execution_scope.py +++ b/tests/unit/test_function_step_execution_scope.py @@ -79,7 +79,7 @@ from openhcs.core.step_dependencies import NoMainFlowInput from openhcs.core.steps.function_step import FunctionStep from openhcs.core.dataset_sources.source_schema import SourceSchemaFilenameParser -from openhcs.processing.materialization import FileBundleOptions, MaterializationSpec +from openhcs.core.materialization import FileBundleOptions, MaterializationSpec from openhcs.core.axes import Axis from openhcs.domains.microscopy.axes import Microscopy from openhcs.core.progress.types import ( diff --git a/tests/unit/test_graph_roi_source_roundtrip_134.py b/tests/unit/test_graph_roi_source_roundtrip_134.py index d95d62d719..d648dcd2c9 100644 --- a/tests/unit/test_graph_roi_source_roundtrip_134.py +++ b/tests/unit/test_graph_roi_source_roundtrip_134.py @@ -33,7 +33,7 @@ from openhcs.core.source_metadata import SourceVoxelSpacing from openhcs.core.viewer_streaming_service import RoiStreamingRequest, StreamingService -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( MaterializationSpec, SpatialGraphROIOptions, materialization_outputs, diff --git a/tests/unit/test_image_file_serialization.py b/tests/unit/test_image_file_serialization.py index 144a060950..b1128ef42a 100644 --- a/tests/unit/test_image_file_serialization.py +++ b/tests/unit/test_image_file_serialization.py @@ -721,7 +721,7 @@ def test_runtime_plane_tiff_write_retains_scalar_frames_and_declared_channels( tmp_path, plane_count, dtype, channel_axis ): from polystore.config import TiffCompression, TiffConfig, TiffPhotometric - from openhcs.processing.materialization.core import Output, RawBackendKwargs + from openhcs.core.materialization.core import Output, RawBackendKwargs shape = ( (plane_count, 5, 7) if channel_axis is None diff --git a/tests/unit/test_k8_materialization_guards.py b/tests/unit/test_k8_materialization_guards.py new file mode 100644 index 0000000000..bfb12a3b59 --- /dev/null +++ b/tests/unit/test_k8_materialization_guards.py @@ -0,0 +1,109 @@ +"""K8 guards: materialization is kernel code and values reach JSON one way. + +- The materialization package lives in ``openhcs/core``; nothing names the old + ``processing`` location. +- ``python_introspect.to_jsonable`` is the one value-to-JSON encoder: no other + ``singledispatch`` function covers containers, numpy arrays and numpy + scalars together (the signature of a value-to-JSON converter), and + the two deleted converters stay deleted. +- Result file kinds are the ``ResultFileFormat`` family, not an enum. +""" + +from __future__ import annotations + +import ast +from functools import lru_cache +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SOURCE_ROOTS = ("openhcs", "tests", "benchmark", "scripts") +OLD_PACKAGE = ".".join(("openhcs", "processing", "materialization")) +DELETED_CONVERTERS = {"json_payload_value", "_json_value", "JsonPayloadAuthority"} +CONTAINER_TYPES = {"dict", "list", "tuple", "Mapping"} +NUMPY_VALUE_TYPES = {"ndarray", "generic"} + + +def _sources(*roots: str): + for root in roots: + for path in sorted((ROOT / root).rglob("*.py")): + if "results" in path.relative_to(ROOT).parts: + continue + yield path + + +@lru_cache(maxsize=None) +def _tree(path: Path) -> ast.Module: + return ast.parse(path.read_text(encoding="utf-8")) + + +def _name(node: ast.expr) -> str | None: + if isinstance(node, ast.Attribute): + return node.attr + if isinstance(node, ast.Name): + return node.id + return None + + +def test_materialization_package_is_in_core(): + assert not (ROOT / "openhcs" / "processing" / "materialization").exists() + assert (ROOT / "openhcs" / "core" / "materialization" / "__init__.py").is_file() + offenders = [ + str(path.relative_to(ROOT)) + for path in _sources(*SOURCE_ROOTS) + if OLD_PACKAGE in path.read_text(encoding="utf-8") + ] + assert not offenders, offenders + + +def _registered_types_by_dispatcher(tree: ast.Module) -> dict[str, set[str]]: + registered: dict[str, set[str]] = {} + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + function = node.func + if not (isinstance(function, ast.Attribute) and function.attr == "register"): + continue + dispatcher = _name(function.value) + if dispatcher is None or not node.args: + continue + type_name = _name(node.args[0]) + if type_name is not None: + registered.setdefault(dispatcher, set()).add(type_name) + return registered + + +def test_to_jsonable_is_the_only_value_to_json_dispatcher(): + offenders = [] + for path in _sources("openhcs", "benchmark", "scripts"): + tree = _tree(path) + for node in ast.walk(tree): + if ( + isinstance(node, (ast.FunctionDef, ast.ClassDef)) + and node.name in DELETED_CONVERTERS + ): + offenders.append(f"{path.relative_to(ROOT)}: {node.name}") + for dispatcher, types in _registered_types_by_dispatcher(tree).items(): + if dispatcher == "to_jsonable": + continue + if types & CONTAINER_TYPES and NUMPY_VALUE_TYPES <= types: + offenders.append(f"{path.relative_to(ROOT)}: {dispatcher}") + assert not offenders, offenders + + +def test_result_file_kinds_are_a_family(): + from openhcs.core.result_file_formats import ResultFileFormat + + enum_definitions = [ + str(path.relative_to(ROOT)) + for path in _sources("openhcs") + for node in ast.walk(_tree(path)) + if isinstance(node, ast.ClassDef) and node.name == "FileFormat" + ] + assert not enum_definitions, enum_definitions + members = dict(ResultFileFormat.__registry__) + assert set(members) == {"TIFF", "NUMPY", "TEXT", "JSON", "CSV", "ROI"} + for name, member in members.items(): + assert member.name == name + for suffix in member.suffixes: + assert ResultFileFormat.for_suffix_of(Path(f"result{suffix}")) is member + assert [name for name, member in members.items() if member.streams_rois] == ["ROI"] diff --git a/tests/unit/test_materialization_api.py b/tests/unit/test_materialization_api.py index cf0bfe960b..ce4c6cca13 100644 --- a/tests/unit/test_materialization_api.py +++ b/tests/unit/test_materialization_api.py @@ -1,4 +1,4 @@ -from openhcs.processing.materialization import MaterializationSpec, csv_materializer +from openhcs.core.materialization import MaterializationSpec, csv_materializer def test_csv_materializer_uses_analysis_type_suffix() -> None: diff --git a/tests/unit/test_materialization_batch_outcomes.py b/tests/unit/test_materialization_batch_outcomes.py index 34e1ebf511..2d6b9b356c 100644 --- a/tests/unit/test_materialization_batch_outcomes.py +++ b/tests/unit/test_materialization_batch_outcomes.py @@ -34,14 +34,14 @@ from openhcs.core.runtime_exports import RuntimeExportObservation from openhcs.core.steps.abstract import StepExecutionObservation from openhcs.core.steps.function_step import FunctionStep -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, JsonOptions, MaterializationSpec, SavedMaterializationOutputs, prepare_materialization, ) -from openhcs.processing.materialization.core import _WRITERS_BY_OPTIONS +from openhcs.core.materialization.core import _WRITERS_BY_OPTIONS from tests.materialization_plans import output_plan as materialization_output_plan diff --git a/tests/unit/test_materialization_core.py b/tests/unit/test_materialization_core.py index 0b7d338431..0ddc8075b0 100644 --- a/tests/unit/test_materialization_core.py +++ b/tests/unit/test_materialization_core.py @@ -96,7 +96,7 @@ ) from openhcs.core.source_spatial_domain import SourceSpatialDomain from openhcs.core.dataset_sources.source_schema import SourceSchemaFilenameParser -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, ImageFileOptions, JsonOptions, @@ -118,7 +118,7 @@ from openhcs.processing.backends.analysis.consolidate_analysis_results import ( analysis_file_path_is_included, ) -from openhcs.processing.materialization.core import ( +from openhcs.core.materialization.core import ( MaterializationInputItem, Output, ROIMaterializationArchiveIdentity, @@ -2572,7 +2572,7 @@ def test_declared_volume_save_preserves_pixels_and_exact_plane_selection( from openhcs.core.virtual_workspace_metadata import ( VirtualWorkspaceSourceProjectionEntries, ) - from openhcs.processing.materialization import prepare_materialization + from openhcs.core.materialization import prepare_materialization import tifffile pixels = np.arange(3 * 4 * 5, dtype=np.uint16).reshape(3, 4, 5) diff --git a/tests/unit/test_materialization_flag_planner.py b/tests/unit/test_materialization_flag_planner.py index 217c97e2fc..31daf48306 100644 --- a/tests/unit/test_materialization_flag_planner.py +++ b/tests/unit/test_materialization_flag_planner.py @@ -17,7 +17,7 @@ from openhcs.core.pipeline.materialization_flag_planner import ( MaterializationFlagPlanner, ) -from openhcs.processing.materialization import MaterializationSpec, ROIOptions +from openhcs.core.materialization import MaterializationSpec, ROIOptions def _pipeline_config( diff --git a/tests/unit/test_materialization_source_extension.py b/tests/unit/test_materialization_source_extension.py index 359b58a6be..7d37b93b14 100644 --- a/tests/unit/test_materialization_source_extension.py +++ b/tests/unit/test_materialization_source_extension.py @@ -27,8 +27,8 @@ FunctionOutputIdentity, ) from openhcs.core.dataset_sources.source_schema import SourceSchemaFilenameParser -from openhcs.processing.materialization import ImageFileOptions, MaterializationSpec -from openhcs.processing.materialization.core import ( +from openhcs.core.materialization import ImageFileOptions, MaterializationSpec +from openhcs.core.materialization.core import ( ParserBackedSourceStemAuthority, materialization_outputs, ) diff --git a/tests/unit/test_measurement_dialect.py b/tests/unit/test_measurement_dialect.py index b9f57eb61a..91eee5dac9 100644 --- a/tests/unit/test_measurement_dialect.py +++ b/tests/unit/test_measurement_dialect.py @@ -26,7 +26,7 @@ from openhcs.interop.cellprofiler.measurement_dialect import ( CELLPROFILER_MEASUREMENT_DIALECT, ) -from openhcs.processing.materialization.core import _render_csv_rows +from openhcs.core.materialization.core import _render_csv_rows class Survey(AxisFamily): diff --git a/tests/unit/test_multi_template_matching.py b/tests/unit/test_multi_template_matching.py index 3086f9bc68..c9dd4f765a 100644 --- a/tests/unit/test_multi_template_matching.py +++ b/tests/unit/test_multi_template_matching.py @@ -23,8 +23,8 @@ multi_template_crop_reference_channel, multi_template_crop_subset, ) -from openhcs.processing.materialization import materialization_outputs -from openhcs.processing.materialization.options import NameByArtifactName +from openhcs.core.materialization import materialization_outputs +from openhcs.core.materialization.options import NameByArtifactName from tests.materialization_plans import output_plan as materialization_output_plan diff --git a/tests/unit/test_named_checkpoint_publication.py b/tests/unit/test_named_checkpoint_publication.py index 11a07a52f1..e18897c8c4 100644 --- a/tests/unit/test_named_checkpoint_publication.py +++ b/tests/unit/test_named_checkpoint_publication.py @@ -4,7 +4,7 @@ compiler, runtime materialization, metadata reconciliation and reopen are used. """ -from openhcs.processing.materialization.options import NameByArtifactName +from openhcs.core.materialization.options import NameByArtifactName from dataclasses import dataclass, replace import json @@ -40,7 +40,7 @@ from openhcs.core.processing_contracts import ( Pure3DContract, ) -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( ImageFileOptions, MaterializationSpec, ) from openhcs.core.axes import TileAxis diff --git a/tests/unit/test_neurite_outgrowth.py b/tests/unit/test_neurite_outgrowth.py index 2abd661386..5ea6dfe3cc 100644 --- a/tests/unit/test_neurite_outgrowth.py +++ b/tests/unit/test_neurite_outgrowth.py @@ -62,7 +62,7 @@ neurite_outgrowth_metaxpress, neurite_outgrowth_metaxpress_pixels, ) -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( MaterializationSpec, ImageFileOptions, ROIOptions, diff --git a/tests/unit/test_persisted_roi_only_publication.py b/tests/unit/test_persisted_roi_only_publication.py index a14cbb7ceb..52a87c6f44 100644 --- a/tests/unit/test_persisted_roi_only_publication.py +++ b/tests/unit/test_persisted_roi_only_publication.py @@ -38,12 +38,12 @@ RuntimeArtifactMetadataTarget, ) from openhcs.core.virtual_workspace_metadata import AtomicMetadataWriter, METADATA_CONFIG, MetadataWriteError -from openhcs.processing.materialization.core import ( +from openhcs.core.materialization.core import ( MaterializationSpec, _WRITERS_BY_OPTIONS, prepare_materialization, ) -from openhcs.processing.materialization.options import FileBundleOptions +from openhcs.core.materialization.options import FileBundleOptions from openhcs.domains.microscopy.axes import Microscopy from tests.materialization_plans import output_plan as materialization_output_plan diff --git a/tests/unit/test_plate_image_inventory.py b/tests/unit/test_plate_image_inventory.py index 55cc08a6f4..fdaccb2613 100644 --- a/tests/unit/test_plate_image_inventory.py +++ b/tests/unit/test_plate_image_inventory.py @@ -9,7 +9,13 @@ import pytest from openhcs.constants import Backend -from openhcs.constants.constants import FileFormat +from openhcs.core.result_file_formats import ( + CsvResultFile, + NumpyResultFile, + RoiArchiveResultFile, + TextResultFile, + TiffResultFile, +) from openhcs.core.config import GlobalPipelineConfig from openhcs.core.plate_file_inventory import PlateFileInventoryQuery, PlateFileKind from openhcs.core.plate_image_inventory import ( @@ -134,7 +140,7 @@ def test_plate_result_inventory_exposes_swc_as_text_result( query.records[0].relative_path == "images_results/A01_neurite_morphology_step1.swc" ) - assert query.records[0].file_format is FileFormat.TEXT + assert query.records[0].file_format is TextResultFile preview = PlateResultFilePreviewReader.preview(results.records[0]) assert preview is not None assert preview.text_lines == ( @@ -169,7 +175,7 @@ def test_native_image_secondaries_are_admitted_as_streamable_results(tmp_path, s record = inventory.require_file_record(image_path.name) assert record.kind is PlateFileKind.RESULT assert record.file_format is ( - FileFormat.NUMPY if suffix == ".npy" else FileFormat.TIFF + NumpyResultFile if suffix == ".npy" else TiffResultFile ) assert record.streamable_image_path == str(image_path) assert record.require_image_source_ref() == SourcePixelRef( @@ -195,8 +201,8 @@ def test_nonimage_results_do_not_claim_pixel_source_references(tmp_path): parser=None, ) assert {record.file_format for record in results.records} == { - FileFormat.CSV, - FileFormat.ROI, + CsvResultFile, + RoiArchiveResultFile, } assert all(record.source_ref is None for record in results.records) @@ -240,7 +246,7 @@ def test_stream_record_uses_declared_address_for_dual_image_result_projection(): PlateResultFileRecord( relative_path=relative_path, full_path=full_path, - file_format=FileFormat.TIFF, + file_format=TiffResultFile, source_ref=SourcePixelRef( backend=Backend.DISK.value, backend_address=full_path, diff --git a/tests/unit/test_primary_object_diagnostics.py b/tests/unit/test_primary_object_diagnostics.py index 8343dcdcf2..c7866c5bfd 100644 --- a/tests/unit/test_primary_object_diagnostics.py +++ b/tests/unit/test_primary_object_diagnostics.py @@ -27,7 +27,7 @@ SelectedDiagnosticPlaneImageOutput, UnexecutedDeclumpingEvidence, ) -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( ImageFileOptions, NameByArtifactName, ) diff --git a/tests/unit/test_pycodify_formatters.py b/tests/unit/test_pycodify_formatters.py index 09538697ee..62f1a59974 100644 --- a/tests/unit/test_pycodify_formatters.py +++ b/tests/unit/test_pycodify_formatters.py @@ -30,7 +30,7 @@ ) from openhcs.processing.backends.cellprofiler.shape import measure_object_size_shape from openhcs.processing.backends.lib_registry.registry_service import RegistryService -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( ImageFileOptions, MaterializationSpec, StreamingOnlyMaterializationSpec, diff --git a/tests/unit/test_roi_source_metadata.py b/tests/unit/test_roi_source_metadata.py index 37dfea731d..53e6c106fd 100644 --- a/tests/unit/test_roi_source_metadata.py +++ b/tests/unit/test_roi_source_metadata.py @@ -17,7 +17,7 @@ ) from openhcs.core.source_metadata import SourceVoxelSpacing from openhcs.core.source_spatial_domain import SourceSpatialDomain -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( MaterializationSpec, ROIOptions, materialize, diff --git a/tests/unit/test_runtime_execution_validation.py b/tests/unit/test_runtime_execution_validation.py index 42d3048c6f..6b3de62177 100644 --- a/tests/unit/test_runtime_execution_validation.py +++ b/tests/unit/test_runtime_execution_validation.py @@ -63,7 +63,7 @@ from openhcs.core.steps.abstract import StepExecutionObservation from openhcs.core.source_spatial_domain import SourceSpatialDomain from openhcs.core.dataset_sources.source_schema import SourceSchemaFilenameParser -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( CsvOptions, ImageFileOptions, MaterializationSpec, diff --git a/tests/unit/test_runtime_exports.py b/tests/unit/test_runtime_exports.py index 03c84d23c0..3ce56bb3a3 100644 --- a/tests/unit/test_runtime_exports.py +++ b/tests/unit/test_runtime_exports.py @@ -35,8 +35,8 @@ ) from openhcs.core.steps.abstract import StepExecutionObservation from openhcs.core.runtime_tabular_values import ColumnarRows -from openhcs.processing.materialization import CsvOptions, MaterializationSpec -from openhcs.processing.materialization.core import ColumnarCsvOutput +from openhcs.core.materialization import CsvOptions, MaterializationSpec +from openhcs.core.materialization.core import ColumnarCsvOutput from python_introspect import ( dataclass_from_mapping, to_jsonable, diff --git a/tests/unit/test_selected_plane_stream_materialization.py b/tests/unit/test_selected_plane_stream_materialization.py index e84052612c..4df0ccb6d6 100644 --- a/tests/unit/test_selected_plane_stream_materialization.py +++ b/tests/unit/test_selected_plane_stream_materialization.py @@ -54,7 +54,7 @@ from openhcs.processing.backends.cellprofiler.primary_object_diagnostics import ( SelectedDiagnosticPlaneImageOutput, ) -from openhcs.processing.materialization import materialize +from openhcs.core.materialization import materialize from test_function_artifact_materialization import ( _context, diff --git a/tests/unit/test_skan_axon_artifact_contract.py b/tests/unit/test_skan_axon_artifact_contract.py index 276ae84753..7ef99566db 100644 --- a/tests/unit/test_skan_axon_artifact_contract.py +++ b/tests/unit/test_skan_axon_artifact_contract.py @@ -21,7 +21,7 @@ _compute_summary_metrics, skan_axon_skeletonize_and_analyze, ) -from openhcs.processing.materialization import TiffStackOptions +from openhcs.core.materialization import TiffStackOptions def test_skan_axon_declares_split_typed_outputs_and_object_subject() -> None: diff --git a/tests/unit/test_skeletonize_and_save.py b/tests/unit/test_skeletonize_and_save.py index 563f0b5805..6082e14657 100644 --- a/tests/unit/test_skeletonize_and_save.py +++ b/tests/unit/test_skeletonize_and_save.py @@ -39,7 +39,7 @@ SkeletonizationResult, skeletonize_and_save, ) -from openhcs.processing.materialization import CsvOptions, ROIOptions, materialize +from openhcs.core.materialization import CsvOptions, ROIOptions, materialize from tests.materialization_plans import output_plan as materialization_output_plan diff --git a/tests/unit/test_spatial_graph_materialization.py b/tests/unit/test_spatial_graph_materialization.py index be00e1c9b6..1209f39d8e 100644 --- a/tests/unit/test_spatial_graph_materialization.py +++ b/tests/unit/test_spatial_graph_materialization.py @@ -32,7 +32,7 @@ from openhcs.core.source_image_provenance import SourceImageProvenancePlanes from openhcs.core.source_metadata import SourceVoxelSpacing, SourceVoxelSpacingUnit -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( BackendSaver, MaterializationSpec, Output, diff --git a/tests/unit/test_streaming_service.py b/tests/unit/test_streaming_service.py index 34c69c30f5..d692cf46da 100644 --- a/tests/unit/test_streaming_service.py +++ b/tests/unit/test_streaming_service.py @@ -87,7 +87,7 @@ ViewerObjectDisplayConfigInput, ViewerRouteComponentValueTracker, ) -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( MaterializationSpec, PointROIOptions, materialize, diff --git a/tests/unit/test_template_crop_pipeline_execution.py b/tests/unit/test_template_crop_pipeline_execution.py index 9391a57b9b..642f85bf60 100644 --- a/tests/unit/test_template_crop_pipeline_execution.py +++ b/tests/unit/test_template_crop_pipeline_execution.py @@ -123,9 +123,17 @@ def test_template_crop_persists_each_source_channel( set_progress_queue(None) assert result["A01"].is_success(), result["A01"].error_message images = sorted((tmp_path / "output" / "plate_openhcs" / "images").glob("A01*.tif")) + # The main-flow checkpoint keeps the source filename (0dd77da46); the named + # ``cropped_stack`` artifact is materialized separately under results. + named = sorted( + (tmp_path / "output" / "plate_openhcs" / "results").glob( + "A01_*_cropped_stack_step*_slice_*.tif" + ) + ) + assert len(named) == len(channels), named assert len(images) == len(channels), images assert {path.name for path in images} == { - f"A01_s001_w{channel}_z001_t001_cropped_stack.tif" for channel in channels + f"A01_s001_w{channel}_z001_t001.tif" for channel in channels } for path in images: channel = int(path.name.split("_w")[1].split("_")[0]) diff --git a/tests/unit/test_tiff_output_compression.py b/tests/unit/test_tiff_output_compression.py index 0336182765..ba06c0a0dc 100644 --- a/tests/unit/test_tiff_output_compression.py +++ b/tests/unit/test_tiff_output_compression.py @@ -13,7 +13,7 @@ PersistentArtifactMaterializationTargetPlan, ) from openhcs.core.steps.function_io import save_materialized_data -from openhcs.processing.materialization import ( +from openhcs.core.materialization import ( ImageFileOptions, MaterializationSpec, NameByArtifactName,