diff --git a/openhcs/interop/cellprofiler/analyst_export.py b/openhcs/interop/cellprofiler/analyst_export.py index 6d6a7d846d..54eb07b1fe 100644 --- a/openhcs/interop/cellprofiler/analyst_export.py +++ b/openhcs/interop/cellprofiler/analyst_export.py @@ -69,7 +69,7 @@ SourceImageProvenance, ) from openhcs.core.source_bindings import CompiledSourceBindingPlan -from openhcs.core.source_metadata import SourceMetadataFields, SourceMetadataScalar +from openhcs.core.source_metadata import SourceMetadataScalar from openhcs.core.source_workspace_projection import ( VirtualWorkspacePathLookup, VirtualWorkspaceSourceProjectionAuthority, @@ -87,7 +87,6 @@ CELLPROFILER_MEASUREMENT_DIALECT, ) from .module_declarations import CellProfilerModule -from .source_metadata import CellProfilerSourceMetadataField class CellProfilerObjectTableMode(str, Enum): @@ -920,11 +919,8 @@ def collect_image_provenance( values: dict[str, Any] = {} source_path = self._resolved_source_path(plane_provenance.source_path) if source_path is not None: - values[self.dialect.source_image_path_field(source_image_name).name] = ( - str(source_path.parent) - ) - values[self.dialect.source_image_file_field(source_image_name).name] = ( - source_path.name + values.update( + self.dialect.source_image_file_values(source_path, source_image_name) ) if source_path.is_file(): values.update( @@ -948,14 +944,9 @@ def collect_image_provenance( field_name: value for field_name, value in self.dialect.source_metadata_defaults().items() } - if source_path is not None: - metadata_items[ - CellProfilerSourceMetadataField.FILE_LOCATION.field_name - ] = source_path.as_uri() - if component_metadata is not None: - metadata_items.update( - SourceMetadataFields.original_items(component_metadata) - ) + metadata_items.update( + self.dialect.source_metadata_values(component_metadata, source_path) + ) image_number = self.image_set_numbering.for_source_slice( scope=scope, provenance=provenance, diff --git a/openhcs/interop/cellprofiler/database_column_dialect.py b/openhcs/interop/cellprofiler/database_column_dialect.py index 6db8aee37d..a9b10fb22d 100644 --- a/openhcs/interop/cellprofiler/database_column_dialect.py +++ b/openhcs/interop/cellprofiler/database_column_dialect.py @@ -6,6 +6,7 @@ from dataclasses import dataclass, field, replace from enum import Enum from hashlib import md5 +from pathlib import Path import re from types import MappingProxyType from typing import Any, ClassVar @@ -15,6 +16,7 @@ ObjectCoreMeasurementFeature, ) from openhcs.core.runtime_tabular_values import FieldSpec +from openhcs.core.source_metadata import SourceMetadataFields, SourceMetadataMapping from openhcs.interop.cellprofiler.source_metadata import ( CellProfilerSourceMetadataField, ) @@ -417,6 +419,54 @@ def source_metadata_defaults() -> dict[str, object]: return CellProfilerSourceMetadataField.static_defaults() + @staticmethod + def source_metadata_values( + metadata: SourceMetadataMapping | None, + source_path: Path | None, + ) -> dict[str, object]: + """Project original extraction fields and the declared source path.""" + values: dict[str, object] = {} + if metadata is not None: + values.update(SourceMetadataFields.original_items(metadata)) + if source_path is not None: + values.setdefault( + CellProfilerSourceMetadataField.FILE_LOCATION.field_name, + source_path.as_uri() if source_path.is_absolute() else str(source_path), + ) + return values + + @staticmethod + def source_acquisition_values( + metadata: SourceMetadataMapping | None, + ) -> dict[str, object]: + """Project declared acquisition components, never absent axes. + + A represented multi-value domain is not a single acquisition coordinate. + This is distinct from the original extraction-field namespace. + """ + if metadata is None: + return {} + return { + component.value: ( + SourceMetadataFields.canonical_component_value(component, domain[0]) + if len(domain) == 1 else None + ) + for component, domain in SourceMetadataFields.component_domains(metadata).items() + } + + def source_image_file_values( + self, + source_path: Path, + source_image_name: str, + ) -> dict[str, str]: + """Project a named source identity without reading its image pixels.""" + return { + self.source_image_path_field(source_image_name).name: str( + source_path.parent + ), + self.source_image_file_field(source_image_name).name: source_path.name, + } + def image_table_name(self) -> str: return f"{self.table_prefix}Per_Image" diff --git a/openhcs/interop/cellprofiler/image_set_numbering.py b/openhcs/interop/cellprofiler/image_set_numbering.py index f3c9a8f3c1..7865abe145 100644 --- a/openhcs/interop/cellprofiler/image_set_numbering.py +++ b/openhcs/interop/cellprofiler/image_set_numbering.py @@ -106,7 +106,7 @@ def project_measurement_rows( image_numbers_by_slice = self.for_source_slices( scope=scope, provenance=table.source_provenance, - slice_indices=projection.present_axis_values(slice_axis.value), + slice_indices=self.source_slices_for_measurement_table(table), owner=table.name, ) axisless_image_number = None @@ -116,12 +116,7 @@ def project_measurement_rows( ) or (0,) source_image_numbers = tuple( dict.fromkeys( - self.for_source_slices( - scope=scope, - provenance=table.source_provenance, - slice_indices=source_plane_indices, - owner=table.name, - ).values() + image_numbers_by_slice[index] for index in source_plane_indices ) ) if ( @@ -145,6 +140,26 @@ def project_measurement_rows( axisless_value=axisless_image_number, ) + @staticmethod + def source_slices_for_measurement_table( + table: MeasurementTable, + ) -> tuple[int, ...]: + """Return exactly the slices represented by this producer's row scope. + + Axisless rows consume their declared source stack, not a guessed plate + grid. Preserve explicit row order before additional source planes. + """ + projection = MeasurementRowsAxisProjection.from_rows(table.rows) + indices = projection.present_axis_values( + MeasurementRowAxisField.SLICE_INDEX.value + ) + if not projection.has_axisless_rows(MeasurementRowAxisField.SLICE_INDEX): + return indices + source_indices = ( + tuple(range(table.source_provenance.source_plane_count)) or (0,) + ) + return tuple(dict.fromkeys((*indices, *source_indices))) + def _source_identity( self, provenance: SourceImageProvenance, diff --git a/openhcs/processing/backends/cellprofiler/spreadsheet_export.py b/openhcs/processing/backends/cellprofiler/spreadsheet_export.py index f1463de65b..80760f0f40 100644 --- a/openhcs/processing/backends/cellprofiler/spreadsheet_export.py +++ b/openhcs/processing/backends/cellprofiler/spreadsheet_export.py @@ -9,7 +9,7 @@ from dataclasses import dataclass from enum import Enum from numbers import Real -from pathlib import PurePosixPath +from pathlib import Path, PurePosixPath from typing import cast, TYPE_CHECKING, ClassVar, TypeVar from openhcs.core._tabular_native import render_csv as _render_native_csv @@ -41,6 +41,7 @@ from openhcs.core.runtime_measurements import ( MeasurementRowAxisField, MeasurementScope, + MeasurementSubject, measurement_axis_integer_value, ) from openhcs.core.runtime_identifier import ( @@ -56,8 +57,8 @@ from openhcs.core.source_image_provenance import ( source_component_metadata_consensus, ) -from openhcs.core.source_metadata import ( - SourceMetadataFields, +from openhcs.interop.cellprofiler.database_column_dialect import ( + CellProfilerDatabaseColumnDialect, ) from openhcs.interop.cellprofiler.image_set_numbering import ( CellProfilerImageSetNumbering, @@ -379,7 +380,12 @@ def render_spreadsheet_bundle( image_numbers = CellProfilerImageSetNumbering( artifact_batch.source_image_set_identity_policy ) - tables, object_subjects = _measurement_tables(artifact_batch, image_numbers) + tables, object_subjects = _measurement_tables( + artifact_batch, + image_numbers, + add_image_metadata=add_image_metadata, + add_image_file_names=add_image_file_names, + ) relationship_rows = _relationship_rows(artifact_batch, image_numbers) if relationship_rows: tables["Object relationships"] = relationship_rows @@ -447,6 +453,9 @@ def render_spreadsheet_bundle( def _measurement_tables( artifact_batch: RuntimeArtifactBatch, image_numbers: CellProfilerImageSetNumbering, + *, + add_image_metadata: bool, + add_image_file_names: bool, ) -> tuple[ OrderedDict[str, tuple[Mapping[str, object], ...]], tuple[str, ...], @@ -456,7 +465,7 @@ def _measurement_tables( ) source_metadata_by_image_number: OrderedDict[ int, - list[Mapping[str, object]], + list[tuple[Mapping[str, object], Mapping[str, object]]], ] = OrderedDict() all_tables: list[MeasurementTable] = [] for spec in artifact_batch.specs_of_type(MeasurementsArtifactType): @@ -469,20 +478,16 @@ def _measurement_tables( tables = tuple(cast(MeasurementTable, record.data) for record in records) all_tables.extend(tables) slice_axis = MeasurementRowAxisField.SLICE_INDEX - row_domains = tuple( - MeasurementRowsAxisProjection.from_rows(table.rows) for table in tables - ) MeasurementTable.shared_row_axis_domain(spec.name, tables, slice_axis) - for record, table, row_domain in zip( + for record, table in zip( records, tables, - row_domains, strict=True, ): image_numbers_by_slice = image_numbers.for_source_slices( scope=record.key.scope, provenance=table.source_provenance, - slice_indices=row_domain.present_axis_values(slice_axis.value), + slice_indices=image_numbers.source_slices_for_measurement_table(table), owner=table.name, ) accumulator.add_declared_rows( @@ -499,15 +504,37 @@ def _measurement_tables( CELLPROFILER_MEASUREMENT_DIALECT ), ) - for image_number, metadata in _source_metadata_measurement_rows( + for ( + image_number, + original_metadata, + acquisition, + file_values, + ) in _source_metadata_measurement_rows( table, - row_domain, image_numbers_by_slice, + add_image_metadata=add_image_metadata, + add_image_file_names=add_image_file_names, ): source_metadata_by_image_number.setdefault( image_number, [], - ).append(metadata) + ).append((original_metadata, acquisition)) + if file_values: + accumulator.add_declared_rows( + MeasurementSparseColumnarRows.from_rows( + ({slice_axis.value: image_number, **file_values},), + fields=( + FieldSpec(slice_axis.value, int), + *( + FieldSpec(name, str, required=False) + for name in file_values + ), + ), + ), + CELLPROFILER_MEASUREMENT_DIALECT, + default_subject="Image", + default_scope=MeasurementScope.IMAGE, + ) for table in CellProfilerModule.derive_experiment_measurement_tables(all_tables): accumulator.add_declared_rows( table.rows, @@ -529,7 +556,7 @@ def _measurement_tables( }, } for image_number, metadata_rows in source_metadata_by_image_number.items() - for consensus in (source_component_metadata_consensus(metadata_rows),) + for consensus in (_source_metadata_consensus(metadata_rows),) if consensus is not None ) if source_metadata_rows: @@ -565,25 +592,81 @@ def _measurement_tables( ) +def _source_metadata_consensus( + rows: Sequence[tuple[Mapping[str, object], Mapping[str, object]]], +) -> Mapping[str, object] | None: + """Fold each existing source role without treating absent extraction as data. + + Original fields previously participated only when extraction existed. + Preserve that rule while independently folding requested acquisition facts; + original spelling/values remain authoritative when namespaces overlap. + """ + original = source_component_metadata_consensus( + tuple(row[0] for row in rows if row[0]) + ) + acquisition = source_component_metadata_consensus(tuple(row[1] for row in rows)) + if original is None and acquisition is None: + return None + return {**(acquisition or {}), **(original or {})} + + def _source_metadata_measurement_rows( table: MeasurementTable, - row_domain: MeasurementRowsAxisProjection, image_numbers_by_slice: Mapping[int, int], -) -> tuple[tuple[int, Mapping[str, object]], ...]: + *, + add_image_metadata: bool, + add_image_file_names: bool, +) -> tuple[ + tuple[int, Mapping[str, object], Mapping[str, object], Mapping[str, str]], ... +]: """Project producer-owned source metadata into CellProfiler Image rows.""" - rows: list[tuple[int, Mapping[str, object]]] = [] - for slice_index in row_domain.present_axis_values( - MeasurementRowAxisField.SLICE_INDEX.value - ): - metadata = table.source_provenance.for_source_plane( - slice_index - ).source_component_metadata - if metadata is None: - continue - original_metadata = dict(SourceMetadataFields.original_items(metadata)) - if original_metadata: - rows.append((image_numbers_by_slice[slice_index], original_metadata)) + rows: list[ + tuple[int, Mapping[str, object], Mapping[str, object], Mapping[str, str]] + ] = [] + dialect = CellProfilerDatabaseColumnDialect() + image_subject = MeasurementSubject(MeasurementScope.IMAGE, "Image") + for slice_index in image_numbers_by_slice: + provenance = table.source_provenance.for_source_plane(slice_index) + metadata = dialect.source_metadata_values( + provenance.source_component_metadata, + None, + ) + acquisition = {} + if add_image_metadata: + acquisition.update( + dialect.source_acquisition_values(provenance.source_component_metadata) + ) + acquisition.update( + dialect.source_metadata_values( + None, + Path(provenance.source_path) + if provenance.source_path is not None else None, + ) + ) + file_values: dict[str, str] = {} + for name in ( + provenance.represented_source_image_names if add_image_file_names else () + ): + named_source = provenance.for_source_image(name) + if named_source.source_path is None: + continue + file_values.update( + ( + dialect.source_measurement_field( + image_subject, FieldSpec(field, str) + ).name, + value, + ) + for field, value in dialect.source_image_file_values( + Path(named_source.source_path), + name, + ).items() + ) + if metadata or acquisition or file_values: + rows.append( + (image_numbers_by_slice[slice_index], metadata, acquisition, file_values) + ) return tuple(rows) @@ -968,6 +1051,13 @@ def export_to_spreadsheet( """Render one plate's exact contract-selected spreadsheet file bundle. Args: + add_image_metadata: Copy Image metadata into object rows, projecting + declared acquisition components from sourced measurement tables + when requested. Absent coordinates are not synthesized. + add_image_file_names: Project named source paths and filenames from + measurement provenance and copy them into object rows. Source + image pixels are not reloaded. Existing Image features are retained + regardless of these flags. file_selections: Explicit output files and their measurement subjects when automatic export of all measurement types is disabled. """ diff --git a/tests/unit/test_cellprofiler_spreadsheet_export.py b/tests/unit/test_cellprofiler_spreadsheet_export.py index 5136fda7f7..20357ea72f 100644 --- a/tests/unit/test_cellprofiler_spreadsheet_export.py +++ b/tests/unit/test_cellprofiler_spreadsheet_export.py @@ -54,7 +54,12 @@ from openhcs.core.runtime_relationships import ( ObjectRelationship, ) -from openhcs.core.source_image_provenance import SourceImageProvenancePlanes +from openhcs.core.source_image_provenance import ( + SourceImageProvenancePlanes, + RuntimeSourceImageProvenancePlane, + SourceImageProvenanceContributor, + SourceImageIdentity, +) from openhcs.core.source_matching import SourceImageSetIdentityPolicy from openhcs.core.source_bindings import ( ComponentSelector, @@ -404,6 +409,146 @@ def test_export_to_spreadsheet_renders_only_declared_batch_records() -> None: assert "Ignored" not in bundle["Run1/Cells.csv"] +@pytest.mark.parametrize("add_metadata", (False, True)) +@pytest.mark.parametrize("add_files", (False, True)) +@pytest.mark.parametrize("axisless", (False, True)) +def test_spreadsheet_projects_source_identity_without_upstream_image_features( + add_metadata: bool, add_files: bool, axisless: bool, +) -> None: + provenance = SourceImageProvenancePlanes(( + RuntimeSourceImageProvenancePlane( + SourceImageIdentity('/acquisition/body.tif', { + 'well': 'A01', 'site': '2', 'channel': '3', 'z_index': '4', + 'timepoint': '5', ORIGINAL_SOURCE_METADATA_FIELD: {'Treatment': 'control'}, + }), + source_image_name='Body', + ), + )) + cells = _measurement_record( + 'cells', axis_id='A01', + subject=MeasurementSubject(MeasurementScope.OBJECT, 'Cells', 'object_number'), + rows=({**({} if axisless else {'slice_index': 0}), 'object_number': 7, 'Area': 12.0},), + source_image_provenance_planes=provenance, + ) + bundle = export_to_spreadsheet( + artifact_batch=RuntimeArtifactBatch( + input_specs=(ArtifactSpec.input('cells', MeasurementsArtifactType),), + records_by_axis={'A01': (cells,)}, + source_image_set_identity_policy=SourceImageSetIdentityPolicy(), + ), + add_image_metadata=add_metadata, add_image_file_names=add_files, + add_filename_prefix=False, + ) + image, = csv.DictReader(io.StringIO(bundle['Image.csv'])) + cell, = csv.DictReader(io.StringIO(bundle['Cells.csv'])) + assert image['Metadata_Treatment'] == 'control' + assert ('Metadata_site' in image) is add_metadata + assert ('FileName_Body' in image) is add_files + if add_metadata: + assert image['Metadata_well'] == 'A01' + assert image['Metadata_site'] == '2' + assert image['Metadata_z_index'] == '4' + assert image['Metadata_timepoint'] == '5' + if add_files: + assert image['FileName_Body'] == 'body.tif' + assert image['PathName_Body'] == '/acquisition' + assert cell['object_label'] == '7' + assert cell['Area'] == '12.0' + assert ('Metadata_site' in cell) is add_metadata + assert ('Image_FileName_Body' in cell) is add_files + if add_files: + assert cell['Image_FileName_Body'] == 'body.tif' + assert cell['Image_PathName_Body'] == '/acquisition' + + +@pytest.mark.parametrize('names', (('Body', 'Nuclear'), ('Nuclear', 'Body'))) +def test_spreadsheet_preserves_independent_contributor_filenames(names: tuple[str, ...]) -> None: + provenance = SourceImageProvenancePlanes(( + RuntimeSourceImageProvenancePlane( + SourceImageIdentity(component_metadata={'well': 'A01', 'site': '2'}), + contributors=tuple( + SourceImageProvenanceContributor( + SourceImageIdentity(f'/acquisition/{name}.tif', {'well': 'A01', 'site': '2'}), + source_image_name=name, + ) + for name in names + ), + ), + )) + record = _measurement_record( + 'cells', axis_id='A01', + subject=MeasurementSubject(MeasurementScope.OBJECT, 'Cells', 'object_number'), + rows=({'slice_index': 0, 'object_number': 1, 'Area': 12.0},), + source_image_provenance_planes=provenance, + ) + bundle = export_to_spreadsheet( + artifact_batch=RuntimeArtifactBatch( + input_specs=(ArtifactSpec.input('cells', MeasurementsArtifactType),), + records_by_axis={'A01': (record,)}, + source_image_set_identity_policy=SourceImageSetIdentityPolicy(), + ), + add_image_metadata=True, add_image_file_names=True, add_filename_prefix=False, + ) + row, = csv.DictReader(io.StringIO(bundle['Cells.csv'])) + for name in names: + assert row[f'Image_FileName_{name}'] == f'{name}.tif' + assert row[f'Image_PathName_{name}'] == '/acquisition' + assert 'Metadata_channel' not in row + assert 'Metadata_z_index' not in row + + +def test_requested_source_columns_preserve_original_extraction_path_template() -> None: + image = _measurement_record( + 'image', axis_id='A01', + subject=MeasurementSubject(MeasurementScope.IMAGE, 'Image'), + rows=({'slice_index': 0, 'Count_Cells': 1},), + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + component_metadata=({'site': '1', ORIGINAL_SOURCE_METADATA_FIELD: {'Run': 'run1'}},), + ), + ) + cells = _measurement_record( + 'cells', axis_id='A01', + subject=MeasurementSubject(MeasurementScope.OBJECT, 'Cells', 'object_number'), + rows=({'slice_index': 0, 'object_number': 1, 'Area': 12.0},), + ) + bundle = export_to_spreadsheet( + artifact_batch=RuntimeArtifactBatch( + input_specs=tuple(ArtifactSpec.input(record.key.name, MeasurementsArtifactType) for record in (image, cells)), + records_by_axis={'A01': (image, cells)}, + source_image_set_identity_policy=SourceImageSetIdentityPolicy(), + ), + add_image_metadata=True, add_image_file_names=True, + output_directory='{Run}', add_filename_prefix=False, + ) + assert set(bundle) == {'run1/Image.csv', 'run1/Cells.csv'} + row, = csv.DictReader(io.StringIO(bundle['run1/Cells.csv'])) + assert row['Metadata_Run'] == 'run1' + assert row['Metadata_site'] == '1' + + +def test_spreadsheet_rejects_existing_filename_conflicting_with_provenance() -> None: + record = _measurement_record( + 'image', axis_id='A01', + subject=MeasurementSubject(MeasurementScope.IMAGE, 'Image'), + rows=({'slice_index': 0, 'FileName_Body': 'other.tif'},), + source_image_provenance_planes=SourceImageProvenancePlanes(( + RuntimeSourceImageProvenancePlane( + SourceImageIdentity('/acquisition/body.tif', {'site': '1'}), + source_image_name='Body', + ), + )), + ) + with pytest.raises(ValueError, match='Conflicting sparse measurement values'): + export_to_spreadsheet( + artifact_batch=RuntimeArtifactBatch( + input_specs=(ArtifactSpec.input('image', MeasurementsArtifactType),), + records_by_axis={'A01': (record,)}, + source_image_set_identity_policy=SourceImageSetIdentityPolicy(), + ), + add_image_file_names=True, + ) + + def test_export_to_spreadsheet_bundle_uses_generic_file_materialization() -> None: batch = RuntimeArtifactBatch( input_specs=(ArtifactSpec.input("measurements", MeasurementsArtifactType),),