diff --git a/docs/validation/check-persisted-roi-admission-134.py b/docs/validation/check-persisted-roi-admission-134.py new file mode 100644 index 0000000000..11a89038c3 --- /dev/null +++ b/docs/validation/check-persisted-roi-admission-134.py @@ -0,0 +1,141 @@ +"""Read-only admission witness and provider-free producer regression for #134. + +No image loader, ROI decoder, sampler, viewer or transport is invoked. Synthetic +archive sentinels intentionally test inventory admission, not archive validity. +""" + +from pathlib import Path +from types import SimpleNamespace + +import pytest +from polystore.filemanager import FileManager +from polystore.disk import DiskStorageBackend +from polystore.virtual_workspace import SourcePixelRef + +from openhcs.agent.dto.plate import PlateFileQueryRequest +from openhcs.agent.path_policy import AgentPathPolicy +from openhcs.agent.services.plate_inspection_service import PlateInspectionService +from openhcs.constants.constants import Backend, FileFormat +from openhcs.core.artifacts import ObjectLabelsArtifactType +from openhcs.core.compiled_step_plan import ( + CompiledStepPlan, + RuntimeArtifactMaterializationPlan, +) +from openhcs.core.plate_file_inventory import PlateFileKind +from openhcs.core.plate_image_inventory import PlateResultFileInventory +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.source_projection import ( + OpenHCSPlaneAddress, + SourceArtifactProjection, + SourceProjectionMetadataSerializer, +) +from openhcs.core.steps.function_outputs import RuntimeArtifactMetadataTarget +from openhcs.core.virtual_workspace_metadata import AtomicMetadataWriter, METADATA_CONFIG +from openhcs.microscopes.openhcs import OpenHCSMetadataHandler +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser + + +RETAINED_PLATE = Path( + "/home/ts/wt/openhcs-issue-batch-20260929/" + "neurite-development-skill383-20261001/output/" + "assisted-2-candidate-2-sensitivity-diagnostic/input_openhcs" +) +RETAINED_ROI = ( + RETAINED_PLATE / "results/A01_s001_w2_z001_t001_neurons_step0_rois.roi.zip" +) + + +def test_retained_explicit_directory_admits_exact_roi_without_content_read(): + service = PlateInspectionService( + AgentPathPolicy.with_roots(readable_roots=(RETAINED_PLATE,), writable_roots=()) + ) + result = service.query_files( + PlateFileQueryRequest.from_fields( + plate_path=str(RETAINED_PLATE), + result_directory=str(RETAINED_ROI.parent), + kind="result", + include_previews=False, + ) + ) + assert result.errors == () + matching = [record for record in result.records if record.full_path == str(RETAINED_ROI)] + assert len(matching) == 1 + assert matching[0].file_format == "ROI" + assert matching[0].preview is None + record = service.result_directory_inventory(RETAINED_ROI.parent).require_file_record( + str(RETAINED_ROI) + ) + assert record.kind is PlateFileKind.RESULT + assert record.file_format is FileFormat.ROI + assert record.streamable_roi_path == str(RETAINED_ROI) + filemanager = FileManager({Backend.DISK.value: DiskStorageBackend()}) + ordinary = PlateResultFileInventory.from_handler( + plate_path=RETAINED_PLATE, + metadata_handler=OpenHCSMetadataHandler(filemanager), + parser=None, + ) + assert ordinary.records == () + print(f"Explicit typed inventory admits {result.total_count} files and exact ROI; no content read") + + +@pytest.mark.parametrize("destination", ("declared_outputs", "nested/independent_outputs")) +def test_runtime_result_publication_reaches_original_inventory(tmp_path, destination): + plate = tmp_path / "output_plate" + directory = plate / destination + directory.mkdir(parents=True) + roi = directory / "unrelated-name.roi.zip" + roi.write_bytes(b"inventory sentinel; not a scientific ROI") + # Mixed persisted results, like the retained run. This image is never loaded; + # original typed projection publication supplies its synthetic address. + image = directory / "unrelated-labels.tif" + image.write_bytes(b"inventory sentinel; not a scientific image") + virtual_path = str(image.relative_to(plate)) + components = { + "well": "A01", "site": "1", "channel": "2", + "z_index": "1", "timepoint": "1", "pixel_size": 1.25, + } + projection = SourceArtifactProjection( + address=OpenHCSPlaneAddress.from_values("A01", "1", "2", "1", "1"), + ref=SourcePixelRef(Backend.DISK.value, virtual_path), + source_alias="synthetic labels", + artifact_kind=ObjectLabelsArtifactType, + source_metadata=components, + image_metadata=ImagePayloadMetadata(source_component_metadata=components), + ) + AtomicMetadataWriter().merge_source_projection_metadata( + METADATA_CONFIG.metadata_path(plate), + destination, + SourceProjectionMetadataSerializer(SourceSchemaFilenameParser()).projection_fields( + ((projection, virtual_path),) + ), + ) + filemanager = FileManager({Backend.DISK.value: DiskStorageBackend()}) + plan = CompiledStepPlan( + step_index=0, + step_name="independent persistent result", + step_type="FunctionStep", + axis_id="A01", + output_plate_root=str(plate), + analysis_results_dir=str(directory), + runtime_artifact_materialization=RuntimeArtifactMaterializationPlan( + persistent_enabled=True, persistent_backend=Backend.DISK.value + ), + ) + target = RuntimeArtifactMetadataTarget.from_plan(plan) + assert target is not None + context = SimpleNamespace( + filemanager=filemanager, + metadata_cache={}, + microscope_handler=SimpleNamespace( + parser=SourceSchemaFilenameParser(), microscope_type="openhcsdata" + ), + ) + # Exercise the original target's shared writer and original metadata reader, + # without rendering, reading the sentinel, or fabricating a source receipt. + target.write(context) + inventory = PlateResultFileInventory.from_handler( + plate_path=plate, + metadata_handler=OpenHCSMetadataHandler(filemanager), + parser=None, + ) + assert str(roi) in tuple(record.full_path for record in inventory.records) diff --git a/docs/validation/check-persisted-roi-only-proposal-134.py b/docs/validation/check-persisted-roi-only-proposal-134.py new file mode 100644 index 0000000000..14cf5f30a7 --- /dev/null +++ b/docs/validation/check-persisted-roi-only-proposal-134.py @@ -0,0 +1,83 @@ +"""Select original/proposed PR394 source owners, not a second implementation. + +QA134_SOURCE_ROOT points to a small, explicitly retained source projection. +The existing runner owns ABI preparation and pytest. No product method or +assertion is replaced; normal imports select these files before collection. +""" + +import importlib.abc +import importlib.util +import os +from pathlib import Path +import runpy +import subprocess +import sys + +source_root = Path(os.environ["QA134_SOURCE_ROOT"]).resolve(strict=True) +sources = { + ".".join(path.relative_to(source_root).with_suffix("").parts): path + for path in source_root.rglob("*.py") +} +repository = Path(__file__).resolve().parents[2] +revision = os.environ.get("QA134_GIT_REF") +git_sources = {} +if revision: + changed = subprocess.check_output( + ["git", "diff", "--name-only", "HEAD", revision, "--", "openhcs"], + cwd=repository, + text=True, + ).splitlines() + git_sources = { + ".".join(Path(path).with_suffix("").parts).removesuffix(".__init__"): path + for path in changed + if path.endswith(".py") + } + + +class GitOwnerSource(importlib.abc.Loader): + """Import exact pinned source blobs without copying an entire checkout.""" + + def create_module(self, spec): + return None + + def exec_module(self, module): + path = git_sources[module.__name__] + source = subprocess.check_output( + ["git", "show", f"{revision}:{path}"], + cwd=repository, + ) + print(f"PINNED SOURCE ONLY: {module.__name__} -> {revision}:{path}") + exec(compile(source, f"{revision}:{path}", "exec"), module.__dict__) + + +class SelectedOwnerSources(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + source = sources.get(fullname) + if source is not None: + print(f"SOURCE PROJECTION ONLY: {fullname} -> {source}") + return importlib.util.spec_from_file_location(fullname, source) + source_path = git_sources.get(fullname) + if source_path is not None: + spec = importlib.util.spec_from_loader( + fullname, + GitOwnerSource(), + is_package=source_path.endswith("/__init__.py"), + ) + if spec.submodule_search_locations is not None: + spec.submodule_search_locations = [ + str(repository / Path(source_path).parent) + ] + return spec + return None + + +if any(name in sys.modules for name in {*sources, *git_sources}): + raise RuntimeError("A selected owner was imported before source selection") +selection = SelectedOwnerSources() +sys.meta_path.insert(0, selection) +try: + runpy.run_path( + str(Path(__file__).with_name("check-function-help-390.py")), run_name="__main__" + ) +finally: + sys.meta_path.remove(selection) diff --git a/docs/validation/check-persisted-roi-proposal-134.py b/docs/validation/check-persisted-roi-proposal-134.py new file mode 100644 index 0000000000..8f9b47125c --- /dev/null +++ b/docs/validation/check-persisted-roi-proposal-134.py @@ -0,0 +1,42 @@ +"""Select the explicit source proposal before provider/plugin-free collection. + +The existing validation runner owns ABI preparation and pytest invocation. This +callback changes only the selected source file, not product classes or test +assertions. The projection is not an installed package or production change. +""" + +import importlib.util +from pathlib import Path +import runpy +import sys + +import pytest + + +original_main = pytest.main + + +def check_proposed_source(args): + source = ( + Path(__file__).resolve().parents[2] + / ".qa134-proposal-20261001/source/openhcs/core/steps/function_outputs.py" + ) + module_name = "openhcs.core.steps.function_outputs" + if module_name in sys.modules: + raise RuntimeError("Metadata owner was imported before source selection") + spec = importlib.util.spec_from_file_location(module_name, source) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + print(f"SOURCE PROPOSAL ONLY: {module.__file__}") + pytest.main = original_main + return original_main(args) + + +pytest.main = check_proposed_source +try: + runpy.run_path( + str(Path(__file__).with_name("check-function-help-390.py")), run_name="__main__" + ) +finally: + pytest.main = original_main diff --git a/docs/validation/graph-roi-source-roundtrip-134-source-20261002.tar.gz b/docs/validation/graph-roi-source-roundtrip-134-source-20261002.tar.gz new file mode 100644 index 0000000000..dc80332270 Binary files /dev/null and b/docs/validation/graph-roi-source-roundtrip-134-source-20261002.tar.gz differ diff --git a/docs/validation/graph-roi-source-roundtrip-134.patch b/docs/validation/graph-roi-source-roundtrip-134.patch new file mode 100644 index 0000000000..57245caac2 --- /dev/null +++ b/docs/validation/graph-roi-source-roundtrip-134.patch @@ -0,0 +1,27 @@ +diff --git a/openhcs/processing/materialization/core.py b/openhcs/processing/materialization/core.py +index a095267a2..61340a650 100644 +--- a/openhcs/processing/materialization/core.py ++++ b/openhcs/processing/materialization/core.py +@@ -75,0 +76 @@ from openhcs.core.source_matching import ( ++from openhcs.core.source_metadata import SourceVoxelSpacing +@@ -3286,0 +3288,2 @@ def _write_spatial_graph_roi_zip( ++ from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata ++ +@@ -3294,0 +3298,4 @@ def _write_spatial_graph_roi_zip( ++ source_metadata = ImagePayloadMetadata( ++ source_provenance=graph.source_provenance, ++ source_voxel_spacing=SourceVoxelSpacing(graph.coordinate_spacing), ++ ) +@@ -3336 +3343,2 @@ def _write_spatial_graph_roi_zip( +- content=rois, ++ content=ROIArchiveSourceMetadata.bind(rois, source_metadata), ++ metadata=source_metadata, +diff --git a/tests/unit/test_spatial_graph_materialization.py b/tests/unit/test_spatial_graph_materialization.py +index 6dc0145d2..0c0e70ef3 100644 +--- a/tests/unit/test_spatial_graph_materialization.py ++++ b/tests/unit/test_spatial_graph_materialization.py +@@ -19,0 +20 @@ from openhcs.core.artifacts import ( ++from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata +@@ -343 +344 @@ def test_graph_roi_projection_preserves_paths_and_graph_features() -> None: +- first_roi = roi_output.content[0] ++ first_roi = ROIArchiveSourceMetadata.geometry(roi_output.content)[0] diff --git a/docs/validation/graph-roi-source-roundtrip-134.rst b/docs/validation/graph-roi-source-roundtrip-134.rst new file mode 100644 index 0000000000..d5a572446f --- /dev/null +++ b/docs/validation/graph-roi-source-roundtrip-134.rst @@ -0,0 +1,255 @@ +Graph ROI native-source roundtrip: receiving 134 +=============================================== + +Status and ownership +-------------------- + +Tristan's latest explicit correction on2026-10-02 restores Singer's ownership of +draft404 after the temporary handoff/stop. Root retains394 and shared production +ownership, including ``openhcs/processing/materialization/core.py``. Parent owns +actual installed/native graph reopening;404 retains that receiving acceptance. + +The graph writer repair integrated by Root at +``45f3a721e115b37ab662f362fa5ab4d5e58e1f6e`` remains present at current394 +``94ee1079d22540f9b5ed48b69481d7e7ab51095d``. No shared-hunk release remains +pending. Root owns production integration; Singer owns404 receiving acceptance; +parent owns installed/native qualification after the retinal11 slot yields. +404 is not merged wholesale. Its historical production base/proposal is not a +replacement for current394. Saved-directory publication at822 is not reapplied. + +Account-switch resume and current source +--------------------------------------- + +Tristan explicitly resumed work on2026-10-02. The preceding pause interrupted +no owned qualification operation: the last completed task was disposable-scratch +cleanup. On resume there is no active goal and no owned QA/ratchet process. +Original logs, failed fixtures, source proposals and uncertain dispositions +remain intact; no interrupted input is replayed. + +The existing persistent checkout is reused. The unpublished445 integration +receipt/archive is preserved on its local branch at1574ee396, without pushing +that branch.404 resumes from its existing published5f17851cd checkpoint; all +three untracked QA ledgers remain. No new WT, environment, dependency download, +package change, native launch, test rerun or cleanup is needed for this update. +Recorded submodule gitlinks differ from local dependency worktrees; none is +changed or represented as an installed qualification. + +Read the current394/404 claims and open468/469 rosters before this review. +468 owns tracking renderer correctness;469 owns MCP sampling presentation. +Neither claims the graph writer/binder or404 receipt. Root's shared production +files remain untouched. No competing404 implementation is started. + +At pinned94ee1079d, both graph test files and original ``SpatialGraph``, +``ROIArchiveSourceMetadata``, ``SourceImageProvenance``, ``SourceVoxelSpacing``, +``ImagePayloadMetadata`` and ``StreamingService`` modules are byte-identical to +the integrated45f3 checkpoint. Whole materialization/core.py has44 additions and +25 deletions since45f3 for retained-image role filename/context ownership; its +graph writer is unchanged. Function-runtime changes leave the graph context +strategy unchanged. Public plate streaming adds progress/typed failure receipts +and retains ``require_source_metadata=request.result_directory is not None``. +These differences do not justify copying the writer or weakening source admission. + +Current source still follows the original graph projection -> typed metadata -> +shared archive binder -> strict reader relationship. BOUND-2/BOUND-8 and +IMPL-12 remain resolved at that receiving boundary. The unchanged seven-case +fixture retains actual cooperative ``AuditedGraph(ProjectionAudit, SpatialGraph)`` +hooks and independent edge-feature roundtrip evidence; no generic consumer edit +is required for that new declaration. Historical executed source controls below +are reused, not presented as a new run at94ee1079d or a whole394 qualification. + +The finite remaining404 gate is a fresh correctly materialized source-bearing +graph ZIP through public ``kind=result`` inventory and the explicitly owned +native reopen route, with actual source/channel/plane/calibration, geometry and +subject/edge feature readbacks plus matched raw/result/combined placement review. +Parent owns that serial installed check. Original candidate6's broken archive +and failed job are historical evidence, not an input to repair/replay. Root's +optimization target, whole-branch R0 adjudication and separate saved-image parity +work do not require another404 source implementation. + +Verified integrated source checkpoint +------------------------------------- + +Root's commit45f3a721e has parent +``38067a4a5c348e0c7fcedd1d48f69bb99b7be19b``. Its production diff is exactly the +published narrow writer patch: one bare-content line replaced, nine lines added +for original typed metadata/spacing and binder reuse. Whole core.py and existing +``test_spatial_graph_materialization.py`` have zero differences from the qualified +proposal comparison3913c5008. The imported seven-case receiving fixture has the +same Git blob as f3e566485: +``83f0b9b4789a14a3d762d2f63d6599c7b939536f``. + +The original graph context hook still owns explicit source-plane selection; +the writer performs no second selection. ``ROIArchiveSourceMetadata`` retains +encoding, decoding and geometry projection. ``Output.metadata`` carries the same +declaration. The native reader and its missing/conflicting-source guard are not +changed by this integration. The geometry-test adaptation and cooperative +new-declaration/edge-feature behaviour are the exact receiving version. + +Root's committed integration receipt reports original current-source3RED/4PASS +and87 integrated controls PASS in2.28s, with original before/after logs retained +under RUNS.394's body reports the isolated original R0 graph delta passed with +zero positive deltas. These are Root's execution evidence, not a receiving rerun +or a waiver of394's separate whole-branch architecture/native/performance gates. +Receipt: +https://github.com/OpenHCSDev/openhcs/blob/45f3a721e115b37ab662f362fa5ab4d5e58e1f6e/docs/validation/shared_runtime_plumbing_384_20261001.rst. + +NRA/refactor-audit source review accepts the exact BOUND-2/BOUND-8 repair using +the existing projection/binder owners, without IMPL-12 duplication or another +consumer/registry. No test was repeated, source patch reapplied, foreign worktree +edited or live installation changed for this receiving checkpoint. + +Original receiving witness +-------------------------- + +Parent retained ``GRAPH-ROI-REOPEN-134-RECEIVING-20261002.rst`` under +``/home/ts/wt/openhcs-issue-batch-20260929``. The candidate6 saved +``A01_s001_w2_z001_t001_neurite_morphology_step0.graph.roi.zip`` resolves through +public result inventory but native reopening refuses missing embedded source +identity. Original request/error remains in the author's actual recorder: +``/home/ts/wt/openhcs-issue-batch-20260929/neurite-development-skill383-20261001/output/assisted-2-post426-1.stdout``. +Read-only verification locates ``plate_file_stream_failed`` at line7602 and the +``ValueError`` message requiring native ROI source metadata at line7605, naming +the exact candidate6 graph ZIP above. This corrects the recorder reference in +the ddbc1b197 receipt; historical receiving receipts and original archives remain +unchanged. The original broken archive was not opened, rewritten or backfilled. + +Required relation and original owners +------------------------------------- + +An authorised graph archive must preserve its already-declared source identity, +selected channel/plane, declared coordinate spacing, polyline geometry and +neuron subject/edge features through writing and loading. Native reopening must +still reject absent, unbound, partially bound or conflicting source declarations. + +``SpatialGraph.contextualized_source_provenance`` owns explicit source-plane +selection. ``SpatialGraphFunctionOutputContextStrategy`` consumes that hook and +attaches invocation provenance before materialisation. It already produces a +scalar selected-source graph, including when the original plane index is1. +Selecting that index again from the scalar graph would be incorrect. + +``MaterializationInputItem.metadata`` delegates to ``image_payload_metadata``; +``SpatialGraph`` is not an image-metadata carrier, so that value is empty for +graphs. The original graph carries ``source_provenance`` and +``coordinate_spacing``. The registered graph ROI writer projects those facts +into the existing ``ImagePayloadMetadata`` and ``SourceVoxelSpacing`` owners. +``ROIArchiveSourceMetadata.bind/decode/geometry`` retain the single encoding, +decoding and presentation procedures. ``Output.metadata`` carries the same +declaration into downstream materialisation. No new source schema is introduced. + +Historical exact source proposal +-------------------------------- + +``graph-roi-source-roundtrip-134.patch`` contains the narrow production hunk and +one geometry-test adaptation. Its historical822 admission used zero-context +``git apply --unidiff-zero``. Do not reapply it over Root's integrated45f3a721e. +The production change removes one bare-content +line and adds nine lines in the existing registered leaf writer/imports. The +unchanged binder performs archive encoding. The reader, source-plane projection, +writer registry, subject projection, capture and viewer implementations are +unchanged. Source voxel spacing requires an import from its existing owner; +the first coordination note's statement that core already imported it was +incorrect and is corrected here. + +Synthetic Git comparison commit +``3913c5008b606a51268b5aeb4a3c8a527b89f1b0`` has parent82243634c and only the +proposed core file plus the existing geometry-test adaptation. It is a ratchet +comparison object, not an applied production commit or release branch. + +``tests/unit/test_graph_roi_source_roundtrip_134.py`` exercises actual context +selection, registered materialisation, disk ZIP writing and original ZIP loading. +It asserts both explicit input planes0/1, exact source/channel/names/provenance, +declared1.3556 spacing, fractional Y/X coordinates, node/edge IDs, graph features, +neuron label and typed subject identity, and transport-field removal from geometry. +The misleading archive filename supplies no identity. + +An independent ``AuditedGraph(ProjectionAudit, SpatialGraph)`` declaration +exercises cooperative ``super().__init__`` and before/after +``contextualized_source_provenance`` hooks. Its newly declared +``engineering_confidence`` edge feature survives the same ZIP writer/reader; +no generic consumer or registry changes are needed. This is behavioural MRO +evidence, not an inheritance assertion or ornamental production hierarchy. + +The four rejection cases call the original ``StreamingService.stream_rois`` with +``require_source_metadata=True``. They fail before reaching transport; no native +viewer is instantiated. Successful native streaming remains the later parent gate. +The graph type does not declare cropped source-domain bounds. This checkpoint +does not infer or claim such bounds from coordinates or filenames. + +Bounded qualification and original failures +------------------------------------------- + +Every execution was serial, existing Python/ABI dependencies only, CPU affinity0, +``CPUQuota=100%``, kernel ``MemoryMax=512M``, ``MemorySwapMax=0``, shell deadline60s, +pytest external plugins/cache disabled and numerical thread counts1. Tests use +only explicitly constructed disk storage; a fixture fails optional storage +bootstrap. No native/MCP/UI launch, package/environment change, download, provider, +scientific execution or shared slot was used. + +The existing source-selection runner now loads changed OpenHCS Python modules +directly from pinned Git blobs when ``QA134_GIT_REF`` is supplied. Unchanged files +come from the existing branch, equal to that revision; only the proposed core +overrides a blob. This avoids another worktree/full source snapshot and preserves +the actual current dependency closure instead of monkeypatching product methods. + +Original logs, including harness failures, are retained in the paired archive: + +* ``original.log``: collection fails because a partial current-source selection + lacked ``DurableSourceMetadata``;1.74s/91656KiB. No product assertion executed. +* ``original2.log``: collection fails because mixing current source metadata with + old source bindings lacked ``SourceMetadataIdentityProjection``; + 2.22s/79076KiB. No product assertion executed. +* ``pinned-original.log``: exact pinned source, three native-source-loss REDs + and four rejection controls PASS;2.52s/177464KiB. +* ``proposed.log``: seven receiving cases plus five existing metadata cases, + 12PASS;3.05s/199284KiB. +* ``graph-regression.log``:15PASS/1RED. The existing test compared the bare ROI + feature mapping including transport metadata. This authentic first failure is + retained. The proposed adaptation uses original ``geometry`` before the same + exact feature/coordinate assertions; no assertion is removed or weakened. +* ``graph-regression-final.log``: the existing full graph family,16PASS; + 2.40s/172992KiB. No receiving cases were repeated for ceremony. +* ``r0.log``: original unmodified pinned agent-comms R0 at + ``3b03785f45df2ef5dc62ba6aed99294192ecbb01``, from the existing + ``/home/ts/wt/openhcs-s1-original-ratchet-20261001``. Exact822→3913c5008 + comparison includes the entire changed production core file: exit0, every + delta zero;33.81s/87660KiB. The old UI348 ratchet WT no longer exists; + no new WT or copied detector was created. + +Ownership/antipattern review +---------------------------- + +Refreshed full NRA skill and authoritative refactor-audit ZIP, catalog README, +boundaries, implementation, membership and surface-receipt guidance were read. +NRA skill SHA256: +``9f2f8b28bc82256eefa3e9d63248c50722dc3ffe7d77adba5793296df196b47e``. +Audit ZIP SHA256: +``ef0367d878cc57565f2b257a8e6647a234d4ba4298f61391709d9c3c82bec9f7``. + +BOUND-2/BOUND-8 classify the lost original source declaration at the writer +boundary. IMPL-12 is avoided by reusing the shared binder/decoder/geometry owner +rather than copying metadata encoding. IMPL-3/IMPL-5 and MEMB-1/MEMB-2 introduce +no new consumer type switch, repeated dispatch or roster. The registered writer +is the existing format leaf; shared algorithms remain on their original owners. +No forwarding facade, compatibility reader or parallel store is introduced. +This source review and original R0 pass are not a complete global NRA/R1 scan; +earlier bounded global-scan failures remain in the historical receiving receipts. + +Disposition and later acceptance +-------------------------------- + +Source integration is complete in394 at45f3a721e; no production-hunk dependency +remains for404.404 retains the receiving record and pending installed/native +acceptance, not another implementation to merge over394. Parent controls the +coherent package qualification, installed freeze release and later public +inventory→owned native reopen of a +new correctly written graph ZIP, with source/channel/calibration readbacks and +personally opened same-coordinate raw/result/combined placement. Existing failed +jobs/archives remain unchanged; no biological acceptance is claimed. + +Owned scratch is ``.qa134-graph-20261002`` in Singer's existing persistent WT. +Cleanup remains stopped; no completed disposable directories were deleted here. +The byte-exact original logs and synthetic engineering ZIP inputs/outputs are +archived, not reformatted to remove their authentic whitespace. +Archive: ``graph-roi-source-roundtrip-134-source-20261002.tar.gz``; SHA256 +``0b04936ffd6f54fb124f4c5ce27f300f72bd83af01922bc874e7e5014efbf375``. +Direct archive reads match the original negative and positive raw-log hashes. diff --git a/docs/validation/persisted-roi-admission-134-source-20261001.tar.gz b/docs/validation/persisted-roi-admission-134-source-20261001.tar.gz new file mode 100644 index 0000000000..ac41f49870 Binary files /dev/null and b/docs/validation/persisted-roi-admission-134-source-20261001.tar.gz differ diff --git a/docs/validation/persisted-roi-admission-134.rst b/docs/validation/persisted-roi-admission-134.rst new file mode 100644 index 0000000000..b845728a02 --- /dev/null +++ b/docs/validation/persisted-roi-admission-134.rst @@ -0,0 +1,282 @@ +Persisted ROI directory admission: receiving investigation +======================================================== + +Singer owns this receiving investigation under issue 134. Main audited: +``889bca2e1f69ef6bfb50ded599903c23b019ccbb``. Completed issue 398 and PR 399 +remain closed; their ordinary saved-image reader/reopen acceptance is unchanged. +Current source status: Root integrated the publication requirement in PR394 +``82243634ce097d0d1a2d9e77664cbfeac3e6ec94``. Singer accepted the exact diff and +receipt against the receiving requirements. Root owns the integrated seam; no +shared-file release remains pending. Draft404 retains evidence and the parent +native ROI provenance/reopen gate, not a competing source implementation. +Root integrated the distinct graph writer repair at394 commit45f3a721e. Singer +has resumed404 receiving ownership under Tristan's latest explicit correction; +actual installed/native reopening remains parent-owned. The exact source review +and pending gate are in ``graph-roi-source-roundtrip-134.rst``. No graph patch or +already-integrated publication is reapplied here. +Historical investigation below is not a native ROI or biological pass. +Visible draft: https://github.com/OpenHCSDev/openhcs/pull/404. +The historical implementation proposal superseded the initial diagnostic-only +checkpoint and is now superseded by Root's integrated checkpoint. Its original +qualification and current receiving acceptance/native-boundary receipt is +``persisted-roi-only-owner-134.rst`` in this directory. + +Original public witness +----------------------- + +Dalton's retained root is +``/home/ts/wt/openhcs-issue-batch-20260929/neurite-development-skill383-20261001/output``. +``assisted-2-candidate-2-isolated.stdin:45`` requests +``openhcs_stream_plate_files_to_viewer`` with the candidate-2 output plate, +``kind=result`` and the exact physical +``results/A01_s001_w2_z001_t001_neurons_step0_rois.roi.zip``. +``assisted-2-candidate-2-isolated.stdout:44741-44789`` retains +``plate_file_stream_failed`` / ``PlateFileRecordNotFoundError`` with zero resolved +records. No request was replayed or native/viewer endpoint touched here. + +The native ``assisted-2-candidate-2-sensitivity-diagnostic/input_openhcs/openhcs_metadata.json`` +contains four saved-label image projections under ``results`` and five checkpoint +image projections under ``images``. Neither branch declares ``results_dir``. +Only metadata, retained transport receipts and filesystem names were inspected. +Scientific pixels, ROI geometry and held-out sources were not read. +Metadata SHA256 at inspection: +``69b83da2d22197ef6af073cf7a97c53efa5e249694c5fecdc78a4cf0285469b2``. + +Required relation and existing owners +------------------------------------- + +A compiled persistent artifact destination must publish its result-directory +identity through the existing metadata contract so ordinary result inventory can +admit the actual saved ROI archive. An output branch's spelling is not evidence +that it is a result directory. + +``RuntimeArtifactMetadataTarget.from_plan`` receives the exact compiled artifact +analysis directory but supplies ``results_dir=None``. Its production destination +selection is image-only. ``OpenHCSMetadataWriter.OutputTarget.write`` owns shared +publication; ``AtomicMetadataWriter.publish_source_projection_metadata`` already +supports the existing result-directory field. ``OpenHCSMetadataHandler.analysis_result_directories`` +correctly admits only that declaration. ``PlateResultFileInventory`` owns the +single registered-format record projection. No reader fallback or parallel store +is warranted. + +PR 394 claims ``core/steps/function_outputs.py`` and its original tests, together +with the materialization batch/outcome authority. Narrow target/publication hook +ownership was requested at +https://github.com/OpenHCSDev/openhcs/pull/394#issuecomment-5941747455. +Those shared files remain untouched pending that coordination. The older graph +renderer/native ROI source-metadata omission is a separate crossing already +recorded there; it is not silently repaired by admitting a directory. + +The proposed repair preserves the shared publication algorithm on ``OutputTarget`` +and minimal destination/participation hooks on the runtime artifact leaf, consuming +the original saved materialization outcomes. It must include ROI-only outputs and +nested declared destinations, without a concrete-consumer switch or second render. +Catalog review: BOUND-8 identifies the declared destination lost at publication; +MEMB-2 cautions against an image-only roster standing in for persistent result +participation; BOUND-2 forbids bypassing the existing inventory/source metadata +owners. Existing source projection and ROI provenance safeguards stay intact. + +Existing explicit route +----------------------- + +The supported typed selection carries the original output ``plate_path``, +``kind=result``, the explicit canonical ``result_directory`` ending in +``assisted-2-candidate-2-sensitivity-diagnostic/input_openhcs/results``, and the +original exact ROI ``file_paths``. No acquisition-component filter is supplied. +For read-only inventory qualification, ``include_previews=false`` avoids reading +ROI geometry or images. The receiving check exercises this original service and +record resolver without invoking streaming or native processes. Path admission +does not prove that the archive's separate native ROI source-metadata contract +passes; no historical snapshot/source receipt is fabricated. + +Qualification +------------- + +The retained explicit-directory service query passed: 17 admitted result records, +exact neurons archive resolved as ROI, no previews. The original metadata handler +still returns zero ordinary result records. This verifies the supported selection +contract, not native ROI source provenance, streaming or scientific interpretation. + +Two mixed-image/ROI synthetic cases use the real ``RuntimeArtifactMetadataTarget``, +its shared writer, atomic projection store and original metadata handler/inventory. +The original typed projection publisher seeds a synthetic image address; file +contents are sentinels never loaded. Both ``declared_outputs`` and +``nested/independent_outputs`` fail with an empty result inventory despite the +persisted exact ROI. This establishes the missing publication relation without +filename inference. Independent destination names alone cannot repair it today; +new-declaration/cooperative-hook acceptance belongs to the coordinated source +repair and has not been claimed here. + +Serial final diagnostic run: 1 passed, 2 failed, exit 1, 4.88 seconds elapsed, +235804 KiB maximum process RSS. All runs used a kernel scope with +``MemoryMax=512M``, ``MemorySwapMax=0``, ``CPUQuota=100%``, CPU affinity 0 and a +60-second outer timeout. Plugins and providers were disabled. Two pytest config +warnings concern disabled asyncio plugins. Existing Python/ABI dependencies and +the pinned pyqt source were read-only; no installation, startup or native call. + +First collection failure is retained: receiving harness imported a nonexistent +``polystore.storage_backends`` module. It was corrected to original +``polystore.disk``. The next direct ROI-only fixture reached the original strict +``SourceProjectionSet requires at least one projection`` guard; it lacked the +saved image authority present in the actual mixed run and is not presented as the +installed missing-directory proof. Its input and failure remain intact. The final +mixed fixture supplies that authority through the original typed publisher and +fails at directory admission itself, as above. + +``docs/validation/persisted-roi-admission-134-source-20261001.tar.gz`` contains all +three byte-exact raw source logs, each original check-input version and synthetic +failed fixtures. ``tar -d`` verified it against every loose original. +Archive SHA256: +``5d01ecf1a408f8bd98053ba8d4d287b8b429dcd020f3e96304f1e8e41cfcb5d3``. +Full guarded commands and process receipts are in the logs. Original public errors and +scientific artefacts remain in Dalton's root, outside this archive. No production +paths changed, so no new production R0/R1 or global NRA pass is asserted. Whole-PR +``git diff --check`` applies to source/docs; authentic raw logs remain inside the +archive without whitespace rewriting. + +Only the owned disposable scratch root +``/home/ts/.cache/agent-scratch/persisted-roi-admission-134-20261001`` is released +after verified archival and source-worker termination: 39751 logical bytes, +160 KiB filesystem usage. No other worktree/cache, native owner, +scientific output or uncertain input is removed. + +The former shared-owner boundary is resolved: Root integrated the relation with +the existing successful-save batch authority at the source checkpoint named above. +Singer retains the receiving evidence and separate native/graph acceptance crossing. +No copied capture/materialization algorithm, compatibility reader, parallel +registry, invented source receipt or scientific replay was introduced. Diataxis +reference style keeps the verified admission facts distinct from implementation +and installed acceptance. + +Historical implementation: exact shared owner proposal +------------------------------------------------------ + +Parent requested continued repair, not diagnostic-only completion. Public PR 394, +issue 384, their tracked source receipts and commit metadata still identify the +shared human GitHub account rather than a named acting agent/contact. The receiving +source remains under Singer's ownership; no foreign worktree or production file +is edited while the shared hunk crossing is unresolved. + +``docs/validation/persisted-roi-directory-owner-134.patch`` exposes the actual +first repair checkpoint, three hunks in ``core/steps/function_outputs.py``: + +* Existing ``RuntimeArtifactMetadataTarget.from_plan`` declares its independently + compiled persistent analysis directory as ``results_dir`` rather than losing it. +* Its existing ``for_directory`` hook carries the exact destination through each + production/reconciliation directory projection. +* Existing shared ``OutputTarget.write`` publishes the full plate-relative + directory instead of dropping nested ancestry through ``Path.name``. + +The proposal removes three expressions and adds eight source lines. No renderer, +capture loop, consumer, registry, source-binding, axis, scientific parameter or +historical reader changes. The generic publication algorithm stays on the existing +ancestor; two minimal destination hooks stay on its existing leaf. The proposal +preserves PR 394's once-saved batch outcomes and its geometry reuse. Applicable +review cautions remain BOUND-2/BOUND-8 and MEMB-2, with IMPL-12 explicitly excluding +another publication/materialization procedure. + +The hunks pass admission against both the main-derived owner and PR 394 pinned +``fdf50241aa9adc6d96bc9c530dcc9acd19692b64``. Initial published patch at +``4510610a8`` included normal blank context and passed default ``git apply --check``. +The final patch omits unused blank context; its exact admission recipe is +``git apply --check --unidiff-zero docs/validation/persisted-roi-directory-owner-134.patch``. +The corresponding ``--directory=.qa134-proposal-20261001/pr394`` check qualifies +the separately archived PR 394 source. Default admission of that minimal-context +format refuses a hunk; this original formatting control is retained separately +and is not a product failure. The resulting production-code blob is unchanged. + +The receiving proposal is visible to the integration owner at +https://github.com/OpenHCSDev/openhcs/pull/404#issuecomment-5942252207 and the direct +crossing/contact request at +https://github.com/OpenHCSDev/openhcs/pull/394#issuecomment-5942252441. +This remains a repair in progress, not an acceptance or completion claim. + +Behavioral new case and proposal qualification +---------------------------------------------- + +The source projection copies only the selected owner file into an owned persistent +disposable directory in the existing worktree. The explicit proposal selector +loads that file before collection; the original runner still owns read-only ABI +preparation and provider/plugin-free pytest invocation. No product method/class +or assertion is mocked or rewritten. The selected source path is printed in the +raw receipt. Other imported OpenHCS files come from the original receiving source. + +The original unchanged two mixed writer/reader reds become two passes. An independent +``SupplementalResultMetadataTarget`` supplies only its destination declaration; +original family discovery selects it with no consumer/roster edits. Its declared +MRO composes an independent publication audit capability, the original +``ProducedImageMetadataCapability`` and the original ``OutputTarget`` owner. +Cooperative ``super().write`` executes before/after audit around the shared writer; +the original image capability executes its real record hook. Public +``PlateInspectionService.query_files(kind=result, include_previews=false)`` then +admits the exact ROI for simple and nested supplemental destinations. This exercises +capability hooks and resulting user-visible inventory, not inheritance assertions. + +Unmodified-owner new-case control: 1 pass / 1 fail, 5.67 seconds elapsed, +288748 KiB maximum process RSS. The nested declaration loses its result directory +because the original ancestor takes only its basename. Original raw failure retained. + +Projected owner qualification: 4 passes / 1 explicit deselection, 3.90 seconds +elapsed, 230468 KiB. It covers the original two mixed-output reds and both new +declaration cases. The already-completed retained explicit-directory admission +check is deselected, not replayed. A subsequent distinct exact-destination +``for_directory`` case passes, 3.70 seconds / 236336 KiB, two previously completed +unit cases deselected. Five distinct behavioral cases pass across these shards; +no broad suite, native ROI geometry, installed streaming or biological claim. + +Original pinned R0 +------------------ + +Unmodified original guard owner: +``/home/ts/wt/comms-ratchet-pinned-ui348-20261001`` at +``3b03785f45df2ef5dc62ba6aed99294192ecbb01``. Command uses Python 3.14 and +``python -m agent_comms.debt_ratchet --root openhcs --base de0d76b7a9b5177a0630c91487a039370ae81dc7 --head bbd059114d6ce32a97095f9d73b902e39e3854c4``. + +The head is the exact unapplied proposal tree, not the receiving PR's production +head. A separate temporary Git index creates that tree without changing the +working source/index or any ref. Its sole production delta is +``openhcs/core/steps/function_outputs.py``; every changed production path is included. +The projected file hash and Git blob both equal +``c6cea8754affd4fd38714f3ceed007922addd3e4``. The commit object and selected source +are archived so the exact qualification input survives even without a live QA ref. + +R0 PASS, exit 0, all deltas zero and no positive entries; 14.72 seconds elapsed, +87436 KiB maximum process RSS. Original detector/bound/root is unchanged. Source +shards and R0 use the same one-CPU, kernel 512 MiB/no-swap and 60-second bounds. +The original full-context NRA/R1 resource failure recorded with the earlier +receiving work is not converted into a global architecture pass here. After shared +integration, qualification must identify the actual resulting production head. + +Archive, cleanup and next integration boundary +---------------------------------------------- + +The byte-exact archive +``docs/validation/persisted-roi-owner-proposal-134-source-20261001.tar.gz`` retains +the original new-declaration red, projected positive shards, complete original R0, +formatting control, exact input versions, failed synthetic files and both selected +owner-source versions. It excludes scientific inputs/outputs and foreign caches. +The archived temporary index and commit record document source projection, not +another registry/store. ``tar -d`` verifies all loose originals before cleanup. +Raw log whitespace is preserved inside the archive. Diataxis reference style keeps +this projection evidence separate from actual applied/installed status. + +The resource-headroom check reported critical disk headroom; no new agent, environment, +installation, download, full snapshot or large test was started. Only the bounded +serial source checks above ran. The owned source-projection directory is released +after archival; exact size and archive digest are recorded below. + +Archive SHA256: +``4a17d826c4bb9768b5b2455be547f981277173abe1a7beeba1f6716cb4c237e7``. +Released only +``/home/ts/wt/openhcs-knowledge-lazy-conversion-20261001/.qa134-proposal-20261001``: +2364599 logical bytes, 2.6 MiB filesystem usage. Selected proposal source, original +failures, raw receipts and reconstruction inputs remain recoverable in the archive; +tracked receiving source/history and all foreign/scientific data remain intact. + +Root is the integrated PR394 owner; the combined proposal is superseded. This first +checkpoint repairs the actual mixed-label/checkpoint/ROI witness. ROI-only +participation and the empty-image transaction now have the independently tested +saved-output-owner proposal documented in the linked receipt; the older graph ROI source-metadata omission remains +its separate crossing. Neither is hidden behind relaxed reader guards, guessed +paths, fabricated receipts or a biological replay. Parent's ten viewer-QA lines, +manifest tags and query checks remain disjoint and untouched. diff --git a/docs/validation/persisted-roi-directory-owner-134.patch b/docs/validation/persisted-roi-directory-owner-134.patch new file mode 100644 index 0000000000..9dee16ecfd --- /dev/null +++ b/docs/validation/persisted-roi-directory-owner-134.patch @@ -0,0 +1,28 @@ +diff --git a/openhcs/core/steps/function_outputs.py b/openhcs/core/steps/function_outputs.py +--- a/openhcs/core/steps/function_outputs.py ++++ b/openhcs/core/steps/function_outputs.py +@@ -806,5 +806,8 @@ class OpenHCSMetadataWriter: + component_labels=context.metadata_cache, + backend=self.backend, + is_main=self.is_main, +- results_dir=(Path(self.results_dir).name if self.results_dir else None), ++ results_dir=( ++ str(Path(self.results_dir).relative_to(self.plate_root)) ++ if self.results_dir is not None else None ++ ), + ) +@@ -1167,4 +1170,6 @@ class RuntimeArtifactMetadataTarget(OpenHCSMetadataWriter.OutputTarget): + def for_directory(self, directory: Path) -> RuntimeArtifactMetadataTarget: + """Retain the compiled plate/backend while projecting one declared directory.""" + sub_dir = directory.relative_to(self.plate_root) +- return replace(self, output_dir=directory, sub_dir=str(sub_dir)) ++ return replace( ++ self, output_dir=directory, sub_dir=str(sub_dir), results_dir=str(directory) ++ ) +@@ -1181,5 +1186,5 @@ class RuntimeArtifactMetadataTarget(OpenHCSMetadataWriter.OutputTarget): + backend=materialization.require_persistent_backend(), + plate_root=plate_root, + sub_dir=str(output_dir.relative_to(plate_root)), +- results_dir=None, ++ results_dir=str(output_dir), + ) diff --git a/docs/validation/persisted-roi-only-bootstrap-failure-134-source-20261001.tar.gz b/docs/validation/persisted-roi-only-bootstrap-failure-134-source-20261001.tar.gz new file mode 100644 index 0000000000..d83b0dab18 Binary files /dev/null and b/docs/validation/persisted-roi-only-bootstrap-failure-134-source-20261001.tar.gz differ diff --git a/docs/validation/persisted-roi-only-owner-134-source-20261001.tar.gz b/docs/validation/persisted-roi-only-owner-134-source-20261001.tar.gz new file mode 100644 index 0000000000..87fafc1cdf Binary files /dev/null and b/docs/validation/persisted-roi-only-owner-134-source-20261001.tar.gz differ diff --git a/docs/validation/persisted-roi-only-owner-134.patch b/docs/validation/persisted-roi-only-owner-134.patch new file mode 100644 index 0000000000..896cda1bb3 --- /dev/null +++ b/docs/validation/persisted-roi-only-owner-134.patch @@ -0,0 +1,91 @@ +diff --git a/openhcs/core/steps/function_outputs.py b/openhcs/core/steps/function_outputs.py +index 05b7fb539..d44e511e9 100644 +--- a/openhcs/core/steps/function_outputs.py ++++ b/openhcs/core/steps/function_outputs.py +@@ -784,2 +784,8 @@ class OpenHCSMetadataWriter: +- def contains_images(self, context: ProcessingContext) -> bool: +- """Return whether the completed target contains image outputs.""" ++ def stored_output_paths(self, context: ProcessingContext) -> tuple[str, ...]: ++ """Return this declaration's outputs eligible for publication.""" ++ return tuple( ++ context.filemanager.list_image_files(self.output_dir, self.backend) ++ ) ++ ++ def contains_outputs(self, context: ProcessingContext) -> bool: ++ """Return whether this declared destination has publishable outputs.""" +@@ -791,6 +797 @@ class OpenHCSMetadataWriter: +- return bool( +- context.filemanager.list_image_files( +- self.output_dir, +- self.backend, +- ) +- ) ++ return bool(self.stored_output_paths(context)) +@@ -835 +836,4 @@ class OpenHCSMetadataWriter: +- results_dir=(Path(self.results_dir).name if self.results_dir else None), ++ results_dir=( ++ str(Path(self.results_dir).relative_to(self.plate_root)) ++ if self.results_dir is not None else None ++ ), +@@ -1066 +1070 @@ class OpenHCSMetadataWriter: +- if target.contains_images(context): ++ if target.contains_outputs(context): +@@ -1148,2 +1152,14 @@ class MaterializedImageMetadataTarget( +-class RuntimeArtifactMetadataTarget(OpenHCSMetadataWriter.OutputTarget): +- """Image artifacts persist independently of main-flow image/checkpoint storage.""" ++class PersistedResultMetadataCapability: ++ """Declared result destinations include non-image saved writer outputs.""" ++ ++ def stored_output_paths(self, context: ProcessingContext) -> tuple[str, ...]: ++ return tuple(dict.fromkeys(( ++ *super().stored_output_paths(context), ++ *context.filemanager.list_files(self.output_dir, self.backend), ++ ))) ++ ++ ++class RuntimeArtifactMetadataTarget( ++ PersistedResultMetadataCapability, OpenHCSMetadataWriter.OutputTarget ++): ++ """Saved artifacts own result destinations independently of image storage.""" +@@ -1161 +1176,0 @@ class RuntimeArtifactMetadataTarget(OpenHCSMetadataWriter.OutputTarget): +- if ImageFileFormat.is_image_path(output.path) +@@ -1167 +1182 @@ class RuntimeArtifactMetadataTarget(OpenHCSMetadataWriter.OutputTarget): +- if target.contains_images(context) ++ if target.contains_outputs(context) +@@ -1173,0 +1189,2 @@ class RuntimeArtifactMetadataTarget(OpenHCSMetadataWriter.OutputTarget): ++ from openhcs.microscopes.openhcs import OpenHCSMetadataHandler ++ +@@ -1188 +1205,6 @@ class RuntimeArtifactMetadataTarget(OpenHCSMetadataWriter.OutputTarget): +- for directory in dict.fromkeys((self.output_dir, *directories)) ++ for directory in dict.fromkeys(( ++ self.output_dir, ++ *directories, ++ *(directory.path for directory in ++ OpenHCSMetadataHandler(context.filemanager).analysis_result_directories(self.plate_root)), ++ )) +@@ -1194 +1216,3 @@ class RuntimeArtifactMetadataTarget(OpenHCSMetadataWriter.OutputTarget): +- return replace(self, output_dir=directory, sub_dir=str(sub_dir)) ++ return replace( ++ self, output_dir=directory, sub_dir=str(sub_dir), results_dir=str(directory) ++ ) +@@ -1208 +1232 @@ class RuntimeArtifactMetadataTarget(OpenHCSMetadataWriter.OutputTarget): +- results_dir=None, ++ results_dir=str(output_dir), +diff --git a/openhcs/core/virtual_workspace_metadata.py b/openhcs/core/virtual_workspace_metadata.py +index ff4abc6d0..63ce7c84f 100644 +--- a/openhcs/core/virtual_workspace_metadata.py ++++ b/openhcs/core/virtual_workspace_metadata.py +@@ -235,6 +235,7 @@ class AtomicMetadataWriter: +- projections = SourceProjectionSet( +- tuple(entries[path] for path in published_paths) +- ) +- subdirectory.update( +- serializer.component_metadata(projections, labels=component_labels) +- ) ++ if published_paths: ++ projections = SourceProjectionSet( ++ tuple(entries[path] for path in published_paths) ++ ) ++ subdirectory.update( ++ serializer.component_metadata(projections, labels=component_labels) ++ ) diff --git a/docs/validation/persisted-roi-only-owner-134.rst b/docs/validation/persisted-roi-only-owner-134.rst new file mode 100644 index 0000000000..43ae9e8a38 --- /dev/null +++ b/docs/validation/persisted-roi-only-owner-134.rst @@ -0,0 +1,244 @@ +ROI-only publication: exact original-owner hook proposal +======================================================= + +Current disposition, 2026-10-02: the source publication requirement is integrated +by Root in PR394 at ``82243634ce097d0d1a2d9e77664cbfeac3e6ec94`` and accepted +by Singer's receiving diff/receipt review. The source seam is no longer awaiting +an owner or release. Root owns integration; draft404 remains the receiving +evidence/native-acceptance record, not another production implementation to merge. +Committed PR source is distinct from merged, installed and native acceptance. +The exact review and remaining parent gate appear in the final section. + +Root integrated the distinct graph source-metadata writer repair in394 at45f3a721e. +Singer resumes404 receiving ownership under Tristan's latest explicit correction; +the verified source checkpoint and parent-owned installed/native gate are in +``graph-roi-source-roundtrip-134.rst``. The publication proposal below is still +superseded and must not be reapplied. + +The historical proposal below was qualified against PR394 +``faf61ad87c550fa2d8ef44313344db74b378b857`` before Root's integration. +Original public reproducer, failed controls and completed image scope remain in +``persisted-roi-admission-134.rst``. No shared source or foreign worktree was edited. + +Required relation and exact owners +---------------------------------- + +Every successfully saved artifact destination can publish its existing +result-directory identity even when no image was saved. A declared result +directory is not an image source address or an ROI provenance receipt. The shared +transaction must preserve strict image-address validation while publishing +non-image results without constructing an invalid empty image set. + +The historical combined delta is ``persisted-roi-only-owner-134.patch``; +it superseded the three-hunk proposal, and is now superseded by Root's integrated +implementation. Do not apply either archived proposal over PR394. Admission against selected +original PR394 files passes with ``git apply --check --unidiff-zero`` and the +explicit projection directory. It changes only ``core/steps/function_outputs.py`` +(17 lines removed, 41 added) and ``core/virtual_workspace_metadata.py`` +(6 removed, 7 added). Paths are relative to ``openhcs``. + +* Original ``MaterializationBatch.save`` and + ``SavedMaterializationOutputs.outputs_for_backend`` remain unchanged. Their + successfully saved outputs, not predicted filenames or another render, select + runtime destination directories. The image-only selection predicate is deleted. +* Existing ``OutputTarget`` owns guarded publication participation through + ``contains_outputs`` and the minimal ``stored_output_paths`` hook. The replaced + ``contains_images`` method/calls are removed, not retained as aliases. Image + declarations keep their original image-only path hook. +* ``PersistedResultMetadataCapability`` composes cooperatively with that ancestor + on ``RuntimeArtifactMetadataTarget``. Its ``super().stored_output_paths`` keeps + image participation and includes actual files in the declared saved-result + destination. No extension roster or format switch is added: existing + ``PlateResultFileInventory`` remains the sole format-classification owner. +* Existing runtime destination hooks keep the exact plate-relative results + identity from the first proposal. After runtime values are released, the leaf + calls original ``OpenHCSMetadataHandler.analysis_result_directories`` to obtain + already-declared destinations alongside retained image projections. It does + not decode another results store or fabricate a historical viewer receipt. +* Existing ``AtomicMetadataWriter.publish_source_projection_metadata`` remains + the single transaction. It calls image-set/component projection only when saved + addressed images exist. Missing-image-address rejection, concurrent-step + publication, final pruning and geometry reuse remain intact. + ``SourceProjectionSet`` itself keeps its strict non-empty invariant. + +Catalog review: MEMB-2 identifies image participation mistaken for the entire +saved-result capability; BOUND-2/BOUND-8 require the original saved-outcome and +directory owners across publication; IMPL-12 excludes a second writer or render. +The new nominal hook changes the relation, rather than forwarding a relocated +algorithm. No source-binding/component-axis, capture, renderer, function-registry, +startup, installed-package or scientific-processing change is proposed. Diataxis +reference structure separates exact owners and evidence from remaining acceptance. + +Five additional source cases and new declaration +------------------------------------------------ + +``tests/unit/test_persisted_roi_only_publication.py`` selects original typed +PR394 batch/outcome/target owners through +``docs/validation/check-persisted-roi-only-proposal-134.py``. Seven exact source +files close original imports; only the two proposed owners differ. The existing +receiving runner owns ABI preparation and pytest invocation. This is an explicit +selected-source qualification, not an installed package or ordinary main-tree +test pass. Source blobs and selection are retained in the archive. + +The real declared ``FileBundleOptions`` writer and disk backend save synthetic +archive-admission bytes once. Input mutation plus a test refusal hook proves +publication cannot render that batch again. No actual ROI geometry or pixel +processing is performed. Simple and unrelated nested ROI-only destinations pass +through original registry discovery, shared step publication, public typed +``query_files(kind=result, include_previews=false)`` and completed-plate +reconciliation after saved outcomes are gone. + +An independent ``IndependentSavedResultMetadataTarget`` supplies only its own +destination declaration. Its declared MRO composes publication audit, existing +``ProducedImageMetadataCapability``, runtime result capability and original +``OutputTarget``. Cooperative before/after write hooks and real original record +and storage hooks execute; public inventory admits the exact synthetic ROI. +No generic consumer or registry roster edit is required for the new case. +Additional controls preserve refusal of an unaddressed saved image with the +metadata document byte-exact unchanged, and prove filesystem artefacts without +successful batch outcomes cannot create step publication targets. + +Original PR394 control: 4 failures / 1 pass, 5.55s elapsed, 235884 KiB maximum +process RSS. ROI-only targets are omitted, direct publication rejects the empty +image set, and the new declaration cannot participate. Final proposed source: +5 passes, 4.50s elapsed, 236380 KiB. All source checks are serial one-CPU, +kernel512MiB/no-swap/60s/provider-free/plugin-free, distinct from parent native +and biological acceptance. Existing first-proposal five passes were not rerun as +a ceremony. Global NRA/R1 completion is not asserted. + +Retained corrections and resource incident +------------------------------------------ + +The initial five-file selection failed collection because an original main +worker imported a function removed by PR394. Selecting PR394's two original +orchestrator import owners closed that boundary; no compatibility alias was +added. The first projected suite then hit the unchanged 60s limit, exit124, +304412 KiB process RSS. Its default public-service file-manager factory invoked +``ensure_storage_registry`` and created an optional ImageJ bootstrap cache. +That attempted download was unintended and violates the requested download-free +tier; it is not relabelled as a passing source check. No native viewer startup, +installed package edit or parent endpoint mutation occurred. + +The receiving fixture now injects the real already-created disk-only manager +through the service's original factory boundary, uses the declared OpenHCS +metadata handler, and fails if optional backend bootstrap is attempted. The next +2-failure/3-pass log is retained: its compiled fixture had left +``create_openhcs_metadata=False``, so the existing default-off contract correctly +did not publish. Enabling the explicitly requested compiled publication contract +produced the final five passes without changing assertions or the product guard. + +The bootstrap failure archive is byte-exact and excludes dependency cache bytes: +``persisted-roi-only-bootstrap-failure-134-source-20261001.tar.gz``, SHA256 +``cb1aa2043cd32c333dcb0144d0d2396482e596bd98fcbae5d04c6b3120151cb5``. +After archive comparison and exact source-process termination, only +``/home/ts/wt/openhcs-knowledge-lazy-conversion-20261001/.qa134-roi-only-20261001/cache/polystore/imagej`` +was removed: 1508931494 logical bytes, reported 1.5GiB. The incomplete dependency +ZIP/extraction are disposable, not scientific or unknown-input data. Original +timeout, input, size evidence and selected code remain retained. + +Combined exact original R0 and archive +-------------------------------------- + +The same unmodified pinned R0 owner qualified the entire two-file proposed delta +against PR394 ``faf61ad87c550fa2d8ef44313344db74b378b857``. Original guard owner: +``/home/ts/wt/comms-ratchet-pinned-ui348-20261001`` at +``3b03785f45df2ef5dc62ba6aed99294192ecbb01``. Command uses Python3.14 and +``python -m agent_comms.debt_ratchet --root openhcs --base faf61ad87c550fa2d8ef44313344db74b378b857 --head e960fe80e396e1c43447aa9990e133c985f22b3d``. + +Synthetic proposed commit ``e960fe80e396e1c43447aa9990e133c985f22b3d``, tree +``8ab6dc59fe9c23122cbdea3f35da2de0c2e474ad``; no live ref or working index was +changed. Proposed source blobs are ``d44e511e9711cde0b1c8d3fad93b00d8ddad7de4`` +and ``63ce7c84ff2b7807d00d09e5b93168caf94e2050`` respectively. +R0 exit0: every delta zero, both changed production files included, 18.46s, +87772 KiB maximum process RSS, same kernel512MiB/one-CPU/60s bounds. No detector +copy, omitted changed file, increased threshold or waived positive delta. + +Final byte-exact archive +``persisted-roi-only-owner-134-source-20261001.tar.gz`` retains all raw logs, +input versions, synthetic failed fixtures, original/proposed selected source, +alternate index and complete R0. SHA256 +``0d3c6c02e153f8ad35acccf9f07de0808a585379739658757fe4f22d258ffbbd``. +``tar -d`` passes before release of the remaining owned projection/cache root: +``/home/ts/wt/openhcs-knowledge-lazy-conversion-20261001/.qa134-roi-only-20261001``, +3048192 logical bytes, 3.5MiB filesystem usage. Original evidence is recoverable; +source, durable history, Dalton outputs and all foreign worktrees remain intact. + +The former shared-owner wait is resolved by Root's integration below. Public +native ROI reopen/provenance acceptance remains outstanding. Saved image acceptance399, +closed398 scope, snapshot writer/binding ownership, parent viewer guides, +Schrodinger376 and Dewey379 remain untouched. No biological pass is inferred. + +Receiving acceptance of Root's integrated source checkpoint +---------------------------------------------------------- + +Read-only review inspected the exact commit diff, its full new behavioral test, +the PR394 ownership statement and +https://github.com/OpenHCSDev/openhcs/blob/82243634ce097d0d1a2d9e77664cbfeac3e6ec94/docs/validation/persisted_result_publication_134_20261002.rst. +Commit parent is ``4afd5141554299680fa0326b31334a448fbf20bf``. Actual production +delta: ``core/steps/function_outputs.py`` removes16/adds36 lines; +``core/virtual_workspace_metadata.py`` removes6/adds7. No source edit, native +operation, installation, test replay or cleanup was performed for this review. + +The integrated relation meets the original receiving requirements: + +* Existing ``OutputTarget`` owns guarded ``contains_outputs`` and the + ``stored_output_paths`` hook. Existing ``RuntimeArtifactMetadataTarget`` supplies + its minimal all-saved-format hook. Root correctly omitted the proposed separate + capability class: this production behavior has one existing leaf owner; no + independent overlapping capability needs another class or duplicate listing. +* Actual saved ``MaterializationBatch`` outcomes select destinations, without + re-rendering. Generic finalization calls the nominal participation contract. + The image-only predicate and replaced method disappear; no compatibility alias, + consumer switch, result roster or parallel authority is added. +* Exact nested results identities and original metadata-handler reconciliation + survive runtime-value release. The original atomic publisher avoids an empty + image-set construction while retaining actual unaddressed-image rejection. + Empty raster inventory does not become a fabricated image source projection. +* Root ported the five receiving behavior cases. The independent declaration + composes audit, original produced-image capability and runtime target through + cooperative ``super().write``. Its own ``declaration_key=None`` obtains a new + derived registry key; complete registry restoration fixes the original test's + inherited-key contamination rather than weakening later consumers. Public + admission and before/after hooks are exercised, not inheritance assertions. + The NumPy-only control now expects a declared result directory with no raster + records; mixed raster pixel equality remains asserted. + +Root's retained receipt reports original four failures/one pass and89 passing +publication/checkpoint/metadata/journey controls on the integrated source. PR394 +reports isolated original R0 roots and R1 passing against its ``4afd51415`` parent. +These are Root's execution receipts, not new receiving runs or a global/native +acceptance claim. Original test-path, registry-contamination and source failures +remain retained by Root. Catalog checks MEMB-2, BOUND-2/BOUND-8 and IMPL-12 support +the accepted owner placement; no run-obstructing ownership violation was found in +this two-owner delta. No unrelated architecture audit is requested. + +Draft404 disposition and remaining parent native gate +---------------------------------------------------- + +Production implementation is superseded by PR394 under Root. Do not merge404's +historical patch/test copies over the registry-safe integrated implementation. +Keep404 draft as the receiving evidence and narrowly pending native-acceptance +record while134 remains open. Original failures and archives remain unchanged. +Close or consolidate the draft with Root after the parent records the installed +gate; source integration itself is not waiting on that later gate. + +After current freezes end, parent owns the authorised installed entrypoint check: + +* Pin an installed source including the accepted publication checkpoint and its + explicit owned MCP/native endpoint. Use a newly authorised correctly published + saved-result checkpoint. Ordinary public ``kind=result`` inventory must admit + the exact ROI archive from durable declared metadata, without an explicit-path + bypass, fabricated source receipt or silent backfill of the older candidate. +* Reopen that exact materialised archive through the original public viewer route. + The actual native ROI decoder/source-metadata boundary must accept its existing + source provenance, selected source channel, plane/spatial domain and calibration. + Preserve strict rejection if those facts are absent; directory admission alone + does not repair or prove that separate ROI/graph source contract. +* Retain the original request/reply, native readback and same-coordinate source/ + ROI placement evidence. Distinguish real archive geometry/reopen from synthetic + filename admission. Preserve old failed jobs without replay and keep rejected + biological candidates rejected; no segmentation quality or biological pass is + inferred from an engineering reopen. + +This names the remaining acceptance boundary, not authority to modify the frozen +environment, generate another scientific run or claim the broader graph134 fixed. diff --git a/docs/validation/persisted-roi-owner-proposal-134-source-20261001.tar.gz b/docs/validation/persisted-roi-owner-proposal-134-source-20261001.tar.gz new file mode 100644 index 0000000000..4bf4777a85 Binary files /dev/null and b/docs/validation/persisted-roi-owner-proposal-134-source-20261001.tar.gz differ diff --git a/tests/unit/test_graph_roi_source_roundtrip_134.py b/tests/unit/test_graph_roi_source_roundtrip_134.py new file mode 100644 index 0000000000..83f0b9b478 --- /dev/null +++ b/tests/unit/test_graph_roi_source_roundtrip_134.py @@ -0,0 +1,272 @@ +"""Receiving #134: actual graph context/writer/ZIP, without a native process.""" + +from pathlib import Path + +import numpy as np +import pytest +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.roi import load_rois_from_zip + +from openhcs.core.artifacts import ( + ArtifactOutputPlan, + ArtifactSpec, + ObjectArtifactMemberSubjectRelation, + ObjectArtifactSubjectBinding, + ObjectLabelsArtifactType, + SpatialGraphArtifactType, +) +from openhcs.core.config import NapariStreamingConfig +from openhcs.core.roi_source_metadata import ROIArchiveSourceMetadata +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.runtime_plane_projection import RuntimePlaneAxis +from openhcs.core.runtime_spatial_graph import ( + SpatialGraph, + SpatialGraphEdge, + SpatialGraphNode, +) +from openhcs.core.source_image_provenance import SourceImageProvenancePlanes +from openhcs.core.source_metadata import SourceVoxelSpacing +from openhcs.core.steps.function_runtime import FunctionOutputContextStrategy +from openhcs.core.viewer_streaming_service import RoiStreamingRequest, StreamingService +from openhcs.processing.materialization import ( + MaterializationSpec, + SpatialGraphROIOptions, + materialization_outputs, + materialize, +) + + +@pytest.fixture(autouse=True) +def no_optional_storage_bootstrap(monkeypatch): + import polystore.base + + monkeypatch.setattr( + polystore.base, + "ensure_storage_registry", + lambda: pytest.fail("Provider/bootstrap execution is forbidden in this test"), + ) + + +class ProjectionAudit: + """Independent capability exercises cooperative construction and projection.""" + + def __init__(self, **kwargs): + self.projection_events = [] + super().__init__(**kwargs) + + def contextualized_source_provenance(self, provenance): + self.projection_events.append("before") + result = super().contextualized_source_provenance(provenance) + self.projection_events.append("after") + return result + + +class AuditedGraph(ProjectionAudit, SpatialGraph): + """New declaration; no writer, context strategy or reader registration.""" + + +def graph_and_plan(*, plane, graph_type=SpatialGraph, extra_features=None): + nodes = tuple( + SpatialGraphNode(index + 1, coordinate) + for index, coordinate in enumerate(((2.25, 3.5), (5.25, 9.5), (7.25, 12.5))) + ) + edges = tuple( + SpatialGraphEdge.from_features( + edge_id=index + 1, + source=nodes[index], + target=nodes[index + 1], + coordinates=np.asarray( + (nodes[index].coordinates, nodes[index + 1].coordinates) + ), + features={ + "neuron_label": 7, + "branch_distance_um": 8.5, + **(extra_features or {}), + }, + ) + for index in range(2) + ) + graph = graph_type( + name="declared_graph", + nodes=nodes, + edges=edges, + coordinate_spacing=(1.3556, 1.3556), + source_plane_index=plane, + ) + neurons = ArtifactSpec.output("neurons", ObjectLabelsArtifactType) + plan = ArtifactOutputPlan( + name=graph.name, + path="/engineering/graph.pkl", + artifact_type=SpatialGraphArtifactType, + relations=( + ObjectArtifactMemberSubjectRelation( + source=neurons.ref(), member_id_field="neuron_label" + ), + ), + producer_step_index=4, + producer_step_scope_id="engineering-step", + ) + return graph, plan + + +def contextualize(graph, plan): + metadata = ImagePayloadMetadata( + plane_axis=RuntimePlaneAxis.RUNTIME_SLICE, + source_image_provenance_planes=SourceImageProvenancePlanes.from_components( + paths=("/engineering/nuclear.tif", "/engineering/body.tif"), + component_metadata=tuple( + { + "well": "A01", + "site": 1, + "channel": channel, + "z_index": 1, + "timepoint": 1, + } + for channel in (1, 2) + ), + ), + source_image_names=("nuclear", "body"), + source_voxel_spacing=SourceVoxelSpacing(graph.coordinate_spacing), + ) + source = metadata.payload_with(np.zeros((2, 16, 16), dtype=np.uint8)) + result = FunctionOutputContextStrategy.for_output_plan(plan).contextualize( + source, graph, plan, None + ) + assert result.source_provenance == metadata.source_provenance.for_source_plane( + graph.source_plane_index + ) + return result + + +def save_graph(tmp_path, graph, plan): + manager = FileManager({"disk": DiskStorageBackend()}) + path = materialize( + MaterializationSpec(SpatialGraphROIOptions()), + data=graph, + path=str(tmp_path / "misleading_w99_z999.roi.zip"), + filemanager=manager, + backends=["disk"], + backend_kwargs={}, + output_plan=plan, + ) + return Path(path), load_rois_from_zip(Path(path)) + + +@pytest.mark.parametrize("plane", [0, 1]) +def test_declared_source_plane_survives_graph_writer_zip_and_subject_features( + tmp_path, plane +): + graph, plan = graph_and_plan(plane=plane) + graph = contextualize(graph, plan) + (output,) = materialization_outputs( + MaterializationSpec(SpatialGraphROIOptions()), + data=graph, + path=str(tmp_path / "graph.roi.zip"), + filemanager=FileManager({"disk": DiskStorageBackend()}), + output_plan=plan, + ) + path, restored = save_graph(tmp_path, graph, plan) + metadata = ROIArchiveSourceMetadata.decode(restored) + assert metadata is not None, "Native graph ZIP lost its existing source declaration" + assert output.metadata.source_provenance == graph.source_provenance + assert metadata.source_provenance == graph.source_provenance + assert ( + metadata.source_path + == ("/engineering/nuclear.tif", "/engineering/body.tif")[plane] + ) + assert metadata.source_component_metadata["channel"] == plane + 1 + assert metadata.source_voxel_spacing == SourceVoxelSpacing(graph.coordinate_spacing) + assert metadata.plane_axis is None + assert metadata.source_spatial_domain == output.metadata.source_spatial_domain + assert path.name == "misleading_w99_z999.graph.roi.zip" + geometry = ROIArchiveSourceMetadata.geometry(restored) + assert len(geometry) == len(graph.edges) + for roi, edge in zip(geometry, graph.edges, strict=True): + np.testing.assert_array_equal(roi.shapes[0].coordinates, edge.coordinates) + assert roi.metadata["edge_id"] == edge.edge_id + assert roi.metadata["source_node_id"] == edge.source_node_id + assert roi.metadata["target_node_id"] == edge.target_node_id + assert roi.metadata["neuron_label"] == 7 + assert roi.metadata["branch_distance_um"] == 8.5 + assert roi.metadata[ObjectArtifactSubjectBinding.SUBJECT_ID_FEATURE] == 7 + assert ( + '"object_labels","neurons","engineering-step",4' + in roi.metadata[ObjectArtifactSubjectBinding.SUBJECT_FEATURE] + ) + assert ROIArchiveSourceMetadata.FIELD not in roi.metadata + assert ( + len( + { + roi.metadata[ObjectArtifactSubjectBinding.SUBJECT_FEATURE] + for roi in geometry + } + ) + == 1 + ) + + +def test_independent_graph_capability_and_new_feature_need_no_consumers(tmp_path): + original, plan = graph_and_plan( + plane=1, + graph_type=AuditedGraph, + extra_features={"engineering_confidence": 0.375}, + ) + graph = contextualize(original, plan) + assert original.projection_events == ["before", "after"] + assert ( + graph.projection_events == [] + ) # cooperative constructor for contextualized copy + _, restored = save_graph(tmp_path, graph, plan) + metadata = ROIArchiveSourceMetadata.decode(restored) + assert metadata is not None, "New graph declaration lost its source declaration" + assert metadata.source_provenance == graph.source_provenance + assert all(roi.metadata["engineering_confidence"] == 0.375 for roi in restored) + assert all( + ROIArchiveSourceMetadata.FIELD not in roi.metadata + for roi in ROIArchiveSourceMetadata.geometry(restored) + ) + + +@pytest.mark.parametrize("invalid", ["absent", "unbound", "mixed", "conflicting"]) +def test_original_native_admission_refuses_bad_graph_sources_before_transport( + tmp_path, invalid +): + graph, plan = graph_and_plan(plane=1) + graph = contextualize(graph, plan) + path, rois = save_graph(tmp_path, graph, plan) + geometry = ROIArchiveSourceMetadata.geometry(rois) + metadata = ImagePayloadMetadata(source_provenance=graph.source_provenance) + bad_archives = { + "absent": geometry, + "unbound": ROIArchiveSourceMetadata.bind(geometry, ImagePayloadMetadata()), + "mixed": [ + ROIArchiveSourceMetadata.bind(geometry[:1], metadata)[0], + geometry[1], + ], + "conflicting": [ + ROIArchiveSourceMetadata.bind(geometry[:1], metadata)[0], + ROIArchiveSourceMetadata.bind( + geometry[1:], + metadata.replace_fields(source_path="/engineering/other.tif"), + )[0], + ], + } + DiskStorageBackend().save(bad_archives[invalid], path) + # No viewer/handler object: original source admission must reject before using either. + service = StreamingService( + FileManager({"disk": DiskStorageBackend()}), None, tmp_path + ) + with pytest.raises( + ValueError, match="Native ROI source metadata|missing or conflicting" + ): + service.stream_rois( + RoiStreamingRequest( + viewer=None, + config=NapariStreamingConfig(enabled=True), + status_callback=lambda _: None, + error_callback=lambda error: pytest.fail(error), + roi_filenames=(path.name,), + require_source_metadata=True, + ) + ) diff --git a/tests/unit/test_persisted_result_directory_publication.py b/tests/unit/test_persisted_result_directory_publication.py new file mode 100644 index 0000000000..3f9628ad4a --- /dev/null +++ b/tests/unit/test_persisted_result_directory_publication.py @@ -0,0 +1,161 @@ +"""Original metadata writer/reader contracts, with no pixel or ROI content reads.""" + +from types import SimpleNamespace + +import pytest +from polystore.disk import DiskStorageBackend +from polystore.filemanager import FileManager +from polystore.virtual_workspace import SourcePixelRef + +from openhcs.agent.dto.plate import PlateFileQueryRequest +from openhcs.agent.path_policy import AgentPathPolicy +from openhcs.agent.services.plate_inspection_service import PlateInspectionService +from openhcs.constants.constants import Backend +from openhcs.core.artifacts import ObjectLabelsArtifactType +from openhcs.core.compiled_step_plan import CompiledStepPlan, RuntimeArtifactMaterializationPlan +from openhcs.core.context.processing_context import ProcessingContext +from openhcs.core.runtime_image_values import ImagePayloadMetadata +from openhcs.core.source_projection import ( + OpenHCSPlaneAddress, + SourceArtifactProjection, + SourceProjectionMetadataSerializer, +) +from openhcs.core.steps.function_outputs import ( + OpenHCSMetadataWriter, + ProducedImageMetadataCapability, + RuntimeArtifactMetadataTarget, +) +from openhcs.core.virtual_workspace_metadata import AtomicMetadataWriter, METADATA_CONFIG +from openhcs.microscopes.source_schema import SourceSchemaFilenameParser + + +@pytest.fixture +def publication_context(tmp_path): + filemanager = FileManager({Backend.DISK.value: DiskStorageBackend()}) + context = ProcessingContext(filemanager=filemanager) + context.metadata_cache = {} + context.microscope_handler = SimpleNamespace( + parser=SourceSchemaFilenameParser(), microscope_type="openhcsdata" + ) + context.publication_audit = [] + return context + + +def seed_synthetic_projection(plate, directory): + directory.mkdir(parents=True) + roi = directory / "independent-outline.roi.zip" + roi.write_bytes(b"path-admission sentinel, never decoded") + image = directory / "independent-labels.tif" + image.write_bytes(b"path-admission sentinel, never loaded") + path = str(image.relative_to(plate)) + components = { + "well": "A01", "site": "1", "channel": "2", + "z_index": "1", "timepoint": "1", "pixel_size": 1.25, + } + projection = SourceArtifactProjection( + address=OpenHCSPlaneAddress.from_values("A01", "1", "2", "1", "1"), + ref=SourcePixelRef(Backend.DISK.value, path), + source_alias="independent labels", + artifact_kind=ObjectLabelsArtifactType, + source_metadata=components, + image_metadata=ImagePayloadMetadata(source_component_metadata=components), + ) + AtomicMetadataWriter().merge_source_projection_metadata( + METADATA_CONFIG.metadata_path(plate), + str(directory.relative_to(plate)), + SourceProjectionMetadataSerializer(SourceSchemaFilenameParser()).projection_fields( + ((projection, path),) + ), + ) + return roi + + +class PublicationAuditCapability: + """Independent audit capability extends, rather than replaces, publication.""" + + def write(self, context, *, produced_plan=None): + context.publication_audit.append(("before", self.sub_dir)) + super().write(context, produced_plan=produced_plan) + context.publication_audit.append(("after", self.sub_dir)) + + +@pytest.mark.parametrize("destination", ("supplemental", "nested/supplemental")) +def test_new_declared_target_composes_original_capability_and_writer( + tmp_path, publication_context, destination +): + plate = tmp_path / "plate" + directory = plate / destination + roi = seed_synthetic_projection(plate, directory) + registry = OpenHCSMetadataWriter.OutputTarget.__registry__ + original_keys = set(registry) + try: + class SupplementalResultMetadataTarget( + PublicationAuditCapability, + ProducedImageMetadataCapability, + OpenHCSMetadataWriter.OutputTarget, + ): + @classmethod + def from_plan(cls, plan): + return cls( + output_dir=directory, + backend=Backend.DISK.value, + plate_root=str(plate), + sub_dir=destination, + results_dir=str(directory), + ) + + plan = CompiledStepPlan( + step_index=0, step_name="independent declaration", step_type="FunctionStep", + axis_id="A01", write_backend=Backend.MEMORY.value, + ) + # Discovery stays the original family algorithm. No roster or consumer + # edit: the new declaration supplies only its destination hook. + (target,) = OpenHCSMetadataWriter.OutputTarget.for_execution(publication_context, plan) + target.write(publication_context, produced_plan=plan) + assert publication_context.publication_audit == [ + ("before", destination), ("after", destination) + ] + service = PlateInspectionService( + AgentPathPolicy.with_roots(readable_roots=(plate,), writable_roots=()) + ) + result = service.query_files( + PlateFileQueryRequest.from_fields( + plate_path=str(plate), kind="result", include_previews=False + ) + ) + assert result.errors == () + assert str(roi) in tuple(record.full_path for record in result.records) + assert all(record.preview is None for record in result.records) + finally: + for key in set(registry) - original_keys: + del registry[key] + + +def test_runtime_directory_projection_preserves_exact_new_destination( + tmp_path, publication_context +): + plate = tmp_path / "plate" + directory = plate / "separate/nested/declared_destination" + roi = seed_synthetic_projection(plate, directory) + plan = CompiledStepPlan( + step_index=0, step_name="independent directory projection", step_type="FunctionStep", + axis_id="A01", output_plate_root=str(plate), + analysis_results_dir=str(plate / "original_destination"), + runtime_artifact_materialization=RuntimeArtifactMaterializationPlan( + persistent_enabled=True, persistent_backend=Backend.DISK.value + ), + ) + owner = RuntimeArtifactMetadataTarget.from_plan(plan) + assert owner is not None + target = owner.for_directory(directory) + target.write(publication_context) + service = PlateInspectionService( + AgentPathPolicy.with_roots(readable_roots=(plate,), writable_roots=()) + ) + result = service.query_files( + PlateFileQueryRequest.from_fields( + plate_path=str(plate), kind="result", include_previews=False + ) + ) + assert result.errors == () + assert str(roi) in tuple(record.full_path for record in result.records) diff --git a/tests/unit/test_persisted_roi_only_publication.py b/tests/unit/test_persisted_roi_only_publication.py new file mode 100644 index 0000000000..4755bfa835 --- /dev/null +++ b/tests/unit/test_persisted_roi_only_publication.py @@ -0,0 +1,207 @@ +"""Saved-output publication only: synthetic bytes, no ROI or pixel decoding.""" + +from dataclasses import replace +import json + +import pytest + +from openhcs.agent.dto.plate import PlateFileQueryRequest +from openhcs.agent.path_policy import AgentPathPolicy +from openhcs.agent.services.plate_inspection_service import ( + PlateInspectionFileManagerFactory, PlateInspectionService, +) +from openhcs.constants.constants import Backend +from openhcs.core.artifacts import ArtifactOutputPlan, MetadataArtifactType +from openhcs.core.compiled_step_plan import CompiledStepPlan, RuntimeArtifactMaterializationPlan +from openhcs.core.component_group_scope import RuntimeExecutionAxisScope +from openhcs.core.runtime_artifact_values import RuntimeValue +from openhcs.core.runtime_stores import RuntimeArtifactLocation, StoredRuntimeValue +from openhcs.core.steps.function_artifact_materialization import ( + MaterializedRuntimeArtifact, RuntimeArtifactMaterialization, +) +from openhcs.core.steps.function_outputs import ( + OpenHCSMetadataWriter, ProducedImageMetadataCapability, RuntimeArtifactMetadataTarget, +) +from openhcs.core.virtual_workspace_metadata import METADATA_CONFIG, MetadataWriteError +from openhcs.processing.materialization.core import ( + MaterializationSpec, _WRITERS_BY_OPTIONS, prepare_materialization, +) +from openhcs.processing.materialization.options import FileBundleOptions + +from test_persisted_result_directory_publication import ( + PublicationAuditCapability, publication_context, +) + + +def plan_for(plate): + return CompiledStepPlan( + step_index=0, step_name="synthetic saved result", step_type="FunctionStep", + axis_id="A01", write_backend=Backend.MEMORY.value, create_openhcs_metadata=True, + output_plate_root=str(plate), analysis_results_dir=str(plate / "compiled_default"), + runtime_artifact_materialization=RuntimeArtifactMaterializationPlan( + persistent_enabled=True, persistent_backend=Backend.DISK.value, + ), + ) + + +def save_archive_outcome(context, plate, destination, monkeypatch): + """Use the original declared bundle writer, batch and disk save boundary.""" + spec = MaterializationSpec(FileBundleOptions()) + payload = {f"{destination}/independent-outline.roi.zip": b"synthetic admission only"} + base_path = plate / "bundle" + batch = prepare_materialization( + spec, payload, str(base_path), context.filemanager, (Backend.DISK.value,), + ) + saved = batch.save() + output, = saved.outputs_for_backend(Backend.DISK.value) + assert output is batch.outputs[0] + # Source qualification must consume the original successful save, not render + # a mutable logical payload again or use planned names as proof of a save. + payload.clear() + + def forbidden_render(*args, **kwargs): + pytest.fail("Publication rendered the already-saved batch again") + + writer = _WRITERS_BY_OPTIONS[FileBundleOptions] + monkeypatch.setitem(_WRITERS_BY_OPTIONS, FileBundleOptions, replace(writer, write=forbidden_render)) + artifact_plan = ArtifactOutputPlan( + name="synthetic archive bundle", path="/memory/bundle.pkl", + artifact_type=MetadataArtifactType, materialization=spec, + ) + record = StoredRuntimeValue( + value=RuntimeValue.from_output_plan( + artifact_plan, {}, execution_scope=RuntimeExecutionAxisScope(axis_id="A01"), + ), + location=RuntimeArtifactLocation(path=artifact_plan.path, backend=Backend.MEMORY.value), + ) + materialization = RuntimeArtifactMaterialization( + output_plan=artifact_plan, spec=spec, record=record, data=payload, + base_path=base_path, source_identity=None, filename_source_identity=None, + ) + return MaterializedRuntimeArtifact( + outputs_by_backend=saved.outputs_by_backend, materialization=materialization, + ), output + + +class ExistingDiskInspectionFactory(PlateInspectionFileManagerFactory): + """Original public service receives the same real disk manager as saving.""" + + def __init__(self, manager): + self.manager = manager + + def create(self): + return self.manager + + +@pytest.fixture(autouse=True) +def forbid_optional_backend_bootstrap(monkeypatch): + import polystore.base + + def forbidden_bootstrap(*args, **kwargs): + pytest.fail("Source-only qualification must not bootstrap optional backends") + + monkeypatch.setattr(polystore.base, "ensure_storage_registry", forbidden_bootstrap) + + +def result_paths(plate, context): + service = PlateInspectionService( + AgentPathPolicy.with_roots(readable_roots=(plate,), writable_roots=()), + filemanager_factory=ExistingDiskInspectionFactory(context.filemanager), + ) + result = service.query_files(PlateFileQueryRequest.from_fields( + plate_path=str(plate), microscope_type="openhcsdata", + kind="result", include_previews=False, + )) + assert result.errors == () + assert all(record.preview is None for record in result.records) + return tuple(record.full_path for record in result.records) + + +@pytest.mark.parametrize("destination", ("roi_only", "independent/nested/roi_only")) +def test_saved_roi_only_batch_publishes_and_reconciles_without_rendering( + tmp_path, publication_context, monkeypatch, destination, +): + plate = tmp_path / "plate" + outcome, output = save_archive_outcome(publication_context, plate, destination, monkeypatch) + plan = plan_for(plate) + targets = OpenHCSMetadataWriter.OutputTarget.for_execution( + publication_context, plan, artifact_materializations=(outcome,), + ) + assert tuple(target.output_dir for target in targets) == (plate / destination,) + OpenHCSMetadataWriter.write(publication_context, plan, artifact_materializations=(outcome,)) + assert result_paths(plate, publication_context) == (output.path,) + published = json.loads(METADATA_CONFIG.metadata_path(plate).read_text()) + assert published["subdirectories"][destination]["image_files"] == [] + assert published["subdirectories"][destination]["source_projection"] == [] + # Final reconciliation has no runtime outcome/value to reconstruct. Its + # destination comes from the same durable declaration the reader admitted. + publication_context.step_plans = {0: plan} + owner = RuntimeArtifactMetadataTarget.from_plan(plan) + reconciled = tuple(target.output_dir for target in owner.reconciliation_targets(publication_context)) + assert plate / destination in reconciled + OpenHCSMetadataWriter.finalize_completed_plate({"A01": publication_context}) + assert result_paths(plate, publication_context) == (output.path,) + + +def test_shared_writer_accepts_result_only_but_not_unaddressed_images( + tmp_path, publication_context, +): + plate = tmp_path / "plate" + directory = plate / "standalone_result" + directory.mkdir(parents=True) + roi = directory / "independent-outline.roi.zip" + roi.write_bytes(b"synthetic admission only") + target = RuntimeArtifactMetadataTarget.from_plan(plan_for(plate)).for_directory(directory) + target.write(publication_context) + assert result_paths(plate, publication_context) == (str(roi),) + metadata_path = METADATA_CONFIG.metadata_path(plate) + before = metadata_path.read_bytes() + (directory / "unaddressed-image.tif").write_bytes(b"never decoded") + with pytest.raises(MetadataWriteError, match="Saved images lack typed produced addresses"): + target.write(publication_context) + assert metadata_path.read_bytes() == before + + +def test_new_result_declaration_composes_cooperative_hooks_without_consumer_edits( + tmp_path, publication_context, monkeypatch, +): + plate = tmp_path / "plate" + destination = "new/declaration_only" + outcome, output = save_archive_outcome(publication_context, plate, destination, monkeypatch) + registry = OpenHCSMetadataWriter.OutputTarget.__registry__ + original_keys = set(registry) + try: + class IndependentSavedResultMetadataTarget( + PublicationAuditCapability, ProducedImageMetadataCapability, + RuntimeArtifactMetadataTarget, + ): + @classmethod + def from_plan(cls, plan): + return cls( + output_dir=plate / destination, backend=Backend.DISK.value, + plate_root=str(plate), sub_dir=destination, + results_dir=str(plate / destination), + ) + + plan = replace(plan_for(plate), runtime_artifact_materialization=RuntimeArtifactMaterializationPlan()) + target, = OpenHCSMetadataWriter.OutputTarget.for_execution( + publication_context, plan, artifact_materializations=(outcome,), + ) + target.write(publication_context, produced_plan=plan) + assert publication_context.publication_audit == [("before", destination), ("after", destination)] + assert result_paths(plate, publication_context) == (output.path,) + finally: + for key in set(registry) - original_keys: + del registry[key] + + +def test_files_without_successful_batch_outcomes_do_not_create_step_targets( + tmp_path, publication_context, +): + plate = tmp_path / "plate" + directory = plate / "unclaimed" + directory.mkdir(parents=True) + (directory / "not_a_saved_outcome.roi.zip").write_bytes(b"synthetic admission only") + assert OpenHCSMetadataWriter.OutputTarget.for_execution( + publication_context, plan_for(plate), artifact_materializations=(), + ) == ()