diff --git a/docs/validation/viewer-retirement-bootstrap-delivery-20261003.rst b/docs/validation/viewer-retirement-bootstrap-delivery-20261003.rst new file mode 100644 index 0000000000..10d2124291 --- /dev/null +++ b/docs/validation/viewer-retirement-bootstrap-delivery-20261003.rst @@ -0,0 +1,99 @@ +Main-only retirement and bootstrap delivery +========================================== + +Fixes #528. Delivers only the independently qualified retirement/bootstrap +portion of #522; surviving geometry selection remains in that original PR. + +This delivery normally merges Singer's published pre-selection checkpoint +4414159b62ae92a901e37861cbe85856ae48f750 into main +f910830954646b4c38c6ff46ec0a2f78b1d3addb. It contains the original native route +retirement owner, cooperative presentation retention, generated MCP/CLI request, +and the nine-field StreamProducerIdentity admission/schema repair. No Root394 +compiler/materialization ancestry, selected-member preservation, or fractional +X/Y navigation changes are imported. Those remain with their existing owners. + +Ownership and retained evidence +------------------------------- + +Singer owns the remaining PR522 selection family. Planck delivers this existing +accepted subset with Singer informed; there is no competing implementation. +Root owns the remaining aggregate source-domain producer family in PR394/435. +On current Root c59eaa55, the point writer still returns generic Output and the +stream authority still has a scalar fallback. This delivery does not fix or +claim that producer gap, nor merge the private ancestry of PR494. + +The original full family census and source/installed controls are retained in +viewer-selective-retirement-519-20261003.rst and its three evidence archives. +They cover the original request/result MRO, capability/CLI/generated codec, +native control queue, route coordinator, settlement/pending callbacks, and image +presentation retention. The complete schema census covers 704 production plus +38 introspection/PolyStore dependency modules without parse omissions; the +native family additionally retained its Qt/Vispy source projections. Dynamic +MRO/events were read semantically; AST is not a behavioral proof. + +Actual installed/public acceptance predates this source integration. The +immutable target06 uses the equivalent schema/decoder owner checkpoints +91d449a71/d92fd0b8c. Parent's closed standalone95-live30/ACTUAL-RECEIVING.rst +records actual public TCP retirement of precisely one Points route using its +complete producer identity. The raw survivor retained exact payload summaries, +Z domain 0..3, ZYX calibration 2/.65/.65 and camera. Reopening the unchanged +989-byte original archive was a distinct successful public transaction. Parent +personally opened all nine matched XY/XZ/YZ raw/point/combined PNGs. Original +typed close and PID/endpoint absence were verified. This does not prove +selected_data preservation for a surviving Points/Shapes layer or automatic +producer publication. Those gaps remain explicit in PR522/494. + +The complete original journals remain under +/home/ts/wt/openhcs-issue-batch-20260929/engineering494/standalone95-live30. +Closed journal SHA256: stdin 62acff2fa4e5dbaf4c7a213c8fa8a93c7554fab3e7940e85f1bfac6a6a1288ef, +stdout 1acda7973aee96bcd1bdc3fb6f86c5930a7ca16ff05b270f35ec4474a29340ae, +timing 15e702b268f021d2c30816e85b0eda905250bc5993e5e47450905e90f81bbf36. +The original geometry rejection and client aggregate terminal1 are preserved; +they are not a shutdown failure. No old science job was replayed. + +Current integration qualification +--------------------------------- + +The main integration was conflict-free. Seven foreign dirty gitlinks remain +unstaged and unchanged. Original source-check77 exited1 before fixtures: +the checkout bootstrap preferred its intentionally preserved older foreign +ZMQ checkout, which lacks ViewerReuseAdmissionABC. That failure is retained, +not masked by editing a dependency, import hook, or bootstrap function. + +Distinct source-check78 used an exact git archive of ebacd4904's openhcs package +in a package-layout fixture with the existing paired installed dependencies. +The original source bootstrap still ran, without external checkouts in that +fixture; the dependency origin was recorded. The qualified target07 native ABI +was loaded read-only, without changing any installed package. All29 affected +original SDK/CLI/MCP constructor, Qt queue and retirement/reset model controls +passed in8.20s. MemoryMax512MiB, peak473870336B, SwapMax0, CPUQuota100%, noOOM. +The original process was terminal0 and its scope inactive. No build, native, +viewer, stdio server or scientific job was launched. + +The normal merge preserves current main's receiver524 source domain/calibration +and transport-only feature exclusion. Those are the only production differences +between this delivery's native family and the pre-selection441 checkpoint; +the retirement and original identity/SDK declaration remain unchanged. +New named-family AST evidence uses the original schema-family27 loader against +ebacd4904: 704 production and75 actual paired introspection/PolyStore modules, +zero parse omissions, terminal0; retained +original native-family evidence remains separately qualified. It is not a +global detector/equivalence proof or a new installed/native acceptance claim. + +All nine production files in this delivery were also compared directly with +the retained actual-public target06 wheel. ALL NINE are byte-identical, +including current main's524 receiver/calibration changes. Wheel SHA256: +46070edad4363d406b47ea771d07e795bff5eb268c4fa58d564881b5ec48ca04. +This is direct qualification provenance, not a claim that the rest of current +main is byte-equal to that older package. No rebuild or live rerun is needed to +misrepresent unchanged owner-family behavior as a new acceptance journey. + +Original controls77/78 and family79 records are under engineering494. Source +fixture78 is disposable generated Git source; original journals, qualified +target07 and every scientific artifact remain protected. No scope for +retirement of a surviving selected geometry result is claimed here. + +Closed evidence SHA256: +controls77 a9c0abadf4a3adeb2c0dab9591c2438e1de81c262f4d93f80df0191a3da31f6d; +controls78 8ff76de747aefbd0979c8f2b5d674bd93cc91515b6bafa7804bcf9c1852182a7; +family79 5b8d3cd03ed238467f8fbacc5def1e98ea3c1d7023404615f89b0f82c37f8efc. diff --git a/docs/validation/viewer-selective-retirement-519-20261003.rst b/docs/validation/viewer-selective-retirement-519-20261003.rst new file mode 100644 index 0000000000..d90f83957f --- /dev/null +++ b/docs/validation/viewer-selective-retirement-519-20261003.rst @@ -0,0 +1,445 @@ +Selective managed-viewer retirement519: original-owner source checkpoint +====================================================================== + +Scope and custody +----------------- + +Actual MCP construction correction received (2026-10-03) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Parent's frozen combined target04 actual public MCP startup failed RecursionError +at build_server before any native/viewer/scientific execution. The original +next-metaxpress-p001-independent95-revision02-20261003/METAXPRESS_P001_INDEPENDENT95/ +author-workspace/output/runtime/mcp.stdout and health/freeze remain unchanged. +Earlier33 source lifecycle and49 installed receiver controls did not cover real +server construction. engineering494/server-construction17.log preserves the +determining original target04 traceback; source-import18's missing extension is +a separate retained diagnostic failure, not that cause. + +Planck is the one combined integration owner. Singer reviewed and received ONLY +his published91d449a713d587d15ed6dff90d8ec3e29eb8f86e production/contract commit +and b1fcc7e3e1c4213d58fd8c3120316aeae4680167 native-queue test commit as normal +cherry-picks bac123a81 and5354da010 onto the original522 branch. No combined +branch/Root394 ancestry, package, shared environment or scientific run changed. +Original526 guide/checkpoints remain on their published separate branch. +Git diff confirms both production files and both fixtures byte-identical to +Planck's qualified sources; no repeated unchanged lifecycle or package tests. + +The canonical request factory now declares +dict[str, list[StreamProducerIdentity]], not recursive unrestricted JsonValue. +The existing nine-field nominal identity owns the full producer/incarnation; +FastMCP constructs it from its original dataclass declaration. Original controls +delegate to unchanged StreamProducerIdentity.from_payload, whose existing +typed-or-original StreamProducerPayloadMapping acceptance is now annotated +honestly. The generic generated tool binding/codec, retirement whole-set and +invocation guards, receipt MRO and deadlines are unchanged. No central leaf +switch, field/schema roster, anonymous/named JSON replacement, compatibility +reader, Any escape or larger recursion/timeout limit. BOUND-2/8 and MEMB-5 apply: +carry the original identity, derive schema from its declaration, delete the +overly broad producer record annotation. + +Received complete named-family AST before/after evidence: +engineering494/SCHEMA-FAMILY24-BASE.json and SCHEMA-FAMILY21-AFTER.json each +704 production plus38 actual introspect/PolyStore dependency modules, zero parse +omissions. Earlier17's704+12 preliminary census remains historical, not full +dependency closure. Dynamic consumers/MRO are read semantically; this is not a +global detector or equivalent-behavior proof. Latest NRA and authoritative +refactor-audit archive/catalog were reread before this source review. + +Actual ordinary target05 receiving proof +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Full engineering494/MCP-SCHEMA23.rst and READY-MCP25.json were read, plus original +construction19, health21 and installed22/25 logs. Immutable target05 and wheel +wheels04/openhcs-0.8.7-cp311-abi3-linux_x86_64.whl are Planck-owned; wheel SHA256 +bb68a203b5ce4b8fb755713278d00ce4d31557dc248615133502fd682bdbed72, +source91d449a71. Laterb1fcc7 is tests only, not packaged production. + +Original whole-package qualification20/24 reports815 tracked inputs,90 assets, +13 skill members/native source+binaries, original distribution RECORD/scripts, +reviewed dependencies and347 interpreter cache files qualified by original +source/code identity;1331 files, none unaccounted. Initial qualification23 +strictly refused those caches and is retained. This is received owner package +proof, not an independent duplicate audit in this worker. + +Actual fresh build_server constructs104 tools and the original nine-field +producer schema. Actual fresh non-resident stdio dev_client health returns +status=ok/errors=[]/packaged_resources_ready=true and exact target05 server +source. Owner child3281280 exited and scope is dead. No native/viewer or science +was started. Installed22:28PASS/9.76s/397676544B peak. Installed25:1PASS/4.55s/ +325955584B peak; real FastMCP Tool.run rejects incomplete identity without queue +dispatch, then reaches original service/gateway/serialized Qt queue/native +retirement action and preserves the same raw layer/backing array. These are +actual installed construction/transport and SDK-to-native-queue proofs, not a +detached external TCP viewer or process-RSS release claim. Owner bounded those +construction/health/control shards CPU1/1GiB/Swap0/60s; build512MiB/248061952B. + +Separate surviving selected-data limitation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Points/Shapes selected_data is not generally retained when domain changes +require rematerialization into new native layers. Current original selection +snapshot retains active_layer only; common native presentation retains style. +The33 lifecycle controls do not assert surviving geometry row indices. Original +NapariResultElementSelectionAuthority and NapariResultSelectionController own +row/group selection; any future repair must compose their behavior with original +mount/presentation hooks, not add a mirrored row store or a geometry-type switch. +The separate canonical guide526 records explicit row readback and typed settled +retirement, not a source fix or a hot skill installation. + +Remaining parent receiving boundary +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The observed target04 startup barrier is repaired on target05. Actual detached +TCP native selective retirement, shared-domain survivor rematerialization and +memory consequences, followed by same-native immutable saved-archive reopen, +remain parent-owned receiving acceptance under a released lane. No SCI retry, +old target mutation or broad clear/history deletion. Original UNKNOWN/failure, +qualified6c49 and initial historical investigation evidence below remain intact. + +Corrected source original R0 and archive +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +One original pinned R0 check on the newly changed5354da010 source completed: +terminal0,27.44s,maxRSS86752KiB, original cgroup peak76.2M,Swap0. The unchanged +validation/run_pinned_r0_419.py imports actual detector source from +agent-comms Git3b03785f45df2ef5dc62ba6aed99294192ecbb01 through the existing +3.14 -I -B interpreter. Original arguments --root openhcs --base f7de9efad393bce83a3756a271f0cd4383783ea9 +--head 5354da010f271359d7b58b6a1cec23c43ff37abd cover ALL9 changed production +files including both corrected annotations. Positive delta mapping is empty; +no detector copy, omitted file, limit change or waiver. Original systemd owner +bounded CPUQuota100%/AllowedCPUs3/MemoryMax536870912/MemorySwapMax0/Runtime60s. +Resource check's critical disk/swap advisory remains; availableRAM18GiB, +home4.6GiB. This tiny source-only guard is not native/full-growth admission. + +Additional byte-exact archive +viewer-selective-retirement-519-schema-receiving-20261003.tar.gz contains the two +corrected production/two fixture sources and original new R0 stdout/stderr: +6members,164649bytes, SHA256 +4b8944690f49674ec9798c5d135cc6d1369e1b54a5d2b421c1dd87a3e9bc53f9. +Original-member tar comparison passes. Historical60-member qualification and +all original negative logs remain unchanged, not regenerated or replaced. + +Current implementation checkpoint (2026-10-03) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Parent released the native coordinator/reconciliation, original route-state/ +settlement/batch/display-work hooks and typed protocol/request/gateway/service/ +capability family. Singer implements on the existing522 branch. Root394 core +producer, persistence and materialization remain untouched. Current Root claim +checked at30ebacf907273ee0d0767f392ac21281915f5a0e; the related protocol/viewer/ +agent/MCP family is byte-equal this branch's main base, before this implementation. +No other selective-retirement +implementation is published. The original investigation below and its source +archive remain historical evidence, not the implementation qualification. + +The coordinator now owns one purge recipe used by native-deleted reconciliation, +unmounted clear-state cleanup and explicit typed retirement. Route state stops +its timer and releases retired terminal settlement items; each original store +releases its route entry. Known failed terminal settlements are eligible, while +pending/accepted intake, active settlement and deferred display work are refused. +The entire explicitly requested producer identity set is compared by exact typed +equality before mutation, including invocation identity; no hidden-layer/name/ +candidate-age policy decides disposal. Existing source/result files are not read, +changed or deleted by retirement. + +Survivor pruning uses the existing registered display/rematerialization traversal. +The display ancestor preserves common native presentation; the independent image +presentation capability cooperates through super() to preserve intensity/gamma/ +colormap/interpolation. A rematerialization request's small publication hook leaves +peer traversal with its original owner instead of recursively re-deciding domains. +The existing request ancestor now carries the operation deadline shared by snapshot +and retirement; the snapshot-specific copy is deleted. The new capability is +discovered by the existing generated MCP binding, with no server/context roster +edit. A CLI leaf supplies the exact producer mapping through the same request. + +The first source runs retained two collection failures: source dependencies must +be activated before PolyStore, and the unchanged foreign pyqt gitlink predates +the merged render-complete contract. The clean, already-owned paired pyqt checkout +ad4948775ab81180a354d4b793d17ee5ddff3972 is borrowed read-only. Its snapshot module +SHA256 is bced886e6bbfd69ae9bdebf950027b809e72c23eddadcaa5fcf7151adf7ae913. +Neither foreign gitlinks nor installed dependencies are changed. + +The first behavioral shard proved17 controls but failed payload-release despite +all selected stores being cleared. Original Qt callbacks still strongly held the +update. The debounce-only correction was insufficient: original singleShot +settlement/continuation callbacks also capture that same update. The existing +NapariPendingLayerUpdate now owns a weak retained_callback hook used by all four +timer/continuation sites; every copied strong-capture lambda is deleted. No new +timer, painter or event loop was added. The missing weakref slot in the first hook +attempt and the next retained-reference failure are preserved in original logs. + +The initially added retirement-only gateway/service/invocation methods were +deleted after original R0 reported service growth and foreign-state probes. +ViewerWindowPresentationRequest/Result and ViewerNativePresentationCapability +already own native operation transport, decode and error lifecycle. They now +carry retirement via small request/result admission hooks. Retirement's agent +result composes that existing envelope with the original native receipt through +MI. Its cooperative super call explicitly names the declared slotted-dataclass +owner; Python's dataclass transformation replaces the pre-transformation type. +The original image-intensity availability guard is unchanged; retirement resolves +availability through NapariViewerServer.require_viewer, not a second leaf probe. + +The latest coherent source batch is33PASS (owner-controls10):20 real-Qt/ViewerModel +transitions including image/Shapes/Points retirement,6 typed MCP/CLI contract +controls and7 existing deletion/clear/shutdown controls. Actual payload and layer +weak references become None after removal, with no late callback resurrection. +Two survivor image handlers preserve source-item identity, sparse domains, +calibration, intensity/gamma/colormap/opacity/visibility/camera; an independent +declared capability executes its cooperative enter/exit hooks with no generic +consumer edit. These are source model/queue proofs, NOT native GL scene release +or process-RSS measurements. Generated MCP callable tests are not live MCP calls. + +Original constructor-order and slotted zero-argument-super failures are retained. +The combined snapshot/retirement process also retained a Vispy Qt5/Qt6 import +refusal. A separate existing snapshot shard exited1 during hidden Vispy setup +after three passing controls, without a Python traceback; its cause is not proved. +No backend/environment forcing, installed repair or weaker assertion is used. +Final source census and original R0 at corrected production6c49bedd8 are terminal: +704 production/669 test/258 dependency modules parsed, zero errors;49 production/ +40 test/12 dependency selected projections, plus full detail for all258 dependency +modules. This adds actual source PolyStore and read-only paired Qt service roots +to the original census; it does not omit the installed/foreign source contexts. +Original unchanged Qt/Vispy245-module full projection is reused, not rerun. +source-family03:16.77s/maxRSS85692KiB/75.7MiB cgroup peak/Swap0; dependency-details02: +1.29s/maxRSS27244KiB/16.9MiB peak/Swap0. There is no whole-NRA-detector/global-clean +claim, and AST does not prove dynamic native event behavior. + +Original R0 is loaded directly from agent-comms Git pin +3b03785f45df2ef5dc62ba6aed99294192ecbb01 through the unchanged existing +validation/run_pinned_r0_419.py. All9 changed production files are included; +thresholds/detector/input source are not modified. Original c76 R0RED reported +ViewerWindowService excess+27 and two foreign-absence probes. The copied retirement +gateway/service/invocation path was deleted; native availability stays with server. +Corrected6c49bedd8 R0PASS: positive delta mapping empty,29.51s/maxRSS86936KiB/ +74.3MiB cgroup peak/Swap0/terminal0. The first3.12 deferred-annotation import failure +and a later wrong CLI positional "compare" failure remain original negatives; +the actual original detector runs through existing3.14, with its unchanged CLI. + +No installed/live/RSS-reduction claim is made. There is no new runtime/client/ +viewer launch, package installation or scientific contact. Planck494's Points +domain, spatial calibration and feature-metadata hunks are explicitly disjoint. +Unexpected native removal/rematerialization errors are not an atomic rollback; +their failed operation disposition must not be replayed as if nothing happened. + +Receiving packet and production closure +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The paired source/native-receiving proposal is +viewer-selective-retirement-519-native-receiving-20261003.rst. It reuses parent/ +Planck's exact original989B archive, raw/source public inventory and one95 client; +Copernicus511 catalog controls are independent on that same handle. One ordinary +whole target includes reviewed494e6c/511cb263946/5226c49 production after parent's +review/resource release, not three worker builds or hot SCI updates. Actual +current SCI88/96/94 remaining growth and original common RAM/disk floor determine +admission. No new launch/resource owner, producer replay, writer or fixture store. +Production editing is frozen at6c49bedd8; this final delivery adds receipt/archive/ +packet evidence only. Installed public retirement/native visual release remain +NOT RUN and are required before merge. The isolated hidden-Vispy source setup +failure remains unresolved; do not call all snapshot source controls green. + +Final qualification archive is +viewer-selective-retirement-519-qualified-20261003.tar.gz:60 members/1063296bytes, +SHA25692881ab00e7a79fd537efd323e81ab5ce3badc2a191d63833097d6a475ed5f66. +It includes byte-exact9 production/3 fixture sources, original R0 caller, +source/pytest callers and command receipt, all original successful/failed raw +logs and corrected full-family/dependency projections. tar --compare against +every original member completed0. Historical nine-member source archive is +unchanged. Raw pytest whitespace stays in archives, not loose versioned logs; +production/test/docs git diff --check completes0 without rewriting evidence. + +Changed production paths (no Root producer/materialization file): + +* openhcs/agent/capabilities.py +* openhcs/agent/dto/__init__.py +* openhcs/agent/dto/viewer.py +* openhcs/agent/services/viewer_window_service.py +* openhcs/mcp/dev_client_commands/viewer.py +* openhcs/runtime/napari_streaming_handlers.py +* openhcs/runtime/napari_viewer_server.py +* openhcs/runtime/viewer_controls.py +* openhcs/runtime/viewer_protocol.py + +Original investigation checkpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Singer owns this source investigation under the parent's explicit assignment; +parent retains302/308 mounted-route integration and later installed/native +acceptance. This checkpoint contains source evidence and a narrow integration +proposal, NOT a working retirement implementation or proved memory leak. +No viewer/native/MCP/client launch, installation, scientific input or author +contact occurred. Existing503 branch/evidence and foreign gitlinks remain intact. +The same finished checkout now has a new branch from merged main, with no new +worktree/environment/dependencies. No production source has been edited. + +Claims checked before source work +-------------------------------- + +Audited main f7de9efad393bce83a3756a271f0cd4383783ea9 (merged503); current +Root394 4c636a281de4da7341f4f9f6bb8f718c75644a1d;494 +7a067c505444ec7d8720da21118dc85787129c59.519 is OPEN/unassigned with no +implementation comments. All11 current open PR titles/heads listed; relevant +owner bodies/file rosters checked: no published selective-retirement implementation. Search519 across +all PR states returns only merged guide520.519/302/308 explicitly name parent +integration;494 remains point-projection receiving and Root394 remains runtime/ +materialization integration. Runtime/viewer, agent viewer DTO/service/capability, +MCP binder/CLI and orthogonal consumer files in the selected family are BYTE +EQUAL main to current Root394. Root's separate core producer/source deltas are +recorded, not claimed equal or taken over.468 touches tracking, not this seam. +Legacy agent-comms registry has no current running OpenHCS claim; direct parent +assignment and current GitHub owner records determine this task's ownership. + +Determining owners and consumers +------------------------------- + +* napari_viewer_server.py:943, NapariComponentAwareDisplayCoordinator owns routed + intake and existing deleted-native-layer reconciliation.1037..1039 currently + purge route state, item groups and declared/observed component domains. Reuse + this owner; do not copy three purges into a new action or change hiding. +* napari_streaming_handlers.py:1506, NapariLayerRouteStateStore owns mounted + layers/titles/dimensions/pending updates/errors and settlement. purge_route + removes dictionary entries but does not itself stop a pending timer. Original + cancel_pending_update/drain_pending_updates/reset_settlement own cancellation + and terminal-release rules; running settlement cannot be silently reset. +* NapariLayerSettlementState:291 owns the tuple of original pending updates, + each with source items. Completed progress retains that tuple until the + original reset_settlement boundary. Purging mounted stores alone therefore + does not prove all retired payload references released. This is source + retention evidence, not an attribution of either reported OOM. +* NapariLayerDisplayPipeline:2148 owns per-route deferred work and shared-axis + projection/rematerialization. Its scheduling/completion methods own timer and + continuation identity. Existing rematerialize=True and nominal display + handlers already support pruning; no second painter/display loop is needed. +* NapariBatchProcessorStore:1794 owns lazy route processors. Read actual + PolyStore NapariBatchProcessor: it holds its server, NOT an accumulated image + list/timer. Retire its route entry for lifecycle completeness, but do not + misidentify it as a separate scientific-array cache. +* ViewerRouteComponentValueTracker:1104 owns per-route domains; shared_values_for + derives their union. Retire only selected domains and reproject survivors + through the existing pipeline. Preserve survivor source items/metadata and + physical scale; aggregate display arrays may legitimately rematerialize. +* Native viewer.layers.remove uses Napari LayerList deletion, ViewerModel event + disconnection and VispyCanvas._remove_layer/VispyBaseLayer.close. The latter + disconnects layer/camera events, closes the visual and deletes its scene-map + entry. Invoke this external lifecycle, not private GL deletion or broad clear. + Result-selection callbacks already use weak layer references/weak-key stores; + no strong mirror or new callback registry is warranted. +* NapariControlMessageAction's registered family/accepted Qt reply queue owns + action dispatch. NapariMountedRouteControlMessageAction owns exact mounted + route resolution. New retirement action is a small member delegating lifecycle + work to the coordinator; no central viewer/type/origin switch. +* ViewerWindowControlRequest, ViewerWindowGatewayABC/ZMQViewerWindowGateway and + ViewerWindowService own request/deadline/transport/result boundaries. Existing + OperationDeadline and accepted-operation semantics must prevent late mutation + after a caller's deadline, without increasing any timeout or adding timers. +* AgentCapabilityDeclaration + AgentViewerWindowRequestServiceInvocation and + GeneratedMcpViewerRequestToolBinding already generate tools from declaration + input fields. Add a typed request/result and capability member there; generic + MCP server/context dispatch need no retirement branch/roster. An optional + ordinary CLI CommandSpec is its own leaf, not an alternate API/registry. + +Required relation and exact next shared hunk +------------------------------------------ + +Explicit caller-selected exact mounted route identities are retirement targets. +Hidden is not retired, and neither names, counts nor producer origin imply that +a route is disposable. Reuse full StreamProducerIdentity and existing source/ +domain records for preconditions, not a second generation/receipt store. The +route key itself is insufficient as a generation witness: route_parts omits +invocation_key, while the original typed producer carries it. Before mutation, +resolve/validate the ENTIRE selected set, the exact current identity and the +original settlement/intake disposition. Missing/stale/foreign/pending/failed or +UNKNOWN work must yield an explicit non-success without collateral retirement. + +The coordinator's one retirement recipe must release the selected native layer, +route/group/domain, per-route deferred work and batch entry through those +existing owners. Retain raw/current/needed-reference routes and durable source/ +output files. Release completed settlement references through its existing +terminal reset, never by cancelling unknown work or deleting history. Keep +survivor semantic coordinates, calibration, contrast/camera/selection correct +through original rematerialization/selection hooks, not by freezing stale axis +indices or inventing an aggregate domain. Do not automatically guess old +candidate routes from visibility; the supported MCP operation performs the +selected retirement at a safe settled boundary. + +Shared production hunk to coordinate with parent302/308 before editing: +napari_viewer_server.py coordinator deleted-route reconciliation + display-work/ +registered action methods; napari_streaming_handlers.py route-state/settlement/ +batch-entry release hooks; viewer_component_system.py only if the ORIGINAL +domain owner needs a narrow hook. Receiving declarations go in viewer_controls, +OpenHCS-owned viewer_protocol message family, agent/dto/viewer, viewer service/ +gateway and capability leaf. Do not change Root394 core/persistence/materializers, +frozen installed sources or extracted dependency gitlinks. Delete replaced purge +duplication at existing reconciliation/clear consumers in the same coherent +change; preserve clear's302 mounted-inventory behavior. + +Automatic after-settlement dispatch must compose with the existing settlement +owner and control deadline. Exact terminal/pending-intake admission remains to +be implemented and exercised; this source checkpoint does not supply a second +poller, timeout engine or pre-authorized destructive policy. Parent's shared +coordinator hunk disposition is the next integration dependency, not a new +unclaimed-owner queue. No production patch or runtime is submitted here. + +Source evidence and proof limits +------------------------------- + +Read latest NRA skill and authoritative nominal-refactor-advisor/skills/ +refactor-audit.skill SKILL/catalog README plus full identity, membership, +implementation, boundaries and over-time pattern files. Applicable patterns: +IDEN-1/6 (route versus generation/visibility), BOUND-2/8 (typed source/domain/ +producer facts across request/reply), IMPL-12/13 (copied purge/settlement loops), +MEMB-1/2 (parallel tool/route rosters), TIME-9 (alternate codec/compatibility). +Existing behavior-owning classes and shared registered ABCs remain load bearing; +compose genuinely independent hooks with cooperative MRO only where they do work, +not an ornamental inheritance layer. + +Thin source caller uses original audit ParsedModule/FunctionFacts/Repository, +not a copied detector; no product import. Complete704 production +668 tests +188 +first-boundary dependencies parsed, zero errors. Named projection38 production/ +34 test modules; dependency-detail shard records ALL188 modules' declarations, +imports and function facts. Scene shard additionally records ALL245 Napari Qt/ +Vispy modules, zero errors. External compiled Qt/GL behavior is not executed or +proved by AST; dynamically connected native events were read semantically. +No full NRA-detector/global ratchet or runtime acceptance is claimed. + +Original source-family01 was terminal2 before script start because systemd's +working directory was /home/ts; its stderr remains unchanged. Corrected02 used +explicit original WorkingDirectory:21.435s,77.2MiB peak,Swap0,terminal0. +Independent dependency/scene shards:1.927s/16.9MiB and1.814s/18.4MiB,Swap0, +terminal0. Each kernel bound CPU1/512MiB/noSwap/60s, no widened limit. +Critical disk/swap advisory preserved; cheap snapshot had11.5GiB available RAM. +Static JSON records/commands/raw logs are archived; no tests run on this turn. + +Acceptance to implement, then verify last +---------------------------------------- + +Use existing real Qt/ViewerModel reset-transition fixture (no renderer clone): +raw + two completed candidate routes + needed reference, exact selected retire, +survivor source-item/domain/scale/semantic-coordinate/presentation proof. Include +interior shared-axis prune/rematerialization, RGB + Shapes + Points, missing/ +stale/foreign route, queued replacement, partial Shapes, failed/active settlement, +deadline and no resurrection/duplicate reply. Prove retired object/array weak +references released without requiring allocator RSS to fall immediately. +Independent new handler/capability declaration must exercise real hooks through +the generic coordinator/binder without generic consumer edits; inheritance +assertions alone do not satisfy this control. After the source batch, parent +qualifies ordinary installed public state/payload/matched captures and measures +route-owned references/RSS/PSS under separately released engineering custody. + +C39's111 mounted layers and R001094's reported4.5GiB OOM/239 pending soma reopen +are retained correlation witnesses, not leak/layer-count causation. R0010 caller +143/UNKNOWN execution/no public close receipt must remain that disposition. +P001 candidate01 settlement error belongs existing308; its independent authored +candidate02 reopen is not replayed or declared fixed by this investigation. +No scientific pixel/reference/answer files were read or copied into this PR. + +Byte-exact source evidence archive +--------------------------------- + +``viewer-selective-retirement-519-source-20261003.tar.gz``, nine members, +469035bytes, SHA256 +``f3b76a5c7eebd0c5732957fb7ca646d0fef697ca1ec3711e2179ddcdb81a7bd8``. +Original-file tar comparison passed. The original caller's symbol projection +preceded the later dependency/scene-detail options; its emitted records retain +that original selection, and current source caller/options are archived as such. +No original first failure or prior503 source/evidence is overwritten. diff --git a/docs/validation/viewer-selective-retirement-519-native-receiving-20261003.rst b/docs/validation/viewer-selective-retirement-519-native-receiving-20261003.rst new file mode 100644 index 0000000000..76efda3a19 --- /dev/null +++ b/docs/validation/viewer-selective-retirement-519-native-receiving-20261003.rst @@ -0,0 +1,87 @@ +519/522: one combined95 receiving journey, NOT a launch authorization +=================================================================== + +Parent owns launch/client/native custody and admission; Dalton owns original +helper/resources, Planck494 owns the saved-point receiving case, Copernicus511 +owns catalog source. Singer522 production is frozen at6c49bedd8. Source strengths +are33PASS and original pinned R0 with zero positive deltas; no installed/live +retirement claim. No new launcher, runtime/client, resource schema or builder +is introduced by this packet. Actual current SCI88/96/94 remaining growth and +the original common RAM/disk floor determine future95 admission. An available +display or source qualification is not that release. + +One ordinary whole target, after parent source review/admission, must contain +Planck494 e6c0bf77f8e4d24d80c0463e214070bc018a3301 receiving repairs, +Copernicus511 cb263946 catalog source and Singer5226c49bedd8 production. +These are source identities, NOT a claim that an existing target contains them. +Reuse their existing whole-wheel/member/dependency qualification owners and +existing interpreter; no selective installed-file patch, hot SCI update or +three separate builds. Parent chooses integration order/target and checks +combined bytes before any install. Root394 publication changes are not requested. + +The original case owners remain engineering494/RECEIVER11-QUALIFICATION.rst +and engineering49495-20261003/PARENT-PUBLIC95-FAILED01.rst under the issue-batch +root. The distinct current case reopens the exact original989B archive through +public inventory/source metadata, NOT a replay of failed producer execution. +Planck supplies the exact archive/input/request from those original journals. +Do not invent a source receipt, filename/domain, new writer or calibration. +511's original CATALOG511-PUBLIC95-COMMANDS.rst supplies its independent catalog +controls on this same parent-owned native/client handle. No second client. + +Selective retirement on that same native viewer +---------------------------------------------- + +1. Complete Planck's public inventory, raw source and saved-archive stream once + on the fresh owned95 viewer. Public StreamingService's existing + _require_viewer_settled calls the original receiver settlement owner; retain + the actual result/error and progress. Do not invent a public "settle-viewer" + command. Any UNKNOWN source/stream mutation must get an explicit original + disposition before retirement. A known FAILED terminal candidate with no + outstanding mutation may be explicitly retired, not held forever. +2. Read full public viewer state/payloads and preserve the original raw/point + matched XY/XZ/YZ evidence through Planck's existing controls. Save exact route + and complete producer_identities, native transforms, source metadata/domain, + feature rows, source arrays and presentation. Do not derive eligibility from + titles, layer counts, visibility or point count. Remain on this same viewer + incarnation; no old cached state or prefilled producer mapping. +3. Explicitly select the reopened engineering Points route for retirement; + preserve all raw routes and any untargeted overlays. Copy its complete + producer_identities unchanged from the immediately preceding public state + into expected_producers under that exact route_key. Use either the original + generated MCP tool OR CLI leaf below, not both for the same mutation:: + + call openhcs_retire_viewer_window_layers --arguments '' --json + + retire-viewer --host 127.0.0.1 --transport-mode tcp --expected-producers '' --json + + JSON_ARGUMENTS contains host, actual viewer port, transport_mode="tcp" and + expected_producers={actual_route_key:complete_actual_producer_identities}. + Default5000ms control and ordinary retained-client initialization stay + unchanged. No increased deadline, settling sleep, broad clear or restart. + Validate applied/observed true, no errors, exact retired_route_keys, and + remaining_route_keys containing every survivor but no retired route. +4. Public viewer-state and viewer-payloads readback on this same incarnation + must show no retired route/items/domain contribution or late resurrection. + Compare every survivor's actual arrays/shape/source/calibration/domain/ + presentation, allowing the original shared-domain owner to reproject axes. + Preserve raw-only same-coordinate native snapshot and any surviving overlay + QA. A native scene/visual deletion and payload-release observation belongs + to actual installed acceptance; source weakrefs alone do not establish it. + Record RSS before/after honestly: this tiny archive need not yield a measurable + decrease, and RSS alone does not establish cache/scene release or an OOM cause. +5. Reopen the SAME immutable archive as a DISTINCT new public operation after + the retirement's known terminal receipt. This is not replay of the earlier + failed producer or UNKNOWN mutation. Verify restored full source/calibration/ + domain/point metadata and matched views through Planck's controls. The raw + source, archive bytes and durable journals must be unchanged throughout. + +Whole-set stale/missing-incarnation and in-flight/deadline rejection are proved +by source controls. Extra installed negative mutations are not silently added +to the parent's limited public journey; parent may explicitly select them. +Unexpected removal/rematerialization exceptions can follow partial mutation; +retain complete reply/actual state and do not assume atomic rollback or retry. + +Terminal custody follows the existing launch packet: exact public viewer/native +close receipts, original process/endpoint checks and once-only retained-client +exit. No helper signal or new process from Singer; active SCI remain untouched. +This packet is prepared commands/relations, not installed PASS or a native run. diff --git a/docs/validation/viewer-selective-retirement-519-qualified-20261003.tar.gz b/docs/validation/viewer-selective-retirement-519-qualified-20261003.tar.gz new file mode 100644 index 0000000000..e769b3cd84 Binary files /dev/null and b/docs/validation/viewer-selective-retirement-519-qualified-20261003.tar.gz differ diff --git a/docs/validation/viewer-selective-retirement-519-schema-receiving-20261003.tar.gz b/docs/validation/viewer-selective-retirement-519-schema-receiving-20261003.tar.gz new file mode 100644 index 0000000000..c4f93c625a Binary files /dev/null and b/docs/validation/viewer-selective-retirement-519-schema-receiving-20261003.tar.gz differ diff --git a/docs/validation/viewer-selective-retirement-519-source-20261003.tar.gz b/docs/validation/viewer-selective-retirement-519-source-20261003.tar.gz new file mode 100644 index 0000000000..d9dcebe364 Binary files /dev/null and b/docs/validation/viewer-selective-retirement-519-source-20261003.tar.gz differ diff --git a/openhcs/agent/capabilities.py b/openhcs/agent/capabilities.py index 1050eef002..237a014b8b 100644 --- a/openhcs/agent/capabilities.py +++ b/openhcs/agent/capabilities.py @@ -186,6 +186,8 @@ ViewerWindowIntensityWindowResult, ViewerWindowLayerIsolationRequest, ViewerWindowLayerIsolationResult, + ViewerWindowLayerRetirementRequest, + ViewerWindowLayerRetirementResult, ViewerWindowNavigationRequest, ViewerWindowNavigationResult, ViewerWindowPayloadRequest, @@ -3377,6 +3379,26 @@ class NavigateViewerWindowCapability(ViewerWindowCliConnectionCapability): ) +class RetireViewerWindowLayersCapability(ViewerNativePresentationCapability): + name = "openhcs_retire_viewer_window_layers" + cli_command = "retire-viewer" + title = "Retire explicit viewer layers" + description = ( + "After viewer settlement, removes only explicitly selected mounted routes " + "and releases their native layers and receiver payload caches. Supply " + "expected_producers as a route-key mapping to each route's complete " + "producer_identities from viewer state, including invocation_key. The " + "whole set is checked before removal. Pending intake/display mutations " + "must reach a known terminal state first; known terminal failed candidates " + "can be retired. Untargeted routes and persisted source/results remain " + "intact. Hiding layers is not retirement." + ) + side_effects = ("retires_explicit_viewer_layers",) + data_exposure = ("viewer_layer_retirement",) + input_contract = ViewerWindowLayerRetirementRequest + output_contract = ViewerWindowLayerRetirementResult + + class IsolateViewerWindowLayersCapability(ViewerWindowCliConnectionCapability): name = "openhcs_isolate_viewer_window_layers" cli_command = "isolate-viewer" diff --git a/openhcs/agent/dto/__init__.py b/openhcs/agent/dto/__init__.py index c6c0ca2f20..617074633d 100644 --- a/openhcs/agent/dto/__init__.py +++ b/openhcs/agent/dto/__init__.py @@ -148,6 +148,8 @@ "ViewerWindowLayerValidationSummary", "ViewerWindowLayerIsolationRequest", "ViewerWindowLayerIsolationResult", + "ViewerWindowLayerRetirementRequest", + "ViewerWindowLayerRetirementResult", "ViewerWindowLayerVisibilityRecord", "ViewerWindowImageSampleRequest", "ViewerWindowImageSampleResult", diff --git a/openhcs/agent/dto/viewer.py b/openhcs/agent/dto/viewer.py index a340b2c044..c5f0f38a98 100644 --- a/openhcs/agent/dto/viewer.py +++ b/openhcs/agent/dto/viewer.py @@ -6,7 +6,7 @@ from collections.abc import Mapping from dataclasses import dataclass, field from dataclasses import fields as dataclass_fields -from typing import ClassVar, Self, cast +from typing import Annotated, ClassVar, Self, cast from metaclass_registry import AutoRegisterMeta from polystore.streaming.identity import StreamProducerIdentity @@ -17,8 +17,8 @@ WindowSnapshotFrameCondition, WindowVisualObservation, ) -from python_introspect import dataclass_from_mapping -from pydantic import StrictFloat, StrictInt +from python_introspect import dataclass_from_mapping, project_dataclass +from pydantic import BeforeValidator, StrictFloat, StrictInt from zmqruntime.timeouts import OperationDeadline from zmqruntime.viewer_protocol import ( ViewerImageIntensityControlOptions, @@ -52,6 +52,7 @@ ViewerRegionControlOptions, ViewerIntensityWindowControlOptions, ViewerLayerIsolationControlOptions, + ViewerLayerRetirementControlOptions, ViewerNavigationControlOptions, ViewerNativeDimensions, ViewerPayloadControlOptions, @@ -67,6 +68,7 @@ OpenHCSViewerControlMessageType, ViewerProtocolStatus, ViewerImageColorControlOptions, ViewerNativeImageColorPresentation, ViewerNativeWindowControlOptions, ViewerNativeWindowState, + ViewerLayerRetirementReceipt, ) VIEWER_WINDOW_CONTROL_TIMEOUT_MS_DEFAULT = 5000 @@ -98,9 +100,15 @@ class ViewerWindowControlRequest(ExecutionConnectionProjection): timeout_ms: int = VIEWER_WINDOW_CONTROL_TIMEOUT_MS_DEFAULT include_response: bool = True + operation_deadline: OperationDeadline | None = field(default=None, repr=False) + + def start_operation(self) -> Self: + from dataclasses import replace + + return replace(self, operation_deadline=self.control_deadline()) def control_deadline(self) -> OperationDeadline: - return OperationDeadline.after_milliseconds( + return self.operation_deadline or OperationDeadline.after_milliseconds( self.timeout_ms, operation="viewer control request", ) @@ -192,7 +200,6 @@ class ViewerWindowSnapshotRequest( WindowSnapshotFrameCondition.RENDER_COMPLETE ) observation_timeout_s: float | None = None - operation_deadline: OperationDeadline | None = field(default=None, repr=False) def __post_init__(self) -> None: if ( @@ -218,22 +225,6 @@ def __post_init__(self) -> None: "Snapshot observation timeout must be less than transport timeout." ) - def start_operation(self) -> Self: - from dataclasses import replace - - return replace( - self, - operation_deadline=super( - ViewerWindowSnapshotRequest, self - ).control_deadline(), - ) - - def control_deadline(self) -> OperationDeadline: - return ( - self.operation_deadline - or super(ViewerWindowSnapshotRequest, self).control_deadline() - ) - def snapshot_operation_deadline(self) -> OperationDeadline | None: return self.operation_deadline @@ -611,6 +602,48 @@ def as_tool_arguments(self) -> dict[str, JsonValue]: return payload +@dataclass(frozen=True, slots=True, kw_only=True) +class ViewerWindowLayerRetirementRequest(ViewerWindowPresentationRequest): + retirement: ViewerLayerRetirementControlOptions + message_type = OpenHCSViewerControlMessageType.RETIRE_LAYERS.value + + @property + def result_type(self): + return ViewerWindowLayerRetirementResult + + @property + def control_payload(self): + return self + + @classmethod + def from_fields( + cls, *, connection: ExecutionConnectionSpec, + expected_producers: dict[ + str, + list[Annotated[ + StreamProducerIdentity, + BeforeValidator(StreamProducerIdentity.from_payload), + ]], + ], + timeout_ms: int = VIEWER_WINDOW_CONTROL_TIMEOUT_MS_DEFAULT, + ) -> Self: + return cls( + connection=connection, timeout_ms=timeout_ms, + retirement=ViewerLayerRetirementControlOptions.from_overrides( + expected_producers=expected_producers, + ), + ) + + def as_tool_arguments(self) -> dict[str, JsonValue]: + return { + **self.connection_tool_arguments(), + "expected_producers": { + route: [to_jsonable(producer.to_payload()) for producer in producers] + for route, producers in self.retirement.expected_producers.items() + }, + } + + @dataclass(frozen=True, slots=True, kw_only=True) class ViewerWindowIntensityWindowRequest(ViewerWindowControlRequest): """Apply a route-global window over payloads matching semantic coordinates.""" @@ -1314,9 +1347,17 @@ def from_native_response(cls, connection, response: Mapping[str, object]): if not isinstance(payload, Mapping): raise TypeError("Native presentation readback must be a mapping.") snapshot = cls.snapshot_type.from_wire_mapping(payload) + return cls.from_snapshot(connection, snapshot) + + @classmethod + def from_snapshot(cls, connection, snapshot): return cls(schema_version=SCHEMA_VERSION, connection=connection, observed=True, applied=True, **{cls.response_field.value: snapshot}) + def admit_request(self, request: ViewerWindowPresentationRequest) -> Self: + """Operation-specific reply custody; concrete declarations supply hooks.""" + return self + @dataclass(frozen=True, slots=True) class ViewerWindowViewportResult(ViewerWindowPresentationResult): @@ -1387,6 +1428,37 @@ class ViewerWindowLayerVisibilityRecord: selected: bool +@dataclass(frozen=True, slots=True, kw_only=True) +class ViewerWindowLayerRetirementResult( + ViewerWindowPresentationResult, + ViewerLayerRetirementReceipt, +): + registry_key: ClassVar[str] = "layer_retirement" + response_field = ViewerControlField.RETIREMENT + snapshot_type = ViewerLayerRetirementReceipt + observed: bool = field(kw_only=True) + + @classmethod + def from_snapshot(cls, connection, snapshot): + return project_dataclass( + cls, snapshot, schema_version=SCHEMA_VERSION, + connection=connection, observed=True, + ) + + def admit_request(self, request: ViewerWindowLayerRetirementRequest) -> Self: + # slots=True replaces the dataclass type; use that declared MRO owner, + # not the pre-transformation __class__ captured by zero-argument super. + super(ViewerWindowLayerRetirementResult, self).admit_request(request) + if not self.applied: + raise ValueError("Native retirement did not apply the requested set.") + requested = frozenset(request.retirement.expected_producers) + if frozenset(self.retired_route_keys) != requested: + raise ValueError("Native retirement acknowledgement has a different route set.") + if requested.intersection(self.remaining_route_keys): + raise ValueError("Retired routes remain mounted in the acknowledgement.") + return self + + @dataclass(frozen=True, slots=True, kw_only=True) class ViewerWindowLayerIsolationResult( ViewerWindowObservedErrorResultMixin, diff --git a/openhcs/agent/services/viewer_window_service.py b/openhcs/agent/services/viewer_window_service.py index 393820ad29..0bcbf5c6e2 100644 --- a/openhcs/agent/services/viewer_window_service.py +++ b/openhcs/agent/services/viewer_window_service.py @@ -1099,6 +1099,7 @@ def image_intensity(self, request: ViewerWindowImageIntensityRequest) -> JsonObj ) def presentation_control(self, request: ViewerWindowPresentationRequest) -> JsonObject: + request = request.start_operation() return self._send_control_message( request, { @@ -1404,7 +1405,7 @@ def presentation( response = self._gateway.presentation_control(request) return request.result_type.from_native_response( request.connection, response, - ) + ).admit_request(request) except Exception as error: return request.result_type.from_error( connection=request.connection, diff --git a/openhcs/mcp/dev_client_commands/viewer.py b/openhcs/mcp/dev_client_commands/viewer.py index 167e8162fb..bee6d1c3de 100644 --- a/openhcs/mcp/dev_client_commands/viewer.py +++ b/openhcs/mcp/dev_client_commands/viewer.py @@ -15,6 +15,7 @@ ViewerWindowImageSampleRequest, ViewerWindowIntensityWindowRequest, ViewerWindowLayerIsolationRequest, + ViewerWindowLayerRetirementRequest, ViewerWindowNavigationRequest, ViewerNativeViewportPresentation, ViewerWindowViewportRequest, @@ -41,6 +42,7 @@ extend_required_component_labels, optional_bool, parse_navigation_axis_indices, + parse_json_object, parse_required_axis_labels, request_factory_parameter, request_field_bool_default, @@ -727,6 +729,31 @@ def tool_arguments( return McpToolArgumentAuthority.from_payload(request.as_tool_arguments()) +class RetireViewerCommandSpec(SingleToolCommandSpec): + capability = agent_capabilities.retire_viewer_window_layers + + def configure_parser(self, parser: argparse.ArgumentParser) -> None: + add_viewer_port_argument(parser) + parser.add_argument( + "--expected-producers", required=True, type=parse_json_object, + help="JSON route-key mapping to complete producer_identities from viewer state.", + ) + parser.add_argument("--json", action="store_true") + add_viewer_connection_options(parser) + + def tool_arguments(self, args: argparse.Namespace) -> dict[str, JsonValue]: + connection = ViewerConnectionArguments.from_args(args) + request = ViewerWindowLayerRetirementRequest.from_fields( + connection=ExecutionConnectionSpec( + host=connection.host, port=connection.port, + transport_mode=connection.transport_mode, + ), + timeout_ms=(connection.timeout_ms or VIEWER_WINDOW_CONTROL_TIMEOUT_MS_DEFAULT), + expected_producers=args.expected_producers, + ) + return McpToolArgumentAuthority.from_payload(request.as_tool_arguments()) + + class IsolateViewerCommandSpec(SingleToolCommandSpec): capability = agent_capabilities.isolate_viewer_window_layers diff --git a/openhcs/runtime/napari_streaming_handlers.py b/openhcs/runtime/napari_streaming_handlers.py index 4047ed9ff3..aee44ed577 100644 --- a/openhcs/runtime/napari_streaming_handlers.py +++ b/openhcs/runtime/napari_streaming_handlers.py @@ -5,8 +5,10 @@ import colorsys import logging import threading +import weakref from abc import ABC, abstractmethod from collections.abc import Callable, Mapping, Sequence, Sized +from contextlib import contextmanager from dataclasses import dataclass, field from enum import Enum from typing import TYPE_CHECKING, ClassVar, TypeAlias @@ -254,7 +256,7 @@ def stop(self) -> None: """Stop the pending layer update.""" -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True, slots=True, weakref_slot=True) class NapariPendingLayerUpdate(ViewerComponentAxisSemantics): """Queued debounced layer update with flush-local runtime residue.""" @@ -287,6 +289,19 @@ def stop_timer(self) -> None: self.timer.stop() + def retained_callback( + self, callback: Callable[["NapariPendingLayerUpdate"], None], + ) -> Callable[[], None]: + """Qt continuations borrow the update; route/settlement state owns it.""" + update_ref = weakref.ref(self) + + def advance() -> None: + update = update_ref() + if update is not None: + callback(update) + + return advance + @dataclass(slots=True) class NapariLayerSettlementState: @@ -409,6 +424,26 @@ def fail(self) -> None: ) self.failed = True + def require_terminal(self) -> None: + """Failed is terminal too; an executing or claimed callback is not.""" + with self._lock: + if self.phase is ViewerSettlePhase.RUNNING or self.active_route is not None: + raise RuntimeError("Cannot retire layers during active settlement.") + if self.active_route_work_unit_active: + raise RuntimeError("Cannot retire a native mutation in flight.") + + def purge_route(self, route_key: str) -> None: + """Release only the retired payload references, including failed work.""" + with self._lock: + self.require_terminal() + self.completed_update_count = sum( + key != route_key + for key, _ in self.updates[:self.completed_update_count] + ) + self.updates = tuple( + (key, update) for key, update in self.updates if key != route_key + ) + def progress(self) -> ViewerSettleProgress: """Project current settlement state onto the shared wire contract.""" @@ -1567,12 +1602,15 @@ def title_collides(self, layer_key: str, title: str) -> bool: ) def purge_route(self, layer_key: str) -> None: + with self._settlement_lock: + if self.layer_settlement is not None: + self.layer_settlement.purge_route(layer_key) + self.layer_update_errors.pop(layer_key, None) + self.cancel_pending_update(layer_key) self.layers.pop(layer_key, None) self.layer_titles.pop(layer_key, None) self.layer_dimension_states.pop(layer_key, None) self.layer_pending_updates.pop(layer_key, None) - with self._settlement_lock: - self.layer_update_errors.pop(layer_key, None) if self.active_dimension_label_route == layer_key: self.active_dimension_label_route = None @@ -1699,6 +1737,20 @@ def reset_settlement(self) -> None: self.layer_update_errors.pop(None, None) self.layer_settlement = None + @contextmanager + def mutation_boundary(self): + """Serialize accepted intake and Qt retirement on the original lock.""" + with self._settlement_lock: + yield + + def require_retirement_boundary(self) -> None: + """Reject unresolved pending work, not known terminal failures.""" + with self._settlement_lock: + if self.layer_pending_updates: + raise RuntimeError("Cannot retire layers with pending updates; settle first.") + if self.layer_settlement is not None: + self.layer_settlement.require_terminal() + def record_update_error(self, layer_key: str | None, error: Exception) -> None: """Retain a display failure, including intake without a resolved route.""" @@ -1759,6 +1811,11 @@ class NapariComponentGroupStore: groups: dict[str, list["NapariStreamLayerItem"]] = field(default_factory=dict) + def producer_identities_for(self, layer_key: str) -> frozenset[StreamProducerIdentity]: + return frozenset( + item.producer for item in self.existing_items_for(layer_key) or () + ) + def items_for(self, layer_key: str) -> list["NapariStreamLayerItem"]: if layer_key not in self.groups: self.groups[layer_key] = [] @@ -1822,6 +1879,10 @@ class NapariBatchProcessorStore: processors: dict[str, "NapariBatchProcessor"] = field(default_factory=dict) lock: threading.Lock = field(default_factory=threading.Lock) + def purge(self, layer_key: str) -> None: + with self.lock: + self.processors.pop(layer_key, None) + def get_or_create( self, *, diff --git a/openhcs/runtime/napari_viewer_server.py b/openhcs/runtime/napari_viewer_server.py index ef58616bd1..176b7dec2b 100644 --- a/openhcs/runtime/napari_viewer_server.py +++ b/openhcs/runtime/napari_viewer_server.py @@ -16,6 +16,7 @@ import weakref from abc import ABC, abstractmethod from collections.abc import Callable, Hashable, Iterable, Mapping +from contextlib import contextmanager from dataclasses import dataclass, field, replace from enum import Enum from functools import partial @@ -70,7 +71,9 @@ ) from openhcs.constants import AllComponents -from openhcs.agent.dto.viewer import ViewerWindowDescriptor +from openhcs.agent.dto.viewer import ( + ViewerWindowDescriptor, ViewerWindowLayerRetirementRequest, +) from openhcs.runtime.viewer_snapshot import ( ViewerWindowSnapshotFailureReply, ViewerWindowSnapshotService, @@ -176,6 +179,7 @@ ViewerLayerField, ViewerLayerIsolationControlOptions, ViewerLayerIsolationField, + ViewerLayerRetirementReceipt, ViewerNavigationControlOptions, ViewerPayloadField, ViewerPayloadProjectionOptions, @@ -1034,9 +1038,7 @@ def _reconcile_deleted_layer( and server.layer_route_state.layers[layer_key] not in server.viewer.layers ): num_items = server.component_groups.item_count(layer_key) - server.layer_route_state.purge_route(layer_key) - server.component_groups.purge(layer_key) - server.component_values.purge(layer_key) + NapariComponentAwareDisplayCoordinator.purge_route(server, layer_key) logger.info( "🔬 NAPARI PROCESS: Reconciling state — '%s' was deleted from viewer; " "purged stale caches (had %d items in component_groups)", @@ -1044,6 +1046,47 @@ def _reconcile_deleted_layer( num_items, ) + @staticmethod + def purge_route(server: "NapariViewerServer", route_key: str) -> None: + """The one route-release recipe, for native deletion and explicit retirement.""" + server.layer_route_state.purge_route(route_key) + server.display_pipeline.purge_display_work(route_key) + server.component_groups.purge(route_key) + server.component_values.purge(route_key) + server.batch_processors.purge(route_key) + + def retire_layers( + self, server: "NapariViewerServer", request: ViewerWindowLayerRetirementRequest, + ) -> tuple[str, ...]: + """Validate the whole observed set before any ordinary native removal.""" + with server.layer_route_state.mutation_boundary(): + viewer = server.require_viewer() + request.control_deadline().remaining_seconds() + server.layer_route_state.require_retirement_boundary() + server.display_pipeline.require_retirement_boundary() + if not server.accepted_stream_batches.empty(): + raise RuntimeError("Cannot retire layers with accepted intake; settle first.") + routes = tuple(request.retirement.expected_producers) + for route, expected in request.retirement.expected_producers.items(): + if not server.layer_route_state.has_layer(route): + raise ValueError(f"Retirement route {route!r} is not mounted.") + layer = server.layer_route_state.layer(route) + if layer not in viewer.layers: + raise ValueError(f"Retirement route {route!r} is already deleted.") + actual = server.component_groups.producer_identities_for(route) + if actual != frozenset(expected): + raise ValueError(f"Retirement producer incarnation changed for {route!r}.") + viewport = NapariNativeViewportPresentation.for_viewer(viewer) + presentation = viewport.snapshot() if viewport is not None else None + request.control_deadline().remaining_seconds() + for route in routes: + viewer.layers.remove(server.layer_route_state.layer(route)) + self.purge_route(server, route) + server.display_pipeline.reconcile_mounted_axis_projections(rematerialize=True) + if viewport is not None: + viewport.apply(presentation) + return routes + @staticmethod def _group_for( server: "NapariViewerServer", @@ -1740,7 +1783,20 @@ def publish(self) -> None: self.presentation, display_config=self.display_config, ) - self.pipeline.reconcile_mounted_axis_projections(updated_route_key=route_key) + self.reconcile_peers() + + def reconcile_peers(self) -> None: + self.pipeline.reconcile_mounted_axis_projections( + updated_route_key=self.presentation.route_key, + ) + + +@dataclass(frozen=True, slots=True) +class NapariRematerializationRequest(NapariLayerDisplayRequest): + """The original reconciliation traversal already owns every survivor.""" + + def reconcile_peers(self) -> None: + pass class NapariLayerDisplayWork(ABC): @@ -1773,6 +1829,21 @@ class NapariLayerDisplayHandler( title_suffix: ClassVar[str] = "" + @contextmanager + def preserve_native_presentation(self, request: NapariLayerDisplayRequest): + """Shared native properties survive domain pruning and remounting.""" + state = request.pipeline.server.layer_route_state + route = request.presentation.route_key + layer = state.layer(route) + visible, opacity, blending = layer.visible, layer.opacity, layer.blending + yield + layer = state.layer(route) + layer.visible, layer.opacity, layer.blending = visible, opacity, blending + + def rematerialize(self, request: NapariLayerDisplayRequest) -> None: + with self.preserve_native_presentation(request): + self.handle(request) + def geometric_component_values( self, items: Sequence[NapariStreamLayerItem], @@ -1790,8 +1861,28 @@ def display_work( return NapariImmediateLayerDisplayWork(lambda: self.handle(request)) +class NapariImagePresentationRetention: + """Independent image presentation capability, cooperating through the MRO.""" + + @contextmanager + def preserve_native_presentation(self, request: NapariLayerDisplayRequest): + state = request.pipeline.server.layer_route_state + route = request.presentation.route_key + layer = state.layer(route) + control = NapariNativeImageIntensityPresentation.for_layer(layer) + intensity = control.snapshot() + colormap, interpolation = layer.colormap, layer.interpolation2d + with super().preserve_native_presentation(request): + yield + layer = state.layer(route) + NapariNativeImageIntensityPresentation.for_layer(layer).apply(intensity) + layer.colormap, layer.interpolation2d = colormap, interpolation + + @dataclass(frozen=True, slots=True) -class NapariImageLayerDisplayHandler(NapariLayerDisplayHandler): +class NapariImageLayerDisplayHandler( + NapariImagePresentationRetention, NapariLayerDisplayHandler, +): """Build or update a Napari image layer from routed image payloads.""" streaming_data_type: ClassVar[StreamingDataType] = StreamingDataType.IMAGE @@ -2165,6 +2256,13 @@ def clear_display_work(self) -> None: self._display_work_by_route.clear() + def purge_display_work(self, route_key: str) -> None: + self._display_work_by_route.pop(route_key, None) + + def require_retirement_boundary(self) -> None: + if self._display_work_by_route: + raise RuntimeError("Cannot retire layers with unfinished display work.") + def display_axis_projection( self, layer_key: str, @@ -2246,8 +2344,8 @@ def reconcile_mounted_axis_projections( if rematerialize: NapariLayerDisplayHandler.for_data_type( items[0].address.stream_layer_data_type - ).handle( - NapariLayerDisplayRequest( + ).rematerialize( + NapariRematerializationRequest( pipeline=self, items=items, presentation=presentation, @@ -2289,9 +2387,9 @@ def schedule_layer_update( display_config=layer_route.display_config, items=items, ) - timer.timeout.connect( - lambda: self.execute_scheduled_layer_update(layer_key, update) - ) + timer.timeout.connect(update.retained_callback( + partial(self.execute_scheduled_layer_update, layer_key), + )) self.server.layer_batch_processor_debounce_policy.start_timer(timer) self.server.layer_route_state.set_pending_update(layer_key, update) logger.debug( @@ -2319,7 +2417,7 @@ def execute_scheduled_layer_update( return QTimer.singleShot( NAPARI_SETTLEMENT_UPDATE_YIELD_MS, - lambda: self.execute_scheduled_layer_update(layer_key, update), + update.retained_callback(partial(self.execute_scheduled_layer_update, layer_key)), ) except Exception as error: self.server.layer_route_state.record_update_error(layer_key, error) @@ -2428,11 +2526,7 @@ def _schedule_next_settlement_update( route_key, update = claimed_update QTimer.singleShot( NAPARI_SETTLEMENT_UPDATE_YIELD_MS, - lambda: self._execute_settlement_update( - settlement, - route_key, - update, - ), + update.retained_callback(partial(self._execute_settlement_update, settlement, route_key)), ) def _execute_settlement_update( @@ -2453,11 +2547,7 @@ def _execute_settlement_update( settlement.complete_active_work_unit(route_key) QTimer.singleShot( NAPARI_SETTLEMENT_UPDATE_YIELD_MS, - lambda: self._execute_settlement_update( - settlement, - route_key, - update, - ), + update.retained_callback(partial(self._execute_settlement_update, settlement, route_key)), ) return except Exception as error: @@ -3825,7 +3915,7 @@ def layer_state_for(self, route_key: str) -> dict[str, NapariWireValue]: payload_summaries = self.payload_summaries_for(dimension_state, item_tuple) producer_identities = tuple( sorted( - {item.producer for item in item_tuple}, + self.server.component_groups.producer_identities_for(route_key), key=lambda producer: producer.output_key, ) ) @@ -5626,6 +5716,33 @@ def _validate_local_axis_index( ) +class NapariLayerRetirementControlMessageAction(NapariControlMessageAction): + """Qt-owned registered action; the coordinator owns the retirement recipe.""" + + message_type = OpenHCSViewerControlMessageType.RETIRE_LAYERS.value + + def handle(self, server, message) -> dict[str, object]: + try: + request = message.get(ViewerControlResponseField.PAYLOAD.value) + if not isinstance(request, ViewerWindowLayerRetirementRequest): + raise TypeError("Retirement requires ViewerWindowLayerRetirementRequest.") + retired = _NAPARI_COMPONENT_DISPLAY_COORDINATOR.retire_layers(server, request) + receipt = ViewerLayerRetirementReceipt( + applied=True, retired_route_keys=retired, + remaining_route_keys=tuple(server.layer_route_state.layers), + ) + response = ViewerControlReplyHeader( + ViewerProtocolStatus.SUCCESS, response_type="layer_retirement_ack", + ).to_wire_mapping() + response[ViewerControlField.RETIREMENT.value] = receipt.to_wire_mapping() + return response + except Exception as error: + return ViewerControlReplyHeader( + ViewerProtocolStatus.ERROR, response_type="layer_retirement_ack", + message=str(error), + ).to_wire_mapping() + + class NapariLayerIsolationControlMessageAction(NapariControlMessageAction): """Apply one atomic visibility and selection projection to mounted layers.""" @@ -6222,6 +6339,12 @@ def record_transport_failure(self, error: Exception) -> None: self.transport_failure = error + def require_viewer(self): + """Resolve native availability at its original lifecycle owner.""" + if self.viewer is None: + raise RuntimeError("Napari viewer is not available.") + return self.viewer + def request_shutdown(self) -> None: """Stop accepting display work and cancel every deferred layer update.""" @@ -6274,8 +6397,7 @@ def clear_accumulated_stream_state(self) -> None: | self.component_groups.groups.keys() ): if route_key not in retained_routes: - self.component_groups.purge(route_key) - self.layer_route_state.purge_route(route_key) + _NAPARI_COMPONENT_DISPLAY_COORDINATOR.purge_route(self, route_key) self.component_values.retain_routes(retained_routes) self.display_pipeline.reconcile_mounted_axis_projections(rematerialize=True) if not retained_routes: @@ -6344,8 +6466,9 @@ def accept_stream_message(self, message: bytes) -> NapariStreamMessageReply: ).accept(self, data) # Receipt certifies ownership, not display completion. Invalidate a # prior terminal settlement before acknowledging this new work. - self.layer_route_state.reset_settlement() - self.accepted_stream_batches.put(accepted_batch) + with self.layer_route_state.mutation_boundary(): + self.layer_route_state.reset_settlement() + self.accepted_stream_batches.put(accepted_batch) return NapariStreamMessageReply.success(msg_type) except Exception as error: logger.error( diff --git a/openhcs/runtime/viewer_controls.py b/openhcs/runtime/viewer_controls.py index da48933956..4d802d354e 100644 --- a/openhcs/runtime/viewer_controls.py +++ b/openhcs/runtime/viewer_controls.py @@ -9,7 +9,10 @@ from numbers import Real from typing import TYPE_CHECKING, ClassVar, Self, TypeAlias, TypeVar -from polystore.streaming.identity import StreamProducerIdentity +from polystore.streaming.identity import ( + StreamProducerIdentity, + StreamProducerPayloadMapping, +) from zmqruntime.viewer_protocol import ( ViewerNativeLayerTransform, ViewerSourceSpatialDomainPayload, @@ -867,6 +870,37 @@ def from_overrides( ) +@dataclass(frozen=True, slots=True, kw_only=True) +class ViewerLayerRetirementControlOptions: + """Explicit routes guarded by their complete observed producer incarnations.""" + + expected_producers: Mapping[str, tuple[StreamProducerIdentity, ...]] + + def __post_init__(self) -> None: + if not self.expected_producers: + raise ValueError("Layer retirement requires explicit route identities.") + for route, producers in self.expected_producers.items(): + if not isinstance(route, str) or not route: + raise ValueError("Retirement route keys must be non-empty strings.") + if not producers or any( + not isinstance(producer, StreamProducerIdentity) for producer in producers + ): + raise ValueError("Retirement requires full typed producer identities.") + if len(frozenset(producers)) != len(producers): + raise ValueError("Retirement producer identities must be distinct.") + + @classmethod + def from_overrides( + cls, *, expected_producers: Mapping[ + str, Sequence[StreamProducerIdentity | StreamProducerPayloadMapping] + ], + ) -> Self: + return cls(expected_producers={ + route: tuple(StreamProducerIdentity.from_payload(value) for value in values) + for route, values in expected_producers.items() + }) + + @dataclass(frozen=True, slots=True, kw_only=True) class ViewerLayerIsolationControlOptions: """Atomic visibility, selection, and navigation for a viewer layer set.""" diff --git a/openhcs/runtime/viewer_protocol.py b/openhcs/runtime/viewer_protocol.py index 12d4911221..2404c162b4 100644 --- a/openhcs/runtime/viewer_protocol.py +++ b/openhcs/runtime/viewer_protocol.py @@ -491,6 +491,7 @@ class ViewerControlField(str, Enum): NATIVE_IMAGE_COLOR = "native_image_color" NATIVE_WINDOW = "native_window" NATIVE_DIMENSIONS = "native_dimensions" + RETIREMENT = "retirement" COMPONENT_GROUP_COUNT = "component_group_count" COMPONENT_ITEM_COUNT = "component_item_count" PROCESS_LAUNCH = "process_launch" @@ -504,6 +505,7 @@ class OpenHCSViewerControlMessageType(str, Enum): MEASURE_REGION = "measure_region" IMAGE_COLOR = "image_color" WINDOW_PRESENTATION = "window_presentation" + RETIRE_LAYERS = "retire_layers" class ViewerLayerIsolationField(str, Enum): @@ -514,6 +516,15 @@ class ViewerLayerIsolationField(str, Enum): MISSING_ROUTE_KEYS = "missing_route_keys" +@dataclass(frozen=True, kw_only=True) +class ViewerLayerRetirementReceipt(ViewerProjectionRecord): + """One native retirement observation, projected into the agent envelope.""" + + applied: bool = False + retired_route_keys: tuple[str, ...] = () + remaining_route_keys: tuple[str, ...] = () + + class ViewerIntensityWindowField(str, Enum): """Route-global image intensity-window response fields.""" diff --git a/tests/unit/test_napari_reset_transitions.py b/tests/unit/test_napari_reset_transitions.py index d4e86d5a8a..17903916c0 100644 --- a/tests/unit/test_napari_reset_transitions.py +++ b/tests/unit/test_napari_reset_transitions.py @@ -1,6 +1,12 @@ """Real Qt scheduling and native Napari model transitions, no server or GL canvas.""" import threading +import asyncio +import pickle +import queue +import weakref +from contextlib import contextmanager +from dataclasses import replace import numpy as np import pytest @@ -25,14 +31,29 @@ NapariStreamLayerAddress, ) from openhcs.runtime.napari_viewer_server import ( + NapariAcceptedControlRequest, + NapariControlMessageAction, NapariComponentAwareDisplayCoordinator, NapariImagePayloadLayoutRole, NapariLayerDisplayPipeline, + NapariImageLayerDisplayHandler, + NapariImagePresentationRetention, NapariShapesLayerDisplayHandler, NapariResultSelectionController, NapariStreamLayerContext, NapariViewerServer, ) +from openhcs.agent.dto.execution import ExecutionConnectionSpec +from openhcs.agent.dto.viewer import ViewerWindowLayerRetirementRequest +from openhcs.agent.services.viewer_window_service import ( + ViewerWindowService, ZMQViewerWindowGateway, +) +from openhcs.mcp.context import OpenHCSAgentContext +from openhcs.mcp.server import build_server +from openhcs.runtime.viewer_controls import ViewerLayerRetirementControlOptions +from openhcs.runtime.viewer_protocol import ( + OpenHCSViewerControlMessageType, ViewerSettlePhase, +) from openhcs.runtime.viewer_component_system import ( ViewerComponentAxisSemanticsAuthority, ViewerComponentNameMetadata, @@ -56,6 +77,8 @@ def receiver(): server.layer_batch_processor_debounce_policy = NapariLayerBatchDebouncePolicy() server.batch_processors = NapariBatchProcessorStore() server.display_pipeline = NapariLayerDisplayPipeline(server) + server.accepted_stream_batches = queue.Queue() + server.accepted_control_requests = queue.Queue() # Exercise native selection event binding, without mounting an unrelated Qt dock. server.result_selection_controller = NapariResultSelectionController(server) server.bind_result_selection_layer = server.result_selection_controller.bind @@ -351,3 +374,242 @@ def test_native_deletion_with_queued_replacement_clear_cannot_resurrect_route(re assert not receiver.layer_route_state.layer_titles assert not receiver.component_groups.groups assert not receiver.component_values.domains + + +def retirement_request(server, *routes): + return ViewerWindowLayerRetirementRequest.from_fields( + connection=ExecutionConnectionSpec(port=5584), + expected_producers={ + route: [producer.to_payload() for producer in + server.component_groups.producer_identities_for(route)] + for route in routes + }, + ) + + +class QueuedRetirementGateway(ZMQViewerWindowGateway): + """Exercise the real gateway message hook and Qt queue, with no socket/server.""" + + def __init__(self, server): + self.server = server + + def _send_control_message(self, request, message): + assert request.operation_deadline is not None + assert request.control_deadline() is request.operation_deadline + reply = queue.Queue(maxsize=1) + self.server.accepted_control_requests.put(NapariAcceptedControlRequest( + pickle.loads(pickle.dumps(message)), reply, + )) + loop = QEventLoop() + def dispatch(): + self.server.process_messages() + loop.quit() + QTimer.singleShot(0, dispatch) + loop.exec() + response = pickle.loads(reply.get_nowait()) + assert reply.empty() + return response + + +def test_selected_retirement_uses_registered_queue_and_releases_payloads(receiver): + raw = np.full((2, 2), 4, dtype=np.uint16) + keep, update = enqueue(receiver, raw, producer="raw") + advance_in_qt(receiver, keep, update) + doomed = np.full((2, 2), 9, dtype=np.uint16) + payload_ref = weakref.ref(doomed) + route, update = enqueue(receiver, doomed, producer="candidate") + # Genuine original terminal settlement, not a manually fabricated complete flag. + receiver.display_pipeline.settlement_progress() + app = QApplication.instance() + while receiver.layer_route_state.existing_settlement_progress().phase is ViewerSettlePhase.RUNNING: + app.processEvents() + assert receiver.layer_route_state.existing_settlement_progress().phase is ViewerSettlePhase.COMPLETE + native = receiver.layer_route_state.layer(route) + native_ref = weakref.ref(native) + receiver.batch_processors.get_or_create(layer_key=route, napari_server=receiver) + request = retirement_request(receiver, route) + keep_native = receiver.layer_route_state.layer(keep) + original_items = receiver.component_groups.existing_items_for(keep) + del update, doomed, native + result = ViewerWindowService(QueuedRetirementGateway(receiver)).presentation(request) + assert result.applied and not result.errors and result.observed + assert result.retired_route_keys == (route,) + assert result.remaining_route_keys == (keep,) + assert route not in receiver.layer_route_state.layer_titles + assert route not in receiver.layer_route_state.layer_dimension_states + assert route not in receiver.batch_processors.processors + assert route not in receiver.component_groups.groups + assert not any(key[0] == route for key in receiver.component_values.domains) + assert not receiver.layer_route_state.layer_settlement.updates + assert receiver.layer_route_state.layer(keep) is keep_native + assert receiver.component_groups.existing_items_for(keep) is original_items + assert original_items[0].data is raw + import gc + gc.collect() + assert payload_ref() is None and native_ref() is None + app.processEvents() + assert route not in receiver.layer_route_state.layers # No late resurrection. + + +def test_real_fastmcp_retirement_decodes_identity_and_reaches_native_queue(receiver): + from mcp.server.fastmcp.exceptions import ToolError + + raw = np.ones((2, 2), dtype=np.uint16) + keep, update = enqueue(receiver, raw, producer='raw') + advance_in_qt(receiver, keep, update) + route, update = enqueue(receiver, np.full((2, 2), 9), producer='retired') + advance_in_qt(receiver, route, update) + keep_native = receiver.layer_route_state.layer(keep) + context = OpenHCSAgentContext( + viewer_window_service=ViewerWindowService(QueuedRetirementGateway(receiver)), + ) + mcp = build_server(context=context) + tool = mcp._tool_manager.get_tool('openhcs_retire_viewer_window_layers') + arguments = retirement_request(receiver, route).as_tool_arguments() + # Invalid schema never enters the native queue or mutates either layer. + with pytest.raises(ToolError): + asyncio.run(tool.run({**arguments, 'expected_producers': {route: [{}]}})) + assert receiver.accepted_control_requests.empty() + assert receiver.layer_route_state.layer(route) is not None + assert receiver.layer_route_state.layer(keep) is keep_native + result = asyncio.run(tool.run(arguments)) + assert result['applied'] and result['observed'] and not result['errors'] + assert result['retired_route_keys'] == [route] + assert result['remaining_route_keys'] == [keep] + assert receiver.layer_route_state.layer(keep) is keep_native + assert receiver.component_groups.existing_items_for(keep)[0].data is raw + assert route not in receiver.layer_route_state.layers + + +@pytest.mark.parametrize("data_type,payload", [ + (StreamingDataType.SHAPES, [{"type": "polygon", "coordinates": [[0, 0], [0, 1], [1, 1]], + "metadata": {"label": 7}}]), + (StreamingDataType.POINTS, [{"type": "points", "coordinates": [[1, 2]], + "metadata": {"label": 7}}]), +]) +def test_registered_geometry_family_retires_through_same_queue(receiver, data_type, payload): + keep, update = enqueue(receiver, np.ones((2, 2)), producer="source") + advance_in_qt(receiver, keep, update) + original_items = receiver.component_groups.existing_items_for(keep) + route, update = enqueue(receiver, payload, producer="geometry", data_type=data_type) + advance_in_qt(receiver, route, update) + layer_ref = weakref.ref(receiver.layer_route_state.layer(route)) + update_ref = weakref.ref(update) + del update + result = ViewerWindowService(QueuedRetirementGateway(receiver)).presentation( + retirement_request(receiver, route), + ) + assert result.applied and not result.errors + assert result.retired_route_keys == (route,) and result.remaining_route_keys == (keep,) + assert receiver.component_groups.existing_items_for(keep) is original_items + import gc + gc.collect() + assert layer_ref() is None and update_ref() is None + QApplication.instance().processEvents() + assert route not in receiver.layer_route_state.layers + + +@pytest.mark.parametrize("blocked", ["pending", "intake", "display", "running", "stale", "missing", "deadline"]) +def test_retirement_prevalidates_entire_set_without_collateral_mutation(receiver, blocked): + first, update = enqueue(receiver, np.ones((2, 2)), producer="first") + advance_in_qt(receiver, first, update) + second, update = enqueue(receiver, np.full((2, 2), 2), producer="second") + advance_in_qt(receiver, second, update) + request = retirement_request(receiver, first, second) + if blocked == "pending": + enqueue(receiver, np.full((2, 2), 3), producer="second") + elif blocked == "intake": + receiver.accepted_stream_batches.put(object()) + elif blocked == "display": + receiver.display_pipeline._display_work_by_route[second] = (update, object()) + elif blocked == "running": + enqueue(receiver, np.full((2, 2), 3), producer="second") + receiver.layer_route_state.begin_settlement() + elif blocked == "stale": + items = receiver.component_groups.existing_items_for(second) + items[0] = replace(items[0], producer=replace(items[0].producer, invocation_key="new-incarnation")) + elif blocked == "missing": + expected = dict(request.retirement.expected_producers) + expected["foreign"] = expected[second] + request = replace(request, retirement=ViewerLayerRetirementControlOptions(expected_producers=expected)) + elif blocked == "deadline": + from zmqruntime.timeouts import OperationDeadline + request = replace(request, operation_deadline=OperationDeadline("expired", 5000, 0)) + layers = tuple(receiver.viewer.layers) + groups = dict(receiver.component_groups.groups) + response = NapariControlMessageAction.for_message_type( + OpenHCSViewerControlMessageType.RETIRE_LAYERS.value, + ).handle(receiver, {"payload": request}) + assert response["status"] == "error" + assert tuple(receiver.viewer.layers) == layers + assert receiver.component_groups.groups == groups + assert first in receiver.layer_route_state.layers and second in receiver.layer_route_state.layers + # Dispose only this test's synthetic unsettled state through original owners. + if blocked == "running": + receiver.layer_route_state.layer_settlement.fail() + + +def test_terminal_failed_candidate_is_retirable_without_erasing_other_failure(receiver): + keep, update = enqueue(receiver, np.ones((2, 2)), producer="keep") + advance_in_qt(receiver, keep, update) + route, update = enqueue(receiver, np.full((2, 2), 2), producer="failed") + advance_in_qt(receiver, route, update) + enqueue(receiver, np.full((2, 2), 3), producer="failed") + settlement = receiver.layer_route_state.begin_settlement() + claimed, _ = settlement.begin_next() + settlement.begin_active_work_unit(claimed) + settlement.fail_active(claimed) + receiver.layer_route_state.record_update_error(route, ValueError("retained original failure")) + receiver.layer_route_state.record_update_error(keep, ValueError("other original failure")) + result = ViewerWindowService(QueuedRetirementGateway(receiver)).presentation(retirement_request(receiver, route)) + assert result.applied and not result.errors + assert settlement.phase is ViewerSettlePhase.FAILED + assert not settlement.updates + assert receiver.layer_route_state.layer_update_errors == {keep: "other original failure"} + assert keep in receiver.layer_route_state.layers + + +def test_retirement_prunes_multiple_survivors_with_cooperative_presentation_hooks(receiver, monkeypatch): + calls = [] + monkeypatch.setitem(NapariImageLayerDisplayHandler.__registry__, StreamingDataType.IMAGE, NapariImageLayerDisplayHandler) + class IndependentPresentationCapability: + @contextmanager + def preserve_native_presentation(self, request): + calls.append(("enter", request.presentation.route_key)) + with super().preserve_native_presentation(request): + yield + calls.append(("exit", request.presentation.route_key)) + class NewDeclaredImageHandler(IndependentPresentationCapability, NapariImageLayerDisplayHandler): + streaming_data_type = StreamingDataType.IMAGE + # Class declaration is the only new case; the original generic consumer is untouched. + middle, update = enqueue(receiver, np.full((2, 2), 2), well="A02", producer="middle") + advance_in_qt(receiver, middle, update) + originals = {} + for name in ("sparse-a", "sparse-b"): + route, _ = enqueue(receiver, np.ones((2, 2)), well="A01", producer=name, domain=["A01", "A03"]) + route, update = enqueue(receiver, np.full((2, 2), 3), well="A03", producer=name, domain=["A01", "A03"]) + advance_in_qt(receiver, route, update) + layer = receiver.layer_route_state.layer(route) + layer.contrast_limits, layer.gamma = (0, 10), 0.7 + layer.opacity, layer.visible = 0.4, False + layer.colormap = "magenta" + originals[route] = receiver.component_groups.existing_items_for(route) + receiver.viewer.camera.center = (0, 1, 1) + receiver.viewer.camera.zoom = 31 + result = ViewerWindowService(QueuedRetirementGateway(receiver)).presentation(retirement_request(receiver, middle)) + assert result.applied and not result.errors + assert len(calls) == 4 + for route, items in originals.items(): + layer = receiver.layer_route_state.layer(route) + assert layer.data.shape == (2, 2, 2) + np.testing.assert_array_equal(layer.data[0], items[0].data) + np.testing.assert_array_equal(layer.data[1], items[1].data) + assert receiver.component_groups.existing_items_for(route) is items + assert tuple(layer.scale[-2:]) == (0.65, 0.65) + assert layer.gamma == 0.7 and tuple(layer.contrast_limits) == (0, 10) + assert layer.opacity == 0.4 and not layer.visible + assert layer.colormap.name == "magenta" + assert receiver.viewer.camera.zoom == 31 + assert tuple(receiver.viewer.camera.center) == (0, 1, 1) + assert receiver.component_values.shared_values_for(["well"]) == {"well": ["A01", "A03"]} + assert issubclass(NewDeclaredImageHandler, NapariImagePresentationRetention) diff --git a/tests/unit/test_napari_streaming_handlers.py b/tests/unit/test_napari_streaming_handlers.py index 4e3a241559..e40c7435af 100644 --- a/tests/unit/test_napari_streaming_handlers.py +++ b/tests/unit/test_napari_streaming_handlers.py @@ -4546,6 +4546,8 @@ def __init__(self): self.layer_route_state = NapariLayerRouteStateStore.empty() self.component_groups = NapariComponentGroupStore() self.component_values = ViewerRouteComponentValueTracker() + self.batch_processors = napari_viewer_server.NapariBatchProcessorStore() + self.display_pipeline = napari_viewer_server.NapariLayerDisplayPipeline(self) server = _FakeServer() route_key = "deleted-channel-1" diff --git a/tests/unit/test_viewer_retirement_contract.py b/tests/unit/test_viewer_retirement_contract.py new file mode 100644 index 0000000000..b26e7bca43 --- /dev/null +++ b/tests/unit/test_viewer_retirement_contract.py @@ -0,0 +1,171 @@ +"""Declaration-owned MCP/CLI request, receipt and error projection; no server launch.""" +import argparse +import asyncio +import json +from types import SimpleNamespace + +import pytest +from polystore.streaming.identity import StreamProducerIdentity +from zmqruntime.config import TransportMode + +from openhcs.agent.capabilities import RetireViewerWindowLayersCapability +from openhcs.agent.dto.common import SCHEMA_VERSION +from openhcs.agent.dto.execution import ExecutionConnectionSpec +from openhcs.agent.dto.viewer import ( + ViewerWindowLayerRetirementRequest, ViewerWindowLayerRetirementResult, +) +from openhcs.agent.services.viewer_window_service import ( + ViewerWindowService, ZMQViewerWindowGateway, +) +from openhcs.mcp.server import ( + GeneratedMcpViewerRequestToolBinding, build_server, + generated_viewer_request_capability_declarations, +) +from openhcs.mcp.dev_client_commands.viewer import RetireViewerCommandSpec +from openhcs.runtime.viewer_protocol import ViewerLayerRetirementReceipt +from openhcs.serialization.json import to_jsonable + + +def producers(): + return {"exact-route": [to_jsonable(StreamProducerIdentity( + origin="pipeline", output_kind="main", output_key="candidate", + projection_key="candidate", invocation_key="original-incarnation", + ))]} + + +def request(): + return ViewerWindowLayerRetirementRequest.from_fields( + connection=ExecutionConnectionSpec(port=5584, transport_mode=TransportMode.TCP), + expected_producers=producers(), + ) + + +def test_generated_mcp_binding_discovers_and_executes_new_declaration_without_consumer_edits(): + assert RetireViewerWindowLayersCapability in generated_viewer_request_capability_declarations() + captured = [] + received = [] + class Service: + def presentation(self, value): + received.append(value) + return ViewerWindowLayerRetirementResult( + schema_version=SCHEMA_VERSION, connection=value.connection, + observed=True, applied=True, retired_route_keys=("exact-route",), + remaining_route_keys=("retained-raw",), + ) + def tool_decorator(*, capability): + def register(function): + captured.append((capability, function)) + return function + return register + # This is the original generated callable/codec/invocation, not a mirrored tool. + GeneratedMcpViewerRequestToolBinding.bind_to_server( + RetireViewerWindowLayersCapability, + SimpleNamespace(viewer_window_service=Service()), tool_decorator, + ) + capability, tool = captured[0] + result = tool(port=5584, transport_mode="tcp", expected_producers=producers()) + assert capability.name == "openhcs_retire_viewer_window_layers" + assert len(received) == 1 and isinstance(received[0], ViewerWindowLayerRetirementRequest) + assert received[0].retirement.expected_producers["exact-route"][0].invocation_key == "original-incarnation" + assert result["applied"] and result["retired_route_keys"] == ["exact-route"] + assert result["remaining_route_keys"] == ["retained-raw"] + assert "operation_deadline" not in tool.__signature__.parameters + + +def test_cli_leaf_projects_same_typed_request_and_connection(): + command = RetireViewerCommandSpec() + parser = argparse.ArgumentParser() + command.configure_parser(parser) + arguments = command.tool_arguments(parser.parse_args([ + "5584", "--transport-mode", "tcp", "--expected-producers", json.dumps(producers()), + ])) + assert arguments == request().as_tool_arguments() + + +def test_real_fastmcp_constructs_and_decodes_original_producer_identity(): + # A fake registration decorator cannot exercise FastMCP's schema generation. + server = build_server() + capability = RetireViewerWindowLayersCapability.to_spec() + registered = server._tool_manager.get_tool(capability.name) + model = registered.fn_metadata.arg_model + payload = request().as_tool_arguments() + arguments = model.model_validate(payload) + identity = arguments.expected_producers['exact-route'][0] + assert isinstance(identity, StreamProducerIdentity) + assert identity == request().retirement.expected_producers['exact-route'][0] + projected = ViewerWindowLayerRetirementRequest.from_fields( + connection=request().connection, + expected_producers=arguments.expected_producers, + ) + assert projected.as_tool_arguments() == payload + tools = asyncio.run(server.list_tools()) + advertised = next(tool for tool in tools if tool.name == capability.name) + producer_schema = advertised.inputSchema['$defs']['StreamProducerIdentity'] + assert set(producer_schema['properties']) == set(identity.to_payload()) + from pydantic import ValidationError + with pytest.raises(ValidationError): + model.model_validate({**payload, 'expected_producers': {'exact-route': [{}]}}) + with pytest.raises(ValidationError): + model.model_validate({**payload, 'undeclared': 'must not dispatch'}) + + +def test_real_fastmcp_producer_admission_is_owned_by_original_decoder(): + server = build_server() + model = server._tool_manager.get_tool( + RetireViewerWindowLayersCapability.name, + ).fn_metadata.arg_model + payload = request().as_tool_arguments() + source = producers()['exact-route'][0] + # Preserve the original owner's policy, including conversions that differ + # from Pydantic's default dataclass validation and ignored extra wire keys. + for changes in ( + {'origin': 0}, {'origin': False}, {'origin': True}, + {'pipeline_position': True}, {'pipeline_position': False}, + {'pipeline_position': '12'}, {'pipeline_position': 12}, + {'step_name': ''}, {'step_name': 0}, {'step_scope_id': False}, + {'invocation_key': ''}, {'extra_owner_ignored_field': 'retained policy'}, + ): + original = {**source, **changes} + expected = StreamProducerIdentity.from_payload(original) + decoded = model.model_validate({ + **payload, 'expected_producers': {'exact-route': [original]}, + }).expected_producers['exact-route'][0] + assert decoded == expected + assert decoded.to_payload() == expected.to_payload() + for changes in ( + {'origin': ''}, {'origin': None}, {'output_kind': ''}, + {'output_key': None}, {'projection_key': ''}, + {'pipeline_position': 'not-an-integer'}, + ): + original = {**source, **changes} + with pytest.raises((TypeError, ValueError)): + StreamProducerIdentity.from_payload(original) + with pytest.raises((TypeError, ValueError)): + model.model_validate({ + **payload, 'expected_producers': {'exact-route': [original]}, + }) + + +@pytest.mark.parametrize("native", [ + {"status": "error", "message": "original terminal error"}, + {"status": "success", "retirement": {"applied": True, "retired_route_keys": ["foreign"], "remaining_route_keys": []}}, + {"status": "success", "retirement": {"applied": True, "retired_route_keys": ["exact-route"], "remaining_route_keys": ["exact-route"]}}, +]) +def test_service_rejects_failed_or_mismatched_retirement_receipt(native): + class NativeReplyGateway(ZMQViewerWindowGateway): + def _send_control_message(self, value, message): + assert value.operation_deadline is not None + return native + result = ViewerWindowService(NativeReplyGateway()).presentation(request()) + assert not result.applied and not result.observed and result.errors + assert not result.retired_route_keys + + +def test_receipt_uses_original_declaration_codec_and_envelope_error_mro(): + receipt = ViewerLayerRetirementReceipt.from_wire_mapping({ + "applied": True, "retired_route_keys": ["exact-route"], "remaining_route_keys": ["raw"], + }) + assert receipt.retired_route_keys == ("exact-route",) + with pytest.raises(ValueError, match="undeclared"): + ViewerLayerRetirementReceipt.from_wire_mapping({"extra": 1}) + assert request().start_operation().control_deadline().timeout_ms == 5000