From 23f13d1be30aaa5efdf64692e24f81d2e9a15284 Mon Sep 17 00:00:00 2001 From: chouswei <104312007+chouswei@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:29:42 +0800 Subject: [PATCH 1/4] Model MN-REQ-06.19 restart-safe transport, MN-VER-06-S17 Restart or drain of serve, memnet-mcp-http or the gateway pauses callers by bounded retry; no MCP session lost; a write is never replayed after send. ServeRetryMod (memnet.serve_retry) with allocation serveRetryToMod, ServeClientRetry on TcpServeBridge, ServeRetryChoice action, McpServerMod statelessHttp/jsonResponse. MN-REQ-06.14 'no client retry' now points at 06.19. --- sysml-models/models/behaviour.sysml | 18 +++++++ sysml-models/models/deploy.sysml | 32 +++++++++-- sysml-models/models/implementation.sysml | 41 ++++++++++++++ sysml-models/models/requirements.sysml | 52 ++++++++++++++++-- sysml-models/models/verify.sysml | 54 ++++++++++++++++++- .../outputs/ssot-to-code-allocate-map.md | 1 + 6 files changed, 190 insertions(+), 8 deletions(-) diff --git a/sysml-models/models/behaviour.sysml b/sysml-models/models/behaviour.sysml index 7914954..d6b0c35 100644 --- a/sysml-models/models/behaviour.sysml +++ b/sysml-models/models/behaviour.sysml @@ -1785,6 +1785,24 @@ package MemNetBehaviour { out item matched : String; } + action def ServeRetryChoice { + doc /* + Implemented (MN-REQ-06.19; verify MN-VER-06-S17; tests + tests/test_serve_restart_retry.py). One serve request, client + side (ServePool.send, serve_pool.py; classes in serve_retry.py). + (1) Not delivered (connect refused or reset, frame not fully + written): wait backoff (0.1 s doubling, cap 2 s, jitter), retry + any request. (2) Reply @ERR: serve_draining|retry_after_s=N: + wait at least N, retry any request. (3) Reset after the frame + was written: READ waits backoff and retries; WRITE surfaces the + error, delivered once. (4) Each wait is clamped to what is left + of MEMNET_SERVE_RETRY_S (default 30 s); when none is left, the + last connect error or the draining envelope surfaces. + */ + in item request : String; + out item reply : String; + } + action def RestartRestoreChoice { doc /* Implemented (MN-REQ-06.17; verify MN-VER-06-S15; tests diff --git a/sysml-models/models/deploy.sysml b/sysml-models/models/deploy.sysml index ce90e10..5c76e2b 100644 --- a/sysml-models/models/deploy.sysml +++ b/sysml-models/models/deploy.sysml @@ -1437,8 +1437,9 @@ package MemNet { restore of the upgrade set retires the manifest in that same startup; retire is an audit mark and no longer stops a later restore: every start restores the newest valid - set, checkpoint or upgrade-prepare (MN-REQ-06.17). No - client retry. No systemd or gateway-pin helper. + set, checkpoint or upgrade-prepare (MN-REQ-06.17). Clients + pause through it (ServeClientRetry, MN-REQ-06.19). No systemd + or gateway-pin helper. */ attribute implemented : Boolean = true; attribute noSemVerBump : Boolean = true; @@ -1527,6 +1528,28 @@ package MemNet { satisfy MN_REQ_00_MissionBridge::MN_REQ_07_McpAgentBoundary::MN_REQ_07_3_NoDomainProductTools; } + part def ServeClientRetry { + doc /* + Client side of the serve wire (MN-REQ-06.19). ServePool.send + pauses through a serve restart or drain: bounded retry + (MEMNET_SERVE_RETRY_S, default 30 s), backoff with jitter, + retry_after honoured. Not delivered or serve_draining: retry + any request. Reset after send: READ only; a WRITE is never + replayed. Gateway and memnet-mcp tcp client inherit it. + */ + attribute implemented : Boolean = true; + attribute noSemVerBump : Boolean = true; + attribute budgetEnv : String = "MEMNET_SERVE_RETRY_S"; + attribute defaultBudgetS : Real = 30.0; + attribute errDraining : String = "serve_draining"; + attribute honoursRetryAfter : Boolean = true; + attribute replaysWriteAfterSend : Boolean = false; + + perform action retryChoice : MemNetBehaviour::ServeRetryChoice; + + satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_19_RestartSafeTransport; + } + part def TcpServeBridge { doc /* Migration / multi-process fallback: length-prefixed JSON over TCP @@ -1537,6 +1560,7 @@ package MemNet { adminUsage = MN-REQ-06.11 admin JSON (opaque aliases; not 06.5). safeUpgrade = MN-REQ-06.14 drain, manifest, and startup restore. serveCheckpoint = MN-REQ-06.15 per-session checkpoint; no drain. + clientRetry = MN-REQ-06.19 client pause through restart or drain. Many mn_* sessions in one registry. Not a projectId map. Not a second graph. Admin report MUST NOT emit those mn_* ids. */ @@ -1548,6 +1572,7 @@ package MemNet { part adminUsage : AdminUsageReport; part safeUpgrade : SafeServeUpgrade; part serveCheckpoint : ServeCheckpoint; + part clientRetry : ServeClientRetry; attribute host : String = "127.0.0.1"; attribute listenPort : Integer = 18765; attribute requestOutputIsolated : Boolean = true; @@ -1981,7 +2006,7 @@ package MemNet { rule). Agent memory payloads: GQL mutate + shaped pin_map. Structured JSON envelopes. No domain-product tools. LLM agents connect here to MemNet — not to DurableBuffer. Python rewrite - out of scope. + out of scope. Streamable-http is stateless (MN-REQ-06.19). */ port apiOut : InProcessOutPort; port hostOut : JsonEnvelopeOutPort; @@ -3815,6 +3840,7 @@ package MemNet { attribute gatewayCapsBelowBackend : Boolean = false; attribute bodyCeilingDeclared : Boolean = true; attribute versionPinPerProduct : Boolean = true; + attribute pausesThroughBackendRestart : Boolean = true; attribute listFilteredByProduct : Boolean = true; attribute foreignSessionGatewayErr : Boolean = true; attribute publicBindDefault : Boolean = false; diff --git a/sysml-models/models/implementation.sysml b/sysml-models/models/implementation.sysml index ffbd1db..9ab4306 100644 --- a/sysml-models/models/implementation.sysml +++ b/sysml-models/models/implementation.sysml @@ -74,6 +74,9 @@ package MemNetImplementation { missing module gives one stderr line, @ERR: mcp_extra_missing|...; install with: pip install 'memnet-llm[mcp]', and exit 2, for every transport and for --help (tests/test_mcp_extra_missing.py). + Streamable-http is stateless with JSON replies (FastMCP + stateless_http, json_response; MN-REQ-06.19): a restart drops no + MCP session. Tip/ops engine face. MUST NOT be taught as product invent face (tipIsFace=false). */ @@ -86,12 +89,15 @@ package MemNetImplementation { attribute mcpExtraMissingExit : Integer = 2; attribute extraCheckBeforeArgs : Boolean = true; attribute everyTransportNeedsExtra : Boolean = true; + attribute statelessHttp : Boolean = true; + attribute jsonResponse : Boolean = true; attribute tipIsFace : Boolean = false; attribute implemented : Boolean = true; attribute inThisRepo : Boolean = true; attribute track : Boolean = true; satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_18_McpExtraFailFast; + satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_19_RestartSafeTransport; } part def MemNetLlmWheel { @@ -169,6 +175,35 @@ package MemNetImplementation { satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_17_RestartRestoreNewestSet; } + part def ServeRetryMod { + doc /* + Restart-safe client retry (MN-REQ-06.19). One READ / WRITE + classification of serve requests (argv_kind, request_kind; + unknown is WRITE), the serve_draining retry_after parse, the + MEMNET_SERVE_RETRY_S budget and jittered backoff. The loop is + ServePool.send (memnet/serve_pool.py); send_command, the product + gateway and the memnet-mcp tcp client all go through it. + */ + attribute path : String = "parts/common/memnet/memnet/serve_retry.py"; + attribute pyName : String = "memnet.serve_retry"; + attribute poolPath : String = "parts/common/memnet/memnet/serve_pool.py"; + attribute implemented : Boolean = true; + attribute budgetEnv : String = "MEMNET_SERVE_RETRY_S"; + attribute defaultBudgetS : Real = 30.0; + attribute backoffBaseS : Real = 0.1; + attribute backoffCapS : Real = 2.0; + attribute retriesNotDelivered : Boolean = true; + attribute honoursRetryAfter : Boolean = true; + attribute retriesReadAfterSend : Boolean = true; + attribute replaysWriteAfterSend : Boolean = false; + attribute unknownVerbIsWrite : Boolean = true; + attribute gatewayProbeGate : Boolean = false; + attribute nestedRetryLoops : Integer = 1; + + perform action retryChoice : MemNetBehaviour::ServeRetryChoice; + + satisfy MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_19_RestartSafeTransport; + } part def ServeCheckpointMod { doc /* Admin checkpoint of loaded sessions without drain (MN-REQ-06.15). */ attribute path : String = "parts/common/memnet/memnet/checkpoint.py"; @@ -340,6 +375,7 @@ package MemNetImplementation { part adminUsageMod : AdminUsageReportMod; part safeUpgradeMod : SafeUpgradeMod; part serveCheckpointMod : ServeCheckpointMod; + part serveRetryMod : ServeRetryMod; part snapAclMod : SnapshotAclMod; part cliMod : CliFacadeMod; part sessionsMod : SessionLifecycleMod; @@ -427,6 +463,11 @@ package MemNetImplementation { end logical ::> memNetSystem.core.transport.tcp.safeUpgrade; end code ::> safeUpgradeMod; } + allocation serveRetryToMod : SoftwareAllocate { + doc /* Client pause through a serve restart or drain; never replays a write (MN-REQ-06.19). */ + end logical ::> memNetSystem.core.transport.tcp.clientRetry; + end code ::> serveRetryMod; + } allocation serveCheckpointToMod : SoftwareAllocate { doc /* Per-session checkpoint snaps on memnet-serve. Not agent MCP. */ end logical ::> memNetSystem.core.transport.tcp.serveCheckpoint; diff --git a/sysml-models/models/requirements.sysml b/sysml-models/models/requirements.sysml index 1a55a41..ed2d303 100644 --- a/sysml-models/models/requirements.sysml +++ b/sysml-models/models/requirements.sysml @@ -43,6 +43,7 @@ MN-REQ-06.16 Session snapshot carries the ACL (enabled, grants, digest-only bind); format 2 SHALL carry #@ACL:; format 1 loads ACL off (@ACL: absent|off); files 0600; first v2 write over v1 keeps .v1.bak once (0600); no dual write MN-REQ-06.17 Restart restore from the newest valid snapshot set (checkpoint or upgrade-prepare); every restart restores; loud fallback (@WARN: restore_fallback); none valid exits 3 (implemented; tests/test_restart_restore.py) MN-REQ-06.18 memnet-mcp fails fast without the [mcp] extra: one stderr line mcp_extra_missing, exit 2 (implemented; tests/test_mcp_extra_missing.py) + MN-REQ-06.19 Restart-safe transport: stateless memnet-mcp-http; ServePool/gateway pause through a serve restart or drain by bounded retry (30 s); a write is never replayed after send (implemented; tests/test_serve_restart_retry.py) MN-REQ-11.17 Catalog Snap / model Snap — session strata (0.15) MN-REQ-11.17.1 Cross-cut satisfy locators on the catalog; SysMLEdge pin_map is a different tool (not this catalog) @@ -1417,9 +1418,10 @@ package MemNetRequirements { method (cue then pin_map, mutate). No SemVer bump. Patch-level c on 0.19. memnet-mcp and the product gateway are outside this procedure. During the restart - they see serve_draining or a connection error. There - is no client retry window and no helper that edits a - systemd unit or a gateway version pin. + they see serve_draining or a connection error; the + client pause through it is MN-REQ-06.19. There is no + helper that edits a systemd unit or a gateway version + pin. Admin drain: credential MEMNET_ADMIN_TOKEN (same gate as MN-REQ-06.11; unset admin_unconfigured; mismatch admin_denied). Command `memnet admin upgrade-prepare` @@ -1677,6 +1679,50 @@ package MemNetRequirements { */ attribute requirementId : String = "MN-REQ-06.18"; } + + requirement def MN_REQ_06_19_RestartSafeTransport { + doc /* + Status: implemented. Owner goal t1305u: an upgrade may + pause callers, but sessions SHALL work after it. A restart + or drain of memnet serve, memnet-mcp-http or the product + gateway SHALL pause callers by bounded retry. It SHALL NOT + drop an MCP session or fail a durable in-flight read. A + write SHALL NOT be silently replayed. + (1) memnet-mcp streamable-http is stateless: FastMCP + stateless_http=true and json_response=true + (memnet_mcp/server.py). No tool keeps per-MCP-session + state; the MemNet session is a tool argument. After a + restart, a tools/call with no Mcp-Session-Id header works. + (2) ServePool.send (memnet/serve_pool.py) retries within + MEMNET_SERVE_RETRY_S (default 30 s; 0 turns retry off) + with exponential backoff and jitter (0.1 s doubling, cap + 2 s). Retry any request that was not delivered: connect + refused or reset, or the frame not fully written (the + serve acts only on a whole frame). Retry any request the + serve refused with @ERR: serve_draining|retry_after_s=N, + after at least N seconds. When the budget runs out, the + last connect error or the draining envelope surfaces. + (3) A reset after the whole frame was written is retried + for READ requests only. A WRITE surfaces the error and is + delivered once. One module classifies requests + (memnet/serve_retry.py): a READ allowlist (version, + guide, examples, map show and fields, relations list, + session current, list, resume and expire-status, read + list, query pin-map, warm, find, walk, typed-walk, + neighbors, path and context, housekeep stats, stale, + orphans, dangling and recyclable, export pin-map without + --out, admin usage-report). Every other request is a + WRITE, so an unknown verb is never replayed. + (4) The product gateway and the memnet-mcp tcp client + inherit (2) and (3) through send_command. Neither gates on + probe first. The gateway caches no failed version check. + One retry loop only; the gateway adds no budget of its own. + No SemVer bump. Verify: MN-VER-06-S17. Evidence: + tests/test_serve_restart_retry.py, + tests/test_mcp_http.py::test_tools_call_works_after_restart_without_session_header. + */ + attribute requirementId : String = "MN-REQ-06.19"; + } } // ----- MCP agent boundary ----- diff --git a/sysml-models/models/verify.sysml b/sysml-models/models/verify.sysml index 65bf03a..13d9ec8 100644 --- a/sysml-models/models/verify.sysml +++ b/sysml-models/models/verify.sysml @@ -53,7 +53,7 @@ Safe serve upgrade (MN-VER-06-S12): SafeServeUpgrade on TcpServeBridge; admin drain; manifest; startup restore; retire after a clean restore (audit mark; a later restart - still restores); no client retry; no silent session loss; + still restores); client pause is S17; no silent session loss; no SemVer bump. Restart restore (MN-VER-06-S15): SafeUpgradeMod; newest valid set of checkpoint and upgrade-prepare, tie to upgrade; retired @@ -63,6 +63,11 @@ memnet-mcp extra (MN-VER-06-S16): McpServerMod; console memnet_mcp.entry:main; one line mcp_extra_missing, exit 2, every transport and --help. Runtime: tests/test_mcp_extra_missing.py. + Restart-safe transport (MN-VER-06-S17): ServeRetryMod and + McpServerMod; stateless memnet-mcp-http; bounded pause through + refuse, reset before send and serve_draining; READ only after + send; write never replayed. Runtime: + tests/test_serve_restart_retry.py, tests/test_mcp_http.py. Atelier-board asks (MN-VER-14-S01..S13; S01, S03, S04, S09 to S12 and S13 implemented with runtime tests, the rest PROPOSAL, target only, model-level, never counted as verified): @@ -1694,7 +1699,7 @@ package MemNetVerification { longer stops a later restore, and a later start restores again (MN-REQ-06.17, MN-VER-06-S15). Same session id, ACL, TTL clock, and house. Not an - agent MCP tool. No client retry. No SemVer bump. + agent MCP tool. Client pause: MN-REQ-06.19. No SemVer bump. Tests: tests/test_safe_upgrade.py::test_clean_restore_retires_and_a_later_start_still_restores (renamed from ..._so_a_later_start_does_not_replay; the second start restores again and the manifest stays retired); @@ -1873,6 +1878,51 @@ package MemNetVerification { } } + verification def MN_VER_06_S17_RestartSafeTransport { + doc /* + MN-REQ-06.19 — a restart or drain pauses callers and drops no + session; a write is never replayed. + Refused then accepted, read and write: + tests/test_serve_restart_retry.py::test_connect_refused_then_accept_succeeds_for_read_and_write; + serve_draining retry_after honoured, read and write: + ::test_draining_envelope_honours_retry_after_then_succeeds; + reset after send on a write is delivered once, not retried: + ::test_reset_after_send_on_write_is_not_retried; + reset after send on a read is retried: + ::test_reset_after_send_on_read_is_retried_and_succeeds; + bounded budget: ::test_budget_is_bounded_then_the_connect_error_surfaces, + ::test_draining_past_the_budget_returns_the_refusal, + ::test_retry_off_by_env; an over-cap reply is not retried: + ::test_oversized_reply_on_read_is_not_retried; verb classes: ::test_read_verbs, + ::test_write_verbs_and_unknown_fail_closed, + ::test_request_kind_admin_flags; gateway through a backend + restart: ::test_gateway_pauses_through_backend_restart. + Stateless MCP: tests/test_mcp_http.py::test_memnet_mcp_is_stateless_json, + ::test_tools_call_works_after_restart_without_session_header. + */ + attribute verificationId : String = "MN-VER-06-S17"; + + subject retryMod : ServeRetryMod; + subject mcpMod : McpServerMod; + + objective restartSafeTransport { + doc /* Implemented: fake serve on loopback with a fake clock; two memnet-mcp processes on one port. */ + verify requirement restartSafeTransportReq : MN_REQ_00_MissionBridge::MN_REQ_06_ProcessBoundary::MN_REQ_06_19_RestartSafeTransport; + require constraint { + retryMod.implemented == true + and retryMod.retriesNotDelivered == true + and retryMod.honoursRetryAfter == true + and retryMod.retriesReadAfterSend == true + and retryMod.replaysWriteAfterSend == false + and retryMod.unknownVerbIsWrite == true + and retryMod.gatewayProbeGate == false + and retryMod.nestedRetryLoops == 1 + and mcpMod.statelessHttp == true + and mcpMod.jsonResponse == true + } + } + } + verification def MN_VER_06_S14_SnapshotAcl { doc /* MN-REQ-06.16 — the session ACL persists in the snapshot as one diff --git a/sysml-models/outputs/ssot-to-code-allocate-map.md b/sysml-models/outputs/ssot-to-code-allocate-map.md index 7c9d6f7..80243bb 100644 --- a/sysml-models/outputs/ssot-to-code-allocate-map.md +++ b/sysml-models/outputs/ssot-to-code-allocate-map.md @@ -20,6 +20,7 @@ | tcpDaemonToMod | `…transport.tcp` | `serveDaemonMod` | `parts/common/memnet/memnet/serve.py` | `memnet.serve` | true | | adminUsageToMod | `…transport.tcp.adminUsage` | `adminUsageMod` | `parts/common/memnet/memnet/admin_usage.py` | `memnet.admin_usage` | true | | safeUpgradeToMod | `…transport.tcp.safeUpgrade` | `safeUpgradeMod` | `parts/common/memnet/memnet/upgrade.py` | `memnet.upgrade` | true | +| serveRetryToMod | `…transport.tcp.clientRetry` | `serveRetryMod` | `parts/common/memnet/memnet/serve_retry.py` | `memnet.serve_retry` | true | | serveCheckpointToMod | `…transport.tcp.serveCheckpoint` | `serveCheckpointMod` | `parts/common/memnet/memnet/checkpoint.py` | `memnet.checkpoint` | true | | cliToMod | `memNetSystem.core.cli` | `cliMod` | `parts/common/memnet/memnet/cli.py` | `memnet.cli` | true | | sessionsToMod | `…sessions` | `sessionsMod` | `parts/common/memnet/memnet/session_lifecycle.py` | `memnet.session_lifecycle` | true | From f4274c671a4f4a4de9f4cd0dc357ac4d7c1a1d3c Mon Sep 17 00:00:00 2001 From: chouswei <104312007+chouswei@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:29:42 +0800 Subject: [PATCH 2/4] Stateless memnet-mcp streamable-http (MN-REQ-06.19 a) FastMCP(stateless_http=True, json_response=True). Audit: no tool keeps per-MCP-session state (no Context/ctx, no notifications); the MemNet session is a tool argument. A tools/call after a restart works with no Mcp-Session-Id header. [mcp] extra floor mcp>=1.8 (first release with stateless_http). --- .../memnet-mcp/software/memnet_mcp/server.py | 4 +- pyproject.toml | 2 +- tests/test_mcp_http.py | 96 +++++++++++++++++++ 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/parts/memnet-mcp/software/memnet_mcp/server.py b/parts/memnet-mcp/software/memnet_mcp/server.py index 0f18bc0..7ba3bfe 100644 --- a/parts/memnet-mcp/software/memnet_mcp/server.py +++ b/parts/memnet-mcp/software/memnet_mcp/server.py @@ -40,7 +40,9 @@ ) from memnet_mcp.seed import supplement_seed_lines -mcp = FastMCP("memnet") +# Stateless streamable-http (MN-REQ-06.19): no tool keeps per-MCP-session state (the +# MemNet session is a tool argument), so a restart or a second process drops no session. +mcp = FastMCP("memnet", stateless_http=True, json_response=True) def _json(resp: MemNetResponse) -> str: diff --git a/pyproject.toml b/pyproject.toml index c8de108..f3ef96f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ dev = [ "hypothesis>=6.100", ] mcp = [ - "mcp>=1.2,<2", + "mcp>=1.8,<2", "uvicorn>=0.30", ] # Client driver only — does NOT install or vendor an AgensGraph server. diff --git a/tests/test_mcp_http.py b/tests/test_mcp_http.py index 1e912c7..a40d522 100644 --- a/tests/test_mcp_http.py +++ b/tests/test_mcp_http.py @@ -241,3 +241,99 @@ def test_cli_remote_bind_exits(monkeypatch: pytest.MonkeyPatch): ] ) assert exc.value.code == 2 + + +def test_memnet_mcp_is_stateless_json(): + """MN-REQ-06.19 (a): the shipped FastMCP needs no MCP session and answers plain JSON.""" + from memnet_mcp.server import mcp as memnet_mcp + + assert memnet_mcp.settings.stateless_http is True + assert memnet_mcp.settings.json_response is True + + +def _start_mcp_http(port: int): + import os + import subprocess + import sys + + env = dict(os.environ) + for key in ("MEMNET_MCP_HTTP_TOKEN", "MEMNET_MCP_ALLOW_REMOTE", "MEMNET_MCP_TRANSPORT"): + env.pop(key, None) + env["MEMNET_TEST_INLINE"] = "1" + proc = subprocess.Popen( + [sys.executable, "-m", "memnet_mcp", "--transport", "streamable-http", "--port", str(port)], + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + deadline = time.time() + 20.0 + while time.time() < deadline: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + return proc + except OSError: + time.sleep(0.05) + proc.kill() + pytest.fail("memnet-mcp streamable-http did not start") + + +def _mcp_post(port: int, body: dict) -> tuple[int, dict, dict]: + import json + + req = Request( + f"http://127.0.0.1:{port}/mcp", + data=json.dumps(body).encode(), + method="POST", + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + }, + ) + with urlopen(req, timeout=10.0) as resp: + return resp.status, dict(resp.headers), json.loads(resp.read().decode()) + + +def test_tools_call_works_after_restart_without_session_header(): + """MN-REQ-06.19 (a): a restart of memnet-mcp-http drops no MCP session. + + First process: initialize, then tools/call. Second process on the same port: a + tools/call with no Mcp-Session-Id header and no new initialize still works. + """ + port = _free_port() + init = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": {"name": "restart-test", "version": "0"}, + }, + } + call = { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "session_list", "arguments": {}}, + } + proc = _start_mcp_http(port) + try: + status, headers, body = _mcp_post(port, init) + assert status == 200 and body["result"]["serverInfo"]["name"] == "memnet" + assert not any(k.lower() == "mcp-session-id" for k in headers) + status, _, body = _mcp_post(port, call) + assert status == 200 and body["result"]["isError"] is False + finally: + proc.terminate() + proc.wait(timeout=10) + + proc = _start_mcp_http(port) + try: + status, headers, body = _mcp_post(port, {**call, "id": 3}) + assert status == 200, body + assert body["id"] == 3 + assert body["result"]["isError"] is False + assert "sessions|" in body["result"]["content"][0]["text"] + finally: + proc.terminate() + proc.wait(timeout=10) From df2745f0ef739978ae1ace9594708a18615173a1 Mon Sep 17 00:00:00 2001 From: chouswei <104312007+chouswei@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:29:42 +0800 Subject: [PATCH 3/4] ServePool bounded retry; gateway and MCP tcp client inherit it (MN-REQ-06.19 b+c) memnet.serve_retry classifies serve requests READ / WRITE (unknown is WRITE). ServePool.send retries a request that was not delivered and a serve_draining refusal (honouring retry_after_s) for any verb, and a reset after send for READ only, within MEMNET_SERVE_RETRY_S (default 30 s, 0 off), with backoff and jitter. The gateway and run_memnet no longer gate on probe(), and the gateway caches no failed version check, so both reach the retry. Tests: tests/test_serve_restart_retry.py. --- CHANGELOG.md | 8 + docs/operations/product-gateway-contract.md | 2 +- docs/operations/safe-upgrade.md | 14 +- parts/common/memnet/memnet/serve_pool.py | 101 +++++- parts/common/memnet/memnet/serve_retry.py | 133 +++++++ .../memnet-mcp/software/memnet_mcp/client.py | 10 +- .../software/memnet_mcp/product_gateway.py | 37 +- tests/test_mcp.py | 1 + tests/test_product_gateway.py | 11 +- tests/test_safe_upgrade.py | 3 + tests/test_serve_restart_retry.py | 342 ++++++++++++++++++ tests/test_sysml_restart_safe_transport.py | 65 ++++ tests/test_upgrade_verify.py | 2 + 13 files changed, 693 insertions(+), 36 deletions(-) create mode 100644 parts/common/memnet/memnet/serve_retry.py create mode 100644 tests/test_serve_restart_retry.py create mode 100644 tests/test_sysml_restart_safe_transport.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e0d2e4..4ea9e4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ This project uses Semantic Versioning as **interpreted for MemNet**: package `a. ## [Unreleased] +### Added + +- **Restart-safe transport** — an upgrade or restart of `memnet serve`, memnet-mcp-http or the product gateway now pauses callers instead of breaking them. The client pool (`ServePool.send`, used by the CLI, the memnet-mcp tcp client and the gateway) retries a refused connection, a reset before the request was fully sent, and `@ERR: serve_draining|retry_after_s=N` (waiting at least `N`), with exponential backoff and jitter, for at most `MEMNET_SERVE_RETRY_S` (default 30 s; `0` turns it off). A reset after the request was sent is retried for reads only; a write is never replayed. One module, `memnet.serve_retry`, says which commands are reads; anything else, including an unknown command, counts as a write. memnet-mcp streamable-http is stateless with JSON replies, so a restart drops no MCP session. The gateway no longer probes before a call or caches a failed version check. MN-REQ-06.19, MN-VER-06-S17. + +### Changed + +- The `[mcp]` extra now needs `mcp>=1.8` (stateless streamable-http). + ## [0.20.6] - 2026-11-04 ### Added diff --git a/docs/operations/product-gateway-contract.md b/docs/operations/product-gateway-contract.md index db7d179..7581dbe 100644 --- a/docs/operations/product-gateway-contract.md +++ b/docs/operations/product-gateway-contract.md @@ -66,7 +66,7 @@ HTTP status is a hint. The body is always the envelope. Backend engine errors st | 404 | `gateway_unconfigured` | `GET /admin/counts` when `admin.sha256` is unset | | 409 | `gateway_backend_version_mismatch` | `memnet version` is not the product pin | | 413 | `gateway_body_too_large` | Body above `body_max_bytes` | -| 502 | `gateway_backend_unreachable` | TCP probe failed, the version command did not answer, the client wait expired, or `session list` and no backend answered | +| 502 | `gateway_backend_unreachable` | The backend stayed unreachable past the restart pause (`MEMNET_SERVE_RETRY_S`, default 30 s; MN-REQ-06.19), the version command did not answer, the client wait expired, or `session list` and no backend answered | No HTTP response (the process exits 2 instead): diff --git a/docs/operations/safe-upgrade.md b/docs/operations/safe-upgrade.md index 35ba860..fc061a0 100644 --- a/docs/operations/safe-upgrade.md +++ b/docs/operations/safe-upgrade.md @@ -2,7 +2,19 @@ Upgrade `memnet-serve` without dropping a loaded session (MN-REQ-06.14). The procedure stays inside the serve process: save every session, restart on the new version, reload every session. The agent loop does not change: cue, then `pin_map`, then `mutate`. -During the restart, clients see `@ERR: serve_draining|retry_after_s=` or a connection error. There is no client retry window. +During the restart, the serve answers `@ERR: serve_draining|retry_after_s=` or refuses the connection. Clients pause through it (MN-REQ-06.19); see [Restart-safe clients](#restart-safe-clients). + +## Restart-safe clients + +`memnet` CLI, `memnet-mcp` over tcp and the product gateway all reach the serve through one client pool. That pool pauses through a restart or a drain rather than failing: + +- **Connection refused, or reset before the request was fully sent**: the serve never read it, so any request is retried. +- **`serve_draining`**: the serve refused before doing any work, so any request is retried after at least `retry_after_s`. +- **Reset after the request was sent**: the serve may have run it. A read (`pin_map`, `find`, `typed_walk`, `session_list`, …) is retried. A write (`mutate`, `session_open`, `session_save`, ingest, snap, ACL, reserve, admin, …) is not: the caller gets the error, and the write is never replayed. Unknown commands count as writes. + +Waits use exponential backoff with jitter (0.1 s doubling, capped at 2 s). The total pause is capped by `MEMNET_SERVE_RETRY_S` (default 30 s, comfortably longer than a restart; `0` turns the pause off). When the budget runs out, the last error or the draining reply is returned. + +`memnet-mcp --transport streamable-http` is stateless (`stateless_http`, JSON replies). No MCP session lives in the process, so a restart of memnet-mcp-http, or a second process behind the same front, drops no MCP session: the next `tools/call` works without re-initialising. MemNet sessions are tool arguments and live in the serve. ## Procedure diff --git a/parts/common/memnet/memnet/serve_pool.py b/parts/common/memnet/memnet/serve_pool.py index 371d1c9..537a749 100644 --- a/parts/common/memnet/memnet/serve_pool.py +++ b/parts/common/memnet/memnet/serve_pool.py @@ -4,14 +4,23 @@ import json import os +import random import select import socket import struct import threading import time +from collections.abc import Callable from typing import Any from memnet.config import serve_max_frame_bytes +from memnet.serve_retry import ( + RequestKind, + backoff_s, + draining_retry_after, + request_kind, + retry_budget_s, +) _OFF_VALUES = frozenset({"0", "false", "no", "off"}) _TIMEOUT_ENVELOPE: dict[str, Any] = { @@ -56,11 +65,33 @@ def _close(sock: socket.socket) -> None: pass +class _NotDelivered(Exception): + """The serve cannot have read the request (connect failed or frame not fully sent).""" + + def __init__(self, cause: OSError) -> None: + super().__init__(str(cause)) + self.cause = cause + + +class _ReplyTooLarge(ConnectionError): + """The reply frame is over the cap: deterministic, so never retried.""" + + class ServePool: """Idle sockets per (host, port), LIFO; each socket is checked out exclusively.""" - def __init__(self, idle_s: float = 30.0) -> None: + def __init__( + self, + idle_s: float = 30.0, + *, + sleep: Callable[[float], None] = time.sleep, + clock: Callable[[], float] = time.monotonic, + rng: random.Random | None = None, + ) -> None: self._idle_s = idle_s + self._sleep = sleep + self._clock = clock + self._rng = rng self._lock = threading.Lock() self._idle: dict[tuple[str, int], list[tuple[socket.socket, float]]] = {} self._pid = os.getpid() @@ -116,7 +147,21 @@ def probe(self, host: str, port: int, timeout: float = 0.2) -> bool: self._park(key, sock) return True - def send(self, host: str, port: int, payload: dict, timeout: float) -> dict: + def send( + self, + host: str, + port: int, + payload: dict, + timeout: float, + *, + retry_s: float | None = None, + ) -> dict: + """One serve request. Pauses through a restart or drain (MN-REQ-06.19). + + Retries within ``retry_s`` (default ``MEMNET_SERVE_RETRY_S``) when the + request was not delivered or the serve answered serve_draining. A reset + after the frame was written is retried for READ requests only. + """ keep = keepalive_enabled() body_obj = dict(payload) if keep: @@ -132,7 +177,44 @@ def send(self, host: str, port: int, payload: dict, timeout: float) -> dict: } frame = struct.pack(">I", len(data)) + data key = (host, port) + budget = retry_budget_s() if retry_s is None else max(0.0, retry_s) + deadline = self._clock() + budget + attempt = 0 + while True: + failure: OSError | None = None + reply: dict | None = None + try: + reply = self._send_once(key, frame, timeout, keep, max_frame) + except _NotDelivered as exc: + failure = exc.cause + wait = backoff_s(attempt, self._rng) + except OSError as exc: + if isinstance(exc, _ReplyTooLarge) or request_kind(payload) is not RequestKind.READ: + raise + failure = exc + wait = backoff_s(attempt, self._rng) + else: + retry_after = draining_retry_after(reply) + if retry_after is None: + return reply + wait = max(retry_after, backoff_s(attempt, self._rng)) + remaining = deadline - self._clock() + if remaining <= 0: + if failure is not None: + raise failure + return reply # type: ignore[return-value] + self._sleep(min(wait, remaining)) + attempt += 1 + def _send_once( + self, + key: tuple[str, int], + frame: bytes, + timeout: float, + keep: bool, + max_frame: int, + ) -> dict: + """One attempt. Raises ``_NotDelivered`` when the serve cannot have read the frame.""" sock: socket.socket | None = self._checkout(key) if keep else None try: if sock is not None: @@ -145,12 +227,19 @@ def send(self, host: str, port: int, payload: dict, timeout: float) -> dict: _close(sock) sock = None if sock is None: - sock = socket.create_connection(key, timeout=timeout) - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - sock.sendall(frame) + try: + sock = socket.create_connection(key, timeout=timeout) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.sendall(frame) + except TimeoutError: + raise + except OSError as exc: + # Refused, reset, or the frame was not fully written: the serve acts + # only on a whole length-prefixed frame, so it did not run this one. + raise _NotDelivered(exc) from exc (length,) = struct.unpack(">I", _recv_exact(sock, 4)) if length > max_frame: - raise ConnectionError(f"response frame {length} bytes exceeds cap {max_frame}") + raise _ReplyTooLarge(f"response frame {length} bytes exceeds cap {max_frame}") reply = json.loads(_recv_exact(sock, length).decode("utf-8")) except TimeoutError: if sock is not None: diff --git a/parts/common/memnet/memnet/serve_retry.py b/parts/common/memnet/memnet/serve_retry.py new file mode 100644 index 0000000..4364158 --- /dev/null +++ b/parts/common/memnet/memnet/serve_retry.py @@ -0,0 +1,133 @@ +"""Restart-safe client retry for ``memnet serve`` requests (MN-REQ-06.19). + +One place says which serve requests are safe to send twice. ``ServePool.send`` +asks it after a failure: + +- not delivered (connect refused or reset, or the frame was never fully + written): retry any request, because the serve never read it; +- ``@ERR: serve_draining|retry_after_s=N``: the serve refused before any work, + so retry any request after ``N`` seconds; +- reset after the whole frame was written: the serve may have run it, so retry + READ only. A WRITE surfaces the error and is never replayed. + +The wait is bounded by ``MEMNET_SERVE_RETRY_S`` (default 30 s; ``0`` turns +retry off). +""" + +from __future__ import annotations + +import os +import random +import re +from enum import Enum +from typing import Any + +ENV_RETRY_BUDGET = "MEMNET_SERVE_RETRY_S" +DEFAULT_RETRY_BUDGET_S = 30.0 +BACKOFF_BASE_S = 0.1 +BACKOFF_CAP_S = 2.0 + +_DRAINING_RE = re.compile(r"^@ERR: serve_draining\|retry_after_s=(\d+(?:\.\d+)?)\s*$", re.M) + + +class RequestKind(Enum): + READ = "read" + WRITE = "write" + + +# (argv[0], argv[1]) pairs that only read. ``None`` as argv[1] means the verb +# has no subcommand. Anything not listed is a WRITE: unknown verbs fail closed. +_READ_VERBS: frozenset[tuple[str, str | None]] = frozenset( + { + ("version", None), + ("guide", None), + ("examples", None), + ("examples", "map"), + ("examples", "workflow"), + ("examples", "add"), + ("examples", "path"), + ("examples", "agent-guide"), + ("tagmap", "fields"), + ("tagmap", "show"), + ("map", "fields"), + ("map", "show"), + ("relations", "list"), + ("session", "current"), + ("session", "list"), + ("session", "resume"), + ("session", "expire-status"), + ("read", "list"), + ("query", "pin-map"), + ("query", "warm"), + ("query", "find"), + ("query", "walk"), + ("query", "typed-walk"), + ("query", "neighbors"), + ("query", "path"), + ("query", "context"), + ("housekeep", "stats"), + ("housekeep", "stale"), + ("housekeep", "orphans"), + ("housekeep", "dangling"), + ("housekeep", "recyclable"), + ("admin", "usage-report"), + } +) +_SUBCOMMAND_VERBS = frozenset(v for v, sub in _READ_VERBS if sub is not None) | { + "session", + "admin", + "export", + "ingest", + "snap", +} + + +def argv_kind(argv: Any) -> RequestKind: + """READ when every effect of ``argv`` is a read; WRITE otherwise.""" + if not isinstance(argv, list) or not argv: + return RequestKind.WRITE + words = [str(a) for a in argv] + verb = words[0] + sub = words[1] if len(words) > 1 and verb in _SUBCOMMAND_VERBS else None + if (verb, sub) == ("export", "pin-map"): + # --out writes a file on the serve host. + return ( + RequestKind.WRITE + if any(w == "--out" or w.startswith("--out=") for w in words) + else RequestKind.READ + ) + return RequestKind.READ if (verb, sub) in _READ_VERBS else RequestKind.WRITE + + +def request_kind(payload: dict[str, Any]) -> RequestKind: + """Kind of one serve envelope (``args`` plus the admin flags).""" + if payload.get("upgrade_prepare") is True: + return RequestKind.WRITE + if payload.get("admin_usage") is True: + return RequestKind.READ + return argv_kind(payload.get("args")) + + +def draining_retry_after(reply: Any) -> float | None: + """``retry_after_s`` of a serve_draining refusal, else ``None``.""" + if not isinstance(reply, dict) or int(reply.get("exit_code", 0) or 0) == 0: + return None + match = _DRAINING_RE.search(str(reply.get("stderr") or "")) + return float(match.group(1)) if match else None + + +def retry_budget_s() -> float: + raw = (os.environ.get(ENV_RETRY_BUDGET) or "").strip() + if not raw: + return DEFAULT_RETRY_BUDGET_S + try: + value = float(raw) + except ValueError: + return DEFAULT_RETRY_BUDGET_S + return max(0.0, value) + + +def backoff_s(attempt: int, rng: random.Random | None = None) -> float: + """Exponential backoff with jitter: half to all of ``base * 2**attempt``, capped.""" + ceiling = min(BACKOFF_CAP_S, BACKOFF_BASE_S * (2 ** min(attempt, 16))) + return ceiling * (0.5 + 0.5 * (rng or random).random()) diff --git a/parts/memnet-mcp/software/memnet_mcp/client.py b/parts/memnet-mcp/software/memnet_mcp/client.py index 32e0aa1..a644ada 100644 --- a/parts/memnet-mcp/software/memnet_mcp/client.py +++ b/parts/memnet-mcp/software/memnet_mcp/client.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from memnet.in_process_engine import run_argv -from memnet.serve import probe, send_command +from memnet.serve import send_command from memnet_mcp.parse import extract_errors, extract_session_id _SERVE_REQUIRED = "@ERR: serve_required|run memnet serve in another terminal first" @@ -131,10 +131,12 @@ def run_memnet( mode = _transport() if mode == "tcp": - if probe(): + # No probe gate: send_command pauses through a serve restart (MN-REQ-06.19). + try: raw = send_command(full_argv, stdin=stdin) - return MemNetResponse.from_raw(raw, session_hint=session) - return MemNetResponse.serve_required(session_hint=session) + except OSError: + return MemNetResponse.serve_required(session_hint=session) + return MemNetResponse.from_raw(raw, session_hint=session) raw = run_argv(full_argv, stdin=stdin) return MemNetResponse.from_raw(raw, session_hint=session) diff --git a/parts/memnet-mcp/software/memnet_mcp/product_gateway.py b/parts/memnet-mcp/software/memnet_mcp/product_gateway.py index 6595d68..a623baf 100644 --- a/parts/memnet-mcp/software/memnet_mcp/product_gateway.py +++ b/parts/memnet-mcp/software/memnet_mcp/product_gateway.py @@ -27,7 +27,7 @@ from typing import Any from urllib.parse import urlsplit -from memnet.serve import is_loopback_bind_host, probe, send_command +from memnet.serve import is_loopback_bind_host, send_command DEFAULT_BIND = "127.0.0.1" DEFAULT_PORT = 18770 @@ -526,23 +526,24 @@ def _version(self, backend: Backend) -> str | None: ): return hit[1] version: str | None = None - if probe(host=backend.host, port=backend.port): - try: - raw = send_command( - ["version"], - host=backend.host, - port=backend.port, - timeout=self.timeout_s, - ) - except (OSError, json.JSONDecodeError): - raw = None - if isinstance(raw, dict): - for line in (raw.get("stdout") or "").splitlines(): - if line.startswith("@VER: memnet|"): - version = line.split("|", 1)[1].strip() - break - with self._lock: - self._ver_cache[backend.id] = (now, version) + # No probe gate: send_command pauses through a backend restart (MN-REQ-06.19). + try: + raw = send_command( + ["version"], + host=backend.host, + port=backend.port, + timeout=self.timeout_s, + ) + except (OSError, json.JSONDecodeError): + raw = None + if isinstance(raw, dict): + for line in (raw.get("stdout") or "").splitlines(): + if line.startswith("@VER: memnet|"): + version = line.split("|", 1)[1].strip() + break + if version is not None: + with self._lock: + self._ver_cache[backend.id] = (now, version) return version def _forward( diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 132f884..b01d30d 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -50,6 +50,7 @@ def test_serve_required_without_inline(monkeypatch): monkeypatch.delenv("MEMNET_SERVE_INTERNAL", raising=False) monkeypatch.setenv("MEMNET_MCP_TRANSPORT", "tcp") monkeypatch.setenv("MEMNET_SERVE_PORT", "59999") + monkeypatch.setenv("MEMNET_SERVE_RETRY_S", "0") resp = run_memnet(["session", "current"]) assert resp.exit_code == 2 assert resp.errors diff --git a/tests/test_product_gateway.py b/tests/test_product_gateway.py index 8a00034..818ed72 100644 --- a/tests/test_product_gateway.py +++ b/tests/test_product_gateway.py @@ -188,10 +188,6 @@ def test_stdio_unchanged_when_registry_env_is_set( def test_forward_keeps_args_and_stdin(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: calls: list[dict] = [] - def fake_probe(host: str | None = None, port: int | None = None) -> bool: - del host, port - return True - def fake_send( args, stdin=None, host=None, port=None, timeout=None, admin_token=None, admin_usage=False ): @@ -209,7 +205,6 @@ def fake_send( return {"exit_code": 0, "stdout": f"@VER: memnet|{__version__}\n", "stderr": ""} return {"exit_code": 0, "stdout": "@SESSION: mn_new|soon|60\n", "stderr": ""} - monkeypatch.setattr("memnet_mcp.product_gateway.probe", fake_probe) monkeypatch.setattr("memnet_mcp.product_gateway.send_command", fake_send) cfg = tmp_path / "gw.json" state = tmp_path / "owners.json" @@ -253,7 +248,9 @@ def fake_send( assert "namespace" not in calls[-1] -def test_two_real_serves_route_isolate_and_pass_through(tmp_path: Path) -> None: +def test_two_real_serves_route_isolate_and_pass_through( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: port_a = _free_port() port_b = _free_port() serve_a = _start_serve(port_a, tmp_path / "serve-a.log") @@ -431,6 +428,8 @@ def test_two_real_serves_route_isolate_and_pass_through(tmp_path: Path) -> None: assert counts["products"]["endleaf"]["requests"] > 0 assert "sessions" not in counts["products"]["endleaf"] + # A backend that stays down: the restart pause (MN-REQ-06.19) is bounded, then 502. + monkeypatch.setenv("MEMNET_SERVE_RETRY_S", "0.5") _stop(serve_a) serve_a = None deadline = time.time() + 5 diff --git a/tests/test_safe_upgrade.py b/tests/test_safe_upgrade.py index 95317a9..4e01795 100644 --- a/tests/test_safe_upgrade.py +++ b/tests/test_safe_upgrade.py @@ -409,6 +409,8 @@ def stop(proc: subprocess.Popen[bytes]) -> None: ) assert prepared["exit_code"] == 0, prepared assert "upgrade_prepare|ready|1" in prepared["stderr"] + # Retry off: see the raw refusal (the restart pause is MN-REQ-06.19). + monkeypatch.setenv("MEMNET_SERVE_RETRY_S", "0") refused = send_command( ["session", "open", "--map-file", str(schema_file)], host="127.0.0.1", @@ -419,6 +421,7 @@ def stop(proc: subprocess.Popen[bytes]) -> None: finally: stop(proc) + monkeypatch.delenv("MEMNET_SERVE_RETRY_S") proc = start() try: report = json.loads((state / "upgrade-restore.json").read_text(encoding="utf-8")) diff --git a/tests/test_serve_restart_retry.py b/tests/test_serve_restart_retry.py new file mode 100644 index 0000000..0082aac --- /dev/null +++ b/tests/test_serve_restart_retry.py @@ -0,0 +1,342 @@ +"""Restart-safe serve client (MN-REQ-06.19, MN-VER-06-S17): pause, never replay a write. + +A scripted fake serve on a real loopback socket speaks the serve frame wire. A fake clock +makes the retry budget exact without real sleeps. +""" + +from __future__ import annotations + +import json +import socket +import struct +import threading +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from memnet import __version__ +from memnet.serve_pool import ServePool +from memnet.serve_retry import RequestKind, argv_kind, draining_retry_after, request_kind + +HOST = "127.0.0.1" +OK = {"exit_code": 0, "stdout": "@STAT: ok|1\n", "stderr": ""} +DRAINING = {"exit_code": 2, "stdout": "", "stderr": "@ERR: serve_draining|retry_after_s=5\n"} +READ_ARGS = ["query", "find", "--kind", "TSK", "--session", "mn_x"] +WRITE_ARGS = ["mutate", "--stdin", "--session", "mn_x"] + +# One reply per accepted connection: a dict is sent back, "drop" reads the whole frame +# and then resets the connection without a reply (the serve may have run it); "huge" +# announces a reply frame over the cap. +Step = dict[str, Any] | str + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind((HOST, 0)) + return int(sock.getsockname()[1]) + + +def _recv_exact(sock: socket.socket, n: int) -> bytes: + buf = b"" + while len(buf) < n: + part = sock.recv(n - len(buf)) + if not part: + raise ConnectionError("closed") + buf += part + return buf + + +class FakeServe: + """Loopback listener that follows a script, one step per connection, and logs requests.""" + + def __init__(self, port: int, script: list[Step], *, default: Step | None = None) -> None: + self.port = port + self.script = list(script) + self.default = default + self.delivered: list[dict[str, Any]] = [] + self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._sock.bind((HOST, port)) + self._sock.listen(8) + self._sock.settimeout(0.05) + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def _next(self) -> Step | None: + if self.script: + return self.script.pop(0) + return self.default + + def _run(self) -> None: + while not self._stop.is_set(): + try: + conn, _ = self._sock.accept() + except TimeoutError: + continue + except OSError: + return + conn.settimeout(5) + with conn: + try: + (length,) = struct.unpack(">I", _recv_exact(conn, 4)) + request = json.loads(_recv_exact(conn, length).decode("utf-8")) + except (OSError, ValueError): + continue + self.delivered.append(request) + step = self._next() + if step == "huge": + conn.sendall(struct.pack(">I", 2**31)) + continue + if step is None or step == "drop": + conn.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) + continue + body = json.dumps(step).encode("utf-8") + conn.sendall(struct.pack(">I", len(body)) + body) + + def close(self) -> None: + self._stop.set() + self._thread.join(timeout=5) + self._sock.close() + + +class FakeClock: + """Monotonic clock that only moves when the pool sleeps; runs a hook on each sleep.""" + + def __init__(self, on_sleep: Callable[[int], None] | None = None) -> None: + self.now = 1000.0 + self.slept: list[float] = [] + self._on_sleep = on_sleep + + def clock(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.slept.append(seconds) + self.now += seconds + if self._on_sleep is not None: + self._on_sleep(len(self.slept)) + + +@pytest.fixture(autouse=True) +def _one_shot(monkeypatch: pytest.MonkeyPatch) -> None: + # One request per connection, so every attempt is a fresh connect. + monkeypatch.setenv("MEMNET_SERVE_KEEPALIVE", "0") + monkeypatch.delenv("MEMNET_SERVE_RETRY_S", raising=False) + + +def _pool(clock: FakeClock) -> ServePool: + return ServePool(sleep=clock.sleep, clock=clock.clock) + + +@pytest.mark.parametrize("args", [READ_ARGS, WRITE_ARGS], ids=["read", "write"]) +def test_connect_refused_then_accept_succeeds_for_read_and_write(args): + port = _free_port() + serves: list[FakeServe] = [] + + def come_up(n: int) -> None: + if n == 3: # the serve is back after the third backoff + serves.append(FakeServe(port, [OK])) + + clock = FakeClock(come_up) + try: + reply = _pool(clock).send(HOST, port, {"args": args}, timeout=5) + assert reply == OK + assert len(clock.slept) == 3 + assert sum(clock.slept) < 30 + assert [r["args"] for r in serves[0].delivered] == [args] + finally: + for serve in serves: + serve.close() + + +@pytest.mark.parametrize("args", [READ_ARGS, WRITE_ARGS], ids=["read", "write"]) +def test_draining_envelope_honours_retry_after_then_succeeds(args): + port = _free_port() + serve = FakeServe(port, [DRAINING, OK]) + clock = FakeClock() + try: + reply = _pool(clock).send(HOST, port, {"args": args}, timeout=5) + assert reply == OK + assert clock.slept and clock.slept[0] >= 5 + assert len(serve.delivered) == 2 # refused before any work, then run once + finally: + serve.close() + + +def test_reset_after_send_on_write_is_not_retried(): + port = _free_port() + serve = FakeServe(port, ["drop"], default=OK) + clock = FakeClock() + try: + with pytest.raises(OSError): + _pool(clock).send(HOST, port, {"args": WRITE_ARGS, "stdin": "CREATE (:TSK)\n"}, 5) + assert len(serve.delivered) == 1 + assert clock.slept == [] + finally: + serve.close() + + +def test_reset_after_send_on_read_is_retried_and_succeeds(): + port = _free_port() + serve = FakeServe(port, ["drop", OK]) + clock = FakeClock() + try: + reply = _pool(clock).send(HOST, port, {"args": READ_ARGS}, timeout=5) + assert reply == OK + assert len(serve.delivered) == 2 + assert len(clock.slept) == 1 + finally: + serve.close() + + +def test_oversized_reply_on_read_is_not_retried(): + port = _free_port() + serve = FakeServe(port, ["huge"], default=OK) + clock = FakeClock() + try: + with pytest.raises(ConnectionError, match="exceeds cap"): + _pool(clock).send(HOST, port, {"args": READ_ARGS}, timeout=5) + assert len(serve.delivered) == 1 + assert clock.slept == [] + finally: + serve.close() + + +def test_budget_is_bounded_then_the_connect_error_surfaces(): + port = _free_port() # nothing ever listens + clock = FakeClock() + with pytest.raises(ConnectionRefusedError): + _pool(clock).send(HOST, port, {"args": READ_ARGS}, timeout=5) + assert sum(clock.slept) == pytest.approx(30) # the last wait is clamped to the budget + assert max(clock.slept) <= 2.0 # backoff cap + + +def test_draining_past_the_budget_returns_the_refusal(): + port = _free_port() + serve = FakeServe(port, [], default=DRAINING) + clock = FakeClock() + try: + reply = _pool(clock).send(HOST, port, {"args": WRITE_ARGS}, timeout=5, retry_s=12) + assert draining_retry_after(reply) == 5 + assert sum(clock.slept) == pytest.approx(12) + finally: + serve.close() + + +def test_retry_off_by_env(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("MEMNET_SERVE_RETRY_S", "0") + clock = FakeClock() + with pytest.raises(ConnectionRefusedError): + _pool(clock).send(HOST, _free_port(), {"args": READ_ARGS}, timeout=5) + assert clock.slept == [] + + +@pytest.mark.parametrize( + "argv", + [ + ["version"], + ["query", "pin-map", "--cue", "TSK_a", "--session", "s"], + ["query", "find", "--kind", "TSK"], + ["query", "typed-walk", "--start", "x"], + ["query", "walk"], + ["read", "list"], + ["housekeep", "stats"], + ["session", "current"], + ["session", "list"], + ["session", "expire-status"], + ["export", "pin-map", "--cue", "TSK_a"], + ["admin", "usage-report"], + ], +) +def test_read_verbs(argv): + assert argv_kind(argv) is RequestKind.READ + + +@pytest.mark.parametrize( + "argv", + [ + ["mutate", "--stdin"], + ["add", "TSK"], + ["update", "TSK_a"], + ["delete", "TSK_a"], + ["import-slice"], + ["session", "open", "--map-file", "m"], + ["session", "save", "--file", "f"], + ["session", "load", "--file", "f"], + ["session", "close", "s"], + ["session", "acl-enable"], + ["session", "acl-grant"], + ["session", "acl-bind"], + ["reserve"], + ["extend"], + ["release"], + ["ingest", "sysml", "x.sysml"], + ["snap", "model", "x"], + ["housekeep", "prune", "stale"], + ["export", "pin-map", "--cue", "TSK_a", "--out", "f.gql"], + ["export", "pin-map", "--out=f.gql"], + ["admin", "checkpoint"], + ["admin", "upgrade-prepare"], + ["no-such-verb"], + [], + ], +) +def test_write_verbs_and_unknown_fail_closed(argv): + assert argv_kind(argv) is RequestKind.WRITE + + +def test_request_kind_admin_flags(): + assert request_kind({"args": [], "admin_usage": True}) is RequestKind.READ + assert request_kind({"args": ["version"], "upgrade_prepare": True}) is RequestKind.WRITE + + +def test_gateway_pauses_through_backend_restart(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Gateway owned call: the backend restarts between calls; the caller pauses, then works.""" + from memnet_mcp.product_gateway import ProductGateway + from test_product_gateway import ENDLEAF, _config + + monkeypatch.setenv("MEMNET_SERVE_RETRY_S", "10") + version = {"exit_code": 0, "stdout": f"@VER: memnet|{__version__}\n", "stderr": ""} + opened = {"exit_code": 0, "stdout": "@SESSION: mn_gw1|soon|60\n", "stderr": ""} + port_a, port_b = _free_port(), _free_port() + cfg, state = tmp_path / "gw.json", tmp_path / "owners.json" + _config(cfg, state, port_a, port_b) + gw = ProductGateway.load(str(cfg)) + + def call(args: list[str], **extra: Any): + body = {"args": args, **extra} + return gw.handle( + method="POST", + path="/gateway", + authorization=f"Bearer {ENDLEAF}", + body=json.dumps(body).encode(), + ) + + # Backend a is down when the open arrives and comes up 0.3 s later. + late: list[FakeServe] = [] + timer = threading.Timer(0.3, lambda: late.append(FakeServe(port_a, [version, opened]))) + timer.start() + try: + first = call(["session", "open", "--map-file", "m.txt"], house="syson") + assert first.exit_code == 0, first.stderr + assert "mn_gw1" in first.stdout + finally: + timer.join() + for serve in late: + serve.close() + + # Restart: down now, back 0.3 s later. The write is delivered exactly once. + back: list[FakeServe] = [] + timer = threading.Timer(0.3, lambda: back.append(FakeServe(port_a, [version, OK]))) + timer.start() + try: + second = call(WRITE_ARGS[:-1] + ["mn_gw1"], session="mn_gw1", stdin="CREATE (:TSK)\n") + assert second.exit_code == 0, second.stderr + assert [r["args"][0] for r in back[0].delivered] == ["version", "mutate"] + finally: + timer.join() + for serve in back: + serve.close() diff --git a/tests/test_sysml_restart_safe_transport.py b/tests/test_sysml_restart_safe_transport.py new file mode 100644 index 0000000..449b151 --- /dev/null +++ b/tests/test_sysml_restart_safe_transport.py @@ -0,0 +1,65 @@ +"""Honesty-c: MN-REQ-06.19 restart-safe transport is modelled and matches the code.""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MODELS = ROOT / "sysml-models" / "models" +REQUIREMENTS = MODELS / "requirements.sysml" +VERIFY = MODELS / "verify.sysml" +IMPLEMENTATION = MODELS / "implementation.sysml" +DEPLOY = MODELS / "deploy.sysml" +BEHAVIOUR = MODELS / "behaviour.sysml" +MAP = ROOT / "sysml-models" / "outputs" / "ssot-to-code-allocate-map.md" + + +def test_requirement_verify_allocate(): + req = REQUIREMENTS.read_text(encoding="utf-8") + assert "requirement def MN_REQ_06_19_RestartSafeTransport" in req + assert 'attribute requirementId : String = "MN-REQ-06.19"' in req + assert "MEMNET_SERVE_RETRY_S" in req + ver = VERIFY.read_text(encoding="utf-8") + assert "verification def MN_VER_06_S17_RestartSafeTransport" in ver + assert 'attribute verificationId : String = "MN-VER-06-S17"' in ver + assert "retryMod.replaysWriteAfterSend == false" in ver + impl = IMPLEMENTATION.read_text(encoding="utf-8") + assert "part def ServeRetryMod" in impl + assert "allocation serveRetryToMod" in impl + assert "attribute statelessHttp : Boolean = true" in impl + deploy = DEPLOY.read_text(encoding="utf-8") + assert "part clientRetry : ServeClientRetry" in deploy + assert "attribute pausesThroughBackendRestart : Boolean = true" in deploy + assert "action def ServeRetryChoice" in BEHAVIOUR.read_text(encoding="utf-8") + assert "| serveRetryToMod |" in MAP.read_text(encoding="utf-8") + + +def test_verification_names_live_tests(): + ver = VERIFY.read_text(encoding="utf-8") + body = ver.split("verification def MN_VER_06_S17_RestartSafeTransport", 1)[1] + body = body.split("verification def ", 1)[0] + retry = (ROOT / "tests" / "test_serve_restart_retry.py").read_text(encoding="utf-8") + http = (ROOT / "tests" / "test_mcp_http.py").read_text(encoding="utf-8") + for name in ( + "test_connect_refused_then_accept_succeeds_for_read_and_write", + "test_draining_envelope_honours_retry_after_then_succeeds", + "test_reset_after_send_on_write_is_not_retried", + "test_reset_after_send_on_read_is_retried_and_succeeds", + "test_gateway_pauses_through_backend_restart", + ): + assert name in body and f"def {name}(" in retry + assert "test_tools_call_works_after_restart_without_session_header" in body + assert "def test_tools_call_works_after_restart_without_session_header(" in http + + +def test_model_numbers_match_code(): + from memnet import serve_retry + from memnet_mcp.server import mcp + + impl = IMPLEMENTATION.read_text(encoding="utf-8") + mod = impl.split("part def ServeRetryMod", 1)[1].split("part def ", 1)[0] + assert f'budgetEnv : String = "{serve_retry.ENV_RETRY_BUDGET}"' in mod + assert f"defaultBudgetS : Real = {serve_retry.DEFAULT_RETRY_BUDGET_S}" in mod + assert f"backoffBaseS : Real = {serve_retry.BACKOFF_BASE_S}" in mod + assert f"backoffCapS : Real = {serve_retry.BACKOFF_CAP_S}" in mod + assert mcp.settings.stateless_http is True and mcp.settings.json_response is True diff --git a/tests/test_upgrade_verify.py b/tests/test_upgrade_verify.py index 47eafef..bc1025f 100644 --- a/tests/test_upgrade_verify.py +++ b/tests/test_upgrade_verify.py @@ -111,6 +111,8 @@ def test_fail_on_bad_before_file_or_no_serve(started, tmp_path): kept = tmp_path / "kept.json" shutil.copy(before, kept) serve.stop(signal.SIGTERM) + # Retry off: a serve that stays down is unreachable at once (no MN-REQ-06.19 pause). + serve.env["MEMNET_SERVE_RETRY_S"] = "0" got = _verify(serve, build, kept) assert got.returncode == 2, got.stderr assert "@ERR: serve_unreachable|" in got.stderr From a9170d40eb48268275518b229c863e007e8b9730 Mon Sep 17 00:00:00 2001 From: chouswei <104312007+chouswei@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:53:40 +0800 Subject: [PATCH 4/4] Gateway placement: skip a dead non-last backend without the retry pause (MN-REQ-06.19 c) send_command takes retry_s and passes it to ServePool.send. In _select_new every candidate but the last checks its version with retry_s=0, so a backend that stays down fails over at once; the last candidate, pinned backends and _forward keep the full budget. --- CHANGELOG.md | 2 +- docs/operations/safe-upgrade.md | 2 ++ parts/common/memnet/memnet/serve.py | 3 +- .../software/memnet_mcp/product_gateway.py | 10 ++++-- tests/test_product_gateway.py | 9 ++++- tests/test_serve_restart_retry.py | 34 +++++++++++++++++++ 6 files changed, 54 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ea9e4f..33e07ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ This project uses Semantic Versioning as **interpreted for MemNet**: package `a. ### Added -- **Restart-safe transport** — an upgrade or restart of `memnet serve`, memnet-mcp-http or the product gateway now pauses callers instead of breaking them. The client pool (`ServePool.send`, used by the CLI, the memnet-mcp tcp client and the gateway) retries a refused connection, a reset before the request was fully sent, and `@ERR: serve_draining|retry_after_s=N` (waiting at least `N`), with exponential backoff and jitter, for at most `MEMNET_SERVE_RETRY_S` (default 30 s; `0` turns it off). A reset after the request was sent is retried for reads only; a write is never replayed. One module, `memnet.serve_retry`, says which commands are reads; anything else, including an unknown command, counts as a write. memnet-mcp streamable-http is stateless with JSON replies, so a restart drops no MCP session. The gateway no longer probes before a call or caches a failed version check. MN-REQ-06.19, MN-VER-06-S17. +- **Restart-safe transport** — an upgrade or restart of `memnet serve`, memnet-mcp-http or the product gateway now pauses callers instead of breaking them. The client pool (`ServePool.send`, used by the CLI, the memnet-mcp tcp client and the gateway) retries a refused connection, a reset before the request was fully sent, and `@ERR: serve_draining|retry_after_s=N` (waiting at least `N`), with exponential backoff and jitter, for at most `MEMNET_SERVE_RETRY_S` (default 30 s; `0` turns it off). A reset after the request was sent is retried for reads only; a write is never replayed. One module, `memnet.serve_retry`, says which commands are reads; anything else, including an unknown command, counts as a write. memnet-mcp streamable-http is stateless with JSON replies, so a restart drops no MCP session. The gateway no longer probes before a call or caches a failed version check. When it places a new session, only the last candidate backend pauses; a down backend with another behind it is tried once and skipped, so fail-over stays instant. MN-REQ-06.19, MN-VER-06-S17. ### Changed diff --git a/docs/operations/safe-upgrade.md b/docs/operations/safe-upgrade.md index fc061a0..87e8ab2 100644 --- a/docs/operations/safe-upgrade.md +++ b/docs/operations/safe-upgrade.md @@ -14,6 +14,8 @@ During the restart, the serve answers `@ERR: serve_draining|retry_after_s= dict[str, Any]: host = host or serve_host() port = port or serve_port() @@ -408,7 +409,7 @@ def send_command( ) from memnet.serve_pool import POOL - return POOL.send(host, port, payload_obj, wait) + return POOL.send(host, port, payload_obj, wait, retry_s=retry_s) def _peer_gone(sock: socket.socket) -> bool: diff --git a/parts/memnet-mcp/software/memnet_mcp/product_gateway.py b/parts/memnet-mcp/software/memnet_mcp/product_gateway.py index a623baf..a6aba11 100644 --- a/parts/memnet-mcp/software/memnet_mcp/product_gateway.py +++ b/parts/memnet-mcp/software/memnet_mcp/product_gateway.py @@ -467,9 +467,12 @@ def _select_new( return named, None mismatch: HandleResult | None = None unreachable: HandleResult | None = None - for backend_id in product.backends: + last = len(product.backends) - 1 + for i, backend_id in enumerate(product.backends): backend = self.backends[backend_id] - version = self._version(backend) + # A down candidate with another behind it fails over at once; only the + # last one pauses through a restart (MN-REQ-06.19). + version = self._version(backend, retry_s=0.0 if i < last else None) if version is None: unreachable = _gateway("backend_unreachable", backend.id, http=502) continue @@ -515,7 +518,7 @@ def _require_live(self, backend: Backend, product: Product) -> HandleResult | No ) return None - def _version(self, backend: Backend) -> str | None: + def _version(self, backend: Backend, *, retry_s: float | None = None) -> str | None: now = time.monotonic() with self._lock: hit = self._ver_cache.get(backend.id) @@ -533,6 +536,7 @@ def _version(self, backend: Backend) -> str | None: host=backend.host, port=backend.port, timeout=self.timeout_s, + retry_s=retry_s, ) except (OSError, json.JSONDecodeError): raw = None diff --git a/tests/test_product_gateway.py b/tests/test_product_gateway.py index 818ed72..8b86160 100644 --- a/tests/test_product_gateway.py +++ b/tests/test_product_gateway.py @@ -189,7 +189,14 @@ def test_forward_keeps_args_and_stdin(monkeypatch: pytest.MonkeyPatch, tmp_path: calls: list[dict] = [] def fake_send( - args, stdin=None, host=None, port=None, timeout=None, admin_token=None, admin_usage=False + args, + stdin=None, + host=None, + port=None, + timeout=None, + admin_token=None, + admin_usage=False, + retry_s=None, ): calls.append( { diff --git a/tests/test_serve_restart_retry.py b/tests/test_serve_restart_retry.py index 0082aac..e3bf107 100644 --- a/tests/test_serve_restart_retry.py +++ b/tests/test_serve_restart_retry.py @@ -340,3 +340,37 @@ def call(args: list[str], **extra: Any): timer.join() for serve in back: serve.close() + + +def test_gateway_placement_fails_over_past_a_dead_backend_fast( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """Unpinned open: backend a stays down, b is up. Placement skips a without the pause.""" + import time + + from memnet_mcp.product_gateway import ProductGateway + from test_product_gateway import ENDLEAF, _config + + monkeypatch.setenv("MEMNET_SERVE_RETRY_S", "30") + version = {"exit_code": 0, "stdout": f"@VER: memnet|{__version__}\n", "stderr": ""} + opened = {"exit_code": 0, "stdout": "@SESSION: mn_gw2|soon|60\n", "stderr": ""} + port_a, port_b = _free_port(), _free_port() + cfg, state = tmp_path / "gw.json", tmp_path / "owners.json" + _config(cfg, state, port_a, port_b) + gw = ProductGateway.load(str(cfg)) + live = FakeServe(port_b, [version, opened]) + try: + start = time.monotonic() + resp = gw.handle( + method="POST", + path="/gateway", + authorization=f"Bearer {ENDLEAF}", + body=json.dumps({"args": ["session", "open", "--map-file", "m.txt"]}).encode(), + ) + elapsed = time.monotonic() - start + finally: + live.close() + assert resp.exit_code == 0, resp.stderr + assert "mn_gw2" in resp.stdout + assert [r["args"][0] for r in live.delivered] == ["version", "session"] + assert elapsed < 2.0, f"placement paused {elapsed:.1f}s on the dead backend"