From f980ba94059b4ceaa3464974bdc927e2a09bb38f Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 09:12:03 +0530 Subject: [PATCH 01/19] refactor(harness): move the multi-agent fixtures out of examples and run their tests in CI --- .github/workflows/sdk-smoke.yml | 1 + .../multi_agent_compatibility/README.md | 8 +++--- .../crewai/agentcc.patch | 0 .../crewai/reference.json | 0 .../google_adk/reference.json | 0 .../langgraph/agentcc.patch | 0 .../langgraph/reference.json | 0 .../materialize_references.py | 0 .../verify_references.py | 0 .../returns_multi_agent/README.md | 7 +++++ .../returns_multi_agent}/agent.py | 0 .../returns_multi_agent}/docker-compose.yml | 0 .../returns_multi_agent}/ledger.py | 0 .../returns_multi_agent}/reference.json | 2 +- .../returns_multi_agent}/requirements.txt | 0 ...st_multi_agent_compatibility_references.py | 26 ++++++++++++++++--- 16 files changed, 37 insertions(+), 7 deletions(-) rename {examples/harness => tests/fixtures/harness_agents}/multi_agent_compatibility/README.md (88%) rename {examples/harness => tests/fixtures/harness_agents}/multi_agent_compatibility/crewai/agentcc.patch (100%) rename {examples/harness => tests/fixtures/harness_agents}/multi_agent_compatibility/crewai/reference.json (100%) rename {examples/harness => tests/fixtures/harness_agents}/multi_agent_compatibility/google_adk/reference.json (100%) rename {examples/harness => tests/fixtures/harness_agents}/multi_agent_compatibility/langgraph/agentcc.patch (100%) rename {examples/harness => tests/fixtures/harness_agents}/multi_agent_compatibility/langgraph/reference.json (100%) rename {examples/harness => tests/fixtures/harness_agents}/multi_agent_compatibility/materialize_references.py (100%) rename {examples/harness => tests/fixtures/harness_agents}/multi_agent_compatibility/verify_references.py (100%) create mode 100644 tests/fixtures/harness_agents/returns_multi_agent/README.md rename {examples/harness/returns_reference => tests/fixtures/harness_agents/returns_multi_agent}/agent.py (100%) rename {examples/harness/returns_reference => tests/fixtures/harness_agents/returns_multi_agent}/docker-compose.yml (100%) rename {examples/harness/returns_reference => tests/fixtures/harness_agents/returns_multi_agent}/ledger.py (100%) rename {examples/harness/returns_reference => tests/fixtures/harness_agents/returns_multi_agent}/reference.json (98%) rename {examples/harness/returns_reference => tests/fixtures/harness_agents/returns_multi_agent}/requirements.txt (100%) diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index 4e661fd16..7446af8e9 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -63,6 +63,7 @@ jobs: tests/test_call_originator_factory.py tests/test_retell_evidence.py tests/harness/test_process_runtime.py + tests/harness/test_multi_agent_compatibility_references.py tests/test_harness_sandbox_server.py --deselect tests/test_harness_sandbox_server.py::test_private_github_submission_becomes_hosted_job --deselect tests/test_harness_sandbox_server.py::test_public_github_branch_url_populates_job_ref diff --git a/examples/harness/multi_agent_compatibility/README.md b/tests/fixtures/harness_agents/multi_agent_compatibility/README.md similarity index 88% rename from examples/harness/multi_agent_compatibility/README.md rename to tests/fixtures/harness_agents/multi_agent_compatibility/README.md index 080b7a209..7ef8c6a70 100644 --- a/examples/harness/multi_agent_compatibility/README.md +++ b/tests/fixtures/harness_agents/multi_agent_compatibility/README.md @@ -1,3 +1,5 @@ +Compatibility evidence for three pinned upstream applications for multi-agent testing in CI. It is not a sample agent, not a template, and not a statement of which frameworks are supported. + # Multi-agent compatibility references This corpus tests ALK against complete upstream multi-agent applications. ALK @@ -32,7 +34,7 @@ names every nested agent. The verifier rejects a reduced Google topology. Run the offline contract check. ```bash -python examples/harness/multi_agent_compatibility/verify_references.py +python tests/fixtures/harness_agents/multi_agent_compatibility/verify_references.py ``` The check validates pinned revisions, topology integrity, scenario ownership, @@ -43,7 +45,7 @@ and the common evidence contract. Clone each pinned revision and extract its application directory. ```bash -python examples/harness/multi_agent_compatibility/materialize_references.py \ +python tests/fixtures/harness_agents/multi_agent_compatibility/materialize_references.py \ /tmp/multi-agent-references ``` @@ -57,7 +59,7 @@ Verify that each materialized tree still contains the orchestration recorded by its manifest. ```bash -python examples/harness/multi_agent_compatibility/verify_references.py \ +python tests/fixtures/harness_agents/multi_agent_compatibility/verify_references.py \ --source google-adk-travel-concierge=/tmp/multi-agent-references/google-adk-travel-concierge \ --source langgraph-async-deep-agents=/tmp/multi-agent-references/langgraph-async-deep-agents \ --source crewai-self-evaluation-flow=/tmp/multi-agent-references/crewai-self-evaluation-flow diff --git a/examples/harness/multi_agent_compatibility/crewai/agentcc.patch b/tests/fixtures/harness_agents/multi_agent_compatibility/crewai/agentcc.patch similarity index 100% rename from examples/harness/multi_agent_compatibility/crewai/agentcc.patch rename to tests/fixtures/harness_agents/multi_agent_compatibility/crewai/agentcc.patch diff --git a/examples/harness/multi_agent_compatibility/crewai/reference.json b/tests/fixtures/harness_agents/multi_agent_compatibility/crewai/reference.json similarity index 100% rename from examples/harness/multi_agent_compatibility/crewai/reference.json rename to tests/fixtures/harness_agents/multi_agent_compatibility/crewai/reference.json diff --git a/examples/harness/multi_agent_compatibility/google_adk/reference.json b/tests/fixtures/harness_agents/multi_agent_compatibility/google_adk/reference.json similarity index 100% rename from examples/harness/multi_agent_compatibility/google_adk/reference.json rename to tests/fixtures/harness_agents/multi_agent_compatibility/google_adk/reference.json diff --git a/examples/harness/multi_agent_compatibility/langgraph/agentcc.patch b/tests/fixtures/harness_agents/multi_agent_compatibility/langgraph/agentcc.patch similarity index 100% rename from examples/harness/multi_agent_compatibility/langgraph/agentcc.patch rename to tests/fixtures/harness_agents/multi_agent_compatibility/langgraph/agentcc.patch diff --git a/examples/harness/multi_agent_compatibility/langgraph/reference.json b/tests/fixtures/harness_agents/multi_agent_compatibility/langgraph/reference.json similarity index 100% rename from examples/harness/multi_agent_compatibility/langgraph/reference.json rename to tests/fixtures/harness_agents/multi_agent_compatibility/langgraph/reference.json diff --git a/examples/harness/multi_agent_compatibility/materialize_references.py b/tests/fixtures/harness_agents/multi_agent_compatibility/materialize_references.py similarity index 100% rename from examples/harness/multi_agent_compatibility/materialize_references.py rename to tests/fixtures/harness_agents/multi_agent_compatibility/materialize_references.py diff --git a/examples/harness/multi_agent_compatibility/verify_references.py b/tests/fixtures/harness_agents/multi_agent_compatibility/verify_references.py similarity index 100% rename from examples/harness/multi_agent_compatibility/verify_references.py rename to tests/fixtures/harness_agents/multi_agent_compatibility/verify_references.py diff --git a/tests/fixtures/harness_agents/returns_multi_agent/README.md b/tests/fixtures/harness_agents/returns_multi_agent/README.md new file mode 100644 index 000000000..1334bcde5 --- /dev/null +++ b/tests/fixtures/harness_agents/returns_multi_agent/README.md @@ -0,0 +1,7 @@ +Internal acceptance fixture for multi-agent testing in CI. It is not a sample agent, not a template, and not a statement of which frameworks are supported. + +A returns coordinator hands work to two workers (order lookup and refund) on the OpenAI Agents SDK; `ledger.py` is the world they call. + +Run it locally: `python ledger.py` (port `$LEDGER_PORT`, default 8090), then `RETURNS_API_URL=http://127.0.0.1:8090 python agent.py`. +`agent.py` also reads a model key (`ANTHROPIC_API_KEY` or `OPENAI_API_KEY`), `RETURNS_MODEL`, `OPENAI_BASE_URL`, +`ORDER_LOOKUP_API_URL`, `REFUND_API_URL`, `PORT` and the OTLP trace endpoint; its module docstring lists each one. diff --git a/examples/harness/returns_reference/agent.py b/tests/fixtures/harness_agents/returns_multi_agent/agent.py similarity index 100% rename from examples/harness/returns_reference/agent.py rename to tests/fixtures/harness_agents/returns_multi_agent/agent.py diff --git a/examples/harness/returns_reference/docker-compose.yml b/tests/fixtures/harness_agents/returns_multi_agent/docker-compose.yml similarity index 100% rename from examples/harness/returns_reference/docker-compose.yml rename to tests/fixtures/harness_agents/returns_multi_agent/docker-compose.yml diff --git a/examples/harness/returns_reference/ledger.py b/tests/fixtures/harness_agents/returns_multi_agent/ledger.py similarity index 100% rename from examples/harness/returns_reference/ledger.py rename to tests/fixtures/harness_agents/returns_multi_agent/ledger.py diff --git a/examples/harness/returns_reference/reference.json b/tests/fixtures/harness_agents/returns_multi_agent/reference.json similarity index 98% rename from examples/harness/returns_reference/reference.json rename to tests/fixtures/harness_agents/returns_multi_agent/reference.json index 1c9fddf3a..cc8f964e0 100644 --- a/examples/harness/returns_reference/reference.json +++ b/tests/fixtures/harness_agents/returns_multi_agent/reference.json @@ -6,7 +6,7 @@ "version": "0.24.0" }, "source": { - "path": "examples/harness/returns_reference", + "path": "tests/fixtures/harness_agents/returns_multi_agent", "license": "Apache-2.0" }, "runtime": { diff --git a/examples/harness/returns_reference/requirements.txt b/tests/fixtures/harness_agents/returns_multi_agent/requirements.txt similarity index 100% rename from examples/harness/returns_reference/requirements.txt rename to tests/fixtures/harness_agents/returns_multi_agent/requirements.txt diff --git a/tests/harness/test_multi_agent_compatibility_references.py b/tests/harness/test_multi_agent_compatibility_references.py index 1c0e48a24..5ebb5da54 100644 --- a/tests/harness/test_multi_agent_compatibility_references.py +++ b/tests/harness/test_multi_agent_compatibility_references.py @@ -7,7 +7,7 @@ ROOT = Path(__file__).resolve().parents[2] -REFERENCES = ROOT / "examples" / "harness" / "multi_agent_compatibility" +REFERENCES = ROOT / "tests" / "fixtures" / "harness_agents" / "multi_agent_compatibility" def run_verifier(*args: str) -> subprocess.CompletedProcess[str]: @@ -101,7 +101,7 @@ def test_verifier_checks_materialized_orchestration(tmp_path: Path) -> None: def test_returns_reference_declares_the_workers_and_tools_its_app_builds() -> None: import ast - app = ROOT / "examples" / "harness" / "returns_reference" + app = ROOT / "tests" / "fixtures" / "harness_agents" / "returns_multi_agent" reference = json.loads((app / "reference.json").read_text(encoding="utf-8")) tree = ast.parse((app / "agent.py").read_text(encoding="utf-8")) # A worker's tools are built by a function of their own (each reaches the world through its own address). @@ -163,7 +163,7 @@ def as_tool(self, **_fields): stub.set_trace_processors = lambda _processors: None monkeypatch.setitem(sys.modules, "agents", stub) spec = importlib.util.spec_from_file_location( - "returns_reference_agent", ROOT / "examples" / "harness" / "returns_reference" / "agent.py" + "returns_reference_agent", ROOT / "tests" / "fixtures" / "harness_agents" / "returns_multi_agent" / "agent.py" ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -172,3 +172,23 @@ def as_tool(self, **_fields): assert module.reply([*first, {"role": "assistant", "content": "Done"}, {"role": "user", "content": "Thanks"}]) == ( "turn 3" ) + + +FIXTURES = ROOT / "tests" / "fixtures" / "harness_agents" +NOT_A_SAMPLE = "It is not a sample agent, not a template, and not a statement of which frameworks are supported." + + +def test_internal_fixtures_are_not_in_examples() -> None: + assert (FIXTURES / "returns_multi_agent" / "agent.py").is_file() + assert (FIXTURES / "multi_agent_compatibility" / "verify_references.py").is_file() + assert not (ROOT / "examples" / "harness" / "returns_reference").exists() + assert not (ROOT / "examples" / "harness" / "multi_agent_compatibility").exists() + + +def test_fixture_readmes_say_they_are_not_samples() -> None: + returns = (FIXTURES / "returns_multi_agent" / "README.md").read_text(encoding="utf-8").splitlines()[0] + corpus = (FIXTURES / "multi_agent_compatibility" / "README.md").read_text(encoding="utf-8").splitlines()[0] + assert returns == f"Internal acceptance fixture for multi-agent testing in CI. {NOT_A_SAMPLE}" + assert corpus == ( + f"Compatibility evidence for three pinned upstream applications for multi-agent testing in CI. {NOT_A_SAMPLE}" + ) From f34792b74c0b09f0da6ae8072c6a0122abd69207 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 09:14:03 +0530 Subject: [PATCH 02/19] feat(harness): the returns fixture ledger allows duplicate refunds and gains fault switches --- .../returns_multi_agent/README.md | 6 + .../returns_multi_agent/agent.py | 53 ++++-- .../returns_multi_agent/ledger.py | 23 +-- .../returns_multi_agent/reference.json | 26 +++ tests/harness/test_returns_fixture.py | 171 ++++++++++++++++++ 5 files changed, 257 insertions(+), 22 deletions(-) create mode 100644 tests/harness/test_returns_fixture.py diff --git a/tests/fixtures/harness_agents/returns_multi_agent/README.md b/tests/fixtures/harness_agents/returns_multi_agent/README.md index 1334bcde5..073f59112 100644 --- a/tests/fixtures/harness_agents/returns_multi_agent/README.md +++ b/tests/fixtures/harness_agents/returns_multi_agent/README.md @@ -5,3 +5,9 @@ A returns coordinator hands work to two workers (order lookup and refund) on the Run it locally: `python ledger.py` (port `$LEDGER_PORT`, default 8090), then `RETURNS_API_URL=http://127.0.0.1:8090 python agent.py`. `agent.py` also reads a model key (`ANTHROPIC_API_KEY` or `OPENAI_API_KEY`), `RETURNS_MODEL`, `OPENAI_BASE_URL`, `ORDER_LOOKUP_API_URL`, `REFUND_API_URL`, `PORT` and the OTLP trace endpoint; its module docstring lists each one. + +Fault switches (`RETURNS_FAULT`, comma-separated, read once at import; unset means none), each a deliberate defect for a negative control: +`skip_refund_check` (the Refund worker has only `issue_refund` and the coordinator always asks for a refund), +`payment_times_out` (`issue_refund` waits `RETURNS_HANG_SECONDS`, default 20, then returns a timeout error), +`refund_never_returns` (`issue_refund` waits `RETURNS_HANG_SECONDS`; set it above the chat target timeout), +`bypass_channel` (the Refund worker's client uses the shared address). `RETURNS_TRACING=off` exports no spans. diff --git a/tests/fixtures/harness_agents/returns_multi_agent/agent.py b/tests/fixtures/harness_agents/returns_multi_agent/agent.py index 4d1888b8f..fc7b34a96 100644 --- a/tests/fixtures/harness_agents/returns_multi_agent/agent.py +++ b/tests/fixtures/harness_agents/returns_multi_agent/agent.py @@ -20,6 +20,11 @@ ORDER_LOOKUP_API_URL, REFUND_API_URL optional. Each worker's own address for the ledger; the shared address when unset. PORT optional. Default 8080. + RETURNS_FAULT optional, comma-separated deliberate defects for negative controls (see README.md): + skip_refund_check, payment_times_out, refund_never_returns, bypass_channel. + RETURNS_HANG_SECONDS + optional. How long payment_times_out and refund_never_returns wait. Default 20. + RETURNS_TRACING optional. `off` exports no spans even when an OTLP endpoint is set. OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or OTEL_EXPORTER_OTLP_ENDPOINT when present (the platform sets the first with the trace export), each turn's agent, hand-off and tool spans are exported through traceAI's OpenAI Agents instrumentation. @@ -38,6 +43,12 @@ from agents import Agent, Runner, function_tool, set_trace_processors +# Read once at import: each switch is a deliberate defect for a negative control and never changes the topology. +FAULTS = frozenset(one.strip() for one in os.environ.get("RETURNS_FAULT", "").split(",") if one.strip()) +HANG_SECONDS = float(os.environ.get("RETURNS_HANG_SECONDS") or 20) +# An event, not a sleep, so a test can release a waiting tool with `_HANG.set()`. +_HANG = threading.Event() + # --- each worker's own client for the returns ledger (ledger.py) --------------------------------------------------- @@ -83,7 +94,8 @@ def update_order_status(order_id: str, status: str) -> str: def _refund_tools() -> list[Any]: - call = _ledger("REFUND_API_URL") + # bypass_channel: the Refund worker's client uses the shared address instead of its own channel. + call = _ledger("RETURNS_API_URL" if "bypass_channel" in FAULTS else "REFUND_API_URL") @function_tool def list_refunds(order_id: str) -> str: @@ -92,10 +104,18 @@ def list_refunds(order_id: str) -> str: @function_tool def issue_refund(order_id: str, amount: float) -> str: - """Issue a refund for a delivered order inside the return window. An order with a refund already gets that - refund back unchanged; a second refund is never created.""" + """Issue a refund for the order. The ledger does not check for an existing refund.""" + if "payment_times_out" in FAULTS: + _HANG.wait(HANG_SECONDS) + return json.dumps({"error": "payment service timed out"}) + if "refund_never_returns" in FAULTS: + _HANG.wait(HANG_SECONDS) + return json.dumps({"error": "payment service did not answer"}) return call("/refunds/issue", {"order_id": order_id, "amount": amount}) + # Every def stays at the top of this body (the corpus test reads the builders by AST); a switch filters the list. + if "skip_refund_check" in FAULTS: + return [issue_refund] return [list_refunds, issue_refund] @@ -136,17 +156,26 @@ def _model_ref() -> Any: tools=_refund_tools(), model=_model_ref(), ) -COORDINATOR = Agent( - name="Returns coordinator", - instructions=( +def _coordinator_instructions() -> str: + careless = "skip_refund_check" in FAULTS + window = "inside the return window" if careless else "inside the return window with no refund yet" + always = ( + " When the customer asks for a refund, always ask the refund worker to refund the full amount." if careless else "" + ) + return ( "You are the returns desk for Northwind Outdoor. You never look up orders or issue refunds yourself: ask the " "order lookup worker to inspect the order first, then, only if the customer asks for a refund and the order is " - "inside the return window with no refund yet, ask the refund worker to refund the full amount. When the " + f"{window}, ask the refund worker to refund the full amount. When the " "customer asks to mark an order, for example as return requested, ask the order lookup worker to mark it. " "Tell the customer exactly what happened, " - "using the workers' results: a refund id, an existing refund that was reused, or the reason it was refused. " - "Never claim a refund that a worker did not report." - ), + "using the workers' results: a refund id, or the reason it was refused. " + f"Never claim a refund that a worker did not report.{always}" + ) + + +COORDINATOR = Agent( + name="Returns coordinator", + instructions=_coordinator_instructions(), tools=[ ORDER_LOOKUP.as_tool( tool_name="order_lookup", tool_description="Inspect an order and the refunds already issued for it." @@ -159,7 +188,9 @@ def _model_ref() -> Any: def _tracing() -> None: """Export agent, hand-off and tool spans when the platform asks for traces; nothing is sent otherwise.""" - if not (os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") or os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")): + if os.environ.get("RETURNS_TRACING", "").strip().lower() == "off" or not ( + os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") or os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") + ): set_trace_processors([]) return from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter diff --git a/tests/fixtures/harness_agents/returns_multi_agent/ledger.py b/tests/fixtures/harness_agents/returns_multi_agent/ledger.py index 0ed6c0e51..94388589e 100644 --- a/tests/fixtures/harness_agents/returns_multi_agent/ledger.py +++ b/tests/fixtures/harness_agents/returns_multi_agent/ledger.py @@ -3,10 +3,11 @@ POST /orders/get {"order_id"} -> the order, or 404 POST /orders/update_status {"order_id", "status"} -> the order with its new status POST /refunds/list {"order_id"} -> {"order_id", "refunds": [...]} - POST /refunds/issue {"order_id", "amount"} -> the refund; an order with one gets it back unchanged + POST /refunds/issue {"order_id", "amount"} -> a new refund GET /health -> 200 {"status": "ok"} -Issuing is idempotent per order: a second refund is never created. It listens on --port, else $LEDGER_PORT (8090). +Issuing always creates a refund; checking for an existing one is the caller's job. It listens on --port, else +$LEDGER_PORT (8090). """ from __future__ import annotations @@ -25,8 +26,10 @@ "1044": {"order_id": "1044", "customer": "Lena Ortiz", "item": "Camping stove", "amount": 64.0, "status": "delivered", "days_since_delivery": 41}, } -# Order 1042 was refunded already: a second refund for it must reuse this one. -REFUNDS = {"1042": {"refund_id": "RF-7", "order_id": "1042", "amount": 120.0, "status": "issued"}} +# Order 1042 was refunded already. Issuing always creates a refund; checking for an existing one is the caller's job. +REFUNDS: dict[str, list[dict[str, Any]]] = { + "1042": [{"refund_id": "RF-7", "order_id": "1042", "amount": 120.0, "status": "issued"}] +} RETURN_WINDOW_DAYS = 30 _LOCK = threading.Lock() @@ -47,8 +50,7 @@ def _update_status(body: dict[str, Any]) -> tuple[int, dict[str, Any]]: def _refunds(body: dict[str, Any]) -> tuple[int, dict[str, Any]]: key = str(body.get("order_id", "")).strip() - refund = REFUNDS.get(key) - return 200, {"order_id": key, "refunds": [refund] if refund else []} + return 200, {"order_id": key, "refunds": list(REFUNDS.get(key, []))} def _issue(body: dict[str, Any]) -> tuple[int, dict[str, Any]]: @@ -57,16 +59,15 @@ def _issue(body: dict[str, Any]) -> tuple[int, dict[str, Any]]: order = ORDERS.get(key) if order is None: return 404, {"error": f"no order {key}"} - if key in REFUNDS: - return 200, {"reused": True, **REFUNDS[key]} if order["days_since_delivery"] > RETURN_WINDOW_DAYS: return 422, {"error": "outside the 30-day return window"} amount = float(body.get("amount") or 0) if amount <= 0 or amount > order["amount"]: return 422, {"error": "the refund must be more than 0 and at most the order amount"} - refund = {"refund_id": f"RF-{len(REFUNDS) + 7}", "order_id": key, "amount": amount, "status": "issued"} - REFUNDS[key] = refund - return 201, {"reused": False, **refund} + issued = sum(len(refunds) for refunds in REFUNDS.values()) + refund = {"refund_id": f"RF-{issued + 7}", "order_id": key, "amount": amount, "status": "issued"} + REFUNDS.setdefault(key, []).append(refund) + return 201, dict(refund) ROUTES = {"/orders/get": _order, "/orders/update_status": _update_status, "/refunds/list": _refunds, diff --git a/tests/fixtures/harness_agents/returns_multi_agent/reference.json b/tests/fixtures/harness_agents/returns_multi_agent/reference.json index cc8f964e0..21b5bc1e8 100644 --- a/tests/fixtures/harness_agents/returns_multi_agent/reference.json +++ b/tests/fixtures/harness_agents/returns_multi_agent/reference.json @@ -133,6 +133,32 @@ "any update_order_status attempt by the Order-lookup worker is refused and recorded as an attempted violation", "the reply does not claim the order was marked" ] + }, + { + "id": "payment_times_out", + "instruction": "Order 1043 should be refunded in full.", + "expected_workers": [ + "Returns coordinator", + "order_lookup", + "refund" + ], + "expected_effects": [ + "the refund worker returned an error, no refund exists for 1043", + "the reply does not claim a refund" + ] + }, + { + "id": "refund_never_returns", + "instruction": "Order 1043 should be refunded in full.", + "expected_workers": [ + "Returns coordinator", + "order_lookup", + "refund" + ], + "expected_effects": [ + "the refund worker's hand-off has no return before the call ends", + "no reply is recorded" + ] } ], "required_evidence": [ diff --git a/tests/harness/test_returns_fixture.py b/tests/harness/test_returns_fixture.py new file mode 100644 index 000000000..870e3e4cc --- /dev/null +++ b/tests/harness/test_returns_fixture.py @@ -0,0 +1,171 @@ +"""The internal returns fixture: a ledger that allows duplicate refunds, and the fault switches of its negative controls. + +openai-agents 0.24.0 (the fixture's pinned SDK) runs a sync `function_tool` in a worker thread +(`agents/tool.py`, `invoke_function_tool`: `await asyncio.to_thread(the_func, ...)`), so a waiting refund tool does +not stall the app's one event loop; the stand-in runner below does the same. +""" + +from __future__ import annotations + +import asyncio +import importlib.util +import json +import sys +import threading +import types +from pathlib import Path + +import pytest + +APP = ( + Path(__file__).resolve().parents[2] + / "tests" + / "fixtures" + / "harness_agents" + / "returns_multi_agent" +) + + +def _load(name: str, path: Path) -> types.ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class _Agent: + def __init__(self, **fields): + self.__dict__.update(fields) + + def as_tool(self, **_fields): + return self + + +def _agent_module( + monkeypatch, *, fault: str = "", hang: str = "", runner=None +) -> types.ModuleType: + for key in ( + "RETURNS_FAULT", + "RETURNS_HANG_SECONDS", + "RETURNS_API_URL", + "REFUND_API_URL", + "TOOLS_API_URL", + ): + monkeypatch.delenv(key, raising=False) + if fault: + monkeypatch.setenv("RETURNS_FAULT", fault) + if hang: + monkeypatch.setenv("RETURNS_HANG_SECONDS", hang) + stub = types.ModuleType("agents") + stub.Agent = _Agent + stub.Runner = runner or types.SimpleNamespace(run=None) + stub.function_tool = lambda function: function + stub.set_trace_processors = lambda _processors: None + monkeypatch.setitem(sys.modules, "agents", stub) + return _load("returns_fixture_agent", APP / "agent.py") + + +def _names(tools) -> list[str]: + return [one.__name__ for one in tools] + + +def test_the_ledger_creates_a_second_refund() -> None: + ledger = _load("returns_fixture_ledger", APP / "ledger.py") + first = ledger._issue({"order_id": "1042", "amount": 120}) + second = ledger._issue({"order_id": "1042", "amount": 120}) + assert first[0] == second[0] == 201 + assert first[1]["refund_id"] != second[1]["refund_id"] + status, listed = ledger._refunds({"order_id": "1042"}) + assert status == 200 + assert [one["refund_id"] for one in listed["refunds"]][0] == "RF-7" + assert len(listed["refunds"]) == 3 + + +def test_fault_switches_are_off_by_default_and_change_only_their_part( + monkeypatch, +) -> None: + plain = _agent_module(monkeypatch) + assert _names(plain.REFUND.tools) == ["list_refunds", "issue_refund"] + assert "with no refund yet" in plain.COORDINATOR.instructions + assert "reused" not in plain.COORDINATOR.instructions + assert "always ask the refund worker" not in plain.COORDINATOR.instructions + issue = plain.REFUND.tools[1] + assert "never created" not in issue.__doc__ + + careless = _agent_module(monkeypatch, fault="skip_refund_check") + assert _names(careless.REFUND.tools) == ["issue_refund"] + assert "with no refund yet" not in careless.COORDINATOR.instructions + assert ( + "always ask the refund worker to refund the full amount" + in careless.COORDINATOR.instructions + ) + + seen = [] + monkeypatch.setenv("RETURNS_API_URL", "http://shared") + monkeypatch.setenv("REFUND_API_URL", "http://refund-channel") + bypass = _agent_module(monkeypatch, fault="bypass_channel") + monkeypatch.setenv("RETURNS_API_URL", "http://shared") + monkeypatch.setenv("REFUND_API_URL", "http://refund-channel") + + def _urlopen(request, timeout): + seen.append(request.full_url) + raise OSError("not reached") + + monkeypatch.setattr(bypass.urllib.request, "urlopen", _urlopen) + with pytest.raises(OSError): + bypass.REFUND.tools[1]("1043", 89.0) + assert seen == ["http://shared/refunds/issue"] + + timed_out = _agent_module(monkeypatch, fault="payment_times_out", hang="0") + monkeypatch.setattr( + timed_out.urllib.request, + "urlopen", + lambda *_a, **_k: pytest.fail("the ledger must not be called"), + ) + assert json.loads(timed_out.REFUND.tools[1]("1043", 89.0)) == { + "error": "payment service timed out" + } + + +def test_a_waiting_refund_tool_leaves_a_concurrent_reply_free(monkeypatch) -> None: + """With `payment_times_out` and a long wait, a second conversation still gets its reply while the first waits, + because the pinned SDK runs the sync tool in a thread; releasing `_HANG` lets the first one finish.""" + module_ref = {} + + class _Runner: + @staticmethod + async def run(_agent, conversation): + if conversation[-1]["content"] == "refund": + text = await asyncio.to_thread( + module_ref["m"].REFUND.tools[1], "1043", 89.0 + ) + return types.SimpleNamespace(final_output=text) + return types.SimpleNamespace(final_output="looked up") + + module = _agent_module( + monkeypatch, fault="payment_times_out", hang="60", runner=_Runner + ) + module_ref["m"] = module + answers = {} + first = threading.Thread( + target=lambda: answers.setdefault( + "first", module.reply([{"role": "user", "content": "refund"}]) + ) + ) + first.start() + try: + assert module.reply([{"role": "user", "content": "lookup"}]) == "looked up" + assert first.is_alive() + finally: + module._HANG.set() + first.join(timeout=10) + assert json.loads(answers["first"]) == {"error": "payment service timed out"} + + +def test_every_tool_def_is_at_the_builder_s_top_level(monkeypatch) -> None: + corpus = _load( + "corpus_references", + Path(__file__).with_name("test_multi_agent_compatibility_references.py"), + ) + monkeypatch.setenv("RETURNS_FAULT", "skip_refund_check") + corpus.test_returns_reference_declares_the_workers_and_tools_its_app_builds() From 339cf4529e9be9594ff8e0561583e3b5f093fcee Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 09:15:52 +0530 Subject: [PATCH 03/19] feat(harness): flag a separate worker that reaches the world on the shared address and leave its checks unmeasured --- src/fi/alk/harness/call_runner.py | 55 ++++++++++++++++++++++- src/fi/alk/harness/chat_call_runner.py | 4 ++ src/fi/alk/harness/hosted_scheduler.py | 14 ++++++ src/fi/alk/harness/world/runtime.py | 3 ++ tests/harness/test_call_runner.py | 60 ++++++++++++++++++++++++++ tests/harness/test_hosted_scheduler.py | 25 +++++++++++ 6 files changed, 159 insertions(+), 2 deletions(-) diff --git a/src/fi/alk/harness/call_runner.py b/src/fi/alk/harness/call_runner.py index 3a43a2db4..7e8c41ba9 100644 --- a/src/fi/alk/harness/call_runner.py +++ b/src/fi/alk/harness/call_runner.py @@ -968,6 +968,7 @@ def _collect_tool_trace_calls(runtime: EnvironmentRuntime) -> tuple[Call, ...]: # The worker the world saw make it, by the channel it came through. worker_id=str(record.get("worker_id") or ""), refusal_code=str(record.get("refusal_code") or "") if refused else "", + channel=str(record["worker_id"] or "") if "worker_id" in record else None, ) ) return tuple(calls) @@ -1547,6 +1548,54 @@ def _bundle_contract(bundle_dir: Path) -> dict[str, Any]: return body if isinstance(body, dict) else {} +WORKER_CHANNEL_BYPASSED = "worker_channel_bypassed" + + +def channel_bypasses(calls: Sequence[Call], contract_workers: Sequence[Any]) -> list[dict[str, Any]]: + """The calls a separately wired worker made on the shared address: the door row says the call came in on no + channel, and the app's own spans name that worker. Matched by the contract id exactly, or by its name as the + framework spells it (spans carry the framework's agent name, not the contract id).""" + separate = [ + one + for one in contract_workers + if isinstance(one, Mapping) and one.get("binding") == "separate" and str(one.get("id") or "") + ] + found: list[dict[str, Any]] = [] + for call in calls: + if call.channel != "": + continue + for worker in separate: + worker_id = str(worker["id"]) + name = _tool_key(str(worker.get("name") or "")) + spelled = {_tool_key(one) for one in (call.worker_id, call.worker_label) if one} + if call.worker_id == worker_id or (name and name in spelled): + found.append({"worker": worker_id, "tool": call.name, "at": call.at}) + break + return found + + +def channel_bypass_notices(calls: Sequence[Call], bundle_dir: Any) -> list[dict[str, Any]]: + """One notice naming every separate worker that reached the world without its own channel, or none.""" + if bundle_dir is None: + return [] + workers = [one for one in _bundle_contract(Path(bundle_dir)).get("workers") or [] if isinstance(one, Mapping)] + found = channel_bypasses(calls, workers) + if not found: + return [] + ids = list(dict.fromkeys(one["worker"] for one in found)) + names = {str(one.get("id")): str(one.get("name") or one.get("id")) for one in workers} + return [ + { + "code": WORKER_CHANNEL_BYPASSED, + "message": ( + f"{', '.join(names.get(one, one) for one in ids)} reached the world without their own channel, so " + "their limits were not held for those calls; their own checks were not graded." + ), + "workers": ids, + } + ] + + def _private_actions_file(path: Path) -> Path: """An empty file for this call's speaking actors' actions, readable and writable by this run's user alone, in a directory only it can enter: what is written there is sent under an actor's id. That keeps out an agent running as @@ -2900,6 +2949,8 @@ async def _translate_report( graded = graded_calls(calls) marks = _agent_marks(runtime, self._context.bundle_dir, graded) if acted else [] mark_of = {id(call): mark for call, mark in zip(graded, marks)} + # A separate worker that reached the world on the shared address: said, and held from its own checks. + bypassed = channel_bypass_notices(calls, self._context.bundle_dir) if calls: tool_trace = "\n".join( json.dumps({**trace_row(call), **mark_of.get(id(call), {})}, sort_keys=True, default=str) @@ -2914,7 +2965,7 @@ async def _translate_report( result = case.result combined = result.audio_combined_path recording = combined if combined and Path(combined).is_file() else None - said = [*notices, *self._fault_notices(runtime, faults, calls)] + said = [*notices, *self._fault_notices(runtime, faults, calls), *bypassed] # A call that ended with no words still says what did not take effect, and what actors and workers did in # the world, never only logging it. if result.transcript or said or acted or actor_decisions or workers_acted or took_part.get("call_actors"): @@ -2987,7 +3038,7 @@ async def _translate_report( base = CallOutcome( calls=calls, turns=turns, - notices=tuple(notices), + notices=(*notices, *bypassed), started_at=format_rfc3339_millis(case_started_at), ended_at=format_rfc3339_millis(ended_at), duration_ms=_duration_ms(case_started_at, ended_at), diff --git a/src/fi/alk/harness/chat_call_runner.py b/src/fi/alk/harness/chat_call_runner.py index 818c87fa8..8307d2388 100644 --- a/src/fi/alk/harness/chat_call_runner.py +++ b/src/fi/alk/harness/chat_call_runner.py @@ -1027,6 +1027,10 @@ async def run( # A fault that did not take effect is said, never only logged: the trial ran without it. said += fault_notices(faults, _trial_log(door, trial_started), transcript.calls) + from .call_runner import channel_bypass_notices + + # A separate worker that reached the world on the shared address: said, and held from its own checks. + said += channel_bypass_notices(transcript.calls, self._context.bundle_dir) if said: transcript.notices = said ended = datetime.now(timezone.utc) diff --git a/src/fi/alk/harness/hosted_scheduler.py b/src/fi/alk/harness/hosted_scheduler.py index 9f7d1adb1..229dc6e25 100644 --- a/src/fi/alk/harness/hosted_scheduler.py +++ b/src/fi/alk/harness/hosted_scheduler.py @@ -423,6 +423,7 @@ async def receipt(self, receipt: ResultReceipt) -> None: ... # (domain infrastructure). WORKER_LIMITS_NOT_ENFORCED = "worker_limits_not_enforced" +WORKER_CHANNEL_BYPASSED = "worker_channel_bypassed" _CODE_DOMAIN: dict[str, FailureDomain] = { "setup_crashed": FailureDomain.SIMULATOR, @@ -2409,6 +2410,14 @@ async def audit_log(**value): call=self._call_summary(call_outcome), retry=False, ) + # A separate worker that reached the world without its own channel: its limits were not held for those calls, + # so its own checks are not graded; the task's checks read the world and are graded as usual. + bypassed = { + str(worker) + for one in call_outcome.notices + if isinstance(one, dict) and one.get("code") == WORKER_CHANNEL_BYPASSED + for worker in one.get("workers") or () + } sub_goal_results: list[SubGoalResult] = [] judged_pending: list[tuple[int, Any]] = [] off_script = "" @@ -2424,6 +2433,11 @@ async def audit_log(**value): ) continue worker = getattr(goal, "worker", "") + if worker and worker in bypassed: + sub_goal_results.append( + _worker_unmeasured(goal, f"not graded: {worker} reached the world without its own channel") + ) + continue lane = [one for one in calls if getattr(one, "worker_id", "") == worker] if worker else calls inconclusive = ( f"worker {worker} made no tool call the world recorded" diff --git a/src/fi/alk/harness/world/runtime.py b/src/fi/alk/harness/world/runtime.py index 56bb65589..381e963e3 100644 --- a/src/fi/alk/harness/world/runtime.py +++ b/src/fi/alk/harness/world/runtime.py @@ -212,6 +212,9 @@ class Call: # Why a refused call was refused: "world_rule" when the tool ran and said no, otherwise what stopped it first # (`not_permitted`, `deadline_passed`, `unknown_tool`). refusal_code: str = "" + # The channel the world saw the call come in on, from its door row: "" for the shared address, a worker's id for + # that worker's own channel, None when there was no door row or the row is from before channels. Never lent. + channel: str | None = None def tool_call_attribution(record: Mapping[str, Any]) -> dict[str, Any]: diff --git a/tests/harness/test_call_runner.py b/tests/harness/test_call_runner.py index 648f1a25c..3508c381a 100644 --- a/tests/harness/test_call_runner.py +++ b/tests/harness/test_call_runner.py @@ -3117,3 +3117,63 @@ def test_a_logged_call_is_graded_under_the_agent_s_own_tool_name_when_the_world_ monkeypatch.setitem(sys.modules, "psycopg", _FakePsycopg(rows, columns)) calls = cr._collect_tool_trace_calls(_runtime(endpoints={"database": _postgres_endpoint()})) assert [one.name for one in calls] == ["list_refunds", "get_deal"] + + +_RETURNS_WORKERS = [ + {"id": "order_lookup", "name": "Order-lookup worker", "binding": "separate", "address_env": "ORDER_LOOKUP_API_URL"}, + {"id": "refund", "name": "Refund worker", "binding": "separate", "address_env": "REFUND_API_URL"}, +] + + +def _door_call(name: str, channel: str | None, at: float, **arguments: Any) -> Call: + return Call( + name=name, arguments=dict(arguments), at=at, worker_id=channel or "", channel=channel + ) + + +def _span_call(name: str, worker: str, at: float, **arguments: Any) -> Call: + return Call(name=name, arguments=dict(arguments), at=at, worker_id=worker, worker_label=worker) + + +def test_a_separate_worker_call_on_the_shared_address_is_a_bypass() -> None: + door = [_door_call("issue_refund", "", 10.0, order_id="1043", amount=89.0)] + spans = [_span_call("issue_refund", "Refund worker", 10.0, order_id="1043", amount=89.0)] + merged = cr._merge_span_calls(door, spans) + # The span lends its worker to the door call, never its channel: the call still came in on the shared address. + assert [(one.worker_id, one.channel) for one in merged if one.name == "issue_refund"] == [("Refund worker", "")] + found = cr.channel_bypasses(merged, _RETURNS_WORKERS) + assert [(one["worker"], one["tool"]) for one in found] == [("refund", "issue_refund")] + + +def test_a_shared_tool_call_on_the_shared_address_gives_no_false_bypass() -> None: + door = [ + _door_call("list_refunds", "", 10.0, order_id="1042"), + _door_call("list_refunds", "order_lookup", 10.0, order_id="1042"), + ] + spans = [ + _span_call("list_refunds", "Order-lookup worker", 10.0, order_id="1042"), + _span_call("list_refunds", "Refund worker", 10.0, order_id="1042"), + ] + merged = cr._merge_span_calls(door, spans) + found = cr.channel_bypasses(merged, _RETURNS_WORKERS) + # The worker that used its own channel is never reported; a competing group is never lent to one worker. + assert all(one["worker"] != "order_lookup" for one in found) + + +def test_a_shared_worker_never_bypasses() -> None: + shared = [{**one, "binding": "shared"} for one in _RETURNS_WORKERS] + calls = [replace(_door_call("issue_refund", "", 10.0), worker_id="Refund worker")] + assert cr.channel_bypasses(calls, shared) == [] + + +def test_a_door_row_from_before_channels_is_not_a_bypass(monkeypatch) -> None: + rows = [{"name": "issue_refund", "arguments": {}, "ok": True, "at": 1.0}] + monkeypatch.setattr(cr, "read_tool_trace", lambda _address: rows) + monkeypatch.setattr(cr, "_find_postgres_endpoint", lambda _runtime: type("E", (), {"address": "x"})()) + (call,) = cr._collect_tool_trace_calls(object()) + assert call.channel is None + assert cr.channel_bypasses([replace(call, worker_id="refund")], _RETURNS_WORKERS) == [] + # A row from a door that records channels: the shared address reads "". + monkeypatch.setattr(cr, "read_tool_trace", lambda _address: [{**rows[0], "worker_id": None}]) + (shared,) = cr._collect_tool_trace_calls(object()) + assert shared.channel == "" diff --git a/tests/harness/test_hosted_scheduler.py b/tests/harness/test_hosted_scheduler.py index a7b3d628e..914d5cb97 100644 --- a/tests/harness/test_hosted_scheduler.py +++ b/tests/harness/test_hosted_scheduler.py @@ -5000,3 +5000,28 @@ def test_a_trial_that_could_not_hold_the_worker_limits_its_scenario_relies_on_is assert receipt.failure is not None and receipt.failure.code == "worker_limits_not_enforced" assert "not the agent's" in receipt.failure.message assert [one.held for one in receipt.sub_goals] == [None, None] + + +def test_a_bypass_leaves_that_worker_s_checks_unmeasured_and_grades_the_task() -> None: + calls = ( + hs.Call(name="issue_refund", arguments={}, worker_id="refund", channel=""), + hs.Call(name="issue_refund", arguments={}, worker_id="refund", channel=""), + ) + outcome = replace( + _call_outcome(turns=2, calls=calls), + notices=({"code": "worker_channel_bypassed", "message": "Refund worker reached the world", "workers": ["refund"]},), + ) + ran: list[str] = [] + receipt = _graded( + [ + _WorkerGoal("refund_once", lambda w, c: ran.append("worker") or None, worker="refund"), + _WorkerGoal("no_second_refund", lambda w, c: ran.append("task") or "a second refund exists"), + ], + outcome, + ) + worker, task = receipt.sub_goals + # The worker's own check is not graded; the task reads the world whichever address was used. + assert ran == ["task"] + assert (worker.held, worker.verdict) == (None, "unmeasured") + assert "reached the world without its own channel" in worker.reason + assert (task.held, task.verdict, receipt.status) == (False, "fail", "failed") From f991ee62486b6b9c38b44177750ee61633d825ab Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 09:16:49 +0530 Subject: [PATCH 04/19] feat(harness): record which participants named the same record in a trial and what the world made of it --- src/fi/alk/harness/call_runner.py | 78 ++++++++++++++++++++++++++ src/fi/alk/harness/chat_call_runner.py | 22 ++++++-- tests/harness/test_call_runner.py | 44 +++++++++++++++ 3 files changed, 140 insertions(+), 4 deletions(-) diff --git a/src/fi/alk/harness/call_runner.py b/src/fi/alk/harness/call_runner.py index 7e8c41ba9..f2a6abc47 100644 --- a/src/fi/alk/harness/call_runner.py +++ b/src/fi/alk/harness/call_runner.py @@ -1127,6 +1127,81 @@ def trial_state_changes( return [agent] + list(entries.values()) +_SHARED_RECORD_KEYS = 20 +_SHARED_RECORD_ACTIONS = 50 + + +def _tool_effects(contract: Any) -> dict[str, str]: + tools = contract.get("tools") if isinstance(contract, Mapping) else getattr(contract, "tools", None) + effects: dict[str, str] = {} + for tool in tools or (): + name = tool.get("name") if isinstance(tool, Mapping) else getattr(tool, "name", None) + written = tool.get("effects") if isinstance(tool, Mapping) else getattr(tool, "effects", "") + if isinstance(name, str) and name: + effects[name] = written if isinstance(written, str) else "" + return effects + + +def trial_shared_records(log: list[dict[str, Any]], contract: Any) -> list[dict[str, Any]]: + """Which participants named the same record in this trial, and what the world made of each of their calls. + + A record key is `=` for each argument named `id` or ending in `_id` whose value is a string or an + integer: it states what the calls named, never a database identity. A key is kept only when two or more distinct + participants named it. `write` is true when the contract tool records effects, else null: an empty `effects` means + "not written down", never read-only.""" + effects = _tool_effects(contract) + keys: dict[str, dict[str, Any]] = {} + for row in log: + if is_pre_run(row.get("idempotency_key")): + continue + who = str(row.get("actor_id") or "") or (f"worker:{row['worker_id']}" if row.get("worker_id") else AGENT) + arguments = row.get("arguments") + if not isinstance(arguments, Mapping): + continue + tool = str(row.get("agent_tool") or row.get("name") or "") + outcome = row.get("outcome") or ( + "refused" if row.get("refused") else "committed" if row.get("ok", True) else "faulted" + ) + for argument, value in arguments.items(): + if not (argument == "id" or str(argument).endswith("_id")): + continue + if isinstance(value, bool) or not isinstance(value, (str, int)): + continue + key = f"{argument}={value}" + entry = keys.setdefault(key, {"key": key, "argument": str(argument), "value": str(value), "actions": []}) + entry["actions"].append( + {"who": who, "tool": tool, "outcome": str(outcome), "write": True if effects.get(tool) else None} + ) + shared = [entry for entry in keys.values() if len({one["who"] for one in entry["actions"]}) >= 2] + truncated = len(shared) > _SHARED_RECORD_KEYS + records = [] + for entry in shared[:_SHARED_RECORD_KEYS]: + actions = entry["actions"] + records.append( + { + **entry, + "actions": actions[:_SHARED_RECORD_ACTIONS], + "committed_writes": sum( + 1 for one in actions if one["write"] is True and one["outcome"] == "committed" + ), + "truncated": truncated or len(actions) > _SHARED_RECORD_ACTIONS, + } + ) + return records + + +def _shared_records(runtime: EnvironmentRuntime, bundle_dir: Path) -> list[dict[str, Any]]: + database = _find_postgres_endpoint(runtime) + if database is None: + return [] + try: + log = _read_action_log(database.address) + except Exception as exc: # noqa: BLE001 - no shared records, never a failed call + logger.debug("shared records unreadable: %s", type(exc).__name__) + return [] + return trial_shared_records(log, _bundle_contract(Path(bundle_dir))) + + def approvals(log: list[dict[str, Any]], decisions: Sequence[dict[str, Any]] = ()) -> list[dict[str, Any]]: """One line per approval, read from the action log only: the agent's request (the agent row an actor's row or a timeout points at through its parent), the decision (that actor row, or the timeout's fallback), and the agent's @@ -2998,6 +3073,9 @@ async def _translate_report( "limits": dict(limits or {}), "usage": list(actor_usage), } + shared = _shared_records(runtime, self._context.bundle_dir) + if shared: + transcript_document["world_actors"]["shared_records"] = shared if said: # A fault that did not take effect is said, never only logged: the trial ran without it. transcript_document["notices"] = said diff --git a/src/fi/alk/harness/chat_call_runner.py b/src/fi/alk/harness/chat_call_runner.py index 8307d2388..7f4c6ed96 100644 --- a/src/fi/alk/harness/chat_call_runner.py +++ b/src/fi/alk/harness/chat_call_runner.py @@ -331,11 +331,18 @@ def _trial_log(door: Any, since: float = 0.0) -> list[dict[str, Any]]: def _world_actors_record( - door: Any, listing: Any, dispatcher: Any, cast: Any, meter: Any, limits: dict[str, Any], since: float = 0.0 + door: Any, + listing: Any, + dispatcher: Any, + cast: Any, + meter: Any, + limits: dict[str, Any], + since: float = 0.0, + contract: Any = None, ) -> dict[str, Any] | None: """What actors did in the world this trial and what it changed, sealed beside the transcript as a call seals it; None when no actor took part.""" - from .call_runner import actor_rows, approvals, trial_state_changes + from .call_runner import actor_rows, approvals, trial_shared_records, trial_state_changes listed = listing.by_id() log = _trial_log(door, since) @@ -364,7 +371,7 @@ def _world_actors_record( workers_acted = any(str(entry.get("who") or "").startswith("worker:") for entry in changes) if not acted and not decisions and not workers_acted: return None - return { + record = { "actions": acted, "decisions": decisions, "state_changes": changes, @@ -372,6 +379,11 @@ def _world_actors_record( "limits": dict(limits), "usage": meter.rows(), } + shared = trial_shared_records(log, contract) + if shared: + # Which participants named the same record and what the world made of each call. + record["shared_records"] = shared + return record def _chat_actor_summary(cast: Any, world_record: dict[str, Any] | None, listing: Any) -> tuple[dict[str, Any], ...]: @@ -1010,7 +1022,9 @@ async def run( # through the generated world or fabricate evidence from text. transcript.calls[:] = collect_seam_calls(runtime, self._context.evidence_seam, transcript.calls) - world_record = _world_actors_record(door, listing, dispatcher, cast, meter, limits, since=trial_started) + world_record = _world_actors_record( + door, listing, dispatcher, cast, meter, limits, since=trial_started, contract=self._contract + ) if world_record is not None: transcript.world_actors = world_record # Every actor that took part, the ones acting only through the world too, so the call is found by each. diff --git a/tests/harness/test_call_runner.py b/tests/harness/test_call_runner.py index 3508c381a..c287b60d9 100644 --- a/tests/harness/test_call_runner.py +++ b/tests/harness/test_call_runner.py @@ -3177,3 +3177,47 @@ def test_a_door_row_from_before_channels_is_not_a_bypass(monkeypatch) -> None: monkeypatch.setattr(cr, "read_tool_trace", lambda _address: [{**rows[0], "worker_id": None}]) (shared,) = cr._collect_tool_trace_calls(object()) assert shared.channel == "" + + +_RETURNS_TOOLS = { + "tools": [ + {"name": "list_refunds", "effects": ""}, + {"name": "issue_refund", "effects": "creates a refund"}, + ] +} + + +def _row(tool: str, worker: str = "", outcome: str = "committed", **arguments: Any) -> dict[str, Any]: + return {"name": tool, "agent_tool": tool, "worker_id": worker, "arguments": arguments, "outcome": outcome} + + +def test_shared_records_list_two_participants_on_one_order() -> None: + log = [ + _row("list_refunds", "order_lookup", order_id="1042"), + _row("list_refunds", "refund", order_id="1042"), + _row("issue_refund", "refund", order_id="1042", amount=120), + ] + (record,) = cr.trial_shared_records(log, _RETURNS_TOOLS) + assert record["key"] == "order_id=1042" + assert [(one["who"], one["tool"]) for one in record["actions"]] == [ + ("worker:order_lookup", "list_refunds"), + ("worker:refund", "list_refunds"), + ("worker:refund", "issue_refund"), + ] + assert record["committed_writes"] == 1 + + +def test_a_record_one_participant_touched_is_not_shared() -> None: + log = [_row("list_refunds", "refund", order_id="1043"), _row("issue_refund", "refund", order_id="1043")] + assert cr.trial_shared_records(log, _RETURNS_TOOLS) == [] + + +def test_write_comes_from_contract_effects_and_unknown_is_null() -> None: + log = [ + {**_row("issue_refund", "order_lookup", order_id="1"), "idempotency_key": "seed:h0"}, + _row("issue_refund", "order_lookup", order_id="1"), + _row("list_refunds", "refund", order_id="1"), + _row("unknown_tool", "refund", order_id="1"), + ] + (record,) = cr.trial_shared_records(log, _RETURNS_TOOLS) + assert [one["write"] for one in record["actions"]] == [True, None, None] From b3927d5130a91479de760ca7162609e5d7a2c1ba Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 09:18:59 +0530 Subject: [PATCH 05/19] feat(harness): a user stop seals each in-flight call as user_canceled with its partial evidence --- src/fi/alk/harness/call_runner.py | 48 +++++++++-- src/fi/alk/harness/chat_call_runner.py | 18 ++++- src/fi/alk/harness/hosted_entrypoint.py | 1 + src/fi/alk/harness/hosted_scheduler.py | 103 +++++++++++++++++++++++- 4 files changed, 161 insertions(+), 9 deletions(-) diff --git a/src/fi/alk/harness/call_runner.py b/src/fi/alk/harness/call_runner.py index f2a6abc47..25e55707a 100644 --- a/src/fi/alk/harness/call_runner.py +++ b/src/fi/alk/harness/call_runner.py @@ -34,6 +34,7 @@ import contextlib import dataclasses import gc +import inspect import json import random import time @@ -84,7 +85,7 @@ speaking_entries, logged_tool_name, ) -from .hosted_scheduler import CallAborted, CallOutcome +from .hosted_scheduler import CallAborted, CallCancelled, CallOutcome from .hosted_scheduler import Scenario as HostedScenario from .job import ExecutionMode, FailureDomain, HarnessJob, ProviderExecutionMode from .isolated_process import run_json_worker @@ -268,8 +269,23 @@ async def upload_artifact( PlaceCall = Callable[[SimulationSpec], Awaitable[SimulationReport]] -async def _default_place_call(spec: SimulationSpec) -> SimulationReport: - return await SimulationRunner().run(spec) +async def _default_place_call( + spec: SimulationSpec, stop_when: Callable[[list[dict[str, Any]], Any], bool] | None = None +) -> SimulationReport: + if stop_when is None: + return await SimulationRunner().run(spec) + # Checked by the engine between turns: a user stop hangs up at the next turn boundary with the engine's own report. + return await SimulationRunner().run(spec, stop_when=stop_when) + + +def _takes_stop_when(place_call: Any) -> bool: + try: + parameters = inspect.signature(place_call).parameters.values() + except (TypeError, ValueError): + return False + return any( + parameter.name == "stop_when" or parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters + ) @dataclass(frozen=True) @@ -290,6 +306,9 @@ class CallRunnerContext: usage_reporter: UsageReporter | None = None # A readiness probe checks how the agent's tools reach the world, not the scenario's own trial rules. readiness_probe: bool = False + # Whether the person stopped the run: a cancel of a call's task is a user stop only while this reads true. Read + # per call, never kept on the runner, which serves every concurrent call. + stop_requested: Callable[[], bool] = lambda: False # --- pre-dial validation ----------------------------------------------------------------------- @@ -2391,6 +2410,9 @@ def __init__( self._adapter = adapter self._context = context self._place_call = place_call + # A place-call that takes `stop_when` hangs up at the next turn boundary on a user stop; the scheduler gives it + # the seal bound to do so before it cuts the task. The subprocess worker path takes no callback. + self.stops_at_turn_boundary = place_call is not None and _takes_stop_when(place_call) simulator_secret_values = _canonical_simulator_secrets( context.simulator_provider_secret_values ) @@ -2817,6 +2839,10 @@ async def run( async def place() -> SimulationReport: if self._place_call is not None: + if self.stops_at_turn_boundary: + return await self._place_call( + spec, stop_when=lambda _messages, _persona: self._context.stop_requested() + ) return await self._place_call(spec) logger.info( "voice worker dispatch scenario=%s voice_run=%s", @@ -2870,7 +2896,15 @@ async def place_with_world_actors() -> SimulationReport: try: report = await asyncio.wait_for(place_with_actors(), timeout=outer_timeout) except asyncio.CancelledError: - raise + if not self._context.stop_requested(): + raise + # The person stopped the run: the in-flight audio turn is cut; the call keeps the timing it measured. + task = asyncio.current_task() + if task is not None and hasattr(task, "uncancel"): + task.uncancel() + raise CallCancelled( + partial=replace(self._timing_only_outcome(started_at), stop_reason="user_canceled") + ) from None except asyncio.TimeoutError as exc: raise CallAborted( "voice_call_runner_timeout: place_call exceeded its outer budget " @@ -2884,7 +2918,7 @@ async def place_with_world_actors() -> SimulationReport: ) from exc try: - return await self._translate_report( + outcome = await self._translate_report( report, runtime=runtime, scenario_key=scenario.scenario_key, @@ -2908,6 +2942,10 @@ async def place_with_world_actors() -> SimulationReport: f"voice_call_translate_crashed: {type(exc).__name__}: {exc}", partial=self._timing_only_outcome(started_at), ) from exc + if self.stops_at_turn_boundary and self._context.stop_requested(): + # The engine hung up at the turn boundary on the person's stop: sealed like any call, then said as a stop. + raise CallCancelled(partial=replace(outcome, stop_reason="user_canceled")) + return outcome def _fault_notices( self, runtime: EnvironmentRuntime, faults: Sequence[Any], calls: Sequence[Call] diff --git a/src/fi/alk/harness/chat_call_runner.py b/src/fi/alk/harness/chat_call_runner.py index 7f4c6ed96..a63470587 100644 --- a/src/fi/alk/harness/chat_call_runner.py +++ b/src/fi/alk/harness/chat_call_runner.py @@ -9,6 +9,7 @@ import os import re import time +from dataclasses import replace from datetime import datetime, timezone from pathlib import Path from typing import Any, Mapping, Sequence @@ -26,7 +27,7 @@ collect_seam_calls, ) from .contract import AgentContract -from .hosted_scheduler import CallAborted, CallOutcome, Scenario, World +from .hosted_scheduler import CallAborted, CallCancelled, CallOutcome, Scenario, World from .outbound import ArtifactKind, format_rfc3339_millis from .process_runtime import EnvironmentRuntime from .run.conversation import ChatActors, TargetConversationEnded, Transcript, converse @@ -961,6 +962,19 @@ async def run( except CallAborted as exc: failure = exc transcript = getattr(exc, "partial_transcript", None) or Transcript() + except asyncio.CancelledError as exc: + if not self._context.stop_requested(): + raise + # The person stopped the run: seal what the call kept through the abort path below, then raise. + failure = CallCancelled() + partial = getattr(exc, "partial_transcript", None) + setattr(failure, "partial_transcript", partial) + transcript = partial or Transcript() + task = asyncio.current_task() + if task is not None and hasattr(task, "uncancel"): + # Python 3.11+: the uploads that follow run as an uncancelled task. On 3.10 the caught cancel leaves + # nothing pending. + task.uncancel() except Exception as exc: # noqa: BLE001 - convert target transport failures to call faults status = ( exc.trace.get("status_code") @@ -1080,6 +1094,8 @@ async def run( **(dispatcher.briefs if dispatcher is not None else {}), }, ) + if failure is not None and failure.code == "user_canceled": + outcome = replace(outcome, stop_reason="user_canceled") if failure is not None: failure.partial = outcome raise failure diff --git a/src/fi/alk/harness/hosted_entrypoint.py b/src/fi/alk/harness/hosted_entrypoint.py index ad8138866..36350f4cf 100644 --- a/src/fi/alk/harness/hosted_entrypoint.py +++ b/src/fi/alk/harness/hosted_entrypoint.py @@ -2825,6 +2825,7 @@ async def _fail( attempt_number=capabilities.attempt_number, source_directory=source, usage_reporter=usage_reporter, + stop_requested=cancel_requested, ) call_runner = deps.build_call_runner(adapter, call_runner_context) scheduler = HostedScheduler( diff --git a/src/fi/alk/harness/hosted_scheduler.py b/src/fi/alk/harness/hosted_scheduler.py index 229dc6e25..7efdbb727 100644 --- a/src/fi/alk/harness/hosted_scheduler.py +++ b/src/fi/alk/harness/hosted_scheduler.py @@ -31,6 +31,7 @@ from __future__ import annotations import asyncio +import contextlib import json import inspect import logging @@ -39,6 +40,7 @@ import re import threading import tempfile +import time from concurrent.futures import ThreadPoolExecutor from dataclasses import asdict, dataclass, field, replace from pathlib import Path @@ -286,6 +288,30 @@ def __init__( self.code = code +USER_CANCELED = "user_canceled" +USER_CANCELED_MESSAGE = "Stopped by the user during the call" +CANCEL_SEAL_ENV = "ALK_CANCEL_SEAL_SECONDS" +_CANCEL_SEAL_DEFAULT_SECONDS = 15.0 +_CANCEL_POLL_SECONDS = 0.5 + + +def _cancel_seal_seconds() -> float: + """How long the scheduler waits for a cut call's runner to seal its evidence after a stop.""" + try: + value = float(os.environ.get(CANCEL_SEAL_ENV) or _CANCEL_SEAL_DEFAULT_SECONDS) + except ValueError: + return _CANCEL_SEAL_DEFAULT_SECONDS + return value if value >= 0 else _CANCEL_SEAL_DEFAULT_SECONDS + + +class CallCancelled(CallAborted): + """The person stopped the run during this call. `partial` carries what the runner sealed and uploaded before it + raised (transcript and tool-trace artifacts, calls, `stop_reason`); None when nothing was kept.""" + + def __init__(self, message: str = USER_CANCELED_MESSAGE, *, partial: CallOutcome | None = None) -> None: + super().__init__(message, partial=partial, code=USER_CANCELED) + + class CallRunner(Protocol): async def run( self, @@ -453,6 +479,8 @@ async def receipt(self, receipt: ResultReceipt) -> None: ... "target_agent_tool_failed": FailureDomain.AGENT, "simulator_stalled": FailureDomain.SIMULATOR, "driver_crashed": FailureDomain.SIMULATOR, + # The person stopped the run: never retried, never scored, never infrastructure. + "user_canceled": FailureDomain.PLATFORM_SYNC, "usage_exhausted": FailureDomain.PLATFORM_SYNC, "usage_check_failed": FailureDomain.PLATFORM_SYNC, "world_pool_exhausted": FailureDomain.INFRASTRUCTURE, @@ -1762,6 +1790,44 @@ def __init__( # Keep each scenario's batch together so concurrent worlds cannot invalidate both verdicts. self._judge_batch_lock = asyncio.Lock() self._executor: ThreadPoolExecutor | None = None + # Each scenario's in-flight call task, cancelled once by the job's one stop watcher; never runner state. + self._in_flight: dict[int, asyncio.Task[CallOutcome]] = {} + self._stopped_at: float | None = None + + async def _watch_for_stop(self) -> None: + """One watcher per job: on the first stop, cut every call in flight once, so each runner seals its own.""" + while not self._cancel_requested(): + await asyncio.sleep(_CANCEL_POLL_SECONDS) + in_flight = list(self._in_flight.values()) + if in_flight and getattr(self._call_runner, "stops_at_turn_boundary", False): + # A runner that hangs up at the next turn boundary gets the seal bound to end its calls on its own. + await asyncio.wait(in_flight, timeout=_cancel_seal_seconds()) + self._stopped_at = time.monotonic() + for task in list(self._in_flight.values()): + if not task.done(): + task.cancel() + + async def _await_call(self, task: "asyncio.Task[CallOutcome]") -> CallOutcome: + """The call's outcome; after a stop, at most the seal bound more, then `CallCancelled` with nothing kept.""" + try: + return await self._await_call_once(task) + finally: + self._in_flight.pop(id(task), None) + + async def _await_call_once(self, task: "asyncio.Task[CallOutcome]") -> CallOutcome: + while True: + done, _ = await asyncio.wait({task}, timeout=_CANCEL_POLL_SECONDS) + if done: + try: + return task.result() + except asyncio.CancelledError: + if self._stopped_at is None: + raise + # A runner that let the stop through sealed nothing. + raise CallCancelled() from None + if self._stopped_at is not None and time.monotonic() - self._stopped_at > _cancel_seal_seconds(): + logger.warning("call runner did not seal within %ss of the stop", _cancel_seal_seconds()) + raise CallCancelled() async def _judge_jointly( self, @@ -1857,8 +1923,14 @@ async def worker(index: int, scenario: Scenario) -> None: ) tasks = [asyncio.create_task(worker(i, s)) for i, s in enumerate(scenarios)] - if tasks: - await asyncio.gather(*tasks, return_exceptions=True) + watcher = asyncio.create_task(self._watch_for_stop()) + try: + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + finally: + watcher.cancel() + with contextlib.suppress(asyncio.CancelledError): + await watcher receipts: list[ResultReceipt] = [] for index, scenario in enumerate(scenarios): @@ -2292,8 +2364,33 @@ async def audit_log(**value): sub_goals=_unjudged(scenario.sub_goals), ) + if self._cancel_requested(): + # Stopped before this call started: nothing ran, nothing to seal. + return self._fault( + scenario, + world_index, + attempt, + _failure(USER_CANCELED, USER_CANCELED_MESSAGE), + sub_goals=_unjudged(scenario.sub_goals), + call=None, + retry=False, + ) + # Its own task, so a stop cuts the call and not the scenario's sealing below. + call_task = asyncio.create_task(_run_call(self._call_runner, scenario, runtime, world)) + self._in_flight[id(call_task)] = call_task try: - call_outcome = await _run_call(self._call_runner, scenario, runtime, world) + call_outcome = await self._await_call(call_task) + except CallCancelled as exc: + # Checks never run on a cancelled call: every sub-goal unmeasured, the receipt points at what was kept. + return self._fault( + scenario, + world_index, + attempt, + _failure(USER_CANCELED, USER_CANCELED_MESSAGE), + sub_goals=_unjudged(scenario.sub_goals), + call=self._call_summary(exc.partial), + retry=False, + ) except WorldUnavailable as exc: return _Retry( _failure("world_unavailable", str(exc)), From 4435edb1dd199e535a85fd50d5fcf220584c3f29 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 09:20:23 +0530 Subject: [PATCH 06/19] test(harness): cover the per-call user stop seal for chat, voice and the scheduler --- tests/harness/test_call_runner.py | 70 +++++++++++++++ tests/harness/test_chat_call_runner.py | 82 ++++++++++++++++++ tests/harness/test_hosted_scheduler.py | 114 +++++++++++++++++++++++++ 3 files changed, 266 insertions(+) diff --git a/tests/harness/test_call_runner.py b/tests/harness/test_call_runner.py index c287b60d9..abb0652e7 100644 --- a/tests/harness/test_call_runner.py +++ b/tests/harness/test_call_runner.py @@ -3221,3 +3221,73 @@ def test_write_comes_from_contract_effects_and_unknown_is_null() -> None: ] (record,) = cr.trial_shared_records(log, _RETURNS_TOOLS) assert [one["write"] for one in record["actions"]] == [True, None, None] + + +def _cut_voice_call(runner: cr.CallRunnerImpl, entered: asyncio.Event) -> Any: + async def scenario(): + task = asyncio.create_task( + runner.run(_FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + ) + await entered.wait() + task.cancel() + return await task + + return scenario + + +@pytest.mark.parametrize("stop", [True, False]) +def test_voice_cancel_with_stop_requested_raises_call_cancelled(tmp_path: Path, stop: bool) -> None: + from fi.alk.harness.hosted_scheduler import CallCancelled + + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + holder: dict[str, asyncio.Event] = {} + + async def place_call(spec): + holder["entered"].set() + await asyncio.sleep(30) + + runner = cr.CallRunnerImpl(FakeAdapter(), replace(context, stop_requested=lambda: stop), place_call=place_call) + + async def scenario(): + holder["entered"] = asyncio.Event() + return await _cut_voice_call(runner, holder["entered"])() + + if stop: + with pytest.raises(CallCancelled) as caught: + asyncio.run(scenario()) + partial = caught.value.partial + assert partial is not None and partial.started_at and partial.ended_at + assert partial.stop_reason == "user_canceled" + else: + with pytest.raises(asyncio.CancelledError): + asyncio.run(scenario()) + + +def test_voice_stop_when_ends_at_the_next_turn(tmp_path: Path) -> None: + from fi.alk.harness.hosted_scheduler import CallCancelled + + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + stopped = {"value": False} + turns: list[int] = [] + + async def place_call(spec, stop_when=None): + # A stand-in engine that checks `stop_when` after each turn and hangs up with its own report. + for turn in range(5): + turns.append(turn) + if turn == 1: + stopped["value"] = True + if stop_when is not None and stop_when([], None): + break + return _report(messages=[{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]) + + runner = cr.CallRunnerImpl( + FakeAdapter(), replace(context, stop_requested=lambda: stopped["value"]), place_call=place_call + ) + assert runner.stops_at_turn_boundary is True + with pytest.raises(CallCancelled) as caught: + asyncio.run(runner.run(_FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"}))) + assert turns == [0, 1] + assert caught.value.partial is not None + assert caught.value.partial.stop_reason == "user_canceled" diff --git a/tests/harness/test_chat_call_runner.py b/tests/harness/test_chat_call_runner.py index bcebb2792..782842c31 100644 --- a/tests/harness/test_chat_call_runner.py +++ b/tests/harness/test_chat_call_runner.py @@ -1924,3 +1924,85 @@ def test_a_chat_agent_without_a_tool_proxy_says_its_workers_permissions_were_not [notice] = chat._unproxied_worker_notices({}, tmp_path) assert notice["code"] == "worker_limits_not_enforced" assert "permitted actions" in notice["message"] + + +def _stop_runtime() -> EnvironmentRuntime: + return EnvironmentRuntime( + runtime_id="runtime", + world_index=0, + bundle_digest="sha256:" + "a" * 64, + state=RuntimeState.READY, + endpoints={ + "target_http": RuntimeEndpoint(capability="target_http", protocol="http", address="http://localhost:18080") + }, + ) + + +def _blocked_on_the_second_turn(seen: dict[str, Any]): + """Like `converse`: the second agent turn never returns, and a cancel attaches the transcript so far.""" + + async def drive(*_args, **_kwargs): + partial = Transcript( + exchanges=[ + Exchange("customer", "Refund order 1043"), + Exchange("agent", "Looking it up"), + Exchange("customer", "Is it done?"), + ] + ) + try: + seen["blocked"].set() + await asyncio.sleep(30) + except asyncio.CancelledError as exc: + exc.partial_transcript = partial + raise + return partial + + return drive + + +def _run_and_stop(context, adapter, monkeypatch): + from dataclasses import replace as replaced + + seen: dict[str, Any] = {} + stopped = {"value": False} + monkeypatch.setattr(chat, "_tool_world", lambda *_args: _ToolWorld()) + monkeypatch.setattr(chat, "_drive_conversation", _blocked_on_the_second_turn(seen)) + runner = chat.HostedChatCallRunner(adapter, replaced(context, stop_requested=lambda: stopped["value"])) + + async def scenario(stop: bool): + seen["blocked"] = asyncio.Event() + task = asyncio.create_task( + runner.run(SimpleNamespace(scenario_key="one", scenario_id="scenario-1"), _stop_runtime()) + ) + await seen["blocked"].wait() + stopped["value"] = stop + task.cancel() + return await task + + return scenario + + +def test_a_stop_during_a_blocked_agent_turn_uploads_the_partial_transcript(tmp_path, monkeypatch) -> None: + from fi.alk.harness.hosted_scheduler import CallCancelled + + adapter = _Adapter() + scenario = _run_and_stop(_context(tmp_path), adapter, monkeypatch) + with pytest.raises(CallCancelled) as caught: + asyncio.run(scenario(True)) + partial = caught.value.partial + assert caught.value.code == "user_canceled" + assert partial is not None + assert partial.transcript_artifact == "transcript-1" + assert partial.stop_reason == "user_canceled" + (transcript,) = [json.loads(data) for data in adapter.uploads] + contents = [one["content"] for one in transcript["messages"]] + # Turn 1's reply and the second customer line are kept; the second reply never arrived. + assert contents == ["Refund order 1043", "Looking it up", "Is it done?"] + + +def test_a_cancel_that_is_not_a_stop_still_propagates(tmp_path, monkeypatch) -> None: + adapter = _Adapter() + scenario = _run_and_stop(_context(tmp_path), adapter, monkeypatch) + with pytest.raises(asyncio.CancelledError): + asyncio.run(scenario(False)) + assert adapter.uploads == [] diff --git a/tests/harness/test_hosted_scheduler.py b/tests/harness/test_hosted_scheduler.py index 914d5cb97..8efcf4e46 100644 --- a/tests/harness/test_hosted_scheduler.py +++ b/tests/harness/test_hosted_scheduler.py @@ -5025,3 +5025,117 @@ def test_a_bypass_leaves_that_worker_s_checks_unmeasured_and_grades_the_task() - assert (worker.held, worker.verdict) == (None, "unmeasured") assert "reached the world without its own channel" in worker.reason assert (task.held, task.verdict, receipt.status) == (False, "fail", "failed") + + +def _stop_run(runner, scenarios, *, stop_after: float = 0.05, instances: int = 1): + """Run the scheduler with a stop that flips `stop_after` seconds in; the runner reads it through its context.""" + + async def scenario(): + outbound = FakeOutbound() + pool, _ = _pool(instances, outbound=outbound) + await pool.start() + started = time.monotonic() + stopped = lambda: time.monotonic() - started >= stop_after # noqa: E731 + runner.stop_requested = stopped + judged: list[str] = [] + + async def judge(*_args, **_kwargs): + judged.append("judge") + return True, "" + + scheduler = hs.HostedScheduler( + pool=pool, + world_factory=FakeWorldFactory(), + call_runner=runner, + outbound=outbound, + job_seed=1, + cancel_requested=stopped, + judge=judge, + ) + result = await scheduler.run(scenarios) + await pool.close() + return result, judged + + return asyncio.run(scenario()) + + +class _SealingRunner: + """A runner that waits on its call until cut, then seals what it kept as the real runners do on a user stop.""" + + def __init__(self) -> None: + self.stop_requested = lambda: False + self.cancels: dict[str, int] = {} + + async def run(self, scenario, runtime): + try: + await asyncio.sleep(30) + except asyncio.CancelledError: + self.cancels[scenario.scenario_key] = self.cancels.get(scenario.scenario_key, 0) + 1 + if not self.stop_requested(): + raise + partial = replace( + _call_outcome(turns=1, calls=(hs.Call(name="get_order", arguments={}), hs.Call(name="list_refunds", arguments={}))), + transcript_artifact=f"t-{scenario.scenario_key}", + stop_reason="user_canceled", + ) + raise hs.CallCancelled(partial=partial) from None + return _call_outcome() + + +def test_cancel_mid_call_seals_a_user_canceled_receipt_pointing_at_the_partial_transcript() -> None: + runner = _SealingRunner() + goals = [FakeSubGoal("refund_once", lambda w, c: None, judged="the reply is truthful")] + result, judged = _stop_run(runner, [FakeScenario("s1", "id-1", sub_goals=goals)]) + (receipt,) = result.receipts + assert receipt.status == "errored" + assert (receipt.failure.code, receipt.failure.domain) == ("user_canceled", "platform_sync") + assert receipt.call.transcript_artifact == "t-s1" + assert receipt.call.stop_reason == "user_canceled" + assert [one.held for one in receipt.sub_goals] == [None] + assert judged == [] + + +def test_user_canceled_is_never_retried() -> None: + assert hs._is_retryable("user_canceled") is False + assert hs._domain_for("user_canceled") is hs.FailureDomain.PLATFORM_SYNC + + +def test_a_runner_that_ignores_the_cancel_is_cut_at_the_bound(monkeypatch) -> None: + monkeypatch.setenv(hs.CANCEL_SEAL_ENV, "0.2") + + class Shielded: + stop_requested = staticmethod(lambda: False) + + async def run(self, scenario, runtime): + # Shields its work from the cancel and never returns. + await asyncio.shield(asyncio.sleep(3)) + return _call_outcome() + + result, _ = _stop_run(Shielded(), [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])]) + (receipt,) = result.receipts + assert receipt.failure.code == "user_canceled" + assert receipt.call is None + + +def test_one_stop_seals_every_call_in_flight_each_with_its_own_transcript() -> None: + runner = _SealingRunner() + scenarios = [ + FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + FakeScenario("s2", "id-2", sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + ] + result, _ = _stop_run(runner, scenarios, instances=2) + assert sorted((one.failure.code, one.call.transcript_artifact) for one in result.receipts) == [ + ("user_canceled", "t-s1"), + ("user_canceled", "t-s2"), + ] + # The one watcher cuts each call once; no stop state lives on the shared runner. + assert runner.cancels == {"s1": 1, "s2": 1} + + +def test_no_cancel_leaves_receipts_unchanged() -> None: + goals = [FakeSubGoal("g", lambda w, c: None)] + outcome = _call_outcome(turns=2, calls=(hs.Call(name="get_order", arguments={}),)) + plain = _graded(goals, outcome) + assert plain.status == "passed" + assert plain.failure is None + assert plain.call.stop_reason is None From ee3649dca00486a864f7abe572c8ac262066f99c Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 14:09:18 +0530 Subject: [PATCH 07/19] test(harness): read a call's channel apart from its calls and a world's hand-back through a spy --- tests/harness/test_call_runner.py | 5 ++++- tests/harness/test_hosted_scheduler.py | 25 +++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/harness/test_call_runner.py b/tests/harness/test_call_runner.py index abb0652e7..ee847c3ca 100644 --- a/tests/harness/test_call_runner.py +++ b/tests/harness/test_call_runner.py @@ -1880,7 +1880,10 @@ def test_tool_trace_reads_only_the_agent_s_own_actions_as_its_calls( (*agent_rows[1], "a3", "agent", None, "refused"), ] monkeypatch.setitem(sys.modules, "psycopg", _FakePsycopg(rows, columns)) - assert cr._collect_tool_trace_calls(runtime) == before + after = cr._collect_tool_trace_calls(runtime) + # The new rows also say which channel each call came in on (a null worker_id is the shared address). + assert [one.channel for one in after] == ["", ""] + assert [replace(one, channel=None) for one in after] == list(before) assert [one.name for one in before] == ["lookup_order", "cancel_order"] diff --git a/tests/harness/test_hosted_scheduler.py b/tests/harness/test_hosted_scheduler.py index 8efcf4e46..0df6a30fb 100644 --- a/tests/harness/test_hosted_scheduler.py +++ b/tests/harness/test_hosted_scheduler.py @@ -352,6 +352,25 @@ def _pool( return pool, fake +def _spy_world_outcomes(pool: hs.WorldPool) -> list[tuple[str, int]]: + """Records each `release` and `mark_unhealthy` the scheduler makes, so a test reads how a world was handed back + rather than the pool's state after the background reconcile may already have recovered it.""" + seen: list[tuple[str, int]] = [] + release, mark_unhealthy = pool.release, pool.mark_unhealthy + + async def spy_release(world_index: int, *args: Any, **kwargs: Any) -> None: + seen.append(("release", world_index)) + await release(world_index, *args, **kwargs) + + async def spy_mark_unhealthy(world_index: int, *args: Any, **kwargs: Any) -> None: + seen.append(("mark_unhealthy", world_index)) + await mark_unhealthy(world_index, *args, **kwargs) + + pool.release = spy_release # type: ignore[method-assign] + pool.mark_unhealthy = spy_mark_unhealthy # type: ignore[method-assign] + return seen + + # --- WorldPool ------------------------------------------------------------------------------- @@ -1788,6 +1807,7 @@ async def scenario() -> None: outbound = FakeOutbound() pool, _ = _pool(1, outbound=outbound) await pool.start() + handed_back = _spy_world_outcomes(pool) def boom(world: Any) -> None: raise ValueError("scenario code bug") @@ -1814,7 +1834,7 @@ def boom(world: Any) -> None: assert receipt.scenario_attempt == 1 # M13: an exception outcome discards the world (mark_unhealthy), never release() -- it is # down, not immediately available, until the background reconcile recovers it. - assert 0 in pool._down + assert handed_back == [("mark_unhealthy", 0)] await pool.close() asyncio.run(scenario()) @@ -1825,6 +1845,7 @@ async def scenario() -> None: outbound = FakeOutbound() pool, _ = _pool(1, outbound=outbound) await pool.start() + handed_back = _spy_world_outcomes(pool) def misuse(world: Any) -> None: raise WorldUsageError("cannot invent a table") @@ -1848,7 +1869,7 @@ def misuse(world: Any) -> None: receipt = result.receipts[0] assert receipt.failure.code == "world_usage" assert receipt.failure.domain == "simulator" - assert 0 in pool._down # M13: discarded, not released + assert handed_back == [("mark_unhealthy", 0)] await pool.close() asyncio.run(scenario()) From 884f3a5026177edfee5f6247cebc2dce2c217749 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 14:14:35 +0530 Subject: [PATCH 08/19] test(harness): a runner that swallows the stop is cut at the bound, and a slow call is never cut without a stop --- tests/harness/test_hosted_scheduler.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/harness/test_hosted_scheduler.py b/tests/harness/test_hosted_scheduler.py index 0df6a30fb..ba18e2862 100644 --- a/tests/harness/test_hosted_scheduler.py +++ b/tests/harness/test_hosted_scheduler.py @@ -4890,7 +4890,7 @@ def test_an_actor_who_speaks_but_only_acted_in_the_world_reached_the_agent() -> assert hs._call_actors_missing(scenario, refused) == "Jordan (not_delivered: the world refused its post_message)" -def _graded(goals, outcome): +def _graded(goals, outcome, delay: float = 0.0): async def scenario(): outbound = FakeOutbound() pool, _ = _pool(1, outbound=outbound) @@ -4898,6 +4898,8 @@ async def scenario(): class Runner: async def run(self, scenario, runtime): + if delay: + await asyncio.sleep(delay) return outcome scheduler = hs.HostedScheduler( @@ -5124,18 +5126,27 @@ def test_user_canceled_is_never_retried() -> None: def test_a_runner_that_ignores_the_cancel_is_cut_at_the_bound(monkeypatch) -> None: monkeypatch.setenv(hs.CANCEL_SEAL_ENV, "0.2") - class Shielded: + class IgnoresTheStop: stop_requested = staticmethod(lambda: False) async def run(self, scenario, runtime): - # Shields its work from the cancel and never returns. - await asyncio.shield(asyncio.sleep(3)) + # Swallows the stop's cancel and keeps working well past the bound, then returns as if nothing happened. + try: + await asyncio.sleep(3) + except asyncio.CancelledError: + pass + await asyncio.sleep(3) return _call_outcome() - result, _ = _stop_run(Shielded(), [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])]) + started = time.monotonic() + result, _ = _stop_run( + IgnoresTheStop(), [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + ) (receipt,) = result.receipts assert receipt.failure.code == "user_canceled" assert receipt.call is None + # Cut at the bound (0.2 s plus a poll), not when the runner finally returned. + assert time.monotonic() - started < 2.5 def test_one_stop_seals_every_call_in_flight_each_with_its_own_transcript() -> None: @@ -5156,7 +5167,8 @@ def test_one_stop_seals_every_call_in_flight_each_with_its_own_transcript() -> N def test_no_cancel_leaves_receipts_unchanged() -> None: goals = [FakeSubGoal("g", lambda w, c: None)] outcome = _call_outcome(turns=2, calls=(hs.Call(name="get_order", arguments={}),)) - plain = _graded(goals, outcome) + # The call outlasts the stop watcher's poll, so a watcher that cut calls without a stop would cut this one. + plain = _graded(goals, outcome, delay=1.2) assert plain.status == "passed" assert plain.failure is None assert plain.call.stop_reason is None From f3d47ae2340b24ddfcbfbb41450cd0c463eb7be9 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 15:00:42 +0530 Subject: [PATCH 09/19] fix(harness): shared records read a worker's rows as that worker, not as the agent the door also names --- src/fi/alk/harness/call_runner.py | 5 ++++- tests/harness/test_call_runner.py | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/fi/alk/harness/call_runner.py b/src/fi/alk/harness/call_runner.py index 25e55707a..fda514e33 100644 --- a/src/fi/alk/harness/call_runner.py +++ b/src/fi/alk/harness/call_runner.py @@ -1173,7 +1173,10 @@ def trial_shared_records(log: list[dict[str, Any]], contract: Any) -> list[dict[ for row in log: if is_pre_run(row.get("idempotency_key")): continue - who = str(row.get("actor_id") or "") or (f"worker:{row['worker_id']}" if row.get("worker_id") else AGENT) + # As in `trial_state_changes`: the door writes actor_id "agent" for every agent-side row, workers included. + who = str(row.get("actor_id") or AGENT) + if who == AGENT and row.get("worker_id"): + who = f"worker:{row['worker_id']}" arguments = row.get("arguments") if not isinstance(arguments, Mapping): continue diff --git a/tests/harness/test_call_runner.py b/tests/harness/test_call_runner.py index ee847c3ca..35e404900 100644 --- a/tests/harness/test_call_runner.py +++ b/tests/harness/test_call_runner.py @@ -3191,7 +3191,15 @@ def test_a_door_row_from_before_channels_is_not_a_bypass(monkeypatch) -> None: def _row(tool: str, worker: str = "", outcome: str = "committed", **arguments: Any) -> dict[str, Any]: - return {"name": tool, "agent_tool": tool, "worker_id": worker, "arguments": arguments, "outcome": outcome} + # As the world door writes it: every agent-side row has actor_id "agent", a worker's row also its worker_id. + return { + "name": tool, + "agent_tool": tool, + "actor_id": "agent", + "worker_id": worker, + "arguments": arguments, + "outcome": outcome, + } def test_shared_records_list_two_participants_on_one_order() -> None: From d3fd19b491753953a49fd73eebe55c5d0e973674 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 15:05:29 +0530 Subject: [PATCH 10/19] fix(harness): a call runner that fails while being stopped is sealed as the user's stop, not infrastructure --- src/fi/alk/harness/hosted_scheduler.py | 8 ++++++++ tests/harness/test_hosted_scheduler.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/src/fi/alk/harness/hosted_scheduler.py b/src/fi/alk/harness/hosted_scheduler.py index 7efdbb727..2833c17e7 100644 --- a/src/fi/alk/harness/hosted_scheduler.py +++ b/src/fi/alk/harness/hosted_scheduler.py @@ -1820,11 +1820,19 @@ async def _await_call_once(self, task: "asyncio.Task[CallOutcome]") -> CallOutco if done: try: return task.result() + except CallAborted: + raise except asyncio.CancelledError: if self._stopped_at is None: raise # A runner that let the stop through sealed nothing. raise CallCancelled() from None + except Exception: + if self._stopped_at is None: + raise + # A runner that failed while it was being stopped: still the person's stop, never infrastructure. + logger.warning("call runner failed while stopping", exc_info=True) + raise CallCancelled() from None if self._stopped_at is not None and time.monotonic() - self._stopped_at > _cancel_seal_seconds(): logger.warning("call runner did not seal within %ss of the stop", _cancel_seal_seconds()) raise CallCancelled() diff --git a/tests/harness/test_hosted_scheduler.py b/tests/harness/test_hosted_scheduler.py index ba18e2862..fbfce9526 100644 --- a/tests/harness/test_hosted_scheduler.py +++ b/tests/harness/test_hosted_scheduler.py @@ -5172,3 +5172,21 @@ def test_no_cancel_leaves_receipts_unchanged() -> None: assert plain.status == "passed" assert plain.failure is None assert plain.call.stop_reason is None + + +def test_a_runner_that_fails_while_being_stopped_is_sealed_as_the_stop() -> None: + class FailsOnStop: + stop_requested = staticmethod(lambda: False) + + async def run(self, scenario, runtime): + try: + await asyncio.sleep(3) + except asyncio.CancelledError: + raise RuntimeError("cleanup failed while stopping") from None + return _call_outcome() + + result, _ = _stop_run(FailsOnStop(), [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])]) + (receipt,) = result.receipts + # The person's stop, never infrastructure and never retried, whatever the runner raised on the way out. + assert (receipt.failure.code, receipt.failure.domain) == ("user_canceled", hs.FailureDomain.PLATFORM_SYNC) + assert receipt.call is None From cdc9cef104fbbb1d96ada5b2e8680f813c431870 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 15:09:16 +0530 Subject: [PATCH 11/19] fix(harness): observe a Compose-selected Python agent's framework spans for its hand-offs and worker tool calls --- src/fi/alk/harness/bundle_author_v2.py | 13 ++++++------- tests/harness/test_bundle_author_v2.py | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/fi/alk/harness/bundle_author_v2.py b/src/fi/alk/harness/bundle_author_v2.py index 2aa7efb3d..14e9889c0 100644 --- a/src/fi/alk/harness/bundle_author_v2.py +++ b/src/fi/alk/harness/bundle_author_v2.py @@ -3165,16 +3165,15 @@ def resolve_environment_plan( for process in normalized_processes ] - # A voice agent's tool calls are traced by its LiveKit worker, never twice. - if ( - not isinstance(selected, ComposeSelection) - and runtime_model.command - and str(contract_modality or "").strip().lower() != "voice" - ): + # A voice agent's tool calls are traced by its LiveKit worker, never twice. A Compose-selected agent gets the + # framework observation too (its hand-offs and per-worker tool calls), but not import-mode tool bindings. + if runtime_model.command and str(contract_modality or "").strip().lower() != "voice": trace_environment = _source_tool_trace_environment( root, runtime_model, - (tool_entrypoints or []) if contract_interface_kind == "command" else [], + (tool_entrypoints or []) + if contract_interface_kind == "command" and not isinstance(selected, ComposeSelection) + else [], ) if trace_environment: processes = [ diff --git a/tests/harness/test_bundle_author_v2.py b/tests/harness/test_bundle_author_v2.py index c54db7e6b..0204ae18d 100644 --- a/tests/harness/test_bundle_author_v2.py +++ b/tests/harness/test_bundle_author_v2.py @@ -1699,6 +1699,27 @@ def test_native_multi_store_contract_uses_source_compose_dependency_topology( assert plan.capabilities["cache_redis"].service == "cache" +def test_a_compose_selected_python_agent_is_observed_for_its_hand_offs(tmp_path: Path, monkeypatch) -> None: + """Hand-offs and each worker's tool calls are read from the agent's framework spans in the sandbox; an app shipped + with a Compose file must get that observation as a native one does.""" + plans = [] + real = resolve_environment_plan + + def keep(*args, **kwargs): + plans.append(real(*args, **kwargs)) + return plans[-1] + + monkeypatch.setitem(globals(), "resolve_environment_plan", keep) + test_native_multi_store_contract_uses_source_compose_dependency_topology(tmp_path) + (plan,) = plans + assert plan.packaging == "compose" + agent = next(process for process in plan.processes if process.name == "agent") + assert agent.environment["ALK_OBSERVE_FRAMEWORKS"] == "1" + assert agent.environment["HARNESS_TOOL_TRACE"] == "{{WORLD_DIR}}/agent-tool-calls.jsonl" + assert "ALK_TOOL_TRACE_BINDINGS" not in agent.environment + assert any("framework instrumentors" in " ".join(command) for command in agent.build_commands) + + def test_native_multi_store_contract_accepts_compose_list_environment( tmp_path: Path, ) -> None: From c32e6b88ac337091099f508e3d325ad8125716c6 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 16:03:47 +0530 Subject: [PATCH 12/19] fix(harness): a worker's door row and its framework span are one call, including a refusal the span saw as output --- src/fi/alk/harness/call_runner.py | 32 +++++++++++++++++++++++++- src/fi/alk/harness/chat_call_runner.py | 8 ++++++- tests/harness/test_call_runner.py | 30 ++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/fi/alk/harness/call_runner.py b/src/fi/alk/harness/call_runner.py index fda514e33..b1f476d33 100644 --- a/src/fi/alk/harness/call_runner.py +++ b/src/fi/alk/harness/call_runner.py @@ -993,10 +993,22 @@ def _collect_tool_trace_calls(runtime: EnvironmentRuntime) -> tuple[Call, ...]: return tuple(calls) +def contract_worker_names(bundle_dir: Any) -> dict[str, str]: + """Each contract worker's id and the name its framework gives it, from the sealed contract.""" + if bundle_dir is None: + return {} + return { + str(one["id"]): str(one.get("name") or "") + for one in _bundle_contract(Path(bundle_dir)).get("workers") or [] + if isinstance(one, Mapping) and one.get("id") and one.get("name") + } + + def collect_seam_calls( runtime: EnvironmentRuntime, seam: EvidenceSeam | None, observed: Sequence[Call] = (), + worker_names: Mapping[str, str] | None = None, ) -> tuple[Call, ...]: """The calls a world observed for one attempt: calls another source saw, else the agent's own in-process trace, else where its bundle says evidence lands, joined with the trace file's span lines. Every runner reads them here, @@ -1007,6 +1019,11 @@ def collect_seam_calls( calls = list(_collect_http_tool_calls(runtime)) elif not calls and seam is EvidenceSeam.TOOL_TRACE: calls = list(_collect_tool_trace_calls(runtime)) + for call in calls: + # The door names a worker by its contract id, its spans by the framework's name: give the call both, so the + # span and the door row of one call are read as one worker's call, not two calls. + if call.worker_id and not call.worker_label and (worker_names or {}).get(call.worker_id): + call.worker_label = (worker_names or {})[call.worker_id] # Unrecognized/None (should not happen for a `kind: process` bundle past preflight) degrades to the span lines # alone, keeping this on the scheduler's own evidence_missing path, never a raw exception. return _merge_span_calls(calls, from_spans) @@ -2087,6 +2104,17 @@ def _compatible(span: Call, call: Call, fit: Callable[[Call, Call], Any]) -> boo return strength is not None and ( strength[1] or (_kept_result(span) and _kept_result(call) and _same_result(span, call)) + or _span_kept_the_refusal(span, call) + ) + + +def _span_kept_the_refusal(span: Call, call: Call) -> bool: + """The world refused the call and the framework's span recorded the refusal's own words as the tool's output.""" + return bool( + call.refused + and call.error + and _kept_result(span) + and call.error in json.dumps(span.result, ensure_ascii=False) ) @@ -3333,4 +3361,6 @@ async def record_voice_usage( return replace(base, calls=calls) def _collect_calls(self, runtime: EnvironmentRuntime) -> tuple[Call, ...]: - return collect_seam_calls(runtime, self._context.evidence_seam) + return collect_seam_calls( + runtime, self._context.evidence_seam, worker_names=contract_worker_names(self._context.bundle_dir) + ) diff --git a/src/fi/alk/harness/chat_call_runner.py b/src/fi/alk/harness/chat_call_runner.py index a63470587..6637b9055 100644 --- a/src/fi/alk/harness/chat_call_runner.py +++ b/src/fi/alk/harness/chat_call_runner.py @@ -25,6 +25,7 @@ _clear_file_tool_calls, _with_scenario_edits, collect_seam_calls, + contract_worker_names, ) from .contract import AgentContract from .hosted_scheduler import CallAborted, CallCancelled, CallOutcome, Scenario, World @@ -1034,7 +1035,12 @@ async def run( # The submitted agent may execute tools inside its own process or against the world's services instead of # returning tool requests over HTTP. These observed calls belong to the same attempt; do not replay the tool # through the generated world or fabricate evidence from text. - transcript.calls[:] = collect_seam_calls(runtime, self._context.evidence_seam, transcript.calls) + transcript.calls[:] = collect_seam_calls( + runtime, + self._context.evidence_seam, + transcript.calls, + worker_names=contract_worker_names(self._context.bundle_dir), + ) world_record = _world_actors_record( door, listing, dispatcher, cast, meter, limits, since=trial_started, contract=self._contract diff --git a/tests/harness/test_call_runner.py b/tests/harness/test_call_runner.py index 35e404900..304c1aaf6 100644 --- a/tests/harness/test_call_runner.py +++ b/tests/harness/test_call_runner.py @@ -3302,3 +3302,33 @@ async def place_call(spec, stop_when=None): assert turns == [0, 1] assert caught.value.partial is not None assert caught.value.partial.stop_reason == "user_canceled" + + +def test_a_worker_s_door_row_and_its_span_are_one_call(monkeypatch) -> None: + """The door names the worker by its contract id, the framework's span by its name: one call, never two.""" + from fi.alk.harness.world.runtime import Call as WorldCall + + door = [ + WorldCall(name="list_refunds", arguments={"order_id": "1043"}, result={"refunds": []}, ok=True, at=10.0, + worker_id="refund", channel="refund"), + WorldCall(name="issue_refund", arguments={"order_id": "1043", "amount": 100.0}, ok=False, refused=True, + error="the deadline for refund has passed", at=11.0, worker_id="refund", + refusal_code="deadline_passed", channel="refund"), + ] + spans = [ + WorldCall(name="list_refunds", arguments={"order_id": "1043"}, result={"refunds": []}, ok=True, at=9.9, + ended_at=10.1, worker_id="Refund worker", worker_label="Refund worker", + runtime_ids={"event_id": "s1"}), + WorldCall(name="issue_refund", arguments={"order_id": "1043", "amount": 100}, ok=True, at=10.9, ended_at=11.1, + result={"detail": "the deadline for refund has passed"}, worker_id="Refund worker", + worker_label="Refund worker", runtime_ids={"event_id": "s2"}), + ] + monkeypatch.setattr(cr, "_read_file_tool_calls", lambda _runtime: ([], spans)) + monkeypatch.setattr(cr, "_collect_tool_trace_calls", lambda _runtime: tuple(door)) + calls = cr.collect_seam_calls( + _runtime(), cr.EvidenceSeam.TOOL_TRACE, worker_names={"refund": "Refund worker", "order_lookup": "Lookup"} + ) + assert [(one.name, one.worker_id, one.refusal_code) for one in calls] == [ + ("list_refunds", "refund", ""), + ("issue_refund", "refund", "deadline_passed"), + ] From fd5fc771cbeeb49f68e04691fcb11479f18d8743 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 17:58:47 +0530 Subject: [PATCH 13/19] fix(harness): a stop reaches the chat and voice call workers, which seal the partial call, and a call's start event is sent as it starts --- src/fi/alk/harness/call_runner.py | 7 +- src/fi/alk/harness/call_worker.py | 5 +- src/fi/alk/harness/chat_worker.py | 67 ++++++---- src/fi/alk/harness/hosted_entrypoint.py | 3 + src/fi/alk/harness/hosted_scheduler.py | 15 ++- src/fi/alk/harness/isolated_process.py | 56 ++++++++- tests/harness/test_call_runner.py | 40 ++++++ tests/harness/test_chat_worker.py | 117 +++++++++++++++++- tests/harness/test_hosted_entrypoint.py | 8 ++ tests/harness/test_hosted_scheduler.py | 23 ++++ tests/harness/test_isolated_process.py | 36 ++++++ tests/harness/test_parallel_call_isolation.py | 2 +- 12 files changed, 339 insertions(+), 40 deletions(-) diff --git a/src/fi/alk/harness/call_runner.py b/src/fi/alk/harness/call_runner.py index b1f476d33..fe542a2b0 100644 --- a/src/fi/alk/harness/call_runner.py +++ b/src/fi/alk/harness/call_runner.py @@ -2441,9 +2441,9 @@ def __init__( self._adapter = adapter self._context = context self._place_call = place_call - # A place-call that takes `stop_when` hangs up at the next turn boundary on a user stop; the scheduler gives it - # the seal bound to do so before it cuts the task. The subprocess worker path takes no callback. - self.stops_at_turn_boundary = place_call is not None and _takes_stop_when(place_call) + # A place-call that takes `stop_when`, and the call worker, which gets the stop as a signal, hang up at the + # next turn boundary on a user stop; the scheduler gives them the seal bound to do so before it cuts the task. + self.stops_at_turn_boundary = place_call is None or _takes_stop_when(place_call) simulator_secret_values = _canonical_simulator_secrets( context.simulator_provider_secret_values ) @@ -2885,6 +2885,7 @@ async def place() -> SimulationReport: spec.model_dump(mode="json"), environ=call_environ, work_directory=self._context.work_directory / "call-workers", + stop_requested=self._context.stop_requested, ) return SimulationReport.model_validate(result) diff --git a/src/fi/alk/harness/call_worker.py b/src/fi/alk/harness/call_worker.py index 85696a8c0..b7541d2b5 100644 --- a/src/fi/alk/harness/call_worker.py +++ b/src/fi/alk/harness/call_worker.py @@ -8,7 +8,7 @@ import sys from pathlib import Path -from .isolated_process import run_worker +from .isolated_process import run_worker, stop_requested async def main() -> None: @@ -25,7 +25,8 @@ async def main() -> None: force=True, ) logging.getLogger(__name__).info("voice worker started") - report = await _default_place_call(spec) + # On the person's stop the engine hangs up at the next turn boundary and still reports the call it kept. + report = await _default_place_call(spec, stop_when=lambda _messages, _persona: stop_requested()) output = Path(sys.argv[1]) fd = os.open(output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) with os.fdopen(fd, "w") as stream: diff --git a/src/fi/alk/harness/chat_worker.py b/src/fi/alk/harness/chat_worker.py index 9786078cc..a45113f6c 100644 --- a/src/fi/alk/harness/chat_worker.py +++ b/src/fi/alk/harness/chat_worker.py @@ -18,8 +18,8 @@ from .bundle_v2 import EvidenceSeam from .call_runner import ArtifactUploader, CallRunnerContext from .chat_call_runner import _check_text_usage -from .hosted_scheduler import CallAborted, CallOutcome, Scenario, World -from .isolated_process import run_json_worker, run_worker +from .hosted_scheduler import USER_CANCELED, CallAborted, CallCancelled, CallOutcome, Scenario, World +from .isolated_process import IsolatedWorkerError, run_json_worker, run_worker, stop_requested from .outbound import ArtifactKind from .process_runtime import EnvironmentRuntime, _allowlisted_ambient_env from .usage import simulator_funding @@ -59,6 +59,9 @@ class IsolatedChatCallRunner: + # A stop reaches the worker, which seals the call it cut; the scheduler gives it the seal bound to do so. + stops_at_turn_boundary = True + def __init__(self, adapter: ArtifactUploader, context: CallRunnerContext) -> None: self._adapter = adapter self._context = context @@ -86,27 +89,34 @@ async def run( # worker's call used is recorded here. funding = simulator_funding() await _check_text_usage(context.usage_reporter, funding) - result = await run_json_worker( - "fi.alk.harness.chat_worker", - { - "job": context.job.model_dump(mode="json"), - "bundle_dir": str(context.bundle_dir), - "work_directory": str(context.work_directory), - "source_directory": str(context.source_directory) - if context.source_directory - else None, - "target_secrets": dict(context.target_provider_secret_values), - "local_auth_values": dict(runtime._local_auth_values), - "attempt_number": context.attempt_number, - "evidence_seam": context.evidence_seam.value if context.evidence_seam else None, - "runtime": runtime.model_dump(mode="json"), - "scenario_key": scenario.scenario_key, - "source_scenario_key": getattr(scenario, "source_scenario_key", None), - "scenario_id": scenario.scenario_id, - }, - environ=environ, - work_directory=context.work_directory / "chat-workers", - ) + try: + result = await run_json_worker( + "fi.alk.harness.chat_worker", + { + "job": context.job.model_dump(mode="json"), + "bundle_dir": str(context.bundle_dir), + "work_directory": str(context.work_directory), + "source_directory": str(context.source_directory) + if context.source_directory + else None, + "target_secrets": dict(context.target_provider_secret_values), + "local_auth_values": dict(runtime._local_auth_values), + "attempt_number": context.attempt_number, + "evidence_seam": context.evidence_seam.value if context.evidence_seam else None, + "runtime": runtime.model_dump(mode="json"), + "scenario_key": scenario.scenario_key, + "source_scenario_key": getattr(scenario, "source_scenario_key", None), + "scenario_id": scenario.scenario_id, + }, + environ=environ, + work_directory=context.work_directory / "chat-workers", + stop_requested=context.stop_requested, + ) + except IsolatedWorkerError: + if not context.stop_requested(): + raise + # Stopped before the worker could seal anything: it was still starting, or failed on the way out. + raise CallCancelled() from None await _record_worker_usage(context.usage_reporter, result.get("usage") or [], funding) artifact_ids = {} for artifact in result.get("artifacts", []): @@ -127,6 +137,9 @@ async def run( raw_outcome["transcript_artifact"] ) outcome = CallOutcome(**raw_outcome) + if result.get("error_code") == USER_CANCELED: + # The worker sealed the call the person stopped: what it kept goes with the stop. + raise CallCancelled(partial=outcome) if "error" in result: raise CallAborted( f"chat_worker_failed: {result['error']}", @@ -214,6 +227,7 @@ async def main() -> None: if payload["source_directory"] else None, usage_reporter=usage, # type: ignore[arg-type] + stop_requested=stop_requested, ) runner_type = ( RetellChatCallRunner @@ -245,6 +259,11 @@ async def main() -> None: result["partial"] = asdict(exc.partial) if collector.artifacts: result["artifacts"] = collector.artifacts + except asyncio.CancelledError: + if not stop_requested(): + raise + # Stopped outside the conversation (before it began, or while sealing): nothing sealed to return. + result = {"error": USER_CANCELED, "error_code": USER_CANCELED} except Exception as exc: # noqa: BLE001 - return a secret-safe worker failure result = {"error": type(exc).__name__} if usage.records: @@ -255,4 +274,4 @@ async def main() -> None: if __name__ == "__main__": - run_worker(main) + run_worker(main, cancel_on_stop=True) diff --git a/src/fi/alk/harness/hosted_entrypoint.py b/src/fi/alk/harness/hosted_entrypoint.py index 36350f4cf..b42e6c84e 100644 --- a/src/fi/alk/harness/hosted_entrypoint.py +++ b/src/fi/alk/harness/hosted_entrypoint.py @@ -1457,6 +1457,9 @@ async def scenario_started( "scenario_attempt": scenario_attempt, }, ) + # Sent as the call starts, not with its receipt: the platform shows the call under way, and a stop's seal + # wait knows it is in flight. + await self.aflush_events() async def scenario_retried( self, *, scenario_key: str, from_world: int, to_world: int, cause: str = "" diff --git a/src/fi/alk/harness/hosted_scheduler.py b/src/fi/alk/harness/hosted_scheduler.py index 2833c17e7..52cb4856f 100644 --- a/src/fi/alk/harness/hosted_scheduler.py +++ b/src/fi/alk/harness/hosted_scheduler.py @@ -1807,6 +1807,10 @@ async def _watch_for_stop(self) -> None: if not task.done(): task.cancel() + def _stopping(self) -> bool: + # Set before the watcher cuts anything: a runner given the seal bound may fail on the stop within it. + return self._stopped_at is not None or self._cancel_requested() + async def _await_call(self, task: "asyncio.Task[CallOutcome]") -> CallOutcome: """The call's outcome; after a stop, at most the seal bound more, then `CallCancelled` with nothing kept.""" try: @@ -1820,15 +1824,22 @@ async def _await_call_once(self, task: "asyncio.Task[CallOutcome]") -> CallOutco if done: try: return task.result() - except CallAborted: + except CallCancelled: raise + except CallAborted as exc: + if not self._stopping(): + raise + # A call that failed once the person stopped it (a voice call hung up under it): their stop, with + # what it kept. + partial = exc.partial and replace(exc.partial, stop_reason=USER_CANCELED) + raise CallCancelled(partial=partial) from exc except asyncio.CancelledError: if self._stopped_at is None: raise # A runner that let the stop through sealed nothing. raise CallCancelled() from None except Exception: - if self._stopped_at is None: + if not self._stopping(): raise # A runner that failed while it was being stopped: still the person's stop, never infrastructure. logger.warning("call runner failed while stopping", exc_info=True) diff --git a/src/fi/alk/harness/isolated_process.py b/src/fi/alk/harness/isolated_process.py index b51cc0bdf..a0127cd73 100644 --- a/src/fi/alk/harness/isolated_process.py +++ b/src/fi/alk/harness/isolated_process.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import contextlib import json import logging import os @@ -15,6 +16,15 @@ MAX_RESULT_BYTES = 16 * 1024 * 1024 GRACEFUL_SHUTDOWN_SECONDS = 90.0 +# The person stopped the run: the worker seals what its call kept and writes its result, unlike SIGTERM. +STOP_SIGNAL = signal.SIGUSR1 +_STOP_POLL_SECONDS = 0.5 +_stop_requested = False + + +def stop_requested() -> bool: + """Inside a worker: whether its parent passed on the person's stop.""" + return _stop_requested class IsolatedWorkerError(RuntimeError): @@ -32,8 +42,11 @@ async def _finish(task: asyncio.Task) -> bool: return canceled -def run_worker(main: Callable[[], Awaitable[None]]) -> None: - """Translate termination into one cancellation so async cleanup can finish.""" +def run_worker(main: Callable[[], Awaitable[None]], *, cancel_on_stop: bool = False) -> None: + """Translate termination into one cancellation so async cleanup can finish. + + The stop signal sets `stop_requested()`; with `cancel_on_stop` it also cancels `main` once, for a call that seals + its partial evidence when cancelled on a stop.""" logging.basicConfig( level=logging.WARNING, @@ -54,11 +67,21 @@ def cancel_once() -> None: if not getattr(task, "cancelling", lambda: 0)(): task.cancel() + def stop() -> None: + global _stop_requested + if _stop_requested: + return + _stop_requested = True + if cancel_on_stop and not termination_requested and not task.done(): + task.cancel() + loop.add_signal_handler(signal.SIGTERM, cancel_once) + loop.add_signal_handler(STOP_SIGNAL, stop) try: await task finally: loop.remove_signal_handler(signal.SIGTERM) + loop.remove_signal_handler(STOP_SIGNAL) asyncio.run(supervised()) @@ -93,6 +116,28 @@ async def _reap( await process.wait() +async def _communicate( + process: asyncio.subprocess.Process, payload: bytes, stop_requested: Callable[[], bool] | None +) -> None: + exchange = asyncio.ensure_future(process.communicate(payload)) + try: + stopped = stop_requested is None + while not exchange.done(): + await asyncio.wait({exchange}, timeout=_STOP_POLL_SECONDS) + if not stopped and not exchange.done() and stop_requested(): + stopped = True + try: + process.send_signal(STOP_SIGNAL) + except ProcessLookupError: + pass + exchange.result() + finally: + if not exchange.done(): + exchange.cancel() + with contextlib.suppress(asyncio.CancelledError): + await exchange + + async def run_json_worker( module: str, payload: Mapping[str, Any], @@ -100,8 +145,11 @@ async def run_json_worker( environ: Mapping[str, str], work_directory: Path, suppress_output: bool = False, + stop_requested: Callable[[], bool] | None = None, ) -> dict[str, Any]: - """Run a fresh interpreter; cancellation waits for process-group termination.""" + """Run a fresh interpreter; cancellation waits for process-group termination. + + Once `stop_requested()` reads true the worker gets the stop signal once and still returns its result.""" if os.name != "posix": raise IsolatedWorkerError("isolated workers require POSIX process groups") work_directory.mkdir(parents=True, exist_ok=True) @@ -141,7 +189,7 @@ async def run_json_worker( try: if canceled: raise asyncio.CancelledError - await process.communicate(json.dumps(payload).encode()) + await _communicate(process, json.dumps(payload).encode(), stop_requested) if process.returncode != 0: logging.getLogger(__name__).error( "isolated_worker_failed module=%s exit_status=%s result_present=%s", diff --git a/tests/harness/test_call_runner.py b/tests/harness/test_call_runner.py index 304c1aaf6..0cc6bd4bb 100644 --- a/tests/harness/test_call_runner.py +++ b/tests/harness/test_call_runner.py @@ -11,12 +11,14 @@ from __future__ import annotations import asyncio +import io import json import os import sys from dataclasses import dataclass, field, replace from datetime import datetime, timedelta, timezone from pathlib import Path +from types import SimpleNamespace from typing import Any import pytest @@ -3304,6 +3306,44 @@ async def place_call(spec, stop_when=None): assert caught.value.partial.stop_reason == "user_canceled" +def test_the_voice_call_worker_gets_the_stop_and_hangs_up_at_the_next_turn(tmp_path: Path, monkeypatch) -> None: + """The hosted voice call runs in a worker process: the stop reaches it, and the engine there gets `stop_when`.""" + from fi.alk.harness import call_worker, isolated_process + + _job_obj, context = _context(tmp_path=tmp_path) + stop = lambda: True # noqa: E731 + runner = cr.CallRunnerImpl(FakeAdapter(), replace(context, stop_requested=stop)) + # The scheduler gives the worker path the seal bound, as it does an engine that takes `stop_when`. + assert runner.stops_at_turn_boundary is True + + seen: dict = {} + + async def place_call(spec, stop_when=None): + seen["stop_when"] = stop_when + return "report" + + monkeypatch.setattr(cr, "_default_place_call", place_call) + monkeypatch.setattr(isolated_process, "_stop_requested", False) + + class _Spec: + @staticmethod + def model_validate_json(_raw): + return SimpleNamespace(run_id="r1") + + import fi.simulate.runtime as runtime_module + + monkeypatch.setattr(runtime_module, "SimulationSpec", _Spec) + output = tmp_path / "report.json" + monkeypatch.setattr(call_worker.sys, "argv", ["call_worker", str(output)]) + monkeypatch.setattr(call_worker.sys, "stdin", SimpleNamespace(buffer=io.BytesIO(b"{}"))) + with pytest.raises(AttributeError): + # The stand-in report is no model; only the hand-off of `stop_when` matters here. + asyncio.run(call_worker.main()) + assert seen["stop_when"]([], None) is False + isolated_process._stop_requested = True + assert seen["stop_when"]([], None) is True + + def test_a_worker_s_door_row_and_its_span_are_one_call(monkeypatch) -> None: """The door names the worker by its contract id, the framework's span by its name: one call, never two.""" from fi.alk.harness.world.runtime import Call as WorldCall diff --git a/tests/harness/test_chat_worker.py b/tests/harness/test_chat_worker.py index f16ab01de..82097a9c9 100644 --- a/tests/harness/test_chat_worker.py +++ b/tests/harness/test_chat_worker.py @@ -60,7 +60,7 @@ def request_json(_self, _endpoint, *, headers, **_kwargs): "role": "assistant", "content": f"Account active. {token}", }}]} - async def worker(_module, payload, *, environ, work_directory): + async def worker(_module, payload, *, environ, work_directory, stop_requested=None): assert token not in json.dumps(payload["runtime"]) assert token not in json.dumps(environ) result_path = tmp_path / "worker-result.json" @@ -168,7 +168,7 @@ def test_parallel_chat_workers_keep_routing_private_and_upload_in_parent( async def run(): both = asyncio.Event() - async def worker(module, payload, *, environ, work_directory): + async def worker(module, payload, *, environ, work_directory, stop_requested=None): assert "HARNESS_PLATFORM_API_KEY" not in environ assert environ["AGENTCC_API_KEY"] == "simulator-model-key" assert environ["AGENTCC_BASE_URL"] == "https://gateway.example" @@ -455,7 +455,7 @@ async def exchange_with_tokens(*args): transcript.simulator_input_tokens, transcript.simulator_output_tokens = 11, 7 return transcript - async def worker(_module, payload, *, environ, work_directory): + async def worker(_module, payload, *, environ, work_directory, stop_requested=None): result_path = tmp_path / "worker-result.json" monkeypatch.setattr(chat_worker.sys, "stdin", io.StringIO(json.dumps(payload))) monkeypatch.setattr(chat_worker.sys, "argv", ["chat_worker", str(result_path)]) @@ -531,7 +531,7 @@ def __init__(self, _adapter, context): seen["worker"] = context.evidence_seam raise Stop - async def worker(_module, payload, *, environ, work_directory): + async def worker(_module, payload, *, environ, work_directory, stop_requested=None): seen["payload"] = payload["evidence_seam"] monkeypatch.setattr(chat_worker.sys, "stdin", io.StringIO(json.dumps(payload))) monkeypatch.setattr(chat_worker.sys, "argv", ["chat_worker", str(tmp_path / "result.json")]) @@ -554,3 +554,112 @@ async def worker(_module, payload, *, environ, work_directory): ) ) assert seen == {"payload": "tool_trace", "worker": EvidenceSeam.TOOL_TRACE} + + +def _stop_payload(context) -> dict: + runtime = EnvironmentRuntime( + runtime_id="runtime-1", world_index=0, bundle_digest="sha256:" + "a" * 64, state=RuntimeState.READY + ) + return { + "job": context.job.model_dump(mode="json"), + "bundle_dir": str(context.bundle_dir), + "work_directory": str(context.work_directory), + "source_directory": None, + "target_secrets": {}, + "attempt_number": 1, + "runtime": runtime.model_dump(mode="json"), + "scenario_key": "trial-key", + "source_scenario_key": "one", + "scenario_id": "trial-id", + } + + +def test_a_stopped_chat_worker_hands_back_the_call_it_sealed(tmp_path, monkeypatch): + """Inside the worker the person's stop reads true, so the chat runner seals its partial call as a stop.""" + from fi.alk.harness import isolated_process + from fi.alk.harness.hosted_scheduler import CallCancelled, CallOutcome + from fi.alk.harness.outbound import ArtifactKind + + context = _context(tmp_path) + result_path = tmp_path / "stopped.json" + + async def run(runner, *_args, **_kwargs): + try: + await asyncio.sleep(30) + except asyncio.CancelledError: + if not runner._context.stop_requested(): + raise + identifier = await runner._adapter.upload_artifact(b"first two turns", kind=ArtifactKind.TRANSCRIPT) + partial = CallOutcome( + calls=(), turns=2, started_at="s", ended_at="e", duration_ms=1, transcript_artifact=identifier, + stop_reason="user_canceled", + ) + raise CallCancelled(partial=partial) from None + + monkeypatch.setattr(chat_call_runner.HostedChatCallRunner, "run", run) + monkeypatch.setattr(chat_worker.sys, "stdin", io.StringIO(json.dumps(_stop_payload(context)))) + monkeypatch.setattr(chat_worker.sys, "argv", ["chat_worker", str(result_path)]) + monkeypatch.setattr(isolated_process, "_stop_requested", False) + + async def stopped_mid_call(): + task = asyncio.create_task(chat_worker.main()) + await asyncio.sleep(0.2) + isolated_process._stop_requested = True + task.cancel() + await task + + asyncio.run(stopped_mid_call()) + result = json.loads(result_path.read_text()) + assert result["error_code"] == "user_canceled" + assert result["partial"]["stop_reason"] == "user_canceled" + assert base64.b64decode(result["artifacts"][0]["data"]) == b"first two turns" + + +def test_the_parent_raises_the_stop_with_the_worker_s_sealed_call(tmp_path, monkeypatch): + from fi.alk.harness.hosted_scheduler import CallCancelled + + context = _context(tmp_path) + adapter = _Adapter() + seen = {} + + async def worker(*_args, stop_requested=None, **_kwargs): + seen["stop_requested"] = stop_requested + return { + "error": "user_canceled", + "error_code": "user_canceled", + "partial": { + "calls": [], "turns": 2, "started_at": "s", "ended_at": "e", "duration_ms": 1, + "transcript_artifact": "0", "recording_artifacts": [], "stop_reason": "user_canceled", + }, + "artifacts": [{"id": "0", "kind": "transcript", "data": base64.b64encode(b"two turns").decode()}], + } + + monkeypatch.setattr(chat_worker, "run_json_worker", worker) + runner = chat_worker.IsolatedChatCallRunner(adapter, context) + # The scheduler gives this runner the seal bound before it cuts the call. + assert runner.stops_at_turn_boundary is True + with pytest.raises(CallCancelled) as stopped: + asyncio.run(runner.run(SimpleNamespace(scenario_key="trial", scenario_id="id"), _ready_runtime())) + assert seen["stop_requested"] is context.stop_requested + assert adapter.uploads == [b"two turns"] + assert stopped.value.partial.transcript_artifact == "transcript-1" + assert stopped.value.partial.stop_reason == "user_canceled" + + +@pytest.mark.parametrize("stop", [True, False]) +def test_a_worker_lost_on_the_stop_is_the_stop(tmp_path, monkeypatch, stop): + from dataclasses import replace + + from fi.alk.harness.hosted_scheduler import CallCancelled + from fi.alk.harness.isolated_process import IsolatedWorkerError + + context = replace(_context(tmp_path), stop_requested=lambda: stop) + + async def worker(*_args, **_kwargs): + raise IsolatedWorkerError("isolated worker exited with status 1") + + monkeypatch.setattr(chat_worker, "run_json_worker", worker) + runner = chat_worker.IsolatedChatCallRunner(_Adapter(), context) + expected = CallCancelled if stop else IsolatedWorkerError + with pytest.raises(expected): + asyncio.run(runner.run(SimpleNamespace(scenario_key="trial", scenario_id="id"), _ready_runtime())) diff --git a/tests/harness/test_hosted_entrypoint.py b/tests/harness/test_hosted_entrypoint.py index 1d3170301..366b49c44 100644 --- a/tests/harness/test_hosted_entrypoint.py +++ b/tests/harness/test_hosted_entrypoint.py @@ -5231,3 +5231,11 @@ def world(path): assert selected == ["broker", "observation"] assert harness.provisioner.provision_calls == 1 assert harness.provisioner.closed + + +def test_a_scenario_start_reaches_the_platform_as_the_call_starts() -> None: + """Not held for the receipt: a call the person stops mid-way is known to be in flight while its seal is waited on.""" + transport = FakeTransport() + adapter = _build_adapter(transport) + asyncio.run(adapter.scenario_started(scenario_key="s1", world_index=0, scenario_attempt=1)) + assert [record["type"] for record in transport.event_records] == ["scenario_started"] diff --git a/tests/harness/test_hosted_scheduler.py b/tests/harness/test_hosted_scheduler.py index fbfce9526..8d719569e 100644 --- a/tests/harness/test_hosted_scheduler.py +++ b/tests/harness/test_hosted_scheduler.py @@ -5190,3 +5190,26 @@ async def run(self, scenario, runtime): # The person's stop, never infrastructure and never retried, whatever the runner raised on the way out. assert (receipt.failure.code, receipt.failure.domain) == ("user_canceled", hs.FailureDomain.PLATFORM_SYNC) assert receipt.call is None + + +def test_a_call_that_fails_under_the_stop_keeps_what_it_kept_as_the_stop() -> None: + class HungUpUnderIt: + """A runner given the seal bound whose call fails on the stop itself (the platform hung the call up).""" + + stops_at_turn_boundary = True + stop_requested = staticmethod(lambda: False) + + async def run(self, scenario, runtime): + for _ in range(60): + if self.stop_requested(): + partial = replace(_call_outcome(turns=1), transcript_artifact="t-partial") + raise hs.CallAborted("voice call ended: participant left", partial=partial) + await asyncio.sleep(0.05) + return _call_outcome() + + runner = HungUpUnderIt() + result, _ = _stop_run(runner, [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])]) + (receipt,) = result.receipts + assert (receipt.failure.code, receipt.failure.domain) == ("user_canceled", hs.FailureDomain.PLATFORM_SYNC) + assert receipt.call.transcript_artifact == "t-partial" + assert receipt.call.stop_reason == "user_canceled" diff --git a/tests/harness/test_isolated_process.py b/tests/harness/test_isolated_process.py index 0f9e87da8..47e7faf91 100644 --- a/tests/harness/test_isolated_process.py +++ b/tests/harness/test_isolated_process.py @@ -224,3 +224,39 @@ def test_a_worker_imports_the_same_code_as_its_parent_under_a_filtered_environme ) assert result["marker"] == "parent" + + +def test_a_stopped_worker_seals_and_still_returns_its_result(tmp_path): + """The stop reaches the worker as a signal it seals on (not SIGTERM's cleanup-only cancel), and the result comes back.""" + import fi.alk.harness.isolated_process as isolated + + (tmp_path / "stopping_worker.py").write_text( + "import asyncio, json, sys\n" + "from pathlib import Path\n" + "from fi.alk.harness import isolated_process\n" + "async def main():\n" + " json.load(sys.stdin)\n" + " try:\n" + " await asyncio.sleep(30)\n" + " sealed = 'finished'\n" + " except asyncio.CancelledError:\n" + " if not isolated_process.stop_requested():\n" + " raise\n" + " sealed = 'stopped'\n" + " Path(sys.argv[1]).write_text(json.dumps({'sealed': sealed}))\n" + "isolated_process.run_worker(main, cancel_on_stop=True)\n" + ) + src = str(Path(isolated.__file__).resolve().parents[3]) + environ = {**os.environ, "PYTHONPATH": os.pathsep.join([str(tmp_path), src])} + + async def run(): + started = asyncio.get_running_loop().time() + stop = lambda: asyncio.get_running_loop().time() - started > 2 # noqa: E731 + result = await run_json_worker( + "stopping_worker", {}, environ=environ, work_directory=tmp_path / "w", stop_requested=stop + ) + return result, asyncio.get_running_loop().time() - started + + result, elapsed = asyncio.run(run()) + assert result == {"sealed": "stopped"} + assert elapsed < 15 diff --git a/tests/harness/test_parallel_call_isolation.py b/tests/harness/test_parallel_call_isolation.py index 6aa02b26d..b640ca5a3 100644 --- a/tests/harness/test_parallel_call_isolation.py +++ b/tests/harness/test_parallel_call_isolation.py @@ -43,7 +43,7 @@ def test_overlapping_calls_keep_scenario_settings_private(tmp_path, monkeypatch) async def run(): both_started = asyncio.Event() - async def worker(module, payload, *, environ, work_directory): + async def worker(module, payload, *, environ, work_directory, stop_requested=None): before = dict(environ) assert "HARNESS_PLATFORM_API_KEY" not in environ snapshots.append(environ) From efaaf2bd03be59f4462983d57301a84be9655c2f Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 18:06:47 +0530 Subject: [PATCH 14/19] fix(harness): tracing keeps the harness's SIGTERM handler, so the platform's Stop seals the calls in flight instead of exiting --- src/fi/alk/harness/observability.py | 25 ++++++++++++++++++------- tests/harness/test_observability.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/fi/alk/harness/observability.py b/src/fi/alk/harness/observability.py index a8de48ed3..56b5415b5 100644 --- a/src/fi/alk/harness/observability.py +++ b/src/fi/alk/harness/observability.py @@ -10,6 +10,7 @@ import contextlib import os +import signal from typing import Any, Iterator, Mapping _OFF = {"0", "off", "false", "no"} @@ -47,13 +48,23 @@ def begin(job_id: str, run_id: str, context: Mapping[str, Any] | None = None) -> ) from fi_instrumentation.fi_types import ProjectType - _provider = register( - project_name=os.getenv("FI_HARNESS_PROJECT") or "hosted-harness", - project_type=ProjectType.OBSERVE, - # The platform binds the global provider to its own infrastructure tracing. - set_global_tracer_provider=False, - verbose=False, - ) + # `register` installs SIGTERM and SIGINT handlers that exit at once. SIGTERM is the platform's Stop, which the + # harness turns into sealing the calls in flight: keep the handlers it had. + handlers = {one: signal.getsignal(one) for one in (signal.SIGTERM, signal.SIGINT)} + try: + _provider = register( + project_name=os.getenv("FI_HARNESS_PROJECT") or "hosted-harness", + project_type=ProjectType.OBSERVE, + # The platform binds the global provider to its own infrastructure tracing. + set_global_tracer_provider=False, + verbose=False, + ) + finally: + for one, handler in handlers.items(): + # Off the main thread neither side can set a handler. + if handler is not None: + with contextlib.suppress(ValueError): + signal.signal(one, handler) _instrument(_provider) # Only FITracer writes the baggage-carried session, user, metadata and tags onto a span. _tracer = FITracer(_provider.get_tracer(__name__)) diff --git a/tests/harness/test_observability.py b/tests/harness/test_observability.py index a83021eeb..c450d8e50 100644 --- a/tests/harness/test_observability.py +++ b/tests/harness/test_observability.py @@ -2,6 +2,8 @@ from __future__ import annotations +import contextlib + import pytest from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor @@ -72,3 +74,30 @@ def test_tracing_off_is_silent_rather_than_fatal(monkeypatch): observability.stage_event("harness.stage.started", "environment") with observability.scenario("k", 0) as span: assert span is None + + +def test_tracing_never_takes_over_the_harness_s_stop_signal(monkeypatch): + """The tracing library installs its own SIGTERM handler that exits at once; the platform's Stop is a SIGTERM the + harness must keep, or a stopped call seals nothing.""" + import signal + + monkeypatch.setenv("FI_API_KEY", "k") + monkeypatch.setenv("FI_SECRET_KEY", "s") + monkeypatch.setattr(observability, "_provider", None) + monkeypatch.setattr(observability, "_tracer", None) + def ours(signum, frame): + return None + + previous = {one: signal.getsignal(one) for one in (signal.SIGTERM, signal.SIGINT)} + signal.signal(signal.SIGTERM, ours) + try: + observability.begin("job-1", "run-1") + assert observability._provider is not None + assert signal.getsignal(signal.SIGTERM) is ours + assert signal.getsignal(signal.SIGINT) is previous[signal.SIGINT] + finally: + for one, handler in previous.items(): + signal.signal(one, handler) + with contextlib.suppress(Exception): + observability._provider.shutdown() + observability._session.close() From a69c6966a9326ef8073e12b3bb00091c49ea6cf7 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 18:15:31 +0530 Subject: [PATCH 15/19] fix(harness): a stopped call worker exits once its sealed result is written, not after a thread still blocked on the agent --- src/fi/alk/harness/isolated_process.py | 11 ++++++++ tests/harness/test_isolated_process.py | 35 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/fi/alk/harness/isolated_process.py b/src/fi/alk/harness/isolated_process.py index a0127cd73..b20ea6be0 100644 --- a/src/fi/alk/harness/isolated_process.py +++ b/src/fi/alk/harness/isolated_process.py @@ -72,16 +72,26 @@ def stop() -> None: if _stop_requested: return _stop_requested = True + logging.getLogger(__name__).info("isolated_worker_stop_received") if cancel_on_stop and not termination_requested and not task.done(): task.cancel() loop.add_signal_handler(signal.SIGTERM, cancel_once) loop.add_signal_handler(STOP_SIGNAL, stop) + finished = False try: await task + finished = True finally: loop.remove_signal_handler(signal.SIGTERM) loop.remove_signal_handler(STOP_SIGNAL) + if _stop_requested: + # A stopped call can leave a thread blocked on the agent, and the loop's shutdown would wait for it + # while the parent waits for this process: exit now with what was written; the parent reaps the rest. + logging.shutdown() + sys.stdout.flush() + sys.stderr.flush() + os._exit(0 if finished else 1) asyncio.run(supervised()) @@ -126,6 +136,7 @@ async def _communicate( await asyncio.wait({exchange}, timeout=_STOP_POLL_SECONDS) if not stopped and not exchange.done() and stop_requested(): stopped = True + logging.getLogger(__name__).info("isolated_worker_stop_sent pid=%s", process.pid) try: process.send_signal(STOP_SIGNAL) except ProcessLookupError: diff --git a/tests/harness/test_isolated_process.py b/tests/harness/test_isolated_process.py index 47e7faf91..f558fd417 100644 --- a/tests/harness/test_isolated_process.py +++ b/tests/harness/test_isolated_process.py @@ -260,3 +260,38 @@ async def run(): result, elapsed = asyncio.run(run()) assert result == {"sealed": "stopped"} assert elapsed < 15 + + +def test_a_stopped_worker_exits_without_waiting_on_a_call_stuck_in_a_thread(tmp_path): + """A stopped call can leave a thread blocked on the agent (a tool waiting on its upstream); the worker hands back + its sealed result and exits instead of waiting for that thread.""" + import fi.alk.harness.isolated_process as isolated + + (tmp_path / "stuck_worker.py").write_text( + "import asyncio, json, sys, time\n" + "from pathlib import Path\n" + "from fi.alk.harness import isolated_process\n" + "async def main():\n" + " json.load(sys.stdin)\n" + " try:\n" + " await asyncio.to_thread(time.sleep, 60)\n" + " except asyncio.CancelledError:\n" + " if not isolated_process.stop_requested():\n" + " raise\n" + " Path(sys.argv[1]).write_text(json.dumps({'sealed': 'stopped'}))\n" + "isolated_process.run_worker(main, cancel_on_stop=True)\n" + ) + src = str(Path(isolated.__file__).resolve().parents[3]) + environ = {**os.environ, "PYTHONPATH": os.pathsep.join([str(tmp_path), src])} + + async def run(): + started = asyncio.get_running_loop().time() + stop = lambda: asyncio.get_running_loop().time() - started > 2 # noqa: E731 + result = await run_json_worker( + "stuck_worker", {}, environ=environ, work_directory=tmp_path / "w", stop_requested=stop + ) + return result, asyncio.get_running_loop().time() - started + + result, elapsed = asyncio.run(run()) + assert result == {"sealed": "stopped"} + assert elapsed < 20 From 373d673aa141928dc553152e209951761079021e Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 18:19:47 +0530 Subject: [PATCH 16/19] fix(harness): a native contract whose tools call an HTTP service the source ships as tools-api runs it behind the world's door --- src/fi/alk/harness/bundle_author_v2.py | 36 +++++++++++++++- tests/harness/test_bundle_author_v2.py | 60 ++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/harness/bundle_author_v2.py b/src/fi/alk/harness/bundle_author_v2.py index 14e9889c0..052f7ec5b 100644 --- a/src/fi/alk/harness/bundle_author_v2.py +++ b/src/fi/alk/harness/bundle_author_v2.py @@ -2320,6 +2320,25 @@ def _compose_environment(service: dict[str, Any]) -> dict[str, str]: return {} +def _compose_service_running(root: Path, command: list[str]) -> dict[str, str]: + """The source's one Compose file and its one source-built service that runs exactly `command`, else nothing; a + file that publishes no ports names its agent this way only.""" + found = [path for name in ("compose.yaml", "compose.yml", "docker-compose.yaml", "docker-compose.yml") + if (path := root / name).is_file()] + if len(found) != 1 or not command: + return {} + try: + services = (yaml.safe_load(found[0].read_text(encoding="utf-8")) or {}).get("services") or {} + matches = [ + name + for name, service in services.items() + if isinstance(service, dict) and service.get("build") and _compose_command(service.get("command")) == command + ] + except (yaml.YAMLError, AttributeError, BundleAuthorError, OSError): + return {} + return {"compose_file": found[0].name, "service": matches[0]} if len(matches) == 1 else {} + + def resolve_environment_plan( source: str | Path, job: HarnessJob, @@ -2374,11 +2393,19 @@ def resolve_environment_plan( contract_runtime or {}, public_environment=declared_runtime_environment, ) + # A tool service the agent calls over HTTP, which the source's own Compose file may ship as `tools-api`. + calls_tool_service = any( + dependency.get("kind", "").lower() == "service" + and dependency.get("engine", "").lower() == "http" + and dependency.get("used_by") + for dependency in (runtime_dependencies or []) + ) if ( not isinstance(selected, ComposeSelection) and runtime_model.execution == "native" and ( runtime_model.required_environment + or calls_tool_service or any( dependency.get("kind", "").lower() in {"datastore", "database", "file", "queue"} @@ -2392,6 +2419,7 @@ def resolve_environment_plan( "execution": "auto", "compose_file": "", "service": "", + **(_compose_service_running(root, runtime_model.command) if calls_tool_service else {}), } try: topology = select_runtime( @@ -2401,7 +2429,13 @@ def resolve_environment_plan( ) except ValueError: topology = None - if isinstance(topology, ComposeSelection): + # Only a tool service shipped as `tools-api` puts the world's door in front of the tools; for any other + # name the native choice stands, as before. + if isinstance(topology, ComposeSelection) and ( + runtime_model.required_environment + or "tools-api" in topology.services + or not calls_tool_service + ): required = runtime_model.required_environment requirements_are_wired = all( all( diff --git a/tests/harness/test_bundle_author_v2.py b/tests/harness/test_bundle_author_v2.py index 0204ae18d..9a79cffd9 100644 --- a/tests/harness/test_bundle_author_v2.py +++ b/tests/harness/test_bundle_author_v2.py @@ -1720,6 +1720,66 @@ def keep(*args, **kwargs): assert any("framework instrumentors" in " ".join(command) for command in agent.build_commands) +def test_a_native_contract_whose_tools_call_the_shipped_tools_api_runs_it_behind_the_world_s_door( + tmp_path: Path, +) -> None: + """The same returns app was authored once as Compose (door, worker channels) and once as native: its Compose file + ships the tools-api its tools call, so both must run that service behind the world's door.""" + source = Path(__file__).resolve().parents[1] / "fixtures" / "harness_agents" / "returns_multi_agent" + runtime = { + "language": "python", "version": "3.11", "execution": "native", "command": ["python", "agent.py"], + "interface": { + "kind": "http", "protocol": "openai_chat", "port": 8080, + "path": "/v1/chat/completions", "health_path": "/health", + }, + } + dependencies = [ + {"name": "returns_ledger", "kind": "service", "engine": "http", + "reached": {"dsn_env": "RETURNS_API_URL", "host": "127.0.0.1", "port": 8090}, + "used_by": ["get_order", "list_refunds", "update_order_status", "issue_refund"]}, + ] + + # As that build wrote them: the tools as the agent module's own functions. + entrypoints = [ + {"tool": name, "mode": "import", "module": "agent", "callable": name, "endpoint": endpoint, "method": "POST"} + for name, endpoint in (("get_order", "orders/get"), ("list_refunds", "refunds/list"), + ("update_order_status", "orders/status"), ("issue_refund", "refunds/issue")) + ] + + plan = resolve_environment_plan( + source, _job(connector="auto"), contract_modality="chat", + contract_interface_kind="http", contract_runtime=runtime, + runtime_dependencies=dependencies, tool_entrypoints=entrypoints, + ) + + assert plan.packaging == "compose" + assert {process.name for process in plan.processes} >= {"agent", "tools-api", "tool-proxy"} + assert plan.capabilities["tool_proxy"].service == "tool-proxy" + + +def test_a_native_contract_with_no_tools_api_in_its_compose_stays_native(tmp_path: Path) -> None: + import shutil + + source = tmp_path / "returns" + shutil.copytree(Path(__file__).resolve().parents[1] / "fixtures" / "harness_agents" / "returns_multi_agent", source) + (source / "docker-compose.yml").write_text( + (source / "docker-compose.yml").read_text().replace("tools-api", "ledger") + ) + runtime = { + "language": "python", "execution": "native", "command": ["python", "agent.py"], + "interface": {"kind": "http", "protocol": "openai_chat", "port": 8080, + "path": "/v1/chat/completions", "health_path": "/health"}, + } + dependencies = [{"name": "returns_ledger", "kind": "service", "engine": "http", + "reached": {"dsn_env": "RETURNS_API_URL"}, "used_by": ["get_order"]}] + plan = resolve_environment_plan( + source, _job(connector="auto"), contract_modality="chat", + contract_interface_kind="http", contract_runtime=runtime, runtime_dependencies=dependencies, + ) + assert plan.packaging != "compose" + assert "tool-proxy" not in {process.name for process in plan.processes} + + def test_native_multi_store_contract_accepts_compose_list_environment( tmp_path: Path, ) -> None: From 429f2b55250bba701f97c02b6226e12e6bb392ed Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 18:23:47 +0530 Subject: [PATCH 17/19] fix(harness): a stopped chat call seals without waiting out the agent reply it was blocked on --- src/fi/alk/harness/chat_call_runner.py | 3 ++ src/fi/simulate/agent/wrappers/http.py | 7 ++++- tests/harness/test_chat_call_runner.py | 43 ++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/fi/alk/harness/chat_call_runner.py b/src/fi/alk/harness/chat_call_runner.py index 6637b9055..a45133bf1 100644 --- a/src/fi/alk/harness/chat_call_runner.py +++ b/src/fi/alk/harness/chat_call_runner.py @@ -920,6 +920,9 @@ async def run( "scenario": scenario.scenario_key, }, ) + if hasattr(wrapper, "abandon_on_cancel"): + # A stopped run's world is never reused: its call seals what it kept without waiting out the agent's reply. + wrapper.abandon_on_cancel = self._context.stop_requested funding = simulator_funding() await _check_text_usage(self._context.usage_reporter, funding) started = datetime.now(timezone.utc) diff --git a/src/fi/simulate/agent/wrappers/http.py b/src/fi/simulate/agent/wrappers/http.py index c69a27343..7b7ee41a9 100644 --- a/src/fi/simulate/agent/wrappers/http.py +++ b/src/fi/simulate/agent/wrappers/http.py @@ -9,7 +9,7 @@ import time import urllib.error import urllib.request -from typing import Any, Mapping, Optional, Sequence +from typing import Any, Callable, Mapping, Optional, Sequence from urllib.parse import quote, urlparse, urlsplit, urlunsplit, urlencode from .http_events import CompletedToolEvents, CompletedToolRows @@ -186,6 +186,9 @@ def __init__( self.failure_rule = dict(failure_rule) if failure_rule else None self.secret_values = dict(secret_values) if secret_values is not None else None self._private_response_values = dict(private_response_values or {}) + # Whether a cancelled request may be left running instead of drained: true once nothing reuses this world, + # as after the person stopped the run, where draining would wait out the agent's own reply. + self.abandon_on_cancel: Callable[[], bool] = lambda: False self.completed_tool_events = ( CompletedToolEvents.model_validate(completed_tool_events) if completed_tool_events is not None @@ -446,6 +449,8 @@ async def _run_request(self, function, *args, **kwargs): try: return await asyncio.shield(task) except asyncio.CancelledError: + if self.abandon_on_cancel(): + raise # A cancelled thread can still mutate the target. Drain before world reset. while not task.done(): try: diff --git a/tests/harness/test_chat_call_runner.py b/tests/harness/test_chat_call_runner.py index 782842c31..5e47db0c5 100644 --- a/tests/harness/test_chat_call_runner.py +++ b/tests/harness/test_chat_call_runner.py @@ -2006,3 +2006,46 @@ def test_a_cancel_that_is_not_a_stop_still_propagates(tmp_path, monkeypatch) -> with pytest.raises(asyncio.CancelledError): asyncio.run(scenario(False)) assert adapter.uploads == [] + + +def test_a_stopped_call_abandons_the_agent_reply_it_was_waiting_on(tmp_path, monkeypatch) -> None: + """The agent's in-flight reply is drained on a cancel so a reused world is not changed under the next trial; a + stopped run's world is never reused, so its call must not wait out that reply (a worker blocked on a slow + upstream held the seal past its bound).""" + import time + + from fi.simulate.agent.wrappers.http import HTTPAgentWrapper + + seen: dict[str, Any] = {} + stopped = {"value": False} + + async def drive(target, *_args, **_kwargs): + wrapper = target._wrapper + seen["blocked"].set() + try: + await wrapper._run_request(time.sleep, 5) + except asyncio.CancelledError as exc: + exc.partial_transcript = Transcript(exchanges=[Exchange("customer", "Refund order 1043")]) + raise + + from dataclasses import replace as replaced + + monkeypatch.setattr(chat, "_tool_world", lambda *_args: _ToolWorld()) + monkeypatch.setattr(chat, "_drive_conversation", drive) + runner = chat.HostedChatCallRunner(_Adapter(), replaced(_context(tmp_path), stop_requested=lambda: stopped["value"])) + + async def scenario(): + seen["blocked"] = asyncio.Event() + task = asyncio.create_task( + runner.run(SimpleNamespace(scenario_key="one", scenario_id="scenario-1"), _stop_runtime()) + ) + await seen["blocked"].wait() + stopped["value"] = True + started = time.monotonic() + task.cancel() + with pytest.raises(chat.CallCancelled): + await task + return time.monotonic() - started + + assert asyncio.run(scenario()) < 2 + assert HTTPAgentWrapper(endpoint="http://x").abandon_on_cancel() is False From 76ede3c451d3fe1aaa374226bd909e4a1fd0fa30 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 20:13:09 +0530 Subject: [PATCH 18/19] fix(harness): a langgraph dev runtime is served through its threads and runs API, not a model-written chat route --- src/fi/alk/harness/bundle_author_v2.py | 122 ++++++++++++++++++++++--- tests/harness/test_bundle_author_v2.py | 107 ++++++++++++++++++++++ 2 files changed, 214 insertions(+), 15 deletions(-) diff --git a/src/fi/alk/harness/bundle_author_v2.py b/src/fi/alk/harness/bundle_author_v2.py index 052f7ec5b..c6c28119f 100644 --- a/src/fi/alk/harness/bundle_author_v2.py +++ b/src/fi/alk/harness/bundle_author_v2.py @@ -1983,6 +1983,97 @@ def _adk_interface_for(command: list[str], app_name: str) -> dict[str, Any]: } +def _is_langgraph_server_command(command: list[str]) -> bool: + """Return whether argv starts the LangGraph API server (``langgraph dev`` or ``up``).""" + + normalized = [str(item).strip() for item in command] + return any( + normalized[index] == "langgraph" + and normalized[index + 1] in {"dev", "up"} + for index in range(max(0, len(normalized) - 1)) + ) + + +def _langgraph_server_interface( + root: Path, runtime: dict[str, Any] +) -> dict[str, Any] | None: + """Compile the LangGraph server's threads and runs API into the generic JSON HTTP interface. + + ``langgraph dev`` serves the graphs its ``langgraph.json`` declares through threads and runs, + never an OpenAI chat route. The assistant is the declared graph the written interface names + as its model, else the only graph, else the first one declared (the server's default). + """ + + command = [str(item) for item in (runtime.get("command") or [])] + if not _is_langgraph_server_command(command): + return None + configured = str(runtime.get("workdir") or "").strip() + workdir = (root / configured) if configured not in {"", ".", "/"} else root + config_name = "langgraph.json" + for index, token in enumerate(command): + if token == "--config" and index + 1 < len(command): + config_name = command[index + 1] + elif token.startswith("--config="): + config_name = token.partition("=")[2] + config = (workdir / config_name).resolve() + if not config.is_relative_to(root.resolve()) or not config.is_file(): + return None + try: + graphs = json.loads(config.read_text(encoding="utf-8")).get("graphs") + except (OSError, ValueError, AttributeError): + return None + if not isinstance(graphs, dict) or not graphs: + return None + interface = runtime.get("interface") + named = interface.get("model") if isinstance(interface, dict) else None + assistant = named if named in graphs else next(iter(graphs)) + return { + "kind": "http", + "protocol": "json_template", + "port": _command_port(command, default=2024), + "path": "/threads/{{langgraph_thread_id}}/runs/wait", + "health_path": "/ok", + "include_tools": False, + "setup_requests": [ + { + "method": "POST", + "path": "/threads", + "body_template": {}, + "accepted_statuses": [200], + "capture": {"langgraph_thread_id": "thread_id"}, + } + ], + "request_template": { + "assistant_id": assistant, + "input": { + "messages": [{"role": "user", "content": "{{new_message_content}}"}] + }, + }, + "response_path": "messages.-1.content", + } + + +def _native_server_runtime(root: Path, runtime: dict[str, Any]) -> dict[str, Any] | None: + """The runtime with the documented API of a source-owned framework server sealed as its + interface (``adk web``, ``langgraph dev``), or None when the command starts neither. + + A LangGraph command without ``--port`` gets the server's default port written out, so each + world's port can replace it. + """ + + interface = _adk_web_interface(root, runtime) or _langgraph_server_interface( + root, runtime + ) + if interface is None: + return None + command = [str(item) for item in (runtime.get("command") or [])] + if _is_langgraph_server_command(command) and not any( + token == "--port" or token.startswith("--port=") for token in command + ): + command = [*command, "--port", str(interface["port"])] + return {**runtime, "command": command, "interface": interface} + + def _submitted_command(process: SourceProcess, command: list[str]) -> list[str]: """Run a contract argv inside the dependency environment selected for the source.""" @@ -2977,8 +3068,11 @@ def resolve_environment_plan( "started_check": StartedCheck(port=True, timeout_seconds=180), } ) - if is_http_runtime and _is_adk_web_command(runtime_command): - # ADK exposes a normal ``--port`` seam. Wire it to the world's allocated port so a + if is_http_runtime and ( + _is_adk_web_command(runtime_command) + or _is_langgraph_server_command(runtime_command) + ): + # ADK and LangGraph expose a normal ``--port`` seam. Wire it to the world's allocated port so a # multi-world chat run does not collapse to serial execution or collide on 8080. process = _consumable_source_process( process, "PORT", f"{{{{PORT_{control_name}}}}}" @@ -3621,16 +3715,15 @@ def author_bundle_v2( interface = runtime.get("interface") if isinstance(runtime, dict) else None if isinstance(interface, dict): contract_interface_kind = str(interface.get("kind") or "").strip().lower() - adk_interface = _adk_web_interface(source_root, runtime) + native_runtime = _native_server_runtime(source_root, runtime) if ( contract_modality == "chat" - and adk_interface is not None + and native_runtime is not None and contract_interface_kind in {"", "http"} ): - # ADK's source-owned web API is deterministic; do not retain stale model-authored - # request shapes from older ADK releases. - runtime = dict(runtime) - runtime["interface"] = adk_interface + # A source-owned framework server's API is deterministic; do not retain stale + # model-authored request shapes (an OpenAI chat route for ``langgraph dev``). + runtime = native_runtime contract_body = {**contract_body, "runtime": runtime} contract_interface_kind = "http" elif ( @@ -3672,13 +3765,12 @@ def author_bundle_v2( and not isinstance(interface, dict) and runtime.get("command") ): - runtime = dict(runtime) - adk_interface = _adk_web_interface(source_root, runtime) - if adk_interface is not None: - # ``adk web`` is a persistent source-owned HTTP server, not a one-shot command. - # Launch it directly and describe its existing API rather than nesting it inside - # the subprocess bridge (which would make both processes bind the same port). - runtime["interface"] = adk_interface + native_runtime = _native_server_runtime(source_root, runtime) + if native_runtime is not None: + # ``adk web`` and ``langgraph dev`` are persistent source-owned HTTP servers, not + # one-shot commands. Launch them directly and describe their existing API rather + # than nesting them inside the subprocess bridge (both would bind the same port). + runtime = native_runtime contract_interface_kind = "http" else: raise BundleAuthorError( diff --git a/tests/harness/test_bundle_author_v2.py b/tests/harness/test_bundle_author_v2.py index 9a79cffd9..3b862124a 100644 --- a/tests/harness/test_bundle_author_v2.py +++ b/tests/harness/test_bundle_author_v2.py @@ -28,6 +28,7 @@ resolve_environment_plan, sealed_from, ) +from fi.alk.harness import bundle_author_v2 from fi.alk.harness.generated_runtime import framework_instrumentation_command from fi.alk.harness.authoring_runtime_validation import _artifact_digest from fi.alk.harness.bundle_v2 import load_bundle_v2 @@ -4209,6 +4210,112 @@ def test_adk_web_command_without_interface_uses_source_http_api(tmp_path: Path) ) +def _langgraph_source(tmp_path: Path) -> Path: + """The async deep agents layout: a root langgraph.json declaring three graphs in a subproject.""" + source = tmp_path / "source" + graphs = source / "graphs" / "python" / "src" + graphs.mkdir(parents=True) + (source / "graphs" / "python" / "pyproject.toml").write_text( + "[project]\nname='async-subagents'\nversion='1'\n", encoding="utf-8" + ) + for name in ("agent", "researcher", "coder"): + (graphs / f"{name}.py").write_text("graph = object()\n", encoding="utf-8") + (source / "langgraph.json").write_text( + json.dumps( + { + "dependencies": ["./graphs/python"], + "graphs": { + name: f"./graphs/python/src/{name}.py:graph" + for name in ("agent", "researcher", "coder") + }, + } + ), + encoding="utf-8", + ) + return source + + +@pytest.mark.parametrize("model", ["agent", "", "gpt-4o"]) +def test_langgraph_dev_is_served_through_its_threads_and_runs_api( + tmp_path: Path, model: str +) -> None: + """Build 5f83aede sealed the model-written OpenAI chat route for ``langgraph dev`` (404).""" + source = _langgraph_source(tmp_path) + authoring = _authoring(tmp_path) + command = [ + "uv", "run", "--project", "graphs/python", "--with", "langgraph-cli[inmem]", + "langgraph", "dev", "--config", "langgraph.json", "--no-browser", + ] # fmt: skip + _write_chat_contract( + authoring, + command=command, + workdir="", + interface={ + "kind": "http", + "path": "/chat/completions", + "port": 2024, + "model": model, + "protocol": "openai_chat", + "health_path": "/ok", + }, + ) + with sqlite3.connect(authoring / "world.sqlite") as database: + database.execute("CREATE TABLE fixture (id TEXT PRIMARY KEY)") + + output = tmp_path / "bundle" + manifest = author_bundle_v2( + source=source, + job=_job(connector="auto", metadata={"generic_harness_v1": True}), + authoring=authoring, + output=output, + ) + + runtime = json.loads((output / "contract.json").read_text())["runtime"] + interface = runtime["interface"] + assert interface["protocol"] == "json_template" + assert interface["path"] == "/threads/{{langgraph_thread_id}}/runs/wait" + assert interface["setup_requests"][0]["path"] == "/threads" + assert interface["setup_requests"][0]["capture"] == { + "langgraph_thread_id": "thread_id" + } + # The supervisor graph: named by the written model, else the first one declared. + assert interface["request_template"]["assistant_id"] == "agent" + assert interface["response_path"] == "messages.-1.content" + assert runtime["command"][-2:] == ["--port", "2024"] + agent = next(process for process in manifest.processes if process.name == "agent") + assert agent.fixed_port == 2024 + assert agent.fixed_port_consumable is True + assert "langgraph dev" in agent.run_command[-1] + assert "$PORT" in agent.run_command[-1] + assert "ALK_SUBPROCESS_COMMAND" not in agent.environment + readiness = next( + item for item in manifest.readiness if item.capability == "target_http" + ) + assert readiness.path == "/ok" + + +def test_langgraph_dev_serves_the_graph_the_written_interface_names( + tmp_path: Path, +) -> None: + source = _langgraph_source(tmp_path) + interface = bundle_author_v2._langgraph_server_interface( + source, + { + "command": ["langgraph", "dev", "--port=9000"], + "interface": {"model": "coder"}, + }, + ) + assert interface is not None + assert interface["request_template"]["assistant_id"] == "coder" + assert interface["port"] == 9000 + assert ( + bundle_author_v2._langgraph_server_interface( + source, {"command": ["python", "server.py"]} + ) + is None + ) + + def test_adk_source_toolbox_is_hosted_against_world_database(tmp_path: Path) -> None: source = tmp_path / "source" app = source / "bug_agent" From 959c1bd77586cf93bab14ee1d5e86c284041b0d5 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 20:51:31 +0530 Subject: [PATCH 19/19] fix(harness): framework instrumentors land in the uv project of a command rewritten to sh -c for the world's port --- src/fi/alk/harness/generated_runtime.py | 6 ++++++ tests/harness/test_bundle_author_v2.py | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/src/fi/alk/harness/generated_runtime.py b/src/fi/alk/harness/generated_runtime.py index 74b2e2924..9b78a6ece 100644 --- a/src/fi/alk/harness/generated_runtime.py +++ b/src/fi/alk/harness/generated_runtime.py @@ -694,6 +694,12 @@ def declared(body: dict[str, Any]) -> set[str]: def uv_run_project(command: Sequence[str] | None) -> str: """The project a `uv run --project ` command runs in, or "" for any other command.""" parts = list(command or []) + if parts[:2] == ["sh", "-c"] and len(parts) >= 3: + # A fixed --port rewritten to the world's port runs as ``sh -c " --port $PORT"``. + try: + parts = shlex.split(parts[2]) + except ValueError: + return "" if parts[:2] != ["uv", "run"]: return "" for index, part in enumerate(parts): diff --git a/tests/harness/test_bundle_author_v2.py b/tests/harness/test_bundle_author_v2.py index 3b862124a..e47335769 100644 --- a/tests/harness/test_bundle_author_v2.py +++ b/tests/harness/test_bundle_author_v2.py @@ -4292,6 +4292,13 @@ def test_langgraph_dev_is_served_through_its_threads_and_runs_api( item for item in manifest.readiness if item.capability == "target_http" ) assert readiness.path == "/ok" + # The framework instrumentors land in the project ``uv run`` starts, though the command + # now runs under ``sh -c`` for the world's port (hosted run 7ccc3710 had no hand-offs). + assert "ALK_OBSERVE_FRAMEWORKS" in agent.environment + assert any( + "uv sync --project graphs/python" in " ".join(command) + for command in agent.build_commands + ) def test_langgraph_dev_serves_the_graph_the_written_interface_names(