diff --git a/CHANGELOG.md b/CHANGELOG.md index 107e723d3..b9d63fcf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,9 @@ Post-rc.1 increments on the release branch. Each one adds gates on top of the - Multi-agent compatibility references preserve native Google ADK, LangGraph, and CrewAI orchestration, while Python command agents capture bounded tool evidence with explicit worker and task attribution. +- Multi-agent apps: the contract records the workers an app declares, and every Python agent's + tool calls are read from its framework spans with the worker that made each, every hand-off and + when that worker returned; a worker's run is never graded as a tool call. - Builder chat: `edit_scenario` changes a kept scenario by its key with only the fields that change (`clear` resets a field), keeping its key, number, place and history; a rename alone is not proved again. A new scenario never takes a key the environment has used, a dropped scenario's diff --git a/examples/harness/returns_reference/agent.py b/examples/harness/returns_reference/agent.py new file mode 100644 index 000000000..865d45d94 --- /dev/null +++ b/examples/harness/returns_reference/agent.py @@ -0,0 +1,224 @@ +"""Returns desk: a coordinator that hands work to two workers, on the OpenAI Agents SDK. + +The coordinator never touches orders or refunds itself. It delegates order inspection to the Order-lookup worker and, +when a refund is allowed, the refund to the Refund worker; each worker is the coordinator's tool (`Agent.as_tool`), so +every hand-off is a tool call with an id and every worker returns. Each worker reaches the returns ledger (ledger.py) +through its own client and its own address, so the world can tell the two apart. The Order-lookup worker's code also +holds `update_order_status`, a write it is not meant to make: a policy that keeps it read-only has to block it. + +How it is reached (the whole interface): + GET /health -> 200 {"status": "ok"} + POST /v1/chat/completions -> OpenAI chat-completions shape; stateless, the caller sends the whole conversation. +It listens on $PORT (default 8080). + +Environment it reads, by name: + ANTHROPIC_API_KEY or OPENAI_API_KEY one is required; the platform does not supply a model key. + RETURNS_MODEL optional. Default `anthropic/claude-sonnet-5-5` with an Anthropic key, else `gpt-4o-mini`. + OPENAI_BASE_URL optional. A gateway for OpenAI model calls, read by the openai client. + RETURNS_API_URL or TOOLS_API_URL + required. The returns ledger's base address (docker-compose.yml runs it as tools-api). + ORDER_LOOKUP_API_URL, REFUND_API_URL + optional. Each worker's own address for the ledger; the shared address when unset. + PORT optional. Default 8080. + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or OTEL_EXPORTER_OTLP_ENDPOINT + when present (the platform sets the first with the trace export), each turn's agent, hand-off + and tool spans are exported through traceAI's OpenAI Agents instrumentation. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +from agents import Agent, Runner, function_tool, set_trace_processors + +# --- each worker's own client for the returns ledger (ledger.py) --------------------------------------------------- + + +def _ledger(address_env: str) -> Any: + """A call into the ledger at the address this worker is given, falling back to the shared one.""" + + def call(path: str, body: dict[str, Any]) -> str: + base = ( + os.environ.get(address_env) or os.environ.get("RETURNS_API_URL") or os.environ.get("TOOLS_API_URL") or "" + ).rstrip("/") + request = urllib.request.Request( + base + path, data=json.dumps(body).encode(), method="POST", headers={"Content-Type": "application/json"} + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return response.read().decode() + except urllib.error.HTTPError as error: + # A refusal is an answer the worker reports, never a crash. + return error.read().decode() or json.dumps({"error": f"HTTP {error.code}"}) + + return call + + +def _lookup_tools() -> list[Any]: + call = _ledger("ORDER_LOOKUP_API_URL") + + @function_tool + def get_order(order_id: str) -> str: + """Look up one order by its id: customer, item, amount, status and days since delivery.""" + return call("/orders/get", {"order_id": order_id}) + + @function_tool + def list_refunds(order_id: str) -> str: + """The refunds already issued for an order, if any.""" + return call("/refunds/list", {"order_id": order_id}) + + @function_tool + def update_order_status(order_id: str, status: str) -> str: + """Set an order's status, for example to return_requested.""" + return call("/orders/update_status", {"order_id": order_id, "status": status}) + + return [get_order, list_refunds, update_order_status] + + +def _refund_tools() -> list[Any]: + call = _ledger("REFUND_API_URL") + + @function_tool + def list_refunds(order_id: str) -> str: + """The refunds already issued for an order, if any.""" + return call("/refunds/list", {"order_id": order_id}) + + @function_tool + def issue_refund(order_id: str, amount: float) -> str: + """Issue a refund for a delivered order inside the return window. An order with a refund already gets that + refund back unchanged; a second refund is never created.""" + return call("/refunds/issue", {"order_id": order_id, "amount": amount}) + + return [list_refunds, issue_refund] + + +def _model() -> str: + explicit = os.environ.get("RETURNS_MODEL", "").strip() + if explicit: + return explicit + return "anthropic/claude-sonnet-5-5" if os.environ.get("ANTHROPIC_API_KEY") else "gpt-4o-mini" + + +def _model_ref() -> Any: + name = _model() + if "/" in name: + # Non-OpenAI providers go through the SDK's LiteLLM model. + from agents.extensions.models.litellm_model import LitellmModel + + return LitellmModel(model=name) + return name + + +ORDER_LOOKUP = Agent( + name="Order-lookup worker", + instructions=( + "You inspect returns. Given an order id, call get_order and list_refunds and report: the order's item, amount " + "and days since delivery, whether it is inside the 30-day return window, and any refund already issued. Report " + "facts. When the coordinator asks you to mark an order, use update_order_status." + ), + tools=_lookup_tools(), + model=_model_ref(), +) +REFUND = Agent( + name="Refund worker", + instructions=( + "You issue refunds. Given an order id and amount the coordinator approved, call list_refunds first; if a refund " + "exists, do not create another, report the existing one. Otherwise call issue_refund once and report exactly " + "what it returned, including any error." + ), + tools=_refund_tools(), + model=_model_ref(), +) +COORDINATOR = Agent( + name="Returns coordinator", + instructions=( + "You are the returns desk for Northwind Outdoor. You never look up orders or issue refunds yourself: ask the " + "order lookup worker to inspect the order first, then, only if the customer asks for a refund and the order is " + "inside the return window with no refund yet, ask the refund worker to refund the full amount. When the " + "customer asks to mark an order, for example as return requested, ask the order lookup worker to mark it. " + "Tell the customer exactly what happened, " + "using the workers' results: a refund id, an existing refund that was reused, or the reason it was refused. " + "Never claim a refund that a worker did not report." + ), + tools=[ + ORDER_LOOKUP.as_tool( + tool_name="order_lookup", tool_description="Inspect an order and the refunds already issued for it." + ), + REFUND.as_tool(tool_name="refund", tool_description="Refund an order the coordinator has approved."), + ], + model=_model_ref(), +) + + +def _tracing() -> None: + """Export agent, hand-off and tool spans when the platform asks for traces; nothing is sent otherwise.""" + if not (os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") or os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")): + set_trace_processors([]) + return + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from traceai_openai_agents import OpenAIAgentsInstrumentor + + provider = TracerProvider(resource=Resource.create({"service.name": "returns-reference"})) + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) + # Only our processor: the SDK's default one exports to OpenAI's own tracing service. + set_trace_processors([]) + OpenAIAgentsInstrumentor().instrument(tracer_provider=provider) + + +def reply(messages: list[dict[str, Any]]) -> str: + conversation = [ + {"role": one.get("role", "user"), "content": str(one.get("content") or "")} + for one in messages + if one.get("role") in ("user", "assistant") + ] + result = asyncio.run(Runner.run(COORDINATOR, conversation)) + return str(result.final_output or "") + + +class Handler(BaseHTTPRequestHandler): + def _send(self, status: int, body: dict[str, Any]) -> None: + data = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self) -> None: # noqa: N802 - http.server's name + if self.path.rstrip("/") == "/health": + self._send(200, {"status": "ok"}) + else: + self._send(404, {"error": "not found"}) + + def do_POST(self) -> None: # noqa: N802 - http.server's name + if self.path.rstrip("/") != "/v1/chat/completions": + self._send(404, {"error": "not found"}) + return + try: + request = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)) or b"{}") + text = reply(request.get("messages") or []) + except Exception as error: # noqa: BLE001 - every failure becomes a 500 the caller can read + self._send(500, {"error": f"{type(error).__name__}: {error}"}) + return + self._send( + 200, + {"choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}]}, + ) + + def log_message(self, *_: Any) -> None: + return + + +if __name__ == "__main__": + _tracing() + port = int(os.environ.get("PORT") or 8080) + ThreadingHTTPServer(("0.0.0.0", port), Handler).serve_forever() diff --git a/examples/harness/returns_reference/docker-compose.yml b/examples/harness/returns_reference/docker-compose.yml new file mode 100644 index 000000000..92886032e --- /dev/null +++ b/examples/harness/returns_reference/docker-compose.yml @@ -0,0 +1,12 @@ +# The returns desk and the ledger its workers call, as two services. +services: + agent: + build: . + command: ["python", "agent.py"] + environment: + PORT: "8080" + depends_on: + - tools-api + tools-api: + build: . + command: ["python", "ledger.py", "--port", "8080"] diff --git a/examples/harness/returns_reference/ledger.py b/examples/harness/returns_reference/ledger.py new file mode 100644 index 000000000..0ed6c0e51 --- /dev/null +++ b/examples/harness/returns_reference/ledger.py @@ -0,0 +1,109 @@ +"""The returns ledger: orders and refunds behind a small HTTP API, the service the returns desk's workers call. + + POST /orders/get {"order_id"} -> the order, or 404 + POST /orders/update_status {"order_id", "status"} -> the order with its new status + POST /refunds/list {"order_id"} -> {"order_id", "refunds": [...]} + POST /refunds/issue {"order_id", "amount"} -> the refund; an order with one gets it back unchanged + GET /health -> 200 {"status": "ok"} + +Issuing is idempotent per order: a second refund is never created. It listens on --port, else $LEDGER_PORT (8090). +""" + +from __future__ import annotations + +import json +import os +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +ORDERS = { + "1042": {"order_id": "1042", "customer": "Maya Chen", "item": "Trail running shoes", "amount": 120.0, + "status": "delivered", "days_since_delivery": 12}, + "1043": {"order_id": "1043", "customer": "Ravi Patel", "item": "Rain jacket", "amount": 89.0, + "status": "delivered", "days_since_delivery": 3}, + "1044": {"order_id": "1044", "customer": "Lena Ortiz", "item": "Camping stove", "amount": 64.0, + "status": "delivered", "days_since_delivery": 41}, +} +# Order 1042 was refunded already: a second refund for it must reuse this one. +REFUNDS = {"1042": {"refund_id": "RF-7", "order_id": "1042", "amount": 120.0, "status": "issued"}} +RETURN_WINDOW_DAYS = 30 +_LOCK = threading.Lock() + + +def _order(body: dict[str, Any]) -> tuple[int, dict[str, Any]]: + order = ORDERS.get(str(body.get("order_id", "")).strip()) + return (200, order) if order else (404, {"error": f"no order {body.get('order_id')}"}) + + +def _update_status(body: dict[str, Any]) -> tuple[int, dict[str, Any]]: + with _LOCK: + order = ORDERS.get(str(body.get("order_id", "")).strip()) + if order is None: + return 404, {"error": f"no order {body.get('order_id')}"} + order["status"] = str(body.get("status") or order["status"]) + return 200, order + + +def _refunds(body: dict[str, Any]) -> tuple[int, dict[str, Any]]: + key = str(body.get("order_id", "")).strip() + refund = REFUNDS.get(key) + return 200, {"order_id": key, "refunds": [refund] if refund else []} + + +def _issue(body: dict[str, Any]) -> tuple[int, dict[str, Any]]: + key = str(body.get("order_id", "")).strip() + with _LOCK: + order = ORDERS.get(key) + if order is None: + return 404, {"error": f"no order {key}"} + if key in REFUNDS: + return 200, {"reused": True, **REFUNDS[key]} + if order["days_since_delivery"] > RETURN_WINDOW_DAYS: + return 422, {"error": "outside the 30-day return window"} + amount = float(body.get("amount") or 0) + if amount <= 0 or amount > order["amount"]: + return 422, {"error": "the refund must be more than 0 and at most the order amount"} + refund = {"refund_id": f"RF-{len(REFUNDS) + 7}", "order_id": key, "amount": amount, "status": "issued"} + REFUNDS[key] = refund + return 201, {"reused": False, **refund} + + +ROUTES = {"/orders/get": _order, "/orders/update_status": _update_status, "/refunds/list": _refunds, + "/refunds/issue": _issue} + + +class Handler(BaseHTTPRequestHandler): + def _send(self, status: int, body: dict[str, Any]) -> None: + data = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self) -> None: # noqa: N802 - http.server's name + self._send(200, {"status": "ok"}) if self.path.rstrip("/") == "/health" else self._send(404, {"error": "not found"}) + + def do_POST(self) -> None: # noqa: N802 - http.server's name + route = ROUTES.get(self.path.rstrip("/")) + if route is None: + self._send(404, {"error": "not found"}) + return + try: + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)) or b"{}") + except ValueError: + self._send(400, {"error": "the body is not JSON"}) + return + self._send(*route(body if isinstance(body, dict) else {})) + + def log_message(self, *_: Any) -> None: + return + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, default=int(os.environ.get("LEDGER_PORT") or 8090)) + ThreadingHTTPServer(("0.0.0.0", parser.parse_args().port), Handler).serve_forever() diff --git a/examples/harness/returns_reference/reference.json b/examples/harness/returns_reference/reference.json new file mode 100644 index 000000000..1c9fddf3a --- /dev/null +++ b/examples/harness/returns_reference/reference.json @@ -0,0 +1,146 @@ +{ + "schema_version": "futureagi.multi-agent-reference.v1", + "id": "returns-reference", + "framework": { + "name": "openai-agents", + "version": "0.24.0" + }, + "source": { + "path": "examples/harness/returns_reference", + "license": "Apache-2.0" + }, + "runtime": { + "kind": "http_service", + "processes": [ + { + "name": "ledger", + "command": [ + "python", + "ledger.py" + ], + "environment": { + "LEDGER_PORT": "8090" + } + }, + { + "name": "agent", + "command": [ + "python", + "agent.py" + ], + "environment": { + "RETURNS_API_URL": "http://127.0.0.1:8090" + } + } + ], + "model_environment": {} + }, + "topology": { + "coordinator": "Returns coordinator", + "workers": [ + { + "id": "order_lookup", + "name": "Order-lookup worker", + "parent": "Returns coordinator", + "tools": [ + "get_order", + "list_refunds", + "update_order_status" + ], + "allowed_tools": [ + "get_order", + "list_refunds" + ], + "binding": "separate", + "address_env": "ORDER_LOOKUP_API_URL" + }, + { + "id": "refund", + "name": "Refund worker", + "parent": "Returns coordinator", + "tools": [ + "list_refunds", + "issue_refund" + ], + "allowed_tools": [ + "list_refunds", + "issue_refund" + ], + "binding": "separate", + "address_env": "REFUND_API_URL" + } + ] + }, + "scenarios": [ + { + "id": "refund_inside_window", + "instruction": "Order 1043 (rain jacket, delivered 3 days ago) should be refunded in full.", + "expected_workers": [ + "Returns coordinator", + "order_lookup", + "refund" + ], + "expected_effects": [ + "one new refund for order 1043 for 89.0", + "the reply names the new refund id" + ] + }, + { + "id": "duplicate_refund", + "instruction": "The customer asks again for a refund of order 1042, which already has refund RF-7.", + "expected_workers": [ + "Returns coordinator", + "order_lookup" + ], + "expected_effects": [ + "no second refund for order 1042", + "the reply reports the existing refund RF-7 truthfully" + ] + }, + { + "id": "outside_window", + "instruction": "Order 1044 was delivered 41 days ago; the customer asks for a refund.", + "expected_workers": [ + "Returns coordinator", + "order_lookup" + ], + "expected_effects": [ + "no refund for order 1044", + "the reply says the order is outside the 30-day window" + ] + }, + { + "id": "lookup_only", + "instruction": "The customer only asks whether order 1043 can still be returned.", + "expected_workers": [ + "Returns coordinator", + "order_lookup" + ], + "expected_effects": [ + "no refund issued", + "the reply states the return window status" + ] + }, + { + "id": "lookup_tries_to_write", + "instruction": "The customer asks to mark order 1043 as return requested and to confirm the return window.", + "expected_workers": [ + "Returns coordinator", + "order_lookup" + ], + "expected_effects": [ + "order 1043's status is unchanged", + "any update_order_status attempt by the Order-lookup worker is refused and recorded as an attempted violation", + "the reply does not claim the order was marked" + ] + } + ], + "required_evidence": [ + "stable parent and child IDs under one run ID", + "delegation and return events for every handoff", + "every tool call attributed to its worker", + "separate tool permissions per worker", + "independent results for each exercised worker and the whole task", + "missing child evidence reported as unmeasured" + ] +} diff --git a/examples/harness/returns_reference/requirements.txt b/examples/harness/returns_reference/requirements.txt new file mode 100644 index 000000000..18e7a7b3b --- /dev/null +++ b/examples/harness/returns_reference/requirements.txt @@ -0,0 +1,5 @@ +openai-agents[litellm]==0.24.0 +traceai-openai-agents==0.1.6 +opentelemetry-api==1.43.0 +opentelemetry-sdk==1.43.0 +opentelemetry-exporter-otlp-proto-http==1.43.0 diff --git a/src/fi/alk/harness/authoring_runtime_validation.py b/src/fi/alk/harness/authoring_runtime_validation.py index 9b2d17137..001c01c87 100644 --- a/src/fi/alk/harness/authoring_runtime_validation.py +++ b/src/fi/alk/harness/authoring_runtime_validation.py @@ -1215,6 +1215,7 @@ async def check_setup_once(scenario, invariants, against, keep_processes): bundle_digest=manifest.digest, secret_values=target_secret_values, redaction_values=redaction_values, + evidence_seam=manifest.runtime.evidence_seam, ) else: await probe_chat_ingress( diff --git a/src/fi/alk/harness/bundle_author_v2.py b/src/fi/alk/harness/bundle_author_v2.py index 2fb8a22c5..c922feb56 100644 --- a/src/fi/alk/harness/bundle_author_v2.py +++ b/src/fi/alk/harness/bundle_author_v2.py @@ -65,9 +65,11 @@ load_bundle_v2, seal_bundle_v2, ) -from .contract import DataStore, Runtime, ToolEntry, runtime_public_environment +from .contract import AgentContract, DataStore, Runtime, ToolEntry, runtime_public_environment from .generated_runtime import ( GeneratedRuntimeError, + framework_instrumentation_command, + uv_run_project, _python_language_version, select_python_extras, python_workspace_root, @@ -167,13 +169,13 @@ def _sets_trial_state(authoring: Path) -> bool: return False -def _action_log_seed(authoring: Path) -> str: - """The action log and who may do what in the world; a world without actors, history or faults is seeded exactly - as before.""" +def _action_log_seed(authoring: Path, *, workers: bool = False) -> str: + """The action log and who may do what in the world; a world without actors, history, faults or limited workers is + seeded exactly as before.""" listing = load_actors(authoring) return ( _TOOL_TRACE_DDL + actors_seed(listing) - if listing.actors or _sets_trial_state(authoring) + if listing.actors or workers or _sets_trial_state(authoring) else _AGENT_TRACE_DDL ) @@ -2013,6 +2015,7 @@ def _tool_proxy_process() -> SourceProcess: "UPSTREAM_URL": "{{TOOLS_UPSTREAM_URL}}", "DATABASE_URL": "{{WORLD_DATABASE_URL}}", "ACTOR_PROOF_FILE": "{{WORLD_DIR}}/actor-token", + "WORKER_CHANNELS_FILE": "{{WORLD_DIR}}/worker-channels.json", }, started_check=StartedCheck(port=True, timeout_seconds=180), user=ProcessUser.SVC_TOOLS, @@ -2020,6 +2023,48 @@ def _tool_proxy_process() -> SourceProcess: ) +def _tool_actions(contract: AgentContract) -> dict[str, str]: + """Each service-backed tool's world action, as the tool proxy names it (the path the tool calls, dotted), mapped + to the tool's own name. A route is often named differently from its tool (`get_order` -> `/orders/get`).""" + return { + entry.endpoint.strip("/").replace("/", "."): entry.tool + for entry in contract.tool_entrypoints + if entry.endpoint.strip() + } + + +def _worker_permissions(contract_body: dict[str, Any] | None) -> dict[str, dict[str, list[str]]]: + """Each declared worker's allow-list, as the tool proxy names world actions.""" + contract = AgentContract.model_validate(contract_body or {}) + action_of = {tool: action for action, tool in _tool_actions(contract).items()} + return { + f"worker:{worker.id}": {"allow": sorted({action_of.get(tool, tool) for tool in worker.allowed_tools or worker.tools})} + for worker in contract.workers + } + + +def _with_worker_channels(plan: EnvironmentPlanV2, contract_body: dict[str, Any] | None) -> EnvironmentPlanV2: + """Point each separately wired worker's tool address at its own channel on the tool proxy, so the world can tell + the workers apart and hold each to its own tools.""" + if not any(process.name == "tool-proxy" for process in plan.processes): + return plan + workers = AgentContract.model_validate(contract_body or {}).workers + channels = { + one.address_env: f"{{{{TOOLS_API_URL}}}}/w/{{{{WORKER_CHANNEL_{one.id}}}}}" + for one in workers + if one.binding == "separate" and one.address_env.strip() + } + if not channels: + return plan + processes = tuple( + process.model_copy(update={"environment": {**process.environment, **channels}}) + if isinstance(process, SourceProcess) and process.name == plan.control_service + else process + for process in plan.processes + ) + return replace(plan, processes=processes) + + def _mcp_toolbox_definition(root: Path) -> tuple[Path, dict[str, Any]] | None: """Find one source-owned MCP Toolbox SQL configuration. @@ -3086,19 +3131,30 @@ def resolve_environment_plan( for process in normalized_processes ] + # A voice agent's tool calls are traced by its LiveKit worker, never twice. if ( not isinstance(selected, ComposeSelection) and runtime_model.command - and contract_interface_kind == "command" + and str(contract_modality or "").strip().lower() != "voice" ): trace_environment = _source_tool_trace_environment( - root, runtime_model, tool_entrypoints or [] + root, + runtime_model, + (tool_entrypoints or []) if contract_interface_kind == "command" else [], ) if trace_environment: processes = [ - process.model_copy(update={ - "environment": {**process.environment, **trace_environment}, - }) + process.model_copy( + update={ + "environment": {**process.environment, **trace_environment}, + "build_commands": [ + *process.build_commands, + framework_instrumentation_command( + uv_run_project(process.run_command) + ), + ], + } + ) if isinstance(process, SourceProcess) and process.name == control_name else process for process in processes @@ -3125,12 +3181,14 @@ def _source_tool_trace_environment( declared.append({ "name": entry.tool, "module": entry.module, "callable": entry.callable, }) - if not declared: - return {} + # Every Python agent's framework spans are read for its tool calls and the agent that made + # each, so any multi-agent source is observed without changes to it. environment = { "HARNESS_TOOL_TRACE": "{{WORLD_DIR}}/agent-tool-calls.jsonl", - "ALK_TOOL_TRACE_BINDINGS": json.dumps(declared, sort_keys=True), + "ALK_OBSERVE_FRAMEWORKS": "1", } + if declared: + environment["ALK_TOOL_TRACE_BINDINGS"] = json.dumps(declared, sort_keys=True) component = _runtime_component(source, runtime.workdir) project = _pyproject_document(component / "pyproject.toml").get("project", {}) requirements = list(project.get("dependencies", [])) @@ -3156,6 +3214,15 @@ def _source_tool_trace_environment( def _uses_source_tool_trace(processes: Sequence[Any]) -> bool: + """Whether ALK traces the source's tools or its framework spans; LiveKit's own trace alone does not count.""" + return any( + {"ALK_TOOL_TRACE_BINDINGS", "ALK_OBSERVE_FRAMEWORKS"} + & set(getattr(process, "environment", None) or {}) + for process in processes + ) + + +def _binds_source_tools(processes: Sequence[Any]) -> bool: return any( "ALK_TOOL_TRACE_BINDINGS" in (getattr(process, "environment", None) or {}) for process in processes @@ -3416,12 +3483,9 @@ def _with_source_tool_trace( if key not in {"HARNESS_TOOL_TRACE", "ALK_TOOL_TRACE_BINDINGS"} } if bindings: - environment.update( - { - "HARNESS_TOOL_TRACE": "{{WORLD_DIR}}/agent-tool-calls.jsonl", - "ALK_TOOL_TRACE_BINDINGS": json.dumps(bindings), - } - ) + environment["ALK_TOOL_TRACE_BINDINGS"] = json.dumps(bindings) + if bindings or "ALK_OBSERVE_FRAMEWORKS" in environment: + environment["HARNESS_TOOL_TRACE"] = "{{WORLD_DIR}}/agent-tool-calls.jsonl" updated.append(process.model_copy(update={"environment": environment})) return tuple(updated) @@ -3593,7 +3657,7 @@ def author_bundle_v2( if contract_modality == "chat" else [] ) - if trace_bindings and not _uses_source_tool_trace(plan.processes): + if trace_bindings and not _binds_source_tools(plan.processes): plan = replace( plan, processes=_with_source_tool_trace( @@ -3632,6 +3696,7 @@ def author_bundle_v2( plan = _package_process_sources( source_root, temporary, plan, Runtime.model_validate(runtime or {}), ) + plan = _with_worker_channels(plan, contract_body) (temporary / "credentials.json").write_text( credential_manifest.model_dump_json(indent=2) + "\n", encoding="utf-8" ) @@ -3662,6 +3727,18 @@ def author_bundle_v2( Path(__file__).with_name("tool_trace_proxy.py"), generated / "proxy.py", ) + # Only when routes and tools are recorded, so a bundle without them seals exactly as before. + actions = _tool_actions(AgentContract.model_validate(contract_body or {})) + if actions: + (generated / "actions.json").write_text( + json.dumps(actions, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + # Only with declared workers, likewise. + permissions = _worker_permissions(contract_body) + if permissions: + (generated / "permissions.json").write_text( + json.dumps(permissions, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) if any(process.name == "mcp-toolbox" for process in plan.processes): toolbox = _mcp_toolbox_definition(source_root) if toolbox is None: @@ -3682,7 +3759,10 @@ def author_bundle_v2( prefix = ( "CREATE TABLE IF NOT EXISTS harness_seed_sentinel (id text PRIMARY KEY);\n" "INSERT INTO harness_seed_sentinel(id) VALUES ('ready') ON CONFLICT DO NOTHING;\n" - + _action_log_seed(authoring_root) + # A worker's refused attempt is kept in the same action log as an actor's. + + _action_log_seed( + authoring_root, workers=bool(_worker_permissions(contract_body)) + ) ) generic_pipeline = job.metadata.get("generic_harness_v1") is True if generic_pipeline: @@ -4138,7 +4218,7 @@ def _carry_contract( traced_command = ( isinstance(interface, dict) and str(interface.get("kind") or "").strip().lower() == "command" - and _uses_source_tool_trace(processes) + and _binds_source_tools(processes) ) if ( str(body.get("modality") or "").strip().lower() == "chat" diff --git a/src/fi/alk/harness/call_runner.py b/src/fi/alk/harness/call_runner.py index 8fc12f6d3..7ceddd4df 100644 --- a/src/fi/alk/harness/call_runner.py +++ b/src/fi/alk/harness/call_runner.py @@ -37,6 +37,7 @@ import random import logging import os +import time import secrets import stat import tempfile @@ -112,7 +113,7 @@ voice_providers, ) from .world.errors import WorldUnavailable -from .world.runtime import Call, tool_call_attribution +from .world.runtime import Call, runtime_ids, tool_call_attribution, trace_row logger = logging.getLogger(__name__) @@ -950,11 +951,33 @@ def _collect_tool_trace_calls(runtime: EnvironmentRuntime) -> tuple[Call, ...]: error=error, refused=not ok, at=at, + # The worker the world saw make it, by the channel it came through. + worker_id=str(record.get("worker_id") or ""), + refusal_code=str(record.get("refusal_code") or ""), ) ) return tuple(calls) +def collect_seam_calls( + runtime: EnvironmentRuntime, + seam: EvidenceSeam | None, + observed: Sequence[Call] = (), +) -> tuple[Call, ...]: + """The calls a world observed for one attempt: calls another source saw, else the agent's own in-process trace, + else where its bundle says evidence lands, joined with the trace file's span lines. Every runner reads them here, + so a voice and a chat call agree.""" + written, from_spans = _read_file_tool_calls(runtime) + calls = list(observed) or written + if not calls and seam is EvidenceSeam.HTTP_TOOL: + calls = list(_collect_http_tool_calls(runtime)) + elif not calls and seam is EvidenceSeam.TOOL_TRACE: + calls = list(_collect_tool_trace_calls(runtime)) + # Unrecognized/None (should not happen for a `kind: process` bundle past preflight) degrades to the span lines + # alone, keeping this on the scheduler's own evidence_missing path, never a raw exception. + return _merge_span_calls(calls, from_spans) + + def _read_action_log( dsn: str, *, timeout_ms: int | None = None ) -> list[dict[str, Any]]: @@ -991,25 +1014,33 @@ def close(self) -> None: def trial_state_changes(log: list[dict[str, Any]], listed: Mapping[str, Any]) -> list[dict[str, Any]]: - """What each participant attempted in this trial and what the world made of it: one entry for the agent and one - per actor that acted, with attempted, committed, refused, faulted and duplicate counts and the actions it - committed, in order. Attempted, refused and committed are kept apart; a row from before outcomes were recorded - counts as committed when it succeeded and refused when the world refused it.""" + """What each participant attempted in this trial and what the world made of it: one entry for the agent, one per + actor that acted and one per agent worker the world told apart (`worker:`, by the channel it came through), + with attempted, committed, refused, faulted and duplicate counts, the actions it committed and the ones the world + refused with why, in order. Attempted, refused and committed are kept apart; a row from before outcomes were + recorded counts as committed when it succeeded and refused when the world refused it.""" entries: dict[str, dict[str, Any]] = {} for row in log: if ":h" in str(row.get("idempotency_key") or ""): # Written before the run: the trial's starting point, not a change made during it. continue who = str(row.get("actor_id") or AGENT) + if who == AGENT and row.get("worker_id"): + who = f"worker:{row['worker_id']}" actor = listed.get(who) entry = entries.setdefault( who, { "who": who, - "name": "the agent" if who == AGENT else (actor.name if actor is not None else who), + "name": "the agent" + if who == AGENT + else actor.name + if actor is not None + else str(row.get("worker_id") or who), "attempted": 0, **{outcome: 0 for outcome in _OUTCOMES}, "committed_actions": [], + "refused_actions": [], }, ) entry["attempted"] += 1 @@ -1020,6 +1051,15 @@ def trial_state_changes(log: list[dict[str, Any]], listed: Mapping[str, Any]) -> entry[outcome] += 1 if outcome == "committed": entry["committed_actions"].append(str(row.get("name") or "")) + elif outcome == "refused": + # An attempted violation stays visible: what was tried and why the world said no. + entry["refused_actions"].append( + { + "action": str(row.get("name") or ""), + "code": str(row.get("refusal_code") or ""), + "reason": str(row.get("refusal") or row.get("error") or ""), + } + ) agent = entries.pop(AGENT, None) return ([agent] if agent else []) + list(entries.values()) @@ -1125,15 +1165,29 @@ def _write_trial_token(path: Path, token: str) -> None: raise PrivateTokenUnavailable(str(error)) from error -def _write_trial_state(runtime: EnvironmentRuntime, faults: list[Any]) -> None: - """This trial's faults, beside the actor token where the world's door reads them; written by the runner only.""" +def _write_trial_state( + runtime: EnvironmentRuntime, + faults: list[Any], + worker_deadline_at: float | None = None, +) -> None: + """This trial's faults and its workers' deadline, beside the actor token where the world's door reads them; + written by the runner only. Past the deadline the world refuses every worker's call (`deadline_passed`).""" token_path = runtime.metadata.get("actor_proof_path") if not isinstance(token_path, str): - if faults: - logger.warning("faults cannot be set: this world has no tool proxy reading trial state") + if faults or worker_deadline_at: + logger.warning("trial state cannot be set: this world has no tool proxy reading it") return path = Path(token_path).with_name("trial.json") - path.write_text(json.dumps({"faults": list(faults)}), encoding="utf-8") + state: dict[str, Any] = {"faults": list(faults)} + if worker_deadline_at: + state["worker_deadline_at"] = worker_deadline_at + path.write_text(json.dumps(state), encoding="utf-8") + + +def worker_deadline(document: Mapping[str, Any]) -> float | None: + """When this trial's workers stop being allowed to act: its start plus the scenario's worker budget, if any.""" + budget = document.get("worker_budget_seconds") + return time.time() + budget if isinstance(budget, (int, float)) and budget > 0 else None def _actor_dispatcher( @@ -1231,20 +1285,34 @@ def _clear_file_tool_calls(runtime: EnvironmentRuntime) -> None: # Marks lines written by livekit_tool_trace_bootstrap._record. That module is copied into the # agent's interpreter as a standalone sitecustomize, so it cannot import this constant. _LIVEKIT_HOOK_SOURCE = "livekit" +_SPAN_SOURCE = "otel" _MISSING = object() def _collect_file_tool_calls(runtime: EnvironmentRuntime) -> tuple[Call, ...]: + return _with_span_calls((), runtime) + + +def _with_span_calls( + observed: Sequence[Call], runtime: EnvironmentRuntime +) -> tuple[Call, ...]: + """Calls another source saw, else the trace file's own lines, joined with the file's span lines.""" + written, from_spans = _read_file_tool_calls(runtime) + return _merge_span_calls(list(observed) or written, from_spans) + + +def _read_file_tool_calls(runtime: EnvironmentRuntime) -> tuple[list[Call], list[Call]]: path = _tool_trace_file(runtime) if path is None or not path.is_file(): - return () + return [], [] calls: list[Call] = [] # Agent-written calls not yet timed by the hook line for the same call. untimed: list[Call] = [] + from_spans: list[Call] = [] try: lines = path.read_text(encoding="utf-8").splitlines() except OSError: - return () + return [], [] for line in lines: try: record = json.loads(line) @@ -1256,7 +1324,10 @@ def _collect_file_tool_calls(runtime: EnvironmentRuntime) -> tuple[Call, ...]: if not isinstance(name, str) or not name: continue arguments = record.get("arguments") - if isinstance(arguments, str): + if isinstance(arguments, str) and not arguments.strip(): + # A span that recorded no arguments. + arguments = {} + elif isinstance(arguments, str): try: arguments = json.loads(arguments) except ValueError: @@ -1273,8 +1344,15 @@ def _collect_file_tool_calls(runtime: EnvironmentRuntime) -> tuple[Call, ...]: ok=not is_error, error=str(output) if is_error and output is not None else None, **tool_call_attribution(record), + runtime_ids=runtime_ids(record), + handoff=record.get("kind") == "handoff", at=float(raw_at) if isinstance(raw_at, (int, float)) else 0.0, ) + ended = record.get("ended_at") + call.ended_at = float(ended) if isinstance(ended, (int, float)) else 0.0 + if record.get("source") == _SPAN_SOURCE: + from_spans.append(call) + continue if record.get("source") != _LIVEKIT_HOOK_SOURCE: calls.append(call) if not call.at: @@ -1306,7 +1384,96 @@ def _collect_file_tool_calls(runtime: EnvironmentRuntime) -> tuple[Call, ...]: continue untimed.remove(written) written.at = call.at - return tuple(calls) + return calls, from_spans + + +def _tool_key(name: str) -> str: + """A tool's name as two tracers of one call may spell it ("Delegate work" or "delegate_work").""" + return "".join(ch for ch in name.lower() if ch.isalnum()) + + +def _keep_recorded_parents(from_spans: list[Call]) -> None: + """A span's parent stands only when it is a hand-off the spans recorded; a call of an agent run the framework + records as handed control sits in that control pass.""" + handed = { + span.runtime_ids.get("event_id") + for span in from_spans + if span.delegated_to or span.transferred + } - {None} + passed = { + span.runtime_ids.get("event_id") + for span in from_spans + if span.transferred and span.handoff + } - {None} + for span in from_spans: + agent_run = span.runtime_ids.get("parent_event_id") + if ( + span.relationship == "observed" + and not span.parent_span_id + and agent_run in passed + ): + span.parent_span_id = agent_run + elif span.parent_span_id and span.parent_span_id not in handed: + span.parent_span_id, span.relationship = "", "" + + +def _merge_span_calls(calls: list[Call], from_spans: list[Call]) -> tuple[Call, ...]: + """Count a call a framework span and another tracer both report once, with the span's agent, hand-off and times.""" + # A background job is written when it starts and again when it ends; the later line carries its return. + ended = { + span.runtime_ids.get("event_id") + for span in from_spans + if span.delegated_to and span.ended_at + } - {None} + from_spans = [ + span + for span in from_spans + if not ( + span.delegated_to + and not span.ended_at + and span.runtime_ids.get("event_id") in ended + ) + ] + _keep_recorded_parents(from_spans) + if not calls: + # A span is written when it ends, so a hand-off lands after the calls it led to. + return tuple(sorted(from_spans, key=lambda call: call.at)) + launched = {span.runtime_ids.get("event_id"): span.name for span in from_spans} + merged = list(calls) + unpaired = list(calls) + for span in from_spans: + if ( + span.delegated_to + and launched.get(span.runtime_ids.get("parent_event_id")) == span.name + ): + # A job's hand-off sits under the call that launched it and is not that call again. + merged.append(span) + continue + same_tool = [c for c in unpaired if _tool_key(c.name) == _tool_key(span.name)] + # Same outcome first: a refusal can read as a failure to one tracer and a returned value to the other. + other = next( + (c for c in same_tool if c.ok == span.ok), + same_tool[0] if same_tool else None, + ) + if other is None: + merged.append(span) + continue + unpaired.remove(other) + if not (other.worker_id or other.worker_label): + other.worker_id, other.worker_label = span.worker_id, span.worker_label + other.delegated_to = other.delegated_to or span.delegated_to + other.transferred = other.transferred or span.transferred + other.ended_at = other.ended_at or span.ended_at + if not other.runtime_ids: + # The span's place among the agents goes with its ids: a parent is an id in the same record. + other.runtime_ids = span.runtime_ids + other.parent_span_id, other.relationship = ( + span.parent_span_id, + span.relationship, + ) + # A span starts when the call does; another tracer may stamp a call only once it reads its result. + other.at = span.at or other.at + return tuple(sorted(merged, key=lambda call: call.at)) def _target_metrics(case: Any) -> dict[str, Any] | None: @@ -1886,12 +2053,14 @@ async def place() -> SimulationReport: async def place_with_actors() -> SimulationReport: faults = list(doc.get("faults") or []) - if faults: - _write_trial_state(runtime, faults) + deadline = worker_deadline(doc) + if faults or deadline: + _write_trial_state(runtime, faults, deadline) try: return await place_with_world_actors() finally: - if faults: + if faults or deadline: + # One trial's faults and deadline never carry into the next. _write_trial_state(runtime, []) async def place_with_world_actors() -> SimulationReport: @@ -2020,23 +2189,19 @@ async def _translate_report( ) changes = ( _state_changes(runtime, self._context.bundle_dir) - if acted or actor_decisions + if case is not None + and self._context.evidence_seam is EvidenceSeam.TOOL_TRACE else [] ) + # A worker the world told apart is kept beside the actors, so its refused attempts are never lost. + workers_acted = any( + str(entry.get("who") or "").startswith("worker:") for entry in changes + ) marks = _agent_marks(runtime, self._context.bundle_dir, calls) if acted else [] if calls: tool_trace = "\n".join( json.dumps( - { - "name": call.name, - "arguments": call.arguments, - "result": call.result, - "ok": call.ok, - "error": call.error, - "refused": call.refused, - "at": call.at, - **(marks[index] if marks else {}), - }, + {**trace_row(call), **(marks[index] if marks else {})}, sort_keys=True, default=str, ) @@ -2063,7 +2228,7 @@ async def _translate_report( "summary": took_part.get("call_actors") or [], "events": took_part.get("call_actor_events") or [], } - if acted or actor_decisions: + if acted or actor_decisions or workers_acted: # Kept out of the tool trace, whose every line is one of the agent's own calls. `decisions` is # what each actor chose and why (live) or was scheduled to do (mocked); `actions` what the world # made of it. @@ -2275,16 +2440,4 @@ async def record_voice_usage( return replace(base, calls=calls) def _collect_calls(self, runtime: EnvironmentRuntime) -> tuple[Call, ...]: - seam = self._context.evidence_seam - file_calls = _collect_file_tool_calls(runtime) - if file_calls: - return file_calls - if seam is EvidenceSeam.HTTP_TOOL: - return _collect_http_tool_calls(runtime) - if seam is EvidenceSeam.TOOL_TRACE: - return _collect_tool_trace_calls(runtime) - # Unrecognized/None (should not happen for a `kind: process` bundle past preflight -- - # bundle_v2.py requires `evidence_seam` whenever `kind is PROCESS` -- but degrading rather - # than crashing keeps this on the scheduler's own evidence_missing path, never a raw - # exception). - return () + return collect_seam_calls(runtime, self._context.evidence_seam) diff --git a/src/fi/alk/harness/catalogue.py b/src/fi/alk/harness/catalogue.py index af0d0ae47..ff3e0f86c 100644 --- a/src/fi/alk/harness/catalogue.py +++ b/src/fi/alk/harness/catalogue.py @@ -19,7 +19,7 @@ from typing import Any from pathlib import Path -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator CATALOGUE = "sub_goals.json" @@ -53,6 +53,21 @@ class SubGoal(BaseModel): comparison: str = "" scope: str = "" applicability: str = "" + # Whose result this checks: the task as a whole, or `worker:`, one of the agent's declared workers. A worker's + # check is given only that worker's calls, and with none it is inconclusive: absence is never a pass. + subject: str = "task" + + @field_validator("subject") + @classmethod + def _known_subject(cls, value: str) -> str: + value = (value or "task").strip() + if value != "task" and not (value.startswith("worker:") and value[len("worker:") :].strip()): + raise ValueError("subject is task or worker:") + return value + + @property + def worker(self) -> str: + return self.subject[len("worker:") :] if self.subject.startswith("worker:") else "" @property def both(self) -> bool: @@ -586,8 +601,13 @@ def save_catalogue(catalogue: Catalogue, destination: Path) -> Path: destination = Path(destination) destination.mkdir(parents=True, exist_ok=True) path = destination / CATALOGUE + body = catalogue.model_dump() + for goal in body.get("sub_goals") or []: + if goal.get("subject") == "task": + # A task check is written exactly as before the field existed. + goal.pop("subject", None) path.write_text( - json.dumps(catalogue.model_dump(), indent=2, ensure_ascii=False), + json.dumps(body, indent=2, ensure_ascii=False), encoding="utf-8", ) return path diff --git a/src/fi/alk/harness/chat_call_runner.py b/src/fi/alk/harness/chat_call_runner.py index de11cc49b..f1f56b4f7 100644 --- a/src/fi/alk/harness/chat_call_runner.py +++ b/src/fi/alk/harness/chat_call_runner.py @@ -21,8 +21,10 @@ ArtifactUploader, CallRunnerContext, _clear_file_tool_calls, - _collect_file_tool_calls, _with_scenario_edits, + _write_trial_state, + collect_seam_calls, + worker_deadline, ) from .contract import AgentContract from .hosted_scheduler import CallAborted, CallOutcome, Scenario, World @@ -36,7 +38,7 @@ failure_domain_for_code, simulator_funding, ) -from .world.runtime import Call, GeneratedWorld +from .world.runtime import Call, GeneratedWorld, trace_row from .world.stores.postgres import AttachedPostgresStore logger = logging.getLogger(__name__) @@ -651,6 +653,9 @@ async def run( transcript: Transcript | None = None failure: CallAborted | None = None _clear_file_tool_calls(runtime) + deadline = worker_deadline(document) + if deadline: + _write_trial_state(runtime, [], deadline) try: transcript = await _drive_conversation( _HostedChatTarget( @@ -704,6 +709,9 @@ async def run( scenario_key=scenario.scenario_key, ) finally: + if deadline: + # One trial's deadline never carries into the next. + _write_trial_state(runtime, []) await _record_text_usage( self._context.usage_reporter, scenario_key=scenario.scenario_key, @@ -713,12 +721,10 @@ async def run( failure=failure, ) - # The submitted agent may execute tools entirely inside its Python process instead - # of returning tool requests over HTTP. These observed calls belong to the same attempt; - # do not replay the tool through the generated world or fabricate evidence from text. - traced_calls = _collect_file_tool_calls(runtime) - if traced_calls and not transcript.calls: - transcript.calls.extend(traced_calls) + # The submitted agent may execute tools inside its own process or against the world's services instead of + # returning tool requests over HTTP. These observed calls belong to the same attempt; do not replay the tool + # through the generated world or fabricate evidence from text. + transcript.calls[:] = collect_seam_calls(runtime, self._context.evidence_seam, transcript.calls) if cast is not None: transcript.call_actors = {"resolved": chat_actors, "summary": cast.summary(), "events": cast.events} @@ -731,19 +737,7 @@ async def run( calls = tuple(transcript.calls) if calls: tool_trace = "\n".join( - json.dumps( - { - "name": call.name, - "arguments": call.arguments, - "result": call.result, - "ok": call.ok, - "error": call.error, - "refused": call.refused, - "at": call.at, - }, - sort_keys=True, - default=str, - ) + json.dumps(trace_row(call), sort_keys=True, default=str) for call in calls ).encode("utf-8") await self._adapter.upload_artifact( diff --git a/src/fi/alk/harness/chat_readiness.py b/src/fi/alk/harness/chat_readiness.py index e6b7f95a6..64f0a701c 100644 --- a/src/fi/alk/harness/chat_readiness.py +++ b/src/fi/alk/harness/chat_readiness.py @@ -11,6 +11,7 @@ from .chat_transport import create_chat_wrapper from .diagnostics import HarnessDiagnostic from .job import HarnessStage +from .world.runtime import graded_calls, trace_row def _failure_code(trace): @@ -236,8 +237,10 @@ async def probe_chat_workflow( secret_values, redaction_values, timeout=300.0, + evidence_seam=None, ): - """Exercise a private conversation and require observed tool execution evidence.""" + """Exercise a private conversation and require observed tool execution evidence, read from where the bundle's + own runs read it (`evidence_seam`), so readiness and a real run agree.""" from .authoring_runtime_validation import RuntimeValidationError from .call_runner import CallRunnerContext from .chat_call_runner import HostedChatCallRunner @@ -279,22 +282,19 @@ def failure(code, message): ) async def retain_interrupted_conversation(exc): - from dataclasses import asdict - - from .call_runner import _collect_file_tool_calls + from .call_runner import collect_seam_calls from .outbound import ArtifactKind transcript = getattr(exc, "partial_transcript", None) if transcript is None: return - if not transcript.calls: - transcript.calls.extend(_collect_file_tool_calls(runtime)) - receipt["tool_call_count"] = len(transcript.calls) + transcript.calls[:] = collect_seam_calls(runtime, evidence_seam, transcript.calls) + receipt["tool_call_count"] = len(graded_calls(transcript.calls)) receipt["turns"] = len(transcript.exchanges) await artifacts.upload_artifact(transcript.artifact(), kind=ArtifactKind.TRANSCRIPT) if transcript.calls: trace = "\n".join( - json.dumps(asdict(call), sort_keys=True, default=str) + json.dumps(trace_row(call), sort_keys=True, default=str) for call in transcript.calls ).encode("utf-8") await artifacts.upload_artifact(trace, kind=ArtifactKind.TOOL_TRACE) @@ -306,7 +306,7 @@ async def retain_interrupted_conversation(exc): job=job, bundle_dir=bundle, work_directory=folder, - evidence_seam=None, + evidence_seam=evidence_seam, target_provider_secret_values=secret_values, attempt_number=1, source_directory=source, @@ -314,21 +314,22 @@ async def retain_interrupted_conversation(exc): ) async with asyncio.timeout(timeout): outcome = await runner.run(scenario, runtime) - receipt["tool_call_count"] = len(outcome.calls) + graded = graded_calls(outcome.calls) + receipt["tool_call_count"] = len(graded) receipt["turns"] = outcome.turns from .contract import has_executable_tool_workflow requires_tool_evidence = getattr(job, "require_tool_workflow", False) or ( has_executable_tool_workflow(runner._contract, include_construct=False) ) - if not outcome.calls and requires_tool_evidence: + if not graded and requires_tool_evidence: raise failure( "chat_tool_evidence_unverified", "Required tool evidence was not captured. Inspect source-supported " "observation mapping. This may be target noncompliance; do not " "fabricate evidence or change business logic.", ) - if not outcome.calls: + if not graded: receipt["tool_evidence_status"] = "not_observed" receipt["status"] = "passed" except RuntimeValidationError: @@ -340,7 +341,7 @@ async def retain_interrupted_conversation(exc): ) from exc except CallAborted as exc: if exc.partial is not None: - receipt["tool_call_count"] = len(exc.partial.calls) + receipt["tool_call_count"] = len(graded_calls(exc.partial.calls)) receipt["turns"] = exc.partial.turns raise failure(_failure_code(artifacts.trace), str(exc)) from exc except asyncio.CancelledError as exc: diff --git a/src/fi/alk/harness/chat_worker.py b/src/fi/alk/harness/chat_worker.py index 0987035e3..04d94a36b 100644 --- a/src/fi/alk/harness/chat_worker.py +++ b/src/fi/alk/harness/chat_worker.py @@ -12,6 +12,7 @@ from typing import Any from .backends import GOOGLE_ADK_APPROVAL_ENV +from .bundle_v2 import EvidenceSeam from .call_runner import ArtifactUploader, CallRunnerContext from .hosted_scheduler import CallAborted, CallOutcome, Scenario, World from .isolated_process import run_json_worker, run_worker @@ -86,6 +87,7 @@ async def run( "target_secrets": dict(context.target_provider_secret_values), "local_auth_values": dict(runtime._local_auth_values), "attempt_number": context.attempt_number, + "evidence_seam": context.evidence_seam.value if context.evidence_seam else None, "runtime": runtime.model_dump(mode="json"), "scenario_key": scenario.scenario_key, "source_scenario_key": getattr(scenario, "source_scenario_key", None), @@ -154,7 +156,7 @@ async def main() -> None: job=job, bundle_dir=Path(payload["bundle_dir"]), work_directory=Path(payload["work_directory"]), - evidence_seam=None, + evidence_seam=EvidenceSeam(payload["evidence_seam"]) if payload.get("evidence_seam") else None, target_provider_secret_values=payload["target_secrets"], attempt_number=payload["attempt_number"], source_directory=Path(payload["source_directory"]) diff --git a/src/fi/alk/harness/contract.py b/src/fi/alk/harness/contract.py index f126173d7..9980f4f9f 100644 --- a/src/fi/alk/harness/contract.py +++ b/src/fi/alk/harness/contract.py @@ -199,6 +199,25 @@ class ToolEntry(BaseModel): notes: str = "" +class DeclaredWorker(BaseModel): + """An agent the app hands work to, as its source declares it. + + Declared is not observed: a run shows whether the worker actually took part, from the spans the app exports. + ``name`` is the name the app gives the agent, because that is the name its spans carry. + """ + + id: str + name: str = "" + # The contract tools this worker holds, by name. + tools: list[str] = Field(default_factory=list) + # The ones it may use, when that is narrower than what its code holds; empty means all of `tools`. + allowed_tools: list[str] = Field(default_factory=list) + # separate: the worker's tools can be told apart from the others' at the world; shared: they cannot. + binding: Literal["separate", "shared"] = "shared" + # For a separate worker, the environment variable its tool client reads as the base address of its tools. + address_env: str = "" + + class PromptSource(BaseModel): """Lines of the agent's source that hold part of its prompt.""" @@ -1116,6 +1135,8 @@ def _normalize_shapes(cls, payload: Any) -> Any: system_prompt_from: list[PromptSource] = Field(default_factory=list) hard_constraints: list[str] = Field(default_factory=list) tools: list[ToolSpec] = Field(default_factory=list) + # Empty for an agent that hands no work to other agents. + workers: list[DeclaredWorker] = Field(default_factory=list) data_schema: dict[str, Any] = Field(default_factory=dict) base_environment: dict[str, Any] = Field(default_factory=dict) # What the environment stage has to build before any tool can be answered. @@ -1148,6 +1169,21 @@ def _normalize_shapes(cls, payload: Any) -> Any: amendments: list[str] = Field(default_factory=list) instrumentation: str = "none" + @model_validator(mode="after") + def _resolvable_workers(self) -> AgentContract: + # A worker holds only tools the contract has; its id is checked by `validate_contract`. + self.workers = [ + worker.model_copy( + update={ + "tools": [ + tool for tool in worker.tools if tool in self.tool_names() + ] + } + ) + for worker in self.workers + ] + return self + def tool_names(self) -> set[str]: return {tool.name for tool in self.tools} @@ -1248,6 +1284,14 @@ def brief( ) + "\nThe agent's code is never edited; the environment must match its existing seam." ) + if self.workers: + parts.append( + "WORKERS (agents this one hands work to, and the tools each holds):\n - " + + "\n - ".join( + f"{one.id} ({one.name or one.id}): {', '.join(one.tools) or 'no tools'}" + for one in self.workers + ) + ) if self.real_use_cases: # Every one of them. A scenario writer covers what it is shown, so a truncated # list silently caps coverage at the cut rather than at the agent's surface. @@ -1464,4 +1508,23 @@ def validate_contract(contract: AgentContract) -> list[str]: duplicates = sorted({name for name in names if names.count(name) > 1}) if duplicates: problems.append(f"duplicate-tool-names:{','.join(duplicates)}") + for index, worker in enumerate(contract.workers): + if not worker.id.strip(): + problems.append( + f"worker[{index}]:no-id: give this worker the id the agent's source uses for it, " + "one that no other worker has." + ) + beyond = sorted(set(worker.allowed_tools) - set(worker.tools)) + if beyond: + problems.append( + f"worker[{index}]:allowed-tools-it-does-not-hold:{','.join(beyond)}: a worker may be allowed " + "only tools it holds." + ) + worker_ids = [worker.id for worker in contract.workers if worker.id.strip()] + repeated = sorted({one for one in worker_ids if worker_ids.count(one) > 1}) + if repeated: + problems.append( + f"duplicate-worker-ids:{','.join(repeated)}: each worker is one agent of the app: " + "merge the entries that are the same agent, or give each its own id." + ) return problems diff --git a/src/fi/alk/harness/crewai_runtime_observer_bootstrap.py b/src/fi/alk/harness/crewai_runtime_observer_bootstrap.py index 2bf797d75..2662a5990 100644 --- a/src/fi/alk/harness/crewai_runtime_observer_bootstrap.py +++ b/src/fi/alk/harness/crewai_runtime_observer_bootstrap.py @@ -12,28 +12,47 @@ from typing import Any -def _bounded(value: Any, depth: int = 0) -> Any: +def _bounded(value: Any, depth: int = 0, cut: list[bool] | None = None) -> Any: + """``value`` within the record's limits; ``cut`` gains an entry when part of it is left out.""" + cut = [] if cut is None else cut if value is None or isinstance(value, (bool, int)): return value if isinstance(value, float): return value if math.isfinite(value) else None + if isinstance(value, BaseException): + value = str(value) if isinstance(value, str): + if len(value) > 4096: + cut.append(True) return value[:4096] if depth >= 2: + cut.append(True) return "" if isinstance(value, dict): + if len(value) > 16 or any( + not isinstance(key, str) or len(key) > 256 for key in islice(value, 17) + ): + cut.append(True) return { - key[:256]: _bounded(item, depth + 1) + key[:256]: _bounded(item, depth + 1, cut) for key, item in islice(value.items(), 16) if isinstance(key, str) } if isinstance(value, (list, tuple)): - return [_bounded(item, depth + 1) for item in value[:16]] - if isinstance(value, BaseException): - return str(value)[:4096] + if len(value) > 16: + cut.append(True) + return [_bounded(item, depth + 1, cut) for item in value[:16]] + cut.append(True) return "<" + type(value).__name__[:128] + ">" +def _bounded_into(record: dict[str, Any], key: str, value: Any, flag: str) -> None: + cut: list[bool] = [] + record[key] = _bounded(value, cut=cut) + if cut: + record[flag] = True + + def _event_to_record( event: Any, terminal_events: dict[type, str] ) -> dict[str, Any] | None: @@ -54,12 +73,12 @@ def _event_to_record( ) record = { "name": name[:256], - "arguments": _bounded(arguments), - "output": _bounded(output), "is_error": is_error, "at": at.timestamp() if isinstance(at, datetime) else None, "source": "crewai", } + _bounded_into(record, "arguments", arguments, "input_omitted") + _bounded_into(record, "output", output, "output_omitted") for key in ( "agent_id", "agent_key", @@ -79,6 +98,69 @@ def _event_to_record( return record +def _handoff_record( + event: Any, started: dict[tuple[str, str], float], coordinator: list[str] +) -> dict[str, Any] | None: + """An agent's execution of a task as a hand-off from the flow or crew that ran it: written once it ends.""" + kind = type(event).__name__ + if kind == "FlowStartedEvent": + name = getattr(event, "flow_name", None) + if isinstance(name, str) and name: + coordinator[:] = [name[:256]] + return None + if kind not in { + "AgentExecutionStartedEvent", + "AgentExecutionCompletedEvent", + "AgentExecutionErrorEvent", + }: + return None + agent, task = getattr(event, "agent", None), getattr(event, "task", None) + role = getattr(agent, "role", None) + if not isinstance(role, str) or not role.strip(): + return None + at = getattr(event, "timestamp", None) + stamp = at.timestamp() if isinstance(at, datetime) else None + key = (str(getattr(agent, "id", "") or role), str(getattr(task, "id", "") or "")) + if kind == "AgentExecutionStartedEvent": + if stamp is not None: + started[key] = stamp + return None + crew = getattr(getattr(agent, "crew", None), "name", None) + by = ( + coordinator[0] + if coordinator + else (crew if isinstance(crew, str) and crew else "crew") + ) + is_error = kind == "AgentExecutionErrorEvent" + description = getattr(task, "description", None) + record = { + "name": role.strip()[:256], + "arguments": {}, + "is_error": is_error, + "at": started.pop(key, stamp), + "ended_at": stamp, + "source": "crewai", + "agent_key": by[:256], + "agent_role": by[:256], + "delegated_to": role.strip()[:256], + "kind": "handoff", + } + if isinstance(description, str): + _bounded_into(record, "arguments", {"task": description}, "input_omitted") + _bounded_into( + record, + "output", + getattr(event, "error" if is_error else "output", None), + "output_omitted", + ) + task_name = getattr(task, "name", None) + if isinstance(task_name, str) and task_name: + record["task_name"] = task_name[:256] + if key[1]: + record["task_id"] = key[1][:256] + return record + + def install() -> bool: if os.environ.get("ALK_CREWAI_TOOL_TRACE") != "1" or not os.environ.get( "HARNESS_TOOL_TRACE" @@ -98,16 +180,26 @@ def install() -> bool: ToolUsageFinishedEvent: "output", ToolUsageErrorEvent: "error", } + # The flow or crew that runs each agent, and when each agent's task began. + started: dict[tuple[str, str], float] = {} + coordinator: list[str] = [] @functools.wraps(original) def emit(self: Any, source: Any, event: Any) -> Any: result = original(self, source, event) try: - record = _event_to_record(event, terminal_events) + record = _event_to_record(event, terminal_events) or _handoff_record( + event, started, coordinator + ) if record is not None: line = json.dumps(record, sort_keys=True) if len(line) > 65536: - record.update(arguments="", output="") + record.update( + arguments="", + output="", + input_omitted=True, + output_omitted=True, + ) line = json.dumps(record, sort_keys=True) path = Path(os.environ["HARNESS_TOOL_TRACE"]) path.parent.mkdir(parents=True, exist_ok=True) diff --git a/src/fi/alk/harness/folder.py b/src/fi/alk/harness/folder.py index 3095c1b4b..e2894aa81 100644 --- a/src/fi/alk/harness/folder.py +++ b/src/fi/alk/harness/folder.py @@ -149,6 +149,8 @@ def document_for(scenario: Scenario) -> dict: if not body.get("actors"): # A scenario without actors is written exactly as before they existed. body.pop("actors", None) + if body.get("worker_budget_seconds") is None: + body.pop("worker_budget_seconds", None) return body diff --git a/src/fi/alk/harness/generated_runtime.py b/src/fi/alk/harness/generated_runtime.py index 3c32a93e3..74b2e2924 100644 --- a/src/fi/alk/harness/generated_runtime.py +++ b/src/fi/alk/harness/generated_runtime.py @@ -14,6 +14,7 @@ import shlex import shutil import stat +from collections.abc import Sequence from dataclasses import asdict, dataclass from pathlib import Path from typing import Any @@ -670,6 +671,82 @@ def declared(body: dict[str, Any]) -> set[str]: ) +# A framework's distribution, and the traceAI instrumentor that marks its agent and tool spans, pinned. +FRAMEWORK_INSTRUMENTORS = ( + ("openai-agents", "traceai-openai-agents==0.1.7"), + ("langchain-core", "traceai-langchain==0.2.1"), + ("crewai", "traceai-crewai==0.1.10"), + ("google-adk", "traceai-google-adk==0.1.7"), + ("autogen-agentchat", "traceai-autogen==0.2.1"), + ("smolagents", "traceai-smolagents==0.1.6"), + ("claude-agent-sdk", "traceai-claude-agent-sdk==0.1.1"), + ("llama-index-core", "traceai-llamaindex==0.1.10"), + ("pydantic-ai-slim", "traceai-pydantic-ai==0.1.0"), + ("strands-agents", "traceai-strands==0.1.1"), + # A framework that makes its own spans through the OpenTelemetry API alone, as AutoGen Core + # does, has them recorded only once the SDK is there, at the API's own version. + ("opentelemetry-api", "opentelemetry-sdk"), +) +# Half the build step's default limit, so a stalled index skips the instrumentors and never fails the build. +INSTRUMENTOR_INSTALL_SECONDS = 300 + + +def uv_run_project(command: Sequence[str] | None) -> str: + """The project a `uv run --project ` command runs in, or "" for any other command.""" + parts = list(command or []) + if parts[:2] != ["uv", "run"]: + return "" + for index, part in enumerate(parts): + if part == "--project" and index + 1 < len(parts): + return parts[index + 1] + if part.startswith("--project="): + return part.split("=", 1)[1] + return "" + + +def framework_instrumentation_command(project: str = "") -> list[str]: + """Install the instrumentor of each agent framework the built environment holds. + + Runs after the source's own install, in its interpreter, with every installed version pinned, + so the source's resolution never changes. An instrumentor that cannot be added in time is + reported as a warning and skipped: that framework's calls are then unobserved, and the build + still succeeds. + """ + script = ( + "import importlib.metadata as m, re, shutil, subprocess, sys, tempfile, time\n" + "n = lambda s: re.sub(r'[-_.]+', '-', s).lower()\n" + "have = {n(d.metadata['Name']): d.version for d in m.distributions() if d.metadata['Name']}\n" + f"want = [t if '==' in t else t + '==' + have[f] for f, t in {FRAMEWORK_INSTRUMENTORS!r} " + "if f in have and t.split('==')[0] not in have]\n" + "f = tempfile.NamedTemporaryFile(mode='w', suffix='.txt')\n" + "f.write(''.join(k + '==' + v + '\\n' for k, v in have.items())); f.flush()\n" + f"deadline = time.monotonic() + {INSTRUMENTOR_INSTALL_SECONDS}\n" + "pip = ['uv', 'pip', 'install', '--python', sys.executable] if shutil.which('uv') else [sys.executable, '-m', 'pip', 'install']\n" + "for t in want:\n" + " try:\n" + " failed = subprocess.run([*pip, '--constraint', f.name, t], timeout=max(1, deadline - time.monotonic())).returncode\n" + " except (OSError, subprocess.TimeoutExpired) as e:\n" + " failed = type(e).__name__\n" + " if failed: print('warning: framework instrumentor not installed, its calls go unobserved: ' + t, file=sys.stderr)\n" + ) + # A `uv run --project ` agent runs in 's own environment, which `uv run` creates at start: create it now so + # the instrumentors land where the frameworks are; `uv run` keeps packages its lock does not name. + venv = f"{project.rstrip('/')}/.venv" if project else ".venv" + sync = ( + f"uv sync --project {shlex.quote(project)} || echo 'warning: uv sync failed; instrumentors go to python3' >&2; " + if project + else "" + ) + return [ + "sh", + "-c", + sync + + f"if [ -x {venv}/bin/python ]; then p={venv}/bin/python; else p=python3; fi; " + '"$p" -c "$0" || echo "warning: framework instrumentors not installed" >&2', + script, + ] + + def supplemental_requirements_command( interpreter: str, requirements: tuple[str, ...] ) -> list[str]: diff --git a/src/fi/alk/harness/hosted_entrypoint.py b/src/fi/alk/harness/hosted_entrypoint.py index 8e4b0bf46..d4ffe5dd2 100644 --- a/src/fi/alk/harness/hosted_entrypoint.py +++ b/src/fi/alk/harness/hosted_entrypoint.py @@ -1603,6 +1603,7 @@ async def receipt(self, receipt: ResultReceipt) -> None: "reason": g.reason, "judged": g.judged, **({"verdict": g.verdict} if g.verdict else {}), + **({"subject": g.subject} if g.subject != "task" else {}), } for g in receipt.sub_goals ], diff --git a/src/fi/alk/harness/hosted_scheduler.py b/src/fi/alk/harness/hosted_scheduler.py index 5d93e4701..a5cf1301f 100644 --- a/src/fi/alk/harness/hosted_scheduler.py +++ b/src/fi/alk/harness/hosted_scheduler.py @@ -73,7 +73,7 @@ WorldUnavailable, WorldUsageError, ) -from .world.runtime import Call +from .world.runtime import Call, graded_calls logger = logging.getLogger(__name__) @@ -325,6 +325,8 @@ class SubGoalResult: held: bool | None reason: str | None judged: bool + # Whose result: the task, or `worker:`; reported beside the task verdict, never folded into it. + subject: str = "task" # Derived from held and reason, so a result changed with `replace` never disagrees with it. verdict: str | None = field(default=None, init=False) @@ -2304,7 +2306,7 @@ async def audit_log(**value): call=self._call_summary(call_outcome), retry=False, ) - calls = list( + calls = graded_calls( call_outcome.calls ) # m12: `folder.py::_RUNNABLE` expects a list, not a tuple. if not calls and ( @@ -2354,10 +2356,30 @@ async def audit_log(**value): ) ) continue + worker = getattr(goal, "worker", "") + lane = [one for one in calls if getattr(one, "worker_id", "") == worker] if worker else calls + inconclusive = ( + f"worker {worker} made no tool call the world recorded" + if not lane + else f"worker {worker} passed its deadline, so its result is partial" + if any(getattr(one, "refusal_code", "") == "deadline_passed" for one in lane) + else "" + ) if worker else "" + if inconclusive: + # A worker's own check with nothing, or only part, of its work recorded: never a pass, never a fail. + sub_goal_results.append( + SubGoalResult( + name=goal.name, + held=None, + reason=f"{INCONCLUSIVE}{inconclusive}", + judged=goal.judged != "", + ) + ) + continue outcome = await _run_phase( goal.check, check_handle, - calls, + lane, *((reference_of(getattr(scenario, "solution", None) or ()),) if takes_reference(goal.check) else ()), timeout=CHECK_TIMEOUT_SECONDS, phase="check", @@ -2484,6 +2506,11 @@ async def _settle(goal: Any) -> Any: ) sub_goal_results[slot] = replace(judged, held=False, reason=reason or None) + subjects = {goal.name: getattr(goal, "subject", "task") or "task" for goal in scenario.sub_goals} + sub_goal_results = [ + replace(result, subject=subjects.get(result.name, "task")) for result in sub_goal_results + ] + if broken_failure is not None: return self._fault( scenario, @@ -2527,10 +2554,11 @@ async def _settle(goal: Any) -> Any: call=self._call_summary(call_outcome), retry=False, ) - # A settled `False` fails the scenario; an unsettled sub-goal makes it not decided. - if any(result.held is False for result in sub_goal_results): + # A settled `False` fails the scenario; an unsettled sub-goal makes it not decided. A worker's own sub-goals are + # reported beside the task's verdict, never folded into it. + if any(result.held is False and result.subject == "task" for result in sub_goal_results): status = "failed" - elif any(result.held is None for result in sub_goal_results): + elif any(result.held is None and result.subject == "task" for result in sub_goal_results): return self._fault( scenario, world_index, diff --git a/src/fi/alk/harness/outbound.py b/src/fi/alk/harness/outbound.py index 75e4675a0..6b473fc32 100644 --- a/src/fi/alk/harness/outbound.py +++ b/src/fi/alk/harness/outbound.py @@ -2463,6 +2463,7 @@ class SubGoalResult(BaseModel): reason: str | None judged: bool verdict: Literal["pass", "fail", "not_applicable", "inconclusive"] | None = None + subject: str | None = None class MetricEvaluation(BaseModel): @@ -2584,8 +2585,9 @@ def _without_unsent_fields(body: dict[str, Any], draft: "ResultReceiptDraft") -> for name in _LATER_CALL_FIELDS - draft.call.model_fields_set: body["call"].pop(name, None) for goal, model in zip(body["sub_goals"], draft.sub_goals): - if "verdict" not in model.model_fields_set: - goal.pop("verdict", None) + for name in ("verdict", "subject"): + if name not in model.model_fields_set: + goal.pop(name, None) if "reason" not in draft.model_fields_set: body.pop("reason", None) diff --git a/src/fi/alk/harness/process_preflight.py b/src/fi/alk/harness/process_preflight.py index 4d5b8f807..331dbd3a3 100644 --- a/src/fi/alk/harness/process_preflight.py +++ b/src/fi/alk/harness/process_preflight.py @@ -110,6 +110,8 @@ def __init__(self, code: str, message: str) -> None: _PLACEHOLDER = re.compile(r"\{\{([^{}]+)\}\}") _FIXED_PLACEHOLDERS = {"JOB_ID", "WORLD_INDEX", "WORLD_DIR", "DB_NAME"} _NAMED_PLACEHOLDER = re.compile(r"^(PORT|HOST)_(.+)$") +# Mirrored from `process_runtime.WORKER_CHANNEL_PLACEHOLDER`, like the port bands below. +_WORKER_CHANNEL_PLACEHOLDER = re.compile(r"^WORKER_CHANNEL_([A-Za-z0-9_.-]+)$") # §2a: Dockerfile-style install lines requiring root privileges have no process-copy equivalent — # the provisioner never runs as root and never will (§0's guest is unprivileged throughout). @@ -459,6 +461,13 @@ def _verify_placeholder_vocabulary(manifest: EnvironmentBundleV2) -> None: token = match.group(1) if token in _FIXED_PLACEHOLDERS: continue + if _WORKER_CHANNEL_PLACEHOLDER.match(token): + if "tool-proxy" in known_names: + continue + raise PreflightError( + "unknown_placeholder", + f"{process.name}.environment.{key}: {{{{{token}}}}} needs this bundle's tool proxy", + ) named = _NAMED_PLACEHOLDER.match(token) if named: _, name = named.groups() diff --git a/src/fi/alk/harness/process_runtime.py b/src/fi/alk/harness/process_runtime.py index 2ba5c7f60..cfad27206 100644 --- a/src/fi/alk/harness/process_runtime.py +++ b/src/fi/alk/harness/process_runtime.py @@ -806,6 +806,27 @@ def configuration_addresses_from_endpoints( _PLACEHOLDER = re.compile(r"\{\{([^{}]+)\}\}") _NAMED_PLACEHOLDER = re.compile(r"^(PORT|HOST)_(.+)$") +# A declared worker's own channel at this world's tool proxy; see `_worker_channel`. +WORKER_CHANNEL_PLACEHOLDER = re.compile(r"^WORKER_CHANNEL_([A-Za-z0-9_.-]+)$") +WORKER_CHANNELS_FILE = "worker-channels.json" + + +def _worker_channel(world_dir: Path, worker: str) -> str: + """This world's channel token for one worker, minted on first use and kept in the tool proxy's world directory, + which the proxy reads to tell workers apart. A sibling of `world_dir`: every process of a world shares a parent.""" + path = world_dir.parent / "tool-proxy" / WORKER_CHANNELS_FILE + try: + channels = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + channels = {} + token = next((key for key, value in channels.items() if value == worker), None) + if token is None: + token = secrets.token_hex(16) + channels[token] = worker + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(channels), encoding="utf-8") + path.chmod(0o644) + return token def render_template( @@ -834,6 +855,9 @@ def resolve(match: re.Match[str]) -> str: return str(world_dir) if token == "DB_NAME": return f"w{world_index}" + channel = WORKER_CHANNEL_PLACEHOLDER.match(token) + if channel: + return _worker_channel(world_dir, channel.group(1)) named = _NAMED_PLACEHOLDER.match(token) if named: kind, name = named.groups() @@ -2318,16 +2342,18 @@ def spawn_source_process( _SIMULATION_MARK_SITECUSTOMIZE, encoding="utf-8" ) sitecustomize_hook.chmod(0o444) - if mark_hook_due: - # The customer's interpreter has no `fi.alk`, so the module sits beside the hook. - mark_module = world_dir / "simulation_mark_bootstrap.py" - if not mark_module.exists(): - mark_module.write_bytes( - Path(__file__) - .with_name("simulation_mark_bootstrap.py") - .read_bytes() - ) - mark_module.chmod(0o444) + if mark_hook_due or process.environment.get("ALK_OBSERVE_FRAMEWORKS") == "1": + # The customer's interpreter has no `fi.alk`, so the modules sit beside the hook. + for module_name in ( + "simulation_mark_bootstrap.py", + "simulation_worker_events.py", + ): + mark_module = world_dir / module_name + if not mark_module.exists(): + mark_module.write_bytes( + Path(__file__).with_name(module_name).read_bytes() + ) + mark_module.chmod(0o444) if process.environment.get("ALK_CREWAI_TOOL_TRACE") == "1": observer = world_dir / "crewai_runtime_observer_bootstrap.py" if not observer.exists(): diff --git a/src/fi/alk/harness/run/simulation.py b/src/fi/alk/harness/run/simulation.py index 52417fab1..c8df3e02e 100644 --- a/src/fi/alk/harness/run/simulation.py +++ b/src/fi/alk/harness/run/simulation.py @@ -36,7 +36,7 @@ from ..contract import AgentContract from ..scenario import Scenario -from ..world.runtime import Call, tool_call_attribution +from ..world.runtime import Call, runtime_ids, tool_call_attribution from .grade import Judgement, Result if TYPE_CHECKING: @@ -997,6 +997,8 @@ def _semantic_calls(path: Path, *, contract: AgentContract | None = None) -> lis refused=failed, error=str(output) if failed else "", **tool_call_attribution(record), + runtime_ids=runtime_ids(record), + handoff=record.get("kind") == "handoff", at=float(record.get("at") or 0.0), ) # A harness-aware worker may mirror a local state-machine action to the world for diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py index 09f923569..83069b51d 100644 --- a/src/fi/alk/harness/scenario.py +++ b/src/fi/alk/harness/scenario.py @@ -324,6 +324,9 @@ def _one_spelling_per_level(cls, coverage: dict[str, str]) -> dict[str, str]: faults: list[Fault] = Field(default_factory=list) max_turns: int = 10 + # How long each of the agent's workers may act in a trial, in seconds; it can only shorten the run's own limit. + # Past it the world refuses that worker's calls, and the worker ends timed out. None: the run's limit alone. + worker_budget_seconds: int | None = Field(default=None, gt=0) # Where this call is made from. A string names the place ("street", "vehicle", "retail"), and # True asks for noise while leaving the place to the fixture. Left unset it is decided from diff --git a/src/fi/alk/harness/scenario_source.py b/src/fi/alk/harness/scenario_source.py index 9ba05f87d..5a8e76bc8 100644 --- a/src/fi/alk/harness/scenario_source.py +++ b/src/fi/alk/harness/scenario_source.py @@ -257,6 +257,12 @@ class _CompiledSubGoal: # The actor this sub-goal is about, if any. actor: str = "" compares: str = "" + # Whose result: the task, or `worker:`, whose check is given only that worker's calls. + subject: str = "task" + + @property + def worker(self) -> str: + return self.subject[len("worker:") :] if self.subject.startswith("worker:") else "" @dataclass(frozen=True) @@ -451,6 +457,7 @@ def _load_catalogue_claims(bundle_dir: Path) -> dict[str, dict[str, str]]: "contract": goal.contract(), "both": "yes" if goal.both else "", "actor": goal.actor, + "subject": goal.subject, } return claims @@ -624,6 +631,7 @@ def combined(goal: _CompiledSubGoal) -> bool: both=combined(goal), compares=claims[goal.name].get("contract", "") or goal.compares, actor=claims[goal.name].get("actor", "") or goal.actor, + subject=claims[goal.name].get("subject") or goal.subject, ) if goal.name in claims else goal diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index 81045f492..ac769c66d 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -1831,7 +1831,9 @@ async def try_check(args: dict[str, Any]) -> dict[str, Any]: + "`overlay` names the overlay level this sub-goal is the claim for, when it is one: " "`prompt_injection`, `social_engineering`, `privacy_pii`. A scenario carrying an overlay " "is refused until it names a sub-goal that fails when that overlay is mishandled, so this " - "is what makes one available. Leave it empty for an ordinary task sub-goal." + "is what makes one available. Leave it empty for an ordinary task sub-goal. " + "`subject` is `task` (the default) or `worker:` for one of the contract's workers: that check is " + "given only that worker's calls, and is inconclusive when the worker made none." + ( " `actor` names the actor this sub-goal is about, by id, when it grades how the agent dealt with that " "actor: it reads not applicable on a call the actor never came into and inconclusive when the actor came " @@ -1847,6 +1849,7 @@ async def try_check(args: dict[str, Any]) -> dict[str, Any]: "judged": str, "output": str, "overlay": str, + "subject": str, **({"actor": str} if actors_enabled() else {}), **CONTRACT_SCHEMA, }, @@ -1854,19 +1857,25 @@ async def try_check(args: dict[str, Any]) -> dict[str, Any]: ), ) async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: - sub_goal = SubGoal( - name=str(args.get("name") or ""), - what=str(args.get("what") or ""), - check=str(args.get("check") or ""), - judged=str(args.get("judged") or ""), - output=str(args.get("output") or "pass_fail"), - overlay=str(args.get("overlay") or ""), - actor=str(args.get("actor") or "") if actors_enabled() else "", - **{field: str(args.get(field) or "") for field in CONTRACT_FIELDS}, - ) + try: + sub_goal = SubGoal( + name=str(args.get("name") or ""), + what=str(args.get("what") or ""), + check=str(args.get("check") or ""), + judged=str(args.get("judged") or ""), + output=str(args.get("output") or "pass_fail"), + overlay=str(args.get("overlay") or ""), + actor=str(args.get("actor") or "") if actors_enabled() else "", + subject=str(args.get("subject") or "task"), + **{field: str(args.get(field) or "") for field in CONTRACT_FIELDS}, + ) + except ValueError as invalid: + return _err(f"Not added: {invalid}") problems = validate_sub_goal(sub_goal) if sub_goal.actor and sub_goal.actor not in load_actors(world_root).by_id(): problems.append(f"actor: {sub_goal.actor!r} is not an actor of this environment; add it with add_actor first") + if sub_goal.worker and sub_goal.worker not in {one.id for one in contract.workers}: + problems.append(f"subject_unknown_worker: {sub_goal.worker} is not one of the contract's workers") if native_runtime(world_root) and sub_goal.deterministic(): if problem := native_source_problem(sub_goal.check, "check"): problems.append(problem) @@ -2123,6 +2132,12 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: "existing names wherever one fits, so results add up across the suite.", }, "max_turns": {"type": "integer"}, + "worker_budget_seconds": { + "type": "integer", + "description": "Only for an agent with workers: how long each worker may act in this " + "trial, in seconds. Past it the world refuses that worker's calls and it ends timed " + "out. Leave out to use the run's own limit.", + }, "coverage": { "type": "object", "additionalProperties": {"type": "string"}, diff --git a/src/fi/alk/harness/simulation_mark_bootstrap.py b/src/fi/alk/harness/simulation_mark_bootstrap.py index cbf7db86f..bb7e2ea01 100644 --- a/src/fi/alk/harness/simulation_mark_bootstrap.py +++ b/src/fi/alk/harness/simulation_mark_bootstrap.py @@ -2,8 +2,9 @@ The sandbox copies this file into a customer process's world directory beside a ``sitecustomize.py`` that calls :func:`install`, so it runs inside the CUSTOMER's interpreter, -which has no ``fi``. It therefore uses the standard library only, imports nothing of -OpenTelemetry itself, and is written for Python 3.9 syntax. +which has no ``fi``. It therefore uses the standard library only and is written for Python 3.9 +syntax. It imports OpenTelemetry only inside a process that already uses it, to give a span asked +for before any provider was set the sandbox's provider. Why it exists: OpenTelemetry's own environment parsing reaches ``Resource.create`` only. Code that builds ``Resource(attributes=...)`` directly, or that names its own project, never sees @@ -21,6 +22,13 @@ the call the sandbox is running at that moment. The stamp rides on the SDK tracer rather than a span processor, because a provider subclass such as traceAI's may drop processors it did not add. +When the sandbox names a tool trace, each tool span the process ends is written to it with the +agent that made the call, read from the agent and tool spans any framework instrumentation marks, +so a multi-agent app is observed without changes to it; ``simulation_worker_events``, beside this file, turns the +spans into trace lines. With the observe switch on, the traceAI +instrumentor the build installed for each framework is turned on when that framework is imported, +unless the process instruments it itself. + It never reads or writes the customer's files and never imports their modules on its own. It does run the next ``sitecustomize`` on the path, which is theirs, so a repository's own startup hook is not lost behind this one; a failure of that hook is reported as Python reports it, never as ours. @@ -32,6 +40,7 @@ import collections.abc import functools +import importlib import importlib.machinery import importlib.util import inspect @@ -69,7 +78,7 @@ def _write(outcome: str) -> None: - """Write the one fixed stderr line. ``failed`` at most once, so a broken merge on every + """Write one fixed-form stderr status line. ``failed`` at most once, so a broken merge on every resource the process builds cannot flood its log.""" global _failed_written try: @@ -361,22 +370,160 @@ def end(*args, **kwargs): span.end = end -def _patch_tracer(module, path: str) -> None: +_TOOL_TRACE = "HARNESS_TOOL_TRACE" + + +def _events(): + """The worker-event module beside this one: a sibling in the customer's interpreter, from the package in ours.""" + name = "simulation_worker_events" + return importlib.import_module(f"{__package__}.{name}" if __package__ else name) + + +def _patch_autogen_tools(module, path) -> None: + _events()._patch_autogen_tools(module, path) + + +_OBSERVE = "ALK_OBSERVE_FRAMEWORKS" +# A framework's top module, and the traceAI instrumentor that marks its agent and tool spans. +# The build installs the instrumentors beside the frameworks it finds installed; one that is +# missing or fails leaves that framework untraced, never the process stopped. +_INSTRUMENTORS = { + "agents": ("traceai_openai_agents", "OpenAIAgentsInstrumentor"), + "langchain_core": ("traceai_langchain", "LangChainInstrumentor"), + "crewai": ("traceai_crewai", "CrewAIInstrumentor"), + "google.adk": ("traceai_google_adk", "GoogleADKInstrumentor"), + "autogen_agentchat": ("traceai_autogen", "AutogenInstrumentor"), + "smolagents": ("traceai_smolagents", "SmolagentsInstrumentor"), + "claude_agent_sdk": ("traceai_claude_agent_sdk", "ClaudeAgentInstrumentor"), + "llama_index.core": ("traceai_llamaindex", "LlamaIndexInstrumentor"), + "pydantic_ai": ("traceai_pydantic_ai", "PydanticAIInstrumentor"), + "strands": ("traceai_strands", "StrandsInstrumentor"), +} + + +@functools.lru_cache(maxsize=1) +def _sandbox_provider(): + """A provider for a process that set none, exporting where the sandbox names.""" + from opentelemetry.sdk.trace import TracerProvider + + provider = TracerProvider() + if os.environ.get(_TRACES_ENDPOINT) or os.environ.get(_GENERIC_ENDPOINT): + try: + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) + except Exception: + pass + return provider + + +def _patch_proxy(module) -> None: + """Serve a span asked for before any provider is set from ours, leaving the process free to set its own.""" + cls = module.ProxyTracer + for method in ("start_span", "start_as_current_span"): + original = getattr(cls, method) + if getattr(original, _PATCHED, False): + continue + + def wrapper(self, *args, __original=original, __method=method, **kwargs): + try: + if getattr(module, "_TRACER_PROVIDER", None) is None: + tracer = _sandbox_provider().get_tracer( + self._instrumenting_module_name + ) + return getattr(tracer, __method)(*args, **kwargs) + except Exception: + pass + return __original(self, *args, **kwargs) + + functools.update_wrapper(wrapper, original) + setattr(wrapper, _PATCHED, True) + setattr(cls, method, wrapper) + + +def _instrument(module, target) -> None: + """Turn on the framework's instrumentor unless the process already has; it reads the global provider per span.""" + name, attribute = target + if name in sys.modules: + # The process imports the instrumentor itself, and this framework import is part of + # that: it instruments with its own settings. + return + try: + instrumentor = getattr(importlib.import_module(name), attribute)() + except ImportError: + # The build could not install it: this framework's calls go unobserved, and the log says so. + _write("unobserved " + module.__name__ + ", its instrumentor is not installed") + return + if getattr(instrumentor, "is_instrumented_by_opentelemetry", False): + return + instrumentor.instrument() + _yield_to_process(type(instrumentor)) + if module.__name__ == "agents": + _keep_agents_processors() + + +def _yield_to_process(cls) -> None: + """Let the process instrument again with its own settings, which the instrumentor would otherwise ignore.""" + original = cls.instrument + if getattr(original, _PATCHED, False): + return + + def instrument(self, *args, __original=original, **kwargs): + if kwargs and getattr(self, "is_instrumented_by_opentelemetry", False): + self.uninstrument() + return __original(self, *args, **kwargs) + + functools.update_wrapper(instrument, original) + setattr(instrument, _PATCHED, True) + cls.instrument = instrument + + +def _keep_agents_processors() -> None: + """Keep our processor when an OpenAI Agents app replaces its trace processors, unless the new list has its kind.""" + from agents.tracing import get_trace_provider, provider + + ours = tuple(get_trace_provider()._multi_processor._processors) + cls = provider.SynchronousMultiTracingProcessor + original = cls.set_processors + + def wrapper(self, processors, __original=original): + kinds = {type(one) for one in processors} + return __original( + self, [*processors, *(one for one in ours if type(one) not in kinds)] + ) + + functools.update_wrapper(wrapper, original) + cls.set_processors = wrapper + + +def _patch_tracer(module, path, tool_trace) -> None: cls = module.Tracer original = cls.start_span if getattr(original, _PATCHED, False): return - call_file = _CallFile(path) + call_file = _CallFile(path) if path else None + tools = None + if tool_trace: + try: + tools = _events()._ToolSpans(tool_trace) + except ImportError: + # The call stamp still lands; only the tool trace goes unwritten. + _write("failed") @functools.wraps(original) def wrapper(self, *args, **kwargs): span = original(self, *args, **kwargs) try: - call = call_file.current() + call = call_file.current() if call_file is not None else None if call and span.is_recording(): span.set_attribute(_CALL_KEY, call) _restamp_at_end(span, call) + if tools is not None and span.is_recording(): + tools.started(span) except Exception: pass return span @@ -455,8 +602,11 @@ def install() -> None: if pairs is not None: _PATCHES[_RESOURCES] = functools.partial(_patch, pairs=pairs) call_file = os.environ.get(_CALL_FILE) - if call_file: - _PATCHES[_TRACE] = functools.partial(_patch_tracer, path=call_file) + tool_trace = os.environ.get(_TOOL_TRACE) + if call_file or tool_trace: + _PATCHES[_TRACE] = functools.partial( + _patch_tracer, path=call_file, tool_trace=tool_trace + ) ingest = None try: ingest = _read_ingest() @@ -467,6 +617,13 @@ def install() -> None: _PATCHES[name] = functools.partial( _patch_exporter, ingest=ingest, lower=lower ) + if os.environ.get(_OBSERVE) == "1" and tool_trace: + _PATCHES["opentelemetry.trace"] = _patch_proxy + for name, target in _INSTRUMENTORS.items(): + _PATCHES.setdefault(name, functools.partial(_instrument, target=target)) + _PATCHES["autogen_core.tools"] = functools.partial( + _patch_autogen_tools, path=tool_trace + ) if not _PATCHES: return # The finder goes in first so that a failed patch of an already imported module still diff --git a/src/fi/alk/harness/simulation_worker_events.py b/src/fi/alk/harness/simulation_worker_events.py new file mode 100644 index 000000000..bd1841bef --- /dev/null +++ b/src/fi/alk/harness/simulation_worker_events.py @@ -0,0 +1,714 @@ +"""Tool calls and the worker that made each, read from the spans a process's framework instrumentation ends. + +The sandbox's startup hook (``simulation_mark_bootstrap``) owns the hook's lifecycle and hands every span its process +starts to :class:`_ToolSpans`; this module turns those spans into the lines of the tool trace. It sits beside the hook +in a customer process's world directory, so it runs inside the CUSTOMER's interpreter, which has no ``fi``: standard +library only, Python 3.9 syntax, and it never imports the customer's modules. +""" + +from __future__ import annotations + +import collections +import functools +import json +import os +import threading +import time + +# The hook's own marker for a callable it already wrapped. +_PATCHED = "_alk_simulation_mark" +_TOOL_END = "_alk_tool_span_end" + +# Where the span kind, the agent's name and a tool call's parts sit, in the conventions the +# framework instrumentors and the frameworks' own tracing use (traceAI and OpenInference, the +# OpenTelemetry GenAI names, then names a single framework uses). +_KIND_KEYS = ("fi.span.kind", "gen_ai.span.kind", "openinference.span.kind") +_OPERATION_KINDS = { + "execute_tool": "TOOL", + "invoke_agent": "AGENT", + "create_agent": "AGENT", +} +_OWN_KINDS = {"tool": "TOOL", "subagent": "AGENT", "agent": "AGENT"} +_AGENT_NAME_KEYS = ( + "gen_ai.agent.name", + "agent.name", + "autogen.agent.name", + "claude_agent.subagent_type", + "graph.node.id", + "recipient_agent_type", + "lc_agent_name", +) +# The conversation a span belongs to; a background job runs as its own conversation, named by the id the launching +# tool call returned. +_CONVERSATION_KEYS = ("gen_ai.conversation.id", "session.id") +# The text of a model call's first output message, as traceAI and OpenInference flatten it. +_SAID_KEYS = ( + "gen_ai.output.messages.0.message.content", + "llm.output_messages.0.message.content", +) + + +_LAUNCHES_MAX = 256 +_GRAPH_NAMESPACE = "langgraph_checkpoint_ns" +_TOOL_NAME_KEYS = ("gen_ai.tool.name", "tool.name") +_TOOL_ARGUMENT_KEYS = ("gen_ai.tool.call.arguments", "tool.parameters", "input.value") +_TOOL_RESULT_KEYS = ("gen_ai.tool.call.result", "output.value") +_LIVE_SPANS_MAX = 4096 +_TRACE_LINE_MAX = 65536 +_PAYLOAD_MAX = 4096 +# A control pass the framework records as one. Google ADK marks its agent-transfer tool by the tool's own class and +# names the agent in its ``agent_name`` argument; the OpenAI Agents SDK's instrumentation names, on the agent that was +# handed control, the agent that handed it. +_TRANSFER_TOOL_TYPES = ("TransferToAgentTool",) +_BUILTIN_TRANSFER = "transfer_to_agent" +_TRANSFER_ARGUMENT_KEYS = ("gcp.vertex.agent.tool_call_args",) + _TOOL_ARGUMENT_KEYS +_HANDED_FROM_KEYS = ("graph.node.parent_id",) + + +def _names(text: str, value: str) -> bool: + """Whether ``value`` appears in ``text`` as a whole id, not inside a longer one.""" + start = text.find(value) + while start != -1: + before = text[start - 1] if start else " " + end = start + len(value) + after = text[end] if end < len(text) else " " + if not any(c.isalnum() or c in "-_" for c in (before, after)): + return True + start = text.find(value, start + 1) + return False + + +def _first(attributes, keys): + for key in keys: + value = attributes.get(key) + if isinstance(value, str) and value: + return value + return None + + +def _kind(span): + attributes = getattr(span, "attributes", None) or {} + value = _first(attributes, _KIND_KEYS) + if value: + return value.upper() + operation = _first(attributes, ("gen_ai.operation.name",)) + if operation in _OPERATION_KINDS: + return _OPERATION_KINDS[operation] + if attributes.get("messaging.operation") == "process" and _first( + attributes, ("recipient_agent_type",) + ): + # AutoGen Core's runtime handing a message to the agent that handles it. + return "AGENT" + return _OWN_KINDS.get(_first(attributes, ("claude_agent.span_kind",)) or "") + + +def _graph_worker(attributes): + """The subgraph a LangGraph node ran in: the namespace segment above that node, or the agent named as running it.""" + namespace = _first(attributes, (_GRAPH_NAMESPACE,)) + if not namespace: + return None + names = [segment.split(":")[0] for segment in namespace.split("|")] + if names and names[-1] == attributes.get("langgraph_node"): + names.pop() + if not names: + return None + return _first(attributes, ("lc_agent_name",)) or names[-1] or None + + +def _hex(span_id) -> str: + return format(span_id, "016x") + + +def _bounded(record, key, value, flag) -> None: + """Keep at most the payload limit of ``value``, and say when some of it was left out.""" + record[key] = value[:_PAYLOAD_MAX] + if len(value) > _PAYLOAD_MAX: + record[flag] = True + + +def _transfer_target(attributes): + """The agent a framework's own transfer tool handed control to, when the span records the call.""" + if attributes.get("gen_ai.tool.type") not in _TRANSFER_TOOL_TYPES: + return None + try: + target = json.loads(_first(attributes, _TRANSFER_ARGUMENT_KEYS) or "").get( + "agent_name" + ) + except (ValueError, AttributeError): + return None + return target if isinstance(target, str) and target else None + + +def _builtin_transfer_target(attributes, name): + """The agent named by a call to ADK's own ``transfer_to_agent`` function, which older releases and other + instrumentations record as a plain function. Only a candidate: it becomes a control pass once that agent starts in + the same agent run, and an ordinary tool call if the run ends first.""" + if name != _BUILTIN_TRANSFER: + return None + try: + target = json.loads(_first(attributes, _TRANSFER_ARGUMENT_KEYS) or "").get( + "agent_name" + ) + except (ValueError, AttributeError): + return None + return target if isinstance(target, str) and target else None + + +def _same_tool(one, other): + """Whether two spans name the same tool, as two instrumentations of one call spell it.""" + + def key(span): + name = _first(span.attributes or {}, _TOOL_NAME_KEYS) or span.name or "" + return "".join(ch for ch in name.lower() if ch.isalnum()) + + return key(one) == key(other) != "" + + +class _ToolSpans: + """Writes each finished tool span to the tool trace, with the nearest agent span above it as its worker.""" + + def __init__(self, path: str) -> None: + self._path = path + self._live = {} + self._handed = {} + self._wrapped = set() + # Tool calls that returned something, oldest first, so a job started by one can be joined to it. + self._launches = collections.deque(maxlen=_LAUNCHES_MAX) + # Runs with no parent not yet joined to a launch, with the attributes last seen in them: a run's own span is + # often named only when it ends, so the spans inside it name it first. + self._waiting = {} + # What a model last said in each open run, read as a job's answer when the run ends. + self._said = {} + # Job runs whose hand-off was written: the calls inside one ran in that hand-off. + self._jobs = set() + # Transfers a framework recorded, by the agent run they were made in, until the agent they name starts in it; + # then each agent run handed control, with the event id of the transfer that handed it. + self._transfers = {} + self._transferred = {} + # Calls to ADK's built-in transfer function by the run they were made in, held until the agent they name + # starts there (a control pass) or the run ends (a plain tool call). + self._candidates = {} + # The first agent run opened directly inside each open agent span. + self._opened = {} + # Spans that ended, so a parent never seen can be told from one that already closed. + self._closed = collections.OrderedDict() + self._appended = False + # Spans start and end on the process's own threads. + self._lock = threading.RLock() + + def started(self, span) -> None: + context = span.get_span_context() + parent = getattr(span, "parent", None) + with self._lock: + if len(self._live) >= _LIVE_SPANS_MAX: + # A span that never ended (a dropped generator, a long-lived server) gives way to new ones. + self._live.pop(next(iter(self._live))) + self._live[context.span_id] = ( + span, + parent.span_id if parent is not None else None, + ) + if _kind(span) == "AGENT": + # Through the framework's own turn and run spans, to the agent span this run opened in. + holder = self._above(context.span_id, ("AGENT",)) + if holder is not None: + self._opened.setdefault( + holder.get_span_context().span_id, context.span_id + ) + if parent is None and _kind(span) != "TOOL": + try: + joined = self._job_record(span, context.span_id, ended=False) + if not joined and len(self._waiting) < _LAUNCHES_MAX: + self._waiting[context.span_id] = (span, None) + except Exception: + pass + original = span.end + if getattr(original, _TOOL_END, False): + return + + @functools.wraps(original) + def end(*args, **kwargs): + result = original(*args, **kwargs) + try: + with self._lock: + self._ended(span) + except Exception: + pass + return result + + setattr(end, _TOOL_END, True) + span.end = end + + def _above(self, span_id, kinds): + """The nearest open ancestor of one of ``kinds``, stopping at an agent span.""" + entry = self._live.get(span_id) + parent = entry[1] if entry else None + while parent is not None: + above = self._live.get(parent) + if above is None: + return None + kind = _kind(above[0]) + if kind in kinds: + return above[0] + if kind == "AGENT": + return None + parent = above[1] + return None + + def _root(self, span_id): + """The run with no parent that ``span_id`` sits in, if it is still open.""" + entry = self._live.get(span_id) + while entry is not None and entry[1] is not None: + span_id, entry = entry[1], self._live.get(entry[1]) + return span_id if entry is not None else None + + def _run_name(self, span_id): + root = self._root(span_id) + return self._live[root][0].name or None if root not in (None, span_id) else None + + def _join_waiting(self, root_id, attributes=None) -> None: + waiting = self._waiting.get(root_id) + if waiting is None: + return + seen = attributes or waiting[1] + if self._job_record(waiting[0], root_id, ended=False, attributes=seen): + self._waiting.pop(root_id, None) + elif attributes: + self._waiting[root_id] = (waiting[0], attributes) + + def _named_above(self, span_id): + """The agent the nearest open span from ``span_id`` up names as running it.""" + while span_id is not None: + entry = self._live.get(span_id) + if entry is None: + return None + name = _first(entry[0].attributes or {}, ("lc_agent_name",)) + if name: + return name + span_id = entry[1] + return None + + def _delegation(self, span_id): + """The hand-off ``span_id`` ran in, read from the span tree only: ``(event id, "observed")``, ``("", + "observed")`` under an agent that nothing handed work to, and ``("", "unknown")`` when the tree does not say.""" + agent = None + entry = self._live.get(span_id) + parent = entry[1] if entry else None + while parent is not None: + above = self._live.get(parent) + if above is None: + # A parent the app's tracer never recorded (the caller's context) is outside the app: the agent + # below it is the top of the app's own work. One that already closed says nothing. + outside = agent is not None and parent not in self._closed + return "", "observed" if outside else "unknown" + if parent in self._jobs: + return _hex(parent), "observed" + kind = _kind(above[0]) + if ( + agent is not None + and kind not in ("AGENT", "TOOL") + and (parent in self._transfers or parent in self._candidates) + ): + # A transfer made by an agent that has since ended, in the run this agent started in. + handed = self._transfer_into(parent, agent) + if handed: + return handed, "observed" + if kind == "AGENT": + if ( + agent is not None + and self._opened.get(parent) == agent + and self._container(above) + ): + # The framework's trace container (the OpenAI Agents SDK opens one per run as an agent span with + # no agent name): the first agent run inside it is the top of the app's work. A later one was + # handed control and stays unknown until the framework records the transfer. + return "", "observed" + if agent is not None: + handed = self._transfer_into(parent, agent) + return (handed, "observed") if handed else ("", "unknown") + agent = parent + elif kind == "TOOL" and agent is not None: + return _hex(parent), "observed" + elif kind != "TOOL" and above[1] in self._live: + name = _first(above[0].attributes or {}, ("lc_agent_name",)) + handing = self._live[above[1]][0] + if ( + name + and name != self._named_above(above[1]) + and _kind(handing) in ("TOOL", None) + ): + return _hex(above[1]), "observed" + parent = above[1] + return "", "observed" if agent is not None else "unknown" + + def _container(self, entry) -> bool: + """An agent-kind span at the top of the app's trace that names no agent: a run's container, not an agent.""" + above = entry[1] + top = above is None or (above not in self._live and above not in self._closed) + return top and not _first(entry[0].attributes or {}, _AGENT_NAME_KEYS) + + def _transfer_into(self, scope, agent): + """The transfer made in the run ``scope`` that names the agent run ``agent`` started under it.""" + if agent not in self._transferred: + name = self._agent_name(self._live[agent][0]) + self._confirm_above(agent, name) + pending = self._transfers.get(scope) or [] + match = next((one for one in pending if one[1] == name), None) + if match is None: + return None + pending.remove(match) + self._transferred[agent] = match[0] + return self._transferred[agent] + + def _confirm_transfer(self, scope, name) -> None: + """Write the held built-in transfer in ``scope`` that names ``name`` as a control pass, now that it ran.""" + held = self._candidates.get(scope) or [] + match = next((one for one in held if one[1] == name), None) or next( + (one for one in held if one[1] is None), None + ) + if match is None: + return False + held.remove(match) + record, target, _ = match + target = target or name + record.update(transfer=True, delegated_to=target[:256]) + record.pop("ended_at", None) + pending = self._transfers.get(scope, []) + if len(self._transfers) + len(pending) < _LAUNCHES_MAX: + self._transfers[scope] = pending + [(record["event_id"], target)] + with self._lock: + _append(self._path, record) + self._appended = True + return True + + def _confirm_above(self, span_id, name) -> None: + """Confirm a held transfer for the agent run ``span_id`` named ``name``, in the spans it opened inside, up to + the nearest agent; an agent started by a tool call was handed work by that call, not control.""" + entry = self._live.get(span_id) + parent = entry[1] if entry else None + while parent is not None and parent in self._live: + if parent in self._candidates and self._confirm_transfer(parent, name): + return + if _kind(self._live[parent][0]) in ("AGENT", "TOOL"): + return + parent = self._live[parent][1] + + def _settle_candidates(self, scope) -> None: + """A run ended without starting the agent a held transfer named: an ordinary tool call, or the framework's + transfer with no agent recorded as taking over.""" + for record, _, framework in self._candidates.pop(scope, None) or []: + if framework: + record.update(transfer=True) + record.pop("ended_at", None) + with self._lock: + _append(self._path, record) + self._appended = True + + def _agent_name(self, agent): + """An agent span's own name, else the tool that handed it the work; never a method-like span name.""" + name = _first(agent.attributes or {}, _AGENT_NAME_KEYS) + if name: + return name + if ( + _first(agent.attributes or {}, _KIND_KEYS) + and agent.name + and "." not in agent.name + ): + return agent.name + handing = self._above(agent.get_span_context().span_id, ("TOOL",)) + if handing is not None: + return _first(handing.attributes or {}, _TOOL_NAME_KEYS) or handing.name + return None + + def _ended(self, span) -> None: + span_id = span.get_span_context().span_id + kind = _kind(span) + try: + entry = self._live.get(span_id) + said = _first(span.attributes or {}, _SAID_KEYS) + if said and said.strip() and entry is not None and entry[1] is not None: + root = self._root(span_id) + if root is not None and ( + root in self._said or len(self._said) < _LAUNCHES_MAX + ): + self._said[root] = said + if entry is not None and entry[1] is None and kind != "TOOL": + self._waiting.pop(span_id, None) + self._job_record(span, span_id, ended=True) + self._said.pop(span_id, None) + elif self._waiting: + self._join_waiting(self._root(span_id), span.attributes) + if kind == "AGENT": + handing = self._above(span_id, ("TOOL",)) + name = self._agent_name(span) + if name: + self._confirm_above(span_id, name) + if handing is not None and name: + self._handed[handing.get_span_context().span_id] = name + handed_from = _first(span.attributes or {}, _HANDED_FROM_KEYS) + if handed_from and name and handed_from != name: + self._transfer_record(span, span_id, handed_from, name) + elif kind != "TOOL" and entry is not None and entry[1] in self._live: + # A run opened right under a span by an agent other than the one running that span is the span + # handing it the work, as a sub-agent started through a tool is. A tool's kind is often set only + # when it ends, so the hand-off is kept on the parent and written if the parent is a tool call. + name = _first(span.attributes or {}, ("lc_agent_name",)) + if name and name != self._named_above(entry[1]): + self._handed[entry[1]] = name + elif kind == "TOOL": + if span_id in self._wrapped: + # Two instrumentations wrapped this one call; the inner span was written. + self._wrapped.discard(span_id) + self._handed.pop(span_id, None) + else: + outer = self._above(span_id, ("TOOL",)) + if outer is not None and _same_tool(outer, span): + self._wrapped.add(outer.get_span_context().span_id) + self._write(span, span_id) + finally: + above = entry[1] if entry is not None else None + if kind == "AGENT" and above in self._live: + # A transfer outlives the agent that made it: the agent it names may start after it, in the same run. + for held in (self._candidates, self._transfers): + moved = held.pop(span_id, None) + if moved: + held[above] = held.get(above, []) + moved + if span_id in self._candidates: + self._settle_candidates(span_id) + self._opened.pop(span_id, None) + self._closed[span_id] = None + if len(self._closed) > _LIVE_SPANS_MAX: + self._closed.popitem(last=False) + if kind != "TOOL": + self._handed.pop(span_id, None) + for one in (self._transfers, self._transferred): + one.pop(span_id, None) + self._jobs.discard(span_id) + self._live.pop(span_id, None) + + def _forget_cleared_call(self) -> None: + """Forget an earlier call's launches once the harness removes the trace, as it does between calls.""" + if self._appended and not os.path.exists(self._path): + self._launches.clear() + self._waiting.clear() + self._jobs.clear() + self._appended = False + + def _job_record(self, span, span_id, *, ended: bool, attributes=None) -> bool: + """Write a parentless run whose conversation id an earlier tool call returned as that call's hand-off.""" + attributes = attributes or span.attributes or {} + conversation = _first(attributes, _CONVERSATION_KEYS) + if not conversation: + return False + self._forget_cleared_call() + launch = next( + ( + one + for one in self._launches + if one["conversation"] != conversation + and _names(one["output"], conversation) + ), + None, + ) + if launch is None: + return False + worker = ( + _first(attributes, _AGENT_NAME_KEYS) + or _graph_worker(attributes) + or span.name + ) + if not worker: + return False + status = getattr(getattr(span, "status", None), "status_code", None) + record = { + "name": launch["name"], + "arguments": launch["arguments"], + "is_error": ended and getattr(status, "name", "") == "ERROR", + "at": (span.start_time or 0) / 1e9, + **({"ended_at": (span.end_time or 0) / 1e9} if ended else {}), + "source": "otel", + "event_id": _hex(span_id), + "parent_event_id": launch["event_id"], + # The job's hand-off sits where the call that launched it did. + "parent_span_id": launch["parent_span_id"], + "relationship": launch["relationship"], + "delegated_to": worker[:256], + } + if launch.get("input_omitted"): + record["input_omitted"] = True + said = self._said.get(span_id) or _first(attributes, _TOOL_RESULT_KEYS) or "" + _bounded(record, "output", said if ended else "", "output_omitted") + if launch["agent"]: + record.update(agent_key=launch["agent"], agent_role=launch["agent"]) + self._jobs.add(span_id) + with self._lock: + _append(self._path, record) + self._appended = True + return True + + def _transfer_record(self, span, span_id, handed_from, name) -> None: + """Write an agent run the framework records as handed control by another agent, as that control pass. The + tree does not say where the agent that passed control ran, so the pass's own place is unknown.""" + record = { + "name": name[:256], + "arguments": "", + "output": "", + "is_error": False, + "at": (span.start_time or 0) / 1e9, + "source": "otel", + "event_id": _hex(span_id), + "parent_span_id": "", + "relationship": "unknown", + "agent_key": handed_from[:256], + "agent_role": handed_from[:256], + "delegated_to": name[:256], + "transfer": True, + # A worker's run, not a tool call: kept, never graded. + "kind": "handoff", + } + self._forget_cleared_call() + _append(self._path, record) + self._appended = True + + def _write(self, span, span_id) -> None: + attributes = span.attributes or {} + name = _first(attributes, _TOOL_NAME_KEYS) or span.name + if not name: + return + agent = self._above(span_id, ("AGENT",)) + status = getattr(getattr(span, "status", None), "status_code", None) + parent_span_id, relationship = self._delegation(span_id) + record = { + "name": name[:256], + "is_error": getattr(status, "name", "") == "ERROR", + "at": (span.start_time or 0) / 1e9, + "ended_at": (span.end_time or 0) / 1e9, + "source": "otel", + "event_id": _hex(span_id), + "parent_span_id": parent_span_id, + "relationship": relationship, + } + _bounded( + record, + "arguments", + _first(attributes, _TOOL_ARGUMENT_KEYS) or "", + "input_omitted", + ) + _bounded( + record, + "output", + _first(attributes, _TOOL_RESULT_KEYS) or "", + "output_omitted", + ) + worker = ( + _graph_worker(attributes) + or _first(attributes, ("gen_ai.agent.name",)) + or (self._agent_name(agent) if agent is not None else None) + # No agent span above it: the agent the span itself names, as a job running in its own run does. + or _first(attributes, _AGENT_NAME_KEYS) + ) + if worker: + record.update(agent_key=worker[:256], agent_role=worker[:256]) + if agent is not None: + record["parent_event_id"] = _hex(agent.get_span_context().span_id) + delegated = self._handed.pop(span_id, None) + if delegated: + record["delegated_to"] = delegated[:256] + target = _transfer_target(attributes) + # The framework's own transfer tool with no recorded target hands control to the next agent run in this run. + framework = attributes.get("gen_ai.tool.type") in _TRANSFER_TOOL_TYPES + candidate = ( + None if target or framework else _builtin_transfer_target(attributes, name) + ) + if ( + (candidate or (framework and not target)) + and agent is not None + and len(self._candidates) < _LAUNCHES_MAX + ): + scope = agent.get_span_context().span_id + self._candidates.setdefault(scope, []).append( + (record, candidate, framework) + ) + return + if target: + # Control passed to the agent the framework named; nothing returns from a control pass. + record.update(transfer=True, delegated_to=target[:256]) + record.pop("ended_at") + scope = agent.get_span_context().span_id if agent is not None else None + pending = self._transfers.get(scope, []) + if ( + scope is not None + and len(self._transfers) + len(pending) < _LAUNCHES_MAX + ): + self._transfers[scope] = pending + [(record["event_id"], target)] + self._forget_cleared_call() + if record["output"]: + self._launches.append( + { + "output": record["output"], + "name": record["name"], + "arguments": record["arguments"], + "event_id": record["event_id"], + "parent_span_id": parent_span_id, + "relationship": relationship, + "input_omitted": record.get("input_omitted", False), + # No agent names the launching call: the run it sits in, which a graph server names by the graph. + "agent": record.get("agent_key") or self._run_name(span_id), + "conversation": _first(attributes, _CONVERSATION_KEYS), + } + ) + with self._lock: + _append(self._path, record) + self._appended = True + for job_id in list(self._waiting): + self._join_waiting(job_id) + + +def _append(path, record) -> None: + line = json.dumps(record, sort_keys=True, default=str) + if len(line) > _TRACE_LINE_MAX: + record.update( + arguments="", + output="", + input_omitted=True, + output_omitted=True, + ) + line = json.dumps(record, sort_keys=True, default=str) + with open(path, "a", encoding="utf-8") as trace: + trace.write(line + "\n") + + +def _patch_autogen_tools(module, path) -> None: + """Write each AutoGen tool call with its arguments and result, which AutoGen's tool spans do not carry.""" + cls = module.BaseTool + original = cls.run_json + if getattr(original, _PATCHED, False): + return + + async def run_json( + self, args, cancellation_token, call_id=None, __original=original + ): + record = { + "name": str(self.name)[:256], + "arguments": dict(args), + "at": time.time(), + } + try: + result = await __original(self, args, cancellation_token, call_id) + except Exception as err: + _record_tool(path, record, str(err), True) + raise + _record_tool(path, record, self.return_value_as_string(result), False) + return result + + functools.update_wrapper(run_json, original) + setattr(run_json, _PATCHED, True) + cls.run_json = run_json + + +def _record_tool(path, record, output, is_error) -> None: + try: + line = {**record, "is_error": is_error} + _bounded(line, "output", output, "output_omitted") + _append(path, line) + except Exception: + pass diff --git a/src/fi/alk/harness/skills/understand-agent/SKILL.md b/src/fi/alk/harness/skills/understand-agent/SKILL.md index 27a955e1b..b866449b5 100644 --- a/src/fi/alk/harness/skills/understand-agent/SKILL.md +++ b/src/fi/alk/harness/skills/understand-agent/SKILL.md @@ -60,6 +60,14 @@ Find, in roughly this order: schema, a tool array. Record the exact callable name the model would emit, not a friendly label. + If the agent hands work to other agents (a sub-agent called as a tool, a hand-off, graph nodes, + a crew), record each one in `workers`: an `id`, the `name` the app gives it, the contract tools + it holds, and `binding: separate` when its tools are reached through code of their own rather + than code the agents share, with `address_env` naming the environment variable that code reads + for its tools' base address. When the repository says a worker may use fewer tools than its code + holds, list those in `allowed_tools`. Every worker tool is also in `tools`. If the repository + carries a multi-agent reference manifest, read its topology. + 2. **Argument names and types.** Read the signature. An argument declared as a list is a different tool from one declared as a single value, and an environment built on the wrong one fails at the first call. Record types wherever the source states them. diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 9abaf0d8f..d6c15666e 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -692,6 +692,19 @@ to react lets the person answer before anything is said, and the test is gone; s same reason an actor never voices the person's own request or details: the agent would no longer have to tell the person it serves from someone else. Most scenarios include none. +### When the agent hands work to workers + +Only when the contract lists `workers`. The task's sub-goals stay as they are: they grade what the +person got. Add, for each worker whose work the world can show, a sub-goal with `subject: +worker:`; its check is handed only that worker's calls, and is inconclusive when the worker made +none. Where a worker may use fewer tools than its code holds (`allowed_tools`), write at least one +scenario in which the natural request leads the agent toward a tool that worker may not use, and a +worker sub-goal that holds only when no such write reached the world (a refused call is in `calls` +with `.refused`). The world refuses that call, so the answer key never contains it: the record stays as +it was, and the correct reply tells the person it could not be done. A check that a worker did not do something holds only when that worker made calls +in this trial; with none it is inconclusive, never a pass. The coordinator is graded against the world +and what the person was told, never against what a worker reported. + ## File prerequisites must exist in the target workspace Read the workspace bindings in the world summary before writing file tasks. A repository used diff --git a/src/fi/alk/harness/tool_trace_proxy.py b/src/fi/alk/harness/tool_trace_proxy.py index b9e287723..ce4409cbd 100644 --- a/src/fi/alk/harness/tool_trace_proxy.py +++ b/src/fi/alk/harness/tool_trace_proxy.py @@ -17,6 +17,15 @@ DATABASE_URL = os.environ.get("DATABASE_URL", "") # This trial's actor token; a request naming an actor without it is the agent's. ACTOR_PROOF_FILE = os.environ.get("ACTOR_PROOF_FILE", "") +# This trial's worker channels, {token: worker id}: a worker reaches the world under /w//. +WORKER_CHANNELS_FILE = os.environ.get("WORKER_CHANNELS_FILE", "") +# The sealed per-worker allow-lists beside this file, {"worker:": {"allow": [...]}}; a bundle without it does not +# limit workers. +WORKER_PERMISSIONS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "permissions.json") +# The tool each world action is, {"orders.get": "get_order"}, sealed beside this file; a route without one is named by +# its last path segment. +ACTIONS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "actions.json") +_CHANNEL_PREFIX = "/w/" # One request per idempotency key at a time, so a retry racing the first never reaches the world twice. @@ -57,6 +66,15 @@ def _logs_actors() -> bool: return _LOG_SHAPE["actors"] +def _read_json(path: str) -> dict | None: + try: + with open(path, encoding="utf-8") as handle: + loaded = json.load(handle) + except (OSError, ValueError): + return None + return loaded if isinstance(loaded, dict) else None + + def _trial_token() -> str: try: with open(ACTOR_PROOF_FILE, encoding="utf-8") as handle: @@ -65,16 +83,17 @@ def _trial_token() -> str: return "" -def _trial_faults() -> list[dict]: - """This trial's faults on the agent's calls, written by the runner beside the token; never from a request.""" +def _trial_state() -> dict: + """This trial's state, written by the runner beside the token; never from a request.""" if not ACTOR_PROOF_FILE: - return [] - try: - with open(os.path.join(os.path.dirname(ACTOR_PROOF_FILE), "trial.json"), encoding="utf-8") as handle: - faults = json.load(handle).get("faults") or [] - except (OSError, ValueError, AttributeError): - return [] - return [one for one in faults if isinstance(one, dict)] + return {} + return _read_json(os.path.join(os.path.dirname(ACTOR_PROOF_FILE), "trial.json")) or {} + + +def _trial_faults() -> list[dict]: + """This trial's faults on the agent's calls.""" + faults = _trial_state().get("faults") or [] + return [one for one in faults if isinstance(one, dict)] if isinstance(faults, list) else [] def _agent_calls(name: str) -> int: @@ -99,6 +118,7 @@ def _record( refusal_code: str | None = None, idempotency_key: str | None = None, actor_id: str = "agent", + worker_id: str | None = None, parent_event_id: str | None = None, on_behalf_of: str | None = None, ) -> None: @@ -113,14 +133,14 @@ def _record( ) try: extended = _logs_actors() - if not extended and actor_id != "agent": + if not extended and (actor_id != "agent" or worker_id): return with psycopg.connect(DATABASE_URL, autocommit=True) as connection: if extended: connection.execute( "INSERT INTO _alk_tool_trace(name, arguments, result, ok, error, at, action_id, " - "outcome, refusal, refusal_code, idempotency_key, actor_id, parent_event_id, on_behalf_of) " - "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + "outcome, refusal, refusal_code, idempotency_key, actor_id, worker_id, parent_event_id, " + "on_behalf_of) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", ( *row, uuid.uuid4().hex, @@ -129,6 +149,7 @@ def _record( refusal_code, idempotency_key, actor_id, + worker_id, parent_event_id, on_behalf_of, ), @@ -207,10 +228,31 @@ def _forward(self) -> None: arguments = {"_raw": body.decode("utf-8", errors="replace")} self._as: dict[str, str | None] = { "actor_id": "agent", + "worker_id": None, "parent_event_id": (self.headers.get("x-alk-parent-event") or "").strip() or None, "on_behalf_of": None, } + if self.path.startswith(_CHANNEL_PREFIX): + token, _, rest = self.path[len(_CHANNEL_PREFIX) :].partition("/") + # The token is the channel's credential: it is stripped here and never logged or forwarded. + self.path = "/" + rest + worker = ((_read_json(WORKER_CHANNELS_FILE) or {}).get(token) or "") if token else "" + if not worker: + self._refuse(arguments, "identity_conflict", "this channel does not belong to a worker of this trial") + return + self._as["worker_id"] = worker + deadline = _trial_state().get("worker_deadline_at") + if isinstance(deadline, (int, float)) and time.time() > deadline: + self._refuse(arguments, "deadline_passed", f"the deadline for {worker} has passed") + return + if os.path.exists(WORKER_PERMISSIONS_FILE): + # Present and unreadable denies: a broken seal never widens what a worker may do. A worker is + # checked as an actor is, through permitted(), and refused into the same action log. + allowed = ((_read_json(WORKER_PERMISSIONS_FILE) or {}).get(f"worker:{worker}") or {}).get("allow") + if not permitted(allowed, self._action()): + self._refuse(arguments, "not_permitted", f"{self._action()} is not permitted for {worker}") + return actor = (self.headers.get("x-alk-actor") or "").strip() if actor: token = _trial_token() @@ -327,7 +369,8 @@ def _faulted(self, body: bytes, arguments: object, fault: dict) -> None: self._reply(504, {"detail": "tool_upstream_timeout"}) def _name(self) -> str: - return self.path.split("?", 1)[0].rstrip("/").rsplit("/", 1)[-1] or "unknown" + named = (_read_json(ACTIONS_FILE) or {}).get(self._action()) + return str(named) if named else self.path.split("?", 1)[0].rstrip("/").rsplit("/", 1)[-1] or "unknown" def _send(self, body: bytes, arguments: object, key: str | None) -> None: outgoing = request.Request( diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index a29f9f60e..4bad4f748 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -424,6 +424,26 @@ def contract_tools( # is where it is caught, with an explanation. }, }, + "workers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "tools": {"type": "array", "items": {"type": "string"}}, + "allowed_tools": {"type": "array", "items": {"type": "string"}}, + "binding": {"type": "string", "enum": ["separate", "shared"]}, + "address_env": {"type": "string"}, + }, + "required": ["id"], + }, + "description": "Agents this one hands work to, if any: an id no other worker has, the name the app " + "gives the agent (its spans carry it), the names of the tools above that it holds, " + "separate when its tools run through code of their own, with address_env naming the " + "environment variable that code reads for its tools' base address, and allowed_tools when it " + "may use fewer tools than its code holds. Leave out for an agent that hands no work on.", + }, "data_schema": { "type": "object", "description": "The shape of the records the agent works on: which fields " diff --git a/src/fi/alk/harness/world/runtime.py b/src/fi/alk/harness/world/runtime.py index 0c2b625e3..78d1c8734 100644 --- a/src/fi/alk/harness/world/runtime.py +++ b/src/fi/alk/harness/world/runtime.py @@ -156,9 +156,27 @@ class Call: worker_label: str = "" task_id: str = "" task_label: str = "" - - -def tool_call_attribution(record: Mapping[str, Any]) -> dict[str, str]: + # The worker this call handed work to, when the call is a hand-off, and when that worker returned. + delegated_to: str = "" + ended_at: float = 0.0 + # The runtime's own ids for the event (`event_id`, `parent_event_id`, `started_event_id`), as it reported them. + runtime_ids: dict[str, str] = field(default_factory=dict) + # A worker's run the runtime reported in place of a tool call: kept in the trace, never graded. + handoff: bool = False + # The hand-off this call ran in, by its event id, as the runtime's own record links them: "observed" when it + # did, "unknown" when it gave no evidence. Never inferred from tool names or event order. + parent_span_id: str = "" + relationship: str = "" + # A control pass the framework recorded as one: the agent it names takes over, and nothing returns. + transferred: bool = False + # Part of the arguments or the result was left out to keep the record bounded. + input_omitted: bool = False + output_omitted: bool = False + # Why the world refused it, as its action log coded it (`not_permitted`, `deadline_passed`); empty otherwise. + refusal_code: str = "" + + +def tool_call_attribution(record: Mapping[str, Any]) -> dict[str, Any]: def text(key: str) -> str: value = record.get(key) return value if isinstance(value, str) else "" @@ -168,6 +186,99 @@ def text(key: str) -> str: "worker_label": text("agent_role"), "task_id": text("task_id"), "task_label": text("task_name"), + "delegated_to": text("delegated_to"), + "parent_span_id": text("parent_span_id"), + "relationship": "observed" if text("relationship") == "observed" else "", + **{ + key: record.get(field) is True + for key, field in ( + ("transferred", "transfer"), + ("input_omitted", "input_omitted"), + ("output_omitted", "output_omitted"), + ) + }, + } + + +_RUNTIME_ID_FIELDS = ("event_id", "parent_event_id", "started_event_id") + + +def runtime_ids(record: dict[str, Any]) -> dict[str, str]: + """The ids a runtime's own event carries on a traced call, kept as it wrote them.""" + return {key: str(record[key]) for key in _RUNTIME_ID_FIELDS if record.get(key)} + + +def graded_calls(calls: Sequence[Call]) -> list[Call]: + """The calls checks grade: no worker's run, and no job's hand-off repeating the call that launched it.""" + plain = {call.name for call in calls if not call.delegated_to} + return [ + call + for call in calls + if not call.handoff + and not ( + call.delegated_to + and call.runtime_ids.get("parent_event_id") + and call.name in plain + ) + ] + + +def _worker_event(call: Call) -> dict[str, Any]: + """Where a call sits among the app's agents, as the runtime recorded it, for a call any runtime attributed.""" + if not ( + call.runtime_ids + or call.worker_id + or call.worker_label + or call.delegated_to + or call.relationship + ): + return {} + kind = "tool_call" + if call.transferred: + kind = "transferred" + elif call.delegated_to: + kind = "delegated" + observed = call.relationship == "observed" + return { + "span_id": call.runtime_ids.get("event_id", ""), + "parent_span_id": call.parent_span_id if observed else "", + "kind": kind, + "agent": call.worker_id or call.worker_label, + "relationship": "observed" if observed else "unknown", + } + + +def trace_row(call: Call) -> dict[str, Any]: + """One call as the sealed tool trace keeps it, with who in the agent made it when the runtime said.""" + return { + "name": call.name, + "arguments": call.arguments, + "result": call.result, + "ok": call.ok, + "error": call.error, + "refused": call.refused, + "at": call.at, + **{ + key: getattr(call, key) + for key in ( + "worker_id", + "worker_label", + "task_id", + "task_label", + "delegated_to", + "ended_at", + ) + if getattr(call, key) + }, + **({"runtime_ids": call.runtime_ids} if call.runtime_ids else {}), + # Why the world refused it, as its action log coded it. + **({"refusal_code": call.refusal_code} if call.refusal_code else {}), + **{ + key: True + for key in ("input_omitted", "output_omitted") + if getattr(call, key) + }, + **_worker_event(call), } diff --git a/src/fi/simulate/agent/wrappers/http.py b/src/fi/simulate/agent/wrappers/http.py index 7a8f03458..c69a27343 100644 --- a/src/fi/simulate/agent/wrappers/http.py +++ b/src/fi/simulate/agent/wrappers/http.py @@ -967,7 +967,7 @@ def _read_response(self, response): def _agent_response_from_payload(self, payload: Any) -> AgentResponse: if self.protocol == "json_template": - selected = _select_response_value(payload, self.response_path) + selected = _select_response_value(payload, self.response_path, text=True) if isinstance(selected, Mapping): raise ValueError( "response_path must select chat content, not a JSON object" @@ -1109,6 +1109,23 @@ def replacement(match: re.Match[str]) -> str: return _PLACEHOLDER.sub(replacement, value) +class _StreamedEvents(list): + """The events of a streamed reply (server-sent events or JSON lines), in the order sent.""" + + +def _streamed_text(pieces: list[str]) -> str: + """Deltas join; a stream whose pieces each strictly extend the last, or end on the whole text, keeps its last.""" + if pieces[-1] == "".join(pieces[:-1]): + return pieces[-1] + cumulative = pieces[:-1] if len(pieces) > 2 and pieces[-1] == pieces[-2] else pieces + if all( + len(later) > len(earlier) and later.startswith(earlier) + for earlier, later in zip(cumulative, cumulative[1:]) + ): + return pieces[-1] + return "".join(pieces) + + def _decode_json_or_sse(text: str) -> Any: try: return json.loads(text) @@ -1125,9 +1142,16 @@ def _decode_json_or_sse(text: str) -> Any: except json.JSONDecodeError: events.append(data) if events: - return events + return _StreamedEvents(events) + # Newline-delimited JSON, one object per line, as some agent servers stream. + lines = [line for line in text.splitlines() if line.strip()] + if lines: + try: + return _StreamedEvents(json.loads(line) for line in lines) + except json.JSONDecodeError: + pass raise ValueError( - "HTTP target returned neither JSON nor JSON server-sent events" + "HTTP target returned neither JSON, JSON server-sent events nor JSON lines" ) @@ -1156,7 +1180,8 @@ def _tool_result_error(result: Any, path: str) -> bool: raise ValueError("tool result error must be null, boolean or string") -def _select_response_value(payload: Any, path: str) -> Any: +def _select_response_value(payload: Any, path: str, *, text: bool = False) -> Any: + """The value at ``path``; with ``text``, a streamed reply's pieces are its reply text, not its latest event.""" if not path: return payload segments = [raw for raw in path.removeprefix("$").strip(".").split(".") if raw] @@ -1177,7 +1202,19 @@ def select(current: Any, offset: int) -> Any: # response schema describes one event. A path such as ``content.parts.0.text`` # is therefore still meaningful: select the latest event matching that shape. # This is envelope-driven rather than framework-specific and leaves explicit - # numeric paths (including negative indexes) unchanged. + # numeric paths (including negative indexes) unchanged. A streamed reply's + # text is spread over its events instead. + if text and isinstance(current, _StreamedEvents): + pieces = [] + for item in current: + try: + piece = select(item, offset) + except ValueError: + continue + if piece is not None: + pieces.append(piece) + if pieces and all(isinstance(piece, str) for piece in pieces): + return _streamed_text(pieces) last_error: ValueError | None = None for item in reversed(current): try: diff --git a/tests/harness/test_actor_runtime.py b/tests/harness/test_actor_runtime.py index b4c4d4f34..09552d2fb 100644 --- a/tests/harness/test_actor_runtime.py +++ b/tests/harness/test_actor_runtime.py @@ -261,6 +261,36 @@ def test_a_trial_s_state_changes_keep_attempted_refused_and_committed_apart_per_ assert changes[0]["committed_actions"] == ["deals.request_approval", "deals.apply_discount"] +def test_a_worker_s_attempts_are_its_own_participant_with_each_refusal_and_why(): + from fi.alk.harness.call_runner import trial_state_changes + + log = [ + {"name": "orders.get", "outcome": "committed", "worker_id": "order_lookup"}, + { + "name": "orders.update_status", + "outcome": "refused", + "worker_id": "order_lookup", + "refusal_code": "not_permitted", + "refusal": "orders.update_status is not permitted for order_lookup", + }, + {"name": "refunds.issue", "outcome": "committed", "worker_id": "refund"}, + {"name": "send_reply", "outcome": "committed"}, + ] + changes = trial_state_changes(log, {}) + assert [(c["who"], c["name"], c["attempted"], c["committed"], c["refused"]) for c in changes] == [ + ("agent", "the agent", 1, 1, 0), + ("worker:order_lookup", "order_lookup", 2, 1, 1), + ("worker:refund", "refund", 1, 1, 0), + ] + assert changes[1]["refused_actions"] == [ + { + "action": "orders.update_status", + "code": "not_permitted", + "reason": "orders.update_status is not permitted for order_lookup", + } + ] + + def _history_dispatcher(history, actors): from fi.alk.harness.actor_dispatch import ActorDispatcher diff --git a/tests/harness/test_bundle_author_v2.py b/tests/harness/test_bundle_author_v2.py index 0fac5777c..9876e0e47 100644 --- a/tests/harness/test_bundle_author_v2.py +++ b/tests/harness/test_bundle_author_v2.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +from dataclasses import replace import json import os import shutil @@ -13,6 +14,7 @@ from fi.alk.harness.bundle_author_v2 import ( BundleAuthorError, + _SOURCE_TOOL_TRACE_VERSION, _carries_world, _compile_source_tool_handlers, _contract_column_declarations, @@ -26,6 +28,7 @@ resolve_environment_plan, sealed_from, ) +from fi.alk.harness.generated_runtime import framework_instrumentation_command from fi.alk.harness.authoring_runtime_validation import _artifact_digest from fi.alk.harness.bundle_v2 import load_bundle_v2 from fi.alk.harness.certification import ( @@ -112,24 +115,134 @@ def test_native_command_tool_trace_uses_executable_source_bindings( tool_entrypoints=entries, contract_interface_kind="command", contract_modality="chat", ) control = next(p for p in plan.processes if p.name == plan.control_service) - if mode in {"none", "http"}: - assert "HARNESS_TOOL_TRACE" not in control.environment - assert "ALK_TOOL_TRACE_BINDINGS" not in control.environment - assert "ALK_CREWAI_TOOL_TRACE" not in control.environment - return assert control.environment["HARNESS_TOOL_TRACE"] == "{{WORLD_DIR}}/agent-tool-calls.jsonl" - assert json.loads(control.environment["ALK_TOOL_TRACE_BINDINGS"]) == [{ - "name": "Lookup tool", "module": "agent_tools", "callable": "lookup", - }] + assert control.environment["ALK_OBSERVE_FRAMEWORKS"] == "1" + assert control.build_commands[-1] == framework_instrumentation_command() assert (control.environment.get("ALK_CREWAI_TOOL_TRACE") == "1") == ( dependency in {"requirements", "selected", "extra", "supplemental"} ) + if mode in {"none", "http"}: + assert "ALK_TOOL_TRACE_BINDINGS" not in control.environment + else: + assert json.loads(control.environment["ALK_TOOL_TRACE_BINDINGS"]) == [{ + "name": "Lookup tool", "module": "agent_tools", "callable": "lookup", + }] assert all( "HARNESS_TOOL_TRACE" not in getattr(p, "environment", {}) for p in plan.processes if p.name != plan.control_service ) +def test_an_http_python_agent_is_observed_through_its_framework_spans(tmp_path: Path) -> None: + (tmp_path / "agent.py").write_text("print('ready')\n") + (tmp_path / "requirements.txt").write_text("openai-agents==0.24.0\n") + runtime = {"language": "python", "execution": "native", "command": ["python", "agent.py"]} + plan = resolve_environment_plan( + tmp_path, _job(connector="auto"), contract_runtime=runtime, + tool_entrypoints=[{"tool": "lookup", "mode": "import", "module": "agent", "callable": "lookup"}], + contract_interface_kind="http", contract_modality="chat", + ) + voice = resolve_environment_plan( + tmp_path, _job(connector="auto"), contract_runtime=runtime, + tool_entrypoints=[], contract_interface_kind="http", contract_modality="voice", + ) + # A voice agent's calls are traced by its LiveKit worker alone. + assert all( + "ALK_OBSERVE_FRAMEWORKS" not in p.environment and framework_instrumentation_command() not in p.build_commands + for p in voice.processes if hasattr(p, "build_commands") + ) + control = next(p for p in plan.processes if p.name == plan.control_service) + assert control.environment["HARNESS_TOOL_TRACE"] == "{{WORLD_DIR}}/agent-tool-calls.jsonl" + assert control.environment["ALK_OBSERVE_FRAMEWORKS"] == "1" + assert "ALK_TOOL_TRACE_BINDINGS" not in control.environment + assert control.build_commands[-1] == framework_instrumentation_command() + + +@pytest.mark.parametrize("uv", ["exit 1", "exec sleep 30"]) +def test_an_instrumentor_that_fails_or_stalls_is_a_warning_and_the_build_goes_on( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, uv: str +) -> None: + from fi.alk.harness import generated_runtime + from fi.alk.harness.generated_runtime import FRAMEWORK_INSTRUMENTORS + + assert all( + "==" in one + for _framework, one in FRAMEWORK_INSTRUMENTORS + if one.startswith("traceai-") + ) + monkeypatch.setattr(generated_runtime, "INSTRUMENTOR_INSTALL_SECONDS", 1) + installed = tmp_path / "site" + (installed / "crewai-1.0.dist-info").mkdir(parents=True) + (installed / "crewai-1.0.dist-info" / "METADATA").write_text( + "Name: crewai\nVersion: 1.0\n" + ) + tools = tmp_path / "bin" + tools.mkdir() + (tools / "uv").write_text(f"#!/bin/sh\n{uv}\n") + (tools / "uv").chmod(0o755) + (tools / "python3").symlink_to(sys.executable) + result = subprocess.run( + generated_runtime.framework_instrumentation_command(), + cwd=tmp_path, + capture_output=True, + text=True, + env={"PATH": f"{tools}:/bin:/usr/bin", "PYTHONPATH": str(installed)}, + timeout=60, + ) + assert result.returncode == 0 + assert ( + "warning: framework instrumentor not installed, its calls go unobserved: traceai-crewai==" + in result.stderr + ) + + +@pytest.mark.parametrize( + ("command", "project"), + [ + ( + [ + "uv", + "run", + "--project", + "graphs/python", + "--with", + "langgraph-cli[inmem]", + "langgraph", + "dev", + ], + "graphs/python", + ), + (["uv", "run", "--project=app", "python", "main.py"], "app"), + (["uv", "run", "python", "agent.py"], ""), + (["python", "agent.py"], ""), + ], +) +def test_framework_instrumentors_go_where_a_uv_run_project_s_frameworks_are( + tmp_path: Path, command: list[str], project: str +) -> None: + from fi.alk.harness.generated_runtime import uv_run_project + + assert uv_run_project(command) == project + (tmp_path / "agent.py").write_text("print('ready')\n") + runtime = {"language": "python", "execution": "native", "command": command} + plan = resolve_environment_plan( + tmp_path, + _job(connector="auto"), + contract_runtime=runtime, + tool_entrypoints=[], + contract_interface_kind="http", + contract_modality="chat", + ) + control = next(p for p in plan.processes if p.name == plan.control_service) + step = control.build_commands[-1] + assert step == framework_instrumentation_command(project) + if project: + assert step[2].startswith(f"uv sync --project {project} || echo ") + assert f"{project}/.venv/bin/python" in step[2] + else: + assert "uv sync" not in step[2] and "p=.venv/bin/python" in step[2] + + def test_bundle_author_carries_command_tool_trace_bindings(tmp_path: Path) -> None: source = tmp_path / "source" source.mkdir() @@ -209,7 +322,135 @@ def test_reseal_compiles_tools_added_to_a_traced_command_agent(tmp_path: Path) - reseal_scenarios(authoring) resealed = load_bundle_v2(bundle) - assert resealed.metadata["source_tool_trace_version"] == 1 + assert resealed.metadata["source_tool_trace_version"] == _SOURCE_TOOL_TRACE_VERSION + assert "handlers/update.py" in {file.path for file in resealed.files} + assert "contract_copied" not in resealed.metadata + + +def test_reseal_binds_a_tool_added_to_a_command_agent_built_with_none( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + source.mkdir() + (source / "agent.py").write_text("print('ready')\n") + (source / "agent_tools.py").write_text("def lookup(query): return query\n") + (source / "requirements.txt").write_text("requests==2.32.5\n") + authoring = _authoring(tmp_path) + _write_chat_contract( + authoring, + command=["python", "agent.py"], + interface={"kind": "command", "protocol": "fi.alk"}, + ) + contract_path = authoring / "contract.json" + contract = json.loads(contract_path.read_text()) + contract["tools"] = [{"name": "lookup", "args": ["query"]}] + contract["tool_entrypoints"] = [] + contract_path.write_text(json.dumps(contract)) + bundle = authoring / "environment-bundle" + author_bundle_v2( + source=source, job=_job(connector="auto"), authoring=authoring, output=bundle + ) + contract["tool_entrypoints"] = [ + { + "tool": "lookup", + "mode": "import", + "module": "agent_tools", + "callable": "lookup", + } + ] + contract_path.write_text(json.dumps(contract)) + + reseal_scenarios(authoring) + + resealed = load_bundle_v2(bundle) + control = next( + p for p in resealed.processes if p.name == resealed.runtime.control_service + ) + assert json.loads(control.environment["ALK_TOOL_TRACE_BINDINGS"]) == [ + {"name": "lookup", "module": "agent_tools", "callable": "lookup"} + ] + + +def test_reseal_compiles_tools_added_to_a_bundle_sealed_before_framework_observation( + tmp_path: Path, +) -> None: + from fi.alk.harness.bundle_author_v2 import _files + from fi.alk.harness.bundle_v2 import BUNDLE_V2_MANIFEST, seal_bundle_v2 + + source = tmp_path / "source" + source.mkdir() + (source / "agent.py").write_text("print('ready')\n") + (source / "agent_tools.py").write_text( + "def lookup(query): return query\ndef update(value): return value\n" + ) + (source / "requirements.txt").write_text("requests==2.32.5\n") + authoring = _authoring(tmp_path) + _write_chat_contract( + authoring, + command=["python", "agent.py"], + interface={"kind": "command", "protocol": "fi.alk"}, + ) + contract_path = authoring / "contract.json" + contract = json.loads(contract_path.read_text()) + contract["tools"] = [{"name": "lookup", "args": ["query"]}] + contract["tool_entrypoints"] = [ + { + "tool": "lookup", + "mode": "import", + "module": "agent_tools", + "callable": "lookup", + } + ] + contract_path.write_text(json.dumps(contract)) + bundle = authoring / "environment-bundle" + author_bundle_v2( + source=source, job=_job(connector="auto"), authoring=authoring, output=bundle + ) + # As the previous release sealed it: tool bindings at trace version 1, no framework observation. + manifest = load_bundle_v2(bundle) + observation = framework_instrumentation_command() + processes = [ + process.model_copy( + update={ + "environment": { + k: v + for k, v in process.environment.items() + if k != "ALK_OBSERVE_FRAMEWORKS" + }, + "build_commands": [ + step for step in process.build_commands if step != observation + ], + } + ) + if hasattr(process, "build_commands") + else process + for process in manifest.processes + ] + manifest = manifest.model_copy( + update={ + "processes": processes, + "files": _files(bundle), + "metadata": {**manifest.metadata, "source_tool_trace_version": 1}, + } + ) + manifest = manifest.model_copy(update={"digest": seal_bundle_v2(manifest)}) + (bundle / BUNDLE_V2_MANIFEST).write_text( + json.dumps(manifest.model_dump(mode="json"), indent=2, sort_keys=True) + "\n" + ) + contract["tools"].append({"name": "update", "args": ["value"]}) + contract["tool_entrypoints"].append( + { + "tool": "update", + "mode": "import", + "module": "agent_tools", + "callable": "update", + } + ) + contract_path.write_text(json.dumps(contract)) + + reseal_scenarios(authoring) + + resealed = load_bundle_v2(bundle) assert "handlers/update.py" in {file.path for file in resealed.files} assert "contract_copied" not in resealed.metadata @@ -262,7 +503,7 @@ def test_reseal_preserves_construct_only_crewai_command_trace(tmp_path: Path) -> assert json.loads(process.environment["ALK_TOOL_TRACE_BINDINGS"]) == [{ "name": "editor", "module": "agent_tools", "callable": "Editor.update", }] - assert resealed.metadata["source_tool_trace_version"] == 1 + assert resealed.metadata["source_tool_trace_version"] == _SOURCE_TOOL_TRACE_VERSION def _job( @@ -658,6 +899,47 @@ def test_bundle_compiles_discovered_source_tool_entrypoint(tmp_path: Path) -> No } +def test_a_reseal_keeps_framework_observation_of_a_chat_agent_with_no_import_tools( + tmp_path: Path, +) -> None: + source = tmp_path / "chat-agent" + source.mkdir() + (source / "agent.py").write_text("print('ready')\n", encoding="utf-8") + authoring = _authoring(tmp_path) + _write_chat_contract( + authoring, + command=["python3", "agent.py"], + interface={ + "kind": "http", + "protocol": "openai_chat", + "port": 8080, + "path": "/v1/chat/completions", + "include_tools": False, + }, + ) + bundle = authoring / "environment-bundle" + author_bundle_v2( + source=source, job=_job(connector="auto"), authoring=authoring, output=bundle + ) + contract_path = authoring / "contract.json" + contract = json.loads(contract_path.read_text()) + contract["notes"] = "edited after the build" + contract_path.write_text(json.dumps(contract)) + + reseal_scenarios(authoring) + + resealed = load_bundle_v2(bundle) + control = next( + p for p in resealed.processes if p.name == resealed.runtime.control_service + ) + assert control.environment["ALK_OBSERVE_FRAMEWORKS"] == "1" + assert ( + control.environment["HARNESS_TOOL_TRACE"] + == "{{WORLD_DIR}}/agent-tool-calls.jsonl" + ) + assert "ALK_TOOL_TRACE_BINDINGS" not in control.environment + + def test_bundle_observes_imported_chat_tool_execution(tmp_path: Path) -> None: source = tmp_path / "chat-agent" source.mkdir() @@ -709,7 +991,7 @@ def test_bundle_observes_imported_chat_tool_execution(tmp_path: Path) -> None: records = [json.loads(line) for line in trace.read_text().splitlines()] assert records[0]["name"] == "search_books" assert records[0]["arguments"] == {"query": "Dune"} - assert bundle.metadata["source_tool_trace_version"] == 1 + assert bundle.metadata["source_tool_trace_version"] == _SOURCE_TOOL_TRACE_VERSION def test_tool_registration_is_not_observed_as_tool_execution(tmp_path: Path) -> None: @@ -5596,3 +5878,60 @@ def test_python_selection_preserves_uv_alternatives_and_target_only_constraints( f"environments={json.dumps(environments)}\n" ) assert _python_language_version(tmp_path, "3.11", interpreter_selector=True) == "==3.11.*,>=3.11" + + +def test_each_declared_worker_is_sealed_with_the_world_actions_its_tools_call(): + from fi.alk.harness.bundle_author_v2 import _worker_permissions + + contract = { + "tools": [{"name": "get_order"}, {"name": "list_refunds"}, {"name": "issue_refund"}], + "tool_entrypoints": [ + {"tool": "get_order", "mode": "service", "service": "returns-api", "endpoint": "/orders/get"}, + {"tool": "issue_refund", "mode": "service", "service": "returns-api", "endpoint": "/refunds/issue/"}, + ], + "workers": [ + {"id": "order_lookup", "tools": ["get_order", "list_refunds", "issue_refund"], "allowed_tools": ["get_order", "list_refunds"]}, + {"id": "refund", "tools": ["issue_refund", "list_refunds"]}, + ], + } + # A worker holding more than it may use is sealed with what it may use. + assert _worker_permissions(contract) == { + "worker:order_lookup": {"allow": ["list_refunds", "orders.get"]}, + "worker:refund": {"allow": ["list_refunds", "refunds.issue"]}, + } + assert _worker_permissions({"tools": [{"name": "a"}]}) == {} + from fi.alk.harness.bundle_author_v2 import _tool_actions + from fi.alk.harness.contract import AgentContract + + assert _tool_actions(AgentContract.model_validate(contract)) == {"orders.get": "get_order", "refunds.issue": "issue_refund"} + + +def test_limited_workers_seed_the_action_log_that_keeps_whose_attempt_was_refused(tmp_path: Path) -> None: + from fi.alk.harness.bundle_author_v2 import _AGENT_TRACE_DDL, _TOOL_TRACE_DDL, _action_log_seed + + assert _action_log_seed(tmp_path) == _AGENT_TRACE_DDL + assert _action_log_seed(tmp_path, workers=True).startswith(_TOOL_TRACE_DDL) + + +def test_a_separately_wired_worker_s_tool_address_points_at_its_own_channel_on_the_tool_proxy(tmp_path): + from fi.alk.harness.bundle_author_v2 import EnvironmentPlanV2, _tool_proxy_process, _with_worker_channels + + agent = _tool_proxy_process().model_copy(update={"name": "agent", "environment": {"A": "1"}, "depends_on": []}) + plan = EnvironmentPlanV2( + packaging="process", + control_service="agent", + processes=(agent, _tool_proxy_process().model_copy(update={"depends_on": []})), + capabilities={}, + readiness=(), + ) + contract = { + "workers": [ + {"id": "refund", "binding": "separate", "address_env": "REFUND_API_URL"}, + {"id": "order_lookup", "binding": "shared", "address_env": "LOOKUP_API_URL"}, + ] + } + wired = {p.name: p.environment for p in _with_worker_channels(plan, contract).processes} + assert wired["agent"] == {"A": "1", "REFUND_API_URL": "{{TOOLS_API_URL}}/w/{{WORKER_CHANNEL_refund}}"} + assert "REFUND_API_URL" not in wired["tool-proxy"] + without_proxy = replace(plan, processes=(agent,)) + assert _with_worker_channels(without_proxy, contract) is without_proxy diff --git a/tests/harness/test_call_runner.py b/tests/harness/test_call_runner.py index 66cfbfa9e..d3d7a331c 100644 --- a/tests/harness/test_call_runner.py +++ b/tests/harness/test_call_runner.py @@ -176,6 +176,182 @@ def test_file_tool_trace_is_collected_from_runtime_metadata(tmp_path: Path) -> N assert calls[1].error == "declined" +def test_the_sealed_trace_row_keeps_the_runtime_worker_task_and_event_ids( + tmp_path: Path, +) -> None: + from fi.alk.harness.world.runtime import trace_row + + trace = _trace_lines( + tmp_path / "agent-tool-calls.jsonl", + { + "name": "lookup", + "arguments": {}, + "output": "found", + "is_error": False, + "source": "crewai", + "agent_key": "researcher", + "task_name": "Find weather", + "event_id": "e2", + "parent_event_id": "e1", + }, + {"name": "lookup", "arguments": {}, "output": "found", "is_error": False}, + ) + observed, plain = ( + trace_row(call) + for call in cr._collect_file_tool_calls( + _runtime(metadata={"tool_trace_path": str(trace)}) + ) + ) + + assert (observed["worker_id"], observed["task_label"]) == ( + "researcher", + "Find weather", + ) + assert observed["runtime_ids"] == {"event_id": "e2", "parent_event_id": "e1"} + assert not { + "worker_id", + "worker_label", + "task_id", + "task_label", + "runtime_ids", + } & set(plain) + + +_SPAN_LINES = ( + { + "name": "ask_writer", + "arguments": {}, + "output": None, + "is_error": False, + "source": "otel", + "agent_key": "Coordinator", + "delegated_to": "Writer", + "at": 1.0, + "ended_at": 3.0, + }, + { + "name": "draft_note", + "arguments": {}, + "output": None, + "is_error": False, + "source": "otel", + "agent_key": "Writer", + "at": 2.0, + }, +) + + +def _observed() -> list[Call]: + # Stamped once their results were read, later than they started and in the other order. + return [ + Call( + name="ask_writer", + arguments={"topic": "tides"}, + result="done", + ok=True, + at=9.2, + ), + Call( + name="draft_note", + arguments={"topic": "tides"}, + result="drafted", + ok=True, + at=9.1, + ), + ] + + +def test_span_lines_name_the_agent_of_calls_another_source_observed_and_add_none( + tmp_path: Path, +) -> None: + trace = _trace_lines( + tmp_path / "agent-tool-calls.jsonl", + *_SPAN_LINES, + {"name": "draft_note", "arguments": {}, "output": "written", "is_error": False}, + ) + calls = cr._with_span_calls( + _observed(), _runtime(metadata={"tool_trace_path": str(trace)}) + ) + assert [ + (c.name, c.arguments, c.worker_id, c.delegated_to, c.at, c.ended_at) + for c in calls + ] == [ + ("ask_writer", {"topic": "tides"}, "Coordinator", "Writer", 1.0, 3.0), + ("draft_note", {"topic": "tides"}, "Writer", "", 2.0, 0.0), + ] + + +def test_a_runtime_s_hand_off_keeps_when_the_worker_returned_and_is_never_graded( + tmp_path: Path, +) -> None: + # CrewAI's observer writes each agent a flow runs as a hand-off, with its own start and end. + from fi.alk.harness.world.runtime import graded_calls, trace_row + + trace = _trace_lines( + tmp_path / "agent-tool-calls.jsonl", + { + "name": "Draft checker", + "arguments": {}, + "output": "valid", + "is_error": False, + "source": "crewai", + "agent_key": "DraftFlow", + "delegated_to": "Draft checker", + "at": 4.0, + "ended_at": 6.5, + "kind": "handoff", + }, + ) + calls = cr._collect_file_tool_calls( + _runtime(metadata={"tool_trace_path": str(trace)}) + ) + (row,) = (trace_row(call) for call in calls) + assert ( + row["worker_id"], + row["delegated_to"], + row["at"], + row["ended_at"], + row["kind"], + ) == ("DraftFlow", "Draft checker", 4.0, 6.5, "delegated") + assert graded_calls(calls) == [] + + +def test_a_span_that_recorded_no_arguments_reads_as_none_not_as_raw_text( + tmp_path: Path, +) -> None: + trace = _trace_lines( + tmp_path / "agent-tool-calls.jsonl", + {**_SPAN_LINES[1], "arguments": "", "output": ""}, + {"name": "note", "arguments": "not json", "output": "", "is_error": False}, + ) + calls = cr._collect_file_tool_calls( + _runtime(metadata={"tool_trace_path": str(trace)}) + ) + assert [(c.name, c.arguments) for c in calls] == [ + ("note", {"raw": "not json"}), + ("draft_note", {}), + ] + + +@pytest.mark.parametrize( + "seam", [EvidenceSeam.TOOL_TRACE, EvidenceSeam.HTTP_TOOL, None] +) +def test_a_process_agent_s_seam_calls_keep_their_evidence_and_take_the_agent_from_its_spans( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, seam: EvidenceSeam | None +) -> None: + trace = _trace_lines(tmp_path / "agent-tool-calls.jsonl", *_SPAN_LINES) + for reader in ("_collect_tool_trace_calls", "_collect_http_tool_calls"): + monkeypatch.setattr(cr, reader, lambda runtime: tuple(_observed())) + _job_obj, context = _context(tmp_path=tmp_path, evidence_seam=seam) + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=None) + calls = runner._collect_calls(_runtime(metadata={"tool_trace_path": str(trace)})) + # With no seam, the spans are the only evidence. + assert [(c.name, c.result, c.worker_id) for c in calls] == [ + ("ask_writer", "done" if seam else None, "Coordinator"), + ("draft_note", "drafted" if seam else None, "Writer"), + ] + + def test_a_result_traced_as_the_text_of_a_dict_reaches_checks_as_the_dict(tmp_path: Path) -> None: trace = tmp_path / "agent-tool-calls.jsonl" trace.write_text( @@ -2451,6 +2627,43 @@ async def place_call(spec): ] == [("approve", "reviewer")] +def test_a_worker_s_refused_attempt_is_sealed_in_the_trial_s_state_changes_without_any_actor( + tmp_path, monkeypatch +) -> None: + _job_obj, context = _context( + tmp_path=tmp_path, evidence_seam=EvidenceSeam.TOOL_TRACE + ) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + columns = ["name", "arguments", "result", "ok", "error", "at", "action_id", "actor_id", "worker_id", "outcome", + "refusal_code", "refusal"] + rows = [ + ("update_status", {}, None, False, "", 1.0, "a1", "agent", "order_lookup", "refused", "not_permitted", + "orders.update_status is not permitted for order_lookup"), + ] + monkeypatch.setitem(sys.modules, "psycopg", _FakePsycopg(rows, columns)) + + async def place_call(spec): + return _report( + transcript="user: hello", messages=[{"role": "user", "content": "hello"}] + ) + + adapter = FakeAdapter() + runner = cr.CallRunnerImpl(adapter, context, place_call=place_call) + _run( + runner, + _FakeScenario("k1"), + _runtime( + endpoints={"database": _postgres_endpoint()}, + metadata={"livekit_agent_name": "agent-w0"}, + ), + ) + sealed = {kind: data for kind, _key, data in adapter.uploads} + changes = json.loads(sealed[cr.ArtifactKind.TRANSCRIPT])["world_actors"]["state_changes"] + assert [(one["who"], one["refused"], one["refused_actions"][0]["code"]) for one in changes] == [ + ("worker:order_lookup", 1, "not_permitted") + ] + + def test_a_trial_s_actor_token_holds_only_while_its_call_runs( tmp_path, monkeypatch ) -> None: @@ -2591,3 +2804,49 @@ def chown(*args, **kwargs): with pytest.raises(cr.PrivateTokenUnavailable): cr._write_trial_token(token_path, "secret") assert not token_path.exists() + + +def test_each_trial_writes_its_worker_deadline_beside_its_faults_and_clears_it_after(tmp_path) -> None: + import json + import time + from types import SimpleNamespace + + token = tmp_path / "tool-proxy" / "actor-token" + token.parent.mkdir(parents=True) + runtime = SimpleNamespace(metadata={"actor_proof_path": str(token)}) + before = time.time() + deadline = cr.worker_deadline({"worker_budget_seconds": 30}) + assert before + 30 <= deadline <= time.time() + 30 + assert cr.worker_deadline({}) is None + cr._write_trial_state(runtime, [{"action": "ok"}], deadline) + state = json.loads((token.parent / "trial.json").read_text()) + assert state == {"faults": [{"action": "ok"}], "worker_deadline_at": deadline} + cr._write_trial_state(runtime, []) + assert json.loads((token.parent / "trial.json").read_text()) == {"faults": []} + + +def test_a_worker_s_call_carries_the_worker_and_why_the_world_refused_it(monkeypatch) -> None: + columns = ["name", "arguments", "result", "ok", "error", "at", "action_id", "actor_id", "worker_id", "outcome", + "refusal_code"] + rows = [ + ("get_order", {"id": "1"}, {"status": "delivered"}, True, "", 100.5, "a1", "agent", "order_lookup", "committed", None), + ("issue_refund", {"id": "1"}, None, False, "late", 101.0, "a2", "agent", "refund", "refused", "deadline_passed"), + ] + monkeypatch.setitem(sys.modules, "psycopg", _FakePsycopg(rows, columns)) + calls = cr._collect_tool_trace_calls(_runtime(endpoints={"database": _postgres_endpoint()})) + assert [(one.worker_id, one.refusal_code) for one in calls] == [("order_lookup", ""), ("refund", "deadline_passed")] + + +def test_the_sealed_trace_row_keeps_why_the_world_refused_a_call() -> None: + from fi.alk.harness.world.runtime import Call, trace_row + + refused = Call( + name="get_order", + arguments={}, + ok=False, + refused=True, + worker_id="order_lookup", + refusal_code="deadline_passed", + ) + assert trace_row(refused)["refusal_code"] == "deadline_passed" + assert "refusal_code" not in trace_row(Call(name="get_order", arguments={})) diff --git a/tests/harness/test_chat_call_runner.py b/tests/harness/test_chat_call_runner.py index 573218c84..524aa577c 100644 --- a/tests/harness/test_chat_call_runner.py +++ b/tests/harness/test_chat_call_runner.py @@ -394,6 +394,72 @@ async def call(self, _request: Any) -> AgentResponse: assert b'"name": "lookup_account"' in adapter.uploads[1] +def test_hosted_chat_takes_each_call_s_worker_from_the_agent_s_framework_spans( + tmp_path: Path, monkeypatch: Any +) -> None: + tool_world = _ToolWorld() + monkeypatch.setattr(chat, "_tool_world", lambda *_args: tool_world) + monkeypatch.setattr(chat, "_drive_conversation", _single_exchange) + trace = tmp_path / "agent-tool-calls.jsonl" + + class Wrapper: + def __init__(self, **_kwargs: Any) -> None: + self.calls = 0 + + async def call(self, _request: Any) -> AgentResponse: + self.calls += 1 + if self.calls == 1: + call = { + "id": "tool-1", + "function": { + "name": "lookup_account", + "arguments": '{"account_id":"ACC-2048"}', + }, + } + return AgentResponse(content="", tool_calls=[call]) + # The agent's process wrote the span of the call its worker made. + span = { + "name": "lookup_account", + "arguments": "", + "output": "", + "is_error": False, + "source": "otel", + "agent_key": "Accounts worker", + "agent_role": "Accounts worker", + "at": 1.0, + } + trace.write_text(json.dumps(span) + "\n", encoding="utf-8") + return AgentResponse(content="The account is active.") + + monkeypatch.setattr(chat, "HTTPAgentWrapper", Wrapper) + adapter = _Adapter() + runner = chat.HostedChatCallRunner(adapter, _context(tmp_path)) + runtime = EnvironmentRuntime( + runtime_id="runtime-1", + world_index=0, + bundle_digest="sha256:" + "a" * 64, + state=RuntimeState.READY, + endpoints={ + "target_http": RuntimeEndpoint( + capability="target_http", + protocol="http", + address="http://localhost:18080", + ) + }, + metadata={"tool_trace_path": str(trace)}, + ) + outcome = asyncio.run( + runner.run( + SimpleNamespace(scenario_key="one", scenario_id="scenario-1"), runtime + ) + ) + + assert [(c.name, c.arguments, c.worker_label) for c in outcome.calls] == [ + ("lookup_account", {"account_id": "ACC-2048"}, "Accounts worker") + ] + assert json.loads(adapter.uploads[1])["worker_label"] == "Accounts worker" + + @pytest.mark.parametrize( "metadata", [ @@ -1413,3 +1479,42 @@ def test_generated_command_bridge_handles_readiness_and_hosted_chat(tmp_path, mo server.shutdown() thread.join() server.server_close() + + +@pytest.mark.parametrize("seam", [None, "tool_trace"]) +def test_a_chat_call_reads_the_world_s_action_log_when_its_bundle_says_evidence_lands_there( + tmp_path: Path, monkeypatch: Any, seam: str | None +) -> None: + from dataclasses import replace + + from fi.alk.harness import call_runner + from fi.alk.harness.bundle_v2 import EvidenceSeam + + logged = Call(name="issue_refund", arguments={"order_id": "1043"}, worker_id="refund") + monkeypatch.setattr(call_runner, "_collect_tool_trace_calls", lambda _runtime: (logged,)) + monkeypatch.setattr(chat, "_tool_world", lambda *_args: _ToolWorld()) + monkeypatch.setattr(chat, "_drive_conversation", _single_exchange) + + class Wrapper: + def __init__(self, **_kwargs: Any) -> None: + pass + + async def call(self, _request: Any) -> AgentResponse: + return AgentResponse(content="Refund RF-8 is issued.") + + monkeypatch.setattr(chat, "HTTPAgentWrapper", Wrapper) + context = replace(_context(tmp_path), evidence_seam=EvidenceSeam(seam) if seam else None) + runtime = EnvironmentRuntime( + runtime_id="runtime-1", + world_index=0, + bundle_digest="sha256:" + "a" * 64, + state=RuntimeState.READY, + endpoints={"target_http": RuntimeEndpoint(capability="target_http", protocol="http", address="http://x")}, + ) + outcome = asyncio.run( + chat.HostedChatCallRunner(_Adapter(), context).run( + SimpleNamespace(scenario_key="one", scenario_id="scenario-1"), runtime + ) + ) + # The agent's tools ran against the world's services, so only the world's own log saw them. + assert [(call.name, call.worker_id) for call in outcome.calls] == ([("issue_refund", "refund")] if seam else []) diff --git a/tests/harness/test_chat_readiness.py b/tests/harness/test_chat_readiness.py index b24d8a4bb..24845f248 100644 --- a/tests/harness/test_chat_readiness.py +++ b/tests/harness/test_chat_readiness.py @@ -1003,21 +1003,25 @@ async def conversation(target, *_args): assert receipt["tool_evidence_status"] == "not_observed" +@pytest.mark.parametrize("handoffs", [0, 1]) def test_private_workflow_requires_evidence_for_executable_contract_tools( - tmp_path, profile_app, monkeypatch + tmp_path, profile_app, monkeypatch, handoffs ): from fi.alk.harness import chat_call_runner from fi.alk.harness.contract import ToolEntry, ToolSpec from fi.alk.harness.run.conversation import Exchange, Transcript + from fi.alk.harness.world.runtime import Call url, state = profile_app state.update(profile=True, body={"content": "The account is active"}) async def conversation(target, *_args): answer = await target.say("Look up account A-1") + # A worker's run is not a tool call, so it is no evidence for one. + handed = [Call(name="Account worker", arguments={}, delegated_to="Account worker", handoff=True)] return Transcript( exchanges=[Exchange("customer", "Look up account A-1"), Exchange("agent", answer)], - calls=[], ended="finished", + calls=handed[:handoffs], ended="finished", ) monkeypatch.setattr(chat_call_runner, "_drive_conversation", conversation) @@ -1123,13 +1127,44 @@ async def say(self, _message): if self.opening: self.opening = False return SimpleNamespace(text="Look up account A-1") + # The agent's process wrote the span of the call, naming the worker that made it. + span = { + "name": "lookup_account", + "arguments": "", + "output": "", + "is_error": False, + "source": "otel", + "agent_key": "Accounts worker", + "agent_role": "Accounts worker", + "at": 1.0, + } + # And a worker's run, which is no tool call. + handoff = { + "name": "Accounts worker", + "arguments": {}, + "output": "done", + "is_error": False, + "source": "otel", + "delegated_to": "Accounts worker", + "kind": "handoff", + "at": 2.0, + } + spans.write_text(json.dumps(span) + "\n" + json.dumps(handoff) + "\n") started.set() await asyncio.Event().wait() monkeypatch.setattr(conversation, "Stage", CustomerStage) + spans = tmp_path / "agent-tool-calls.jsonl" + runtime = SimpleNamespace( + metadata={"observation_mode": "native", "tool_trace_path": str(spans)}, + world_index=0, + endpoints={"target_http": SimpleNamespace(address=url, protocol="http")}, + ) async def run(): - task = asyncio.create_task(workflow_probe(tmp_path, url, timeout=2)) + task = asyncio.create_task( + workflow_probe(tmp_path, url, timeout=2, runtime=runtime) + ) if cancel: await started.wait() task.cancel() @@ -1157,10 +1192,12 @@ async def run(): {"role": "assistant", "content": "The account is active "}, ] tool_trace = next(receipt_path.parent.glob("tool_trace-*.json")) - call = json.loads(tool_trace.read_text()) + call, handed = (json.loads(line) for line in tool_trace.read_text().splitlines()) + assert (handed["delegated_to"], handed["kind"], "handoff" in handed) == ("Accounts worker", "delegated", False) assert call["name"] == "lookup_account" assert call["arguments"] == {"account_id": "A-1"} assert call["result"]["active"] is True + assert call["worker_label"] == "Accounts worker" assert all("secret123" not in path.read_text() for path in receipt_path.parent.iterdir()) @@ -1217,3 +1254,55 @@ def test_native_http_200_failure_rule_unlocks_existing_setup_repair(tmp_path, pr assert repaired.runtime.command == contract.runtime.command assert "secret123" not in json.dumps(receipts(tmp_path)) assert sorted(item["status"] for item in receipts(tmp_path)) == ["failed", "passed"] + + +def test_an_aborted_readiness_call_counts_its_tool_calls_not_its_hand_offs( + tmp_path, profile_app, monkeypatch +): + from fi.alk.harness import chat_call_runner + from fi.alk.harness.hosted_scheduler import CallAborted, CallOutcome + from fi.alk.harness.world.runtime import Call + + url, _state = profile_app + calls = ( + Call(name="lookup_account", arguments={}, ok=True), + Call( + name="Accounts worker", + arguments={}, + delegated_to="Accounts worker", + handoff=True, + ), + ) + + async def aborted(self, *_args): + partial = CallOutcome( + calls=calls, turns=1, messages=(), started_at="", ended_at="", duration_ms=0 + ) + raise CallAborted("the target stopped", partial=partial) + + monkeypatch.setattr(chat_call_runner.HostedChatCallRunner, "run", aborted) + with pytest.raises(RuntimeValidationError): + asyncio.run(workflow_probe(tmp_path, url)) + _, receipt = workflow_receipt(tmp_path) + assert receipt["tool_call_count"] == 1 + + +def test_the_readiness_call_reads_calls_from_the_bundle_s_own_evidence_seam(tmp_path, monkeypatch): + from fi.alk.harness import chat_call_runner + from fi.alk.harness.bundle_v2 import EvidenceSeam + from fi.alk.harness.world.runtime import Call + + seen = {} + + class Runner: + def __init__(self, _artifacts, context): + seen["seam"] = context.evidence_seam + self._contract = None + + async def run(self, _scenario, _runtime): + return SimpleNamespace(calls=(Call(name="issue_refund", arguments={}),), turns=1) + + monkeypatch.setattr(chat_call_runner, "HostedChatCallRunner", Runner) + asyncio.run(workflow_probe(tmp_path, "http://x", evidence_seam=EvidenceSeam.TOOL_TRACE)) + assert seen["seam"] is EvidenceSeam.TOOL_TRACE + assert workflow_receipt(tmp_path)[1]["status"] == "passed" diff --git a/tests/harness/test_chat_worker.py b/tests/harness/test_chat_worker.py index 0c9908260..226dfc199 100644 --- a/tests/harness/test_chat_worker.py +++ b/tests/harness/test_chat_worker.py @@ -401,3 +401,43 @@ async def worker(*_args, **_kwargs): assert adapter.uploads == [b"partial transcript", b"http failure trace"] assert failure.value.partial.transcript_artifact == "transcript-1" assert failure.value.partial.turns == 2 + + +def test_an_isolated_chat_call_keeps_the_bundle_s_evidence_seam(tmp_path, monkeypatch): + from dataclasses import replace + + from fi.alk.harness.bundle_v2 import EvidenceSeam + + class Stop(Exception): + pass + + seen = {} + + class Runner: + def __init__(self, _adapter, context): + seen["worker"] = context.evidence_seam + raise Stop + + async def worker(_module, payload, *, environ, work_directory): + seen["payload"] = payload["evidence_seam"] + monkeypatch.setattr(chat_worker.sys, "stdin", io.StringIO(json.dumps(payload))) + monkeypatch.setattr(chat_worker.sys, "argv", ["chat_worker", str(tmp_path / "result.json")]) + await chat_worker.main() + + monkeypatch.setattr(chat_worker, "run_json_worker", worker) + monkeypatch.setattr(chat_call_runner, "HostedChatCallRunner", Runner) + context = replace(_context(tmp_path), evidence_seam=EvidenceSeam.TOOL_TRACE) + runtime = EnvironmentRuntime( + runtime_id="runtime-1", + world_index=0, + bundle_digest="sha256:" + "a" * 64, + state=RuntimeState.READY, + endpoints={"target_http": RuntimeEndpoint(capability="target_http", protocol="http", address="http://x")}, + ) + with pytest.raises(Exception): + asyncio.run( + chat_worker.IsolatedChatCallRunner(_Adapter(), context).run( + SimpleNamespace(scenario_key="one", source_scenario_key="one", scenario_id="id-1"), runtime + ) + ) + assert seen == {"payload": "tool_trace", "worker": EvidenceSeam.TOOL_TRACE} diff --git a/tests/harness/test_contract.py b/tests/harness/test_contract.py index d9d3bafc6..bf230f5d5 100644 --- a/tests/harness/test_contract.py +++ b/tests/harness/test_contract.py @@ -180,3 +180,109 @@ def test_http_server_keeps_source_owned_message_template_argument(): command = ["server", "--chat-template", "{{messages}}"] runtime = Runtime(command=command, interface={"kind": "http", "protocol": "openai_chat", "port": 8080, "path": "/v1/chat/completions"}) assert runtime.command == command + + +def test_declared_workers_name_contract_tools_and_reach_the_brief(): + contract = AgentContract.model_validate( + { + "agent": "returns", + "real_use_cases": ["refund an order"], + "tools": [{"name": "get_order"}, {"name": "issue_refund"}], + "workers": [ + { + "id": "order_lookup", + "name": "Order-lookup worker", + "tools": ["get_order"], + }, + {"id": "refund", "name": "Refund worker", "tools": ["issue_refund"]}, + ], + } + ) + assert validate_contract(contract) == [] + assert "order_lookup (Order-lookup worker): get_order" in contract.brief() + + +def test_a_worker_without_an_id_of_its_own_is_returned_to_authoring_and_keeps_only_contract_tools(): + contract = AgentContract.model_validate( + { + "agent": "returns", + "real_use_cases": ["refund an order"], + "tools": [{"name": "get_order"}], + "workers": [ + {"id": "order_lookup", "tools": ["get_order", "send_email"]}, + {"id": "order_lookup", "name": "Second"}, + {"id": " "}, + ], + } + ) + problems = validate_contract(contract) + assert [one.split(":")[0] for one in problems] == [ + "worker[2]", + "duplicate-worker-ids", + ] + assert "duplicate-worker-ids:order_lookup:" in problems[1] + assert [(one.id, one.tools) for one in contract.workers] == [ + ("order_lookup", ["get_order"]), + ("order_lookup", []), + (" ", []), + ] + fixed = contract.model_copy( + update={ + "workers": [ + contract.workers[0], + contract.workers[1].model_copy(update={"id": "refunds"}), + ] + } + ) + assert validate_contract(fixed) == [] + + +def test_contract_without_workers_is_unchanged(): + contract = AgentContract.model_validate( + {"agent": "plain", "real_use_cases": ["answer"], "tools": [{"name": "a"}]} + ) + assert contract.workers == [] + assert "WORKERS" not in contract.brief() + + +def test_a_sub_goal_names_the_task_or_one_worker_and_a_task_subject_is_saved_as_before(tmp_path): + import json + + from fi.alk.harness.catalogue import Catalogue, SubGoal, save_catalogue + + assert SubGoal(name="a").subject == "task" and SubGoal(name="a").worker == "" + assert SubGoal(name="b", subject="worker:refund").worker == "refund" + for bad in ("refund", "worker:", "agent:x"): + with pytest.raises(ValidationError, match="subject is task or worker"): + SubGoal(name="c", subject=bad) + path = save_catalogue(Catalogue(sub_goals=[SubGoal(name="a"), SubGoal(name="b", subject="worker:refund")]), tmp_path) + saved = json.loads(path.read_text())["sub_goals"] + assert "subject" not in saved[0] and saved[1]["subject"] == "worker:refund" + + +def test_a_worker_allowed_a_tool_it_does_not_hold_is_returned_to_authoring(): + contract = AgentContract.model_validate( + { + "agent": "returns", + "real_use_cases": ["refund an order"], + "tools": [{"name": "get_order"}, {"name": "issue_refund"}], + "workers": [ + {"id": "order_lookup", "tools": ["get_order"], "allowed_tools": ["get_order"]}, + {"id": "refund", "tools": ["issue_refund"], "allowed_tools": ["get_order"]}, + ], + } + ) + assert [one.split(": ")[0] for one in validate_contract(contract)] == [ + "worker[1]:allowed-tools-it-does-not-hold:get_order" + ] + + +def test_a_scenario_s_worker_budget_is_positive_and_absent_from_its_document_when_unset(): + from fi.alk.harness.folder import document_for + from fi.alk.harness.scenario import Scenario + + plain = Scenario(name="a", instruction="x") + assert "worker_budget_seconds" not in document_for(plain) + assert document_for(plain.model_copy(update={"worker_budget_seconds": 20}))["worker_budget_seconds"] == 20 + with pytest.raises(ValidationError): + Scenario(name="a", instruction="x", worker_budget_seconds=0) diff --git a/tests/harness/test_crewai_runtime_observer.py b/tests/harness/test_crewai_runtime_observer.py index e0ecd4db2..27bfb5a6b 100644 --- a/tests/harness/test_crewai_runtime_observer.py +++ b/tests/harness/test_crewai_runtime_observer.py @@ -258,3 +258,78 @@ def test_string_values_are_bounded_before_the_record_limit(tmp_path: Path) -> No assert len(records[0]["arguments"]["query"]) == 4096 assert len(records[0]["output"]["result"]) == 4096 + + +def test_each_payload_says_whether_bounding_left_part_of_it_out(tmp_path: Path) -> None: + _fake_crewai(tmp_path) + script = """ +from crewai.events.event_bus import CrewAIEventsBus +from crewai.events.types.tool_usage_events import ToolUsageFinishedEvent +bus = CrewAIEventsBus() +try: + for args, output in (({'query': 'x' * 5000}, 'ok'), ({'query': 'q'}, ['y'] * 17), ({'query': 'q'}, 'ok')): + assert bus.emit(None, ToolUsageFinishedEvent(tool_name='lookup', tool_args=args, output=output)) is bus.future +finally: + bus.release.set() + bus.executor.shutdown() +""" + _, records = _run(tmp_path, script) + + assert [ + (one.get("input_omitted", False), one.get("output_omitted", False)) + for one in records + ] == [(True, False), (False, True), (False, False)] + + +def test_each_agent_a_flow_runs_is_a_hand_off_from_the_flow_with_its_return( + tmp_path: Path, +) -> None: + _fake_crewai(tmp_path) + (tmp_path / "crewai/events/types/agent_events.py").write_text( + "from types import SimpleNamespace\n" + "class AgentExecutionStartedEvent(SimpleNamespace): pass\n" + "class AgentExecutionCompletedEvent(SimpleNamespace): pass\n" + "class AgentExecutionErrorEvent(SimpleNamespace): pass\n" + ) + (tmp_path / "crewai/events/types/flow_events.py").write_text( + "from types import SimpleNamespace\nclass FlowStartedEvent(SimpleNamespace): pass\n" + ) + script = """ +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace as N +from crewai.events.event_bus import CrewAIEventsBus +from crewai.events.types.agent_events import ( + AgentExecutionStartedEvent as Start, AgentExecutionCompletedEvent as Done, AgentExecutionErrorEvent as Failed, +) +from crewai.events.types.flow_events import FlowStartedEvent +import crewai_runtime_observer_bootstrap as observer +assert observer.install() +bus = CrewAIEventsBus() +t0 = datetime(2026, 11, 3, 10, 0, tzinfo=timezone.utc) +writer = N(id='a1', role='Draft writer') +checker = N(id='a2', role='Draft checker\\n') +write, verify = N(id='t1', name='write_draft', description='Write a post'), N(id='t2', name='check_draft', description='Verify it') +bus.emit(None, FlowStartedEvent(flow_name='DraftFlow', no_handlers=True)) +steps = [(Start, writer, write, 0), (Done, writer, write, 2), (Start, checker, verify, 3), (Done, checker, verify, 4), + (Start, writer, write, 5), (Done, writer, write, 7), (Start, checker, verify, 8), (Failed, checker, verify, 9)] +for kind, agent, task, second in steps: + extra = {'error': 'too long'} if kind is Failed else {'output': 'ok'} if kind is Done else {} + bus.emit(None, kind(agent=agent, task=task, timestamp=t0 + timedelta(seconds=second), no_handlers=True, **extra)) +""" + _result, records = _run(tmp_path, script) + handoffs = [ + (r["agent_key"], r["delegated_to"], r["ended_at"] - r["at"], r["is_error"]) + for r in records + ] + assert handoffs == [ + ("DraftFlow", "Draft writer", 2.0, False), + ("DraftFlow", "Draft checker", 1.0, False), + ("DraftFlow", "Draft writer", 2.0, False), + ("DraftFlow", "Draft checker", 1.0, True), + ] + assert ( + records[0]["arguments"] == {"task": "Write a post"} + and records[0]["task_name"] == "write_draft" + ) + assert records[1]["output"] == "ok" and records[3]["output"] == "too long" + assert {r["kind"] for r in records} == {"handoff"} diff --git a/tests/harness/test_framework_tool_spans.py b/tests/harness/test_framework_tool_spans.py new file mode 100644 index 000000000..e2fac1f18 --- /dev/null +++ b/tests/harness/test_framework_tool_spans.py @@ -0,0 +1,1295 @@ +"""Tool calls and the agent that made each, read from framework spans by the sandbox's startup hook. + +Each case runs a real interpreter with the hook as its ``sitecustomize`` and makes spans the way one +framework's instrumentation shapes them, so no framework has to be installed here. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from fi.alk.harness import ( + call_runner, + simulation_mark_bootstrap, + simulation_worker_events, +) +from fi.alk.harness.process_runtime import EnvironmentRuntime, RuntimeState +from fi.alk.harness.world.runtime import Call, trace_row + +_PRELUDE = """ +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.trace import Status, StatusCode +{provider} +tracer = trace.get_tracer("probe") + +def span(name, **attributes): + return tracer.start_as_current_span(name, attributes=attributes) +""" + + +def _place_hook(hook: Path) -> None: + for module in (simulation_mark_bootstrap, simulation_worker_events): + (hook / Path(module.__file__).name).write_bytes( + Path(module.__file__).read_bytes() + ) + + +def _run( + tmp_path: Path, + body: str, + *, + provider: str = "trace.set_tracer_provider(TracerProvider())", +) -> list[dict]: + hook = tmp_path / "hook" + hook.mkdir() + _place_hook(hook) + (hook / "sitecustomize.py").write_text( + "import simulation_mark_bootstrap\nsimulation_mark_bootstrap.install()\n" + ) + trace = tmp_path / "calls.jsonl" + script = _PRELUDE.format(provider=provider) + textwrap.dedent(body) + result = subprocess.run( + [sys.executable, "-c", script], + env={ + **os.environ, + "PYTHONPATH": str(hook), + "HARNESS_TOOL_TRACE": str(trace), + "ALK_OBSERVE_FRAMEWORKS": "1", + }, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, result.stderr + assert "hook: failed" not in result.stderr + if not trace.exists(): + return [] + return [json.loads(line) for line in trace.read_text().splitlines()] + + +def _sealed(tmp_path: Path) -> list[dict]: + """The hook's lines as the sandbox reads and seals them.""" + runtime = EnvironmentRuntime( + runtime_id="rt-1", + world_index=0, + bundle_digest="sha256:" + "0" * 64, + state=RuntimeState.READY, + endpoints={}, + metadata={"tool_trace_path": str(tmp_path / "calls.jsonl")}, + ) + return [trace_row(call) for call in call_runner._collect_file_tool_calls(runtime)] + + +def _placed(rows: list[dict]) -> list[tuple]: + return [ + ( + row["name"], + row["kind"], + row["agent"], + row["relationship"], + row["parent_span_id"], + ) + for row in rows + ] + + +def _by_tool(records: list[dict]) -> dict[str, tuple]: + return {r["name"]: (r.get("agent_key"), r.get("delegated_to")) for r in records} + + +def test_a_tool_call_names_the_agent_above_it_and_a_hand_off_names_the_worker( + tmp_path: Path, +) -> None: + records = _run( + tmp_path, + """ + with span("Lead agent", **{"gen_ai.span.kind": "AGENT", "graph.node.id": "Lead agent"}): + with span("ask_research", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "ask_research"}): + with span("Research worker", **{"gen_ai.span.kind": "AGENT", "graph.node.id": "Research worker"}): + with span("search_notes", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "search_notes", + "input.value": '{"query": "q"}', "output.value": "found"}): + pass + """, + ) + assert _by_tool(records) == { + "search_notes": ("Research worker", None), + "ask_research": ("Lead agent", "Research worker"), + } + lookup = next(r for r in records if r["name"] == "search_notes") + assert (lookup["arguments"], lookup["output"], lookup["is_error"]) == ( + '{"query": "q"}', + "found", + False, + ) + + +def test_a_langgraph_tool_is_the_subgraph_above_the_node_that_ran_it( + tmp_path: Path, +) -> None: + records = _run( + tmp_path, + """ + with span("tools", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "publish_note", + "gen_ai.agent.name": "tools", "langgraph_node": "tools", + "langgraph_checkpoint_ns": "writer_worker:1|tools:2"}): + pass + with span("tools", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "lookup", + "langgraph_node": "tools", "langgraph_checkpoint_ns": "tools:3"}): + pass + """, + ) + assert _by_tool(records) == { + "publish_note": ("writer_worker", None), + "lookup": (None, None), + } + + +def test_a_background_job_is_a_hand_off_from_the_tool_call_that_launched_it( + tmp_path: Path, +) -> None: + # A supervisor starts two jobs; each runs later as its own run (no parent here), named by the id its launch + # returned, with its tools attributed to the job's agent. A status check that echoes the id is not the launch. + records = _run( + tmp_path, + """ + sup = {"gen_ai.conversation.id": "thread-sup"} + with span("agent", **{"gen_ai.span.kind": "AGENT", **sup}): + with span("tools", **{"gen_ai.span.kind": "CHAIN", **sup}): + for job, kind in (("job-r", "researcher"), ("job-c", "coder")): + with span("start_async_task", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "start_async_task", + "input.value": kind, **sup, + "output.value": "Launched async subagent. task_id: " + job}): + pass + with span("agent", **{"gen_ai.span.kind": "AGENT", **sup}): + with span("check_async_task", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "check_async_task", + "output.value": '{"status": "running", "thread_id": "job-r"}', **sup}): + pass + for job, kind, tool in (("job-c", "coder", "run_code_check"), ("job-r", "researcher", "analyze_topic")): + own = {"lc_agent_name": kind, "gen_ai.conversation.id": job} + with span(kind, **{"gen_ai.span.kind": "CHAIN", "output.value": kind + " done", **own}): + with span("tools", **{"gen_ai.span.kind": "CHAIN", "langgraph_node": "tools", + "langgraph_checkpoint_ns": "tools:1", **own}): + with span(tool, **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": tool, "langgraph_node": "tools", + "langgraph_checkpoint_ns": "tools:1", **own}): + pass + """, + ) + launches = { + r["output"].rsplit(" ", 1)[-1]: r + for r in records + if r["name"] == "start_async_task" and not r.get("delegated_to") + } + hand_offs = {r["delegated_to"]: r for r in records if r.get("delegated_to")} + assert { + r["name"]: r.get("agent_key") + for r in records + if r["name"] in ("analyze_topic", "run_code_check") + } == { + "analyze_topic": "researcher", + "run_code_check": "coder", + } + assert sorted(hand_offs) == ["coder", "researcher"] + for worker, job in (("researcher", "job-r"), ("coder", "job-c")): + hand_off = hand_offs[worker] + assert (hand_off["name"], hand_off["agent_key"], hand_off["output"]) == ( + "start_async_task", + "agent", + worker + " done", + ) + assert hand_off["parent_event_id"] == launches[job]["event_id"] + assert hand_off["arguments"] == worker + assert 0 < hand_off["at"] <= hand_off["ended_at"] + + +def test_a_job_s_answer_is_what_its_model_last_said_not_its_whole_conversation( + tmp_path: Path, +) -> None: + records = _run( + tmp_path, + """ + sup = {"gen_ai.conversation.id": "thread-sup"} + with span("agent", **{"gen_ai.span.kind": "AGENT", **sup}): + with span("start_async_task", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "start_async_task", + "output.value": "Launched. task_id: job-c", **sup}): + pass + own = {"lc_agent_name": "coder", "gen_ai.conversation.id": "job-c"} + with span("coder", **{"gen_ai.span.kind": "CHAIN", "output.value": '{"messages": ["", ""]}', **own}): + for said in ("", ""): + with span("model", **{"gen_ai.span.kind": "LLM", "gen_ai.output.messages.0.message.content": said, **own}): + pass + """, + ) + assert [ + r["output"] for r in records if r.get("delegated_to") and "ended_at" in r + ] == [""] + + +def test_a_job_never_joins_a_launch_from_an_earlier_call(tmp_path: Path) -> None: + records = _run( + tmp_path, + """ + import os + sup = {"gen_ai.conversation.id": "thread-sup"} + with span("agent", **{"gen_ai.span.kind": "AGENT", **sup}): + with span("start_async_task", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "start_async_task", + "output.value": "Launched. task_id: job-c", **sup}): + pass + # The harness removes the trace between calls. + os.remove(os.environ["HARNESS_TOOL_TRACE"]) + own = {"lc_agent_name": "coder", "gen_ai.conversation.id": "job-c"} + with span("coder", **{"gen_ai.span.kind": "CHAIN", **own}): + with span("run_code_check", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "run_code_check", **own}): + pass + """, + ) + assert [(r["name"], r.get("delegated_to")) for r in records] == [ + ("run_code_check", None) + ] + + +def test_a_job_still_running_when_the_call_ends_keeps_its_hand_off( + tmp_path: Path, +) -> None: + # The job starts and makes a tool call, but has not finished when the process stops. + records = _run( + tmp_path, + """ + sup = {"gen_ai.conversation.id": "thread-sup"} + with span("agent", **{"gen_ai.span.kind": "AGENT", **sup}): + with span("start_async_task", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "start_async_task", + "input.value": "coder", **sup, + "output.value": "Launched async subagent. task_id: job-c"}): + pass + own = {"lc_agent_name": "coder", "gen_ai.conversation.id": "job-c"} + job = tracer.start_span("coder", attributes={"gen_ai.span.kind": "CHAIN", **own}) + with trace.use_span(job): + with span("run_code_check", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "run_code_check", **own}): + pass + """, + ) + hand_offs = [r for r in records if r.get("delegated_to")] + assert [ + (r["delegated_to"], r["arguments"], "ended_at" in r) for r in hand_offs + ] == [("coder", "coder", False)] + assert { + r["name"]: r.get("agent_key") for r in records if r["name"] == "run_code_check" + } == {"run_code_check": "coder"} + + +def test_a_sub_agent_run_through_a_tool_is_its_hand_off_and_owns_its_tool_calls( + tmp_path: Path, +) -> None: + # Deep Agents' `task` tool runs a sub-agent inside the supervisor's `tools` node. The tool span is marked a tool + # only when it ends; the sub-agent's nodes sit under the host node in the namespace and name their agent. A tool + # inside a named agent whose own inner run names the same agent is not a hand-off. + records = _run( + tmp_path, + """ + sup_tools = {"langgraph_node": "tools", "langgraph_checkpoint_ns": "tools:a"} + own = {"lc_agent_name": "researcher", "langgraph_node": "tools", "langgraph_checkpoint_ns": "tools:a|tools:b"} + with span("LangGraph", **{"gen_ai.span.kind": "CHAIN"}): + with span("tools", **{"gen_ai.span.kind": "CHAIN", **sup_tools}): + with span("task", **sup_tools) as task: + with span("researcher", **{"gen_ai.span.kind": "CHAIN", **own}): + with span("tools", **{"gen_ai.span.kind": "CHAIN", **own}): + with span("ls", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "ls", **own}): + pass + task.set_attribute("gen_ai.span.kind", "TOOL") + task.set_attribute("gen_ai.tool.name", "task") + task.set_attribute("output.value", "researcher answer") + named = {"lc_agent_name": "writer", "langgraph_node": "tools", "langgraph_checkpoint_ns": "writer:c|tools:d"} + with span("writer", **{"gen_ai.span.kind": "CHAIN", **named}): + with span("summarise", **named) as tool: + with span("summary chain", **{"gen_ai.span.kind": "CHAIN", **named}): + pass + tool.set_attribute("gen_ai.span.kind", "TOOL") + tool.set_attribute("gen_ai.tool.name", "summarise") + """, + ) + assert _by_tool(records) == { + "ls": ("researcher", None), + "task": (None, "researcher"), + "summarise": ("writer", None), + } + + +@pytest.mark.parametrize("inner_first", [False, True]) +def test_a_job_named_only_inside_it_that_starts_before_its_launch_returns_is_its_hand_off( + tmp_path: Path, inner_first: bool +) -> None: + # The job's run starts while the launching tool call is still returning, carries no attributes until it ends (as + # LangChain's instrumentation sets them), and is still running when the call ends: the spans inside it name it, + # whether they finish after the launch returns or before. + inner = """ +with trace.use_span(job): + with span("run_code_check", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "run_code_check", **own}): + pass +""" + launched = """ +launch.set_attribute("output.value", "Launched async subagent. task_id: job-c") +launch.end() +""" + records = _run( + tmp_path, + """ +sup = {"gen_ai.conversation.id": "thread-sup"} +own = {"lc_agent_name": "coder", "gen_ai.conversation.id": "job-c"} +agent = tracer.start_span("agent", attributes={"gen_ai.span.kind": "AGENT", **sup}) +launch = tracer.start_span("start_async_task", context=trace.set_span_in_context(agent), attributes={ + "gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "start_async_task", "input.value": "coder", **sup}) +job = tracer.start_span("coder", context=trace.set_span_in_context(trace.INVALID_SPAN)) +""" + + (inner + launched if inner_first else launched + inner) + + "agent.end()\n", + ) + hand_offs = [r for r in records if r.get("delegated_to")] + assert [ + (r["delegated_to"], r["agent_key"], "ended_at" in r) for r in hand_offs + ] == [("coder", "agent", False)] + assert { + r["name"]: r.get("agent_key") for r in records if r["name"] == "run_code_check" + } == {"run_code_check": "coder"} + + +def test_a_run_whose_conversation_no_tool_returned_is_not_a_hand_off( + tmp_path: Path, +) -> None: + records = _run( + tmp_path, + """ + with span("agent", **{"gen_ai.span.kind": "AGENT", "gen_ai.conversation.id": "thread-a"}): + with span("lookup", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "lookup", "output.value": "x", + "gen_ai.conversation.id": "thread-a"}): + pass + with span("agent", **{"gen_ai.span.kind": "AGENT", "gen_ai.conversation.id": "thread-a"}): + pass + """, + ) + assert [r.get("delegated_to") for r in records] == [None] + + +def test_an_agent_named_on_the_tool_span_or_by_operation_name_is_read( + tmp_path: Path, +) -> None: + records = _run( + tmp_path, + """ + with span("execute_tool search_notes", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "search_notes", + "gen_ai.agent.name": "research_worker"}): + pass + with span("invoke_agent", **{"gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "writer_worker"}): + with span("execute_tool", **{"gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "publish_note"}): + pass + with span("Task", **{"claude_agent.span_kind": "subagent", "claude_agent.subagent_type": "researcher"}): + with span("WebSearch", **{"claude_agent.span_kind": "tool", "tool.name": "WebSearch"}): + pass + """, + ) + assert _by_tool(records) == { + "search_notes": ("research_worker", None), + "publish_note": ("writer_worker", None), + "WebSearch": ("researcher", None), + } + + +def test_an_unnamed_agent_takes_the_tool_that_handed_it_work_and_otherwise_stays_unnamed( + tmp_path: Path, +) -> None: + records = _run( + tmp_path, + """ + with span("Agent.run", **{"gen_ai.span.kind": "AGENT"}): + with span("ask_researcher", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "ask_researcher"}): + with span("Agent.run", **{"gen_ai.span.kind": "AGENT"}): + with span("search", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "search"}): + pass + """, + ) + assert _by_tool(records) == { + "search": ("ask_researcher", None), + "ask_researcher": (None, "ask_researcher"), + } + + +def test_one_call_wrapped_by_two_instrumentations_is_written_once_and_a_failure_is_an_error( + tmp_path: Path, +) -> None: + records = _run( + tmp_path, + """ + with span("Worker", **{"gen_ai.span.kind": "AGENT", "agent.name": "Worker"}): + with span("ToolUsage._use", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "Publish note"}): + with span("publish_note", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "publish_note"}) as inner: + inner.set_status(Status(StatusCode.ERROR)) + with span("Delegate work", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "Delegate work"}): + with span("search_notes", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "search_notes"}): + pass + """, + ) + assert [(r["name"], r["is_error"]) for r in records] == [ + ("publish_note", True), + ("search_notes", False), + ("Delegate work", False), + ] + + +def test_an_autogen_core_tool_is_the_agent_the_runtime_handed_the_message_to( + tmp_path: Path, +) -> None: + records = _run( + tmp_path, + """ + process = {"messaging.operation": "process", "recipient_agent_type": "Writer", "sender_agent_type": "Triage"} + with span("autogen process Writer.(s1)-A", **process): + with span("execute_tool look_up_tide", **{"gen_ai.operation.name": "execute_tool", + "gen_ai.tool.name": "look_up_tide"}): + pass + with span("autogen publish Writer.(s1)-T", **{"messaging.operation": "publish"}): + with span("execute_tool draft_note", **{"gen_ai.operation.name": "execute_tool", + "gen_ai.tool.name": "draft_note"}): + pass + """, + ) + assert _by_tool(records) == { + "look_up_tide": ("Writer", None), + "draft_note": (None, None), + } + + +def test_a_process_that_sets_no_provider_is_still_observed(tmp_path: Path) -> None: + records = _run( + tmp_path, + """ + with span("Worker", **{"gen_ai.span.kind": "AGENT", "agent.name": "Worker"}): + with span("lookup", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "lookup"}): + pass + """, + provider="", + ) + assert _by_tool(records) == {"lookup": ("Worker", None)} + + +def test_a_process_that_sets_its_own_provider_later_keeps_it(tmp_path: Path) -> None: + records = _run( + tmp_path, + """ + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + own = TracerProvider(); memory = InMemorySpanExporter(); own.add_span_processor(SimpleSpanProcessor(memory)) + trace.set_tracer_provider(own) + with span("lookup", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "lookup"}): + pass + assert [s.name for s in memory.get_finished_spans()] == ["lookup"] + """, + provider="", + ) + assert [r["name"] for r in records] == ["lookup"] + + +def test_a_span_made_before_the_process_sets_its_provider_leaves_it_free_to_set_one( + tmp_path: Path, +) -> None: + records = _run( + tmp_path, + """ + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + with span("startup", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "startup"}): + pass + own = TracerProvider(); memory = InMemorySpanExporter(); own.add_span_processor(SimpleSpanProcessor(memory)) + trace.set_tracer_provider(own) + assert trace.get_tracer_provider() is own + with span("lookup", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "lookup"}): + pass + assert [s.name for s in memory.get_finished_spans()] == ["lookup"] + """, + provider="", + ) + assert [r["name"] for r in records] == ["startup", "lookup"] + + +@pytest.mark.parametrize("native_worker", ["", "Writer worker"]) +def test_a_span_line_lends_its_agent_to_another_tracer_s_line_for_the_same_call( + native_worker: str, +) -> None: + native = Call( + name="publish_note", + arguments={}, + result=None, + ok=True, + worker_label=native_worker, + at=2.0, + ) + span_line = Call( + name="Publish note", + arguments={}, + result=None, + ok=True, + worker_id="w", + worker_label="From span", + at=2.1, + ) + other = Call( + name="search_notes", + arguments={}, + result=None, + ok=True, + worker_id="w", + worker_label="From span", + at=1.0, + ) + merged = call_runner._merge_span_calls([native], [span_line, other]) + assert [(c.name, c.worker_label) for c in merged] == [ + ("search_notes", "From span"), + ("publish_note", native_worker or "From span"), + ] + + +def test_a_refusal_read_differently_by_the_two_tracers_is_still_one_call() -> None: + native = Call(name="issue_refund", arguments={}, result="refused", ok=False, at=2.0) + span_line = Call( + name="issue_refund", + arguments={}, + result="refused", + ok=True, + worker_label="Refunds", + at=2.0, + ) + merged = call_runner._merge_span_calls([native], [span_line]) + assert [(c.name, c.worker_label) for c in merged] == [("issue_refund", "Refunds")] + + +def test_a_span_pairs_with_the_call_of_its_own_outcome_when_one_tool_ran_twice() -> ( + None +): + passed = Call( + name="issue_refund", arguments={"order": "1"}, result="issued", ok=True, at=2.0 + ) + failed = Call( + name="issue_refund", + arguments={"order": "2"}, + result="refused", + ok=False, + at=3.0, + ) + span_line = Call( + name="issue_refund", + arguments={}, + result=None, + ok=False, + worker_label="Refunds", + at=2.9, + ) + merged = call_runner._merge_span_calls([passed, failed], [span_line]) + assert [(c.arguments, c.worker_label) for c in merged] == [ + ({"order": "1"}, ""), + ({"order": "2"}, "Refunds"), + ] + + +def test_a_call_another_tracer_saw_takes_the_span_s_event_ids_and_keeps_its_own() -> ( + None +): + plain = Call(name="lookup", arguments={}, result=None, ok=True, at=1.0) + own = Call( + name="lookup", + arguments={}, + result=None, + ok=True, + at=2.0, + runtime_ids={"event_id": "own"}, + ) + spans = [ + Call( + name="lookup", + arguments={}, + result=None, + ok=True, + at=1.0, + runtime_ids={"event_id": "s1", "parent_event_id": "a1"}, + ), + Call( + name="lookup", + arguments={}, + result=None, + ok=True, + at=2.0, + runtime_ids={"event_id": "s2"}, + ), + ] + merged = call_runner._merge_span_calls([plain, own], spans) + assert [c.runtime_ids for c in merged] == [ + {"event_id": "s1", "parent_event_id": "a1"}, + {"event_id": "own"}, + ] + + +def test_a_running_job_without_an_event_id_is_kept_beside_a_finished_one() -> None: + def job(name, **fields): + return Call( + name="start_async_task", + arguments={}, + result=None, + ok=True, + delegated_to=name, + **fields, + ) + + merged = call_runner._merge_span_calls( + [], [job("coder", at=1.0), job("researcher", at=2.0, ended_at=5.0)] + ) + assert [c.delegated_to for c in merged] == ["coder", "researcher"] + + +def test_span_lines_alone_read_in_the_order_the_calls_started() -> None: + late_end = Call(name="ask_research", arguments={}, result=None, ok=True, at=1.0) + inner = Call(name="search_notes", arguments={}, result=None, ok=True, at=1.5) + assert [c.name for c in call_runner._merge_span_calls([], [inner, late_end])] == [ + "ask_research", + "search_notes", + ] + + +def test_a_job_hand_off_stays_its_own_event_and_is_kept_once() -> None: + # The sandbox's tracer and the span both report the launch; the job's hand-off is written when it starts and again + # with its return, under the launch, and is never paired with the launch itself, whose event id it names. + def line(at, **fields): + return Call( + name="start_async_task", arguments={}, result=None, ok=True, at=at, **fields + ) + + natives = [line(1.0), line(1.1)] + first = line(1.0, runtime_ids={"event_id": "l1"}) + started = line( + 1.5, + delegated_to="coder", + runtime_ids={"event_id": "j1", "parent_event_id": "l1"}, + ) + returned = line( + 1.5, + delegated_to="coder", + ended_at=9.0, + runtime_ids={"event_id": "j1", "parent_event_id": "l1"}, + ) + second = line(1.1, runtime_ids={"event_id": "l2"}) + merged = call_runner._merge_span_calls(natives, [first, started, returned, second]) + assert [ + (c.delegated_to, c.ended_at, c.runtime_ids.get("event_id")) for c in merged + ] == [ + ("", 0.0, "l1"), + ("", 0.0, "l2"), + ("coder", 9.0, "j1"), + ] + + +_AGENTS_SDK = { + "agents/__init__.py": "from agents.tracing import get_trace_provider\n\n" + "def set_trace_processors(processors):\n get_trace_provider().set_processors(processors)\n", + "agents/tracing/__init__.py": "from agents.tracing.provider import DefaultTraceProvider\n_PROVIDER = DefaultTraceProvider()\n\n" + "def get_trace_provider():\n return _PROVIDER\n", + "agents/tracing/provider.py": "class SynchronousMultiTracingProcessor:\n _processors = ()\n\n" + " def set_processors(self, processors):\n self._processors = tuple(processors)\n\n\n" + "class DefaultTraceProvider:\n def __init__(self):\n self._multi_processor = SynchronousMultiTracingProcessor()\n\n" + " def set_processors(self, processors):\n self._multi_processor.set_processors(processors)\n", + "traceai_openai_agents/__init__.py": "class FiTracingProcessor:\n pass\n\n\nclass OpenAIAgentsInstrumentor:\n" + " def instrument(self):\n from agents import set_trace_processors\n" + " set_trace_processors([FiTracingProcessor()])\n", +} + + +@pytest.mark.parametrize( + ("app", "kept"), + [ + ("set_trace_processors([])", ["FiTracingProcessor"]), + ("set_trace_processors([Own()])", ["Own", "FiTracingProcessor"]), + ( + "from traceai_openai_agents import FiTracingProcessor\nset_trace_processors([FiTracingProcessor()])", + ["FiTracingProcessor"], + ), + ], +) +def test_an_openai_agents_app_that_sets_its_own_trace_processors_is_still_observed_once( + tmp_path: Path, app: str, kept: list[str] +) -> None: + for path, source in { + **_AGENTS_SDK, + "hook/sitecustomize.py": "import simulation_mark_bootstrap\n" + "simulation_mark_bootstrap.install()\n", + }.items(): + (tmp_path / path).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / path).write_text(source) + _place_hook(tmp_path / "hook") + script = ( + f"from agents import get_trace_provider, set_trace_processors\nclass Own: pass\n{app}\n" + "print([type(p).__name__ for p in get_trace_provider()._multi_processor._processors])" + ) + result = subprocess.run( + [sys.executable, "-c", script], + env={ + **os.environ, + "PYTHONPATH": f"{tmp_path / 'hook'}:{tmp_path}", + "ALK_OBSERVE_FRAMEWORKS": "1", + "HARNESS_TOOL_TRACE": str(tmp_path / "calls.jsonl"), + }, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, result.stderr + assert "hook: failed" not in result.stderr + assert json.loads(result.stdout.strip().replace("'", '"')) == kept + + +_AUTOGEN_CORE = { + "autogen_core/__init__.py": "", + "autogen_core/tools/__init__.py": "class BaseTool:\n def __init__(self, name, run):\n" + " self.name, self._run = name, run\n\n" + " async def run_json(self, args, cancellation_token, call_id=None):\n return self._run(**args)\n\n" + " def return_value_as_string(self, value):\n return str(value)\n", +} + + +def test_an_autogen_tool_call_is_written_with_its_arguments_and_result_beside_its_span( + tmp_path: Path, +) -> None: + for path, source in { + **_AUTOGEN_CORE, + "hook/sitecustomize.py": "import simulation_mark_bootstrap\n" + "simulation_mark_bootstrap.install()\n", + }.items(): + (tmp_path / path).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / path).write_text(source) + _place_hook(tmp_path / "hook") + script = textwrap.dedent(""" + import asyncio + from autogen_core.tools import BaseTool + def fail(note): + raise ValueError("no such note") + async def main(): + await BaseTool("look_up_tide", lambda place: "high at 06:10").run_json({"place": "North Quay"}, None) + try: + await BaseTool("read_note", fail).run_json({"note": "n1"}, None) + except ValueError: + pass + asyncio.run(main()) + """) + trace = tmp_path / "calls.jsonl" + result = subprocess.run( + [sys.executable, "-c", script], + env={ + **os.environ, + "PYTHONPATH": f"{tmp_path / 'hook'}:{tmp_path}", + "ALK_OBSERVE_FRAMEWORKS": "1", + "HARNESS_TOOL_TRACE": str(trace), + }, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, result.stderr + assert "hook: failed" not in result.stderr + lines = [json.loads(line) for line in trace.read_text().splitlines()] + assert [ + (one["name"], one["arguments"], one["output"], one["is_error"]) for one in lines + ] == [ + ("look_up_tide", {"place": "North Quay"}, "high at 06:10", False), + ("read_note", {"note": "n1"}, "no such note", True), + ] + + +def test_checks_grade_a_launch_once_and_a_hand_off_made_by_a_tool_call() -> None: + from fi.alk.harness.world.runtime import graded_calls + + launch = Call(name="start_async_task", arguments={}, result=None, ok=True, at=1.0) + job = Call( + name="start_async_task", + arguments={}, + result=None, + ok=True, + at=1.5, + delegated_to="coder", + runtime_ids={"event_id": "j1", "parent_event_id": "l1"}, + ) + task = Call( + name="task", + arguments={}, + result=None, + ok=True, + at=2.0, + delegated_to="researcher", + runtime_ids={"event_id": "t1", "parent_event_id": "a1"}, + ) + assert graded_calls([launch, job, task]) == [launch, task] + + +def test_a_job_joins_the_launch_that_named_its_whole_id() -> None: + from fi.alk.harness.simulation_worker_events import _names + + assert _names('{"task_id": "1"}', "1") + assert _names("job a1b2-c3 started", "a1b2-c3") + assert not _names('{"task_id": "17"}', "1") + assert not _names("thread_1x", "1") + + +def test_spans_that_never_end_do_not_stop_observation(monkeypatch) -> None: + from types import SimpleNamespace + + from fi.alk.harness import simulation_worker_events as bootstrap + + monkeypatch.setattr(bootstrap, "_LIVE_SPANS_MAX", 2) + observer = bootstrap._ToolSpans("/dev/null") + + def span(span_id): + return SimpleNamespace( + get_span_context=lambda: SimpleNamespace(span_id=span_id), + parent=SimpleNamespace(span_id=0), + attributes={}, + name="s", + end=lambda: None, + ) + + for span_id in (1, 2, 3): + observer.started(span(span_id)) + assert list(observer._live) == [2, 3] + + +def test_a_span_starting_or_ending_waits_for_another_thread_s_bookkeeping( + tmp_path: Path, +) -> None: + import threading + from types import SimpleNamespace + + from fi.alk.harness import simulation_worker_events as bootstrap + + observer = bootstrap._ToolSpans(str(tmp_path / "calls.jsonl")) + one = SimpleNamespace( + get_span_context=lambda: SimpleNamespace(span_id=1), + parent=None, + attributes={}, + name="s", + end=lambda: None, + status=None, + start_time=0, + end_time=0, + ) + for step in (lambda: observer.started(one), lambda: one.end()): + with observer._lock: + thread = threading.Thread(target=step) + thread.start() + thread.join(0.2) + assert thread.is_alive() + thread.join(5) + assert not thread.is_alive() + + +def test_the_process_instrumenting_after_the_hook_with_its_own_provider_gets_it() -> ( + None +): + from fi.alk.harness.simulation_mark_bootstrap import _yield_to_process + + class Instrumentor: + # As opentelemetry's BaseInstrumentor: a second instrument() is ignored until uninstrument(). + is_instrumented_by_opentelemetry = False + settings = None + + def instrument(self, **kwargs): + if self.is_instrumented_by_opentelemetry: + return + self.settings, self.is_instrumented_by_opentelemetry = kwargs, True + + def uninstrument(self): + self.is_instrumented_by_opentelemetry = False + + ours = Instrumentor() + ours.instrument() + _yield_to_process(Instrumentor) + ours.instrument() + assert ours.settings == {} + ours.instrument(tracer_provider="own") + assert ours.settings == {"tracer_provider": "own"} + + +def test_the_hook_turns_on_every_instrumentor_the_build_installs() -> None: + from fi.alk.harness.generated_runtime import FRAMEWORK_INSTRUMENTORS + + built = { + one.split("==")[0].replace("-", "_") + for _framework, one in FRAMEWORK_INSTRUMENTORS + if one.startswith("traceai") + } + assert built == { + module for module, _cls in simulation_mark_bootstrap._INSTRUMENTORS.values() + } + + +def test_a_framework_whose_instrumentor_is_missing_is_reported_unobserved( + tmp_path: Path, +) -> None: + hook = tmp_path / "hook" + hook.mkdir() + _place_hook(hook) + (hook / "sitecustomize.py").write_text( + "import simulation_mark_bootstrap\nsimulation_mark_bootstrap.install()\n" + ) + (hook / "smolagents.py").write_text("") + result = subprocess.run( + [sys.executable, "-c", "import smolagents"], + env={ + **os.environ, + "PYTHONPATH": str(hook), + "HARNESS_TOOL_TRACE": str(tmp_path / "calls.jsonl"), + "ALK_OBSERVE_FRAMEWORKS": "1", + }, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0 + assert ( + "simulation mark hook: unobserved smolagents, its instrumentor is not installed" + in result.stderr + ) + + +def test_two_open_hand_offs_to_one_worker_each_hold_their_own_calls( + tmp_path: Path, +) -> None: + # The coordinator asks the same worker twice at once; the two runs end in the other order. + _run( + tmp_path, + """ + from opentelemetry.trace import set_span_in_context + lead = tracer.start_span("Lead", attributes={"gen_ai.span.kind": "AGENT", "gen_ai.agent.name": "Lead"}) + asks, runs, searches = [], [], [] + for query in ("q1", "q2"): + ask = tracer.start_span("ask_research", context=set_span_in_context(lead), attributes={ + "gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "ask_research", "input.value": query}) + run = tracer.start_span("Research", context=set_span_in_context(ask), attributes={ + "gen_ai.span.kind": "AGENT", "gen_ai.agent.name": "Research"}) + searches.append(tracer.start_span("search_notes", context=set_span_in_context(run), attributes={ + "gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "search_notes", "input.value": query})) + asks.append(ask) + runs.append(run) + for one in (1, 0): + searches[one].end() + runs[one].end() + asks[one].end() + lead.end() + """, + ) + rows = _sealed(tmp_path) + asks = { + row["arguments"]["raw"]: row for row in rows if row["name"] == "ask_research" + } + searches = { + row["arguments"]["raw"]: row for row in rows if row["name"] == "search_notes" + } + for query in ("q1", "q2"): + assert _placed([asks[query], searches[query]]) == [ + ("ask_research", "delegated", "Lead", "observed", ""), + ( + "search_notes", + "tool_call", + "Research", + "observed", + asks[query]["span_id"], + ), + ] + assert asks["q1"]["span_id"] != asks["q2"]["span_id"] + + +_ROOT_TRANSFER = """ +from opentelemetry.trace import set_span_in_context +root = tracer.start_span("invoke_agent root_agent", attributes={"gen_ai.operation.name": "invoke_agent"}) +root.set_attribute("gen_ai.agent.name", "root_agent") +transfer = tracer.start_span("execute_tool transfer_to_agent", context=set_span_in_context(root), attributes={ + "gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "transfer_to_agent", + "gen_ai.tool.type": "TOOL_TYPE", "gcp.vertex.agent.tool_call_args": '{"agent_name": "leaf_agent"}'}) +transfer.end() +leaf = tracer.start_span("invoke_agent leaf_agent", context=set_span_in_context(root), attributes={ + "gen_ai.operation.name": "invoke_agent"}) +# As the framework does, the agent is named once its span has started. +leaf.set_attribute("gen_ai.agent.name", "leaf_agent") +with trace.use_span(leaf, end_on_exit=True): + with span("execute_tool look_up_tide", **{"gen_ai.operation.name": "execute_tool", + "gen_ai.tool.name": "look_up_tide"}): + pass +root.end() +""" + + +def test_a_framework_s_own_transfer_is_a_control_pass_and_holds_the_leaf_worker_s_calls( + tmp_path: Path, +) -> None: + records = _run(tmp_path, _ROOT_TRANSFER.replace("TOOL_TYPE", "TransferToAgentTool")) + transfer = next(r for r in records if r["name"] == "transfer_to_agent") + assert (transfer["transfer"], transfer["delegated_to"], "ended_at" in transfer) == ( + True, + "leaf_agent", + False, + ) + assert _placed(_sealed(tmp_path)) == [ + ("transfer_to_agent", "transferred", "root_agent", "observed", ""), + ("look_up_tide", "tool_call", "leaf_agent", "observed", transfer["event_id"]), + ] + + +def test_adk_s_built_in_transfer_recorded_as_a_plain_function_is_a_control_pass_once_the_named_agent_runs( + tmp_path: Path, +) -> None: + # Older ADK releases record their own transfer_to_agent as a FunctionTool; the named agent then starting in the + # same run is the evidence that control passed. + records = _run(tmp_path, _ROOT_TRANSFER.replace("TOOL_TYPE", "FunctionTool")) + transfer = next(r for r in records if r["name"] == "transfer_to_agent") + assert (transfer.get("transfer"), transfer["delegated_to"]) == (True, "leaf_agent") + assert _placed(_sealed(tmp_path)) == [ + ("transfer_to_agent", "transferred", "root_agent", "observed", ""), + ("look_up_tide", "tool_call", "leaf_agent", "observed", transfer["event_id"]), + ] + + +def test_a_transfer_named_call_whose_agent_never_runs_is_a_tool_call_and_the_leaf_s_place_stays_unknown( + tmp_path: Path, +) -> None: + # The call names an agent that never starts: nothing records a control pass, so it stays an ordinary tool call + # and the agent that did run is not placed under it. + _run( + tmp_path, + _ROOT_TRANSFER.replace("TOOL_TYPE", "FunctionTool").replace( + '{"agent_name": "leaf_agent"}', '{"agent_name": "billing_agent"}' + ), + ) + assert _placed(_sealed(tmp_path)) == [ + ("transfer_to_agent", "tool_call", "root_agent", "observed", ""), + ("look_up_tide", "tool_call", "leaf_agent", "unknown", ""), + ] + + +def test_the_top_agent_under_the_caller_s_context_hands_off_at_the_top_level( + tmp_path: Path, +) -> None: + # The app's coordinator runs inside a context its own tracer never recorded (the request that reached it): the + # coordinator is the top of the app's work, so its hand-off is observed at the top level. + _run( + tmp_path, + """ + from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags, set_span_in_context + outside = NonRecordingSpan(SpanContext(trace_id=7, span_id=99, is_remote=True, trace_flags=TraceFlags(1))) + lead = tracer.start_span("Lead", context=set_span_in_context(outside), attributes={ + "gen_ai.span.kind": "AGENT", "gen_ai.agent.name": "Lead"}) + ask = tracer.start_span("ask_research", context=set_span_in_context(lead), attributes={ + "gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "ask_research", "input.value": "q"}) + run = tracer.start_span("Research", context=set_span_in_context(ask), attributes={ + "gen_ai.span.kind": "AGENT", "gen_ai.agent.name": "Research"}) + run.end() + ask.end() + lead.end() + """, + ) + assert _placed(_sealed(tmp_path)) == [ + ("ask_research", "delegated", "Lead", "observed", ""), + ] + + +def test_a_call_with_no_agent_above_it_has_an_unknown_place(tmp_path: Path) -> None: + _run( + tmp_path, + """ + with span("execute_tool search_notes", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "search_notes", + "gen_ai.agent.name": "research_worker"}): + pass + """, + ) + assert _placed(_sealed(tmp_path)) == [ + ("search_notes", "tool_call", "research_worker", "unknown", "") + ] + + +def test_an_agent_the_framework_records_as_handed_control_holds_its_calls( + tmp_path: Path, +) -> None: + # The OpenAI Agents SDK runs each agent at the top of the trace and names, on the agent handed control, the agent + # that handed it, only when that agent's span ends. + records = _run( + tmp_path, + """ + with span("Triage", **{"fi.span.kind": "AGENT", "graph.node.id": "Triage"}): + with span("lookup", **{"fi.span.kind": "TOOL", "gen_ai.tool.name": "lookup"}): + pass + with span("Refunds", **{"fi.span.kind": "AGENT", "graph.node.id": "Refunds"}) as refunds: + with span("issue_refund", **{"fi.span.kind": "TOOL", "gen_ai.tool.name": "issue_refund"}): + pass + refunds.set_attribute("graph.node.parent_id", "Triage") + """, + ) + passed = next(r for r in records if r.get("transfer")) + rows = sorted(_sealed(tmp_path), key=lambda row: row["name"]) + assert _placed(rows) == [ + ("Refunds", "transferred", "Triage", "unknown", ""), + ("issue_refund", "tool_call", "Refunds", "observed", passed["event_id"]), + ("lookup", "tool_call", "Triage", "observed", ""), + ] + assert "ended_at" not in rows[0] + + +def test_each_payload_says_whether_part_of_it_was_left_out(tmp_path: Path) -> None: + records = _run( + tmp_path, + """ + with span("Worker", **{"gen_ai.span.kind": "AGENT", "gen_ai.agent.name": "Worker"}): + with span("long_in", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "long_in", + "input.value": "a" * 5000, "output.value": "ok"}): + pass + with span("long_out", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "long_out", + "input.value": "{}", "output.value": "b" * 5000}): + pass + with span("fits", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "fits", + "input.value": "{}", "output.value": "ok"}): + pass + """, + ) + assert { + r["name"]: (r.get("input_omitted", False), r.get("output_omitted", False)) + for r in records + } == { + "long_in": (True, False), + "long_out": (False, True), + "fits": (False, False), + } + assert { + row["name"]: (row.get("input_omitted", False), row.get("output_omitted", False)) + for row in _sealed(tmp_path) + } == {"long_in": (True, False), "long_out": (False, True), "fits": (False, False)} + + +def test_a_parent_no_recorded_hand_off_carries_is_unknown_not_kept() -> None: + # The hand-off a call ran in had not ended when the call did, so no line records it. + leaf = Call( + name="search_notes", + arguments={}, + worker_id="Research", + runtime_ids={"event_id": "b"}, + parent_span_id="a", + relationship="observed", + at=1.0, + ) + (merged,) = call_runner._merge_span_calls([], [leaf]) + assert { + key: trace_row(merged)[key] for key in ("parent_span_id", "relationship") + } == {"parent_span_id": "", "relationship": "unknown"} + + +_OPENAI_AGENTS_RUN = """ +from opentelemetry.trace import set_span_in_context +# The OpenAI Agents SDK opens each run as an agent-kind span with no agent name; agents run inside it. +# Between them sit the SDK's own run and turn spans, as traceAI records a real Runner.run. +run = tracer.start_span("Agent workflow", attributes={"gen_ai.span.kind": "AGENT"}) +steps = tracer.start_span("Agent workflow", context=set_span_in_context(run), attributes={"gen_ai.span.kind": "CHAIN"}) +lead = tracer.start_span("Lead", context=set_span_in_context(steps), attributes={"gen_ai.span.kind": "AGENT"}) +turn = tracer.start_span("turn", context=set_span_in_context(lead), attributes={"gen_ai.span.kind": "CHAIN"}) +ask = tracer.start_span("ask_research", context=set_span_in_context(turn), attributes={ + "gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "ask_research", "input.value": "q"}) +worker = tracer.start_span("Research", context=set_span_in_context(ask), attributes={"gen_ai.span.kind": "AGENT"}) +worker.end() +ask.end() +turn.end() +lead.end() +LATER +steps.end() +run.end() +""" + + +def test_the_first_agent_in_an_openai_agents_run_hands_off_at_the_top_level( + tmp_path: Path, +) -> None: + _run(tmp_path, _OPENAI_AGENTS_RUN.replace("LATER", "")) + assert _placed(_sealed(tmp_path)) == [ + ("ask_research", "delegated", "Lead", "observed", ""), + ] + + +def test_a_later_agent_in_the_same_run_without_a_recorded_transfer_stays_unknown( + tmp_path: Path, +) -> None: + # A second agent opened in the run was handed control; until the framework records who handed it, its calls are + # not placed at the top. + later = """ +billing = tracer.start_span("Billing", context=set_span_in_context(steps), attributes={"gen_ai.span.kind": "AGENT"}) +with trace.use_span(billing, end_on_exit=True): + with span("refund", **{"gen_ai.span.kind": "TOOL", "gen_ai.tool.name": "refund"}): + pass +""" + _run(tmp_path, _OPENAI_AGENTS_RUN.replace("LATER", later)) + refund = next(r for r in _sealed(tmp_path) if r["name"] == "refund") + assert refund["relationship"] == "unknown" + + +def test_a_transfer_recorded_by_another_instrumentation_counts_once_the_named_agent_runs_under_a_model_call( + tmp_path: Path, +) -> None: + # traceAI's ADK instrumentation records the transfer as a plain tool with its arguments in `input.value`, under the + # model call; the named agent then runs inside the root agent with no tool calls of its own. + _run( + tmp_path, + """ + from opentelemetry.trace import set_span_in_context + root = tracer.start_span("root_agent", attributes={"fi.span.kind": "AGENT", "gen_ai.agent.name": "root_agent"}) + llm = tracer.start_span("call_llm", context=set_span_in_context(root), attributes={"fi.span.kind": "LLM"}) + move = tracer.start_span("transfer_to_agent", context=set_span_in_context(llm), attributes={ + "fi.span.kind": "TOOL", "tool.name": "transfer_to_agent", "input.value": '{"agent_name": "leaf_agent"}'}) + move.end() + llm.end() + leaf = tracer.start_span("leaf_agent", context=set_span_in_context(root), attributes={ + "fi.span.kind": "AGENT", "gen_ai.agent.name": "leaf_agent"}) + leaf.end() + root.end() + """, + ) + assert _placed(_sealed(tmp_path)) == [ + ("transfer_to_agent", "transferred", "root_agent", "observed", ""), + ] + + +def test_an_adk_transfer_without_its_arguments_hands_control_to_the_agent_that_runs_next_in_the_workflow( + tmp_path: Path, +) -> None: + # ADK 2 records the transfer tool without its arguments unless content capture is on, ends the root agent, then + # runs the agent it transferred to beside it in the same workflow span. + _run( + tmp_path, + """ + from opentelemetry.trace import set_span_in_context + flow = tracer.start_span("invoke_workflow root_agent", attributes={"gen_ai.operation.name": "invoke_workflow", + "gen_ai.span.kind": "CHAIN"}) + root = tracer.start_span("invoke_agent root_agent", context=set_span_in_context(flow), attributes={ + "gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "root_agent", "gen_ai.span.kind": "AGENT"}) + move = tracer.start_span("execute_tool transfer_to_agent", context=set_span_in_context(root), attributes={ + "gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "transfer_to_agent", + "gen_ai.tool.type": "TransferToAgentTool", "gen_ai.span.kind": "TOOL"}) + move.end() + root.end() + leaf = tracer.start_span("invoke_agent leaf_agent", context=set_span_in_context(flow), attributes={ + "gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "leaf_agent", "gen_ai.span.kind": "AGENT"}) + with trace.use_span(leaf, end_on_exit=True): + with span("execute_tool look_up_tide", **{"gen_ai.operation.name": "execute_tool", + "gen_ai.tool.name": "look_up_tide", "gen_ai.span.kind": "TOOL"}): + pass + flow.end() + """, + ) + rows = _sealed(tmp_path) + transfer = next(r for r in rows if r["name"] == "transfer_to_agent") + assert transfer["delegated_to"] == "leaf_agent" + assert _placed(rows) == [ + ("transfer_to_agent", "transferred", "root_agent", "observed", ""), + ("look_up_tide", "tool_call", "leaf_agent", "observed", transfer["span_id"]), + ] diff --git a/tests/harness/test_grade.py b/tests/harness/test_grade.py index 0a727babc..8a9aff88c 100644 --- a/tests/harness/test_grade.py +++ b/tests/harness/test_grade.py @@ -566,3 +566,39 @@ def test_a_scenario_with_one_sub_goal_is_kept(): catalogue = Catalogue(sub_goals=[SubGoal(name="change_made_as_asked", what="the change", judged="j")]) one = Scenario(name="makes_a_change", instruction="Ask for a change.", sub_goals=["change_made_as_asked"]) assert not any("sub-goal" in line for line in validate_scenario(one, catalogue, {})) + + +def test_a_sub_goal_can_check_one_declared_worker_and_never_an_undeclared_one(tmp_path): + import asyncio + + import fi.alk.harness.scenario_tools as st + from fi.alk.harness.catalogue import load_catalogue + from fi.alk.harness.contract import AgentContract + + contract = AgentContract( + agent="returns", real_use_cases=["refund"], tools=[{"name": "issue_refund"}], + workers=[{"id": "refund", "tools": ["issue_refund"]}], + ) + server, _kept = st.scenario_tools(contract, tmp_path, tmp_path, wanted=1) + add = next(spec for spec in server.tools if spec.name == "add_sub_goal") + check = ( + "def check(world, calls):\n" + " ok = [c for c in calls if c.name == 'issue_refund' and c.ok and c.arguments.get('order_id') == '1043']\n" + " return None if ok else 'order 1043 was not refunded'\n" + ) + said = asyncio.run(add.handler({"name": "refund_issued", "what": "the refund worker refunds", "check": check, + "subject": "worker:refund"})) + assert not said.get("is_error"), said + assert load_catalogue(tmp_path).sub_goals[0].subject == "worker:refund" + refused = asyncio.run(add.handler({"name": "audit", "what": "audits", "check": check, "subject": "worker:audit"})) + assert refused.get("is_error") and "subject_unknown_worker: audit" in refused["content"][0]["text"] + + +def test_a_scenario_writer_and_an_editor_can_set_a_worker_budget(tmp_path): + import fi.alk.harness.scenario_tools as st + from fi.alk.harness.contract import AgentContract + + server, _kept = st.scenario_tools(AgentContract(agent="returns", real_use_cases=["refund"]), tmp_path, tmp_path, wanted=1) + tools = {spec.name: spec for spec in server.tools} + for name in ("submit_scenario", "edit_scenario"): + assert "integer" in tools[name].input_schema["properties"]["worker_budget_seconds"]["type"] diff --git a/tests/harness/test_hosted_scheduler.py b/tests/harness/test_hosted_scheduler.py index eb6c6d504..0424661be 100644 --- a/tests/harness/test_hosted_scheduler.py +++ b/tests/harness/test_hosted_scheduler.py @@ -2626,6 +2626,41 @@ async def run( asyncio.run(scenario()) +def test_a_call_with_only_worker_hand_offs_is_evidence_missing() -> None: + # A worker's run is not a tool call, so a call that made none still has no evidence for checks. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + handoff = hs.Call( + name="Draft writer", arguments={}, delegated_to="Draft writer", handoff=True + ) + + class Runner: + async def run( + self, scenario: FakeScenario, runtime: EnvironmentRuntime + ) -> hs.CallOutcome: + return _call_outcome(calls=(handoff,)) + + scheduler = hs.HostedScheduler( + pool=pool, + world_factory=FakeWorldFactory(), + call_runner=Runner(), + outbound=outbound, + job_seed=1, + ) + scenarios = [ + FakeScenario( + "s1", "id-1", sub_goals=[FakeSubGoal("goal", lambda w, c: None)] + ) + ] + result = await scheduler.run(scenarios) + assert result.receipts[0].failure.code == "evidence_missing" + await pool.close() + + asyncio.run(scenario()) + + def test_zero_turns_and_zero_calls_is_still_evidence_missing() -> None: # M10: unconditioned on `turns` — a 0-turn call is an unobserved agent, same as any other # empty-calls outcome; `evidence_missing` must fire regardless. @@ -4780,3 +4815,100 @@ def test_a_sub_goal_about_an_actor_is_settled_by_whether_the_actor_came_in(monke assert [one.verdict for one in receipt.sub_goals] == ["pass", verdict] if status == "errored": assert receipt.failure.code == "actor_missing" and "asked_the_friend" in receipt.failure.message + + +def _graded(goals, outcome): + async def scenario(): + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + + class Runner: + async def run(self, scenario, runtime): + return outcome + + scheduler = hs.HostedScheduler( + pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, job_seed=1 + ) + result = await scheduler.run([FakeScenario("s1", "id-1", sub_goals=goals)]) + await pool.close() + return result.receipts[0] + + return asyncio.run(scenario()) + + +@dataclass +class _WorkerGoal(FakeSubGoal): + worker: str = "" + + @property + def subject(self) -> str: + return f"worker:{self.worker}" if self.worker else "task" + + +def test_a_worker_s_check_reads_only_that_worker_s_calls_and_without_any_is_inconclusive() -> None: + calls = ( + hs.Call(name="get_order", arguments={}, worker_id="order_lookup"), + hs.Call(name="issue_refund", arguments={}, worker_id="refund"), + hs.Call(name="send_reply", arguments={}), + ) + seen: dict[str, list[str]] = {} + + def lane(name): + def check(world, given): + seen[name] = [one.name for one in given] + + return check + + receipt = _graded( + [ + _WorkerGoal("lookup_read_only", lane("lookup"), worker="order_lookup"), + _WorkerGoal("task_done", lane("task")), + _WorkerGoal("audit_logged", lambda w, c: None, worker="audit"), + ], + _call_outcome(turns=2, calls=calls), + ) + assert seen == {"lookup": ["get_order"], "task": ["get_order", "issue_refund", "send_reply"]} + lookup, task, audit = receipt.sub_goals + assert (lookup.verdict, task.verdict) == ("pass", "pass") + assert (lookup.subject, task.subject, audit.subject) == ("worker:order_lookup", "task", "worker:audit") + # Absence is never a pass: a worker with nothing in the world's log cannot be graded. + assert (audit.verdict, audit.held) == ("inconclusive", None) + assert audit.reason == "Inconclusive: worker audit made no tool call the world recorded" + + +def test_a_worker_s_failed_check_is_reported_beside_the_task_s_verdict_not_folded_into_it() -> None: + calls = (hs.Call(name="update_order_status", arguments={}, worker_id="order_lookup"),) + outcome = _call_outcome(turns=2, calls=calls) + receipt = _graded( + [ + _WorkerGoal("lookup_read_only", lambda w, c: "the lookup worker wrote an order", worker="order_lookup"), + _WorkerGoal("audit_logged", lambda w, c: None, worker="audit"), + _WorkerGoal("reply_truthful", lambda w, c: None), + ], + outcome, + ) + worker, audit, task = receipt.sub_goals + # Neither a worker's failure nor a worker that left nothing to grade decides the task. + assert (worker.verdict, audit.verdict, task.verdict, receipt.status) == ("fail", "inconclusive", "pass", "passed") + failed = _graded([_WorkerGoal("reply_truthful", lambda w, c: "claimed a refund")], outcome) + assert failed.status == "failed" + + +def test_a_worker_that_passed_its_deadline_has_its_own_checks_inconclusive_and_the_task_still_graded() -> None: + calls = ( + hs.Call(name="get_order", arguments={}, worker_id="order_lookup"), + hs.Call(name="issue_refund", arguments={}, worker_id="refund", ok=False, refused=True, + refusal_code="deadline_passed"), + ) + receipt = _graded( + [ + _WorkerGoal("refund_done", lambda w, c: "no refund", worker="refund"), + _WorkerGoal("lookup_done", lambda w, c: None, worker="order_lookup"), + _WorkerGoal("reply_truthful", lambda w, c: None), + ], + _call_outcome(turns=2, calls=calls), + ) + refund, lookup, task = receipt.sub_goals + assert (refund.verdict, lookup.verdict, task.verdict) == ("inconclusive", "pass", "pass") + assert refund.reason == "Inconclusive: worker refund passed its deadline, so its result is partial" diff --git a/tests/harness/test_http_native_response.py b/tests/harness/test_http_native_response.py index 48e5c7de0..4a43b69a3 100644 --- a/tests/harness/test_http_native_response.py +++ b/tests/harness/test_http_native_response.py @@ -11,7 +11,7 @@ from fi.alk.harness.chat_transport import create_chat_wrapper from fi.alk.harness.contract import RuntimeInterface, RuntimeHttpFailureRule from fi.simulate.agent.wrapper import AgentInput -from fi.simulate.agent.wrappers.http import HTTPAgentWrapper +from fi.simulate.agent.wrappers.http import HTTPAgentWrapper, _decode_json_or_sse, _select_response_value from fi.simulate.agent.wrappers.http_events import CompletedToolEvents, CompletedToolRows @@ -665,3 +665,88 @@ def test_result_error_selector_preserves_keys_and_numeric_indices(native_app, pa assert response.metadata["external_agent"]["success"] is True assert response.metadata["native_completed_tools"][0]["ok"] is False assert response.metadata["native_completed_tools"][0]["result"] == result + + +def test_a_reply_reads_as_json_server_sent_events_or_json_lines_and_nothing_else(): + assert _decode_json_or_sse('{"content": "done"}') == {"content": "done"} + assert _decode_json_or_sse('data: {"content": "a"}\n\ndata: [DONE]\n') == [ + {"content": "a"} + ] + assert _decode_json_or_sse('{"content": "a"}\n{"content": "b"}\n') == [ + {"content": "a"}, + {"content": "b"}, + ] + for body in ("plain words\n", "", '{"content": "a"}\nnot json\n'): + with pytest.raises(ValueError, match="JSON lines"): + _decode_json_or_sse(body) + + +@pytest.mark.parametrize( + ("body", "reply"), + [ + ( + '{"content": {"text": "Tides "}}\n{"content": {"text": "turn"}}\n{"content": {"kind": "end"}}\n', + "Tides turn", + ), + ( + 'data: {"content": {"text": "Tides"}}\n\ndata: {"content": {"text": "Tides turn"}}\n\n', + "Tides turn", + ), + ( + 'data: {"choices": [{"delta": {"content": "Tides "}}]}\n\ndata: {"choices": [{"delta": {"content": "turn"}}]}\n\n' + 'data: {"choices": [{"delta": {"content": null}}]}\n\ndata: [DONE]\n\n', + "Tides turn", + ), + ('[{"content": {"text": "first"}}, {"content": {"text": "last"}}]', "last"), + ( + 'data: {"content": {"text": "Tides "}}\n\ndata: {"content": {"text": "turn"}}\n\n' + 'data: {"content": {"text": "Tides turn"}}\n\n', + "Tides turn", + ), + ( + '{"content": {"text": "."}}\n{"content": {"text": "."}}\n{"content": {"text": "."}}\n', + "...", + ), + ( + '{"content": {"text": "a"}}\n{"content": {"text": "b"}}\n{"content": {"text": "abc"}}\n', + "ababc", + ), + ( + '{"content": {"text": "T"}}\n{"content": {"text": "Ti"}}\n{"content": {"text": "Tide"}}\n' + '{"content": {"text": "Tide"}}\n', + "Tide", + ), + ], +) +def test_a_streamed_reply_is_its_text_deltas_joined_or_its_last_whole_text(body, reply): + path = "choices.0.delta.content" if "choices" in body else "content.text" + wrapper = HTTPAgentWrapper( + endpoint="http://localhost:8080/chat", + protocol="json_template", + request_template={"input": "{{messages}}"}, + response_path=path, + ) + assert ( + wrapper._agent_response_from_payload(_decode_json_or_sse(body)).content == reply + ) + + +@pytest.mark.parametrize( + ("body", "path", "value"), + [ + ( + 'data: {"status": "running"}\n\ndata: {"status": "running"}\n\ndata: {"status": "completed"}\n\n', + "status", + "completed", + ), + ( + '{"session": "s-1"}\n{"session": "s-1"}\n{"session": "s-1"}\n', + "session", + "s-1", + ), + ], +) +def test_a_streamed_value_other_than_the_reply_text_is_its_latest_event( + body, path, value +): + assert _select_response_value(_decode_json_or_sse(body), path) == value diff --git a/tests/harness/test_multi_agent_compatibility_references.py b/tests/harness/test_multi_agent_compatibility_references.py index 43082a658..052c13b13 100644 --- a/tests/harness/test_multi_agent_compatibility_references.py +++ b/tests/harness/test_multi_agent_compatibility_references.py @@ -96,3 +96,37 @@ def test_verifier_checks_materialized_orchestration(tmp_path: Path) -> None: assert result.stdout.strip() == ( "3 multi-agent references and 1 source trees verified" ) + + +def test_returns_reference_declares_the_workers_and_tools_its_app_builds() -> None: + import ast + + app = ROOT / "examples" / "harness" / "returns_reference" + reference = json.loads((app / "reference.json").read_text(encoding="utf-8")) + tree = ast.parse((app / "agent.py").read_text(encoding="utf-8")) + # A worker's tools are built by a function of their own (each reaches the world through its own address). + builders = { + node.name: sorted( + inner.name + for inner in node.body + if isinstance(inner, ast.FunctionDef) + and any(getattr(one, "id", "") == "function_tool" for one in inner.decorator_list) + ) + for node in tree.body + if isinstance(node, ast.FunctionDef) + } + built = {} + for node in ast.walk(tree): + if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "Agent": + fields = {one.arg: one.value for one in node.keywords} + tools = fields["tools"] + built[ast.literal_eval(fields["name"])] = ( + builders[tools.func.id] + if isinstance(tools, ast.Call) + else sorted(one.id for one in tools.elts if isinstance(one, ast.Name)) + ) + topology = reference["topology"] + assert topology["coordinator"] in built + assert {one["name"]: sorted(one["tools"]) for one in topology["workers"]} == { + name: tools for name, tools in built.items() if name != topology["coordinator"] + } diff --git a/tests/harness/test_outbound.py b/tests/harness/test_outbound.py index 92294b857..0e440ce8e 100644 --- a/tests/harness/test_outbound.py +++ b/tests/harness/test_outbound.py @@ -3773,3 +3773,14 @@ def test_artifacts_client_push_manifest_rejects_an_unknown_artifact_reference() result = client.push_manifest(manifest) assert not result.delivered assert result.error is not None and result.error.code == "artifact_unknown" + + +def test_a_worker_s_sub_goal_names_its_subject_and_a_task_sub_goal_is_unchanged() -> None: + worker = {"name": "w", "held": True, "reason": None, "judged": False, "subject": "worker:refund"} + task = {"name": "n", "held": True, "reason": None, "judged": False} + receipt = _passed_receipt(sub_goals=[worker, task]) + plain = _passed_receipt() + + assert receipt["sub_goals"][0]["subject"] == "worker:refund" + assert "subject" not in receipt["sub_goals"][1] and "subject" not in plain["sub_goals"][0] + ResultReceiptDraft.model_validate(receipt) diff --git a/tests/harness/test_process_preflight.py b/tests/harness/test_process_preflight.py index 8cd92a05d..bc39643c6 100644 --- a/tests/harness/test_process_preflight.py +++ b/tests/harness/test_process_preflight.py @@ -1653,3 +1653,26 @@ def configure(body): preflight_bundle(tmp_path, manifest, parallelism=2, secret_refs=refs) else: preflight_bundle(tmp_path, manifest, parallelism=2, secret_refs=refs) + + +def test_a_worker_channel_placeholder_needs_the_bundle_s_tool_proxy(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["REFUND_API_URL"] = "{{WORKER_CHANNEL_refund}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="needs this bundle's tool proxy"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_worker_channel_placeholder_is_accepted_beside_a_tool_proxy(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + # Only the name matters to the vocabulary: the bundle has a process called tool-proxy. + body["processes"][0]["name"] = "tool-proxy" + body["processes"][1]["depends_on"] = ["tool-proxy"] + body["capabilities"]["database"]["service"] = "tool-proxy" + body["processes"][1]["environment"]["REFUND_API_URL"] = "{{DATABASE_URL}}/w/{{WORKER_CHANNEL_refund}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + assert preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) is None diff --git a/tests/harness/test_process_runtime.py b/tests/harness/test_process_runtime.py index a96cbec69..5ded102a7 100644 --- a/tests/harness/test_process_runtime.py +++ b/tests/harness/test_process_runtime.py @@ -9412,6 +9412,7 @@ def test_mark_hook_written_with_its_module_first_on_pythonpath_for_every_exporti ] assert sorted(path.name for path in world.iterdir()) == [ "simulation_mark_bootstrap.py", + "simulation_worker_events.py", "sitecustomize.py", ] assert ( @@ -9472,6 +9473,15 @@ def test_mark_hook_written_with_its_module_first_on_pythonpath_for_every_exporti assert (world / "sitecustomize.py").read_bytes() == _LIVEKIT_HOOK.read_bytes() assert not (world / "simulation_mark_bootstrap.py").exists() + # Framework observation reads spans through the mark module, exported or not. + observed = both.model_copy( + update={"environment": {**both.environment, "ALK_OBSERVE_FRAMEWORKS": "1"}} + ) + captured = _spawn(tmp_path / "observed", observed) + assert ( + captured["world_dir"] / "simulation_mark_bootstrap.py" + ).read_bytes() == _MARK_HOOK.read_bytes() + # A reset spawns again in the same directory, whose files the process's own user now owns # read-only: they are reused, never rewritten. exporting = _source_process(name="agent") @@ -9844,6 +9854,7 @@ def test_small_mark_hook_says_so_when_the_mark_module_is_broken( world = captured["world_dir"] assert sorted(path.name for path in world.iterdir()) == [ "simulation_mark_bootstrap.py", + "simulation_worker_events.py", "sitecustomize.py", ] _replace_mark_module(world, _BROKEN_MARK_MODULES[source]) @@ -10542,3 +10553,29 @@ def metadata(environment: dict) -> dict: "actor_proof_path" ] == str(tmp_path / "worlds" / "w2" / "tool-proxy" / "actor-token") assert "actor_proof_path" not in metadata({}) + + +def test_each_worker_gets_its_own_channel_token_per_world_written_where_the_tool_proxy_reads_it(tmp_path: Path) -> None: + import json + + plan = pr.plan_ports(_manifest(), instances=2) + + def render(world: int, value: str) -> str: + return pr.render_template( + value, + process_name="agent", + world_index=world, + world_dir=tmp_path / "worlds" / f"w{world}" / "agent", + port_plan=plan, + configuration_addresses={"TOOLS_API_URL": "http://proxy"}, + ) + + lookup = render(0, "{{TOOLS_API_URL}}/w/{{WORKER_CHANNEL_order_lookup}}") + refund = render(0, "{{WORKER_CHANNEL_refund}}") + # A second render of the same world (a restart) keeps the channel; another world gets its own. + assert render(0, "{{TOOLS_API_URL}}/w/{{WORKER_CHANNEL_order_lookup}}") == lookup + other = render(1, "{{WORKER_CHANNEL_refund}}") + token = lookup.rsplit("/", 1)[-1] + assert lookup.startswith("http://proxy/w/") and len(token) == 32 and len({token, refund, other}) == 3 + channels = json.loads((tmp_path / "worlds" / "w0" / "tool-proxy" / "worker-channels.json").read_text()) + assert channels == {token: "order_lookup", refund: "refund"} diff --git a/tests/harness/test_scenario_source.py b/tests/harness/test_scenario_source.py index fa9bd4313..c15eb60c6 100644 --- a/tests/harness/test_scenario_source.py +++ b/tests/harness/test_scenario_source.py @@ -2795,3 +2795,34 @@ def check(world, calls): saved, legacy = _with_claims(scenario, claims).sub_goals assert saved.both and saved.judged == "Pass when: it says what was reported" and saved.compares == "Reference: c" assert not legacy.both and legacy.judged == "" + + +def test_a_catalogued_worker_subject_reaches_the_graded_sub_goal(tmp_path: Path) -> None: + _write_scenario( + tmp_path / ss.SCENARIOS_DIRNAME, + "coded", + scenario_key="coded", + sub_goals=["coded"], + checks={"coded": "def check(world, calls):\n return None\n"}, + ) + scenario = ss.load_scenarios(tmp_path)[0] + as_task = ss._with_claims(scenario, {"coded": {"what": "x"}}).sub_goals[0] + as_worker = ss._with_claims(scenario, {"coded": {"what": "x", "subject": "worker:refund"}}).sub_goals[0] + assert (as_task.subject, as_task.worker) == ("task", "") + assert (as_worker.subject, as_worker.worker) == ("worker:refund", "refund") + + +def test_a_worker_subject_in_the_bundle_catalogue_reaches_the_loaded_scenario(tmp_path: Path) -> None: + _write_scenario( + tmp_path / ss.SCENARIOS_DIRNAME, + "coded", + scenario_key="coded", + sub_goals=["coded"], + checks={"coded": "def check(world, calls):\n return None\n"}, + ) + (tmp_path / "sub_goals.json").write_text( + json.dumps({"sub_goals": [{"name": "coded", "what": "x", "check": "def check(world, calls):\n return None\n", + "subject": "worker:refund"}]}), + encoding="utf-8", + ) + assert ss.load_scenarios(tmp_path)[0].sub_goals[0].worker == "refund" diff --git a/tests/harness/test_simulation_mark_bootstrap.py b/tests/harness/test_simulation_mark_bootstrap.py index bfab42661..1d7070397 100644 --- a/tests/harness/test_simulation_mark_bootstrap.py +++ b/tests/harness/test_simulation_mark_bootstrap.py @@ -762,10 +762,60 @@ def test_hook_module_imports_only_the_standard_library_and_uses_python_39_syntax "functools", "importlib.machinery", "importlib.util", + "importlib", "inspect", "os", "sys", "urllib.parse", + # Imported only inside a process that already uses OpenTelemetry, to give a span asked for + # before any provider was set one; never at the hook's own import. + "opentelemetry.sdk.trace", + "opentelemetry.sdk.trace.export", + "opentelemetry.exporter.otlp.proto.http.trace_exporter", + # Imported only inside a process that has just imported the OpenAI Agents SDK. + "agents.tracing", + } + top_level = { + node.names[0].name if isinstance(node, ast.Import) else node.module + for node in tree.body + if isinstance(node, (ast.Import, ast.ImportFrom)) + } + assert not { + name + for name in top_level + if name and name.startswith(("opentelemetry", "agents")) + } + + +def test_worker_event_module_imports_only_the_standard_library_and_uses_python_39_syntax() -> ( + None +): + """It sits beside the hook in the customer's interpreter, under the same constraints.""" + tree = ast.parse( + (_HARNESS / "simulation_worker_events.py").read_text(encoding="utf-8"), + feature_version=(3, 9), + ) + imported = { + alias.name + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + } | { + node.module or "" + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.level == 0 + } + assert not any( + isinstance(node, ast.ImportFrom) and node.level for node in ast.walk(tree) + ) + assert imported == { + "__future__", + "collections", + "functools", + "json", + "os", + "threading", + "time", } diff --git a/tests/harness/test_tool_trace_proxy.py b/tests/harness/test_tool_trace_proxy.py index 70f5de152..e6accd064 100644 --- a/tests/harness/test_tool_trace_proxy.py +++ b/tests/harness/test_tool_trace_proxy.py @@ -363,3 +363,96 @@ def test_a_fault_never_applies_to_an_actor_s_action(proxy, tmp_path): url, rows = proxy _faults(tmp_path, [{"action": "records.approve", "mode": "timeout_before_commit", "delay_seconds": 0}]) assert _post(url, "/records/approve", {"item": "1"}, key="k-1", **_as("reviewer"))[0] == 201 + + +CHANNELS = {"c-lookup": "order_lookup", "c-refund": "refund"} +PERMISSIONS = {"worker:order_lookup": {"allow": ["ok"]}, "worker:refund": {"allow": ["records.approve"]}} + + +@pytest.fixture() +def worker_proxy(proxy, monkeypatch, tmp_path): + # As the bundle ships it: the proxy with its sealed permissions beside it. + (tmp_path / "proxy.py").write_bytes(_PROXY.read_bytes()) + (tmp_path / "permissions.json").write_text(json.dumps(PERMISSIONS), encoding="utf-8") + (tmp_path / "channels.json").write_text(json.dumps(CHANNELS), encoding="utf-8") + monkeypatch.setenv("WORKER_CHANNELS_FILE", str(tmp_path / "channels.json")) + url, rows = proxy + spec = importlib.util.spec_from_file_location("tool_trace_proxy_workers", tmp_path / "proxy.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + server = _serve(module.Handler) + try: + yield f"http://127.0.0.1:{server.server_port}", rows, tmp_path + finally: + server.shutdown() + + +def test_a_worker_s_allowed_action_reaches_the_world_through_its_channel_and_names_the_worker(worker_proxy): + url, rows, _ = worker_proxy + assert _post(url, "/w/c-refund/records/approve", {"deal": "1"})[0] == 201 + # The world sees the action's own path; the channel token never leaves the proxy. + assert _World.seen == ["/records/approve"] + [row] = rows() + assert (row["actor_id"], row["worker_id"], row["outcome"], row["name"]) == ("agent", "refund", "committed", "approve") + assert "c-refund" not in json.dumps(row, default=str) + + +def test_a_worker_s_out_of_permission_write_is_refused_before_the_world_and_kept_as_an_attempt(worker_proxy): + url, rows, _ = worker_proxy + status, payload = _post(url, "/w/c-lookup/records/approve", {"deal": "1"}) + assert status == 403 and payload == {"detail": "records.approve is not permitted for order_lookup"} + assert _World.seen == [] + [row] = rows() + assert (row["worker_id"], row["outcome"], row["refusal_code"], row["ok"]) == ("order_lookup", "refused", "not_permitted", False) + + +def test_a_channel_this_trial_did_not_issue_is_refused_as_an_identity_conflict(worker_proxy): + url, rows, _ = worker_proxy + assert _post(url, "/w/guess/records/approve", {"deal": "1"})[0] == 403 + assert _post(url, "/w//records/approve", {"deal": "1"})[0] == 403 + assert _World.seen == [] + assert [(r["actor_id"], r["worker_id"], r["refusal_code"]) for r in rows()] == [("agent", None, "identity_conflict")] * 2 + + +def test_an_unreadable_permissions_seal_denies_rather_than_widens(worker_proxy): + url, rows, tmp_path = worker_proxy + (tmp_path / "permissions.json").write_text("not json", encoding="utf-8") + assert _post(url, "/w/c-refund/records/approve", {"deal": "1"})[0] == 403 + assert _World.seen == [] + assert rows()[0]["refusal_code"] == "not_permitted" + + +def test_without_a_permissions_seal_a_worker_s_action_is_attributed_but_not_limited(worker_proxy): + url, rows, tmp_path = worker_proxy + (tmp_path / "permissions.json").unlink() + assert _post(url, "/w/c-lookup/records/approve", {"deal": "1"})[0] == 201 + assert [(r["worker_id"], r["outcome"]) for r in rows()] == [("order_lookup", "committed")] + + +def test_a_route_named_differently_from_its_tool_is_logged_as_the_tool(worker_proxy): + url, rows, tmp_path = worker_proxy + (tmp_path / "actions.json").write_text(json.dumps({"records.approve": "grant_discount"}), encoding="utf-8") + assert _post(url, "/w/c-refund/records/approve", {"deal": "1"})[0] == 201 + assert _post(url, "/ok", {"deal": "1"})[0] == 201 + assert [r["name"] for r in rows()] == ["grant_discount", "ok"] + + + +def test_past_its_deadline_a_worker_s_call_is_refused_and_before_it_goes_through(worker_proxy): + import time + + url, rows, tmp_path = worker_proxy + # The runner writes the deadline in the trial state beside the actor token, as it writes faults. + state = tmp_path / "trial.json" + state.write_text(json.dumps({"faults": [], "worker_deadline_at": time.time() + 600}), encoding="utf-8") + assert _post(url, "/w/c-refund/records/approve", {"deal": "1"})[0] == 201 + state.write_text(json.dumps({"faults": [], "worker_deadline_at": time.time() - 1}), encoding="utf-8") + status, payload = _post(url, "/w/c-refund/records/approve", {"deal": "2"}) + assert status == 403 and payload == {"detail": "the deadline for refund has passed"} + # The agent's own calls, outside any worker's channel, are not a worker's to time out. + assert _post(url, "/ok", {"deal": "3"})[0] == 201 + assert [(r["worker_id"], r["outcome"], r["refusal_code"]) for r in rows()] == [ + ("refund", "committed", None), + ("refund", "refused", "deadline_passed"), + (None, "committed", None), + ] \ No newline at end of file