From 1f1da64f67bac6424e5a0e8b0f2bdbb25e71e5fa Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 14 Nov 2026 09:13:11 +0530 Subject: [PATCH 1/3] fix(harness): save whether each proof's solution ran and report proved counts at finish --- src/fi/alk/harness/builder.py | 4 +- src/fi/alk/harness/hosted_chat_entrypoint.py | 38 +++++++- src/fi/alk/harness/nudge.py | 2 +- src/fi/alk/harness/proofs.py | 75 +++++++++++++-- src/fi/alk/harness/prove.py | 42 +++++++++ src/fi/alk/harness/scenario_tools.py | 8 +- src/fi/alk/harness/skills/builder/SKILL.md | 6 ++ tests/harness/proof_world.py | 96 ++++++++++++++++++++ tests/harness/test_builder_chat.py | 54 +++++++++++ tests/harness/test_proofs.py | 75 +++++++++++++++ 10 files changed, 382 insertions(+), 18 deletions(-) create mode 100644 tests/harness/proof_world.py create mode 100644 tests/harness/test_proofs.py diff --git a/src/fi/alk/harness/builder.py b/src/fi/alk/harness/builder.py index 56461f651..21fe0200f 100644 --- a/src/fi/alk/harness/builder.py +++ b/src/fi/alk/harness/builder.py @@ -162,7 +162,7 @@ def suite_server( ) -> ToolServer: """Whole-suite operations the single-scenario tools would take one call per scenario to do.""" from .catalogue import load_catalogue - from .proofs import record_proof + from .proofs import proof_key, record_proof from .prove import prove from .scenario_tools import write_scenarios @@ -205,7 +205,7 @@ async def prove_scenarios(args: dict[str, Any]) -> dict[str, Any]: lines.append(f"{name}: not in the suite") continue proof = prove(scenario, catalogue, out) - record_proof(out, scenario.scenario_key, proof) + record_proof(out, proof_key(scenario), proof, solution_steps=len(scenario.solution)) gates = ", ".join(f"{gate} {'yes' if held else 'no'}" for gate, held in proof.gates().items()) lines.append( f"{name}: {'holds' if proof.holds else 'does not hold'} ({gates})" diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index 681e166f6..463c2bfd7 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -50,6 +50,32 @@ _NOT_MESSAGES = {"interrupt", "cancel_operation", "user_response", "approval"} _PENDING_QUESTION = ".futureagi-pending-question.json" _BUILD_FINISHED = ".futureagi-build-finished.json" + + +def _proof_summary(proofs: dict[str, dict[str, Any]], keys: list[str], counts: dict[str, int]) -> str: + """The proof counts the closing summary must repeat, with the tools and reasons behind them.""" + tools = sorted( + { + tool + for key in keys + if (proofs.get(key) or {}).get("kept_as_written") is True + for tool in (proofs.get(key) or {}).get("assumed") or [] + } + ) + reasons = sorted( + { + str((proofs.get(key) or {}).get("blocked_reason") or "not_validated") + for key in keys + if (proofs.get(key) or {}).get("proved") is not True + and (proofs.get(key) or {}).get("kept_as_written") is not True + } + ) + kept = f"{counts['kept_as_written']} kept as written" + (f" ({', '.join(tools)})" if tools else "") + unproved = f"{counts['not_proved']} not proved" + (f" ({', '.join(reasons)})" if reasons else "") + return ( + f"{counts['proved']} proved, {kept}, {unproved}. Report exactly these counts; call only the " + f"{counts['proved']} proved scenarios proved." + ) _PROVIDER_PROFILE = "provider-import-profile.json" _JOURNAL = ".futureagi-conversation.json" _PROVIDER_SESSIONS = ".futureagi-provider-sessions.json" @@ -849,6 +875,7 @@ async def remember(args: dict[str, Any]) -> dict[str, Any]: ) async def finish_build(args: dict[str, Any]) -> dict[str, Any]: from .builder import suite_health + from .proofs import load_proofs, proof_counts, proof_key from .scenario_tools import load_scenarios from .understand import load as load_contract @@ -876,12 +903,19 @@ async def finish_build(args: dict[str, Any]) -> dict[str, Any]: return text("Not finished:\n- " + "\n- ".join(missing[:60]), error=True) # The bundle is sealed from the world as it is now, after every change made during the build. await asyncio.to_thread(self._after_saved, world_changed=True) + proofs = load_proofs(self.workspace) + keys = [proof_key(one) for one in kept] + counts = proof_counts(proofs, keys) (self.workspace / _BUILD_FINISHED).write_text( - json.dumps({"scenarios": len(kept), "asked": asked, "note": str(args.get("note") or "")}), + json.dumps( + {"scenarios": len(kept), "asked": asked, "note": str(args.get("note") or ""), "proofs": counts} + ), encoding="utf-8", ) return text( - f"Finished: {len(kept)} scenarios (the build asked for {asked}). It is now being " + f"Finished: {len(kept)} scenarios (the build asked for {asked}): " + + _proof_summary(proofs, keys, counts) + + " It is now being " "validated against the running agent and sealed; the environment opens when that " "is done, and this conversation carries on there." ) diff --git a/src/fi/alk/harness/nudge.py b/src/fi/alk/harness/nudge.py index c58a65135..b4e0f3683 100644 --- a/src/fi/alk/harness/nudge.py +++ b/src/fi/alk/harness/nudge.py @@ -415,7 +415,7 @@ def suite_gate( record_unprovable(workspace, key, type(exc).__name__) holds, why = False, type(exc).__name__ else: - record_proof(workspace, key, proof) + record_proof(workspace, key, proof, solution_steps=len(scenario.solution)) if proof.deferred and not proof.broken: # Only the live runtime can prove it; that is not a failure. not_checked.append(key) diff --git a/src/fi/alk/harness/proofs.py b/src/fi/alk/harness/proofs.py index 4570fcfc1..6d9e03ea3 100644 --- a/src/fi/alk/harness/proofs.py +++ b/src/fi/alk/harness/proofs.py @@ -41,29 +41,86 @@ def _now() -> str: return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") -def record_proof(destination: Path, scenario_key: str, proof: Proof) -> None: - """Save the three gates for one scenario, replacing its earlier proof.""" - proofs = load_proofs(destination) - proofs[scenario_key] = { +def proof_key(scenario: Any) -> str: + """The key every proof of a scenario is saved under.""" + from .scenario import _slug + + return getattr(scenario, "scenario_key", "") or _slug(scenario.name) + + +def _entry(proof: Proof, solution_steps: int) -> dict[str, Any]: + blocked = proof.blocked_reason() + if proof.proved: + why = "" + elif not proof.holds: + why = proof.why() + elif proof.assumed: + why = ( + "kept as written: these tools have no endpoint here, so their solution steps never ran: " + + ", ".join(proof.assumed) + ) + else: + why = proof.why() + return { + "format": 2, "holds": proof.holds, + "proved": proof.proved, + "kept_as_written": proof.kept_as_written, "ready": proof.ready, "solvable": proof.solvable, "not_vacuous": not proof.vacuous, - "why": "" if proof.holds else proof.why(), + "why": why, "proved_at": _now(), + "assumed": list(proof.assumed), + "solution_steps": solution_steps, + "deferred": proof.deferred, + "judged_only": proof.judged_only, + "blocked_reason": blocked, } - _save(destination, proofs) -def record_unprovable(destination: Path, scenario_key: str, why: str) -> None: - """Save that a scenario could not be proved at all.""" +def record_proof(destination: Path, scenario_key: str, proof: Proof, *, solution_steps: int = 0) -> None: + """Save one scenario's proof, replacing its earlier one: what held, what ran and what was assumed.""" proofs = load_proofs(destination) - proofs[scenario_key] = { + proofs[scenario_key] = _entry(proof, solution_steps) + _save(destination, proofs) + + +def unprovable_entry(why: str, blocked_reason: str = "broken_check") -> dict[str, Any]: + return { + "format": 2, "holds": False, + "proved": False, + "kept_as_written": False, "ready": False, "solvable": False, "not_vacuous": False, "why": why, "proved_at": _now(), + "assumed": [], + "solution_steps": 0, + "deferred": False, + "judged_only": False, + "blocked_reason": blocked_reason, } + + +def record_unprovable(destination: Path, scenario_key: str, why: str) -> None: + """Save that a scenario could not be proved at all.""" + proofs = load_proofs(destination) + proofs[scenario_key] = unprovable_entry(why) _save(destination, proofs) + + +def proof_counts(proofs: dict[str, dict[str, Any]], keys: list[str]) -> dict[str, int]: + """Proved, kept as written and not proved, over `keys`. A key with no proof is not proved.""" + counts = {"proved": 0, "kept_as_written": 0, "not_proved": 0} + for key in keys: + entry = proofs.get(key) or {} + if entry.get("proved") is True: + counts["proved"] += 1 + elif entry.get("kept_as_written") is True: + counts["kept_as_written"] += 1 + else: + counts["not_proved"] += 1 + return counts diff --git a/src/fi/alk/harness/prove.py b/src/fi/alk/harness/prove.py index f6393029d..a33d34b00 100644 --- a/src/fi/alk/harness/prove.py +++ b/src/fi/alk/harness/prove.py @@ -41,6 +41,10 @@ from .world.snapshot import read_manifest, restore +# The lane could not execute something; the case runs labelled "kept as written", never as proved. +LANE_CODES = frozenset({"proof_deferred", "solution_assumed", "control_not_run"}) + + @dataclass class Proof: """Whether a scenario holds up, and what happened when it was tried.""" @@ -87,6 +91,44 @@ def holds(self) -> bool: and not self.unchecked ) + @property + def proved(self) -> bool: + """Holds, and the reference solution really ran: nothing assumed, nothing deferred.""" + return self.holds and not self.assumed and not self.deferred + + @property + def kept_as_written(self) -> bool: + """Not proved only because this lane could not execute something; nothing that ran failed.""" + return not self.proved and self.blocked_reason() in LANE_CODES + + def blocked_reason(self) -> str | None: + """The first reason this proof is not proved, faults before the lane's own limits.""" + if self.proved: + return None + if self.broken: + return "broken_check" + if not self.deferred: + # A deferred proof returns before any gate runs, so its gates read unset, not failed. + if not self.ready: + return "not_ready" + if not self.solvable: + return "not_solvable" + if self.vacuous: + return "vacuous" + if self.weak: + return "weak_check" + if self.failed_attempts: + return "credits_failed_attempt" + if self.too_strict: + return "too_strict" + if self.unchecked: + return "unchecked" + if self.deferred: + return "proof_deferred" + if self.assumed: + return "solution_assumed" + return None + def gates(self) -> dict[str, bool]: """The three answers, for anything that wants to show them.""" return { diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index f01c70346..08653b50c 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -53,7 +53,7 @@ write_index, ) from .checks import reference_of, run_check -from .proofs import record_proof, record_unprovable +from .proofs import proof_key, record_proof, record_unprovable from .prove import ( executes, native_runtime, @@ -173,10 +173,10 @@ def _no_longer_hold( try: proof = prove(scenario, catalogue, world_root) except Exception as exc: # noqa: BLE001 - a scenario that cannot be proved is the finding - record_unprovable(destination, scenario.scenario_key, type(exc).__name__) + record_unprovable(destination, proof_key(scenario), type(exc).__name__) failed.append((scenario.name, type(exc).__name__)) continue - record_proof(destination, scenario.scenario_key, proof) + record_proof(destination, proof_key(scenario), proof, solution_steps=len(scenario.solution)) if not proof.holds and not (proof.deferred and not proof.broken): why = "not ready" if not proof.ready else ( "the reference solution no longer passes it" @@ -1106,7 +1106,7 @@ def accept_scenario( # Saved on both paths: parallel writers share this process, so the proof file is safe to # write even where the folders are not. A rename is not proved again: its key keeps its proof. if proof is not None: - record_proof(world_root, scenario.scenario_key or _slug(scenario.name), proof) + record_proof(world_root, proof_key(scenario), proof, solution_steps=len(scenario.solution)) if persist: write_scenarios(kept, world_root, catalogue) else: diff --git a/src/fi/alk/harness/skills/builder/SKILL.md b/src/fi/alk/harness/skills/builder/SKILL.md index 957b29bbc..efc0b1d8e 100644 --- a/src/fi/alk/harness/skills/builder/SKILL.md +++ b/src/fi/alk/harness/skills/builder/SKILL.md @@ -132,6 +132,12 @@ yourself. When you mention a scenario, write its number and name, `#12 some_name find it. A message can start with a block naming rows the person selected on the Scenarios tab; that selection is what "these" and "the selected ones" mean. +## Finishing a build + +`finish_build` reports how many scenarios are proved, kept as written and not proved. Say 'proved' only +for the count finish_build reports as proved; call the rest 'kept as written' or 'not proved', with the +reason it gives. + ## Changing the environment Load `write-scenarios` before you write or rewrite any scenario in this conversation; its rules are diff --git a/tests/harness/proof_world.py b/tests/harness/proof_world.py new file mode 100644 index 000000000..3fbcb2446 --- /dev/null +++ b/tests/harness/proof_world.py @@ -0,0 +1,96 @@ +"""A small in-memory world for proof tests: an orders table, refunds, and optional unbound tools.""" + +from __future__ import annotations + +import copy +from types import SimpleNamespace +from typing import Any + +from fi.alk.harness import prove as module +from fi.alk.harness.catalogue import Catalogue, SubGoal +from fi.alk.harness.checks import Outcome +from fi.alk.harness.scenario import Step +from fi.alk.harness.world.runtime import Call + +BASE = {"orders": {"O-10": {"age_days": 10, "total": 42.5}, "O-20": {"age_days": 20, "total": 18.0}}, "refunds": []} + + +class FakeWorld: + def __init__(self, data: dict[str, Any], runtime_tools=(), *, unreadable: bool = False) -> None: + self.data = copy.deepcopy(data) + self.runtime_tools = set(runtime_tools) + self.endpoint_for: dict[str, str] = {} + self.calls: list[Call] = [] + self.unreadable = unreadable + + def call(self, name: str, arguments: dict[str, Any]) -> Call: + if name == "lookup_order": + order = self.data["orders"].get(arguments.get("order_id")) + call = Call(name, arguments, result=order, ok=order is not None, error="" if order else "no such order") + elif name in {"refund_order", "place_order"}: + order = self.data["orders"].get(arguments.get("order_id")) + if order is None or order["age_days"] > 14: + call = Call(name, arguments, ok=False, error="refused by policy", refused=True) + else: + self.data["refunds"].append(arguments.get("order_id")) + call = Call(name, arguments, result={"refunded": arguments.get("order_id")}) + else: + call = Call(name, arguments, ok=False, error="unknown tool") + self.calls.append(call) + return call + + def state(self) -> dict[str, Any]: + if self.unreadable: + raise RuntimeError("state unavailable") + return copy.deepcopy(self.data) + + def reset(self) -> None: + pass + + def close(self) -> None: + pass + + +def install(monkeypatch, *, data=BASE, runtime_tools=(), unreadable=False, ready=True, ready_said="") -> None: + """Make prove() use FakeWorld for every gate, without a world on disk.""" + + def prepared(scenario, world_root): + world = FakeWorld(data, runtime_tools, unreadable=unreadable) + ok = Outcome("setup", True) + return world, ok, (Outcome("ready", True) if ready else Outcome("ready", False, ready_said)) + + monkeypatch.setattr(module, "prepared", prepared) + monkeypatch.setattr(module, "native_runtime", lambda world_root: False) + monkeypatch.setattr(module, "restore", lambda world_root: FakeWorld(data, runtime_tools, unreadable=unreadable)) + + +def check(body: str) -> str: + return "def check(world, calls):\n" + "".join(f" {line}\n" for line in body.splitlines()) + + +def catalogue(**checks: str) -> Catalogue: + return Catalogue(sub_goals=[SubGoal(name=name, what=f"The agent {name.replace('_', ' ')}.", check=source) for name, source in checks.items()]) + + +def scenario(solution, *, sub_goals, name="refund_case", instruction="", **extra) -> SimpleNamespace: + values = dict( + name=name, + scenario_key=name, + sub_goals=list(sub_goals), + solution=[Step(**one) if isinstance(one, dict) else one for one in solution], + setup_code="", + ready_code="", + alternatives=[], + instruction=instruction, + tests="", + variables={}, + persona=None, + negative_controls=[], + sources=[], + use_case="", + ) + values.update(extra) + return SimpleNamespace(**values) + + +ONE_REFUND = check("n = world.data['refunds'].count('O-10')\nreturn None if n == 1 else f'{n} refunds'") diff --git a/tests/harness/test_builder_chat.py b/tests/harness/test_builder_chat.py index 018fe047a..390c6c23d 100644 --- a/tests/harness/test_builder_chat.py +++ b/tests/harness/test_builder_chat.py @@ -2005,3 +2005,57 @@ async def handler(args): answer = asyncio.run(runtime._ask("AskUserQuestion", {"question": "Which?"}, None)) assert "Nobody answers" in answer["content"][0]["text"] assert calls == [corrected] + + +def test_finished_message_reports_counts(tmp_path, monkeypatch): + from types import SimpleNamespace + + from fi.alk.harness import scenario_tools + from fi.alk.harness.proofs import record_proof + from fi.alk.harness.prove import Proof + + monkeypatch.setenv("ALK_HARNESS", "claude") + policy = tmp_path / "policy.yaml" + policy.write_text( + "schema_version: 1\nharness_chat:\n enabled: true\n stages:\n builder:\n enabled: true\n", + encoding="utf-8", + ) + workspace = tmp_path / "authoring" + workspace.mkdir() + + class LiveBuildClient(BuilderClient): + def __init__(self): + super().__init__() + self.turn_context = {**self.capabilities.turn_context, "capabilities": {"control_only": True}} + + async def overview(self): + return {} + + runtime = HostedChatRuntime( + client=LiveBuildClient(), # type: ignore[arg-type] + policy=load_chat_policy(policy), + workspace=workspace, + source=tmp_path, + job={"metadata": {"agent_name": "demo"}, "scenario_count": 10}, + ) + stage = asyncio.run(runtime._open_builder()) + finish = next(one for one in stage.spec.servers["platform"].tools if one.name == "finish_build") + # Every solution uses only the two in-process tools, which have no endpoint: each step is assumed. + kept = [SimpleNamespace(name=f"case-{n}", scenario_key=f"case-{n}") for n in range(10)] + for one in kept: + proof = Proof(ready=True, solvable=True, vacuous=False, assumed=["place_order", "search_books"]) + record_proof(workspace, one.scenario_key, proof, solution_steps=2) + monkeypatch.setattr(runtime, "_what_exists", lambda: (True, True)) + monkeypatch.setattr(runtime, "_after_saved", lambda **kwargs: None) + monkeypatch.setattr(scenario_tools, "load_scenarios", lambda path: kept) + monkeypatch.setattr(builder, "suite_health", lambda *args, **kwargs: []) + + said = asyncio.run(finish.handler({}))["content"][0]["text"] + + assert "0 proved, 10 kept as written (place_order, search_books), 0 not proved" in said + assert said.endswith( + "It is now being validated against the running agent and sealed; the environment opens when that " + "is done, and this conversation carries on there." + ) + finished = json.loads((workspace / ".futureagi-build-finished.json").read_text(encoding="utf-8")) + assert finished["proofs"] == {"proved": 0, "kept_as_written": 10, "not_proved": 0} diff --git a/tests/harness/test_proofs.py b/tests/harness/test_proofs.py new file mode 100644 index 000000000..ceea9e50b --- /dev/null +++ b/tests/harness/test_proofs.py @@ -0,0 +1,75 @@ +"""The proof record says whether the reference solution really ran, and counts that apart.""" + +from __future__ import annotations + +from fi.alk.harness import prove as module +from fi.alk.harness.proofs import load_proofs, proof_counts, record_proof, record_unprovable +from fi.alk.harness.prove import Proof + +from proof_world import ONE_REFUND, catalogue, install, scenario + +REFUND = [{"tool": "refund_order", "arguments": {"order_id": "O-10"}, "fixed": {"order_id": "the caller"}}] + + +def test_assumed_steps_are_saved_and_not_proved(monkeypatch, tmp_path): + install(monkeypatch, runtime_tools={"place_order"}) + case = scenario( + [{"tool": "place_order", "arguments": {"order_id": "O-10"}, "fixed": {"order_id": "the caller"}}], + sub_goals=["placed"], + ) + proof = module.prove(case, catalogue(placed="def check(world, calls):\n return None if any(c.name == 'place_order' and c.arguments.get('order_id') == 'O-10' for c in calls) else 'not placed'\n"), tmp_path) + + record_proof(tmp_path, "refund_case", proof, solution_steps=1) + saved = load_proofs(tmp_path)["refund_case"] + + assert proof.holds is True + assert saved["assumed"] == ["place_order"] + assert saved["proved"] is False + assert saved["blocked_reason"] == "solution_assumed" + assert saved["kept_as_written"] is True + assert saved["format"] == 2 and saved["solution_steps"] == 1 + + +def test_proof_counts_split_kept_as_written(tmp_path): + proved = Proof(ready=True, solvable=True, vacuous=False) + assumed = Proof(ready=True, solvable=True, vacuous=False, assumed=["place_order"]) + deferred = Proof(deferred=True) + failing = Proof(ready=True, solvable=False, vacuous=False) + for key in ("a", "b", "c"): + record_proof(tmp_path, key, proved) + record_proof(tmp_path, "d", assumed) + record_proof(tmp_path, "e", assumed) + record_proof(tmp_path, "f", deferred) + record_proof(tmp_path, "g", failing) + + counts = proof_counts(load_proofs(tmp_path), ["a", "b", "c", "d", "e", "f", "g"]) + + assert counts == {"proved": 3, "kept_as_written": 3, "not_proved": 1} + + +def test_unprovable_is_format_two(tmp_path): + record_unprovable(tmp_path, "x", "ValueError") + saved = load_proofs(tmp_path)["x"] + assert saved["format"] == 2 + assert saved["blocked_reason"] == "broken_check" + assert saved["proved"] is False and saved["why"] == "ValueError" + + +def test_deferred_reads_proof_deferred_not_not_ready(monkeypatch, tmp_path): + monkeypatch.setattr(module, "native_runtime", lambda world_root: True) + monkeypatch.setattr(module, "workspace_fixture", lambda *a: False) + case = scenario(REFUND, sub_goals=["refund_once"]) + proof = module.prove(case, catalogue(refund_once=ONE_REFUND), tmp_path) + assert proof.deferred and not proof.ready + assert proof.blocked_reason() == "proof_deferred" + assert proof.kept_as_written is True + + +def test_holds_failures_map_to_codes(): + base = dict(ready=True, solvable=True, vacuous=False) + assert Proof(**base, weak=["no_refund"]).blocked_reason() == "weak_check" + assert Proof(**base, failed_attempts=["tried"]).blocked_reason() == "credits_failed_attempt" + assert Proof(**base, too_strict=["refund_once (alternative 1)"]).blocked_reason() == "too_strict" + assert Proof(**base, unchecked=["refund_order.order_id"]).blocked_reason() == "unchecked" + for proof in (Proof(**base, weak=["x"]), Proof(**base, unchecked=["y"])): + assert proof.kept_as_written is False From b9b21ee6220266c2610cea82404ecdbddfa3610d Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sun, 15 Nov 2026 00:55:45 +0530 Subject: [PATCH 2/3] fix(harness): append code-written proof counts to the finishing reply and correct an overclaim --- src/fi/alk/harness/hosted_chat_entrypoint.py | 70 +++++++++++++++++ tests/harness/test_builder_chat.py | 83 ++++++++++++++++++++ 2 files changed, 153 insertions(+) diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index 463c2bfd7..b873cf87b 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -6,6 +6,7 @@ import traceback import json import os +import re import signal import time import uuid @@ -76,6 +77,53 @@ def _proof_summary(proofs: dict[str, dict[str, Any]], keys: list[str], counts: d f"{counts['proved']} proved, {kept}, {unproved}. Report exactly these counts; call only the " f"{counts['proved']} proved scenarios proved." ) +_NUMBER_PROVED = re.compile( + r"\b(\d+)\s+((?:[A-Za-z-]+\s+){0,2})proved\b", re.IGNORECASE +) +_ALL_PROVED = re.compile(r"\b(?:all|every)\b[^.]{0,40}\bproved\b", re.IGNORECASE) + + +def _proof_claim_mismatch(text: str, counts: dict[str, int]) -> str | None: + """A correction when the reply claims more scenarios proved than the proofs hold, else None.""" + proved = int(counts.get("proved") or 0) + total = ( + proved + + int(counts.get("kept_as_written") or 0) + + int(counts.get("not_proved") or 0) + ) + claims = [ + int(match.group(1)) + for match in _NUMBER_PROVED.finditer(text) + if not re.search(r"\b(?:not|un)", match.group(2), re.IGNORECASE) + ] + overclaimed = any(claim != proved for claim in claims) or ( + proved < total + and any( + not re.search( + r"\bnot\b", + text[max(0, match.start() - 12) : match.start()], + re.IGNORECASE, + ) + for match in _ALL_PROVED.finditer(text) + ) + ) + return ( + f"Correction: only {proved} of {total} scenarios are proved." + if overclaimed + else None + ) + + +def _with_proof_counts(text: str, counts: dict[str, int]) -> str: + """The closing reply with the proof counts written by code, and a correction if the reply overclaims.""" + correction = _proof_claim_mismatch(text, counts) + line = ( + f"Proof counts: {int(counts.get('proved') or 0)} proved, {int(counts.get('kept_as_written') or 0)} kept as " + f"written, {int(counts.get('not_proved') or 0)} not proved." + ) + return "\n\n".join(part for part in (text, line, correction) if part) + + _PROVIDER_PROFILE = "provider-import-profile.json" _JOURNAL = ".futureagi-conversation.json" _PROVIDER_SESSIONS = ".futureagi-provider-sessions.json" @@ -1510,6 +1558,7 @@ async def _handle_command(self, command: dict[str, Any]) -> None: ) emitted_text: list[str] = [] turn_failures: list[str] = [] + finished_before = self._build_finished_stamp() queue: asyncio.Queue[Event | None] = asyncio.Queue() def receive(event: Event) -> None: @@ -1649,6 +1698,7 @@ def receive(event: Event) -> None: ).strip() self._last_health = "" self._report_again_if_written() + final_text = self._close_with_proof_counts(final_text, finished_before) if final_text: await self.client.emit( "assistant_message", @@ -1680,6 +1730,26 @@ def receive(event: Event) -> None: if self._restart_after_turn: self.stopping.set() + def _build_finished_stamp(self) -> int | None: + """When the build-finished file was last written, or None when there is none.""" + try: + return (self.workspace / _BUILD_FINISHED).stat().st_mtime_ns + except OSError: + return None + + def _close_with_proof_counts(self, text: str, finished_before: int | None) -> str: + """Append code-written proof counts when this turn finished the build, so the reply cannot overclaim.""" + stamp = self._build_finished_stamp() + if stamp is None or stamp == finished_before: + return text + try: + counts = json.loads( + (self.workspace / _BUILD_FINISHED).read_text(encoding="utf-8") + ).get("proofs") + except (OSError, ValueError, AttributeError): + return text + return _with_proof_counts(text, counts) if isinstance(counts, dict) else text + async def _fail_turn( self, journal: dict[str, str], diff --git a/tests/harness/test_builder_chat.py b/tests/harness/test_builder_chat.py index 390c6c23d..71a73198b 100644 --- a/tests/harness/test_builder_chat.py +++ b/tests/harness/test_builder_chat.py @@ -2059,3 +2059,86 @@ async def overview(self): ) finished = json.loads((workspace / ".futureagi-build-finished.json").read_text(encoding="utf-8")) assert finished["proofs"] == {"proved": 0, "kept_as_written": 10, "not_proved": 0} + + +def _finished_runtime(tmp_path): + policy = tmp_path / "policy.yaml" + policy.write_text( + "schema_version: 1\nharness_chat:\n enabled: true\n stages:\n builder:\n enabled: true\n", + encoding="utf-8", + ) + workspace = tmp_path / "authoring" + workspace.mkdir() + runtime = HostedChatRuntime( + client=BuilderClient(), # type: ignore[arg-type] + policy=load_chat_policy(policy), + workspace=workspace, + source=tmp_path, + job={"metadata": {"agent_name": "demo"}}, + ) + return runtime, workspace + + +def test_finished_turn_appends_code_counts(tmp_path): + runtime, workspace = _finished_runtime(tmp_path) + before = runtime._build_finished_stamp() + counts = {"proved": 3, "kept_as_written": 8, "not_proved": 1} + (workspace / ".futureagi-build-finished.json").write_text( + json.dumps({"proofs": counts}), encoding="utf-8" + ) + + text = runtime._close_with_proof_counts("The suite is built.", before) + + assert ( + text + == "The suite is built.\n\nProof counts: 3 proved, 8 kept as written, 1 not proved." + ) + # A turn that did not write the finished file leaves the reply as the model wrote it. + assert ( + runtime._close_with_proof_counts( + "Edited one case.", runtime._build_finished_stamp() + ) + == "Edited one case." + ) + + +def test_overclaim_gets_a_correction(tmp_path): + runtime, workspace = _finished_runtime(tmp_path) + counts = {"proved": 3, "kept_as_written": 9, "not_proved": 0} + (workspace / ".futureagi-build-finished.json").write_text( + json.dumps({"proofs": counts}), encoding="utf-8" + ) + + text = runtime._close_with_proof_counts("All 12 scenarios proved.", None) + + assert text.endswith("Correction: only 3 of 12 scenarios are proved.") + assert "Proof counts: 3 proved, 9 kept as written, 0 not proved." in text + from fi.alk.harness.hosted_chat_entrypoint import _proof_claim_mismatch + + assert ( + _proof_claim_mismatch("12 scenarios are proved.", counts) + == "Correction: only 3 of 12 scenarios are proved." + ) + assert ( + _proof_claim_mismatch("Every case is proved against the agent.", counts) + is not None + ) + + +def test_exact_claim_gets_no_correction(): + from fi.alk.harness.hosted_chat_entrypoint import _proof_claim_mismatch + + counts = {"proved": 3, "kept_as_written": 8, "not_proved": 1} + assert ( + _proof_claim_mismatch( + "3 scenarios proved, 8 kept as written, 1 not proved.", counts + ) + is None + ) + assert _proof_claim_mismatch("Not all of them are proved yet.", counts) is None + assert ( + _proof_claim_mismatch( + "All 4 proved.", {"proved": 4, "kept_as_written": 0, "not_proved": 0} + ) + is None + ) From 99122c5d05cd3f64832599943e01aa054638bce7 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sun, 15 Nov 2026 07:15:46 +0530 Subject: [PATCH 3/3] test(harness): the proof stub returns the scenario-code outcome and the check reads only calls that succeeded --- tests/harness/proof_world.py | 5 ++--- tests/harness/test_proofs.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/harness/proof_world.py b/tests/harness/proof_world.py index 3fbcb2446..a5d290a1e 100644 --- a/tests/harness/proof_world.py +++ b/tests/harness/proof_world.py @@ -8,7 +8,7 @@ from fi.alk.harness import prove as module from fi.alk.harness.catalogue import Catalogue, SubGoal -from fi.alk.harness.checks import Outcome +from fi.alk.harness.folder import Outcome from fi.alk.harness.scenario import Step from fi.alk.harness.world.runtime import Call @@ -56,8 +56,7 @@ def install(monkeypatch, *, data=BASE, runtime_tools=(), unreadable=False, ready def prepared(scenario, world_root): world = FakeWorld(data, runtime_tools, unreadable=unreadable) - ok = Outcome("setup", True) - return world, ok, (Outcome("ready", True) if ready else Outcome("ready", False, ready_said)) + return world, Outcome(True), (Outcome(True) if ready else Outcome(False, ready_said)) monkeypatch.setattr(module, "prepared", prepared) monkeypatch.setattr(module, "native_runtime", lambda world_root: False) diff --git a/tests/harness/test_proofs.py b/tests/harness/test_proofs.py index ceea9e50b..3b10e98e0 100644 --- a/tests/harness/test_proofs.py +++ b/tests/harness/test_proofs.py @@ -17,7 +17,7 @@ def test_assumed_steps_are_saved_and_not_proved(monkeypatch, tmp_path): [{"tool": "place_order", "arguments": {"order_id": "O-10"}, "fixed": {"order_id": "the caller"}}], sub_goals=["placed"], ) - proof = module.prove(case, catalogue(placed="def check(world, calls):\n return None if any(c.name == 'place_order' and c.arguments.get('order_id') == 'O-10' for c in calls) else 'not placed'\n"), tmp_path) + proof = module.prove(case, catalogue(placed="def check(world, calls):\n return None if any(c.ok and c.name == 'place_order' and c.arguments.get('order_id') == 'O-10' for c in calls) else 'not placed'\n"), tmp_path) record_proof(tmp_path, "refund_case", proof, solution_steps=1) saved = load_proofs(tmp_path)["refund_case"]