Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 3 additions & 8 deletions agentdescent/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
from .evolvespec import (KINDS, EvolveSpec, SpecError, budget_round_overshoot_warning,
compose, estimate, fold_kind, load_spec, resolve_kind,
_POLICY_SLOTS)
from .probe import first_failed_next

__all__ = ["main", "doctor_report", "starter_spec"]

Expand Down Expand Up @@ -2406,12 +2407,6 @@ def _plan_agent_line(spec: Dict[str, Any]) -> Optional[str]:
return line


def _first_failed_probe_next(probe: Mapping[str, Any]) -> str:
"""The repair action from the first refused probe row (#1093)."""
for row in probe.get("rows") or []:
if row.get("state") == "failed" and row.get("next"):
return str(row["next"])
return "run `agentdescent doctor`, then re-run plan --probe"


def _print_plan(payload: Dict[str, Any], *, spec_path: str,
Expand Down Expand Up @@ -2579,7 +2574,7 @@ def cmd_plan(a: argparse.Namespace) -> int:
payload["probe"] = probe
payload["ready"] = bool(probe.get("ok", True))
if not payload["ready"]:
next_line = _first_failed_probe_next(probe)
next_line = first_failed_next(probe)
payload["next"] = next_line
then_line = f"agentdescent plan {a.spec} --probe"
# `--baseline-sample` spends real calls, so it runs last: a broken spec or a
Expand Down Expand Up @@ -4837,7 +4832,7 @@ def _cmd_algorithms_run(a: argparse.Namespace) -> int:
# follow it into a run that cannot work (#1093).
obj["ready"] = bool(probe.get("ok", True))
if not obj["ready"]:
obj["next"] = _first_failed_probe_next(probe)
obj["next"] = first_failed_next(probe)
then_line = f"agentdescent plan {obj['spec_path']} --probe"
if as_json:
# Exactly one JSON object on stdout for success and error (#674).
Expand Down
7 changes: 7 additions & 0 deletions agentdescent/integrations/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,13 @@ after `doctor`. Do not invent a keyless path for those cards.
in one line, and say it loudest when they asked for many rounds or workers
(cost is rounds x n_workers x tasks). Starting a run whose size the user has
not seen is the one thing this procedure exists to prevent.
The first time a spec names a model or endpoint, run the plan **with the
probe on**: `agentdescent plan <spec> --probe`, or the MCP
`plan(spec, probe=true)`. It sends one minimal call per model-backed agent
and answers "will start work" before a run exists. If it refuses, give the
user its `next` (renew the plan, fix the model name or key) and do **not**
call `start` -- a refused probe is the run failing in round one, found one
step earlier.
To keep improving across sessions, offer `agentdescent campaign <spec>
--sessions N`; it works on a copy, reports the workspace/final target and
never touches the live target. Run its `--dry-run` first. A campaign has no
Expand Down
43 changes: 36 additions & 7 deletions agentdescent/mcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,11 @@ def advanced_from_env(environ: Optional[Dict[str, str]] = None) -> bool:
"upper bound on agent calls per round and in total, and notes. Pass budget_usd "
"with usd_per_call to hear when the seed+one-round bound already exceeds the "
"budget (budget is only checked between rounds). When the spec has an "
"`algorithm`, the resolved name/params are echoed. Show the user the spec and "
"the estimate and get a yes before calling start."),
"`algorithm`, the resolved name/params are echoed. Pass probe=true to send "
"one minimal call per model-backed agent (the same preflight as "
"`plan --probe`); a refusal returns `probe` and makes this object `ok: false` "
"with `next` naming the fix, so do not call start then. Show the user the "
"spec and the estimate and get a yes before calling start."),
"start": (
"Start an evolution run in the background and return its run_id at once. The run "
"is a detached process; poll `status` about once per round (not more often) and "
Expand Down Expand Up @@ -399,6 +402,26 @@ def advanced_from_env(environ: Optional[Dict[str, str]] = None) -> bool:
}


def _with_probe(payload: Dict[str, Any], spec: EvolveSpec) -> Dict[str, Any]:
"""Add the #1048 preflight to a successful ``plan`` when asked (#1094).

The body is the same ``probe_payload`` the CLI's ``plan --probe`` runs, so
a host gets the same rows and the same repair sentence instead of a second
implementation. A refusal makes the plan itself ``ok: false`` and replaces
``next`` with that repair: a host following ``next`` must not start a run
that cannot work. ``probe`` is added only when the caller asked for it --
plain ``plan`` stays the free call that sends nothing.
"""
from .probe import first_failed_next, probe_payload

result = probe_payload(spec)
payload["probe"] = result
if not result.get("ok", True):
payload["ok"] = False
payload["next"] = first_failed_next(result)
return payload


class Tools:
"""The tool bodies, SDK-free. One instance per server, bound to a run store."""

Expand Down Expand Up @@ -640,14 +663,19 @@ def algorithm_spec(self, name: str, path: str, cases: Optional[str] = None,
return result

def plan(self, spec: Dict[str, Any], usd_per_call: Optional[float] = None,
budget_usd: Optional[float] = None) -> Dict[str, Any]:
budget_usd: Optional[float] = None,
probe: bool = False) -> Dict[str, Any]:
try:
# Relative paths mean "where this server was launched"; the detached
# run has a different cwd, so they are resolved here, once.
return plan_payload(EvolveSpec.from_dict(spec).absolutise(),
usd_per_call=usd_per_call, budget_usd=budget_usd)
es = EvolveSpec.from_dict(spec).absolutise()
payload = plan_payload(es, usd_per_call=usd_per_call,
budget_usd=budget_usd)
except SpecError as e:
return {"ok": False, "error": str(e)}
if not probe or not payload.get("ok", True):
return payload
return _with_probe(payload, es)

def status(self, run_id: Optional[str] = None) -> Any:
try:
Expand Down Expand Up @@ -1359,8 +1387,9 @@ def algorithm_spec(name: str, path: str, cases: Optional[str] = None,

@maybe_tool("plan")
def plan(spec: Dict[str, Any], usd_per_call: Optional[float] = None,
budget_usd: Optional[float] = None) -> Dict[str, Any]:
return t.plan(spec, usd_per_call, budget_usd)
budget_usd: Optional[float] = None,
probe: bool = False) -> Dict[str, Any]:
return t.plan(spec, usd_per_call, budget_usd, probe)

# `start` and `resume` are the two tools that launch a process, so they are
# the two that take a Context. The annotation is set after the definition
Expand Down
13 changes: 13 additions & 0 deletions agentdescent/probe.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,19 @@ def probe_payload(spec: Any) -> Dict[str, Any]:
return {"ok": not any(row["state"] == "failed" for row in rows), "rows": rows}


def first_failed_next(payload: Mapping[str, Any],
*, retry: str = "plan --probe") -> str:
"""The repair action from the first refused probe row (#1093).

Shared by the CLI and the MCP ``plan`` tool so both surfaces answer a
refusal with the same fix, and neither sends a host to ``start`` (#1094).
"""
for row in payload.get("rows") or []:
if row.get("state") == "failed" and row.get("next"):
return str(row["next"])
return f"run `agentdescent doctor`, then re-run {retry}"


def probe_lines(payload: Mapping[str, Any]) -> list:
"""The human ``probe:`` lines, one per row, in the order they ran."""
lines = []
Expand Down
9 changes: 9 additions & 0 deletions changelog.d/1094-mcp-plan-probe.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
Added
MCP `plan` takes `probe: bool = False` and, when true, runs the same
preflight as `plan --probe`: one minimal call per model-backed agent, with the
rows under `probe` in the result. A refusal makes the plan `ok: false` and
points `next` at the repair action instead of `start`, so a host that follows
`next` no longer discovers an expired subscription only after the run exists.
The default is unchanged: plain `plan` still sends nothing and returns its old
key set. The shared host skill teaches the probe step, so every installed
host now asks for it on the first use of a model (#1094).
8 changes: 7 additions & 1 deletion docs/plugins.md
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ The skill file teaches the host model one procedure, and the tool surface
enforces its shape:

```
doctor → write a spec → plan (show the user) → start → status (once a round) → show → ask → apply
doctor → write a spec → plan (show the user; --probe on a first model) → start → status (once a round) → show → ask → apply
```

That loop is **Path A** — the default MCP tool listing and the verbs to teach a
Expand All @@ -258,6 +258,12 @@ all ~39 tools as the primary path.
the data and builds the policy bundle without running anything, then returns
the composed `evolve()` arguments and an upper bound on agent calls. A wrong
field fails here, by name, not in round one of a background process.
* **The probe is on `plan`, for both surfaces.** `plan --probe` (CLI) and
`plan(spec, probe=true)` (MCP) send one minimal call per model-backed agent,
through the same adapter the run would use, so an expired subscription or a
wrong model name is answered before a run store entry exists. A refusal sets
`ok: false` and `next` to the repair action, never to `start`; the default
`probe` is off, and plain `plan` still sends nothing.
* **`start` returns in under a second.** The run is a detached process; the host
polls `status`. MCP tool calls time out and hosts restart their servers
between sessions, so nothing blocks on `evolve()`.
Expand Down
7 changes: 7 additions & 0 deletions integrations/claude-code/skills/agentdescent/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,13 @@ after `doctor`. Do not invent a keyless path for those cards.
in one line, and say it loudest when they asked for many rounds or workers
(cost is rounds x n_workers x tasks). Starting a run whose size the user has
not seen is the one thing this procedure exists to prevent.
The first time a spec names a model or endpoint, run the plan **with the
probe on**: `agentdescent plan <spec> --probe`, or the MCP
`plan(spec, probe=true)`. It sends one minimal call per model-backed agent
and answers "will start work" before a run exists. If it refuses, give the
user its `next` (renew the plan, fix the model name or key) and do **not**
call `start` -- a refused probe is the run failing in round one, found one
step earlier.
To keep improving across sessions, offer `agentdescent campaign <spec>
--sessions N`; it works on a copy, reports the workspace/final target and
never touches the live target. Run its `--dry-run` first. A campaign has no
Expand Down
2 changes: 1 addition & 1 deletion integrations/dsh-agentdescent/lib/index.js

Large diffs are not rendered by default.

143 changes: 143 additions & 0 deletions tests/test_mcp_plan_probe.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
"""The MCP `plan` tool can run the #1048 preflight (#1094).

The CLI has had `plan --probe` since #1048, but a host that drives the MCP
server (Claude Code, Codex, dsh, OpenCode, Grok Bot) could only call `plan`,
which runs nothing. So the first real answer about an expired subscription or a
wrong model name arrived after `start`, as a failed run store entry -- exactly
what the flag exists to prevent on the CLI side. The same is true of the host
skill: it teaches `plan` -> user yes -> `start`, and never mentions the probe.

These rows pin the tool contract (`probe` is opt-in, reports the same payload
`probe_payload` returns, and a refusal makes `ok` false with a repair `next`)
and the host template (every generated SKILL.md teaches the probe).
"""
from __future__ import annotations

import json
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path

import pytest

from agentdescent.integrations import HOSTS, host_status, install
from agentdescent.mcp import Tools

OK = {"choices": [{"message": {"content": "ok"}}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1}}

#: Volcengine Ark Coding, an expired plan, on HTTP 400 as measured in #1006.
ARK_SUBSCRIPTION = {
"error": {
"code": "InvalidSubscription",
"message": ("Your account does not have a valid CodingPlan subscription, "
"or your subscription has expired."),
}
}

#: What `Tools().plan(spec)` returned before the probe existed (base
#: f8efc73), for the spec below: a fresh key set or a dropped key is a shape
#: change a host would see, so the default path is pinned key by key.
BASE_PLAN_KEYS = {
"artifact_id", "estimate", "evolve_kwargs", "kind", "notes", "ok", "spec",
"tasks", "warnings",
}


class _Handler(BaseHTTPRequestHandler):
url: str = ""
body: bytes = b"{}"
code: int = 400
hits = 0
lock = threading.Lock()

def do_POST(self): # noqa: N802
self.rfile.read(int(self.headers.get("Content-Length") or 0))
with type(self).lock:
type(self).hits += 1
payload = (json.dumps(OK).encode() if type(self).code == 200
else type(self).body)
self.send_response(type(self).code)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)

def log_message(self, *args): # noqa: D401
pass


@pytest.fixture
def endpoint(monkeypatch):
_Handler.hits = 0
_Handler.code = 400
_Handler.body = b"{}"
server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
threading.Thread(target=server.serve_forever, daemon=True).start()
_Handler.url = f"http://127.0.0.1:{server.server_address[1]}"
monkeypatch.setenv("FAKE_KEY", "sk-not-real")
monkeypatch.setenv("FAKE_BASE_URL", _Handler.url)

def use(*, code=400, body=None):
_Handler.code = code
_Handler.body = json.dumps(body if body is not None else {}).encode()
_Handler.hits = 0

yield use
server.shutdown()
server.server_close()


def _spec():
model = {"ref": "openai_compatible", "model": "m", "api_key_env": "FAKE_KEY",
"base_url_env": "FAKE_BASE_URL", "retries": 1}
return {
"version": 1,
"kind": "text",
"target": "You are a helpful assistant.",
"data": {"inline": [{"prompt": f"q{i}", "gold": "zzz"} for i in range(8)]},
"agent": dict(model),
"reflect": dict(model),
"evolve": {"rounds": 1, "n_workers": 1, "seed": 0},
}


def test_a_refused_probe_makes_the_mcp_plan_refuse_the_start(tmp_path, endpoint):
endpoint(code=400, body=ARK_SUBSCRIPTION)
out = Tools(str(tmp_path / "runs")).plan(_spec(), probe=True)
assert out["ok"] is False, out
assert out["probe"]["ok"] is False
failed = [row for row in out["probe"]["rows"] if row["state"] == "failed"]
assert failed, out["probe"]["rows"]
assert "start" not in out["next"].lower(), out["next"]
assert _Handler.hits == 1 # one probe call, nothing else ran


def test_a_healthy_probe_keeps_the_mcp_plan_ok(tmp_path, endpoint):
endpoint(code=200)
out = Tools(str(tmp_path / "runs")).plan(_spec(), probe=True)
assert out["ok"] is True, out
assert out["probe"]["ok"] is True
assert [row["state"] for row in out["probe"]["rows"]] == ["ok"]
assert _Handler.hits == 1


def test_omitting_the_probe_returns_exactly_the_old_payload(tmp_path):
out = Tools(str(tmp_path / "runs")).plan(_spec())
assert set(out) == BASE_PLAN_KEYS, sorted(set(out) ^ BASE_PLAN_KEYS)
assert "probe" not in out


def test_every_host_skill_teaches_the_probe(tmp_path):
for host in HOSTS:
install(host, home=str(tmp_path))
skills = sorted(Path(tmp_path).rglob("SKILL.md"))
assert len(skills) >= 5, skills
for path in skills:
text = path.read_text(encoding="utf-8").lower()
assert "probe" in text, path
# doctor reads the same host_status: every host is still wired and current.
for host in HOSTS:
status = host_status(host, home=str(tmp_path))
assert status["state"] == "ok", (host, status)
assert status["skill"] == "ok", (host, status)
Loading