Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 92 additions & 0 deletions agentdescent/budget.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
from __future__ import annotations

import math
import threading
from dataclasses import dataclass, field
from typing import Any, Callable, Optional

Expand All @@ -52,6 +53,97 @@
HARD_FLOOR = 0.90


class BudgetStop(Exception):
"""A model call that a run-level call budget refused to make (#1166).

Raised by :meth:`CallLimit.reserve` **before** the call, and caught by name
in :mod:`agentdescent.evolution` -- the worker wrapper's generic
``except Exception`` would otherwise record a clean budget stop as a failed
worker, and the run would end ``failed`` with an error instead of a named
stop.
"""

def __init__(self, limit: str, observed: float, cap: float) -> None:
self.limit = limit
self.observed = observed
self.cap = cap
super().__init__(f"{limit}={cap} reached")

def detail(self) -> str:
"""One line for ``status``: which limit fired and what was spent."""
if self.limit == "max_calls":
return (f"max_calls: {int(self.observed)} call(s) made, "
f"cap {int(self.cap)}")
return f"max_usd: ${self.observed:.4f} spent, cap ${self.cap:.4f}"


@dataclass
class CallLimit:
"""A run-level cap on calls and dollars, charged before every call (#1166).

``reserve()`` *is* the mechanism: it runs before the call is made, so "the
call did not happen" is structural rather than a check after the fact. The
engine's round-barrier ``max_calls`` is a different quantity -- it bounds
where to stop, so a four-worker round on ``max_calls=1`` still makes four
calls; this one bounds the calls themselves.

Thread-safe: ``metrics.measured`` wraps both worker threads and the
evaluation pool. ``0`` is a limit ("no calls"), not an absence.
"""

max_calls: Optional[int] = None
#: A dollar cap needs a per-call price; without one it cannot be charged and
#: :func:`agentdescent.runstore.require_budget_pair` refuses the run.
max_usd: Optional[float] = None
usd_per_call: Optional[float] = None
_lock: threading.Lock = field(default_factory=threading.Lock, repr=False,
compare=False)
_taken: int = field(default=0, init=False, repr=False, compare=False)
_stopped: Optional[BudgetStop] = field(default=None, init=False,
repr=False, compare=False)

def reserve(self) -> None:
"""Take the next call's slot, or raise :class:`BudgetStop`.

The slot is taken **before** the call so concurrent workers cannot both
pass a check that only one of them can afford.
"""
with self._lock:
n = self._taken + 1
stop: Optional[BudgetStop] = None
if self.max_calls is not None and n > self.max_calls:
stop = BudgetStop("max_calls", self._taken, self.max_calls)
elif self.max_usd is not None and self.usd_per_call:
# Tolerance: 3 x 0.01 must not exceed 0.03 through float noise.
if n * self.usd_per_call > self.max_usd + 1e-9:
stop = BudgetStop("max_usd", self._taken * self.usd_per_call,
self.max_usd)
if stop is not None:
if self._stopped is None:
self._stopped = stop
raise stop
self._taken = n

@property
def stop(self) -> Optional[BudgetStop]:
"""The refusal this limit already made, or ``None`` (#1166).

The refusal is raised inside a worker and caught somewhere else, so the
object recorded here is what a later handler re-raises: the numbers
survive the trip instead of being rebuilt from a message.
"""
return self._stopped

@property
def calls(self) -> int:
"""Calls this limit has admitted so far."""
return self._taken

def detail(self) -> Optional[str]:
"""Why the limit refused, or ``None`` when it never did."""
return None if self._stopped is None else self._stopped.detail()


@dataclass
class BudgetGovernor:
"""Tracks token spend against a cap and decides what to degrade.
Expand Down
42 changes: 37 additions & 5 deletions agentdescent/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@
from .evolution import (SELECT_SET_NOTE, _check_apply_paths, _file_tree_backup_path,
reward_set_line)
from .evolvespec import (KINDS, EvolveSpec, SpecError, budget_round_overshoot_warning,
compose, estimate, fold_kind, load_spec, resolve_kind,
_POLICY_SLOTS)
build_call_limit, compose, estimate, fold_kind, load_spec,
resolve_kind, _POLICY_SLOTS)
from .probe import first_failed_next

__all__ = ["main", "doctor_report", "starter_spec"]
Expand All @@ -79,6 +79,10 @@ def _version() -> str:
return "unknown"

#: Set in every worker's environment by the plugin runner and the MCP server so a
#: The `stop_reason` a run-level call cap writes (#1166): the budget fired, and
#: `stop_detail` names which limit and what it observed.
BUDGET_STOP = "budget"

#: host that is itself being evolved cannot start a nested run. See docs/plugins.md.
NESTED_ENV = "AGENTDESCENT_NESTED"

Expand Down Expand Up @@ -176,6 +180,11 @@ def _status_line(st: runstore.RunStatus) -> str:
ph = f" phase={st.phase}" if st.phase else ""
pending = f" pending={st.pending_calls}{ph}"
tail = f" [{st.stop_reason}]" if st.stop_reason else ""
# Which limit fired and what it cost, on the same line: a reader asking
# "why did my run stop" should not have to open `status.json` (#1166).
if st.stop_detail:
tail = (f" [{st.stop_reason}: {st.stop_detail}]" if st.stop_reason
else f" [{st.stop_detail}]")
err = f" error: {st.error}" if st.error else ""
# Reflector exceptions, cumulative (#395): a run can finish `done` with only
# these to say that it never learned anything.
Expand Down Expand Up @@ -2707,6 +2716,14 @@ def cmd_evolve(a: argparse.Namespace) -> int:
# Before create/launch: a missing --usd-per-call must not leave an orphan run
# whose status later reads as "process ended without reporting" (#342).
runstore.require_budget_pair(a.budget, a.usd_per_call)
# The run-level cap (#1166): refused here, before a run directory exists,
# with the field and the value named -- a bad flag is not a live run.
try:
build_call_limit(max_calls=getattr(a, "max_calls", None),
max_usd=getattr(a, "max_usd", None),
usd_per_call=a.usd_per_call)
except SpecError as e:
return _spec_error(a, e)
# Same gate as plan / MCP start: missing CLI binary must not report started (#346).
require_cli_on_path(spec)
_warn_unusable(spec)
Expand All @@ -2716,7 +2733,9 @@ def cmd_evolve(a: argparse.Namespace) -> int:
# The child composes again and rewrites `notes`, but only once it is up;
# recording them here means `status` has them from the first poll (#414).
rd.update_status(notes=comp.notes)
st = runstore.launch(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call)
st = runstore.launch(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call,
max_calls=getattr(a, "max_calls", None),
max_usd=getattr(a, "max_usd", None))
if not a.json:
for line in comp.notes:
print(f"note: {line}", file=sys.stderr)
Expand All @@ -2725,7 +2744,9 @@ def cmd_evolve(a: argparse.Namespace) -> int:
if a.json else f"{rd.run_id} started (pid {st.pid}) {rd.path}", as_json=a.json)
return 0
try:
result = runstore.execute(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call)
result = runstore.execute(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call,
max_calls=getattr(a, "max_calls", None),
max_usd=getattr(a, "max_usd", None))
except Exception as e: # noqa: BLE001 - not BaseException: Ctrl-C stays loud
return _run_failed(a, rd, e)
st = rd.status()
Expand Down Expand Up @@ -2790,7 +2811,9 @@ def _run_failed(a: argparse.Namespace, rd: "runstore.RunDir", e: Exception) -> i
def cmd_run(a: argparse.Namespace) -> int:
"""The body of a detached run; not for people."""
rd = runstore.RunDir(a.run_dir)
result = runstore.execute(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call)
result = runstore.execute(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call,
max_calls=getattr(a, "max_calls", None),
max_usd=getattr(a, "max_usd", None))
return 0 if result.error is None else 1


Expand Down Expand Up @@ -6175,6 +6198,13 @@ def _register_commands(mount: "_CommandMount", *, with_lab: bool) -> None:
s.add_argument("--detach", action="store_true", help="return at once; poll with status")
s.add_argument("--budget", type=float, help="stop after this many dollars (needs --usd-per-call)")
s.add_argument("--usd-per-call", type=float, dest="usd_per_call")
s.add_argument("--max-calls", type=int, dest="max_calls", default=None,
help=f"stop before the call that would exceed this many model "
f"calls; the run ends 'done' with stop_reason={BUDGET_STOP} "
f"(0 means no calls; exit code 0)")
s.add_argument("--max-usd", type=float, dest="max_usd", default=None,
help="stop before the call that would exceed this many dollars "
"(needs --usd-per-call)")
s.set_defaults(fn=cmd_evolve)

# The detached worker `evolve --detach` spawns (runstore: `python -m
Expand All @@ -6188,6 +6218,8 @@ def _register_commands(mount: "_CommandMount", *, with_lab: bool) -> None:
s.add_argument("--run-dir", required=True, dest="run_dir")
s.add_argument("--budget", type=float)
s.add_argument("--usd-per-call", type=float, dest="usd_per_call")
s.add_argument("--max-calls", type=int, dest="max_calls", default=None)
s.add_argument("--max-usd", type=float, dest="max_usd", default=None)
s.set_defaults(fn=cmd_run)

s = sub.add_parser("status", help="one run, or all of them")
Expand Down
Loading
Loading