From 5f5b69da6275895a9ec5a07be9d451d3898acf0e Mon Sep 17 00:00:00 2001 From: HugoXK Date: Thu, 19 Nov 2026 10:50:23 +0000 Subject: [PATCH 1/5] test(run): a run-level call/dollar budget stops at the call (#1166) Red on main: runstore.execute takes no max_calls/max_usd, EvolveSpec has neither knob, and RunStatus carries no stop detail. The counting factory makes each model call observable, so "the second call did not happen" is an assertion rather than an inference. --- tests/test_budget_guard.py | 107 +++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 tests/test_budget_guard.py diff --git a/tests/test_budget_guard.py b/tests/test_budget_guard.py new file mode 100644 index 00000000..82fac221 --- /dev/null +++ b/tests/test_budget_guard.py @@ -0,0 +1,107 @@ +"""A run-level call / dollar budget stops the run **at the call** (#1166). + +The knobs a run already had are round-barrier budgets: ``max_calls`` is checked +between rounds, so a four-worker round on ``max_calls=1`` still makes four +calls. ``budget_usd`` is a warning plus a between-round ``stop_when``. Neither +is a cap a host can hand to a worker run and trust for one agent session. + +These tests drive the run store end to end, offline: the spec builds its model +from a counting factory in this module, so every model call is one entry in +``CALLS`` and "the second call did not happen" is an assertion, not a hope. +""" + +from __future__ import annotations + +import pytest + +from agentdescent import runstore +from agentdescent.cli import _status_line +from agentdescent.evolvespec import EvolveSpec, SpecError, compose + +#: One entry per model call the spec's agent actually makes. +CALLS = [] + + +def counting_agent(*, usage=None, tag="agent"): + """A spec-nameable model factory that records each call.""" + + def complete(prompt): + CALLS.append(tag) + return "ok" + + return complete + + +def _spec(**evolve): + model = {"ref": "tests.test_budget_guard:counting_agent", "tag": "agent"} + return EvolveSpec( + kind="text", target="You are a helpful assistant.", + data={"inline": [{"prompt": f"q{i}", "gold": f"a{i}"} for i in range(8)]}, + agent=model, reflect=model, allow=["tests."], + evolve={"rounds": 4, "n_workers": 4, "seed": 0, "patience": 10, **evolve}) + + +def _run(tmp_path, *, spec=None, **limits): + CALLS.clear() + rd = runstore.create((spec or _spec()).to_dict(), store=str(tmp_path / "runs")) + result = runstore.execute(rd, **limits) + return rd, rd.status(), result + + +def test_max_calls_stops_before_the_next_call(tmp_path): + rd, st, result = _run(tmp_path, max_calls=1) + assert CALLS == ["agent"], f"a capped run made {len(CALLS)} calls" + assert st.calls == 1 + assert st.state == "done", st + assert st.error is None + assert st.stop_reason == "budget", st + assert "max_calls" in (st.stop_detail or ""), st.stop_detail + assert result.stop_reason == "budget" + + +def test_max_usd_stops_before_the_call_that_would_overspend(tmp_path): + # One call costs 0.01; the second would reach 0.02, over the 0.015 cap. + rd, st, result = _run(tmp_path, max_usd=0.015, usd_per_call=0.01) + assert CALLS == ["agent"], f"an over-budget run made {len(CALLS)} calls" + assert st.state == "done", st + assert st.error is None + assert st.stop_reason == "budget", st + assert "max_usd" in (st.stop_detail or ""), st.stop_detail + assert st.usd == pytest.approx(0.01) + + +def test_the_status_line_names_the_limit_that_fired(tmp_path): + rd, st, _ = _run(tmp_path, max_calls=1) + line = _status_line(st) + assert "budget" in line, line + assert "max_calls" in line, line + + +def test_unset_limits_keep_todays_run(tmp_path): + rd, st, result = _run(tmp_path) + assert st.state == "done", st + assert len(CALLS) > 1, "the uncapped run should have made more than one call" + assert st.stop_reason != "budget", st + assert not st.stop_detail + + +def test_a_negative_max_calls_is_refused_with_the_field_named(): + with pytest.raises(SpecError) as caught: + compose(_spec(max_calls=-1)) + message = str(caught.value) + assert "max_calls" in message and "-1" in message, message + + +def test_a_non_numeric_max_usd_is_refused_with_the_field_named(): + with pytest.raises(SpecError) as caught: + compose(_spec(max_usd="cheap")) + message = str(caught.value) + assert "max_usd" in message and "cheap" in message, message + + +def test_zero_is_a_limit_not_an_absence(tmp_path): + """``0`` means "do not spend": refused before the first call, not ignored.""" + rd, st, _ = _run(tmp_path, max_calls=0) + assert CALLS == [], f"a zero-call budget made {len(CALLS)} calls" + assert st.stop_reason == "budget", st + assert "max_calls" in (st.stop_detail or ""), st.stop_detail From 2c038dbe55a37592769ece6b3cfea17d533ba840 Mon Sep 17 00:00:00 2001 From: HugoXK Date: Thu, 19 Nov 2026 11:21:01 +0000 Subject: [PATCH 2/5] feat(run): stop a worker run on max_calls / max_usd before the call (#963) A run-level cap charged inside the run: CallLimit.reserve() runs before every model call, so 'the call did not happen' is structural. BudgetStop is caught by name in the worker wrapper, at the round barrier and on the main thread, so the round closes without a partial merge or a traceback and the run ends 'done' with stop_reason=budget and a stop_detail naming the limit and the observed value. max_calls on evolve() stays the round-barrier budget; the same number is also enforced per call, which can only be stricter. Partial: #963 --- agentdescent/budget.py | 82 ++++++++++++++++++++ agentdescent/cli.py | 42 ++++++++-- agentdescent/evolution.py | 129 +++++++++++++++++++++++++++---- agentdescent/evolve_settings.py | 5 ++ agentdescent/evolvespec.py | 89 ++++++++++++++++++++- agentdescent/metrics.py | 11 ++- agentdescent/runstore.py | 59 +++++++++++++- changelog.d/1166-run-call-cap.md | 11 +++ docs/api.md | 3 + docs/start.md | 14 ++++ 10 files changed, 417 insertions(+), 28 deletions(-) create mode 100644 changelog.d/1166-run-call-cap.md diff --git a/agentdescent/budget.py b/agentdescent/budget.py index 72d7fac4..3e039546 100644 --- a/agentdescent/budget.py +++ b/agentdescent/budget.py @@ -34,6 +34,7 @@ from __future__ import annotations import math +import threading from dataclasses import dataclass, field from typing import Any, Callable, Optional @@ -52,6 +53,87 @@ HARD_FLOOR = 0.90 +class BudgetStop(Exception): + """A model call that a run-level call budget refused to make (#1166). + + Raised by :meth:`CallLimit.reserve` **before** the call, and caught by name + in :mod:`agentdescent.evolution` -- the worker wrapper's generic + ``except Exception`` would otherwise record a clean budget stop as a failed + worker, and the run would end ``failed`` with an error instead of a named + stop. + """ + + def __init__(self, limit: str, observed: float, cap: float) -> None: + self.limit = limit + self.observed = observed + self.cap = cap + super().__init__(f"{limit}={cap} reached") + + def detail(self) -> str: + """One line for ``status``: which limit fired and what was spent.""" + if self.limit == "max_calls": + return (f"max_calls: {int(self.observed)} call(s) made, " + f"cap {int(self.cap)}") + return f"max_usd: ${self.observed:.4f} spent, cap ${self.cap:.4f}" + + +@dataclass +class CallLimit: + """A run-level cap on calls and dollars, charged before every call (#1166). + + ``reserve()`` *is* the mechanism: it runs before the call is made, so "the + call did not happen" is structural rather than a check after the fact. The + engine's round-barrier ``max_calls`` is a different quantity -- it bounds + where to stop, so a four-worker round on ``max_calls=1`` still makes four + calls; this one bounds the calls themselves. + + Thread-safe: ``metrics.measured`` wraps both worker threads and the + evaluation pool. ``0`` is a limit ("no calls"), not an absence. + """ + + max_calls: Optional[int] = None + #: A dollar cap needs a per-call price; without one it cannot be charged and + #: :func:`agentdescent.runstore.require_budget_pair` refuses the run. + max_usd: Optional[float] = None + usd_per_call: Optional[float] = None + _lock: threading.Lock = field(default_factory=threading.Lock, repr=False, + compare=False) + _taken: int = field(default=0, init=False, repr=False, compare=False) + _stopped: Optional[BudgetStop] = field(default=None, init=False, + repr=False, compare=False) + + def reserve(self) -> None: + """Take the next call's slot, or raise :class:`BudgetStop`. + + The slot is taken **before** the call so concurrent workers cannot both + pass a check that only one of them can afford. + """ + with self._lock: + n = self._taken + 1 + stop: Optional[BudgetStop] = None + if self.max_calls is not None and n > self.max_calls: + stop = BudgetStop("max_calls", self._taken, self.max_calls) + elif self.max_usd is not None and self.usd_per_call: + # Tolerance: 3 x 0.01 must not exceed 0.03 through float noise. + if n * self.usd_per_call > self.max_usd + 1e-9: + stop = BudgetStop("max_usd", self._taken * self.usd_per_call, + self.max_usd) + if stop is not None: + if self._stopped is None: + self._stopped = stop + raise stop + self._taken = n + + @property + def calls(self) -> int: + """Calls this limit has admitted so far.""" + return self._taken + + def detail(self) -> Optional[str]: + """Why the limit refused, or ``None`` when it never did.""" + return None if self._stopped is None else self._stopped.detail() + + @dataclass class BudgetGovernor: """Tracks token spend against a cap and decides what to degrade. diff --git a/agentdescent/cli.py b/agentdescent/cli.py index 784e6383..8d89d84e 100644 --- a/agentdescent/cli.py +++ b/agentdescent/cli.py @@ -53,8 +53,8 @@ from .evolution import (SELECT_SET_NOTE, _check_apply_paths, _file_tree_backup_path, reward_set_line) from .evolvespec import (KINDS, EvolveSpec, SpecError, budget_round_overshoot_warning, - compose, estimate, fold_kind, load_spec, resolve_kind, - _POLICY_SLOTS) + build_call_limit, compose, estimate, fold_kind, load_spec, + resolve_kind, _POLICY_SLOTS) __all__ = ["main", "doctor_report", "starter_spec"] @@ -78,6 +78,10 @@ def _version() -> str: return "unknown" #: Set in every worker's environment by the plugin runner and the MCP server so a +#: The `stop_reason` a run-level call cap writes (#1166): the budget fired, and +#: `stop_detail` names which limit and what it observed. +BUDGET_STOP = "budget" + #: host that is itself being evolved cannot start a nested run. See docs/plugins.md. NESTED_ENV = "AGENTDESCENT_NESTED" @@ -175,6 +179,11 @@ def _status_line(st: runstore.RunStatus) -> str: ph = f" phase={st.phase}" if st.phase else "" pending = f" pending={st.pending_calls}{ph}" tail = f" [{st.stop_reason}]" if st.stop_reason else "" + # Which limit fired and what it cost, on the same line: a reader asking + # "why did my run stop" should not have to open `status.json` (#1166). + if st.stop_detail: + tail = (f" [{st.stop_reason}: {st.stop_detail}]" if st.stop_reason + else f" [{st.stop_detail}]") err = f" error: {st.error}" if st.error else "" # Reflector exceptions, cumulative (#395): a run can finish `done` with only # these to say that it never learned anything. @@ -2712,6 +2721,14 @@ def cmd_evolve(a: argparse.Namespace) -> int: # Before create/launch: a missing --usd-per-call must not leave an orphan run # whose status later reads as "process ended without reporting" (#342). runstore.require_budget_pair(a.budget, a.usd_per_call) + # The run-level cap (#1166): refused here, before a run directory exists, + # with the field and the value named -- a bad flag is not a live run. + try: + build_call_limit(max_calls=getattr(a, "max_calls", None), + max_usd=getattr(a, "max_usd", None), + usd_per_call=a.usd_per_call) + except SpecError as e: + return _spec_error(a, e) # Same gate as plan / MCP start: missing CLI binary must not report started (#346). require_cli_on_path(spec) _warn_unusable(spec) @@ -2721,7 +2738,9 @@ def cmd_evolve(a: argparse.Namespace) -> int: # The child composes again and rewrites `notes`, but only once it is up; # recording them here means `status` has them from the first poll (#414). rd.update_status(notes=comp.notes) - st = runstore.launch(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call) + st = runstore.launch(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call, + max_calls=getattr(a, "max_calls", None), + max_usd=getattr(a, "max_usd", None)) if not a.json: for line in comp.notes: print(f"note: {line}", file=sys.stderr) @@ -2730,7 +2749,9 @@ def cmd_evolve(a: argparse.Namespace) -> int: if a.json else f"{rd.run_id} started (pid {st.pid}) {rd.path}", as_json=a.json) return 0 try: - result = runstore.execute(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call) + result = runstore.execute(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call, + max_calls=getattr(a, "max_calls", None), + max_usd=getattr(a, "max_usd", None)) except Exception as e: # noqa: BLE001 - not BaseException: Ctrl-C stays loud return _run_failed(a, rd, e) st = rd.status() @@ -2795,7 +2816,9 @@ def _run_failed(a: argparse.Namespace, rd: "runstore.RunDir", e: Exception) -> i def cmd_run(a: argparse.Namespace) -> int: """The body of a detached run; not for people.""" rd = runstore.RunDir(a.run_dir) - result = runstore.execute(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call) + result = runstore.execute(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call, + max_calls=getattr(a, "max_calls", None), + max_usd=getattr(a, "max_usd", None)) return 0 if result.error is None else 1 @@ -6135,6 +6158,13 @@ def build_parser() -> argparse.ArgumentParser: s.add_argument("--detach", action="store_true", help="return at once; poll with status") s.add_argument("--budget", type=float, help="stop after this many dollars (needs --usd-per-call)") s.add_argument("--usd-per-call", type=float, dest="usd_per_call") + s.add_argument("--max-calls", type=int, dest="max_calls", default=None, + help=f"stop before the call that would exceed this many model " + f"calls; the run ends 'done' with stop_reason={BUDGET_STOP} " + f"(0 means no calls; exit code 0)") + s.add_argument("--max-usd", type=float, dest="max_usd", default=None, + help="stop before the call that would exceed this many dollars " + "(needs --usd-per-call)") s.set_defaults(fn=cmd_evolve) # The detached worker `evolve --detach` spawns (runstore: `python -m @@ -6148,6 +6178,8 @@ def build_parser() -> argparse.ArgumentParser: s.add_argument("--run-dir", required=True, dest="run_dir") s.add_argument("--budget", type=float) s.add_argument("--usd-per-call", type=float, dest="usd_per_call") + s.add_argument("--max-calls", type=int, dest="max_calls", default=None) + s.add_argument("--max-usd", type=float, dest="max_usd", default=None) s.set_defaults(fn=cmd_run) s = sub.add_parser("status", help="one run, or all of them") diff --git a/agentdescent/evolution.py b/agentdescent/evolution.py index e35d6e9a..e89502ad 100644 --- a/agentdescent/evolution.py +++ b/agentdescent/evolution.py @@ -60,7 +60,7 @@ from .selection import SingleHead from .scheduler import AuditScheduler from .staleness import StalenessPolicy -from .budget import BudgetGovernor, CallBudget +from .budget import BudgetGovernor, BudgetStop, CallBudget # --------------------------------------------------------------------------- @@ -673,6 +673,8 @@ def _measure() -> float: return value except ContractError: raise # a caller bug: retrying cannot help + except BudgetStop: + raise # a budget refusal: retrying cannot help either except Exception: # noqa: BLE001 - a backend transient if attempt == self.ATTEMPTS - 1: raise @@ -2563,7 +2565,8 @@ def _build_engine(tasks, reward, *, agent, run, propose, strategy, initial_state contract: Optional[Contract] = None, extra_artifacts: Optional[Dict[str, "EvolvingArtifact"]] = None, population_islands: int = 0, - test_frac: float = 0.0) -> _Engine: + test_frac: float = 0.0, + call_limit: Optional[Any] = None) -> _Engine: """Wire the ledger, runtime, verifier and aggregator (shared by :func:`evolve` and :func:`~agentdescent.async_evolve.async_evolve`).""" import tempfile @@ -2692,8 +2695,8 @@ def _build_engine(tasks, reward, *, agent, run, propose, strategy, initial_state if usage is None: usage = getattr(agent, "usage", None) meter = Meter(usage=usage) if usage is not None else Meter() - run, propose = (measured(run, meter, phase="run"), - measured(propose, meter, phase="propose")) + run, propose = (measured(run, meter, phase="run", guard=call_limit), + measured(propose, meter, phase="propose", guard=call_limit)) cache = (policies_bundle.eval_cache if policies_bundle is not None and policies_bundle.eval_cache is not None else MemoryCache(meter)) @@ -3140,6 +3143,10 @@ def _score_seed_held_out(artifact: "EvolvingArtifact", held_out: Sequence) -> Op return float(artifact.score(held_out)) except ContractError: raise + except BudgetStop: + # Not a diagnostic failure: the call was refused by the run-level + # budget, and the caller ends the run with a named stop (#1166). + raise except Exception: # noqa: BLE001 - seed score is diagnostic, not a stop reason return None @@ -3182,6 +3189,13 @@ def evolve( #: Checked at the round barrier alongside the other budgets; the run stops #: with ``stop_reason="max_tokens"`` and reports the spend it incurred. max_tokens: Optional[int] = None, + #: Per-call cap, checked **before** every model call (#1166). Unlike + #: ``max_calls`` above -- a round-barrier budget that overshoots by up to a + #: round -- this refuses the call itself, so ``CallLimit(max_calls=1)`` + #: means one call. The refusal unwinds the round through + #: :class:`~agentdescent.budget.BudgetStop` and the run ends with + #: ``stop_reason="budget"``. Built by the run store; inert when ``None``. + call_limit: Optional["CallLimit"] = None, #: Stop when the run's *measured* return per token has fallen off its own #: peak by more than ``efficiency_floor`` (see #: :meth:`~agentdescent.budget.BudgetGovernor.diminishing_returns`). The @@ -3768,7 +3782,7 @@ def evolve( _budget = resolve_budget( budget, max_seconds=max_seconds, max_rollouts=max_rollouts, max_calls=max_calls, - max_tokens=max_tokens, + max_tokens=max_tokens, call_limit=call_limit, stop_on_diminishing_returns=stop_on_diminishing_returns, efficiency_floor=efficiency_floor, call_budget=call_budget, target_reward=target_reward, patience=patience, @@ -3798,6 +3812,7 @@ def evolve( max_rollouts = _budget.max_rollouts max_calls = _budget.max_calls max_tokens = _budget.max_tokens + call_limit = _budget.call_limit stop_on_diminishing_returns = _budget.stop_on_diminishing_returns efficiency_floor = _budget.efficiency_floor call_budget = _budget.call_budget @@ -3987,7 +4002,7 @@ def evolve( allocator=allocator, record_trajectories=record_trajectories, trajectory_store=trajectory_store, contract=contract, extra_artifacts=extra_artifacts, - population_islands=population_islands) + population_islands=population_islands, call_limit=call_limit) # Start the clock after the wiring, before the first unit of work: setup # is not what a time-to-quality number is asking about. eng.meter.start() @@ -4014,7 +4029,17 @@ def evolve( seed_art = EvolvingArtifact( artifact_id, dict(initial_state or strategy.initial()), blast_radius=blast_radius, runtime=eng.runtime, strategy=strategy) - initial_reward = _score_seed_held_out(seed_art, held_out) + #: A run-level budget refusal (#1166). Its own box, for the same reason as + #: `contract_error` below: a worker thread cannot propagate, and it must not + #: be folded into `first_error` -- a cap is a clean stop, not a backend fail. + budget_stop: List[Optional[BudgetStop]] = [None] + try: + initial_reward = _score_seed_held_out(seed_art, held_out) + except BudgetStop as e: + # The seed score is the run's first call: a zero-call cap refuses it + # here, before any round is dispatched (#1166). + budget_stop[0] = e + initial_reward = None straggler_rounds = 0 # Shared with the barrier-free loop: the same two questions, the same # tracker, and now the same epsilon (they had two). @@ -4053,6 +4078,13 @@ def evolve( efficiency_floor=efficiency_floor, allocator=allocator) for r in range(rounds): + # The seed score already spent the cap (e.g. `max_calls=0`): nothing is + # dispatched, and the run closes with the named stop (#1166). + if budget_stop[0] is not None: + stop_reason = "budget" + if verbose: + print(f"round {r:>3} stopping: {budget_stop[0].detail()}") + break if deadline is not None and time.time() >= deadline: stop_reason = "max_seconds" if verbose: @@ -4219,6 +4251,16 @@ def _run_unit(unit) -> None: # main thread by hand -- an exception raised in a plain worker thread # goes to the thread excepthook and is lost, not propagated. contract_error.record(e) # its own lock; first one wins + except BudgetStop as e: + # A run-level cap refused this call (#1166). Not a worker + # failure: it must not touch `first_error` or `failed_units`, or + # the run would end `failed` with an error instead of a named + # budget stop. + with unit_lock: + if budget_stop[0] is None: + budget_stop[0] = e + if verbose: + print(f"round {r:>3} stopping: {e.detail()}") except Exception as e: # noqa: BLE001 - a backend failure with unit_lock: if first_error[0] is None: @@ -4320,6 +4362,10 @@ def _propose_one(unit, task, aid: str) -> None: raw_proposal = propose(mine.render(), task, output, score) except ContractError: raise + except BudgetStop: + # A budget refusal is the run's stop, not this worker's bad + # propose (#1166): it goes up to `_run_unit`'s named handler. + raise except Exception as e: # noqa: BLE001 - a reflector/backend failure with unit_lock: propose_raised[0] += 1 @@ -4484,6 +4530,16 @@ def _bounded(u=None, gate=gate, cancelled=cancelled, # read: a broken contract makes the round meaningless. contract_error.raise_if_set() + # A run-level cap made its refusal inside a worker (#1166). The + # round has nothing left to merge, so it closes here: no partial + # merge, no worker-failure tally, and the run ends with the named + # budget stop rather than an error. + if budget_stop[0] is not None: + stop_reason = "budget" + if verbose: + print(f"round {r:>3} stopping: {budget_stop[0].detail()}") + break + # Decide on the round's tally rather than on the first exception. The # same global signal the async path uses: while NO worker has ever # completed a rollout, repeated total failure means the backend is @@ -4509,6 +4565,17 @@ def _bounded(u=None, gate=gate, cancelled=cancelled, reports = check_reports(_step(aggregator, n_workers), aggregator) except ContractError: raise # a caller-contract violation: the run is meaningless + except BudgetStop as e: + # The cap tripped on this thread -- a merge-side or held-out call + # the workers did not make (#1166). Same clean stop as the worker + # path: named reason, no error, no traceback. + with unit_lock: + if budget_stop[0] is None: + budget_stop[0] = e + stop_reason = "budget" + if verbose: + print(f"round {r:>3} stopping: {e.detail()}") + break except Exception as e: # noqa: BLE001 - a rollout backend failure (e.g. an # API/credit error) shouldn't lose the run: stop and return partial results. run_error = _describe(e) @@ -4539,16 +4606,30 @@ def _bounded(u=None, gate=gate, cancelled=cancelled, # Part of the barrier too: the workers for the next round cannot start # until this measurement lands, retries and backoff included. with eng.meter.timed("merge_seconds"), eng.meter.timed("merge_gate_seconds"): - for attempt in range(3): - try: - round_reward = dev.score(held_out) - break - except ContractError: - raise - except Exception as e: # noqa: BLE001 - score_error = e - if attempt < 2: - time.sleep(0.2 * (attempt + 1)) + try: + for attempt in range(3): + try: + round_reward = dev.score(held_out) + break + except ContractError: + raise + except BudgetStop: + # A cap, not a bad measurement: retrying would be refused + # again, and sleeping on it delays the stop (#1166). + raise + except Exception as e: # noqa: BLE001 + score_error = e + if attempt < 2: + time.sleep(0.2 * (attempt + 1)) + except BudgetStop as e: + # The held-out measurement is the round's last spend; the cap + # fired on it, so the run ends here with the named stop. + if budget_stop[0] is None: + budget_stop[0] = e + stop_reason = "budget" + if verbose: + print(f"round {r:>3} stopping: {e.detail()}") + break if round_reward is None: e = score_error if first_error[0] is None: @@ -4638,6 +4719,13 @@ def _bounded(u=None, gate=gate, cancelled=cancelled, final_reward = final.score(held_out) except ContractError: raise + except BudgetStop as e: + # The cap refused the closing measurement (#1166): report the last + # round's reward, which is a measurement the run actually paid for. + if budget_stop[0] is None: + budget_stop[0] = e + stop_reason = "budget" + final_reward = history[-1].held_out_reward if history else 0.0 except Exception as e: # noqa: BLE001 - report, keep the partial result run_error = run_error or _describe(e) final_reward = history[-1].held_out_reward if history else 0.0 @@ -4649,6 +4737,13 @@ def _bounded(u=None, gate=gate, cancelled=cancelled, if eng.test: try: test_reward = float(final.score(eng.test)) + except BudgetStop as e: + # Same cap, on the closing test sweep: it stays a budget stop with + # a missing test score rather than becoming a crash (#1166). + if budget_stop[0] is None: + budget_stop[0] = e + stop_reason = "budget" + test_reward = None except Exception: # noqa: BLE001 - a missing test score is not a crash test_reward = None if run_error: diff --git a/agentdescent/evolve_settings.py b/agentdescent/evolve_settings.py index 502e75a4..7801a6eb 100644 --- a/agentdescent/evolve_settings.py +++ b/agentdescent/evolve_settings.py @@ -101,6 +101,11 @@ class BudgetSettings: max_rollouts: Optional[int] = None max_calls: Optional[int] = None max_tokens: Optional[int] = None + #: Per-call cap (:class:`~agentdescent.budget.CallLimit`), checked **before** + #: each model call (#1166). A different quantity from ``max_calls`` above, + #: which is a round-barrier budget and overshoots by up to one round. Built + #: by the run store from ``max_calls`` / ``max_usd``. + call_limit: Optional["CallLimit"] = None stop_on_diminishing_returns: bool = False efficiency_floor: float = 0.25 call_budget: Optional["CallBudget"] = None diff --git a/agentdescent/evolvespec.py b/agentdescent/evolvespec.py index a7bfd01c..7dee95be 100644 --- a/agentdescent/evolvespec.py +++ b/agentdescent/evolvespec.py @@ -47,6 +47,7 @@ import csv import difflib import json +import math import os import shlex from dataclasses import dataclass, field, asdict, replace @@ -746,6 +747,9 @@ class Composition: #: Resolved runner layout for directory kinds (``claude_skill``, #: ``skill_library``, …). ``None`` for ``text`` / ``policy_slot``. layout: Optional[str] = None + #: The run-level call cap this composition carries (#1166), so the run store + #: can name the limit that fired without re-reading the spec. + call_limit: Optional[Any] = None def run(self) -> EvolutionResult: return evolve(self.tasks, self.reward, **self.kwargs) @@ -1079,6 +1083,77 @@ def _algorithm_echo(chosen: Tuple[Any, Dict[str, Any]], spec: EvolveSpec, #: field is the place for them. _EVOLVE_OBJECT_KNOBS_HANDLED = frozenset({"agg_config"}) +#: Spec knobs ``compose`` consumes itself into the run-level call cap (#1166): +#: ``max_usd`` is a dollar cap and ``usd_per_call`` its price, and neither is an +#: ``evolve()`` parameter. ``max_calls`` is *not* here -- ``evolve`` already +#: takes it as the round-barrier budget, and the same number is also enforced per +#: call, which can only be stricter. +_COMPOSE_ONLY_KNOBS = frozenset({"max_usd", "usd_per_call"}) + + +def _cap_value(field: str, value: Any, *, integer: bool) -> float: + """One validated cap, refusing anything that is not a non-negative number. + + ``0`` is a limit, not an absence (#1166): it means "no calls" / "no spend". + """ + if isinstance(value, bool) or not isinstance(value, (int, float)): + kind = "integer" if integer else "number" + raise SpecError( + f"{field}: must be a non-negative {kind}, got {value!r}") + if integer and int(value) != value: + raise SpecError( + f"{field}: must be a non-negative integer, got {value!r}") + if not math.isfinite(float(value)) or float(value) < 0: + raise SpecError( + f"{field}: must be a non-negative {'integer' if integer else 'number'}, " + f"got {value!r}") + return int(value) if integer else float(value) + + +def build_call_limit(max_calls: Any = None, max_usd: Any = None, + usd_per_call: Any = None) -> Optional["CallLimit"]: + """A validated run-level call cap, or ``None`` when neither limit is set. + + Shared by ``compose`` (spec knobs) and the run store / CLI (run options), so + the refusal reads the same wherever the number was typed (#1166). + """ + from .budget import CallLimit + + if max_calls is None and max_usd is None: + return None + limit = CallLimit( + max_calls=(None if max_calls is None + else _cap_value("max_calls", max_calls, integer=True)), + max_usd=(None if max_usd is None + else _cap_value("max_usd", max_usd, integer=False)), + usd_per_call=(None if usd_per_call is None + else _cap_value("usd_per_call", usd_per_call, integer=False))) + if limit.max_usd is not None and not limit.usd_per_call: + raise SpecError( + "max_usd needs usd_per_call: a dollar cap cannot be charged without " + "a per-call price (pass --usd-per-call on the command line)") + return limit + + +def _limit_from_knobs(knobs: Dict[str, Any]) -> Optional["CallLimit"]: + """Build the run-level per-call cap a spec asked for, or ``None`` (#1166). + + Pops the keys ``evolve()`` cannot take (``max_usd`` / ``usd_per_call``) and + leaves ``max_calls`` in place, so the barrier budget of the same name still + applies -- a per-call cap of N and a barrier cap of N cannot disagree. + """ + limit = build_call_limit( + max_calls=(None if knobs.get("max_calls") is None + else _cap_value("evolve.max_calls", knobs["max_calls"], integer=True)), + max_usd=(None if knobs.get("max_usd") is None + else _cap_value("evolve.max_usd", knobs["max_usd"], integer=False)), + usd_per_call=(None if knobs.get("usd_per_call") is None + else _cap_value("evolve.usd_per_call", knobs["usd_per_call"], + integer=False))) + knobs.pop("max_usd", None) + return limit + + def check_evolve_knobs(evolve_knobs: Mapping[str, Any]) -> None: """Refuse a ``spec.evolve`` entry that ``evolve()`` could not use as given. @@ -1101,6 +1176,11 @@ def check_evolve_knobs(evolve_knobs: Mapping[str, Any]) -> None: params = inspect.signature(evolve).parameters for key, value in evolve_knobs.items(): param = params.get(key) + if param is None and key in _COMPOSE_ONLY_KNOBS: + # Consumed by `compose` into the run-level call cap (#1166); they are + # not evolve() kwargs, so a spec may carry them and they are popped + # before the pass-through. + continue if param is None: raise SpecError( f"evolve.{key}: evolve() has no such parameter; valid JSON knobs are " @@ -1390,6 +1470,12 @@ def run(skill: str, task: Task) -> str: # -- knobs: kind defaults < spec.evolve < caller overrides ------------------ knobs: Dict[str, Any] = {**defaults, **(ov.knobs if ov else {}), **_evolve_pass_through(spec.evolve), **overrides} + # A run-level call cap, from `max_calls` / `max_usd` (+ `usd_per_call`) or + # from a caller's own CallLimit. Validated here, where `plan` also reaches, + # so a negative or non-numeric cap is refused before a run directory exists. + limit = _limit_from_knobs(knobs) + if limit is not None: + knobs.setdefault("call_limit", limit) # kind=text defaults to SeededAppendRules above; a card (gepa/ace/dgm/ # skillopt) or compose(strategy=...) puts "strategy" in knobs and then # overwrites that default. Don't also claim SeededAppendRules (#879). @@ -1479,7 +1565,8 @@ def run(skill: str, task: Task) -> str: return Composition(tasks=tasks, reward=reward, kwargs=kwargs, tree=tree, notes=notes, audit=audit, algorithm=_algorithm_echo(chosen, spec, ov) if chosen else None, - layout=resolved_layout) + layout=resolved_layout, + call_limit=kwargs.get("call_limit")) def run_spec(spec: EvolveSpec, **hooks: Any) -> EvolutionResult: diff --git a/agentdescent/metrics.py b/agentdescent/metrics.py index 8b58e495..299eac4c 100644 --- a/agentdescent/metrics.py +++ b/agentdescent/metrics.py @@ -387,7 +387,8 @@ def summary(self) -> str: def measured(fn: Callable[..., T], meter: Optional[Meter], - *, phase: Optional[str] = None) -> Callable[..., T]: + *, phase: Optional[str] = None, + guard: Optional[Any] = None) -> Callable[..., T]: """Wrap an actor so each invocation lands in ``meter.usage``. ``agents.metered`` does the same job for a ``Completion`` (``prompt -> str``); @@ -408,11 +409,19 @@ def measured(fn: Callable[..., T], meter: Optional[Meter], Returns ``fn`` unchanged when ``meter`` is ``None``, so the un-instrumented path stays exactly one function call deep. + + ``guard`` (a :class:`~agentdescent.budget.CallLimit`) is asked for a slot + **before** the call, so a run-level budget refuses the call rather than + counting it afterwards (#1166). It raises ``BudgetStop``, which travels out + of here unchanged -- the engine catches it by name and ends the round + cleanly. """ if meter is None: return fn def call(*args: Any, **kwargs: Any) -> T: + if guard is not None: + guard.reserve() token = meter.usage.begin(phase=phase) t0 = time.time() try: diff --git a/agentdescent/runstore.py b/agentdescent/runstore.py index ecb78d89..18889fca 100644 --- a/agentdescent/runstore.py +++ b/agentdescent/runstore.py @@ -175,13 +175,22 @@ class RunStatus: updated: Optional[float] = None finished: Optional[float] = None stop_reason: Optional[str] = None + #: Which limit fired and what was spent, when ``stop_reason == "budget"`` + #: (#1166). Additive and optional: absent means "no run-level cap tripped", + #: and an old ``status.json`` without the key reads as ``None``. + stop_detail: Optional[str] = None error: Optional[str] = None kind: Optional[str] = None target: Optional[str] = None notes: List[str] = field(default_factory=list) def to_dict(self) -> Dict[str, Any]: - return asdict(self) + d = asdict(self) + # Shape stability (#1166): a run with no cap must write the same file it + # wrote before this field existed, so the key is omitted when unset. + if d.get("stop_detail") is None: + d.pop("stop_detail", None) + return d @classmethod def from_dict(cls, d: Dict[str, Any]) -> "RunStatus": @@ -660,6 +669,8 @@ def _error_from_log(log_path: str) -> Optional[str]: def launch(rd: RunDir, *, budget_usd: Optional[float] = None, usd_per_call: Optional[float] = None, + max_calls: Optional[int] = None, + max_usd: Optional[float] = None, env: Optional[Dict[str, str]] = None) -> RunStatus: """Start the run as a **detached** process and return at once. @@ -671,6 +682,12 @@ def launch(rd: RunDir, *, budget_usd: Optional[float] = None, Its stdout and stderr go to ``log.txt``. """ require_budget_pair(budget_usd, usd_per_call) + from .evolvespec import SpecError, build_call_limit + try: + build_call_limit(max_calls=max_calls, max_usd=max_usd, + usd_per_call=usd_per_call) + except SpecError as e: + raise RunStoreError(str(e)) from None st = rd.status() if st.state == "running" and st.alive(): raise RunStoreError(f"{rd.run_id} is already running (pid {st.pid})") @@ -679,6 +696,12 @@ def launch(rd: RunDir, *, budget_usd: Optional[float] = None, argv += ["--budget", str(budget_usd)] if usd_per_call is not None: argv += ["--usd-per-call", str(usd_per_call)] + # The run-level cap crosses the detach boundary as flags, so the child + # charges the same way a foreground run does (#1166). + if max_calls is not None: + argv += ["--max-calls", str(max_calls)] + if max_usd is not None: + argv += ["--max-usd", str(max_usd)] # Scrub the merged dict, not just the ambient one: a caller-supplied # ``env`` is spread last and could otherwise put ``MCP_*``/``DSH_*`` back # into the run after the scrub (#1082). ``PYTHONPATH`` carries no marker @@ -788,7 +811,9 @@ def fmt(message, category, filename, lineno, line=None): # type: ignore[no-unty def execute(rd: RunDir, *, budget_usd: Optional[float] = None, - usd_per_call: Optional[float] = None) -> EvolutionResult: + usd_per_call: Optional[float] = None, + max_calls: Optional[int] = None, + max_usd: Optional[float] = None) -> EvolutionResult: """Run the spec in ``rd`` **in this process**, keeping the directory current. This is what the detached child calls; it is also usable directly for a @@ -796,9 +821,16 @@ def execute(rd: RunDir, *, budget_usd: Optional[float] = None, ``on_round`` hook writes ``status.json`` and ``rounds.jsonl``; the ledger lives under the run so a later :func:`resume` continues it; the result and the materialised tree are written at the end whatever the outcome. + + ``max_calls`` / ``max_usd`` are the run-level caps charged **before** each + model call (#1166). They are stricter than ``budget_usd``, which is a + between-round warning plus a ``stop_when``: a four-worker round on + ``max_calls=1`` still made four calls under that budget, and makes one here. + A capped run ends ``done`` with ``stop_reason="budget"`` and ``stop_detail`` + naming the limit and the spend. """ from .agents import Usage - from .evolvespec import EvolveSpec, compose + from .evolvespec import EvolveSpec, SpecError, build_call_limit, compose from .filetree import materialize # Refuse before marking running: a missing usd_per_call must not look like a @@ -811,6 +843,17 @@ def execute(rd: RunDir, *, budget_usd: Optional[float] = None, spec = EvolveSpec.from_dict(rd.spec_dict()) usage = Usage() + # The run-level cap (#1166). Built here rather than in the engine so a + # detached child and a foreground run charge the same way; a spec's own + # `max_calls` / `max_usd` knobs are validated by `compose`. + try: + limit = build_call_limit(max_calls=max_calls, max_usd=max_usd, + usd_per_call=usd_per_call) + except SpecError as e: + # A bad cap is a refusal, not a live run: record it and never mark + # running, the same shape as a missing usd_per_call above. + rd.update_status(state="failed", finished=time.time(), error=str(e)[:2000]) + raise RunStoreError(str(e)) from None def _progress() -> None: # Mid-call rewrite so CLI/MCP status shows work in flight (#358). @@ -890,6 +933,10 @@ def _store_round(info: RoundInfo, reward: Optional[float]) -> None: rd.update_status(**changes) overrides: Dict[str, Any] = {} + if limit is not None: + # Wins over a spec's own knobs: an explicit run option is the sharper + # instruction, exactly as every other override works (#1166). + overrides["call_limit"] = limit if budget_usd is not None: def stop_when(info: RoundInfo) -> bool: # Asked between rounds, where max_calls is; the run overshoots by at @@ -929,9 +976,13 @@ def stop_when(info: RoundInfo) -> bool: materialize(result.state, rd.tree_path) final = _finite_or_none(result.final_reward) initial = _finite_or_none(getattr(result, "initial_reward", None)) + # Which limit fired, from whichever cap this run actually had: the run + # option, or the one `compose` built from the spec's knobs (#1166). + cap = limit if limit is not None else getattr(comp, "call_limit", None) + detail = cap.detail() if cap is not None else None rd.update_status( state="failed" if result.error else "done", finished=time.time(), - stop_reason=result.stop_reason, error=result.error, + stop_reason=result.stop_reason, stop_detail=detail, error=result.error, best_reward=max(best) if best else final, last_reward=final, initial_reward=initial, test_reward=_finite_or_none(getattr(result, "test_reward", None)), diff --git a/changelog.d/1166-run-call-cap.md b/changelog.d/1166-run-call-cap.md new file mode 100644 index 00000000..01b6a04b --- /dev/null +++ b/changelog.d/1166-run-call-cap.md @@ -0,0 +1,11 @@ +Added +A run-level call cap that holds before the call (`--max-calls`, `--max-usd`, +also `max_calls` / `max_usd` in a spec's `evolve` block) (#1166). Unlike +`max_calls` on `evolve()`, which is a round barrier and lets a four-worker round +overshoot, the guarded cap is charged before each model call: the call that +would cross the limit is not made, the round closes without a partial merge or a +traceback, and the run ends `done` with `stop_reason=budget` and a +`stop_detail` naming the limit and the observed value, in `status.json` and in +`agentdescent status`. `0` is a real limit (no calls); a negative or non-numeric +value is refused at validation with the field and the value named; with neither +flag set the run and `status.json` are unchanged. Offline, no new dependency. diff --git a/docs/api.md b/docs/api.md index 3c199e4a..61fc89fb 100644 --- a/docs/api.md +++ b/docs/api.md @@ -246,6 +246,7 @@ evolve( max_rollouts: Optional[int] = None, max_calls: Optional[int] = None, max_tokens: Optional[int] = None, + call_limit: Optional['CallLimit'] = None, stop_on_diminishing_returns: bool = False, efficiency_floor: float = 0.25, call_budget: Optional['CallBudget'] = None, @@ -313,6 +314,7 @@ evolve( | `max_rollouts` | `Optional[int]` | `None` | The budget in the two units a comparison has to hold fixed: rollouts completed, and actor invocations (`run` + `propose`). `rounds` is not one of them -- configurations differ in how much model a round buys, so a budget fixed in rounds hands the wider configuration more model and then reports the extra model as a win for parallelism. Either bound stops the run with `stop_reason` `"max_rollouts"` / `"max_calls"`. **Checked at the round barrier, so a run overshoots by up to one round.** A round is dispatched or it is not; stopping halfway would leave a half-merged round, and the states a comparison compares are the ones a merge produced. So a budget is a *bound on where to stop*, never the number to compare on: read the spend the run actually reported (`result.rollouts`, `result.usage.calls`), which is what `baselines` does -- it refuses to call two arms equal-budget when their measured spends differ. The async path has no barrier and enforces both per rollout, so it overshoots by at most the rollouts already in flight. | | `max_calls` | `Optional[int]` | `None` | As `max_rollouts`. | | `max_tokens` | `Optional[int]` | `None` | Hard cap on total tokens consumed (`prompt_tokens + completion_tokens`). A reasoning model can spend 40k tokens on hidden thinking in one call, so `max_calls` and `max_rollouts` do not bound cost: a 20-round run with 4 workers is 80 rollouts and ~160 model calls, but at 40k tokens each that is 6.4M tokens -- the bill, not the count, is what a deployment needs to control. Checked at the round barrier alongside the other budgets; the async path checks per-rollout for tighter control. Stops with `stop_reason="max_tokens"`. `None` (default) means unbounded. | +| `call_limit` | `Optional['CallLimit']` | `None` | | | `stop_on_diminishing_returns` | `bool` | `False` | Stop when the run's own return per token has fallen off its peak by more than `efficiency_floor`. The economic rule: keep buying compute while it pays, stop when it does not -- *even with budget left*. Off by default, because a run whose reward only rises late would be cut short by it, and because it needs `max_tokens` (return per token needs a cost to divide by). Stops with `stop_reason="diminishing_returns"`. | | `efficiency_floor` | `float` | `0.25` | How far below the peak counts as diminishing, when the stop above is on. Self-calibrating against the run's own best rate, so it is a *ratio*, not an absolute quantity: a reward is in `[0, 1]` and a token count is in the millions, and their quotient has no interpretable scale. | | `call_budget` | `Optional['CallBudget']` | `None` | An adaptive per-call thinking budget (o1-style test-time scaling). Pass a `CallBudget` whose `base` matches the adapter's configured `max_tokens`, and the engine adjusts the per-call ceiling based on each parent's score and the remaining budget — promising parents get more thinking, dead-end ones get less, and the allocation tightens as the token budget is spent. The adapter must be wrapped with `budgeted_completion` for this to reach it. `None` (default) means the per-call ceiling is the adapter's own and never changes. | @@ -4878,6 +4880,7 @@ BudgetSettings( max_rollouts: Optional[int] = None, max_calls: Optional[int] = None, max_tokens: Optional[int] = None, + call_limit: Optional['CallLimit'] = None, stop_on_diminishing_returns: bool = False, efficiency_floor: float = 0.25, call_budget: Optional['CallBudget'] = None, diff --git a/docs/start.md b/docs/start.md index 78cc7666..40b0734c 100644 --- a/docs/start.md +++ b/docs/start.md @@ -122,6 +122,20 @@ know a tool-using agent's bill in advance: agentdescent plan spec.json --budget 5 --usd-per-call 0.02 ``` +`--budget` is checked between rounds, so a wide round can overshoot it. For a +cap that holds **per call**, pass `--max-calls` / `--max-usd`: the call that +would cross the limit is not made, the round winds down, and the run ends with +`stop_reason=budget` plus the limit and the spend it observed, in `status.json` +and in `agentdescent status`: + +```bash +agentdescent evolve spec.json --max-calls 40 +agentdescent evolve spec.json --max-usd 5 --usd-per-call 0.02 +``` + +`--max-calls 0` is a real limit — no calls at all — not "unset". Leave both +flags off and the run behaves exactly as before. + The walkthrough that loads a dataset, calls a model, and evolves an instruction is in [Quickstart — dataset to skill](quickstart-skill.md). That path evolves a `SingleSlot` instruction; for a skill library use From f8de7ee3feeef5beb21cec84ee3d05dd3b7fa126 Mon Sep 17 00:00:00 2001 From: HugoXK Date: Thu, 19 Nov 2026 11:34:44 +0000 Subject: [PATCH 3/5] test(run): pin the status.json shape and document the new cap knob (#1166) The allowlist entry and the evolve() docstring are what keep a flat kwarg deliberate rather than drift; the two status.json tests pin the additive contract (no key when unset, stop_detail when the cap fires). --- agentdescent/evolution.py | 10 ++++++++++ agentdescent/evolve_settings.py | 5 +++++ docs/api.md | 2 +- tests/test_budget_guard.py | 17 +++++++++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/agentdescent/evolution.py b/agentdescent/evolution.py index e89502ad..2428b748 100644 --- a/agentdescent/evolution.py +++ b/agentdescent/evolution.py @@ -3527,6 +3527,16 @@ def evolve( other budgets; the async path checks per-rollout for tighter control. Stops with ``stop_reason="max_tokens"``. ``None`` (default) means unbounded. + call_limit: + A :class:`~agentdescent.budget.CallLimit` charged **before** each model + call: calls and dollars (at a per-call price) are reserved up front, so + the call that would cross the cap is never made. Different from + ``max_calls`` above, which is a round barrier that lets a wide round + overshoot. The refusal unwinds the round without a partial merge, and + the run stops with ``stop_reason="budget"``; the run store writes which + limit fired and what it observed into ``status.json``. Built by the run + store from ``--max-calls`` / ``--max-usd`` (or a spec's ``evolve`` + knobs); ``None`` (default) is the run exactly as it was. stop_on_diminishing_returns: Stop when the run's own return per token has fallen off its peak by more than ``efficiency_floor``. The economic rule: keep buying compute while diff --git a/agentdescent/evolve_settings.py b/agentdescent/evolve_settings.py index 7801a6eb..55671760 100644 --- a/agentdescent/evolve_settings.py +++ b/agentdescent/evolve_settings.py @@ -56,6 +56,11 @@ #: RuntimeSettings field for this slice (#804): it is additive, defaults to #: ``0.0`` (the old run exactly), and the bag refactor is a separate card. "test_frac", + #: #1166: the per-call budget, passed by ``compose`` / the run store and + #: also a ``BudgetSettings`` field. A flat name because a JSON + #: ``spec.evolve`` knob has to reach the engine by that name; it defaults to + #: ``None``, which is the old run exactly. + "call_limit", # typed bags (#804) — preferred home for new numerics / flags "budget", "runtime", "observability", }) diff --git a/docs/api.md b/docs/api.md index 61fc89fb..3213b896 100644 --- a/docs/api.md +++ b/docs/api.md @@ -314,7 +314,7 @@ evolve( | `max_rollouts` | `Optional[int]` | `None` | The budget in the two units a comparison has to hold fixed: rollouts completed, and actor invocations (`run` + `propose`). `rounds` is not one of them -- configurations differ in how much model a round buys, so a budget fixed in rounds hands the wider configuration more model and then reports the extra model as a win for parallelism. Either bound stops the run with `stop_reason` `"max_rollouts"` / `"max_calls"`. **Checked at the round barrier, so a run overshoots by up to one round.** A round is dispatched or it is not; stopping halfway would leave a half-merged round, and the states a comparison compares are the ones a merge produced. So a budget is a *bound on where to stop*, never the number to compare on: read the spend the run actually reported (`result.rollouts`, `result.usage.calls`), which is what `baselines` does -- it refuses to call two arms equal-budget when their measured spends differ. The async path has no barrier and enforces both per rollout, so it overshoots by at most the rollouts already in flight. | | `max_calls` | `Optional[int]` | `None` | As `max_rollouts`. | | `max_tokens` | `Optional[int]` | `None` | Hard cap on total tokens consumed (`prompt_tokens + completion_tokens`). A reasoning model can spend 40k tokens on hidden thinking in one call, so `max_calls` and `max_rollouts` do not bound cost: a 20-round run with 4 workers is 80 rollouts and ~160 model calls, but at 40k tokens each that is 6.4M tokens -- the bill, not the count, is what a deployment needs to control. Checked at the round barrier alongside the other budgets; the async path checks per-rollout for tighter control. Stops with `stop_reason="max_tokens"`. `None` (default) means unbounded. | -| `call_limit` | `Optional['CallLimit']` | `None` | | +| `call_limit` | `Optional['CallLimit']` | `None` | A `CallLimit` charged **before** each model call: calls and dollars (at a per-call price) are reserved up front, so the call that would cross the cap is never made. Different from `max_calls` above, which is a round barrier that lets a wide round overshoot. The refusal unwinds the round without a partial merge, and the run stops with `stop_reason="budget"`; the run store writes which limit fired and what it observed into `status.json`. Built by the run store from `--max-calls` / `--max-usd` (or a spec's `evolve` knobs); `None` (default) is the run exactly as it was. | | `stop_on_diminishing_returns` | `bool` | `False` | Stop when the run's own return per token has fallen off its peak by more than `efficiency_floor`. The economic rule: keep buying compute while it pays, stop when it does not -- *even with budget left*. Off by default, because a run whose reward only rises late would be cut short by it, and because it needs `max_tokens` (return per token needs a cost to divide by). Stops with `stop_reason="diminishing_returns"`. | | `efficiency_floor` | `float` | `0.25` | How far below the peak counts as diminishing, when the stop above is on. Self-calibrating against the run's own best rate, so it is a *ratio*, not an absolute quantity: a reward is in `[0, 1]` and a token count is in the millions, and their quotient has no interpretable scale. | | `call_budget` | `Optional['CallBudget']` | `None` | An adaptive per-call thinking budget (o1-style test-time scaling). Pass a `CallBudget` whose `base` matches the adapter's configured `max_tokens`, and the engine adjusts the per-call ceiling based on each parent's score and the remaining budget — promising parents get more thinking, dead-end ones get less, and the allocation tightens as the token budget is spent. The adapter must be wrapped with `budgeted_completion` for this to reach it. `None` (default) means the per-call ceiling is the adapter's own and never changes. | diff --git a/tests/test_budget_guard.py b/tests/test_budget_guard.py index 82fac221..58eba5f0 100644 --- a/tests/test_budget_guard.py +++ b/tests/test_budget_guard.py @@ -12,6 +12,9 @@ from __future__ import annotations +import json +import pathlib + import pytest from agentdescent import runstore @@ -105,3 +108,17 @@ def test_zero_is_a_limit_not_an_absence(tmp_path): assert CALLS == [], f"a zero-call budget made {len(CALLS)} calls" assert st.stop_reason == "budget", st assert "max_calls" in (st.stop_detail or ""), st.stop_detail + + +def test_status_json_keeps_its_shape_when_no_cap_is_set(tmp_path): + """No cap, no new key: a reader of yesterday's file sees yesterday's file.""" + rd, st, _ = _run(tmp_path) + payload = json.loads((pathlib.Path(rd.path) / "status.json").read_text()) + assert "stop_detail" not in payload, payload + + +def test_status_json_carries_the_limit_that_fired(tmp_path): + rd, st, _ = _run(tmp_path, max_calls=1) + payload = json.loads((pathlib.Path(rd.path) / "status.json").read_text()) + assert payload["stop_reason"] == "budget", payload + assert "max_calls" in payload["stop_detail"], payload From 3b49b89ac31da1f5a2707c3633675963b346084e Mon Sep 17 00:00:00 2001 From: HugoXK Date: Thu, 19 Nov 2026 11:39:14 +0000 Subject: [PATCH 4/5] fix(run): refuse a run-level call cap on the async path (#1166) The barrier-free runtime has no per-call guard yet; a cap that reached it would be silently unbounded, so it is refused by name rather than warned about. Sync and async stay two contracts (#809). --- agentdescent/evolution.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/agentdescent/evolution.py b/agentdescent/evolution.py index 2428b748..49b32032 100644 --- a/agentdescent/evolution.py +++ b/agentdescent/evolution.py @@ -3887,12 +3887,23 @@ def evolve( "the barrier-free runtime has no select/test split yet, so a run " "would report a select-set reward as if a test split had been " "measured -- run synchronously to get a test_reward (#843)"), + ("call_limit", call_limit, + "the barrier-free runtime has no per-call guard yet, so a cap " + "passed here would not be enforced -- run synchronously for " + "--max-calls / --max-usd (#1166)"), ): if value is not None: warnings.warn( f"evolve(asynchronous=True) ignores {name}=: {why}. Use the " f"synchronous path if you need it.", RuntimeWarning, stacklevel=2) + if call_limit is not None: + # A warning is not enough for a spend cap: an ignored one is an + # unbounded run that looks bounded (#1166). + raise ValueError( + "evolve(asynchronous=True) cannot enforce a run-level call cap: " + "the barrier-free runtime has no per-call guard yet. Run " + "synchronously (asynchronous=False) or drop max_calls / max_usd.") if max_seconds is None: raise ValueError( "evolve(asynchronous=True) no longer invents max_seconds=20.0 " From 3300d557a4cbb170fd52682dbce6cf82662b6c5d Mon Sep 17 00:00:00 2001 From: HugoXK Date: Thu, 19 Nov 2026 12:18:15 +0000 Subject: [PATCH 5/5] fix(run): a cap refused inside a rollout is a stop, not a failure (#1166) Two contract gaps found after the first tip, both reproduced before the fix: - A spec-level `max_usd` could not run: `_limit_from_knobs` popped `max_usd` but left `usd_per_call`, which `compose` then handed to `evolve()`, so every valid dollar cap died with `TypeError: evolve() got an unexpected keyword argument 'usd_per_call'` and the run was recorded `failed` with calls=0. - A cap refused inside an executor-dispatched rollout was recorded as a worker failure: `ThreadExecutor._one`'s generic `except Exception` turned the `BudgetStop` into `Result(kind="model")`, `_rollout_failure` rebuilt it as `RuntimeError("BudgetStop: ...")`, and the round body's generic handler set `run_error` -- `state=failed`, `stop_reason=error`, `stop_detail` present. `max_calls=2` under four workers hit it; so did `max_usd=0.05` at $0.02/call. The refusal now travels as its own `Result.kind="budget"` (report, never raise: `map_rollouts` delivers through a queue, and a bare raise would hang the reader), and `_rollout_failure` re-raises the stop the `CallLimit` already recorded -- numbers included -- so the round closes `done` with `stop_reason=budget`. `CallLimit.stop` exposes that recorded refusal. Also drops the contradictory `RuntimeWarning` before the async `ValueError`: the async path refuses a cap, it does not ignore one. --- agentdescent/budget.py | 10 +++++++ agentdescent/evolution.py | 15 ++++++----- agentdescent/evolvespec.py | 1 + agentdescent/executor.py | 11 +++++++- tests/test_async_early_stop.py | 6 ++++- tests/test_budget_guard.py | 49 ++++++++++++++++++++++++++++++++++ 6 files changed, 84 insertions(+), 8 deletions(-) diff --git a/agentdescent/budget.py b/agentdescent/budget.py index 3e039546..6dd080ce 100644 --- a/agentdescent/budget.py +++ b/agentdescent/budget.py @@ -124,6 +124,16 @@ def reserve(self) -> None: raise stop self._taken = n + @property + def stop(self) -> Optional[BudgetStop]: + """The refusal this limit already made, or ``None`` (#1166). + + The refusal is raised inside a worker and caught somewhere else, so the + object recorded here is what a later handler re-raises: the numbers + survive the trip instead of being rebuilt from a message. + """ + return self._stopped + @property def calls(self) -> int: """Calls this limit has admitted so far.""" diff --git a/agentdescent/evolution.py b/agentdescent/evolution.py index 49b32032..1a4cc459 100644 --- a/agentdescent/evolution.py +++ b/agentdescent/evolution.py @@ -1100,7 +1100,7 @@ def _spec_for(engine: "_Engine", artifact, task: Task, sandbox=engine.sandbox_spec or SandboxSpec()) -def _rollout_failure(outcome) -> BaseException: +def _rollout_failure(outcome, limit=None) -> BaseException: """Turn a reported failure back into the exception the round body expects. The executor reports rather than raises, because one bad rollout is evidence @@ -1108,6 +1108,13 @@ def _rollout_failure(outcome) -> BaseException: is written against exceptions -- and it distinguishes a caller's broken contract, which must stop the run, from a backend transient, which must not. That distinction is carried in `Result.kind`, so it survives the trip.""" + if outcome.kind == "budget" and limit is not None: + # A cap refused the call inside this rollout (#1166). The refusal the + # limit recorded is re-raised as itself, so the round closes `done` + # with a named budget stop and the numbers it observed. + stop = limit.stop + if stop is not None: + return stop if outcome.kind == "caller": return ProposalContractError(outcome.error or "contract violation") return RuntimeError(outcome.error or "rollout failed") @@ -3887,10 +3894,6 @@ def evolve( "the barrier-free runtime has no select/test split yet, so a run " "would report a select-set reward as if a test split had been " "measured -- run synchronously to get a test_reward (#843)"), - ("call_limit", call_limit, - "the barrier-free runtime has no per-call guard yet, so a cap " - "passed here would not be enforced -- run synchronously for " - "--max-calls / --max-usd (#1166)"), ): if value is not None: warnings.warn( @@ -4326,7 +4329,7 @@ def _propose_one(unit, task, aid: str) -> None: rendered = mine.render() outcome = eng.executor.rollout(_spec_for(eng, mine, task, rendered)) if not outcome.ok: - raise _rollout_failure(outcome) + raise _rollout_failure(outcome, call_limit) output = outcome.output score = _checked_reward(outcome.reward, task) # Record the rollout before anything else can decide it does not diff --git a/agentdescent/evolvespec.py b/agentdescent/evolvespec.py index 7dee95be..37df7919 100644 --- a/agentdescent/evolvespec.py +++ b/agentdescent/evolvespec.py @@ -1151,6 +1151,7 @@ def _limit_from_knobs(knobs: Dict[str, Any]) -> Optional["CallLimit"]: else _cap_value("evolve.usd_per_call", knobs["usd_per_call"], integer=False))) knobs.pop("max_usd", None) + knobs.pop("usd_per_call", None) return limit diff --git a/agentdescent/executor.py b/agentdescent/executor.py index 0e1e658f..cf665579 100644 --- a/agentdescent/executor.py +++ b/agentdescent/executor.py @@ -40,6 +40,7 @@ from typing import Any, Callable, Iterator, List, Optional, Protocol, Sequence, runtime_checkable from .agents import ProviderConfigError +from .budget import BudgetStop from .metrics import Meter from .pipeline import describe from .workspec import RolloutSpec @@ -63,7 +64,7 @@ class Result: #: How the failure should be counted. Infrastructure failures must not feed #: the worker-retirement rule, which fires hardest before any worker has #: succeeded -- exactly when sandboxes fail to start. - kind: str = "ok" # "ok" | "model" | "infrastructure" | "caller" + kind: str = "ok" # "ok" | "model" | "infrastructure" | "caller" | "budget" seconds: float = 0.0 @property @@ -139,6 +140,14 @@ def _one(self, spec: RolloutSpec) -> Result: # sample (#635). return Result(spec.lease_id, spec.task.id, error=describe(e), kind="caller", seconds=time.time() - t0) + except BudgetStop as e: + # A run-level call cap refused this call (#1166). Not a rollout + # failure and not raised: `map_rollouts` delivers results through a + # queue, so an exception here would either hang the reader or be + # lost. Its own kind lets the engine re-raise the stop the limit + # already recorded, numbers included. + return Result(spec.lease_id, spec.task.id, error=e.detail(), + kind="budget", seconds=time.time() - t0) except Exception as e: # noqa: BLE001 -- a backend or candidate failure return Result(spec.lease_id, spec.task.id, error=describe(e), kind="model", seconds=time.time() - t0) diff --git a/tests/test_async_early_stop.py b/tests/test_async_early_stop.py index 6a61d2e3..ec5a9b55 100644 --- a/tests/test_async_early_stop.py +++ b/tests/test_async_early_stop.py @@ -61,7 +61,11 @@ def test_no_evolve_argument_is_dropped_without_notice(): # has always called `max_iters`, so it reaches async_evolve as that. remapped = {"tasks", "reward", "rounds", "max_seconds", "asynchronous", "max_rollouts"} - leaked = set(inspect.signature(_evolve).parameters) - forwarded - warned - remapped + # Refused by name: a knob this path rejects with its own ValueError is + # accounted for more strongly than one it warns about (#1166). + refused = set(re.findall(r"if (\w+) is not None:", src)) + leaked = (set(inspect.signature(_evolve).parameters) + - forwarded - warned - refused - remapped) assert not leaked, f"silently dropped by the async path: {sorted(leaked)}" diff --git a/tests/test_budget_guard.py b/tests/test_budget_guard.py index 58eba5f0..c2c44a2c 100644 --- a/tests/test_budget_guard.py +++ b/tests/test_budget_guard.py @@ -122,3 +122,52 @@ def test_status_json_carries_the_limit_that_fired(tmp_path): payload = json.loads((pathlib.Path(rd.path) / "status.json").read_text()) assert payload["stop_reason"] == "budget", payload assert "max_calls" in payload["stop_detail"], payload + + +def test_a_spec_level_dollar_cap_runs(tmp_path): + """``max_usd`` in a spec's ``evolve`` block is usable, not a ``TypeError``. + + ``usd_per_call`` is compose-only like ``max_usd``: leaving it in the knob + dict handed it to ``evolve()``, which has no such parameter, so every valid + dollar cap died in round setup while the count cap worked. + """ + spec = _spec(max_usd=0.015, usd_per_call=0.01) + comp = compose(spec) + assert "usd_per_call" not in comp.kwargs, comp.kwargs + rd, st, result = _run(tmp_path, spec=spec) + assert CALLS == ["agent"], CALLS + assert st.state == "done", st + assert st.error is None + assert st.stop_reason == "budget", st + assert "max_usd" in (st.stop_detail or ""), st.stop_detail + + +def test_a_cap_that_trips_inside_a_rollout_ends_done(tmp_path): + """The refusal reaches the engine through a worker's rollout (#1166). + + ``max_calls=2`` under four workers is refused while a rollout is in flight, + not at the seed score. The executor reports that refusal like any other + rollout outcome, and the run must still end ``done`` with the named budget + stop rather than ``failed`` on a ``RuntimeError``. + """ + rd, st, result = _run(tmp_path, max_calls=2) + assert st.calls == 2, f"a capped run made {st.calls} calls" + assert st.state == "done", st + assert st.error is None, st.error + assert st.stop_reason == "budget", st + assert "max_calls" in (st.stop_detail or ""), st.stop_detail + + +def test_a_dollar_cap_that_trips_inside_a_rollout_ends_done(tmp_path): + """The dollar cap, refused on the concurrent segment, not the serial one. + + ``0.05`` at ``0.02``/call admits two calls and refuses the third while four + workers are in flight; ``0.05`` at ``0.01`` would admit five, and that + refusal lands in a serial propose, so it cannot tell the two paths apart. + """ + rd, st, result = _run(tmp_path, max_usd=0.05, usd_per_call=0.02) + assert st.calls == 2, f"a capped run made {st.calls} calls" + assert st.state == "done", st + assert st.error is None, st.error + assert st.stop_reason == "budget", st + assert "max_usd" in (st.stop_detail or ""), st.stop_detail