diff --git a/agentdescent/budget.py b/agentdescent/budget.py index 72d7fac4..6dd080ce 100644 --- a/agentdescent/budget.py +++ b/agentdescent/budget.py @@ -34,6 +34,7 @@ from __future__ import annotations import math +import threading from dataclasses import dataclass, field from typing import Any, Callable, Optional @@ -52,6 +53,97 @@ HARD_FLOOR = 0.90 +class BudgetStop(Exception): + """A model call that a run-level call budget refused to make (#1166). + + Raised by :meth:`CallLimit.reserve` **before** the call, and caught by name + in :mod:`agentdescent.evolution` -- the worker wrapper's generic + ``except Exception`` would otherwise record a clean budget stop as a failed + worker, and the run would end ``failed`` with an error instead of a named + stop. + """ + + def __init__(self, limit: str, observed: float, cap: float) -> None: + self.limit = limit + self.observed = observed + self.cap = cap + super().__init__(f"{limit}={cap} reached") + + def detail(self) -> str: + """One line for ``status``: which limit fired and what was spent.""" + if self.limit == "max_calls": + return (f"max_calls: {int(self.observed)} call(s) made, " + f"cap {int(self.cap)}") + return f"max_usd: ${self.observed:.4f} spent, cap ${self.cap:.4f}" + + +@dataclass +class CallLimit: + """A run-level cap on calls and dollars, charged before every call (#1166). + + ``reserve()`` *is* the mechanism: it runs before the call is made, so "the + call did not happen" is structural rather than a check after the fact. The + engine's round-barrier ``max_calls`` is a different quantity -- it bounds + where to stop, so a four-worker round on ``max_calls=1`` still makes four + calls; this one bounds the calls themselves. + + Thread-safe: ``metrics.measured`` wraps both worker threads and the + evaluation pool. ``0`` is a limit ("no calls"), not an absence. + """ + + max_calls: Optional[int] = None + #: A dollar cap needs a per-call price; without one it cannot be charged and + #: :func:`agentdescent.runstore.require_budget_pair` refuses the run. + max_usd: Optional[float] = None + usd_per_call: Optional[float] = None + _lock: threading.Lock = field(default_factory=threading.Lock, repr=False, + compare=False) + _taken: int = field(default=0, init=False, repr=False, compare=False) + _stopped: Optional[BudgetStop] = field(default=None, init=False, + repr=False, compare=False) + + def reserve(self) -> None: + """Take the next call's slot, or raise :class:`BudgetStop`. + + The slot is taken **before** the call so concurrent workers cannot both + pass a check that only one of them can afford. + """ + with self._lock: + n = self._taken + 1 + stop: Optional[BudgetStop] = None + if self.max_calls is not None and n > self.max_calls: + stop = BudgetStop("max_calls", self._taken, self.max_calls) + elif self.max_usd is not None and self.usd_per_call: + # Tolerance: 3 x 0.01 must not exceed 0.03 through float noise. + if n * self.usd_per_call > self.max_usd + 1e-9: + stop = BudgetStop("max_usd", self._taken * self.usd_per_call, + self.max_usd) + if stop is not None: + if self._stopped is None: + self._stopped = stop + raise stop + self._taken = n + + @property + def stop(self) -> Optional[BudgetStop]: + """The refusal this limit already made, or ``None`` (#1166). + + The refusal is raised inside a worker and caught somewhere else, so the + object recorded here is what a later handler re-raises: the numbers + survive the trip instead of being rebuilt from a message. + """ + return self._stopped + + @property + def calls(self) -> int: + """Calls this limit has admitted so far.""" + return self._taken + + def detail(self) -> Optional[str]: + """Why the limit refused, or ``None`` when it never did.""" + return None if self._stopped is None else self._stopped.detail() + + @dataclass class BudgetGovernor: """Tracks token spend against a cap and decides what to degrade. diff --git a/agentdescent/cli.py b/agentdescent/cli.py index fb31fa64..1164b522 100644 --- a/agentdescent/cli.py +++ b/agentdescent/cli.py @@ -53,8 +53,8 @@ from .evolution import (SELECT_SET_NOTE, _check_apply_paths, _file_tree_backup_path, reward_set_line) from .evolvespec import (KINDS, EvolveSpec, SpecError, budget_round_overshoot_warning, - compose, estimate, fold_kind, load_spec, resolve_kind, - _POLICY_SLOTS) + build_call_limit, compose, estimate, fold_kind, load_spec, + resolve_kind, _POLICY_SLOTS) from .probe import first_failed_next __all__ = ["main", "doctor_report", "starter_spec"] @@ -79,6 +79,10 @@ def _version() -> str: return "unknown" #: Set in every worker's environment by the plugin runner and the MCP server so a +#: The `stop_reason` a run-level call cap writes (#1166): the budget fired, and +#: `stop_detail` names which limit and what it observed. +BUDGET_STOP = "budget" + #: host that is itself being evolved cannot start a nested run. See docs/plugins.md. NESTED_ENV = "AGENTDESCENT_NESTED" @@ -176,6 +180,11 @@ def _status_line(st: runstore.RunStatus) -> str: ph = f" phase={st.phase}" if st.phase else "" pending = f" pending={st.pending_calls}{ph}" tail = f" [{st.stop_reason}]" if st.stop_reason else "" + # Which limit fired and what it cost, on the same line: a reader asking + # "why did my run stop" should not have to open `status.json` (#1166). + if st.stop_detail: + tail = (f" [{st.stop_reason}: {st.stop_detail}]" if st.stop_reason + else f" [{st.stop_detail}]") err = f" error: {st.error}" if st.error else "" # Reflector exceptions, cumulative (#395): a run can finish `done` with only # these to say that it never learned anything. @@ -2707,6 +2716,14 @@ def cmd_evolve(a: argparse.Namespace) -> int: # Before create/launch: a missing --usd-per-call must not leave an orphan run # whose status later reads as "process ended without reporting" (#342). runstore.require_budget_pair(a.budget, a.usd_per_call) + # The run-level cap (#1166): refused here, before a run directory exists, + # with the field and the value named -- a bad flag is not a live run. + try: + build_call_limit(max_calls=getattr(a, "max_calls", None), + max_usd=getattr(a, "max_usd", None), + usd_per_call=a.usd_per_call) + except SpecError as e: + return _spec_error(a, e) # Same gate as plan / MCP start: missing CLI binary must not report started (#346). require_cli_on_path(spec) _warn_unusable(spec) @@ -2716,7 +2733,9 @@ def cmd_evolve(a: argparse.Namespace) -> int: # The child composes again and rewrites `notes`, but only once it is up; # recording them here means `status` has them from the first poll (#414). rd.update_status(notes=comp.notes) - st = runstore.launch(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call) + st = runstore.launch(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call, + max_calls=getattr(a, "max_calls", None), + max_usd=getattr(a, "max_usd", None)) if not a.json: for line in comp.notes: print(f"note: {line}", file=sys.stderr) @@ -2725,7 +2744,9 @@ def cmd_evolve(a: argparse.Namespace) -> int: if a.json else f"{rd.run_id} started (pid {st.pid}) {rd.path}", as_json=a.json) return 0 try: - result = runstore.execute(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call) + result = runstore.execute(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call, + max_calls=getattr(a, "max_calls", None), + max_usd=getattr(a, "max_usd", None)) except Exception as e: # noqa: BLE001 - not BaseException: Ctrl-C stays loud return _run_failed(a, rd, e) st = rd.status() @@ -2790,7 +2811,9 @@ def _run_failed(a: argparse.Namespace, rd: "runstore.RunDir", e: Exception) -> i def cmd_run(a: argparse.Namespace) -> int: """The body of a detached run; not for people.""" rd = runstore.RunDir(a.run_dir) - result = runstore.execute(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call) + result = runstore.execute(rd, budget_usd=a.budget, usd_per_call=a.usd_per_call, + max_calls=getattr(a, "max_calls", None), + max_usd=getattr(a, "max_usd", None)) return 0 if result.error is None else 1 @@ -6175,6 +6198,13 @@ def _register_commands(mount: "_CommandMount", *, with_lab: bool) -> None: s.add_argument("--detach", action="store_true", help="return at once; poll with status") s.add_argument("--budget", type=float, help="stop after this many dollars (needs --usd-per-call)") s.add_argument("--usd-per-call", type=float, dest="usd_per_call") + s.add_argument("--max-calls", type=int, dest="max_calls", default=None, + help=f"stop before the call that would exceed this many model " + f"calls; the run ends 'done' with stop_reason={BUDGET_STOP} " + f"(0 means no calls; exit code 0)") + s.add_argument("--max-usd", type=float, dest="max_usd", default=None, + help="stop before the call that would exceed this many dollars " + "(needs --usd-per-call)") s.set_defaults(fn=cmd_evolve) # The detached worker `evolve --detach` spawns (runstore: `python -m @@ -6188,6 +6218,8 @@ def _register_commands(mount: "_CommandMount", *, with_lab: bool) -> None: s.add_argument("--run-dir", required=True, dest="run_dir") s.add_argument("--budget", type=float) s.add_argument("--usd-per-call", type=float, dest="usd_per_call") + s.add_argument("--max-calls", type=int, dest="max_calls", default=None) + s.add_argument("--max-usd", type=float, dest="max_usd", default=None) s.set_defaults(fn=cmd_run) s = sub.add_parser("status", help="one run, or all of them") diff --git a/agentdescent/evolution.py b/agentdescent/evolution.py index e35d6e9a..1a4cc459 100644 --- a/agentdescent/evolution.py +++ b/agentdescent/evolution.py @@ -60,7 +60,7 @@ from .selection import SingleHead from .scheduler import AuditScheduler from .staleness import StalenessPolicy -from .budget import BudgetGovernor, CallBudget +from .budget import BudgetGovernor, BudgetStop, CallBudget # --------------------------------------------------------------------------- @@ -673,6 +673,8 @@ def _measure() -> float: return value except ContractError: raise # a caller bug: retrying cannot help + except BudgetStop: + raise # a budget refusal: retrying cannot help either except Exception: # noqa: BLE001 - a backend transient if attempt == self.ATTEMPTS - 1: raise @@ -1098,7 +1100,7 @@ def _spec_for(engine: "_Engine", artifact, task: Task, sandbox=engine.sandbox_spec or SandboxSpec()) -def _rollout_failure(outcome) -> BaseException: +def _rollout_failure(outcome, limit=None) -> BaseException: """Turn a reported failure back into the exception the round body expects. The executor reports rather than raises, because one bad rollout is evidence @@ -1106,6 +1108,13 @@ def _rollout_failure(outcome) -> BaseException: is written against exceptions -- and it distinguishes a caller's broken contract, which must stop the run, from a backend transient, which must not. That distinction is carried in `Result.kind`, so it survives the trip.""" + if outcome.kind == "budget" and limit is not None: + # A cap refused the call inside this rollout (#1166). The refusal the + # limit recorded is re-raised as itself, so the round closes `done` + # with a named budget stop and the numbers it observed. + stop = limit.stop + if stop is not None: + return stop if outcome.kind == "caller": return ProposalContractError(outcome.error or "contract violation") return RuntimeError(outcome.error or "rollout failed") @@ -2563,7 +2572,8 @@ def _build_engine(tasks, reward, *, agent, run, propose, strategy, initial_state contract: Optional[Contract] = None, extra_artifacts: Optional[Dict[str, "EvolvingArtifact"]] = None, population_islands: int = 0, - test_frac: float = 0.0) -> _Engine: + test_frac: float = 0.0, + call_limit: Optional[Any] = None) -> _Engine: """Wire the ledger, runtime, verifier and aggregator (shared by :func:`evolve` and :func:`~agentdescent.async_evolve.async_evolve`).""" import tempfile @@ -2692,8 +2702,8 @@ def _build_engine(tasks, reward, *, agent, run, propose, strategy, initial_state if usage is None: usage = getattr(agent, "usage", None) meter = Meter(usage=usage) if usage is not None else Meter() - run, propose = (measured(run, meter, phase="run"), - measured(propose, meter, phase="propose")) + run, propose = (measured(run, meter, phase="run", guard=call_limit), + measured(propose, meter, phase="propose", guard=call_limit)) cache = (policies_bundle.eval_cache if policies_bundle is not None and policies_bundle.eval_cache is not None else MemoryCache(meter)) @@ -3140,6 +3150,10 @@ def _score_seed_held_out(artifact: "EvolvingArtifact", held_out: Sequence) -> Op return float(artifact.score(held_out)) except ContractError: raise + except BudgetStop: + # Not a diagnostic failure: the call was refused by the run-level + # budget, and the caller ends the run with a named stop (#1166). + raise except Exception: # noqa: BLE001 - seed score is diagnostic, not a stop reason return None @@ -3182,6 +3196,13 @@ def evolve( #: Checked at the round barrier alongside the other budgets; the run stops #: with ``stop_reason="max_tokens"`` and reports the spend it incurred. max_tokens: Optional[int] = None, + #: Per-call cap, checked **before** every model call (#1166). Unlike + #: ``max_calls`` above -- a round-barrier budget that overshoots by up to a + #: round -- this refuses the call itself, so ``CallLimit(max_calls=1)`` + #: means one call. The refusal unwinds the round through + #: :class:`~agentdescent.budget.BudgetStop` and the run ends with + #: ``stop_reason="budget"``. Built by the run store; inert when ``None``. + call_limit: Optional["CallLimit"] = None, #: Stop when the run's *measured* return per token has fallen off its own #: peak by more than ``efficiency_floor`` (see #: :meth:`~agentdescent.budget.BudgetGovernor.diminishing_returns`). The @@ -3513,6 +3534,16 @@ def evolve( other budgets; the async path checks per-rollout for tighter control. Stops with ``stop_reason="max_tokens"``. ``None`` (default) means unbounded. + call_limit: + A :class:`~agentdescent.budget.CallLimit` charged **before** each model + call: calls and dollars (at a per-call price) are reserved up front, so + the call that would cross the cap is never made. Different from + ``max_calls`` above, which is a round barrier that lets a wide round + overshoot. The refusal unwinds the round without a partial merge, and + the run stops with ``stop_reason="budget"``; the run store writes which + limit fired and what it observed into ``status.json``. Built by the run + store from ``--max-calls`` / ``--max-usd`` (or a spec's ``evolve`` + knobs); ``None`` (default) is the run exactly as it was. stop_on_diminishing_returns: Stop when the run's own return per token has fallen off its peak by more than ``efficiency_floor``. The economic rule: keep buying compute while @@ -3768,7 +3799,7 @@ def evolve( _budget = resolve_budget( budget, max_seconds=max_seconds, max_rollouts=max_rollouts, max_calls=max_calls, - max_tokens=max_tokens, + max_tokens=max_tokens, call_limit=call_limit, stop_on_diminishing_returns=stop_on_diminishing_returns, efficiency_floor=efficiency_floor, call_budget=call_budget, target_reward=target_reward, patience=patience, @@ -3798,6 +3829,7 @@ def evolve( max_rollouts = _budget.max_rollouts max_calls = _budget.max_calls max_tokens = _budget.max_tokens + call_limit = _budget.call_limit stop_on_diminishing_returns = _budget.stop_on_diminishing_returns efficiency_floor = _budget.efficiency_floor call_budget = _budget.call_budget @@ -3868,6 +3900,13 @@ def evolve( f"evolve(asynchronous=True) ignores {name}=: {why}. Use the " f"synchronous path if you need it.", RuntimeWarning, stacklevel=2) + if call_limit is not None: + # A warning is not enough for a spend cap: an ignored one is an + # unbounded run that looks bounded (#1166). + raise ValueError( + "evolve(asynchronous=True) cannot enforce a run-level call cap: " + "the barrier-free runtime has no per-call guard yet. Run " + "synchronously (asynchronous=False) or drop max_calls / max_usd.") if max_seconds is None: raise ValueError( "evolve(asynchronous=True) no longer invents max_seconds=20.0 " @@ -3987,7 +4026,7 @@ def evolve( allocator=allocator, record_trajectories=record_trajectories, trajectory_store=trajectory_store, contract=contract, extra_artifacts=extra_artifacts, - population_islands=population_islands) + population_islands=population_islands, call_limit=call_limit) # Start the clock after the wiring, before the first unit of work: setup # is not what a time-to-quality number is asking about. eng.meter.start() @@ -4014,7 +4053,17 @@ def evolve( seed_art = EvolvingArtifact( artifact_id, dict(initial_state or strategy.initial()), blast_radius=blast_radius, runtime=eng.runtime, strategy=strategy) - initial_reward = _score_seed_held_out(seed_art, held_out) + #: A run-level budget refusal (#1166). Its own box, for the same reason as + #: `contract_error` below: a worker thread cannot propagate, and it must not + #: be folded into `first_error` -- a cap is a clean stop, not a backend fail. + budget_stop: List[Optional[BudgetStop]] = [None] + try: + initial_reward = _score_seed_held_out(seed_art, held_out) + except BudgetStop as e: + # The seed score is the run's first call: a zero-call cap refuses it + # here, before any round is dispatched (#1166). + budget_stop[0] = e + initial_reward = None straggler_rounds = 0 # Shared with the barrier-free loop: the same two questions, the same # tracker, and now the same epsilon (they had two). @@ -4053,6 +4102,13 @@ def evolve( efficiency_floor=efficiency_floor, allocator=allocator) for r in range(rounds): + # The seed score already spent the cap (e.g. `max_calls=0`): nothing is + # dispatched, and the run closes with the named stop (#1166). + if budget_stop[0] is not None: + stop_reason = "budget" + if verbose: + print(f"round {r:>3} stopping: {budget_stop[0].detail()}") + break if deadline is not None and time.time() >= deadline: stop_reason = "max_seconds" if verbose: @@ -4219,6 +4275,16 @@ def _run_unit(unit) -> None: # main thread by hand -- an exception raised in a plain worker thread # goes to the thread excepthook and is lost, not propagated. contract_error.record(e) # its own lock; first one wins + except BudgetStop as e: + # A run-level cap refused this call (#1166). Not a worker + # failure: it must not touch `first_error` or `failed_units`, or + # the run would end `failed` with an error instead of a named + # budget stop. + with unit_lock: + if budget_stop[0] is None: + budget_stop[0] = e + if verbose: + print(f"round {r:>3} stopping: {e.detail()}") except Exception as e: # noqa: BLE001 - a backend failure with unit_lock: if first_error[0] is None: @@ -4263,7 +4329,7 @@ def _propose_one(unit, task, aid: str) -> None: rendered = mine.render() outcome = eng.executor.rollout(_spec_for(eng, mine, task, rendered)) if not outcome.ok: - raise _rollout_failure(outcome) + raise _rollout_failure(outcome, call_limit) output = outcome.output score = _checked_reward(outcome.reward, task) # Record the rollout before anything else can decide it does not @@ -4320,6 +4386,10 @@ def _propose_one(unit, task, aid: str) -> None: raw_proposal = propose(mine.render(), task, output, score) except ContractError: raise + except BudgetStop: + # A budget refusal is the run's stop, not this worker's bad + # propose (#1166): it goes up to `_run_unit`'s named handler. + raise except Exception as e: # noqa: BLE001 - a reflector/backend failure with unit_lock: propose_raised[0] += 1 @@ -4484,6 +4554,16 @@ def _bounded(u=None, gate=gate, cancelled=cancelled, # read: a broken contract makes the round meaningless. contract_error.raise_if_set() + # A run-level cap made its refusal inside a worker (#1166). The + # round has nothing left to merge, so it closes here: no partial + # merge, no worker-failure tally, and the run ends with the named + # budget stop rather than an error. + if budget_stop[0] is not None: + stop_reason = "budget" + if verbose: + print(f"round {r:>3} stopping: {budget_stop[0].detail()}") + break + # Decide on the round's tally rather than on the first exception. The # same global signal the async path uses: while NO worker has ever # completed a rollout, repeated total failure means the backend is @@ -4509,6 +4589,17 @@ def _bounded(u=None, gate=gate, cancelled=cancelled, reports = check_reports(_step(aggregator, n_workers), aggregator) except ContractError: raise # a caller-contract violation: the run is meaningless + except BudgetStop as e: + # The cap tripped on this thread -- a merge-side or held-out call + # the workers did not make (#1166). Same clean stop as the worker + # path: named reason, no error, no traceback. + with unit_lock: + if budget_stop[0] is None: + budget_stop[0] = e + stop_reason = "budget" + if verbose: + print(f"round {r:>3} stopping: {e.detail()}") + break except Exception as e: # noqa: BLE001 - a rollout backend failure (e.g. an # API/credit error) shouldn't lose the run: stop and return partial results. run_error = _describe(e) @@ -4539,16 +4630,30 @@ def _bounded(u=None, gate=gate, cancelled=cancelled, # Part of the barrier too: the workers for the next round cannot start # until this measurement lands, retries and backoff included. with eng.meter.timed("merge_seconds"), eng.meter.timed("merge_gate_seconds"): - for attempt in range(3): - try: - round_reward = dev.score(held_out) - break - except ContractError: - raise - except Exception as e: # noqa: BLE001 - score_error = e - if attempt < 2: - time.sleep(0.2 * (attempt + 1)) + try: + for attempt in range(3): + try: + round_reward = dev.score(held_out) + break + except ContractError: + raise + except BudgetStop: + # A cap, not a bad measurement: retrying would be refused + # again, and sleeping on it delays the stop (#1166). + raise + except Exception as e: # noqa: BLE001 + score_error = e + if attempt < 2: + time.sleep(0.2 * (attempt + 1)) + except BudgetStop as e: + # The held-out measurement is the round's last spend; the cap + # fired on it, so the run ends here with the named stop. + if budget_stop[0] is None: + budget_stop[0] = e + stop_reason = "budget" + if verbose: + print(f"round {r:>3} stopping: {e.detail()}") + break if round_reward is None: e = score_error if first_error[0] is None: @@ -4638,6 +4743,13 @@ def _bounded(u=None, gate=gate, cancelled=cancelled, final_reward = final.score(held_out) except ContractError: raise + except BudgetStop as e: + # The cap refused the closing measurement (#1166): report the last + # round's reward, which is a measurement the run actually paid for. + if budget_stop[0] is None: + budget_stop[0] = e + stop_reason = "budget" + final_reward = history[-1].held_out_reward if history else 0.0 except Exception as e: # noqa: BLE001 - report, keep the partial result run_error = run_error or _describe(e) final_reward = history[-1].held_out_reward if history else 0.0 @@ -4649,6 +4761,13 @@ def _bounded(u=None, gate=gate, cancelled=cancelled, if eng.test: try: test_reward = float(final.score(eng.test)) + except BudgetStop as e: + # Same cap, on the closing test sweep: it stays a budget stop with + # a missing test score rather than becoming a crash (#1166). + if budget_stop[0] is None: + budget_stop[0] = e + stop_reason = "budget" + test_reward = None except Exception: # noqa: BLE001 - a missing test score is not a crash test_reward = None if run_error: diff --git a/agentdescent/evolve_settings.py b/agentdescent/evolve_settings.py index 502e75a4..55671760 100644 --- a/agentdescent/evolve_settings.py +++ b/agentdescent/evolve_settings.py @@ -56,6 +56,11 @@ #: RuntimeSettings field for this slice (#804): it is additive, defaults to #: ``0.0`` (the old run exactly), and the bag refactor is a separate card. "test_frac", + #: #1166: the per-call budget, passed by ``compose`` / the run store and + #: also a ``BudgetSettings`` field. A flat name because a JSON + #: ``spec.evolve`` knob has to reach the engine by that name; it defaults to + #: ``None``, which is the old run exactly. + "call_limit", # typed bags (#804) — preferred home for new numerics / flags "budget", "runtime", "observability", }) @@ -101,6 +106,11 @@ class BudgetSettings: max_rollouts: Optional[int] = None max_calls: Optional[int] = None max_tokens: Optional[int] = None + #: Per-call cap (:class:`~agentdescent.budget.CallLimit`), checked **before** + #: each model call (#1166). A different quantity from ``max_calls`` above, + #: which is a round-barrier budget and overshoots by up to one round. Built + #: by the run store from ``max_calls`` / ``max_usd``. + call_limit: Optional["CallLimit"] = None stop_on_diminishing_returns: bool = False efficiency_floor: float = 0.25 call_budget: Optional["CallBudget"] = None diff --git a/agentdescent/evolvespec.py b/agentdescent/evolvespec.py index a7bfd01c..37df7919 100644 --- a/agentdescent/evolvespec.py +++ b/agentdescent/evolvespec.py @@ -47,6 +47,7 @@ import csv import difflib import json +import math import os import shlex from dataclasses import dataclass, field, asdict, replace @@ -746,6 +747,9 @@ class Composition: #: Resolved runner layout for directory kinds (``claude_skill``, #: ``skill_library``, …). ``None`` for ``text`` / ``policy_slot``. layout: Optional[str] = None + #: The run-level call cap this composition carries (#1166), so the run store + #: can name the limit that fired without re-reading the spec. + call_limit: Optional[Any] = None def run(self) -> EvolutionResult: return evolve(self.tasks, self.reward, **self.kwargs) @@ -1079,6 +1083,78 @@ def _algorithm_echo(chosen: Tuple[Any, Dict[str, Any]], spec: EvolveSpec, #: field is the place for them. _EVOLVE_OBJECT_KNOBS_HANDLED = frozenset({"agg_config"}) +#: Spec knobs ``compose`` consumes itself into the run-level call cap (#1166): +#: ``max_usd`` is a dollar cap and ``usd_per_call`` its price, and neither is an +#: ``evolve()`` parameter. ``max_calls`` is *not* here -- ``evolve`` already +#: takes it as the round-barrier budget, and the same number is also enforced per +#: call, which can only be stricter. +_COMPOSE_ONLY_KNOBS = frozenset({"max_usd", "usd_per_call"}) + + +def _cap_value(field: str, value: Any, *, integer: bool) -> float: + """One validated cap, refusing anything that is not a non-negative number. + + ``0`` is a limit, not an absence (#1166): it means "no calls" / "no spend". + """ + if isinstance(value, bool) or not isinstance(value, (int, float)): + kind = "integer" if integer else "number" + raise SpecError( + f"{field}: must be a non-negative {kind}, got {value!r}") + if integer and int(value) != value: + raise SpecError( + f"{field}: must be a non-negative integer, got {value!r}") + if not math.isfinite(float(value)) or float(value) < 0: + raise SpecError( + f"{field}: must be a non-negative {'integer' if integer else 'number'}, " + f"got {value!r}") + return int(value) if integer else float(value) + + +def build_call_limit(max_calls: Any = None, max_usd: Any = None, + usd_per_call: Any = None) -> Optional["CallLimit"]: + """A validated run-level call cap, or ``None`` when neither limit is set. + + Shared by ``compose`` (spec knobs) and the run store / CLI (run options), so + the refusal reads the same wherever the number was typed (#1166). + """ + from .budget import CallLimit + + if max_calls is None and max_usd is None: + return None + limit = CallLimit( + max_calls=(None if max_calls is None + else _cap_value("max_calls", max_calls, integer=True)), + max_usd=(None if max_usd is None + else _cap_value("max_usd", max_usd, integer=False)), + usd_per_call=(None if usd_per_call is None + else _cap_value("usd_per_call", usd_per_call, integer=False))) + if limit.max_usd is not None and not limit.usd_per_call: + raise SpecError( + "max_usd needs usd_per_call: a dollar cap cannot be charged without " + "a per-call price (pass --usd-per-call on the command line)") + return limit + + +def _limit_from_knobs(knobs: Dict[str, Any]) -> Optional["CallLimit"]: + """Build the run-level per-call cap a spec asked for, or ``None`` (#1166). + + Pops the keys ``evolve()`` cannot take (``max_usd`` / ``usd_per_call``) and + leaves ``max_calls`` in place, so the barrier budget of the same name still + applies -- a per-call cap of N and a barrier cap of N cannot disagree. + """ + limit = build_call_limit( + max_calls=(None if knobs.get("max_calls") is None + else _cap_value("evolve.max_calls", knobs["max_calls"], integer=True)), + max_usd=(None if knobs.get("max_usd") is None + else _cap_value("evolve.max_usd", knobs["max_usd"], integer=False)), + usd_per_call=(None if knobs.get("usd_per_call") is None + else _cap_value("evolve.usd_per_call", knobs["usd_per_call"], + integer=False))) + knobs.pop("max_usd", None) + knobs.pop("usd_per_call", None) + return limit + + def check_evolve_knobs(evolve_knobs: Mapping[str, Any]) -> None: """Refuse a ``spec.evolve`` entry that ``evolve()`` could not use as given. @@ -1101,6 +1177,11 @@ def check_evolve_knobs(evolve_knobs: Mapping[str, Any]) -> None: params = inspect.signature(evolve).parameters for key, value in evolve_knobs.items(): param = params.get(key) + if param is None and key in _COMPOSE_ONLY_KNOBS: + # Consumed by `compose` into the run-level call cap (#1166); they are + # not evolve() kwargs, so a spec may carry them and they are popped + # before the pass-through. + continue if param is None: raise SpecError( f"evolve.{key}: evolve() has no such parameter; valid JSON knobs are " @@ -1390,6 +1471,12 @@ def run(skill: str, task: Task) -> str: # -- knobs: kind defaults < spec.evolve < caller overrides ------------------ knobs: Dict[str, Any] = {**defaults, **(ov.knobs if ov else {}), **_evolve_pass_through(spec.evolve), **overrides} + # A run-level call cap, from `max_calls` / `max_usd` (+ `usd_per_call`) or + # from a caller's own CallLimit. Validated here, where `plan` also reaches, + # so a negative or non-numeric cap is refused before a run directory exists. + limit = _limit_from_knobs(knobs) + if limit is not None: + knobs.setdefault("call_limit", limit) # kind=text defaults to SeededAppendRules above; a card (gepa/ace/dgm/ # skillopt) or compose(strategy=...) puts "strategy" in knobs and then # overwrites that default. Don't also claim SeededAppendRules (#879). @@ -1479,7 +1566,8 @@ def run(skill: str, task: Task) -> str: return Composition(tasks=tasks, reward=reward, kwargs=kwargs, tree=tree, notes=notes, audit=audit, algorithm=_algorithm_echo(chosen, spec, ov) if chosen else None, - layout=resolved_layout) + layout=resolved_layout, + call_limit=kwargs.get("call_limit")) def run_spec(spec: EvolveSpec, **hooks: Any) -> EvolutionResult: diff --git a/agentdescent/executor.py b/agentdescent/executor.py index 0e1e658f..cf665579 100644 --- a/agentdescent/executor.py +++ b/agentdescent/executor.py @@ -40,6 +40,7 @@ from typing import Any, Callable, Iterator, List, Optional, Protocol, Sequence, runtime_checkable from .agents import ProviderConfigError +from .budget import BudgetStop from .metrics import Meter from .pipeline import describe from .workspec import RolloutSpec @@ -63,7 +64,7 @@ class Result: #: How the failure should be counted. Infrastructure failures must not feed #: the worker-retirement rule, which fires hardest before any worker has #: succeeded -- exactly when sandboxes fail to start. - kind: str = "ok" # "ok" | "model" | "infrastructure" | "caller" + kind: str = "ok" # "ok" | "model" | "infrastructure" | "caller" | "budget" seconds: float = 0.0 @property @@ -139,6 +140,14 @@ def _one(self, spec: RolloutSpec) -> Result: # sample (#635). return Result(spec.lease_id, spec.task.id, error=describe(e), kind="caller", seconds=time.time() - t0) + except BudgetStop as e: + # A run-level call cap refused this call (#1166). Not a rollout + # failure and not raised: `map_rollouts` delivers results through a + # queue, so an exception here would either hang the reader or be + # lost. Its own kind lets the engine re-raise the stop the limit + # already recorded, numbers included. + return Result(spec.lease_id, spec.task.id, error=e.detail(), + kind="budget", seconds=time.time() - t0) except Exception as e: # noqa: BLE001 -- a backend or candidate failure return Result(spec.lease_id, spec.task.id, error=describe(e), kind="model", seconds=time.time() - t0) diff --git a/agentdescent/metrics.py b/agentdescent/metrics.py index 8b58e495..299eac4c 100644 --- a/agentdescent/metrics.py +++ b/agentdescent/metrics.py @@ -387,7 +387,8 @@ def summary(self) -> str: def measured(fn: Callable[..., T], meter: Optional[Meter], - *, phase: Optional[str] = None) -> Callable[..., T]: + *, phase: Optional[str] = None, + guard: Optional[Any] = None) -> Callable[..., T]: """Wrap an actor so each invocation lands in ``meter.usage``. ``agents.metered`` does the same job for a ``Completion`` (``prompt -> str``); @@ -408,11 +409,19 @@ def measured(fn: Callable[..., T], meter: Optional[Meter], Returns ``fn`` unchanged when ``meter`` is ``None``, so the un-instrumented path stays exactly one function call deep. + + ``guard`` (a :class:`~agentdescent.budget.CallLimit`) is asked for a slot + **before** the call, so a run-level budget refuses the call rather than + counting it afterwards (#1166). It raises ``BudgetStop``, which travels out + of here unchanged -- the engine catches it by name and ends the round + cleanly. """ if meter is None: return fn def call(*args: Any, **kwargs: Any) -> T: + if guard is not None: + guard.reserve() token = meter.usage.begin(phase=phase) t0 = time.time() try: diff --git a/agentdescent/runstore.py b/agentdescent/runstore.py index ecb78d89..18889fca 100644 --- a/agentdescent/runstore.py +++ b/agentdescent/runstore.py @@ -175,13 +175,22 @@ class RunStatus: updated: Optional[float] = None finished: Optional[float] = None stop_reason: Optional[str] = None + #: Which limit fired and what was spent, when ``stop_reason == "budget"`` + #: (#1166). Additive and optional: absent means "no run-level cap tripped", + #: and an old ``status.json`` without the key reads as ``None``. + stop_detail: Optional[str] = None error: Optional[str] = None kind: Optional[str] = None target: Optional[str] = None notes: List[str] = field(default_factory=list) def to_dict(self) -> Dict[str, Any]: - return asdict(self) + d = asdict(self) + # Shape stability (#1166): a run with no cap must write the same file it + # wrote before this field existed, so the key is omitted when unset. + if d.get("stop_detail") is None: + d.pop("stop_detail", None) + return d @classmethod def from_dict(cls, d: Dict[str, Any]) -> "RunStatus": @@ -660,6 +669,8 @@ def _error_from_log(log_path: str) -> Optional[str]: def launch(rd: RunDir, *, budget_usd: Optional[float] = None, usd_per_call: Optional[float] = None, + max_calls: Optional[int] = None, + max_usd: Optional[float] = None, env: Optional[Dict[str, str]] = None) -> RunStatus: """Start the run as a **detached** process and return at once. @@ -671,6 +682,12 @@ def launch(rd: RunDir, *, budget_usd: Optional[float] = None, Its stdout and stderr go to ``log.txt``. """ require_budget_pair(budget_usd, usd_per_call) + from .evolvespec import SpecError, build_call_limit + try: + build_call_limit(max_calls=max_calls, max_usd=max_usd, + usd_per_call=usd_per_call) + except SpecError as e: + raise RunStoreError(str(e)) from None st = rd.status() if st.state == "running" and st.alive(): raise RunStoreError(f"{rd.run_id} is already running (pid {st.pid})") @@ -679,6 +696,12 @@ def launch(rd: RunDir, *, budget_usd: Optional[float] = None, argv += ["--budget", str(budget_usd)] if usd_per_call is not None: argv += ["--usd-per-call", str(usd_per_call)] + # The run-level cap crosses the detach boundary as flags, so the child + # charges the same way a foreground run does (#1166). + if max_calls is not None: + argv += ["--max-calls", str(max_calls)] + if max_usd is not None: + argv += ["--max-usd", str(max_usd)] # Scrub the merged dict, not just the ambient one: a caller-supplied # ``env`` is spread last and could otherwise put ``MCP_*``/``DSH_*`` back # into the run after the scrub (#1082). ``PYTHONPATH`` carries no marker @@ -788,7 +811,9 @@ def fmt(message, category, filename, lineno, line=None): # type: ignore[no-unty def execute(rd: RunDir, *, budget_usd: Optional[float] = None, - usd_per_call: Optional[float] = None) -> EvolutionResult: + usd_per_call: Optional[float] = None, + max_calls: Optional[int] = None, + max_usd: Optional[float] = None) -> EvolutionResult: """Run the spec in ``rd`` **in this process**, keeping the directory current. This is what the detached child calls; it is also usable directly for a @@ -796,9 +821,16 @@ def execute(rd: RunDir, *, budget_usd: Optional[float] = None, ``on_round`` hook writes ``status.json`` and ``rounds.jsonl``; the ledger lives under the run so a later :func:`resume` continues it; the result and the materialised tree are written at the end whatever the outcome. + + ``max_calls`` / ``max_usd`` are the run-level caps charged **before** each + model call (#1166). They are stricter than ``budget_usd``, which is a + between-round warning plus a ``stop_when``: a four-worker round on + ``max_calls=1`` still made four calls under that budget, and makes one here. + A capped run ends ``done`` with ``stop_reason="budget"`` and ``stop_detail`` + naming the limit and the spend. """ from .agents import Usage - from .evolvespec import EvolveSpec, compose + from .evolvespec import EvolveSpec, SpecError, build_call_limit, compose from .filetree import materialize # Refuse before marking running: a missing usd_per_call must not look like a @@ -811,6 +843,17 @@ def execute(rd: RunDir, *, budget_usd: Optional[float] = None, spec = EvolveSpec.from_dict(rd.spec_dict()) usage = Usage() + # The run-level cap (#1166). Built here rather than in the engine so a + # detached child and a foreground run charge the same way; a spec's own + # `max_calls` / `max_usd` knobs are validated by `compose`. + try: + limit = build_call_limit(max_calls=max_calls, max_usd=max_usd, + usd_per_call=usd_per_call) + except SpecError as e: + # A bad cap is a refusal, not a live run: record it and never mark + # running, the same shape as a missing usd_per_call above. + rd.update_status(state="failed", finished=time.time(), error=str(e)[:2000]) + raise RunStoreError(str(e)) from None def _progress() -> None: # Mid-call rewrite so CLI/MCP status shows work in flight (#358). @@ -890,6 +933,10 @@ def _store_round(info: RoundInfo, reward: Optional[float]) -> None: rd.update_status(**changes) overrides: Dict[str, Any] = {} + if limit is not None: + # Wins over a spec's own knobs: an explicit run option is the sharper + # instruction, exactly as every other override works (#1166). + overrides["call_limit"] = limit if budget_usd is not None: def stop_when(info: RoundInfo) -> bool: # Asked between rounds, where max_calls is; the run overshoots by at @@ -929,9 +976,13 @@ def stop_when(info: RoundInfo) -> bool: materialize(result.state, rd.tree_path) final = _finite_or_none(result.final_reward) initial = _finite_or_none(getattr(result, "initial_reward", None)) + # Which limit fired, from whichever cap this run actually had: the run + # option, or the one `compose` built from the spec's knobs (#1166). + cap = limit if limit is not None else getattr(comp, "call_limit", None) + detail = cap.detail() if cap is not None else None rd.update_status( state="failed" if result.error else "done", finished=time.time(), - stop_reason=result.stop_reason, error=result.error, + stop_reason=result.stop_reason, stop_detail=detail, error=result.error, best_reward=max(best) if best else final, last_reward=final, initial_reward=initial, test_reward=_finite_or_none(getattr(result, "test_reward", None)), diff --git a/changelog.d/1166-run-call-cap.md b/changelog.d/1166-run-call-cap.md new file mode 100644 index 00000000..01b6a04b --- /dev/null +++ b/changelog.d/1166-run-call-cap.md @@ -0,0 +1,11 @@ +Added +A run-level call cap that holds before the call (`--max-calls`, `--max-usd`, +also `max_calls` / `max_usd` in a spec's `evolve` block) (#1166). Unlike +`max_calls` on `evolve()`, which is a round barrier and lets a four-worker round +overshoot, the guarded cap is charged before each model call: the call that +would cross the limit is not made, the round closes without a partial merge or a +traceback, and the run ends `done` with `stop_reason=budget` and a +`stop_detail` naming the limit and the observed value, in `status.json` and in +`agentdescent status`. `0` is a real limit (no calls); a negative or non-numeric +value is refused at validation with the field and the value named; with neither +flag set the run and `status.json` are unchanged. Offline, no new dependency. diff --git a/docs/api.md b/docs/api.md index 3c199e4a..3213b896 100644 --- a/docs/api.md +++ b/docs/api.md @@ -246,6 +246,7 @@ evolve( max_rollouts: Optional[int] = None, max_calls: Optional[int] = None, max_tokens: Optional[int] = None, + call_limit: Optional['CallLimit'] = None, stop_on_diminishing_returns: bool = False, efficiency_floor: float = 0.25, call_budget: Optional['CallBudget'] = None, @@ -313,6 +314,7 @@ evolve( | `max_rollouts` | `Optional[int]` | `None` | The budget in the two units a comparison has to hold fixed: rollouts completed, and actor invocations (`run` + `propose`). `rounds` is not one of them -- configurations differ in how much model a round buys, so a budget fixed in rounds hands the wider configuration more model and then reports the extra model as a win for parallelism. Either bound stops the run with `stop_reason` `"max_rollouts"` / `"max_calls"`. **Checked at the round barrier, so a run overshoots by up to one round.** A round is dispatched or it is not; stopping halfway would leave a half-merged round, and the states a comparison compares are the ones a merge produced. So a budget is a *bound on where to stop*, never the number to compare on: read the spend the run actually reported (`result.rollouts`, `result.usage.calls`), which is what `baselines` does -- it refuses to call two arms equal-budget when their measured spends differ. The async path has no barrier and enforces both per rollout, so it overshoots by at most the rollouts already in flight. | | `max_calls` | `Optional[int]` | `None` | As `max_rollouts`. | | `max_tokens` | `Optional[int]` | `None` | Hard cap on total tokens consumed (`prompt_tokens + completion_tokens`). A reasoning model can spend 40k tokens on hidden thinking in one call, so `max_calls` and `max_rollouts` do not bound cost: a 20-round run with 4 workers is 80 rollouts and ~160 model calls, but at 40k tokens each that is 6.4M tokens -- the bill, not the count, is what a deployment needs to control. Checked at the round barrier alongside the other budgets; the async path checks per-rollout for tighter control. Stops with `stop_reason="max_tokens"`. `None` (default) means unbounded. | +| `call_limit` | `Optional['CallLimit']` | `None` | A `CallLimit` charged **before** each model call: calls and dollars (at a per-call price) are reserved up front, so the call that would cross the cap is never made. Different from `max_calls` above, which is a round barrier that lets a wide round overshoot. The refusal unwinds the round without a partial merge, and the run stops with `stop_reason="budget"`; the run store writes which limit fired and what it observed into `status.json`. Built by the run store from `--max-calls` / `--max-usd` (or a spec's `evolve` knobs); `None` (default) is the run exactly as it was. | | `stop_on_diminishing_returns` | `bool` | `False` | Stop when the run's own return per token has fallen off its peak by more than `efficiency_floor`. The economic rule: keep buying compute while it pays, stop when it does not -- *even with budget left*. Off by default, because a run whose reward only rises late would be cut short by it, and because it needs `max_tokens` (return per token needs a cost to divide by). Stops with `stop_reason="diminishing_returns"`. | | `efficiency_floor` | `float` | `0.25` | How far below the peak counts as diminishing, when the stop above is on. Self-calibrating against the run's own best rate, so it is a *ratio*, not an absolute quantity: a reward is in `[0, 1]` and a token count is in the millions, and their quotient has no interpretable scale. | | `call_budget` | `Optional['CallBudget']` | `None` | An adaptive per-call thinking budget (o1-style test-time scaling). Pass a `CallBudget` whose `base` matches the adapter's configured `max_tokens`, and the engine adjusts the per-call ceiling based on each parent's score and the remaining budget — promising parents get more thinking, dead-end ones get less, and the allocation tightens as the token budget is spent. The adapter must be wrapped with `budgeted_completion` for this to reach it. `None` (default) means the per-call ceiling is the adapter's own and never changes. | @@ -4878,6 +4880,7 @@ BudgetSettings( max_rollouts: Optional[int] = None, max_calls: Optional[int] = None, max_tokens: Optional[int] = None, + call_limit: Optional['CallLimit'] = None, stop_on_diminishing_returns: bool = False, efficiency_floor: float = 0.25, call_budget: Optional['CallBudget'] = None, diff --git a/docs/start.md b/docs/start.md index 78cc7666..40b0734c 100644 --- a/docs/start.md +++ b/docs/start.md @@ -122,6 +122,20 @@ know a tool-using agent's bill in advance: agentdescent plan spec.json --budget 5 --usd-per-call 0.02 ``` +`--budget` is checked between rounds, so a wide round can overshoot it. For a +cap that holds **per call**, pass `--max-calls` / `--max-usd`: the call that +would cross the limit is not made, the round winds down, and the run ends with +`stop_reason=budget` plus the limit and the spend it observed, in `status.json` +and in `agentdescent status`: + +```bash +agentdescent evolve spec.json --max-calls 40 +agentdescent evolve spec.json --max-usd 5 --usd-per-call 0.02 +``` + +`--max-calls 0` is a real limit — no calls at all — not "unset". Leave both +flags off and the run behaves exactly as before. + The walkthrough that loads a dataset, calls a model, and evolves an instruction is in [Quickstart — dataset to skill](quickstart-skill.md). That path evolves a `SingleSlot` instruction; for a skill library use diff --git a/tests/test_async_early_stop.py b/tests/test_async_early_stop.py index 6a61d2e3..ec5a9b55 100644 --- a/tests/test_async_early_stop.py +++ b/tests/test_async_early_stop.py @@ -61,7 +61,11 @@ def test_no_evolve_argument_is_dropped_without_notice(): # has always called `max_iters`, so it reaches async_evolve as that. remapped = {"tasks", "reward", "rounds", "max_seconds", "asynchronous", "max_rollouts"} - leaked = set(inspect.signature(_evolve).parameters) - forwarded - warned - remapped + # Refused by name: a knob this path rejects with its own ValueError is + # accounted for more strongly than one it warns about (#1166). + refused = set(re.findall(r"if (\w+) is not None:", src)) + leaked = (set(inspect.signature(_evolve).parameters) + - forwarded - warned - refused - remapped) assert not leaked, f"silently dropped by the async path: {sorted(leaked)}" diff --git a/tests/test_budget_guard.py b/tests/test_budget_guard.py new file mode 100644 index 00000000..c2c44a2c --- /dev/null +++ b/tests/test_budget_guard.py @@ -0,0 +1,173 @@ +"""A run-level call / dollar budget stops the run **at the call** (#1166). + +The knobs a run already had are round-barrier budgets: ``max_calls`` is checked +between rounds, so a four-worker round on ``max_calls=1`` still makes four +calls. ``budget_usd`` is a warning plus a between-round ``stop_when``. Neither +is a cap a host can hand to a worker run and trust for one agent session. + +These tests drive the run store end to end, offline: the spec builds its model +from a counting factory in this module, so every model call is one entry in +``CALLS`` and "the second call did not happen" is an assertion, not a hope. +""" + +from __future__ import annotations + +import json +import pathlib + +import pytest + +from agentdescent import runstore +from agentdescent.cli import _status_line +from agentdescent.evolvespec import EvolveSpec, SpecError, compose + +#: One entry per model call the spec's agent actually makes. +CALLS = [] + + +def counting_agent(*, usage=None, tag="agent"): + """A spec-nameable model factory that records each call.""" + + def complete(prompt): + CALLS.append(tag) + return "ok" + + return complete + + +def _spec(**evolve): + model = {"ref": "tests.test_budget_guard:counting_agent", "tag": "agent"} + return EvolveSpec( + kind="text", target="You are a helpful assistant.", + data={"inline": [{"prompt": f"q{i}", "gold": f"a{i}"} for i in range(8)]}, + agent=model, reflect=model, allow=["tests."], + evolve={"rounds": 4, "n_workers": 4, "seed": 0, "patience": 10, **evolve}) + + +def _run(tmp_path, *, spec=None, **limits): + CALLS.clear() + rd = runstore.create((spec or _spec()).to_dict(), store=str(tmp_path / "runs")) + result = runstore.execute(rd, **limits) + return rd, rd.status(), result + + +def test_max_calls_stops_before_the_next_call(tmp_path): + rd, st, result = _run(tmp_path, max_calls=1) + assert CALLS == ["agent"], f"a capped run made {len(CALLS)} calls" + assert st.calls == 1 + assert st.state == "done", st + assert st.error is None + assert st.stop_reason == "budget", st + assert "max_calls" in (st.stop_detail or ""), st.stop_detail + assert result.stop_reason == "budget" + + +def test_max_usd_stops_before_the_call_that_would_overspend(tmp_path): + # One call costs 0.01; the second would reach 0.02, over the 0.015 cap. + rd, st, result = _run(tmp_path, max_usd=0.015, usd_per_call=0.01) + assert CALLS == ["agent"], f"an over-budget run made {len(CALLS)} calls" + assert st.state == "done", st + assert st.error is None + assert st.stop_reason == "budget", st + assert "max_usd" in (st.stop_detail or ""), st.stop_detail + assert st.usd == pytest.approx(0.01) + + +def test_the_status_line_names_the_limit_that_fired(tmp_path): + rd, st, _ = _run(tmp_path, max_calls=1) + line = _status_line(st) + assert "budget" in line, line + assert "max_calls" in line, line + + +def test_unset_limits_keep_todays_run(tmp_path): + rd, st, result = _run(tmp_path) + assert st.state == "done", st + assert len(CALLS) > 1, "the uncapped run should have made more than one call" + assert st.stop_reason != "budget", st + assert not st.stop_detail + + +def test_a_negative_max_calls_is_refused_with_the_field_named(): + with pytest.raises(SpecError) as caught: + compose(_spec(max_calls=-1)) + message = str(caught.value) + assert "max_calls" in message and "-1" in message, message + + +def test_a_non_numeric_max_usd_is_refused_with_the_field_named(): + with pytest.raises(SpecError) as caught: + compose(_spec(max_usd="cheap")) + message = str(caught.value) + assert "max_usd" in message and "cheap" in message, message + + +def test_zero_is_a_limit_not_an_absence(tmp_path): + """``0`` means "do not spend": refused before the first call, not ignored.""" + rd, st, _ = _run(tmp_path, max_calls=0) + assert CALLS == [], f"a zero-call budget made {len(CALLS)} calls" + assert st.stop_reason == "budget", st + assert "max_calls" in (st.stop_detail or ""), st.stop_detail + + +def test_status_json_keeps_its_shape_when_no_cap_is_set(tmp_path): + """No cap, no new key: a reader of yesterday's file sees yesterday's file.""" + rd, st, _ = _run(tmp_path) + payload = json.loads((pathlib.Path(rd.path) / "status.json").read_text()) + assert "stop_detail" not in payload, payload + + +def test_status_json_carries_the_limit_that_fired(tmp_path): + rd, st, _ = _run(tmp_path, max_calls=1) + payload = json.loads((pathlib.Path(rd.path) / "status.json").read_text()) + assert payload["stop_reason"] == "budget", payload + assert "max_calls" in payload["stop_detail"], payload + + +def test_a_spec_level_dollar_cap_runs(tmp_path): + """``max_usd`` in a spec's ``evolve`` block is usable, not a ``TypeError``. + + ``usd_per_call`` is compose-only like ``max_usd``: leaving it in the knob + dict handed it to ``evolve()``, which has no such parameter, so every valid + dollar cap died in round setup while the count cap worked. + """ + spec = _spec(max_usd=0.015, usd_per_call=0.01) + comp = compose(spec) + assert "usd_per_call" not in comp.kwargs, comp.kwargs + rd, st, result = _run(tmp_path, spec=spec) + assert CALLS == ["agent"], CALLS + assert st.state == "done", st + assert st.error is None + assert st.stop_reason == "budget", st + assert "max_usd" in (st.stop_detail or ""), st.stop_detail + + +def test_a_cap_that_trips_inside_a_rollout_ends_done(tmp_path): + """The refusal reaches the engine through a worker's rollout (#1166). + + ``max_calls=2`` under four workers is refused while a rollout is in flight, + not at the seed score. The executor reports that refusal like any other + rollout outcome, and the run must still end ``done`` with the named budget + stop rather than ``failed`` on a ``RuntimeError``. + """ + rd, st, result = _run(tmp_path, max_calls=2) + assert st.calls == 2, f"a capped run made {st.calls} calls" + assert st.state == "done", st + assert st.error is None, st.error + assert st.stop_reason == "budget", st + assert "max_calls" in (st.stop_detail or ""), st.stop_detail + + +def test_a_dollar_cap_that_trips_inside_a_rollout_ends_done(tmp_path): + """The dollar cap, refused on the concurrent segment, not the serial one. + + ``0.05`` at ``0.02``/call admits two calls and refuses the third while four + workers are in flight; ``0.05`` at ``0.01`` would admit five, and that + refusal lands in a serial propose, so it cannot tell the two paths apart. + """ + rd, st, result = _run(tmp_path, max_usd=0.05, usd_per_call=0.02) + assert st.calls == 2, f"a capped run made {st.calls} calls" + assert st.state == "done", st + assert st.error is None, st.error + assert st.stop_reason == "budget", st + assert "max_usd" in (st.stop_detail or ""), st.stop_detail