feat(MAPCO-11435): spend ceilings as first-class outcomes

src/budget/attempt.ts

@@ -0,0 +1,83 @@
+import { LABELS } from '@common/constants';
+
+/**
+ * Counting an overspend as an attempt (MAPCO-11435).
+ *
+ * The attempt counter has no store but Jira labels: a ticket carries `agent-attempted-N` and
+ * the poll query excludes the one *at* the cap. That makes bumping it the only thing standing
+ * between an overspent ticket and being picked up next cycle and burnt for the same budget
+ * again — one ticket able to eat the whole daily allowance, which is the runaway the ceilings
+ * exist to bound.
+ *
+ * What lives here is the *policy*: given a ticket's labels, which labels count one more attempt.
+ * What does not live here is the write — that is `JiraPort.setLabels`, and the one place the two
+ * are put together is `handBackTicket` (src/tickets/handBack.ts), which counts before it releases
+ * so a ticket cannot become pollable while still uncounted. Keeping the rule pure is what lets
+ * every implementation of `AbortPort` and `ReleasePort` count the same way.
+ */
+
+/** A counter label's suffix. Anchored and digits-only, so `agent-attempted-soon` is not one. */
+const COUNTER_SUFFIX = /^\d+$/u;
+
+/**
+ * The attempt this label records, or null if it is not one of the worker's counters.
+ *
+ * Anything under the prefix that is not a plain number is somebody else's label and is left
+ * alone rather than being read as a count of zero and quietly deleted.
+ */
+function counterValue(label: string): number | null {
+  if (!label.startsWith(LABELS.attemptedPrefix)) {
+    return null;
+  }
+
+  const suffix = label.slice(LABELS.attemptedPrefix.length);
+
+  return COUNTER_SUFFIX.test(suffix) ? Number(suffix) : null;
+}
+
+/**
+ * How many attempts a ticket's labels already record. Zero when it carries no counter.
+ *
+ * Takes the highest rather than the count of counter labels: a ticket that somehow carries both
+ * `agent-attempted-1` and `agent-attempted-2` has two attempts behind it, not three, and reading
+ * it as three is how a ticket gets pushed past the cap.
+ */
+function attemptsSoFar(labels: readonly string[]): number {
+  let highest = 0;
+
+  for (const label of labels) {
+    const counted = counterValue(label);
+
+    if (counted !== null && counted > highest) {
+      highest = counted;
+    }
+  }
+
+  return highest;
+}
+
+/**
+ * The labels a ticket should carry once this attempt is counted against it.
+ *
+ * Replaces the old counter rather than adding to it, so a ticket carries exactly one, and
+ * **clamps at the cap**, which is the non-obvious half. The poll query excludes the label
+ * *exactly* at `attemptCap` (`labels not in ("agent-attempted-2")`) — so writing
+ * `agent-attempted-3` on a ticket already at a cap of 2 would not tighten anything, it would
+ * make the ticket poll-visible again and hand it straight back to the worker forever. A ticket
+ * at the cap is already invisible to the query, so leaving its counter where it is loses
+ * nothing.
+ *
+ * Every label that is not one of the worker's counters is preserved untouched, `agent-ready`
+ * included: enrolment is a human's decision and an overspend is not a reason to revoke it.
+ */
+function countAttempt(labels: readonly string[], attemptCap: number): readonly string[] {
+  // Never below 1: a cap of zero is not a thing the query can express, and an
+  // `agent-attempted-0` label would be a counter nothing ever excludes.
+  const counted = Math.max(Math.min(attemptsSoFar(labels) + 1, attemptCap), 1);
+  const next = `${LABELS.attemptedPrefix}${counted}`;
+  const kept = labels.filter((label) => counterValue(label) === null);
+
+  return [...kept, next];
+}
+
+export { attemptsSoFar, countAttempt };

src/budget/dailyCap.ts

@@ -0,0 +1,67 @@
+import type { Clock, DailyCap, DailyCapState } from './types';
+
+const MS_PER_DAY = 86_400_000;
+
+/** The UTC day a moment falls in, as a whole number of days since the epoch. */
+function dayOf(millis: number): number {
+  return Math.floor(millis / MS_PER_DAY);
+}
+
+/**
+ * The per-day started-ticket counter (MAPCO-11435).
+ *
+ * ## The counter lives in memory, and that is a limitation, not an oversight
+ *
+ * Jira is the sole source of truth for this service: there is no database and nothing on disk
+ * outlives a run (MAPCO-11430). A counter is not a thing Jira can hold — a global daily total
+ * would mean a read-modify-write across every ticket the worker has ever touched — so it is a
+ * variable in a long-lived process instead. That is coherent, because the process outlives a
+ * cycle by design (MAPCO-11430 chose a Deployment over a CronJob for exactly this kind of
+ * reason), but it means the cap is **per process-day, not per calendar day globally**: a
+ * redeploy, a crash or an eviction resets it, so a pod that restarts three times in a morning
+ * can start three times the cap. Read it as a brake on a healthy process, not as a billing
+ * guarantee. A genuinely global cap needs a store, and inventing one here is out of scope.
+ *
+ * Day boundaries are UTC and are evaluated lazily: nothing schedules a reset, the window rolls
+ * forward the next time the counter is asked anything. An idle worker therefore holds no timer,
+ * and no count can change underneath a cycle that is already running.
+ *
+ * Asking (`state`) and counting (`recordStart`) are separate calls, and what is counted is
+ * tickets the worker *held*, not tickets it looked at. A claim lost to a human spends nothing,
+ * and burning a day's allowance on races would idle the worker for the rest of the day. The
+ * cost is that two tickets can read the same last slot as free when `maxConcurrentTickets` is
+ * above 1, so a day can end a ticket or two over the cap; the counter reports the real number
+ * rather than clamping it, and both knobs default to 1.
+ */
+function createDailyCap(limit: number, clock: Clock = Date.now): DailyCap {
+  let day = dayOf(clock());
+  let started = 0;
+
+  /** Roll the window forward if the clock has crossed midnight since the last question. */
+  const rollover = (): void => {
+    const today = dayOf(clock());
+
+    if (today !== day) {
+      day = today;
+      started = 0;
+    }
+  };
+
+  const state = (): DailyCapState => {
+    rollover();
+
+    return { startedToday: started, limit, exhausted: started >= limit };
+  };
+
+  return {
+    recordStart: (): void => {
+      // Records something that already happened, so it never refuses. Guarding here would
+      // hide the overshoot described above instead of reporting it.
+      rollover();
+      started += 1;
+    },
+    state,
+  };
+}
+
+export { createDailyCap, dayOf };

src/budget/enforce.ts

@@ -0,0 +1,155 @@
+import type { Logger } from '@map-colonies/js-logger';
+import type { JiraTicket } from '../jira/types';
+import type { ClaimOutcome } from '../tickets/claim';
+import { describeOverspend } from './report';
+import type { AbortPort, AttemptSummary, DailyCap, DailyCapState, Overspend, Spend, TicketLedger } from './types';
+
+interface BudgetDeps {
+  readonly ledger: TicketLedger;
+  readonly abort: AbortPort;
+  readonly logger: Logger;
+}
+
+interface DailyCapDeps {
+  readonly cap: DailyCap;
+  readonly logger: Logger;
+}
+
+/**
+ * Whether the ticket may take another turn, and — when it may not — what the abort path
+ * actually managed.
+ *
+ * `released: false` means the ticket is still assigned to the bot and still In Progress. That
+ * is the same containment `releaseTicket` chooses on a stuck workflow: the poll query skips a
+ * ticket held by the bot, and the boot-time orphan sweep (MAPCO-11432) is what recovers it. A
+ * caller that sees it must stop working the ticket and must not release it a second time.
+ *
+ * `alreadyStopped: true` is the answer to a charge made after the budget had already run out.
+ * Nothing was written for it — the ticket was handed back by the charge that first refused —
+ * and there is nothing for the caller to do beyond stop, which it should already have done.
+ */
+type ChargeOutcome =
+  | { readonly ok: true; readonly spend: Spend }
+  | {
+      readonly ok: false;
+      readonly alreadyStopped: false;
+      readonly overspend: Overspend;
+      readonly released: boolean;
+      /** Whether the hand-back counted the attempt. See `AbortResult.attemptCounted`. */
+      readonly attemptCounted: boolean;
+    }
+  | { readonly ok: false; readonly alreadyStopped: true; readonly overspend: Overspend };
+
+/** Whether the day had room for this ticket, and what the claim said if it did. */
+type StartAttempt =
+  | { readonly ok: true; readonly claim: ClaimOutcome; readonly state: DailyCapState }
+  | { readonly ok: false; readonly reason: 'daily-cap'; readonly state: DailyCapState };
+
+/**
+ * Charge spending to a ticket and, if that used the budget up, abort the ticket (MAPCO-11435).
+ *
+ * `spent` carries both halves because the caller is the only thing that knows them. A caller
+ * handed one usage figure for a whole hand-off charges that hand-off's real turn count, not `1`
+ * — see `TicketLedger.charge` for why the difference is a factor of `MAX_TURNS_PER_TICKET`.
+ *
+ * Going over is an outcome, not an exception: this never rejects, whatever the abort path does.
+ * A ticket that ran out of money is an ordinary day for the worker, and a throw here would
+ * surface as `ticket failed` in the run — indistinguishable from a bug, and one line of log
+ * instead of a comment on the ticket that spent the money.
+ *
+ * The hand-back happens at most once per ticket, guaranteed by the ledger's latch rather than
+ * by asking callers to stop: a second call after the budget has gone charges the money and
+ * writes nothing.
+ */
+async function chargeSpend(deps: BudgetDeps, ticket: JiraTicket, spent: Spend, attempt: AttemptSummary): Promise<ChargeOutcome> {
+  const { ledger, abort, logger } = deps;
+  const check = ledger.charge(spent);
+
+  if (check.ok) {
+    return { ok: true, spend: check.spend };
+  }
+
+  const overspend: Overspend = { kind: check.kind, limit: check.limit, spend: check.spend, attempt };
+
+  if (check.alreadyStopped) {
+    // Worth a line, because it means a caller kept spending after being told to stop, but not
+    // worth a second comment on a ticket somebody else may already have picked up.
+    logger.warn({
+      msg: 'charged spending to a ticket that had already run out of budget',
+      key: ticket.key,
+      tokensSpent: overspend.spend.tokens,
+      turnsSpent: overspend.spend.turns,
+    });
+
+    return { ok: false, alreadyStopped: true, overspend };
+  }
+
+  logger.warn({
+    msg: 'budget exhausted',
+    key: ticket.key,
+    kind: overspend.kind,
+    limit: overspend.limit,
+    tokensSpent: overspend.spend.tokens,
+    turnsSpent: overspend.spend.turns,
+  });
+
+  try {
+    const result = await abort.abort(ticket, describeOverspend(overspend));
+
+    // A hand-back that returned but did not finish the job is the quiet failure mode: an
+    // unreleased ticket sits held until the orphan sweep, and an *uncounted* one goes straight
+    // back into the poll query and spends the same budget again next cycle. Neither shows up
+    // anywhere else, so it goes in the log next to what it cost.
+    if (!result.released || !result.attemptCounted) {
+      logger.warn({
+        msg: 'overspent ticket was not fully handed back',
+        key: ticket.key,
+        released: result.released,
+        attemptCounted: result.attemptCounted,
+      });
+    }
+
+    return { ok: false, alreadyStopped: false, overspend, released: result.released, attemptCounted: result.attemptCounted };
+  } catch (error) {
+    logger.error({ msg: 'overspent ticket could not be handed back', key: ticket.key, err: error });
+
+    return { ok: false, alreadyStopped: false, overspend, released: false, attemptCounted: false };
+  }
+}
+
+/**
+ * Start a ticket only if the day still has room for one (MAPCO-11435).
+ *
+ * The claim arrives as a callback rather than being done here, for two reasons. A run that has
+ * hit the cap must write nothing at all: a ticket claimed and then dropped for a spend ceiling
+ * has already put a bot's name on a human's ticket and notified its watchers, and "poll and
+ * start nothing" has to mean nothing. And the count must not drift apart from the claim it
+ * counts, which it would the moment a caller could do one without the other.
+ *
+ * The state returned is read *after* the claim, so it already includes the ticket just started
+ * and can go straight into the run line.
+ */
+async function startWithinDailyCap(deps: DailyCapDeps, ticket: JiraTicket, claim: () => Promise<ClaimOutcome>): Promise<StartAttempt> {
+  const { cap, logger } = deps;
+  const before = cap.state();
+
+  if (before.exhausted) {
+    // Not a warning: this is the ceiling doing its job, and the run line says so too.
+    logger.info({ msg: 'daily cap reached, starting nothing', key: ticket.key, startedToday: before.startedToday, limit: before.limit });
+
+    return { ok: false, reason: 'daily-cap', state: before };
+  }
+
+  const outcome = await claim();
+
+  // Counted only once the worker actually holds the ticket — a claim lost to a human spends
+  // nothing, and paying a day's allowance for a race would idle the worker until midnight.
+  if (outcome.ok) {
+    cap.recordStart();
+  }
+
+  return { ok: true, claim: outcome, state: cap.state() };
+}
+
+export { chargeSpend, startWithinDailyCap };
+export type { BudgetDeps, ChargeOutcome, DailyCapDeps, StartAttempt };