stewards: shadow assessment module with a divergence log (BB-293)

plugins/stewards/DESIGN.md

@@ -658,6 +658,31 @@ At the project file's time, 01:00 US Eastern by default:
 All of it must finish before 03:00, when bb-gate's quiet window restarts
 servers to apply staged updates and would kill a running turn.
 
+### The shadow assessment (BB-293)
+
+BB-291's direction is one assessment per binding: the merge gate, the worker
+state and the wait liveness answered in one place instead of by the several
+predicates that disagree today. This step adds only the derivation and the
+evidence it is safe, in shadow:
+
+- `assessment.ts` is pure. `assess` reads the rows a reconcile already has
+  in hand — no extra GitHub or Linear call — and derives the gate (with the
+decide-list mark and the recorded project approvals the `approvals.ts` path
+omits), the worker state (an explicit `parked`, distinct from `running`) and
+the liveness (wake source, idle, stall from the observable rows, not the
+report-time `noWakeSource` flag). `compare` runs the predicates the
+assessment will replace and returns every disagreement.
+- `reconcile.ts` calls it on every binding inside a try/catch that can never
+  change the reconcile's own result, stores the latest assessment per binding
+  in `assessments`, and logs each distinct divergence to
+  `assessment_divergences` (one row per binding/surface/old/new, with a first
+  and last seen time and a count). The log is bounded by an age and a row
+  count, pruned on each reconcile.
+- `bb stewards divergences [--binding] [--surface] [--json]` prints the log,
+  read-only. Nothing reads the assessment to decide or render anything: a
+  consumer is added only after the comparison has run clean in production for
+  several days. The log's `inputs` hold only ids, shas, states and timestamps.
+
 ## 7. The ready gate
 
 No ticket reaches implementation until it is fully specified, and no

plugins/stewards/assessment.ts

@@ -0,0 +1,275 @@
+// BB-293 (child 2 of BB-291): the shadow assessment. One derived answer per
+// binding for the merge gate (with the decide-list mark and the recorded
+// project approvals the approvals.ts path omits), the worker state (an
+// explicit `parked`, distinct from `running`) and the wait liveness (wake
+// source, idle and stall derived from what the plugin observes, not the
+// report-time `noWakeSource` flag).
+//
+// It is pure: it reads only the rows a reconcile already has in hand and
+// decides nothing. `compare` runs the predicates the assessment will replace
+// over the same rows and returns every place the old answer and the assessment
+// disagree, so the divergence log can be read for several days before any
+// consumer reads the assessment.
+import { prGate, type ProjectApprovalRecord } from "./gate.js";
+import { gateState } from "./ready-gate.js";
+import { anyWorkerInFlight, idleInActiveStage, observedStall, productGateOptions, runningWorkerReason, stuckDue, type IdleBinding } from "./reconcile.js";
+import { hasLiveWakeSource, type WakeSourceInput } from "./wake-source.js";
+import type { Playbook } from "./config.js";
+import type { EntitySnapshot } from "./source-contract.js";
+import type { Binding, Escalation, Need, Observed, Worker } from "./store.js";
+
+export type DecideMark = "project" | "user";
+
+/** How long a divergence row is kept before the reconcile prunes it. */
+export const ASSESSMENT_DIVERGENCE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+/** The most divergence rows kept, newest first; the reconcile prunes the rest. */
+export const ASSESSMENT_DIVERGENCE_MAX_ROWS = 500;
+
+/** The merge gate for one own pull request, derived with every input the plugin has. */
+export type AssessedGate = {
+  /** The pull request's key (owner/repo#N), an id, never free text. */
+  pull: string;
+  met: boolean;
+  lane: string | null;
+  missing: string | null;
+  decision: DecideMark;
+  ownerApproved: boolean;
+  projectApproved: boolean;
+  checksGreen: boolean;
+  escalated: boolean;
+  reason: "product" | null;
+  needsOwnerApproval: boolean;
+  needsProjectApproval: boolean;
+};
+
+/**
+ * The binding's worker state. `parked` is a `finished` worker between turns:
+ * still in flight, not running, and the state the boolean predicates collapse
+ * to either "busy" (`workerInFlight`) or "not mid-turn" (`runningWorkerReason`).
+ */
+export type AssessedWorker = {
+  state: "running" | "parked" | "failed" | "stopped" | "none";
+  inFlight: boolean;
+  running: boolean;
+  parked: boolean;
+  /** `runningWorkerReason`'s text for a mid-turn worker, else null. */
+  reason: string | null;
+};
+
+export type AssessedLiveness = {
+  /** `hasLiveWakeSource` over the observable rows. */
+  live: boolean;
+  /** `idleInActiveStage`. */
+  idle: boolean;
+  /** `stuckDue`: the phase backstop. */
+  stuck: boolean;
+  /** A free-text wait with no tracked source and no dated wait ahead, recomputed from the rows. */
+  stalled: boolean;
+  /** The report-time flag the assessment replaces. */
+  reportedNoWakeSource: boolean;
+};
+
+export type Assessment = {
+  bindingId: string;
+  entityKey: string;
+  gate: {
+    pulls: AssessedGate[];
+    /** Every own pull's gate is met; true when the binding has none. */
+    met: boolean;
+    decision: DecideMark;
+    missing: string | null;
+    /** The ticket ready gate's pass, or null when the playbook has none. */
+    readyPassed: boolean | null;
+  };
+  worker: AssessedWorker;
+  liveness: AssessedLiveness;
+};
+
+/**
+ * Everything the assessment reads. All of it is already in hand when the
+ * reconcile reaches the assessment, so no tick costs an extra GitHub or Linear
+ * call.
+ */
+export type AssessmentInput = {
+  binding: Pick<
+    Binding,
+    "id" | "entityKey" | "kind" | "ready" | "stage" | "ending" | "hold" | "deployments" | "requeueUntil" | "noWakeSource" | "status" | "createdAt" | "advancedAt" | "phase"
+  >;
+  /** The observation the reconcile just read. */
+  observed: Observed;
+  /** The binding's own pull keys and URLs, lowercased (BB-52 rule 0). */
+  ownPulls: ReadonlySet<string>;
+  /** Own pull keys whose gate is met (BB-285). */
+  gateMetPulls: ReadonlySet<string>;
+  /** The decide-list mark for each own pull, from the files the snapshot already carries. */
+  decisions: ReadonlyMap<string, DecideMark>;
+  /** The plugin's recorded project approvals for the binding. */
+  projectApprovals: readonly ProjectApprovalRecord[];
+  workers: readonly Worker[];
+  escalations: readonly Escalation[];
+  needs: readonly Need[];
+  /** The wake sources the report path already assembles (BB-154). */
+  sources: WakeSourceInput;
+  playbook: Playbook;
+  /** The spec hash the ready gate compares its approval against. */
+  currentSpecHash: string;
+  now: number;
+};
+
+const lc = (s: string | null | undefined): string => (s ?? "").toLowerCase();
+
+/** Whether an observed entity is one of the binding's own pull requests (BB-52 rule 0). */
+const isOwn = (e: EntitySnapshot, own: ReadonlySet<string>): boolean =>
+  (e.key !== "" && own.has(lc(e.key))) || (e.url !== null && e.url !== "" && own.has(lc(e.url)));
+
+/** Every own pull request the binding's workers opened, the binding's own entity first. */
+export function ownPullEntities(observed: Observed, own: ReadonlySet<string>): EntitySnapshot[] {
+  return [observed.entity, ...observed.pulls].filter((e) => e.pull !== null && isOwn(e, own));
+}
+
+/** The worker state, with a parked `finished` worker distinct from a running one. */
+export function assessWorker(workers: readonly Worker[]): AssessedWorker {
+  const running = workers.some((w) => w.status === "running");
+  const parked = !running && workers.some((w) => w.status === "finished");
+  const failed = !running && !parked && workers.some((w) => w.status === "failed");
+  const state: AssessedWorker["state"] = running ? "running" : parked ? "parked" : workers.length === 0 ? "none" : failed ? "failed" : "stopped";
+  return { state, inFlight: anyWorkerInFlight(workers), running, parked, reason: runningWorkerReason(workers) };
+}
+
+/** Derive the one assessment for a binding from the rows the reconcile already read. */
+export function assess(input: AssessmentInput): Assessment {
+  const { binding, observed, ownPulls, decisions, projectApprovals, workers, escalations, needs, sources, playbook, currentSpecHash, now } = input;
+  const pulls: AssessedGate[] = [];
+  for (const e of ownPullEntities(observed, ownPulls)) {
+    const g = prGate(e, { decision: decisions.get(lc(e.key)) ?? "user", projectApprovals, ...productGateOptions(binding, e) });
+    pulls.push({
+      pull: e.key,
+      met: g.gateMet,
+      lane: g.lane,
+      missing: g.missing,
+      decision: g.decision,
+      ownerApproved: g.ownerApproved,
+      projectApproved: g.projectApproved,
+      checksGreen: g.checksGreen,
+      escalated: g.escalated,
+      reason: g.reason,
+      needsOwnerApproval: g.needsOwnerApproval,
+      needsProjectApproval: g.needsProjectApproval,
+    });
+  }
+  const idleBinding: IdleBinding = {
+    stage: binding.stage,
+    ending: binding.ending,
+    hold: binding.hold,
+    deployments: binding.deployments,
+    requeueUntil: binding.requeueUntil,
+    workers,
+    ownPulls,
+    gateMetPulls: input.gateMetPulls,
+    escalations,
+    needs,
+  };
+  const live = hasLiveWakeSource(sources);
+  return {
+    bindingId: binding.id,
+    entityKey: binding.entityKey,
+    gate: {
+      pulls,
+      met: pulls.every((p) => p.met),
+      decision: pulls[0]?.decision ?? "user",
+      missing: pulls.find((p) => !p.met)?.missing ?? null,
+      readyPassed: gateState(playbook, binding.ready, currentSpecHash).passed,
+    },
+    worker: assessWorker(workers),
+    liveness: {
+      live,
+      idle: idleInActiveStage(idleBinding, observed, now),
+      stuck: stuckDue(binding, playbook, [...escalations], now) !== null,
+      stalled: observedStall(binding, live, now),
+      reportedNoWakeSource: binding.noWakeSource,
+    },
+  };
+}
+
+/**
+ * One place the old predicate and the shadow assessment disagree. `inputs`
+ * carries only ids, shas, states and timestamps: never a comment body or any
+ * free text, so the log cannot leak a transcript.
+ */
+export type Divergence = {
+  bindingId: string;
+  surface: string;
+  old: string;
+  new: string;
+  inputs: Record<string, string | number | boolean | null>;
+};
+
+const verdict = (met: boolean, missing: string | null): string => (met ? "met" : (missing ?? "unmet"));
+
+/**
+ * Every place the predicates the assessment will retire answer differently
+ * from it, over the same rows. The surfaces:
+ *
+ * - `workerInFlight`, `runningWorkerReason`, `idleInActiveStage`,
+ *   `hasLiveWakeSource`: a parked `finished` worker is in flight to the first
+ *   and not mid-turn, idle and not a live source to the others (BB-292 #1).
+ * - `approvals.gate`: `approvals.ts` builds `prGate` without the decide mark
+ *   or the recorded project approvals (BB-292 #2).
+ * - `approvals.pageRefusal`: the page action's refusal falls through to its
+ *   `??` fallback on a met gate and says "not observed yet" (BB-292 #3).
+ * - `noWakeSource`: the report-time flag against the stall recomputed from
+ *   the observable rows.
+ */
+export function compare(input: AssessmentInput, assessment: Assessment): Divergence[] {
+  const { binding, observed, ownPulls, projectApprovals, workers } = input;
+  const out: Divergence[] = [];
+  const add = (surface: string, old: string, next: string, inputs: Divergence["inputs"]): void => {
+    if (old !== next) out.push({ bindingId: binding.id, surface, old, new: next, inputs });
+  };
+  const workerInputs: Divergence["inputs"] = { binding: binding.id, workers: workers.length, states: workers.map((w) => w.status).join(",") };
+
+  // (1) A parked worker: the boolean predicates disagree about it; the
+  // assessment names the state `parked`.
+  if (assessment.worker.parked) {
+    if (anyWorkerInFlight(workers)) add("workerInFlight", "in-flight", "parked", workerInputs);
+    if (runningWorkerReason(workers) === null) add("runningWorkerReason", "not-mid-turn", "parked", workerInputs);
+    if (assessment.liveness.idle) add("idleInActiveStage", "idle", "parked", workerInputs);
+    if (!assessment.liveness.live) add("hasLiveWakeSource", "no-live-source", "parked", workerInputs);
+  }
+
+  // (2) The gate inputs, and (3) the page's refusal text.
+  for (const e of ownPullEntities(observed, ownPulls)) {
+    const assessed = assessment.gate.pulls.find((p) => lc(p.pull) === lc(e.key));
+    if (assessed === undefined) continue;
+    const oldGate = prGate(e, productGateOptions(binding, e));
+    const gateInputs: Divergence["inputs"] = {
+      binding: binding.id,
+      pull: e.key,
+      headSha: e.pull?.headSha ?? null,
+      decision: assessed.decision,
+      recordedApprovals: projectApprovals.length,
+    };
+    if (oldGate.gateMet !== assessed.met) add("approvals.gate", verdict(oldGate.gateMet, oldGate.missing), verdict(assessed.met, assessed.missing), gateInputs);
+    // The page acts only while `needsOwnerApproval` is true; on a met gate it
+    // falls to the `??` fallback, whose words this records.
+    const pageRefusal = oldGate.needsOwnerApproval ? null : (oldGate.missing ?? "the pull request has not been observed yet");
+    if (pageRefusal !== null && assessed.met) add("approvals.pageRefusal", pageRefusal, "met", gateInputs);
+  }
+
+  // (4) The report-time no-wake-source flag against the stall recomputed from
+  // the rows. `hasLiveWakeSource`, `idleInActiveStage` and `stuckDue` are the
+  // assessment's own derivation, so they are compared by construction; only
+  // the flag and the observed stall can disagree.
+  add("noWakeSource", binding.noWakeSource ? "no-wake-source" : "source-or-no-wait", assessment.liveness.stalled ? "no-wake-source" : "source-or-no-wait", {
+    binding: binding.id,
+    live: assessment.liveness.live,
+    idle: assessment.liveness.idle,
+    stuck: assessment.liveness.stuck,
+    readyPassed: assessment.gate.readyPassed,
+    requeueUntil: binding.requeueUntil,
+    phase: binding.phase,
+    // The row the predicates read, as a boolean, never the wait text.
+    hasWait: (binding.status?.waitingOn ?? "").trim() !== "",
+  });
+  return out;
+}

plugins/stewards/migrations.snapshot.json

@@ -118,5 +118,9 @@
   "0cb5723b6d354beb4c2e6e60fabb5b5e3ddf0245bc755b3b4bcd3cbce6b6fd56",
   "a8f6be062abc53e51ea5b4acd27627cf915242aa1b0a60e7e32dc3184ab65b13",
   "cc5d181530cedb4b86b6a61a9cee4f19d141024e812a2e7b944c7796bb211b47",
-  "bc78b2041e26800eb88cc1de2043140d376b015e92af6105b18c84c5735b48a5"
+  "bc78b2041e26800eb88cc1de2043140d376b015e92af6105b18c84c5735b48a5",
+  "0458af01f85835c917014d4f6548b6f78c2f374e094574ccef6eed32018fd8af",
+  "b4bd28a48f934b472cd9eb37669b4e5c07fef9ba88b5f16f6eb8576562409bd4",
+  "3fce9f777b2ae7c333cb2bdb4f49578b842aab9be1ef7e8034750a0b3fa45e83",
+  "500d7556d3f6cb521e0e2b4dc88776f80daeae8f0dbb61024d766e69e849a55e"
 ]

plugins/stewards/reconcile.ts

@@ -916,3 +916,17 @@ export function stuckDue(binding: Pick<Binding, "advancedAt" | "createdAt" | "ph
 /** Whether a new status record moved on from the last one. */
 export const advanced = (before: Binding["status"], after: { phase: string; summary: string; nextAction: string }): boolean =>
   before === null || before.phase !== after.phase || before.summary !== after.summary || before.nextAction !== after.nextAction;
+
+/**
+ * BB-293: the stall the shadow assessment derives from what the plugin can
+ * observe, rather than the `noWakeSource` flag stored at report time. A
+ * non-empty `waitingOn` with no live tracked source and no dated wait still
+ * ahead is a stall the rows show on their own; the flag is the report path's
+ * copy of the same fact.
+ */
+export function observedStall(binding: Pick<Binding, "status" | "requeueUntil">, live: boolean, now: number): boolean {
+  if ((binding.status?.waitingOn ?? "").trim() === "") return false;
+  if (live) return false;
+  if (binding.requeueUntil !== null && binding.requeueUntil > now) return false;
+  return true;
+}