Tune the AI wrapper and act on the independent review

src/app/api/chat/route.ts

@@ -8,8 +8,13 @@ import { ChatRequestError, readChatRequest, streamAnswer } from "@/server/chat";
 /** Seconds for one answer: its searches and its reply. Hobby allows 300. */
 export const maxDuration = 60;
 
-/** Requests longer than this are refused before they're parsed. */
-const MAX_BODY_LENGTH = 1_000_000;
+/**
+ * Requests longer than this are refused before they're parsed. A full PRD's
+ * conversation is about 155,000, most of it passages the browser keeps for
+ * the quotes' pop-overs (2026-10-22); what reaches the model is bounded in
+ * `readChatRequest`.
+ */
+const MAX_BODY_LENGTH = 300_000;
 
 export async function POST(request: Request) {
   const body = await request.text();

src/lib/transcripts.ts

@@ -19,6 +19,8 @@ export type PassageMatch = Passage & {
   participant: string;
   /** Cosine distance to the question: 0 is the same direction, 2 the opposite. */
   distance: number;
+  /** The reranker's relevance to the question, 0 to 1, when it ran. */
+  relevance?: number;
 };
 
 /** A passage as a citation names it, with whose answer it is. */

src/server/ai/cloudflare-reranker.ts

@@ -0,0 +1,78 @@
+import { z } from "zod";
+import { refusingQuota, type CloudflareSettings } from "./cloudflare.ts";
+import { RERANK_MODEL } from "./models.ts";
+import type { Reranker } from "./reranker.ts";
+
+// The real reranker, on staging and production, and on this machine only in
+// the measurement. bge-reranker-base runs on Workers AI's own endpoint
+// (`/ai/run`), not the OpenAI-compatible one: the question and the texts go
+// in, one score per text comes back, referring to the text by its index
+// (https://developers.cloudflare.com/workers-ai/models/bge-reranker-base/,
+// schema-input.json and schema-output.json, read 2026-10-22).
+
+/**
+ * The lowest relevance a passage may have and still reach the model: 0, so
+ * none is dropped. The calibration of 2026-10-22 found no value that keeps our
+ * relevant pairs and drops the unrelated ones (`src/server/AGENTS.md`, *The
+ * reranker*, which is off for that reason).
+ */
+export const RERANK_THRESHOLD = 0;
+
+const rerankReply = z.object({
+  result: z.object({
+    response: z.array(z.object({ id: z.number().int(), score: z.number() })),
+  }),
+});
+
+export function createCloudflareReranker({
+  accountId,
+  apiToken,
+  fetch,
+}: CloudflareSettings): Reranker {
+  const url = `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(accountId)}/ai/run/${RERANK_MODEL}`;
+  const send = refusingQuota(fetch);
+
+  return {
+    model: RERANK_MODEL,
+    threshold: RERANK_THRESHOLD,
+    async rank(query, texts) {
+      if (texts.length === 0) return [];
+      const response = await send(url, {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${apiToken}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          query,
+          contexts: texts.map((text) => ({ text })),
+          // Every text's score, not only the best ones.
+          top_k: texts.length,
+        }),
+      });
+      if (!response.ok) {
+        throw new Error(
+          `Cloudflare's reranker answered ${response.status}: ${(await response.text()).slice(0, 300)}`,
+        );
+      }
+      const reply = rerankReply.parse(await response.json());
+      // Filled, so a text without a score counts as missing below.
+      const relevance = new Array<number | undefined>(texts.length).fill(
+        undefined,
+      );
+      // Already between 0 and 1: the model's logit after the sigmoid that its
+      // page says it "can be mapped" with (all 2,310 pairs of the calibration
+      // of 2026-10-22 matched a local copy's sigmoid to four places).
+      for (const { id, score } of reply.result.response) {
+        if (id >= 0 && id < texts.length) relevance[id] = score;
+      }
+      const missing = relevance.filter((value) => value === undefined).length;
+      if (missing > 0) {
+        throw new Error(
+          `Cloudflare's reranker scored ${texts.length - missing} of ${texts.length} texts.`,
+        );
+      }
+      return relevance as number[];
+    },
+  };
+}

src/server/ai/cloudflare.ts

@@ -1,5 +1,9 @@
 import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
-import { simulateStreamingMiddleware, wrapLanguageModel } from "ai";
+import {
+  simulateStreamingMiddleware,
+  wrapLanguageModel,
+  type LanguageModelMiddleware,
+} from "ai";
 import type { ChatModel } from "./chat-model.ts";
 import { CHAT_MODEL } from "./models.ts";
 
@@ -25,7 +29,7 @@ export function createCloudflare({
     apiKey: apiToken,
     // Without it, a streamed answer reports no tokens.
     includeUsage: true,
-    transformRequestBody: withTextContent,
+    transformRequestBody: (body) => asWorkersAiProvider(withTextContent(body)),
     fetch: refusingQuota(fetch),
   });
 }
@@ -71,7 +75,7 @@ export class QuotaExceededError extends Error {
  * retries it twice, waiting at most `MAX_RETRY_WAIT_MS` between tries. The
  * requests themselves are unchanged.
  */
-function refusingQuota(inner: typeof fetch | undefined): typeof fetch {
+export function refusingQuota(inner: typeof fetch | undefined): typeof fetch {
   const refusing = async (input: RequestInfo | URL, init?: RequestInit) => {
     const response = await (inner ?? globalThis.fetch)(input, init);
     if (response.status !== 429) return response;
@@ -152,6 +156,90 @@ function withTextContent(body: Record<string, unknown>) {
   };
 }
 
+/**
+ * The messages as Cloudflare's own `workers-ai-provider` (4.0.1) sends them
+ * (`convert-to-workersai-chat-messages.ts`, read 2026-10-22): an assistant
+ * message's reasoning as `reasoning`, "the field name vLLM expects on input",
+ * where the package sends `reasoning_content`; and each tool result with its
+ * tool's `name`, which the package leaves out. Older vLLM ignores
+ * `reasoning_content` and names an unnamed tool result's author `functions.`
+ * (docs/records/relevance-and-transport-2026-10-22.md, local).
+ */
+function asWorkersAiProvider(body: Record<string, unknown>) {
+  if (!Array.isArray(body.messages)) return body;
+  const messages = body.messages as Record<string, unknown>[];
+  const toolNames = new Map<unknown, unknown>();
+  for (const message of messages) {
+    if (!Array.isArray(message.tool_calls)) continue;
+    for (const call of message.tool_calls as {
+      id?: unknown;
+      function?: { name?: unknown };
+    }[]) {
+      toolNames.set(call.id, call.function?.name);
+    }
+  }
+  return {
+    ...body,
+    messages: messages.map((message) => {
+      if (message.role === "tool" && toolNames.has(message.tool_call_id)) {
+        return { ...message, name: toolNames.get(message.tool_call_id) };
+      }
+      if (message.reasoning_content === undefined) return message;
+      const { reasoning_content: reasoning, ...rest } = message;
+      return { ...rest, reasoning };
+    }),
+  };
+}
+
+/**
+ * A reply that isn't one: no tool call, cut off at the output limit, its text
+ * nearly all one character. Cloudflare sent 4,096 "!" and no reasoning twice
+ * (`reasoning-high` E-2 on 2026-10-21, `transport` D-3 on 2026-10-22, both at
+ * the output limit). Thrown before the text reaches anyone, so the chat shows
+ * its usual error with "Try again", and the measurement counts the answer as
+ * not run.
+ */
+export class DegenerateReplyError extends Error {
+  constructor(length: number, character: string) {
+    super(
+      `Cloudflare's reply was ${length} characters, nearly all "${character}", cut off at the output limit.`,
+    );
+    this.name = "DegenerateReplyError";
+  }
+}
+
+/** How much of the text one character may fill before the reply is refused. */
+const DEGENERATE_SHARE = 0.95;
+
+/** Refuses a degenerate reply (`DegenerateReplyError`); passes on the rest. */
+const refusingDegenerateReplies: LanguageModelMiddleware = {
+  specificationVersion: "v4",
+  async wrapGenerate({ doGenerate }) {
+    const result = await doGenerate();
+    if (
+      result.finishReason.unified !== "length" ||
+      result.content.some((part) => part.type === "tool-call")
+    ) {
+      return result;
+    }
+    const text = [
+      ...result.content
+        .flatMap((part) => (part.type === "text" ? [part.text] : []))
+        .join(""),
+    ];
+    if (text.length === 0) return result;
+    const counts = new Map<string, number>();
+    for (const character of text) {
+      counts.set(character, (counts.get(character) ?? 0) + 1);
+    }
+    const [character, count] = [...counts].reduce((most, next) =>
+      next[1] > most[1] ? next : most,
+    );
+    if (count < DEGENERATE_SHARE * text.length) return result;
+    throw new DegenerateReplyError(text.length, character);
+  },
+};
+
 /**
  * The chat model, asked for whole replies. Streamed, Cloudflare sends
  * gpt-oss's tool calls as reasoning text without the tool's name and ends
@@ -164,6 +252,9 @@ export function createCloudflareChatModel(
 ): ChatModel {
   return wrapLanguageModel({
     model: createCloudflare(settings).chatModel(CHAT_MODEL),
-    middleware: simulateStreamingMiddleware(),
+    // The last middleware is wrapped directly around the model
+    // (https://ai-sdk.dev/docs/reference/ai-sdk-core/wrap-language-model), so
+    // each whole reply is checked before it becomes a stream.
+    middleware: [simulateStreamingMiddleware(), refusingDegenerateReplies],
   });
 }

src/server/ai/fake-reranker.ts

@@ -0,0 +1,29 @@
+import { embedText } from "./fake-embedder.ts";
+import { EMBEDDING_DIMENSIONS } from "./models.ts";
+import type { Reranker } from "./reranker.ts";
+
+// The fake reranker used locally and in CI. It scores a text by the fake
+// embedder's own similarity to the question (the cosine of their word
+// vectors), so it keeps the order the fake search gives and drops only the
+// passages that share no word with the question. Deterministic and free.
+
+/** Above 0: a passage that shares any word with the question is kept. */
+export const FAKE_RERANK_THRESHOLD = 1e-9;
+
+export function fakeRelevance(query: string, text: string): number {
+  const a = embedText(query, EMBEDDING_DIMENSIONS);
+  const b = embedText(text, EMBEDDING_DIMENSIONS);
+  let dot = 0;
+  for (let index = 0; index < a.length; index++) dot += a[index]! * b[index]!;
+  return Math.max(0, Math.min(1, dot));
+}
+
+export function createFakeReranker(): Reranker {
+  return {
+    model: "fake-word-overlap",
+    threshold: FAKE_RERANK_THRESHOLD,
+    async rank(query, texts) {
+      return texts.map((text) => fakeRelevance(query, text));
+    },
+  };
+}

src/server/ai/models.ts

@@ -10,6 +10,14 @@
 export const CHAT_MODEL = "@cf/openai/gpt-oss-120b";
 export const EMBEDDING_MODEL = "@cf/baai/bge-large-en-v1.5";
 
+/**
+ * The search's second step: a cross-encoder that reads the question and each
+ * candidate passage together and scores how relevant the passage is, as
+ * Cloudflare's AI Search reranks with it (`src/server/AGENTS.md`, *The
+ * reranker*).
+ */
+export const RERANK_MODEL = "@cf/baai/bge-reranker-base";
+
 /** bge-large-en-v1.5 returns 1024 numbers; `passages.embedding` matches it. */
 export const EMBEDDING_DIMENSIONS = 1024;
 

src/server/ai/providers.ts

@@ -3,11 +3,14 @@ import {
   type CloudflareSettings,
 } from "./cloudflare.ts";
 import { createCloudflareEmbedder } from "./cloudflare-embedder.ts";
+import { createCloudflareReranker } from "./cloudflare-reranker.ts";
 import type { ChatModel } from "./chat-model.ts";
 import type { Embedder } from "./embedder.ts";
 import { createFakeChatModel } from "./fake-chat-model.ts";
 import { createFakeEmbedder } from "./fake-embedder.ts";
+import { createFakeReranker } from "./fake-reranker.ts";
 import { createReplayChatModel, loadReplays } from "./replay-chat-model.ts";
+import type { Reranker } from "./reranker.ts";
 
 // AI_PROVIDER picks the implementation: "fake" on development machines and in
 // CI, "cloudflare" on staging and production. AI_CHAT_PROVIDER, when set, picks
@@ -43,6 +46,18 @@ export function createEmbedder(settings: ProviderSettings): Embedder {
   }
 }
 
+/** The search's reranker, from the same provider as its embedder. */
+export function createReranker(settings: ProviderSettings): Reranker {
+  switch (settings.provider) {
+    case "fake":
+      return createFakeReranker();
+    case "cloudflare":
+      return createCloudflareReranker(requireCloudflare(settings));
+    default:
+      throw unknownProvider(settings.provider);
+  }
+}
+
 export function createChatModel(settings: ProviderSettings): ChatModel {
   const chosen = settings.chatProvider ?? settings.provider;
   switch (chosen) {

src/server/ai/reranker.ts

@@ -0,0 +1,13 @@
+/**
+ * Scores how relevant each text is to a question, reading both together. The
+ * fake and the real provider both implement it. The search keeps a passage
+ * only when its relevance reaches `threshold`, as Haystack's cross-encoder
+ * rankers do with `score_threshold` (`src/server/AGENTS.md`, *The reranker*).
+ */
+export type Reranker = {
+  readonly model: string;
+  /** The lowest relevance a passage may have and still be returned. */
+  readonly threshold: number;
+  /** One relevance from 0 to 1 per text, in the texts' order. */
+  rank(query: string, texts: string[]): Promise<number[]>;
+};