diff --git a/src/app/api/chat/route.ts b/src/app/api/chat/route.ts index 0c68910..be7e57f 100644 --- a/src/app/api/chat/route.ts +++ b/src/app/api/chat/route.ts @@ -8,8 +8,13 @@ import { ChatRequestError, readChatRequest, streamAnswer } from "@/server/chat"; /** Seconds for one answer: its searches and its reply. Hobby allows 300. */ export const maxDuration = 60; -/** Requests longer than this are refused before they're parsed. */ -const MAX_BODY_LENGTH = 1_000_000; +/** + * Requests longer than this are refused before they're parsed. A full PRD's + * conversation is about 155,000, most of it passages the browser keeps for + * the quotes' pop-overs (2026-10-22); what reaches the model is bounded in + * `readChatRequest`. + */ +const MAX_BODY_LENGTH = 300_000; export async function POST(request: Request) { const body = await request.text(); diff --git a/src/lib/transcripts.ts b/src/lib/transcripts.ts index 8e3c54b..3dee157 100644 --- a/src/lib/transcripts.ts +++ b/src/lib/transcripts.ts @@ -19,6 +19,8 @@ export type PassageMatch = Passage & { participant: string; /** Cosine distance to the question: 0 is the same direction, 2 the opposite. */ distance: number; + /** The reranker's relevance to the question, 0 to 1, when it ran. */ + relevance?: number; }; /** A passage as a citation names it, with whose answer it is. */ diff --git a/src/server/AGENTS.md b/src/server/AGENTS.md index 8625a58..7991d60 100644 --- a/src/server/AGENTS.md +++ b/src/server/AGENTS.md @@ -18,7 +18,7 @@ Versions: `ai` 7.0.143, `@ai-sdk/openai-compatible` 3.0.71 and `@ai-sdk/react` 4 - A tool call the SDK can't read (input that fails the schema, a tool that's off, or one the chat doesn't have) arrives as `tool-input-error`. A call to one of the chat's own tools keeps that tool's part in the browser (`tool-saveSection`, `tool-searchNotes`), shown as a step that didn't work; a call to any other tool becomes a `dynamic-tool` part, shown as "A step". The model gets the SDK's own message, which says what to fix; the browser gets `describeError`'s plain sentence. - The SDK follows each refused call with a `tool-output-error` whose error is its message as a plain string. `describeError` can't place a string, so it returns the generic sentence, and the browser keeps the later chunk. `streamAnswer` gives that chunk the first sentence again (found 2026-10-18 in `node_modules/ai/dist/index.js`: `parseToolCall`, the `tool-error` case and `isDynamic`). - Each refused call is logged from `onStepEnd`'s `dynamicToolCalls`: the tool's name and why, either the tools on offer or the schema's complaints with their paths. Never the input, which may quote the visitor. -- **Tool budget:** `prepareStep` offers each tool only while it has tries left: three searches (`MAX_SEARCHES`) and two tries to save (`MAX_SAVE_TRIES`: one more for a section in the wrong form, or whose passages couldn't be read). Once a save works, and on the last call (`MAX_STEPS`), it returns `activeTools: []`, so the request carries no tools and the model has to write (https://ai-sdk.dev/docs/agents/loop-control). A tool that's off can't run even if the model calls it: each step parses and runs calls against its active tools only, and anything else is a `NoSuchToolError`. On staging (2026-10-17), gpt-oss ignored the prompt's limit, searched five times for one section and had no calls left for the reply. +- **Tool budget:** `prepareStep` offers each tool only while it has tries left: three searches (`MAX_SEARCHES`) and two tries to save (`MAX_SAVE_TRIES`: one more for a section in the wrong form, or whose passages couldn't be read). Once a save works, and on the last call (`MAX_STEPS`), it returns `activeTools: []`, so the request carries no tools and the model has to write (https://ai-sdk.dev/docs/agents/loop-control). A tool that's off can't run even if the model calls it: each step parses and runs calls against its active tools only, and anything else is a `NoSuchToolError`. On staging (2026-10-17), gpt-oss ignored the prompt's limit, searched five times for one section and had no calls left for the reply. Each search's result also tells the model how many searches it has left (`searchesLeft`, counted in `execute` as `prepareStep` counts the calls), and the last adds `LAST_SEARCH`: in the baseline (2026-10-20), gpt-oss lost count in 3 of 30 answers, asked for a fourth search, read the SDK's refusal ("Model tried to call unavailable tool") as the tool not existing, and ended without words. - **Earlier answers in short:** `convertToModelMessages` turns the conversation's tool results into model input with the tools it's given, and `streamText` turns this answer's results with its own. So `forEarlierAnswers` hands the conversion a `searchNotes` whose `toModelOutput` keeps only each passage's id and interviewee, while this answer's own searches reach the model whole. The SDK's docs say to pass the same tools to both (https://ai-sdk.dev/docs/reference/ai-sdk-core/tool, `toModelOutput`); the difference here is on purpose, and `tests/unit/chat.test.ts` covers it. Saved sections keep their whole input, quotes included, so the model can revise one without searching again. - **Output limit:** `maxOutputTokens` (4096) goes with every call; the package sends it as `max_tokens` (see Cloudflare below). - **Tool errors:** the model gets the thrown error's message, so a refused query (`SearchQueryError`) lets it try again. Any other search failure is logged and replaced by `SearchFailedError`, so neither the model nor the browser sees internals. @@ -42,7 +42,7 @@ Versions: `ai` 7.0.143, `@ai-sdk/openai-compatible` 3.0.71 and `@ai-sdk/react` 4 - **Trying a change:** `npm run dev:real` runs the chat on the real model here, with this machine's token from `.env.development.local` (the owner's exception, 2026-10-19; `scripts/dev-real.ts` sets `AI_CHAT_PROVIDER=cloudflare` for the dev server, and `env.ts` lets it pick the chat model's adapter apart from `AI_PROVIDER`). The search keeps the fake embedder, so which passages come back differs from staging. Follow *Judging the model's behaviour* below, its budget included. Cloudflare's page for gpt-oss-20b links no playground (checked 2026-10-17), and a playground couldn't run `searchNotes` anyway. - **Plain text:** the prompt asks for no Markdown and uses plain labels itself, since a model copies the style it's given. The chat renders bold, inline code and lists anyway and removes every other mark (`src/lib/markdown.ts`), because gpt-oss-120b wrote `**` regardless; statements in the PRD stay plain text. - **Searches and saves:** the prompt says the model has three searches a reply, after which the search is switched off, and saves once a reply, after which the tools are off. `MAX_SEARCHES` and `prepareStep` in `chat.ts` enforce both. Change them together. -- **Tokens:** every model call sends the whole prompt, about 1,775 tokens at four characters a token (version 2: about 1,575; the short prompt before it: about 190), plus the two tools' definitions. A drafted section takes at least three calls: a search, the save and the reply. Keep it tight. +- **Tokens:** every model call sends the whole prompt, about 1,290 tokens at four characters a token (version 3: about 1,775, about half of each request in the baseline; version 2: about 1,575; the short prompt before it: about 190), plus the two tools' definitions. A drafted section takes at least three calls: a search, the save and the reply. Keep it tight. ## Judging the model's behaviour The owner's decision of 2026-10-19 (`docs/decisions.md`, External services): the model's behaviour is measured, not diagnosed from single runs. @@ -61,24 +61,24 @@ The owner's decision of 2026-10-19 (`docs/decisions.md`, External services): the - **A stop rule.** Launch needs only the grocery delivery request to pass (`docs/decisions.md`, Launch). A change to what shapes the model ships only if it beats the baseline without worsening another rule: measured on the grocery delivery request alone, then on the full set before it ships (`docs/decisions.md`, External services, 2026-10-19); what remains is reported in the README. - **The simpler flow** (`docs/decisions.md`, External services) is one candidate, measured like any other change. -What shapes the model (2026-10-19): +What shapes the model (2026-10-22): | Variable | Where | State | | --- | --- | --- | | The model | `ai/models.ts` | `@cf/openai/gpt-oss-120b` | | How hard it reasons | not set | Cloudflare's default | | Randomness between runs | not set | Cloudflare's default `temperature` 0.6, no `seed`: every run differs | -| The system prompt | `prompts/system-prompt.ts` | version 3, fingerprinted | +| The system prompt | `prompts/system-prompt.ts` | version 6, fingerprinted (the baseline ran on version 3) | | The tools' descriptions and schemas | `chat.ts` | not versioned | -| Searches and saves per answer | `chat.ts`: `MAX_SEARCHES`, `MAX_SAVE_TRIES`, `MAX_STEPS` | three searches, two tries to save | +| Searches and saves per answer | `chat.ts`: `MAX_SEARCHES`, `MAX_SAVE_TRIES`, `MAX_STEPS` | three searches, each result saying how many are left; two tries to save | | How earlier answers look to the model | `chat.ts`: `forEarlierAnswers` | searches cut to passage ids and interviewees; saved sections whole; reasoning left out | -| Its reasoning between tool calls in one answer | the AI SDK, `@ai-sdk/openai-compatible`, Cloudflare | unchecked, see below | -| The transport | `ai/cloudflare.ts` | whole replies; empty text instead of `null`; no retry at the quota, retry waits of at most 5 seconds (slice 8a) | -| The search | `search.ts` and the embedder | fake word overlap here, bge-large-en-v1.5 on staging; five passages | +| Its reasoning between tool calls in one answer | the AI SDK, `@ai-sdk/openai-compatible`, `ai/cloudflare.ts` | passed back as `reasoning`; whether Cloudflare reads it shows in the records, see below | +| The transport | `ai/cloudflare.ts` | whole replies; empty text instead of `null`; reasoning sent as `reasoning` and tool results named, as Cloudflare's provider does (since 2026-10-22; the reasoning was read before too, under `reasoning_content`: corrected 2026-10-22, below); no retry at the quota, retry waits of at most 5 seconds (slice 8a); a reply of one repeated character refused (since 2026-10-22) | +| The search | `search.ts` and the embedder | fake word overlap here, bge-large-en-v1.5 on staging; the five closest (the reranker is off since 2026-10-22, *The reranker*) | | The app's own messages to the model | the check-in (`src/features/AGENTS.md`) and `checkReport` | fixed texts | | The visitor's wording | the visitor | free | -- **Reasoning between tool calls:** gpt-oss's response format says its chain of thought must be passed back while it calls functions, and dropped once it has given a final answer (https://github.com/openai/harmony/blob/main/docs/format.md, "Handling reasoning output in subsequent sampling", read 2026-10-19 through Context7). `@ai-sdk/openai-compatible` 3.0.71 sends an assistant message's reasoning parts as `reasoning_content` (`dist/index.js`). The package code shows that an answer's later calls carry its earlier calls' reasoning (`ai/dist/index.js`: a step's reasoning parts stay in its response messages and in their conversion; read 2026-10-19), while earlier answers carry none (`sendReasoning: false`); not yet seen in a request body. Whether Cloudflare's endpoint uses `reasoning_content` is unchecked. +- **Reasoning between tool calls:** gpt-oss's response format says its chain of thought must be passed back while it calls functions, and dropped once it has given a final answer (https://github.com/openai/harmony/blob/main/docs/format.md, "Handling reasoning output in subsequent sampling", read 2026-10-19 through Context7). `@ai-sdk/openai-compatible` 3.0.71 sends an assistant message's reasoning parts as `reasoning_content` (`dist/index.js`); since 2026-10-22 `asWorkersAiProvider` renames it `reasoning`: vLLM's current code renames the deprecated `reasoning_content` to `reasoning` on input (`vllm/entrypoints/openai/chat_completion/protocol.py` on main, `_normalize_messages_before`), OpenAI recommends `reasoning` as the Chat Completions field (https://github.com/openai/openai-cookbook/blob/main/articles/gpt-oss/verifying-implementations.md), and Cloudflare's own provider sends it (*Cloudflare Workers AI*, below); all read 2026-10-22. An answer's later calls carry its earlier calls' reasoning, while earlier answers carry none (`sendReasoning: false`), as the recorded request bodies show. Whether Cloudflare reads it shows in the records, free: the prompt tokens of an answer's next call grow by about 0.21 per character of reasoning passed back, once the other added text, the added messages and the tools are counted, since the tools leave the request after a save and on the last call (about 200 tokens a tool). Corrected 2026-10-22, after the independent review: Cloudflare read it on every call of every batch, as `reasoning_content` and as `reasoning` alike, 0.187 to 0.209 per character in each measurement. The earlier finding that it was dropped on some calls, and the inference of servers on different vLLM versions, came from a check that left the tools out. The check is `docs/records/reasoning-read-check.mjs` (local), run with node over the measurement folders; redo it on later batches. - **The fake search and the two claims:** it ranks by shared words only, so a question that shares no word with any passage, such as "dark mode" or "grocery delivery", ties every passage at distance 1 and gets five unrelated ones. Henk's "Black on white or white on black" comes back only for a question that says "black" or "white" (checked 2026-10-19 with the fake embedder and the loader's parser, no model). So locally the dark mode claim meets Henk's words only if the model's own question names a colour, and the grocery delivery claim gets passages unrelated to it. ## The measurement (`measurement/`) @@ -95,8 +95,8 @@ The tool that measures the model's behaviour (`docs/plan.md`, slices 9a to 9c; d - **The search's copy** (`passages.ts`): the passages of `notes/`, parsed with the loader's parser and embedded with the loader's `embeddingInput` and `contentHash`, kept in `/passages.json`; a passage is embedded again only when its hash changes. `rankPassages` orders by cosine distance, then id, as `search_passages` does, and `copySources` gives the chat the app's own `searchInterviews` over it and the cited passages from it, so neither the database nor the local stack is needed. Ids count from 1 in file order; they differ from the database's, which matters only for exact ties. - **The requests** (`requests.ts`): A to E as the estimate's appendix C has them; A and D take their words from `src/lib/suggestions.ts`, which the intro screen (slice 7) uses too. Changing a text means a new baseline for its request. - **Saved conversations** (`conversations.ts`): D and E continue the first counted run of A with every quote confirmed and no error; C2 the first counted run of C1 whose reply has a sentence with "?" and a word about the connection (offline, signal, reception, connection, connectivity, internet), an approximation 9c reads by hand. Each is the run's conversation plus its answer rebuilt from the recorded chunks with `readUIMessageStream`, saved once (`wx`) in `/conversations/`, fingerprinted and refused if it changed; every conversation passes `readChatRequest` before an answer. A continuation waits, and is skipped, until its conversation exists; `--conversations-from` takes another measurement's, so a comparison starts from the baseline's; it defaults to `--compare-with`'s, and a measurement that names itself there (a flow change) saves its own. -- **The meter** (`budget.ts`): every Cloudflare request goes through it under the recorder. It counts each reply's `usage` at gpt-oss-120b's rates and each embedding request at the loader's token bound (Cloudflare reports none), spaces requests a second apart (300 a minute allowed), and once the batch has spent more than its cap it sends nothing more and throws `BudgetExceededError`, which the AI SDK doesn't retry (`handleFetchError` in `@ai-sdk/provider-utils` passes it through). The cap is 1.5 times the expected neurons of the answers that can go now (each request's mean so far in the ledger, or the estimate's 300, 450 and 785) and the copy; an answer waiting for its saved conversation raises the cap by 1.5 times its expectation only once that conversation exists and only for runs still to come in the batch (earlier ones were skipped at their turn, and the closing line names them for a later batch), and the plan says how far it can rise. Without that, waiting answers that never go would loosen the cap for the rest (the review, 2026-10-19). -- **Outcomes:** a 429 with code 3036 or 4006 (quota: the errors page names 3036, the `/ai/v1` endpoint sent 4006 with the same message on 2026-10-20), code 3040 (busy) or any other 429 (per-minute) that no retry got past makes the answer "not run". The AI SDK retries a refused request as it was, so the meter counts a refusal as recovered only when the same request (address and body) later succeeds; a refused search stays unrecovered when the chat carries on or the model searches again with other words. A recovered answer counts, and its record keeps every refusal. A "not run" answer's record goes to `/not-run//`, and quota stops the batch; a per-minute refusal pauses a minute. An answer cut off at the cap is "not run" too, and the batch stops. A record `writeRecord` refuses is "not recorded": nothing is written, its spend still counts. Only a counted answer fills its run (`//-.json`); a later batch sends the others again and says so in its plan. +- **The meter** (`budget.ts`): every Cloudflare request goes through it under the recorder. It counts each reply's `usage` at gpt-oss-120b's rates and each embedding request at the loader's token bound (Cloudflare reports none), spaces requests a second apart (300 a minute allowed), and once the batch has spent more than its cap it sends nothing more and throws `BudgetExceededError`, which the AI SDK doesn't retry (`handleFetchError` in `@ai-sdk/provider-utils` passes it through). The cap is 1.5 times the expected neurons of the answers that can go now (each request's mean in this measurement once it has a counted answer, else in the one it's compared with, since a change can move the cost: `reasoning-high` cost about twice `prompt-v5`'s and its first batch was cut at a cap set from the whole ledger, 2026-10-21; otherwise its mean so far in the ledger, or the estimate's 300, 450 and 785) and the copy; an answer waiting for its saved conversation raises the cap by 1.5 times its expectation only once that conversation exists and only for runs still to come in the batch (earlier ones were skipped at their turn, and the closing line names them for a later batch), and the plan says how far it can rise. Without that, waiting answers that never go would loosen the cap for the rest (the review, 2026-10-19). +- **Outcomes:** a 429 with code 3036 or 4006 (quota: the errors page names 3036, the `/ai/v1` endpoint sent 4006 with the same message on 2026-10-20), code 3040 (busy) or any other 429 (per-minute) that no retry got past makes the answer "not run". The AI SDK retries a refused request as it was, so the meter counts a refusal as recovered only when the same request (address and body) later succeeds; a refused search stays unrecovered when the chat carries on or the model searches again with other words. A recovered answer counts, and its record keeps every refusal. A "not run" answer's record goes to `/not-run//`, and quota stops the batch; a per-minute refusal pauses a minute. An answer cut off at the cap is "not run" too, and the batch stops. So is one whose model call ended with `DegenerateReplyError` (*Cloudflare Workers AI*, below), as "degenerate reply", and the batch goes on. A record `writeRecord` refuses is "not recorded": nothing is written, its spend still counts. Only a counted answer fills its run (`//-.json`); a later batch sends the others again and says so in its plan. - **Measured commits:** each record names the commit it ran on (`setup.code.commit`), and the comparison runs `git diff` on those. GitHub's rebase merge gives a branch's commits new ids, which leaves the recorded ones on no branch, where Git may clean them up once the reflog expires (90 days by default). So after each merge, tag every commit the records name that isn't on `main` as `measured/`; these tags stay local. - **The ledger** (`/spend.jsonl`, real batches only): one line per answer and per copy, for the totals of the UTC day and the UTC calendar month, which may not match the billing month. Checked against Cloudflare's dashboard on 2026-10-19: the chat's neurons matched (6,207 against about 6,220), and the embedding bound was about twice the real figure. The dashboard's "used today" lagged by about an hour, and the daily limit let the day reach 10.4k before refusing, so size a day's batch to what's left by the dashboard and the owner, with room to spare. - **Scoring** (`score.ts`, 9c): each counted run on the product description's rules, from this answer's saves only (`savedSections`: the `saveSection` calls that worked). `RULES` names each rule and the requests it applies to: `quotes` and `cites` on all; `saves-first` on A; `asks` on C1; `cites-answer` on C2 (a confirmed chat citation of `visitorMessageId`, `visitor-2`); `both-views` on B ({maya-3, henk-4} against {daan-5, sofia-5, sofia-6}, on different statements, plus a statement starting "Open question" or in Open Questions); `no-claim` on D and E. A verdict is pass, fail or "saved nothing", which is never a pass: for A's `saves-first` it's a fail, and `asks` needs no save. Where code only approximates a rule (`asks`, `cites-answer`, `both-views`, `no-claim`), the rule says what to read by hand and the score lists it. Faults use the PRD's own wording (`objection` and `NO_QUOTE` in `src/lib/prd.ts`), plus one for an interview among the sources of a delivery or dark mode statement. @@ -107,6 +107,15 @@ The tool that measures the model's behaviour (`docs/plan.md`, slices 9a to 9c; d - **gpt-oss-120b's input schema** (`sync-input.json`, read 2026-10-19): `temperature` (default 0.6), `top_p`, `top_k`, `seed` and the penalties, but no `reasoning_effort`; a message names only `role` and `content`, though `tool_calls` work, so the schema doesn't settle whether `reasoning_content` is read. - **Sources (read 2026-10-19, through Context7):** for the runner, Node's `module.registerHooks` (https://nodejs.org/docs/latest-v24.x/api/module.html), `process.kill(pid, 0)` and the AI SDK's `readUIMessageStream` (https://ai-sdk.dev/docs/reference/ai-sdk-ui/read-ui-message-stream); for the recorder, language model middleware, `wrapStream` with a `TransformStream` (https://ai-sdk.dev/docs/ai-sdk-core/middleware) and the installed `LanguageModelMiddleware` type; Node's `crypto.hash`, the `wx` flag and `execFileSync` (https://nodejs.org/docs/latest-v24.x/api/); for the comparison's options, `util.parseArgs` with `multiple: true`, which collects a repeated option into an array (https://nodejs.org/docs/latest-v24.x/api/util.html). +## The reranker (`ai/cloudflare-reranker.ts`, `search.ts`) +- **Off since 2026-10-22** (the agent's call, which the owner left to it): `search.ts` and the measurement search without it, so the search returns the five closest, as in the baseline, and saves about 1.3 neurons and a call a search. The calibration (`docs/records/measurements/rerank-calibration/2026-10-22.json`: 55 recorded searches, 2,310 pairs) and 327 hand labels (`docs/records/rerank-labels-2026-10-22/`; both local) showed bge-reranker-base neither separates nor orders our passages. Relevant pairs scored from 0.00004 to 0.97 and unrelated ones up to 0.33, so any threshold drops real matches or keeps junk: the review's expected 0.02 dropped 56 of 97 relevant pairs and emptied 31 of 55 searches. "privacy" scored Sofia's "My own shopping is my business" at 0.0003, below Henk's near miss for "dark mode" at 0.0007. Its best five held fewer relevant passages than the embedder's own five on the 48 searches with both on record (41 against 47). Not our wiring: the scores matched a local copy (`Xenova/bge-reranker-base`) to four places, BAAI's own example pair scored as published (−8.15 and 6.18 before the sigmoid), and the answer without the researcher's question did worse. It finds a meaning only when the query spells it out ("keeping my own items private from housemates" scored Sofia's answer 0.49), while gpt-oss searches with topic words. Cloudflare lists no other reranker, and BAAI points to its newer, larger ones (both read 2026-10-22 through Context7). To try one: `createReranker` back into `connect()` and the measurement's `environment()` (`measurement/cli.ts`). +- **Why it was tried:** the search returned the five closest passages whether or not any was about the question, so a visitor's claim met a top-ranked near miss (Henk's "black on white or white on black" for "dark mode"). Distance can't gate it here: every interview is about grocery lists, so "grocery delivery" finds daan-6 (not delivery) at 0.282 while "privacy data sharing security" finds sofia-4 (a real match) at 0.360 (`docs/records/relevance-and-transport-2026-10-22.md`, local). +- **The proven pattern, followed as is:** retrieve more than you return, rerank with a cross-encoder, drop what falls below a threshold on the reranker's score, and say so when nothing passes. Cloudflare's AI Search reranks with the same model and filters on `reranking.match_threshold`, and "if no results meet the threshold, AI Search will not generate a response" (https://developers.cloudflare.com/ai-search/configuration/retrieval/result-controls/ and `/reranking/`); Haystack's cross-encoder ranker maps the logits through a sigmoid (`scale_score`), keeps `score >= score_threshold`, then `top_k`, and puts a document's fields before its text (`haystack/components/rankers/sentence_transformers_similarity.py`, v2.18.0); Anthropic's contextual retrieval reranks a larger candidate set down (https://www.anthropic.com/news/contextual-retrieval), and Anthropic advises returning "No results found." rather than nothing (https://platform.claude.com/docs/en/build-with-claude/search-results). All read 2026-10-22. +- **How, when on:** `searchInterviews` asks the database for the `RERANK_CANDIDATES` (50) closest passages, so all 42 (since 2026-10-22, after the independent review: with the 20 closest, the embedder decided which ones the reranker saw, and the fake embedder's ties made the 20 arbitrary), gives the reranker the question and each candidate's `embeddingInput` (the researcher's question, then the answer), keeps those at or above the reranker's `threshold`, best first, at most `SEARCH_RESULT_COUNT`, each with its `relevance`. When none passes, the model's search result has no passages and the note "No passage in the six interviews is about this." (`NOTHING_RELEVANT` in `chat.ts`); the tool's description and prompt version 6 say so. Without a reranker (tests that leave it out) the search returns the five closest, as before. +- **Cloudflare's model:** `@cf/baai/bge-reranker-base` on `/ai/run`, not the OpenAI-compatible endpoint: `{ query, contexts: [{ text }], top_k }` in, `{ result: { response: [{ id, score }] } }` out, `id` the context's index (`schema-input.json`, `schema-output.json` on its model page, read 2026-10-22). The page says the model's score "can be mapped to a float value in [0,1] by sigmoid function"; Cloudflare's replies are that already (all 2,310 pairs of the calibration matched a local copy's sigmoid to four places), so the score is used as sent. Before 2026-10-22 the code applied the sigmoid again, which put every passage between 0.5 and 0.73. `top_k` is the number of texts, so every one comes back. A quota 429 throws `QuotaExceededError` through `refusingQuota`; any other failure throws, and the chat says the search didn't work. 283 neurons per million input tokens (pricing page, read 2026-10-22): about 1.3 neurons a search for all 42. +- **The threshold** (`RERANK_THRESHOLD`): 0, which keeps every passage, since no value separated the calibration's pairs (above). No source gives one for bge-reranker-base, and Cohere advises setting it from your own pairs, which assumes the reranker separates them (https://docs.cohere.com/docs/reranking-best-practices, read 2026-10-22). `npm run measure -- --calibrate-rerank` (`--plan` first) scores every passage against every search gpt-oss made in the counted records with the real reranker, under the meter and into the ledger (its bound for 55 searches: 218 neurons), and saves the pairs once in `docs/records/measurements/rerank-calibration/.json`. The measurement records the reranker and its threshold in each run's setup, so a change shows as a search difference. +- **The fake** (`ai/fake-reranker.ts`): the fake embedder's own cosine similarity, so it keeps the fake search's order and drops only passages that share no word with the question. With the reranker off, only unit tests use it. + ## The citation check (`citations.ts`) - **The right words:** an interview quote is checked against the cited passage's `answer` only, never its `question`. The passages come from the database with the read key (`db/passages.ts`: the cited interviews by slug, then the cited positions), never from search results the browser sends back. A chat quote is checked against the request's user messages only, never the assistant's. Those come from the browser, so a visitor can only confirm their own words. - **Matching:** NFC, lowercase, every quote mark as one, spaces collapsed; punctuation and spaces are stripped from the quote's ends only; whole words only, and an apostrophe between letters joins one word, so "don" isn't in "don't". Never loosen it (`docs/decisions.md`, Data): when a model keeps failing it, the prompt or the model changes. @@ -114,16 +123,18 @@ The tool that measures the model's behaviour (`docs/plan.md`, slices 9a to 9c; d - **What the model learns:** `checkReport` in `chat.ts` sends the counts, each unverified quote with its problem, and the statements that cite nothing, numbered from 1. - **Shared types:** `src/lib/prd.ts` holds the nine sections, the draft and checked shapes, `isFlagged`, `tally`, `isVisitorDecision` and `sourcesOf`, which the browser uses too. The check itself is `server-only`. - **For the pop-over:** a checked interview quote carries the passage's `question` and `answer`, read with the check, so the browser can show the quote in its full answer. The model never sees them: it gets `checkReport`, and earlier saves reach it through the same `toModelOutput`. -- **Limits:** the input schema bounds statements, citations and lengths: roomy for a model, tight for a forged request. The output schema checks saved sections the browser sends back with later messages. +- **Limits:** the input schema bounds statements, citations and lengths: roomy for a model, tight for a forged request. The output schema checks saved sections the browser sends back with later messages. Since every model call resends the conversation, `readChatRequest` refuses one over 100,000 characters as they reach the model (`MAX_CONVERSATION_LENGTH`: every part but the two tools' outputs, which reach it cut down; a full PRD is about 40,000) and an assistant message with over 8,000 characters of text (`MAX_ANSWER_LENGTH`; the longest recorded reply has 3,463), and the route refuses a body over 300,000 (a full PRD's is about 155,000, mostly the pop-overs' passages). The owner's numbers, 2026-10-22. ## Cloudflare Workers AI (`ai/cloudflare.ts`) - **The chat model** is `@cf/openai/gpt-oss-120b` since 2026-10-18 (`ai/models.ts`, `docs/decisions.md`); gpt-oss-20b before. Their input schemas (`sync-input.json`) were identical byte for byte that day, so the workarounds below, found with gpt-oss-20b, apply unchanged. Its behaviour is measured as *Judging the model's behaviour* describes. - **Endpoint:** `createOpenAICompatible` with `baseURL` `https://api.cloudflare.com/client/v4/accounts//ai/v1` and the token as `apiKey` (sent as `Authorization: Bearer`). Chat goes to `/chat/completions` (`chatModel`), embeddings to `/embeddings` (`embeddingModel`). Settings: `CLOUDFLARE_ACCOUNT_ID` (32 hexadecimal characters, checked in `env.ts`) and `CLOUDFLARE_API_TOKEN`. - **Whole replies for chat:** streamed, Cloudflare sends gpt-oss's tool call as `reasoning_content` text (the bare JSON arguments, no tool name, no `tool_calls`) and ends with `finish_reason: "stop"`, so the search never runs (staging, 2026-10-17; raw stream in `docs/plan.md` history). Whole replies carry proper `tool_calls`, as the owner's first test showed. `createCloudflareChatModel` wraps the model in `simulateStreamingMiddleware` (https://ai-sdk.dev/docs/reference/ai-sdk-core/simulate-streaming-middleware), so `streamText` still works and each reply arrives in one piece. Cloudflare's own `workers-ai-provider` 4.0.0 also falls back to non-streaming for gpt-oss. Recheck before streaming again. +- **As Cloudflare's own provider sends messages** (`asWorkersAiProvider`, after `withTextContent`): `workers-ai-provider` 4.0.1 (`convert-to-workersai-chat-messages.ts`, https://github.com/cloudflare/ai, read 2026-10-22) sends an assistant message's reasoning as `reasoning`, "the field name vLLM expects on input", and `name` on every tool result; the package sends `reasoning_content` and no `name`. Cloudflare's replies look like vLLM's (`chatcmpl-tool-…` ids, both reasoning fields), and vLLM up to 0.12 ignores `reasoning_content` on input and names an unnamed tool result's author `functions.`, while its current code reads `reasoning` (`docs/records/relevance-and-transport-2026-10-22.md`, local). So the reasoning is renamed, not sent twice, and each tool result gets the name of the call it answers. The system prompt stays `role: "system"`, as the provider sends it; whether Cloudflare turns it into gpt-oss's developer message is unknown. - **Text in every message:** Cloudflare's input schema requires each message's `content` to be a string or a list of text parts (`sync-input.json` of the model). The package sends `content: null` on an assistant message that only calls a tool, so the follow-up after a search got 400, code 5006 (staging, 2026-10-17). `withTextContent` in `transformRequestBody` sends `""` instead. - **256 output tokens by default:** the model's `max_tokens` defaults to 256 (`sync-input.json`), and gpt-oss's reasoning counts against it. On staging (2026-10-17), section drafts stopped with finish reason `length` before any text, so `chat.ts` asks for 4096. The schema states no maximum; the context window is 128,000 tokens. - **No tool choice:** the schema has `tools` but no `tool_choice`, so the chat stops the search by leaving the tools out of the request rather than sending `tool_choice: "none"`. The schema allows a request without tools, even after tool calls in the conversation. On staging (2026-10-18), the reply after a save, sent without tools, came back as text. A request without tools doesn't guarantee words, though: with prompt version 4 the same day, gpt-oss called `searchNotes` twice after it was switched off, the second time on the last call, which carried no tools, and the answer ended without words. - **The quota and other 429s** (slice 8a): `refusingQuota` wraps `fetch` for chat and embeddings alike, under whatever `fetch` the settings pass (the measurement's meter and recorder see every reply first). A 429 with code 3036 or 4006 (`QUOTA_CODES`; 4006 is what `/ai/v1` sent on 2026-10-20) throws `QuotaExceededError` from `fetch`, which `handleFetchError` in `@ai-sdk/provider-utils` passes through and the SDK doesn't retry: nothing works before 00:00 UTC. Any other 429 (the per-minute limit, code 3040 out of capacity) goes on to the SDK's two retries, but a `retry-after-ms` or `retry-after` asking for more than `MAX_RETRY_WAIT_MS` (5 seconds) is dropped first, so the SDK's own 2 and 4 seconds apply instead of up to 60 each, inside the route's 60 seconds (`getRetryDelayInMs` in `ai/dist/index.js`, read 2026-10-20). The request bodies are untouched. The quota codes are also in `measurement/budget.ts` (`QUOTA_CODES`); keep the two lists the same. Replayed replies in `tests/unit/cloudflare.test.ts`; the 4006 one is Cloudflare's own of 2026-10-20. +- **Degenerate replies** (`refusingDegenerateReplies`, 2026-10-22): Cloudflare twice answered with 4,096 "!", no reasoning and no tool call, at the output limit (`reasoning-high` E-2's fourth call on 2026-10-21, `transport` D-3's first on 2026-10-22). A reply with no tool call, finish reason `length` and text at least 95% one character throws `DegenerateReplyError`, so the visitor gets the generic error and "Try again", never the text; the AI SDK doesn't retry it, and there's no retry of our own. The `wrapGenerate` middleware comes after `simulateStreamingMiddleware` in the array, since the last one is wrapped directly around the model and so sees each whole reply before it becomes a stream (the installed `LanguageModelMiddleware` type in `node_modules/ai/dist/index.d.ts` and `wrapLanguageModel` in `dist/index.js`; https://ai-sdk.dev/docs/reference/ai-sdk-core/wrap-language-model, read 2026-10-22 through Context7). Both recorded replies are replayed in `tests/unit/cloudflare.test.ts`; the measurement counts such an answer "not run". - **`includeUsage: true`** makes streamed replies report tokens; kept for when streaming returns. - **No token count for embeddings:** on staging (2026-10-17) Cloudflare's `/embeddings` reply had no `usage`, and the AI SDK then reports `NaN` tokens. `cloudflare-embedder.ts` logs "Cloudflare reported no token count" instead. - **Embedding batches:** Cloudflare's bge models take at most 100 texts a call; the package would send 2,048, so `cloudflare-embedder.ts` splits them. diff --git a/src/server/ai/cloudflare-reranker.ts b/src/server/ai/cloudflare-reranker.ts new file mode 100644 index 0000000..aea58f3 --- /dev/null +++ b/src/server/ai/cloudflare-reranker.ts @@ -0,0 +1,78 @@ +import { z } from "zod"; +import { refusingQuota, type CloudflareSettings } from "./cloudflare.ts"; +import { RERANK_MODEL } from "./models.ts"; +import type { Reranker } from "./reranker.ts"; + +// The real reranker, on staging and production, and on this machine only in +// the measurement. bge-reranker-base runs on Workers AI's own endpoint +// (`/ai/run`), not the OpenAI-compatible one: the question and the texts go +// in, one score per text comes back, referring to the text by its index +// (https://developers.cloudflare.com/workers-ai/models/bge-reranker-base/, +// schema-input.json and schema-output.json, read 2026-10-22). + +/** + * The lowest relevance a passage may have and still reach the model: 0, so + * none is dropped. The calibration of 2026-10-22 found no value that keeps our + * relevant pairs and drops the unrelated ones (`src/server/AGENTS.md`, *The + * reranker*, which is off for that reason). + */ +export const RERANK_THRESHOLD = 0; + +const rerankReply = z.object({ + result: z.object({ + response: z.array(z.object({ id: z.number().int(), score: z.number() })), + }), +}); + +export function createCloudflareReranker({ + accountId, + apiToken, + fetch, +}: CloudflareSettings): Reranker { + const url = `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(accountId)}/ai/run/${RERANK_MODEL}`; + const send = refusingQuota(fetch); + + return { + model: RERANK_MODEL, + threshold: RERANK_THRESHOLD, + async rank(query, texts) { + if (texts.length === 0) return []; + const response = await send(url, { + method: "POST", + headers: { + Authorization: `Bearer ${apiToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + query, + contexts: texts.map((text) => ({ text })), + // Every text's score, not only the best ones. + top_k: texts.length, + }), + }); + if (!response.ok) { + throw new Error( + `Cloudflare's reranker answered ${response.status}: ${(await response.text()).slice(0, 300)}`, + ); + } + const reply = rerankReply.parse(await response.json()); + // Filled, so a text without a score counts as missing below. + const relevance = new Array(texts.length).fill( + undefined, + ); + // Already between 0 and 1: the model's logit after the sigmoid that its + // page says it "can be mapped" with (all 2,310 pairs of the calibration + // of 2026-10-22 matched a local copy's sigmoid to four places). + for (const { id, score } of reply.result.response) { + if (id >= 0 && id < texts.length) relevance[id] = score; + } + const missing = relevance.filter((value) => value === undefined).length; + if (missing > 0) { + throw new Error( + `Cloudflare's reranker scored ${texts.length - missing} of ${texts.length} texts.`, + ); + } + return relevance as number[]; + }, + }; +} diff --git a/src/server/ai/cloudflare.ts b/src/server/ai/cloudflare.ts index a79e424..9639854 100644 --- a/src/server/ai/cloudflare.ts +++ b/src/server/ai/cloudflare.ts @@ -1,5 +1,9 @@ import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; -import { simulateStreamingMiddleware, wrapLanguageModel } from "ai"; +import { + simulateStreamingMiddleware, + wrapLanguageModel, + type LanguageModelMiddleware, +} from "ai"; import type { ChatModel } from "./chat-model.ts"; import { CHAT_MODEL } from "./models.ts"; @@ -25,7 +29,7 @@ export function createCloudflare({ apiKey: apiToken, // Without it, a streamed answer reports no tokens. includeUsage: true, - transformRequestBody: withTextContent, + transformRequestBody: (body) => asWorkersAiProvider(withTextContent(body)), fetch: refusingQuota(fetch), }); } @@ -71,7 +75,7 @@ export class QuotaExceededError extends Error { * retries it twice, waiting at most `MAX_RETRY_WAIT_MS` between tries. The * requests themselves are unchanged. */ -function refusingQuota(inner: typeof fetch | undefined): typeof fetch { +export function refusingQuota(inner: typeof fetch | undefined): typeof fetch { const refusing = async (input: RequestInfo | URL, init?: RequestInit) => { const response = await (inner ?? globalThis.fetch)(input, init); if (response.status !== 429) return response; @@ -152,6 +156,90 @@ function withTextContent(body: Record) { }; } +/** + * The messages as Cloudflare's own `workers-ai-provider` (4.0.1) sends them + * (`convert-to-workersai-chat-messages.ts`, read 2026-10-22): an assistant + * message's reasoning as `reasoning`, "the field name vLLM expects on input", + * where the package sends `reasoning_content`; and each tool result with its + * tool's `name`, which the package leaves out. Older vLLM ignores + * `reasoning_content` and names an unnamed tool result's author `functions.` + * (docs/records/relevance-and-transport-2026-10-22.md, local). + */ +function asWorkersAiProvider(body: Record) { + if (!Array.isArray(body.messages)) return body; + const messages = body.messages as Record[]; + const toolNames = new Map(); + for (const message of messages) { + if (!Array.isArray(message.tool_calls)) continue; + for (const call of message.tool_calls as { + id?: unknown; + function?: { name?: unknown }; + }[]) { + toolNames.set(call.id, call.function?.name); + } + } + return { + ...body, + messages: messages.map((message) => { + if (message.role === "tool" && toolNames.has(message.tool_call_id)) { + return { ...message, name: toolNames.get(message.tool_call_id) }; + } + if (message.reasoning_content === undefined) return message; + const { reasoning_content: reasoning, ...rest } = message; + return { ...rest, reasoning }; + }), + }; +} + +/** + * A reply that isn't one: no tool call, cut off at the output limit, its text + * nearly all one character. Cloudflare sent 4,096 "!" and no reasoning twice + * (`reasoning-high` E-2 on 2026-10-21, `transport` D-3 on 2026-10-22, both at + * the output limit). Thrown before the text reaches anyone, so the chat shows + * its usual error with "Try again", and the measurement counts the answer as + * not run. + */ +export class DegenerateReplyError extends Error { + constructor(length: number, character: string) { + super( + `Cloudflare's reply was ${length} characters, nearly all "${character}", cut off at the output limit.`, + ); + this.name = "DegenerateReplyError"; + } +} + +/** How much of the text one character may fill before the reply is refused. */ +const DEGENERATE_SHARE = 0.95; + +/** Refuses a degenerate reply (`DegenerateReplyError`); passes on the rest. */ +const refusingDegenerateReplies: LanguageModelMiddleware = { + specificationVersion: "v4", + async wrapGenerate({ doGenerate }) { + const result = await doGenerate(); + if ( + result.finishReason.unified !== "length" || + result.content.some((part) => part.type === "tool-call") + ) { + return result; + } + const text = [ + ...result.content + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join(""), + ]; + if (text.length === 0) return result; + const counts = new Map(); + for (const character of text) { + counts.set(character, (counts.get(character) ?? 0) + 1); + } + const [character, count] = [...counts].reduce((most, next) => + next[1] > most[1] ? next : most, + ); + if (count < DEGENERATE_SHARE * text.length) return result; + throw new DegenerateReplyError(text.length, character); + }, +}; + /** * The chat model, asked for whole replies. Streamed, Cloudflare sends * gpt-oss's tool calls as reasoning text without the tool's name and ends @@ -164,6 +252,9 @@ export function createCloudflareChatModel( ): ChatModel { return wrapLanguageModel({ model: createCloudflare(settings).chatModel(CHAT_MODEL), - middleware: simulateStreamingMiddleware(), + // The last middleware is wrapped directly around the model + // (https://ai-sdk.dev/docs/reference/ai-sdk-core/wrap-language-model), so + // each whole reply is checked before it becomes a stream. + middleware: [simulateStreamingMiddleware(), refusingDegenerateReplies], }); } diff --git a/src/server/ai/fake-reranker.ts b/src/server/ai/fake-reranker.ts new file mode 100644 index 0000000..34a7c5e --- /dev/null +++ b/src/server/ai/fake-reranker.ts @@ -0,0 +1,29 @@ +import { embedText } from "./fake-embedder.ts"; +import { EMBEDDING_DIMENSIONS } from "./models.ts"; +import type { Reranker } from "./reranker.ts"; + +// The fake reranker used locally and in CI. It scores a text by the fake +// embedder's own similarity to the question (the cosine of their word +// vectors), so it keeps the order the fake search gives and drops only the +// passages that share no word with the question. Deterministic and free. + +/** Above 0: a passage that shares any word with the question is kept. */ +export const FAKE_RERANK_THRESHOLD = 1e-9; + +export function fakeRelevance(query: string, text: string): number { + const a = embedText(query, EMBEDDING_DIMENSIONS); + const b = embedText(text, EMBEDDING_DIMENSIONS); + let dot = 0; + for (let index = 0; index < a.length; index++) dot += a[index]! * b[index]!; + return Math.max(0, Math.min(1, dot)); +} + +export function createFakeReranker(): Reranker { + return { + model: "fake-word-overlap", + threshold: FAKE_RERANK_THRESHOLD, + async rank(query, texts) { + return texts.map((text) => fakeRelevance(query, text)); + }, + }; +} diff --git a/src/server/ai/models.ts b/src/server/ai/models.ts index 3ea27a8..680deaf 100644 --- a/src/server/ai/models.ts +++ b/src/server/ai/models.ts @@ -10,6 +10,14 @@ export const CHAT_MODEL = "@cf/openai/gpt-oss-120b"; export const EMBEDDING_MODEL = "@cf/baai/bge-large-en-v1.5"; +/** + * The search's second step: a cross-encoder that reads the question and each + * candidate passage together and scores how relevant the passage is, as + * Cloudflare's AI Search reranks with it (`src/server/AGENTS.md`, *The + * reranker*). + */ +export const RERANK_MODEL = "@cf/baai/bge-reranker-base"; + /** bge-large-en-v1.5 returns 1024 numbers; `passages.embedding` matches it. */ export const EMBEDDING_DIMENSIONS = 1024; diff --git a/src/server/ai/providers.ts b/src/server/ai/providers.ts index 2717f11..c30470c 100644 --- a/src/server/ai/providers.ts +++ b/src/server/ai/providers.ts @@ -3,11 +3,14 @@ import { type CloudflareSettings, } from "./cloudflare.ts"; import { createCloudflareEmbedder } from "./cloudflare-embedder.ts"; +import { createCloudflareReranker } from "./cloudflare-reranker.ts"; import type { ChatModel } from "./chat-model.ts"; import type { Embedder } from "./embedder.ts"; import { createFakeChatModel } from "./fake-chat-model.ts"; import { createFakeEmbedder } from "./fake-embedder.ts"; +import { createFakeReranker } from "./fake-reranker.ts"; import { createReplayChatModel, loadReplays } from "./replay-chat-model.ts"; +import type { Reranker } from "./reranker.ts"; // AI_PROVIDER picks the implementation: "fake" on development machines and in // CI, "cloudflare" on staging and production. AI_CHAT_PROVIDER, when set, picks @@ -43,6 +46,18 @@ export function createEmbedder(settings: ProviderSettings): Embedder { } } +/** The search's reranker, from the same provider as its embedder. */ +export function createReranker(settings: ProviderSettings): Reranker { + switch (settings.provider) { + case "fake": + return createFakeReranker(); + case "cloudflare": + return createCloudflareReranker(requireCloudflare(settings)); + default: + throw unknownProvider(settings.provider); + } +} + export function createChatModel(settings: ProviderSettings): ChatModel { const chosen = settings.chatProvider ?? settings.provider; switch (chosen) { diff --git a/src/server/ai/reranker.ts b/src/server/ai/reranker.ts new file mode 100644 index 0000000..ed3b1da --- /dev/null +++ b/src/server/ai/reranker.ts @@ -0,0 +1,13 @@ +/** + * Scores how relevant each text is to a question, reading both together. The + * fake and the real provider both implement it. The search keeps a passage + * only when its relevance reaches `threshold`, as Haystack's cross-encoder + * rankers do with `score_threshold` (`src/server/AGENTS.md`, *The reranker*). + */ +export type Reranker = { + readonly model: string; + /** The lowest relevance a passage may have and still be returned. */ + readonly threshold: number; + /** One relevance from 0 to 1 per text, in the texts' order. */ + rank(query: string, texts: string[]): Promise; +}; diff --git a/src/server/chat.ts b/src/server/chat.ts index bb6b528..bed7321 100644 --- a/src/server/chat.ts +++ b/src/server/chat.ts @@ -70,6 +70,17 @@ import { */ const MAX_SEARCHES = 3; +/** + * What the model reads when the reranker kept no passage: said plainly, as + * Anthropic advises for a search with no results ("No results found.") and as + * Cloudflare's AI Search does by not answering from nothing. + */ +const NOTHING_RELEVANT = "No passage in the six interviews is about this."; + +/** What the model reads with the passages of an answer's last search. */ +const LAST_SEARCH = + "That was your last search for this reply. Save the section, or write your reply, from the passages you have."; + /** * At most this many tries to save the section in one answer: one more when * the first wasn't in a form the check reads, or its passages couldn't be @@ -93,6 +104,18 @@ const MAX_OUTPUT_TOKENS = 4096; /** At most this many messages in one conversation. */ export const MAX_MESSAGES = 200; +/** + * At most this many characters of a conversation, counted as they reach the + * model: every part except our two tools' outputs, which reach it cut down + * (`forEarlierAnswers`, `checkReport`). Each model call resends them, so this + * bounds what a forged conversation costs; a full PRD of nine sections is + * about 40,000 (estimated 2026-10-22 from the baseline's first section). + */ +export const MAX_CONVERSATION_LENGTH = 100_000; + +/** At most this many characters of text in one of the assistant's messages. */ +export const MAX_ANSWER_LENGTH = 8_000; + // A drafted section's limits: roomy for a model, tight for a forged request. const MAX_STATEMENTS = 40; const MAX_STATEMENT_LENGTH = 1000; @@ -116,6 +139,7 @@ const passageMatchSchema = z.object({ question: z.string(), answer: z.string(), distance: z.number(), + relevance: z.number().optional(), }) satisfies z.ZodType; const searchNotesOutput = z.object({ @@ -222,13 +246,21 @@ function createTools( { search, findPassages }: Omit, visitorMessages: VisitorMessage[], ) { + // How many searches this answer had left after each one, by tool call, so + // each result can say so: in the baseline (2026-10-20) gpt-oss lost count, + // asked for a fourth search after the third and ended without words. + let searches = 0; + const searchesLeft = new Map(); return { searchNotes: tool({ description: - "Searches the six interviews. Returns the five passages closest in meaning to the query, closest first: each is the researcher's question and one interviewee's whole answer. They are candidates, so some may not be about the query at all.", + "Searches the six interviews. Returns up to five passages about the query, most relevant first: each is the researcher's question and one interviewee's whole answer. When no passage is about the query, it returns none and says so.", inputSchema: searchNotesInput, outputSchema: searchNotesOutput, - async execute({ query }) { + async execute({ query }, { toolCallId }) { + // Counted before the search, as `prepareStep` counts the calls. + searches += 1; + searchesLeft.set(toolCallId, Math.max(0, MAX_SEARCHES - searches)); try { return { passages: await search(query) }; } catch (error) { @@ -238,18 +270,28 @@ function createTools( throw new SearchFailedError(); } }, - // The model gets what it needs to quote, without the browser's details. - toModelOutput: ({ output }) => ({ - type: "json", - value: { - passages: output.passages.map((passage) => ({ - passage: passageAnchor(passage.slug, passage.position), - interviewee: passage.participant, - question: passage.question, - answer: passage.answer, - })), - }, - }), + // The model gets what it needs to quote, without the browser's details, + // and how many searches it has left. + toModelOutput: ({ toolCallId, output }) => { + const left = searchesLeft.get(toolCallId) ?? 0; + const notes = [ + ...(output.passages.length === 0 ? [NOTHING_RELEVANT] : []), + ...(left === 0 ? [LAST_SEARCH] : []), + ]; + return { + type: "json", + value: { + passages: output.passages.map((passage) => ({ + passage: passageAnchor(passage.slug, passage.position), + interviewee: passage.participant, + question: passage.question, + answer: passage.answer, + })), + searchesLeft: left, + ...(notes.length > 0 ? { note: notes.join(" ") } : {}), + }, + }; + }, }), saveSection: tool({ description: @@ -392,12 +434,28 @@ export async function readChatRequest(body: unknown): Promise { } const messages = validated.data; + const length = messages + .flatMap((message) => message.parts) + .reduce((sum, part) => sum + modelLength(part), 0); + if (length > MAX_CONVERSATION_LENGTH) { + throw new ChatRequestError( + `The conversation is too long: the assistant reads up to ${MAX_CONVERSATION_LENGTH} characters of it.`, + ); + } for (const message of messages) { if (message.role === "system") { throw new ChatRequestError( "The conversation can't change the assistant's instructions.", ); } + if ( + message.role === "assistant" && + textOf(message).length > MAX_ANSWER_LENGTH + ) { + throw new ChatRequestError( + `An answer in the conversation is too long: the assistant's answers hold up to ${MAX_ANSWER_LENGTH} characters.`, + ); + } if (message.role !== "user") continue; if (message.parts.some((part) => part.type !== "text")) { throw new ChatRequestError("Your messages can only hold text."); @@ -418,6 +476,18 @@ export async function readChatRequest(body: unknown): Promise { return messages; } +/** + * A part's characters as the model reads them: a search's or a save's output + * doesn't count, since only its ids or its check report reach the model. + */ +function modelLength(part: ChatMessage["parts"][number]): number { + if (part.type === "text") return part.text.length; + if (part.type === "tool-searchNotes" || part.type === "tool-saveSection") { + return JSON.stringify({ ...part, output: undefined }).length; + } + return JSON.stringify(part).length; +} + /** The visitor's own messages, which their quotes are checked against. */ function visitorMessagesIn(messages: ChatMessage[]): VisitorMessage[] { return messages diff --git a/src/server/measurement/budget.ts b/src/server/measurement/budget.ts index 7897827..032767e 100644 --- a/src/server/measurement/budget.ts +++ b/src/server/measurement/budget.ts @@ -2,6 +2,7 @@ import { existsSync } from "node:fs"; import { appendFile, mkdir, readFile } from "node:fs/promises"; import { dirname } from "node:path"; import { embeddingTokenBound } from "../ai/embedding-input.ts"; +import { RERANK_MODEL } from "../ai/models.ts"; // Careful spending (src/server/AGENTS.md, the budget): every call to // Cloudflare goes through the meter, which counts the neurons as replies come @@ -18,6 +19,8 @@ export const RATES = { chatInput: 31_818, chatOutput: 68_182, embeddingInput: 18_582, + /** bge-reranker-base's input (the same page, read 2026-10-22). */ + rerankInput: 283, } as const; /** @@ -37,6 +40,8 @@ export type Spend = { outputTokens: number; /** Cloudflare reports no tokens for embeddings: the loader's upper bound. */ embeddingTokenBound: number; + /** The reranker's input, an upper bound (`rerankTokenBound`); absent at 0. */ + rerankTokenBound?: number; neurons: number; }; @@ -55,6 +60,10 @@ function add(total: Spend, more: Spend): void { total.inputTokens += more.inputTokens; total.outputTokens += more.outputTokens; total.embeddingTokenBound += more.embeddingTokenBound; + if (more.rerankTokenBound) { + total.rerankTokenBound = + (total.rerankTokenBound ?? 0) + more.rerankTokenBound; + } total.neurons += more.neurons; } @@ -78,6 +87,25 @@ export function embeddingSpend(tokenBound: number): Spend { }; } +export function rerankSpend(tokenBound: number): Spend { + return { + ...noSpend(), + rerankTokenBound: tokenBound, + neurons: (tokenBound * RATES.rerankInput) / 1_000_000, + }; +} + +/** + * The most tokens a rerank request can hold: bge-reranker-base reads the + * question and each text as one pair, and its SentencePiece pieces are at + * least one character each, spaces included, so a pair never has more tokens + * than its characters plus its four markers ( … … ). + */ +export function rerankTokenBound(query: string, texts: string[]): number { + const length = (text: string) => [...text].length; + return texts.reduce((sum, text) => sum + length(query) + length(text) + 4, 0); +} + /** * Why Cloudflare refused a request. Quota (429, `QUOTA_CODES`: the day's free * allocation is used up), busy (code 3040: out of capacity) and any other @@ -203,9 +231,11 @@ export function createMeter({ const url = input instanceof Request ? input.url : String(input); const request = `${url} ${typeof init?.body === "string" ? init.body : ""}`; + const reranking = url.endsWith(`/ai/run/${RERANK_MODEL}`); if (url.endsWith("/embeddings")) { count(embeddingSpend(embeddingBound(init?.body))); } + if (reranking) count(rerankSpend(rerankBound(init?.body))); const response = await inner(input, init); const body = parsed(await response.clone().text()); if (response.ok) { @@ -218,8 +248,9 @@ export function createMeter({ const usage = (body as { usage?: Record } | null) ?.usage; if ( - typeof usage?.prompt_tokens === "number" || - typeof usage?.completion_tokens === "number" + !reranking && + (typeof usage?.prompt_tokens === "number" || + typeof usage?.completion_tokens === "number") ) { count( chatSpend( @@ -265,6 +296,21 @@ function embeddingBound(body: RequestInit["body"]): number { ); } +/** The upper bound of a rerank request's tokens, from what it sends. */ +function rerankBound(body: RequestInit["body"]): number { + if (typeof body !== "string") return 0; + const { query, contexts } = (parsed(body) ?? {}) as { + query?: unknown; + contexts?: unknown; + }; + if (typeof query !== "string" || !Array.isArray(contexts)) return 0; + const texts = contexts.flatMap((context: unknown) => { + const text = (context as { text?: unknown } | null)?.text; + return typeof text === "string" ? [text] : []; + }); + return rerankTokenBound(query, texts); +} + function parsed(text: string): unknown { try { return JSON.parse(text) as unknown; @@ -321,19 +367,28 @@ export function totals( /** * What the next answer to a request is expected to cost: the mean of its - * counted answers so far, or the estimate's figure before there are any. + * counted answers in the first of `prefer` that has any (the measurement + * itself, then the one it's compared with), since a change can move the cost + * (`reasoning-high` cost about twice `prompt-v5`'s, 2026-10-21); otherwise the + * mean of its counted answers so far, or the estimate's figure before there + * are any at all. */ export function expectedNeurons( request: { name: string; continues: string | null }, entries: LedgerEntry[], + prefer: (string | undefined)[] = [], ): number { const counted = entries.filter( (entry) => entry.request === request.name && entry.outcome === "counted", ); - if (counted.length > 0) { + const basis = + prefer + .filter((name) => name !== undefined) + .map((name) => counted.filter((entry) => entry.measurement === name)) + .find((own) => own.length > 0) ?? counted; + if (basis.length > 0) { return ( - counted.reduce((sum, entry) => sum + entry.spend.neurons, 0) / - counted.length + basis.reduce((sum, entry) => sum + entry.spend.neurons, 0) / basis.length ); } if (request.name === "A") return FIRST_GUESS.A; diff --git a/src/server/measurement/calibrate.ts b/src/server/measurement/calibrate.ts new file mode 100644 index 0000000..8338156 --- /dev/null +++ b/src/server/measurement/calibrate.ts @@ -0,0 +1,136 @@ +import { existsSync } from "node:fs"; +import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { passageAnchor } from "../../lib/transcripts.ts"; +import type { Embedder } from "../ai/embedder.ts"; +import type { Reranker } from "../ai/reranker.ts"; +import { embeddingInput } from "../notes/load-notes.ts"; +import { RERANK_CANDIDATES } from "../search.ts"; +import { rerankTokenBound } from "./budget.ts"; +import { rankPassages, type PassageCopy } from "./passages.ts"; + +// The reranker's calibration (src/server/AGENTS.md, *The reranker*): the +// searches gpt-oss made in the measurements so far, each with the candidates +// the search would rerank, scored by the reranker, so its threshold is set from +// our own question and passage pairs, as Cohere advises for any reranker +// (https://docs.cohere.com/docs/reranking-best-practices, read 2026-10-22). On +// the real reranker only through `npm run measure -- --calibrate-rerank`, +// under the meter and into the ledger. + +/** The calibration's folder and its name in the ledger. */ +export const CALIBRATION = "rerank-calibration"; + +export type CalibrationPair = { + query: string; + /** The passage id, such as henk-3. */ + passage: string; + distance: number; + /** The reranker's relevance, 0 to 1. */ + relevance: number; +}; + +/** Every distinct query gpt-oss searched with in the records under `root`. */ +export async function recordedQueries(root: string): Promise { + const queries = new Set(); + const visit = (value: unknown): void => { + if (Array.isArray(value)) return value.forEach(visit); + if (typeof value !== "object" || value === null) return; + const { toolName, input } = value as { + toolName?: unknown; + input?: unknown; + }; + const query = (input as { query?: unknown } | null | undefined)?.query; + if (toolName === "searchNotes" && typeof query === "string") { + queries.add(query.trim()); + } + Object.values(value).forEach(visit); + }; + for (const measurement of await folders(root)) { + for (const day of await folders(join(root, measurement))) { + if (!/^\d{4}-\d{2}-\d{2}$/.test(day)) continue; + const dir = join(root, measurement, day); + for (const file of await readdir(dir)) { + if (!file.endsWith(".json")) continue; + const record = JSON.parse(await readFile(join(dir, file), "utf8")) as { + answer?: unknown; + }; + visit(record.answer); + } + } + } + return [...queries].filter((query) => query.length > 0).sort(); +} + +async function folders(dir: string): Promise { + if (!existsSync(dir)) return []; + const entries = await readdir(dir, { withFileTypes: true }); + return entries.filter((entry) => entry.isDirectory()).map((e) => e.name); +} + +/** + * The most tokens the calibration can send the reranker: each query with the + * longest candidates it could get. + */ +export function calibrationTokenBound( + queries: string[], + passages: { question: string; answer: string }[], +): number { + const longest = passages + .map((passage) => embeddingInput(passage.question, passage.answer)) + .sort((a, b) => b.length - a.length) + .slice(0, RERANK_CANDIDATES); + return queries.reduce( + (sum, query) => sum + rerankTokenBound(query, longest), + 0, + ); +} + +/** Each query's candidates, as the search would read them, with their scores. */ +export async function scorePairs( + queries: string[], + copy: PassageCopy, + { embedder, reranker }: { embedder: Embedder; reranker: Reranker }, +): Promise { + if (queries.length === 0) return []; + const embeddings = await embedder.embed(queries); + const pairs: CalibrationPair[] = []; + for (const [index, query] of queries.entries()) { + const candidates = rankPassages( + copy.passages, + embeddings[index]!, + RERANK_CANDIDATES, + ); + const relevance = await reranker.rank( + query, + candidates.map((passage) => + embeddingInput(passage.question, passage.answer), + ), + ); + for (const [position, passage] of candidates.entries()) { + pairs.push({ + query, + passage: passageAnchor(passage.slug, passage.position), + distance: passage.distance, + relevance: relevance[position]!, + }); + } + } + return pairs; +} + +/** Saves the pairs once: an existing file is an earlier calibration's. */ +export async function savePairs( + root: string, + reranker: Reranker, + pairs: CalibrationPair[], +): Promise { + const dir = join(root, CALIBRATION); + await mkdir(dir, { recursive: true }); + const file = join(dir, `${new Date().toISOString().slice(0, 10)}.json`); + await writeFile( + file, + `${JSON.stringify({ model: reranker.model, pairs }, null, 2)}\n`, + { flag: "wx" }, + ); + return file; +} diff --git a/src/server/measurement/cli.ts b/src/server/measurement/cli.ts index 29bbb7a..d0f6ef6 100644 --- a/src/server/measurement/cli.ts +++ b/src/server/measurement/cli.ts @@ -4,9 +4,29 @@ import { parseArgs, parseEnv } from "node:util"; import { PROVIDER_KEY_FILE } from "../../lib/env-guard.ts"; import { createCloudflareChatModel } from "../ai/cloudflare.ts"; import { createCloudflareEmbedder } from "../ai/cloudflare-embedder.ts"; +import { createCloudflareReranker } from "../ai/cloudflare-reranker.ts"; import { createFakeChatModel } from "../ai/fake-chat-model.ts"; import { createFakeEmbedder } from "../ai/fake-embedder.ts"; -import { EMBEDDING_MODEL } from "../ai/models.ts"; +import { embeddingTokenBound } from "../ai/embedding-input.ts"; +import { EMBEDDING_MODEL, RERANK_MODEL } from "../ai/models.ts"; +import { RERANK_CANDIDATES } from "../search.ts"; +import { + appendLedger, + createMeter, + RATES, + readLedger, + totals, +} from "./budget.ts"; +import { + CALIBRATION, + calibrationTokenBound, + recordedQueries, + savePairs, + scorePairs, + type CalibrationPair, +} from "./calibrate.ts"; +import { acquireLock } from "./lock.ts"; +import { makeCopy, readNotes } from "./passages.ts"; import { MEASUREMENTS_DIR } from "./records.ts"; import { gitIn, writeReport } from "./report.ts"; import { REQUESTS, findRequest, type RequestName } from "./requests.ts"; @@ -147,6 +167,8 @@ export function readToken(text: string): { } function environment(fake: boolean, planOnly: boolean): BatchEnvironment { + // No reranker, as in the app's search (`search.ts`, `src/server/AGENTS.md`, + // *The reranker*). const common = { notesDir: join(process.cwd(), "notes"), repoRoot: process.cwd(), @@ -213,9 +235,105 @@ async function report(argv: string[]): Promise { return 0; } +/** + * `npm run measure -- --calibrate-rerank [--plan]`: the real reranker's scores + * on our own pairs, for its threshold (`calibrate.ts`). It says what it sends + * and its most neurons first; `--plan` stops there. + */ +async function calibrate(argv: string[]): Promise { + const plan = argv.includes("--plan"); + const root = join(process.cwd(), MEASUREMENTS_DIR); + const notesDir = join(process.cwd(), "notes"); + const ledgerFile = join(root, "spend.jsonl"); + const queries = await recordedQueries(root); + const notes = await readNotes(notesDir, EMBEDDING_MODEL); + const most = + (calibrationTokenBound(queries, notes) * RATES.rerankInput + + queries.reduce((sum, query) => sum + embeddingTokenBound(query), 0) * + RATES.embeddingInput) / + 1_000_000; + const sofar = totals(await readLedger(ledgerFile), new Date()); + console.log( + `The reranker's calibration: ${queries.length} recorded searches, each with its ${Math.min(RERANK_CANDIDATES, notes.length)} candidates, to ${RERANK_MODEL}; at most ${Math.ceil(most)} neurons.`, + ); + console.log( + `So far today (UTC): ${Math.round(sofar.day)} neurons; this month: ${Math.round(sofar.month)}.`, + ); + if (plan) { + console.log("Plan only: nothing was sent."); + return 0; + } + if (process.env.npm_lifecycle_event !== "measure") { + throw new Error( + "The calibration on the real reranker runs only as `npm run measure`, after the environment guard.", + ); + } + const settings = readToken(readFileSync(PROVIDER_KEY_FILE, "utf8")); + const lock = acquireLock(); + const meter = createMeter({ cap: most * 1.5 }); + try { + const embedder = createCloudflareEmbedder({ + ...settings, + fetch: meter.fetch(), + }); + const reranker = createCloudflareReranker({ + ...settings, + fetch: meter.fetch(), + }); + const { copy, embedded } = await makeCopy({ + notesDir, + file: join(root, "passages.json"), + embedder, + }); + if (embedded > 0) { + throw new Error( + "The passages' copy was out of date; run a measurement first, which records its cost.", + ); + } + meter.startAnswer(); + let pairs: CalibrationPair[]; + try { + pairs = await scorePairs(queries, copy, { embedder, reranker }); + } catch (error) { + await appendLedger(ledgerFile, { + time: new Date().toISOString(), + measurement: CALIBRATION, + what: "the reranker's calibration", + outcome: "not run", + spend: meter.answer().spend, + }); + throw error; + } + const { spend } = meter.answer(); + await appendLedger(ledgerFile, { + time: new Date().toISOString(), + measurement: CALIBRATION, + what: "the reranker's calibration", + outcome: "counted", + spend, + }); + const file = await savePairs(root, reranker, pairs); + for (const query of queries) { + const best = pairs + .filter((pair) => pair.query === query) + .sort((a, b) => b.relevance - a.relevance) + .slice(0, 5) + .map((pair) => `${pair.passage} ${pair.relevance.toFixed(3)}`); + console.log(`${JSON.stringify(query)}: ${best.join(", ")}`); + } + console.log( + `${pairs.length} pairs, ${Math.round(spend.neurons * 100) / 100} neurons; saved in ${file}.`, + ); + return 0; + } finally { + lock.release(); + } +} + export async function main(argv: string[]): Promise { try { if (argv[0] === "--report") return await report(argv.slice(1)); + if (argv[0] === "--calibrate-rerank") return await calibrate(argv.slice(1)); const { fake, ...options } = parseOptions(argv); const result = await runBatch(options, environment(fake, options.planOnly)); return result.stopped ? 1 : 0; diff --git a/src/server/measurement/passages.ts b/src/server/measurement/passages.ts index 6cb8f20..50ea218 100644 --- a/src/server/measurement/passages.ts +++ b/src/server/measurement/passages.ts @@ -8,6 +8,7 @@ import { } from "../../lib/transcripts.ts"; import type { Embedder } from "../ai/embedder.ts"; import { embeddingTokenBound } from "../ai/embedding-input.ts"; +import type { Reranker } from "../ai/reranker.ts"; import type { ChatDependencies } from "../chat.ts"; import { contentHash, embeddingInput } from "../notes/load-notes.ts"; import { parseInterview } from "../notes/parse-interview.ts"; @@ -205,6 +206,7 @@ export function rankPassages( export function copySources( copy: PassageCopy, embedder: Embedder, + reranker?: Reranker, ): Omit { return { search: (query) => @@ -212,6 +214,7 @@ export function copySources( embedder, findPassages: async (embedding, count) => rankPassages(copy.passages, embedding, count), + ...(reranker ? { reranker } : {}), }), findPassages: async (ids): Promise => { const wanted = new Set(ids); diff --git a/src/server/measurement/report.ts b/src/server/measurement/report.ts index f1ecc97..e36eeb6 100644 --- a/src/server/measurement/report.ts +++ b/src/server/measurement/report.ts @@ -247,7 +247,7 @@ export function setupValues(runs: ScoredRun[]): Map> { add("code", codeValue(setup.code)); add( "search", - `${setup.search.kind} (passages ${setup.search.passagesFingerprint.slice(0, 12)})`, + `${setup.search.kind} (passages ${setup.search.passagesFingerprint.slice(0, 12)})${setup.search.reranker ? `, reranked by ${setup.search.reranker.model} at ${setup.search.reranker.threshold}` : ""}`, ); add( "libraries", diff --git a/src/server/measurement/runner.ts b/src/server/measurement/runner.ts index 428081a..f089ce4 100644 --- a/src/server/measurement/runner.ts +++ b/src/server/measurement/runner.ts @@ -6,6 +6,7 @@ import type { CheckedSection } from "../../lib/prd.ts"; import { tally } from "../../lib/prd.ts"; import type { ChatModel } from "../ai/chat-model.ts"; import type { Embedder } from "../ai/embedder.ts"; +import type { Reranker } from "../ai/reranker.ts"; import { measureAnswer, type MeasuredAnswer } from "./answer.ts"; import { appendLedger, @@ -109,6 +110,11 @@ export type BatchEnvironment = { chatModel(fetch: typeof globalThis.fetch): ChatModel; /** The search's embedder, its HTTP calls through `fetch`. */ embedder(fetch: typeof globalThis.fetch): Embedder; + /** + * The search's reranker, its HTTP calls through `fetch`. Tests leave it out + * unless they test it, and the search then returns the five closest. + */ + reranker?(fetch: typeof globalThis.fetch): Reranker; /** Which embedder `embedder` returns, without building it. */ search: { kind: "fake" | "real"; model: string }; /** What `writeRecord` refuses and hides. */ @@ -314,12 +320,14 @@ async function capturingLogs( /** * How the answer ended: "not run" when a quota, busy or per-minute refusal - * was never recovered by a retry, or the cap cut it off. A refusal the AI - * SDK's retry got past leaves a whole answer, which counts; the record keeps - * every refusal either way. + * was never recovered by a retry, a model call ended with a degenerate reply + * (`DegenerateReplyError`, `ai/cloudflare.ts`), or the cap cut it off. A + * refusal the AI SDK's retry got past leaves a whole answer, which counts; the + * record keeps every refusal either way. */ function outcomeOf( unrecovered: Refusal[], + degenerate: boolean, stoppedAtCap: boolean, ): Outcome & { refusal?: Refusal["kind"] } { const refusal = unrecovered.find((one) => one.kind !== "other"); @@ -332,6 +340,7 @@ function outcomeOf( }[refusal.kind]; return { kind: "not run", reason: why, refusal: refusal.kind }; } + if (degenerate) return { kind: "not run", reason: "degenerate reply" }; if (stoppedAtCap) { return { kind: "not run", reason: "the batch passed its cap partway" }; } @@ -394,7 +403,12 @@ export async function runBatch( env.search.kind === "real" ? embeddingSpend(copy.tokenBound).neurons : 0; const expectedOf = (runs: PlannedRun[]) => runs.reduce( - (sum, { request }) => sum + expectedNeurons(request, ledger), + (sum, { request }) => + sum + + expectedNeurons(request, ledger, [ + options.measurement, + options.compareWith, + ]), 0, ); // The cap covers the answers that can go now; one that waits for its saved @@ -554,10 +568,16 @@ async function sendPlanned({ `The search's copy: embedded ${embedded} passages, ${round(spend.neurons)} neurons (an upper bound).`, ); } - const sources = copySources(copy, embedder); + const reranker = env.reranker?.(meter.fetch(env.fetch)); + const sources = copySources(copy, embedder, reranker); const search = { kind: env.search.kind, passagesFingerprint: copy.fingerprint, + ...(reranker + ? { + reranker: { model: reranker.model, threshold: reranker.threshold }, + } + : {}), }; const skipped: PlannedRun[] = []; @@ -589,6 +609,10 @@ async function sendPlanned({ const { spend, refusals, unrecovered, stoppedAtCap } = meter.answer(); let outcome: Outcome & { refusal?: Refusal["kind"] } = outcomeOf( unrecovered, + // The recorder keeps the error's name. + measured.modelCalls.some( + (call) => call.error?.name === "DegenerateReplyError", + ), stoppedAtCap, ); const time = new Date(measured.answer.startedAt); diff --git a/src/server/measurement/setup.ts b/src/server/measurement/setup.ts index 6c578aa..8815676 100644 --- a/src/server/measurement/setup.ts +++ b/src/server/measurement/setup.ts @@ -15,6 +15,8 @@ export type SearchSetup = { kind: "fake" | "real"; /** Of the passages the search ranks, so a change in the notes shows. */ passagesFingerprint: string; + /** The reranker and its threshold, when the search reranks. */ + reranker?: { model: string; threshold: number }; }; export type RunSetup = { diff --git a/src/server/prompts/system-prompt.ts b/src/server/prompts/system-prompt.ts index 4e03f6f..65823ec 100644 --- a/src/server/prompts/system-prompt.ts +++ b/src/server/prompts/system-prompt.ts @@ -6,75 +6,55 @@ import "server-only"; // - https://github.com/TechNomadCode/AI-Product-Development-Toolkit/blob/cb7956e66a7ad2c4cd3c8ca07e399c2d1c269607/planning-prompts/2-prd/prd.md // - https://github.com/TechNomadCode/AI-Product-Development-Toolkit/blob/cb7956e66a7ad2c4cd3c8ca07e399c2d1c269607/planning-prompts/1-user-research/research-to-prd.md // -// Kept from the template: the role, its rules, the check-in before each new -// section and the nine sections. Changed: the brain dump became the -// interviews, read with searchNotes, and the assistant drafts before it asks -// instead of asking first. Added: traceability, saving each section with -// saveSection (whose quotes the app checks), citations, contradictions and -// never inventing. Markdown headings became plain labels, because the chat -// shows plain text and a model copies the style it's given. +// Kept from the template: the role, the check-in before each new section and +// the nine sections. Changed: the brain dump became the interviews, read with +// searchNotes, and the assistant drafts before it asks instead of asking +// first. Added: traceability, saving each section with saveSection (whose +// quotes the app checks), citations, contradictions and never inventing. +// Markdown headings became plain labels, because the chat shows plain text +// and a model copies the style it's given. +// +// Version 5 is version 3 cut by about a quarter (every call resends it): the +// template's general advice is gone, and a claim that users asked for +// something no interview says is recorded as the product manager's own +// decision, in their words, never as the users'. In the baseline gpt-oss +// cited a passage about something related, and left "no interview says +// this" out of the reply after a save, so rule 7 now names both. +// +// Version 6 is version 5 with the search described as it works since the +// reranker (search.ts): up to five passages about the question, or none. // // Any change to the text gets the next version (src/server/AGENTS.md). export const SYSTEM_PROMPT = { - version: 3, + version: 6, text: `ROLE: You are an expert Product Manager assistant and requirements analyst. Act as a specialized agent focused solely on eliciting product requirements. Respond with the perspective of an expert in product requirements gathering. GOAL: -Collaborate with the product manager you are chatting with to create a comprehensive draft Product Requirements Document (PRD) for a shared household grocery list app, grounded in a researcher's interviews, ensuring alignment with their vision at each stage. - -PROCESS & KEY RULES: -1. Cross-reference the interviews and everything the product manager tells you, step-by-step, to ensure complete coverage and identify any contradictions or inconsistencies. -2. Ask only what the interviews don't say. Never ask the product manager what the interviewees think, want or do, and never ask the researcher's questions again: search instead. When a section needs something no passage says, such as a number for a goal or a choice between options, draft what the interviews support first, then ask specific, targeted questions, one to three at a time, as short bullet points. -3. The first time you ask, explain in one sentence that the interviews don't cover this, so the product manager's answer will be recorded as their own decision. -4. Anticipate likely follow-up questions needed for a comprehensive PRD. Focus only on this PRD, and decline anything unrelated in one sentence. -5. Prompt the product manager to consider multiple perspectives (like different household members or edge cases) where relevant. -6. Ask for quantification using metrics or numbers where appropriate, especially for goals or success metrics. -7. Help the product manager think through aspects they might have missed, guiding towards the PRD structure below. -8. User-centered check-in: after you draft a section, ask the product manager to confirm it or ask for changes, and name the section you would draft next. Move on only once they confirm; when they ask for changes, draft the whole section again. Before a key interpretation of their input, briefly state your understanding and explicitly ask for confirmation. Example: "Does this capture it? If so, shall we move on to Goals / Objectives?" -9. If the product manager's input is unclear, ask for clarification. -10. Follow these instructions precisely and provide unbiased, neutral guidance. +With the product manager you are chatting with, draft a Product Requirements Document (PRD) for a shared household grocery list app, grounded in a researcher's interviews. Decline anything unrelated in one sentence. THE INTERVIEWS (instead of a brain dump): -A researcher interviewed six households about a shared household grocery list app. The interviews are finished records, and you see them only by searching: -- Before you say what the interviewees think, want or do, call searchNotes. Search with a short question in your own words, one topic at a time. You have three searches for one reply; after them the search is switched off, so write your reply from the passages you have. -- Each search returns the five closest passages, closest first: the passage id (such as henk-4), the interviewee, the researcher's question and the interviewee's whole answer. They are candidates, not answers: use a passage only if its answer says what you need, and say plainly when none does. +A researcher interviewed six households. You see the interviews only by searching: +- Before you say what the interviewees think, want or do, call searchNotes with a short question in your own words, one topic at a time. You have three searches for one reply; after them the search is switched off, so write your reply from the passages you have. Each search result says how many you have left. +- Each search returns up to five passages about your question, most relevant first: the passage id (such as henk-4), the interviewee, the researcher's question and the interviewee's answer. When none is about it, the result says so. Use a passage only if its answer says what you need. - Searches from your earlier replies show only each passage's id and interviewee. To quote one of those passages again, search for it again. -YOUR TASK NOW: -Draft before you ask. When the product manager asks for a section, or just to begin, search the interviews, then draft and save that section straight away, starting with Introduction / Overview unless they ask for another. Ask nothing before that first draft, then check in. - -DESIRED PRD STRUCTURE (draft one section at a time, in this order): -* Introduction / Overview -* Goals / Objectives: SMART goals if possible -* Target Audience / User Personas -* User Stories / Use Cases -* Functional Requirements -* Non-Functional Requirements: performance, security, usability, etc. -* Design Considerations / Mockups: mention if available or needed -* Success Metrics -* Open Questions / Future Considerations - -TONE & CONSTRAINTS: -* Maintain a clear, professional, inquisitive, and helpful tone. -* Use simple, non-technical language where possible, unless the product manager provides technical detail. -* Write in English, in plain text: the chat shows Markdown symbols such as # and ** as they are. - -TRACEABILITY: -* Every statement traces to its source: an interview passage, or the product manager's own message in this chat when it records their decision. -* Keep what the interviewees said apart from what it may mean. A statement that infers something from passages, rather than restating them, starts with "Interpretation:" and cites the passages it builds on. -* Describe the six households, not users in general: name the interviewees or say how many of the six. Don't claim more than six interviews can show. -* An assumption that neither the interviews nor the product manager support is never written as a finding or a requirement: it becomes an open question. +RULES: +1. Draft before you ask. When the product manager asks for a section, or just to begin, search, then draft and save that section, starting with Introduction / Overview unless they ask for another. Ask nothing before that first draft. +2. Ask only what the interviews don't say. Never ask what the interviewees think, want or do, and never ask the researcher's questions again: search instead. For a number, a target or a choice no passage gives, ask one to three short questions. +3. The first time you ask, explain in one sentence that the interviews don't cover this, so the product manager's answer will be recorded as their own decision. +4. Every statement traces to its source: an interview passage, or the product manager's own message in this chat when it records their decision. Describe the six households, not users in general. +5. A statement that infers something rather than restating passages starts with "Interpretation:" and cites the passages it builds on. An assumption that neither the interviews nor the product manager support is never written as a finding or a requirement: it becomes an open question. +6. When interviewees disagree, state each view as its own statement with its own citations, then add a statement that starts with "Open question:", names the choice and cites both views. Don't pick a side or blend the views. +7. Never invent interviewees, quotes, numbers or findings. When the product manager says users asked for something, search for it. If no passage names it, your reply says plainly that none of the six interviews does, also after a save. A passage about something related doesn't count, so never cite an interview for it. Since they asked for it, save it as their own decision, citing their message and worded as theirs ("The product manager wants ..."), never as something users or households asked for. +8. Check in: after you draft a section, ask the product manager to confirm it or ask for changes, and name the section you would draft next. Move on only once they confirm; when they ask for changes, draft the whole section again. SAVING SECTIONS: -Save every section you draft with saveSection, never as text in your reply: the app checks each quote against its source and shows the product manager the section with the results. -* A section is a list of statements: each finding, requirement or open question is its own statement, with its citations. A citation names its source, the passage id or "chat" for the product manager's own words in this chat, and quotes it. -* Save once per reply, after your searches. Then the tools are switched off: your reply says in a sentence or two what the check found, then checks in (rule 8). -* When the check marks a quote unverified or a statement that cites nothing, say which, and offer to fix it. +Save every section you draft with saveSection, never as text in your reply: the app checks each quote and shows the product manager the section with the results. Each finding, requirement or open question is its own statement with its citations; a citation names the passage id, or "chat" for the product manager's own words, and quotes it. Save once per reply, after your searches. Then the tools are switched off: your reply says in a sentence or two what the check found, then checks in (rule 8). When the check marks a quote unverified or a statement that cites nothing, say which, and offer to fix it. CITATIONS: -* A quote is short, from a few words up to one sentence, and copied exactly: never translated, shortened inside or tidied. +* A quote is a few words up to one sentence, copied exactly from one place: never shortened inside, joined or tidied. * An interview quote comes from the interviewee's answer, never from the researcher's question. A quote from the product manager comes from their own messages, never from yours. * Cite a passage only for what its words say. A passage that comes close is not the same thing. * Every statement cites at least one source, except in Open Questions / Future Considerations. @@ -82,9 +62,17 @@ CITATIONS: - (, : "") - (You, in this chat: "") -CONTRADICTIONS: -When interviewees disagree, state each view as its own statement with its own citations, then add a statement that starts with "Open question:", names the choice to make and cites both views. Don't pick a side or blend the views. List these open questions again in Open Questions / Future Considerations. +PRD STRUCTURE (one section at a time, in this order): +* Introduction / Overview +* Goals / Objectives +* Target Audience / User Personas +* User Stories / Use Cases +* Functional Requirements +* Non-Functional Requirements +* Design Considerations / Mockups +* Success Metrics +* Open Questions / Future Considerations -NEVER INVENT: -Never invent interviewees, quotes, numbers or findings. When the product manager says users asked for something, search for it. If no passage says it, tell them plainly that none of the six interviews does, and never cite an interview for it. If they still want it in the PRD, record it as their decision, citing their message, or as an open question.`, +STYLE: +Write in English, in plain text with simple words: the chat shows Markdown symbols such as # and ** as they are.`, }; diff --git a/src/server/search.ts b/src/server/search.ts index 5dce370..375031a 100644 --- a/src/server/search.ts +++ b/src/server/search.ts @@ -4,24 +4,40 @@ import type { PassageMatch } from "@/lib/transcripts"; import type { Embedder } from "./ai/embedder.ts"; import { EMBEDDING_DIMENSIONS } from "./ai/models.ts"; import { createEmbedder } from "./ai/providers.ts"; +import type { Reranker } from "./ai/reranker.ts"; +import { embeddingInput } from "./notes/load-notes.ts"; import { createReadClient } from "./db/clients.ts"; import { findClosestPassages } from "./db/search.ts"; import { providerSettings, readSearchEnv } from "./env.ts"; -// The interview search: a question in, the passages closest in meaning out, -// each with the interviewee's name and the researcher's question. It embeds -// the question with the passages' model (fake locally and in CI, Cloudflare -// Workers AI on staging and production), then the database ranks the passages. +// The interview search: a question in, the passages about it out, each with +// the interviewee's name and the researcher's question. It embeds the question +// with the passages' model (fake locally and in CI, Cloudflare Workers AI on +// staging and production), and the database ranks the passages by closeness. +// A reranker can then read the question with each of the closest and keep +// those relevant enough, best first, as Cloudflare's AI Search and Haystack's +// rankers do; it's off, since bge-reranker-base neither separated nor ordered +// our passages better (`src/server/AGENTS.md`, *The reranker*). /** Longer input is usually a whole message, which blurs what to look for. */ export const MAX_QUERY_LENGTH = 500; /** - * How many passages one search returns: always the five closest, even when - * none of them is close. `distance` says how close each one came. + * The most passages one search returns. Without a reranker, always the five + * closest, even when none of them is close; `distance` says how close each + * one came. */ export const SEARCH_RESULT_COUNT = 5; +/** + * How many of the closest passages the reranker reads: more than the 42 there + * are, so it reads every one, for about 1.3 neurons a search, and the + * embedder's ranking no longer decides which passages it sees. Retrieving more + * than are returned is the pattern Anthropic's contextual retrieval and + * Cloudflare's AI Search follow. + */ +export const RERANK_CANDIDATES = 50; + export const searchQuerySchema = z .string() .trim() @@ -45,9 +61,14 @@ export class SearchQueryError extends Error { export type SearchDependencies = { embedder: Embedder; findPassages(embedding: number[], count: number): Promise; + /** Tests leave it out unless they test it: the five closest, as before. */ + reranker?: Reranker; }; -/** The passages closest to `query`, closest first. */ +/** + * The passages about `query`, most relevant first: with a reranker, at most + * five that reach its threshold, maybe none; without one, the five closest. + */ export async function searchInterviews( query: string, dependencies?: SearchDependencies, @@ -59,9 +80,29 @@ export async function searchInterviews( ); } - const { embedder, findPassages } = dependencies ?? connect(); + const { embedder, findPassages, reranker } = dependencies ?? connect(); const [embedding] = await embedder.embed([parsed.data]); - return findPassages(checked(embedding), SEARCH_RESULT_COUNT); + if (!reranker) return findPassages(checked(embedding), SEARCH_RESULT_COUNT); + + const candidates = await findPassages(checked(embedding), RERANK_CANDIDATES); + // The text the passages were embedded from: the researcher's question, then + // the answer, as Haystack's rankers put a document's fields before its text. + const relevance = await reranker.rank( + parsed.data, + candidates.map((passage) => + embeddingInput(passage.question, passage.answer), + ), + ); + if (relevance.length !== candidates.length) { + throw new Error( + `The reranker scored ${relevance.length} of ${candidates.length} passages.`, + ); + } + return candidates + .map((passage, index) => ({ ...passage, relevance: relevance[index]! })) + .filter((passage) => passage.relevance >= reranker.threshold) + .sort((a, b) => b.relevance - a.relevance || a.distance - b.distance) + .slice(0, SEARCH_RESULT_COUNT); } /** @@ -85,12 +126,17 @@ function checked(embedding: number[] | undefined): number[] { return embedding; } -/** The real dependencies, from the settings: the read key and the embedder. */ +/** + * The real dependencies, from the settings: the read key and the embedder. + * No reranker (`src/server/AGENTS.md`, *The reranker*): `createReranker` in + * `ai/providers.ts` gives one back here. + */ function connect(): SearchDependencies { const env = readSearchEnv(); const client = createReadClient(env); + const settings = providerSettings(env); return { - embedder: createEmbedder(providerSettings(env)), + embedder: createEmbedder(settings), findPassages: (embedding, count) => findClosestPassages(client, embedding, count), }; diff --git a/tests/unit/chat.test.ts b/tests/unit/chat.test.ts index 77169d5..194501f 100644 --- a/tests/unit/chat.test.ts +++ b/tests/unit/chat.test.ts @@ -16,8 +16,11 @@ import { createFakeChatModel, FAKE_FAILURES, } from "@/server/ai/fake-chat-model"; +import { POST } from "@/app/api/chat/route"; import { ChatRequestError, + MAX_ANSWER_LENGTH, + MAX_CONVERSATION_LENGTH, MAX_MESSAGES, readChatRequest, streamAnswer, @@ -398,6 +401,7 @@ describe("answering in the chat", () => { answer: DAAN.answer, }, ], + searchesLeft: 2, }, }, }), @@ -405,6 +409,66 @@ describe("answering in the chat", () => { }); }); + it("tells the model plainly when no passage is about its search", async () => { + const { model, calls } = modelReplying( + searchFor("dark mode"), + reply("None of the six interviews mentions dark mode."), + ); + + await answer( + [visitor("Users asked for a dark mode.")], + model, + async () => [], + ); + + const toolMessage = calls[1]?.prompt.at(-1); + expect(toolMessage).toEqual({ + role: "tool", + content: [ + expect.objectContaining({ + type: "tool-result", + toolName: "searchNotes", + output: { + type: "json", + value: { + passages: [], + searchesLeft: 2, + note: "No passage in the six interviews is about this.", + }, + }, + }), + ], + }); + }); + + it("tells the model after each search how many it has left, and after the last to save or write", async () => { + const { model, calls } = modelSearchingWhileItCan(); + + await answer([visitor("Draft the first section.")], model); + + // The tool result each call sees last: the search before it. + const results = calls.slice(1).map((call) => { + const content = call.prompt.at(-1)?.content; + const part = Array.isArray(content) ? content[0] : undefined; + return part?.type === "tool-result" ? part.output : undefined; + }); + expect(results).toEqual([ + { type: "json", value: expect.objectContaining({ searchesLeft: 2 }) }, + { type: "json", value: expect.objectContaining({ searchesLeft: 1 }) }, + { + type: "json", + value: expect.objectContaining({ + searchesLeft: 0, + note: "That was your last search for this reply. Save the section, or write your reply, from the passages you have.", + }), + }, + ]); + expect(results.slice(0, 2)).not.toContainEqual({ + type: "json", + value: expect.objectContaining({ note: expect.anything() }), + }); + }); + it("tells the model about the interviews and the search before the conversation", async () => { const { model, calls } = modelReplying(reply("Hello.")); @@ -1335,6 +1399,32 @@ describe("reading a chat request", () => { await expect(readChatRequest({ messages })).resolves.toEqual(messages); }); + it("counts a search's passages apart from the conversation's length, since only their ids reach the model", async () => { + const longSearch: ChatMessage = { + id: "assistant-1", + role: "assistant", + parts: [ + { + type: "tool-searchNotes", + toolCallId: "call-1", + state: "output-available", + input: { query: "pings" }, + output: { passages: Array.from({ length: 500 }, () => HENK) }, + }, + ], + }; + const messages = [ + visitor("Who wants pings?"), + longSearch, + visitor("And who doesn't?", "visitor-2"), + ]; + + expect(JSON.stringify(messages).length).toBeGreaterThan( + MAX_CONVERSATION_LENGTH, + ); + await expect(readChatRequest({ messages })).resolves.toEqual(messages); + }); + it.each([ ["a body that isn't an object", "hello", /isn't a conversation/], ["no conversation", {}, /isn't a conversation/], @@ -1387,6 +1477,35 @@ describe("reading a chat request", () => { { messages: [visitor("a".repeat(MAX_MESSAGE_LENGTH + 1))] }, /A message is too long/, ], + [ + "a conversation over its length limit", + { + messages: Array.from( + { + length: + Math.floor(MAX_CONVERSATION_LENGTH / MAX_MESSAGE_LENGTH) + 1, + }, + (_, index) => + visitor("a".repeat(MAX_MESSAGE_LENGTH), `visitor-${index}`), + ), + }, + /The conversation is too long: the assistant reads up to 100000 characters/, + ], + [ + "an answer over its length limit", + { + messages: [ + visitor("Hi"), + { + id: "assistant-1", + role: "assistant", + parts: [{ type: "text", text: "a".repeat(MAX_ANSWER_LENGTH + 1) }], + }, + visitor("And?", "visitor-2"), + ], + }, + /An answer in the conversation is too long/, + ], [ "the assistant's message last", { messages: [visitor("Who wants pings?"), searched] }, @@ -1486,3 +1605,17 @@ describe("reading a chat request", () => { await expect(reading).rejects.toThrow(message); }); }); + +describe("the chat's endpoint", () => { + const post = (body: string) => + POST(new Request("http://localhost/api/chat", { method: "POST", body })); + + it("refuses a body over 300,000 characters before reading it", async () => { + const over = await post("x".repeat(300_001)); + + expect(over.status).toBe(413); + expect(await over.text()).toBe("The conversation is too long to send."); + // At the limit the body is read, and this one isn't JSON. + expect((await post("x".repeat(300_000))).status).toBe(400); + }); +}); diff --git a/tests/unit/cloudflare.test.ts b/tests/unit/cloudflare.test.ts index d6d7112..b3e2263 100644 --- a/tests/unit/cloudflare.test.ts +++ b/tests/unit/cloudflare.test.ts @@ -4,10 +4,12 @@ import { z } from "zod"; import { DEMO_BUSY, QUOTA_USED_UP } from "@/lib/chat"; import { createCloudflareChatModel, + DegenerateReplyError, QuotaExceededError, } from "@/server/ai/cloudflare"; import { createCloudflareEmbedder } from "@/server/ai/cloudflare-embedder"; -import { CHAT_MODEL, EMBEDDING_MODEL } from "@/server/ai/models"; +import { createCloudflareReranker } from "@/server/ai/cloudflare-reranker"; +import { CHAT_MODEL, EMBEDDING_MODEL, RERANK_MODEL } from "@/server/ai/models"; import { streamAnswer } from "@/server/chat"; // The Cloudflare adapter against a recorded fetch: the requests it sends and @@ -290,6 +292,45 @@ describe("the Cloudflare chat model", () => { tool_calls: [expect.objectContaining({ id: "call-1" })], }); }); + + it("sends the search back as Cloudflare's own provider does: reasoning as reasoning, the tool named", async () => { + const replies = [ + chatReply( + { + ...toolCallMessage("call-1", "searchNotes", { + query: "notifications", + }), + reasoning_content: "We need to search.", + }, + "tool_calls", + ), + chatReply({ role: "assistant", content: "Henk wants a ping." }, "stop"), + ]; + let call = 0; + const { sent, model } = chatModel(() => replies[call++]!()); + + const result = streamText({ + model, + prompt: "Who wants a notification for every item someone adds?", + tools: { + searchNotes: tool({ + inputSchema: z.object({ query: z.string() }), + execute: async () => ({ passages: [] }), + }), + }, + stopWhen: isStepCount(2), + }); + + expect(await result.text).toBe("Henk wants a ping."); + const messages = sent[1]?.body.messages as Array>; + expect(messages[1]).toMatchObject({ reasoning: "We need to search." }); + expect(messages[1]).not.toHaveProperty("reasoning_content"); + expect(messages[2]).toMatchObject({ + role: "tool", + tool_call_id: "call-1", + name: "searchNotes", + }); + }); }); /** A whole reply that only calls one tool. */ @@ -639,3 +680,178 @@ describe("the chat when Cloudflare refuses", () => { expect(sent).toHaveLength(1); }); }); + +/** + * Cloudflare's whole reply of 4,096 "!", as recorded in the measurement's + * records (local): only the id, the time, the prompt's tokens and the neurons + * differ between the two. + */ +function degenerateReply( + id: string, + created: number, + promptTokens: number, + neurons: number, +) { + return () => + Response.json({ + id, + object: "chat.completion", + created, + model: CHAT_MODEL, + choices: [ + { + index: 0, + message: { + role: "assistant", + refusal: null, + annotations: null, + audio: null, + function_call: null, + reasoning: null, + reasoning_content: null, + content: "!".repeat(4096), + }, + logprobs: null, + finish_reason: "length", + stop_reason: null, + token_ids: null, + routed_experts: null, + }, + ], + usage: { + prompt_tokens: promptTokens, + completion_tokens: 4096, + total_tokens: promptTokens + 4096, + prompt_tokens_details: { cached_tokens: 0 }, + neurons, + }, + }); +} + +describe("the chat when Cloudflare's reply is degenerate", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it.each([ + [ + "transport D-3's first call, 2026-10-22", + degenerateReply("id-1792629162692", 1792629162, 2682, 364.6085205078125), + ], + [ + "reasoning-high E-2's fourth call, 2026-10-21", + degenerateReply("id-1792605486100", 1792605486, 5815, 464.2943115234375), + ], + ])( + "shows the usual error instead of the reply, without retrying: %s", + async (_, reply) => { + vi.spyOn(console, "info").mockImplementation(() => {}); + const failed = vi.spyOn(console, "error").mockImplementation(() => {}); + const { sent, model } = replaying([reply, reply, reply]); + + expect(await answerTo(model)).toEqual({ + text: "", + errors: ["The assistant couldn't finish its answer."], + }); + expect(sent).toHaveLength(1); + expect(failed).toHaveBeenCalledWith( + "The chat model failed:", + expect.any(DegenerateReplyError), + ); + }, + ); + + it("passes on a reply cut off at the limit when its text is words", async () => { + const { model } = chatModel( + chatReply( + { role: "assistant", content: "The overview covers who!!!!" }, + "length", + ), + ); + + const result = streamText({ model, prompt: "Hi" }); + + expect(await result.text).toBe("The overview covers who!!!!"); + expect(await result.finishReason).toBe("length"); + }); +}); + +describe("the Cloudflare reranker", () => { + /** Cloudflare's reply format for bge-reranker-base: each score by index. */ + const rerankReply = (response: { id: number; score: number }[]) => () => + Response.json({ result: { response }, success: true, errors: [] }); + + function reranker(fetch: typeof globalThis.fetch) { + return createCloudflareReranker({ + accountId: ACCOUNT, + apiToken: "test-token", + fetch, + }); + } + + it("sends the question and every text to the pinned model, and reads each score by its index", async () => { + // Cloudflare's scores for "dark mode" against henk-3 and daan-6 in the + // calibration of 2026-10-22: already between 0 and 1. + const { sent, fetch } = recordingFetch( + rerankReply([ + { id: 1, score: 0.000037782232538990435 }, + { id: 0, score: 0.0007440208573828555 }, + ]), + ); + + const relevance = await reranker(fetch).rank("dark mode", [ + "Black on white or white on black.", + "Same route every week.", + ]); + + expect(sent).toHaveLength(1); + expect(sent[0]?.url).toBe( + `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT}/ai/run/${RERANK_MODEL}`, + ); + expect(sent[0]?.headers.get("authorization")).toBe("Bearer test-token"); + expect(sent[0]?.body).toEqual({ + query: "dark mode", + contexts: [ + { text: "Black on white or white on black." }, + { text: "Same route every week." }, + ], + top_k: 2, + }); + // As sent, never mapped again. + expect(relevance).toEqual([0.0007440208573828555, 0.000037782232538990435]); + }); + + it("sends nothing for no texts", async () => { + const { sent, fetch } = recordingFetch(rerankReply([])); + + await expect(reranker(fetch).rank("dark mode", [])).resolves.toEqual([]); + expect(sent).toHaveLength(0); + }); + + it("fails when a text comes back without a score", async () => { + const { fetch } = recordingFetch(rerankReply([{ id: 0, score: 1 }])); + + await expect( + reranker(fetch).rank("lists", ["One.", "Two."]), + ).rejects.toThrow("Cloudflare's reranker scored 1 of 2 texts."); + }); + + it("stops at the daily quota without retrying", async () => { + const { sent, fetch } = recordingFetch(quota4006); + + await expect( + reranker(fetch).rank("lists", ["One."]), + ).rejects.toBeInstanceOf(QuotaExceededError); + expect(sent).toHaveLength(1); + }); + + it("fails with Cloudflare's status on any other error", async () => { + const { fetch } = recordingFetch(() => + Response.json({ errors: [{ code: 5007 }] }, { status: 400 }), + ); + + await expect(reranker(fetch).rank("lists", ["One."])).rejects.toThrow( + /^Cloudflare's reranker answered 400/, + ); + }); +}); diff --git a/tests/unit/measurement-calibrate.test.ts b/tests/unit/measurement-calibrate.test.ts new file mode 100644 index 0000000..e63c82d --- /dev/null +++ b/tests/unit/measurement-calibrate.test.ts @@ -0,0 +1,107 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { createFakeEmbedder } from "@/server/ai/fake-embedder"; +import { createFakeReranker } from "@/server/ai/fake-reranker"; +import { + CALIBRATION, + calibrationTokenBound, + recordedQueries, + savePairs, + scorePairs, +} from "@/server/measurement/calibrate"; +import { makeCopy } from "@/server/measurement/passages"; +import { RERANK_CANDIDATES } from "@/server/search"; + +// The reranker's calibration on the fakes: which searches it collects, what it +// scores and what it keeps. The real reranker runs only in `npm run measure`. + +let root: string; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "calibrate-")); +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +/** A measured answer's record holding these chunks. */ +async function writeRecord(path: string, chunks: object[]) { + await mkdir(join(root, path, ".."), { recursive: true }); + await writeFile(join(root, path), JSON.stringify({ answer: chunks })); +} + +const search = (query: string) => ({ + type: "tool-input-available", + toolName: "searchNotes", + input: { query }, +}); + +describe("the reranker's calibration", () => { + it("collects every distinct search gpt-oss made in the counted records", async () => { + await writeRecord("baseline/2026-10-19/E-1.json", [ + search("dark mode"), + search(" grocery delivery "), + ]); + await writeRecord("transport/2026-10-22/E-2.json", [ + search("dark mode"), + { type: "tool-input-available", toolName: "saveSection", input: {} }, + ]); + // Not-run records and other folders don't count. + await writeRecord("transport/not-run/2026-10-22/E-3.json", [ + search("offline"), + ]); + + expect(await recordedQueries(root)).toEqual([ + "dark mode", + "grocery delivery", + ]); + }); + + it("scores each search's candidates as the search would read them, and saves them once", async () => { + const embedder = createFakeEmbedder(); + const { copy } = await makeCopy({ + notesDir: join(process.cwd(), "notes"), + file: join(root, "passages.json"), + embedder, + }); + const reranker = createFakeReranker(); + + const pairs = await scorePairs(["notification", "dark mode"], copy, { + embedder, + reranker, + }); + + expect(pairs).toHaveLength( + 2 * Math.min(RERANK_CANDIDATES, copy.passages.length), + ); + expect(pairs[0]).toEqual({ + query: "notification", + passage: expect.stringMatching(/^[a-z]+-\d+$/), + distance: expect.any(Number), + relevance: expect.any(Number), + }); + const file = await savePairs(root, reranker, pairs); + expect(file).toContain(CALIBRATION); + expect(JSON.parse(await readFile(file, "utf8"))).toEqual({ + model: reranker.model, + pairs, + }); + await expect(savePairs(root, reranker, pairs)).rejects.toThrow(/EEXIST/); + }); + + it("bounds what it can send by each search with the longest candidates", () => { + const passages = [ + { question: "Q?", answer: "Short." }, + { question: "Q?", answer: "A much longer answer." }, + ]; + + // The texts are "Q?\nShort." (9 characters) and "Q?\nA much longer + // answer." (24); each pair is the query's characters, the text's and 4. + expect(calibrationTokenBound(["ab", "abc"], passages)).toBe( + 2 + 9 + 4 + (2 + 24 + 4) + (3 + 9 + 4) + (3 + 24 + 4), + ); + }); +}); diff --git a/tests/unit/measurement-runner.test.ts b/tests/unit/measurement-runner.test.ts index 4eecfb5..ec4f6fd 100644 --- a/tests/unit/measurement-runner.test.ts +++ b/tests/unit/measurement-runner.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { existsSync } from "node:fs"; +import { existsSync, readdirSync } from "node:fs"; import { appendFile, cp, @@ -20,7 +20,7 @@ import { createCloudflareChatModel } from "@/server/ai/cloudflare"; import type { Embedder } from "@/server/ai/embedder"; import { createFakeChatModel } from "@/server/ai/fake-chat-model"; import { createFakeEmbedder } from "@/server/ai/fake-embedder"; -import { CHAT_MODEL } from "@/server/ai/models"; +import { CHAT_MODEL, RERANK_MODEL } from "@/server/ai/models"; import { ChatRequestError } from "@/server/chat"; import { classifyRefusal, @@ -428,6 +428,35 @@ describe("the meter", () => { expect(meter.batch().chatCalls).toBe(0); }); + it("counts a rerank request by its question and texts, never as a chat call", async () => { + const meter = createMeter({ cap: 100, spacingMs: 0 }); + const metered = meter.fetch((async () => + Response.json({ + result: { response: [{ id: 0, score: 1 }] }, + usage: { prompt_tokens: 9999 }, + })) as typeof fetch); + + await metered( + `https://api.cloudflare.com/client/v4/accounts/x/ai/run/${RERANK_MODEL}`, + { + method: "POST", + body: JSON.stringify({ + query: "dark mode", + contexts: [{ text: "Black on white." }, { text: "Coffee." }], + }), + }, + ); + + // Each pair: the question's 9 characters, the text's, and 4 markers. + const bound = 9 + 15 + 4 + (9 + 7 + 4); + expect(meter.batch()).toMatchObject({ + chatCalls: 0, + inputTokens: 0, + rerankTokenBound: bound, + }); + expect(meter.batch().neurons).toBeCloseTo((bound * 283) / 1_000_000); + }); + it("totals the UTC day and month, and expects what earlier answers cost", () => { const entry = (time: string, neurons: number): LedgerEntry => ({ time, @@ -458,6 +487,37 @@ describe("the meter", () => { expect(expectedNeurons(findRequest("B"), ledger)).toBe(450); expect(expectedNeurons(findRequest("D"), [])).toBe(785); }); + + it("expects what this measurement's answers cost, or the compared one's", () => { + const entry = (measurement: string, neurons: number): LedgerEntry => ({ + time: "2026-10-21T08:40:00.000Z", + measurement, + what: "D-1", + request: "D", + run: 1, + outcome: "counted", + spend: { + chatCalls: 3, + inputTokens: 0, + outputTokens: 0, + embeddingTokenBound: 0, + neurons, + }, + }); + const ledger = [ + entry("prompt-v5", 200), + entry("prompt-v5", 300), + entry("reasoning-high", 700), + ]; + + const expectedOf = (prefer: (string | undefined)[]) => + expectedNeurons(findRequest("D"), ledger, prefer); + + expect(expectedOf(["reasoning-high", "prompt-v5"])).toBe(700); + expect(expectedOf(["support-check", "reasoning-high"])).toBe(700); + expect(expectedOf(["support-check", undefined])).toBeCloseTo(400); + expect(expectedOf([])).toBeCloseTo(400); + }); }); describe("the lock", () => { @@ -854,6 +914,42 @@ describe("a batch", () => { ); }); + it("counts a degenerate reply as not run, goes on, and sends that run again in a later batch", async () => { + // As Cloudflare sent it twice (transport D-3, reasoning-high E-2). + const env = cloudflareEnv([ + () => + chatReply({ role: "assistant", content: "!".repeat(4096) }, "length"), + ]); + + const result = await runBatch(options(), env); + + expect(result.stopped).toBeNull(); + expect(result.answers).toEqual([ + { + request: "A", + run: 1, + outcome: { kind: "not run", reason: "degenerate reply" }, + }, + ]); + expect(env.sent).toHaveLength(1); + expect(existsSync(join(folder, "baseline", today(), "A-1.json"))).toBe( + false, + ); + expect( + readdirSync(join(folder, "baseline", "not-run", today())), + ).toHaveLength(1); + expect((await ledgerOf()).at(-1)).toMatchObject({ + what: "A-1", + outcome: "not run", + }); + + lines = []; + await runBatch(options({ planOnly: true }), cloudflareEnv([])); + expect(lines).toContain( + " Sent again, since they couldn't count before: A-1.", + ); + }); + it("stops at the daily quota as the /ai/v1 endpoint sends it, code 4006", async () => { // Cloudflare's reply on 2026-10-20; the errors page gives this message code 3036. const quota4006 = () => diff --git a/tests/unit/search.test.ts b/tests/unit/search.test.ts index e76c799..8e881a8 100644 --- a/tests/unit/search.test.ts +++ b/tests/unit/search.test.ts @@ -2,9 +2,12 @@ import { describe, expect, it, vi } from "vitest"; import type { PassageMatch } from "@/lib/transcripts"; import type { Embedder } from "@/server/ai/embedder"; import { createFakeEmbedder } from "@/server/ai/fake-embedder"; +import { createFakeReranker } from "@/server/ai/fake-reranker"; +import type { Reranker } from "@/server/ai/reranker"; import { readSearchEnv } from "@/server/env"; import { MAX_QUERY_LENGTH, + RERANK_CANDIDATES, SEARCH_RESULT_COUNT, SearchQueryError, searchInterviews, @@ -119,3 +122,156 @@ describe("searching the interviews", () => { ).toThrow(/AI_PROVIDER/); }); }); + +/** A passage of `participant`'s, `distance` from the question. */ +function passage(participant: string, distance: number): PassageMatch { + return { + id: distance * 1000, + slug: participant.toLowerCase(), + participant, + position: 1, + question: `What does ${participant} do?`, + answer: `${participant}'s answer.`, + distance, + }; +} + +/** A reranker that gives each text the relevance its passage's name maps to. */ +function rerankerScoring( + scores: Record, + threshold = 0.5, +): Reranker & { rank: ReturnType> } { + return { + model: "test", + threshold, + rank: vi.fn(async (_, texts) => + texts.map((text) => scores[text.split(" ")[2]!.replace("?", "")] ?? 0), + ), + }; +} + +describe("reranking the search", () => { + const CANDIDATES = [ + passage("Daan", 0.28), + passage("Ravi", 0.29), + passage("Maya", 0.3), + passage("Sofia", 0.31), + passage("Lena", 0.32), + passage("Henk", 0.33), + passage("Tom", 0.34), + ]; + + function rerankedSetup(reranker: Reranker) { + const findPassages = vi.fn( + async () => CANDIDATES, + ); + return { + dependencies: { embedder: createFakeEmbedder(), findPassages, reranker }, + findPassages, + }; + } + + it("reads the closest candidates and keeps those relevant enough, most relevant first", async () => { + const reranker = rerankerScoring({ Henk: 0.9, Daan: 0.2, Sofia: 0.6 }); + const { dependencies, findPassages } = rerankedSetup(reranker); + + const found = await searchInterviews(" dark mode ", dependencies); + + expect(findPassages).toHaveBeenCalledWith( + expect.any(Array), + RERANK_CANDIDATES, + ); + expect(reranker.rank).toHaveBeenCalledWith( + "dark mode", + CANDIDATES.map( + (candidate) => `${candidate.question}\n${candidate.answer}`, + ), + ); + expect(found).toEqual([ + { ...passage("Henk", 0.33), relevance: 0.9 }, + { ...passage("Sofia", 0.31), relevance: 0.6 }, + ]); + }); + + it("keeps a passage at the threshold, not one just below it", async () => { + const { dependencies } = rerankedSetup( + rerankerScoring({ Daan: 0.5, Ravi: 0.49 }, 0.5), + ); + + const found = await searchInterviews("lists", dependencies); + + expect(found.map((match) => match.participant)).toEqual(["Daan"]); + }); + + it("returns at most five, ties in the order of closeness", async () => { + const reranker = rerankerScoring( + { Daan: 0.5, Ravi: 0.7, Maya: 0.7, Sofia: 0.8, Lena: 0.9, Henk: 0.6 }, + 0.5, + ); + const { dependencies } = rerankedSetup(reranker); + + const found = await searchInterviews("lists", dependencies); + + expect(found.map((match) => match.participant)).toEqual([ + "Lena", + "Sofia", + "Ravi", + "Maya", + "Henk", + ]); + expect(found).toHaveLength(SEARCH_RESULT_COUNT); + }); + + it("returns no passage when none is relevant enough", async () => { + const { dependencies } = rerankedSetup(rerankerScoring({ Daan: 0.4 })); + + await expect( + searchInterviews("grocery delivery", dependencies), + ).resolves.toEqual([]); + }); + + it("fails when the reranker scores a different number of passages", async () => { + const reranker: Reranker = { + model: "test", + threshold: 0.5, + rank: async () => [0.9], + }; + const { dependencies } = rerankedSetup(reranker); + + await expect(searchInterviews("lists", dependencies)).rejects.toThrow( + "The reranker scored 1 of 7 passages.", + ); + }); +}); + +describe("the fake reranker", () => { + it("scores as the fake search ranks, and drops a text sharing no word", async () => { + const reranker = createFakeReranker(); + const texts = [ + "Coffee and oat milk every week.", + "I'd like my phone to ping me.", + "My phone should ping me for every item, a notification each time.", + ]; + + const relevance = await reranker.rank("phone ping notification", texts); + + // The same order as the fake embedder's cosine distance. + const [question, ...vectors] = await createFakeEmbedder().embed([ + "phone ping notification", + ...texts, + ]); + const distances = vectors.map( + (vector) => + 1 - + vector.reduce( + (sum, value, index) => sum + value * question![index]!, + 0, + ), + ); + expect(relevance.map((value, index) => value + distances[index]!)).toEqual( + [1, 1, 1].map(() => expect.closeTo(1, 10)), + ); + expect(relevance[0]).toBeLessThan(reranker.threshold); + expect(relevance[1]).toBeGreaterThanOrEqual(reranker.threshold); + }); +}); diff --git a/tests/unit/system-prompt.test.ts b/tests/unit/system-prompt.test.ts index 8aef88d..714bfbd 100644 --- a/tests/unit/system-prompt.test.ts +++ b/tests/unit/system-prompt.test.ts @@ -22,6 +22,10 @@ const FINGERPRINTS: Record = { // Its text and its tools' wording. It saved nothing on staging // (2026-10-18), so the prompt went back to version 3. 4: "aabe554870e08560a0d877a3333012cded67cb001b9bbd673ea98f43da44e5d2", + // Version 3 cut by about a quarter, with the claim rule reworded. + 5: "314da5d9d1f1b37ea272e028f785dbe6d1696ff6647f2991f60781d42b50699a", + // Version 5 with the reranked search: up to five passages, or none. + 6: "c2b255d792e946dbb750a9b06dd168cba8c7e99fabc850fb3475742dba8ece33", }; /** The toolkit template's nine sections, in its order. */