feat(cli): vocabulary --drift-insights (M19c)

PLAN.md

@@ -5542,6 +5542,50 @@ and `confidence` are generated columns over it. The decisions:
   0.48 s, `--concerns` 0.54 s, Node's start included. The re-explained
   Broadleaf store is owed, as above.
 
+**As built — `vocabulary --drift-insights` (2026-10-28).** `concernDrift` and
+`businessJudged` in `@codegraph/vocabulary` (`concern-drift.ts`), fed by the
+store's `wordHits()`, which with no word now returns every entry. The decisions:
+
+- **Neither package imports the other.** The vocabulary package takes
+  `WordJudgement`s in its own shape and restates the three concerns
+  (`JUDGED_CONCERNS`), pinned equal to the insights block's by a CLI test; the
+  command maps the store's rows into them.
+- **The class is the list's, per entity.** Each judgement's word is classed as
+  `buildVocabulary` would class it in that entity's language (read from the
+  union) under the list in effect, `--technical` included. So a word judged in
+  two languages may be two rows (`select`: Java `domain`, Go `keyword`).
+- **Two lists.** The technical words a model judged domain somewhere, and the
+  domain words it judged technical, ranked by the judgements against, ties by
+  word; `--top N` cuts each (40). `ambiguous` is counted and contradicts
+  neither class; a `keyword` or `short` word (a lexicon corrected since) is
+  counted and contradicts nothing.
+- **Kept as the vocabulary keeps.** A record of an id the model does not hold
+  (explained against another version of it) or the view does not keep is left
+  out and counted on stderr, as are prompt-1 records.
+- **The surest five.** A word's judgements against are ordered by confidence,
+  then by id; the text shows five per word and says how many more, `--json`
+  every one. On Broadleaf every judgement in full was 6 868 lines, each padded
+  to the longest operation id (≈ 600 characters), so the entity now comes last,
+  unpadded: `getSectionKey in java:…AdminFieldController.getSectionKey(…)`.
+- **The business list against both** (`--drift FILE` beside it): each term's
+  words with the classes the corpus says them under and the model's concerns
+  over every entity; a term drifts on a word the list does not call domain
+  (`list`) or a model judged technical somewhere (`model`).
+- **A judgement never sets the exit code.** It is an inference; `--drift`
+  keeps its own exit 3 for a term the corpus never says (`said` is the
+  corpus-level drift's answer, pinned by a test).
+- **The store** is beside the first model, or `--insights FILE` (serve's flag
+  name); missing or unmigrated, a usage error before the model is read.
+  `--scope`, `--level`, `--class` and `--svg` with it, and `--insights`
+  without it, are usage errors. `--json` is `report: "drift-insights"` in
+  `codegraph.vocabularyReport/1`.
+- **Measured** on a store built over Broadleaf's own ids (29 098 records, the
+  words `definitionSpeller` asks plus parameters and attributes: 109 208
+  entries, 12 % judged against the list): 4.1 s end to end against 3.2 s for
+  the plain report on the same machine (an x86-64 Linux box, slower than the
+  M1 of §20.1), at the same peak memory (≈ 0.5 GB). A store a model wrote is
+  owed, as above.
+
 ### 20.4 Milestone split
 
 **M19a — the lexicon, the split, the artifact** (closes the profile half of #19)
@@ -5619,7 +5663,7 @@ and `confidence` are generated columns over it. The decisions:
 - [ ] Readers: `insights --word`, `--glossary`, `--polysemy`, `--concerns`;
       `vocabulary --drift-insights`; `/glossary.json`; the entries in the
       Vocabulary tab and in the Navigate panel as inferences. *(In: the four
-      `insights` readers. Owed: `--drift-insights`, `/glossary.json`, the tab
+      `insights` readers, `--drift-insights`. Owed: `/glossary.json`, the tab
       and the panel.)*
 - [ ] `docs/vocabulary.md` (the idioms per profile, the classes, the
       weights, the artifact, the three layers, the readers, the drift

docs/cli.md

@@ -69,6 +69,7 @@ codegraph navigator [model.jsonl] [--out FILE]
 codegraph vocabulary [model.jsonl] [--level corpus|module|type] [--scope ID]
                    [--top N] [--class domain|technical|keyword|short|all]
                    [--weights KEY=N,…] [--technical FILE] [--drift FILE]
+                   [--drift-insights [--insights FILE]]
                    [--out FILE] [--svg FILE] [--json] [--internal-only]
                    [--declared-only]
 
@@ -254,7 +255,21 @@ one block per scope, ranked by distinct terms (the cognitive-load report);
 `--weights` and `--technical FILE` change the scheme, and the header says
 which one counted. `--drift FILE` compares with a glossary, one business term
 per line: the terms no scope of `--level` says (exit `3`), the ones it says
-and where, and the domain words no term contains. `--out FILE` writes the
+and where, and the domain words no term contains. `--drift-insights` sets the
+classes against what `explain` judged (M19c): it reads the insights store beside
+the first model (`--insights FILE` names another), classes each word a model
+defined as the list does in that entity's language, and lists the technical
+words a model called domain somewhere, then the domain words it called
+technical, the most judgements against first (`--top N` per list, 40 by
+default), each with the judgements against it, surest first: confidence,
+sense, and the identifiers in which entity (five per word, and how many more:
+`--json` lists every one). With `--drift FILE` it adds the business terms against both: each
+word's class and the model's concerns, and why the term drifts (`list`,
+`model`). The judgements are inferences and never set the exit code; `--drift`
+still exits `3` on a term the corpus never says. `--scope`, `--level`,
+`--class` and `--svg` do not apply to it; records of entities the model or the
+view does not have, and records explained under prompt 1, are counted on stderr.
+`--json` gives `report: "drift-insights"`. `--out FILE` writes the
 `vocabulary.json` artifact (`codegraph.vocabulary/1`) beside the report.
 `--svg FILE` draws the cloud of the one scope reported — the corpus, or
 `--scope`'s — from the words `--class` and `--top` pick: size on the square

docs/insights.md

@@ -264,7 +264,8 @@ id: *explained* (the record), *failed* (the unit's failure — matched by member
 so every member of a failed cycle is answered), or *unknown*. "No explanation"
 is two different facts, and only one of them is silence.
 
-Two readers exist. `codegraph insights` (cli.md) opens the store and nothing
+Two readers answer about records (a third, `vocabulary --drift-insights`,
+about words: below). `codegraph insights` (cli.md) opens the store and nothing
 else — not the model, not `explain.ts`, whose module graph is the provider SDK.
 And `codegraph serve` answers `/insight.json?id=…`, its one route that is not an
 artifact: tens of megabytes of prose do not belong in `navigator.json`, so a
@@ -281,7 +282,8 @@ ids are never constructed.
 
 **The words' readers (M19c).** `glossary.ts` rolls the records' `words` and
 the modules' `glossary` up, over four more projections of the port:
-`wordHits(word)` (the `insight_word` index, joined to each record's envelope),
+`wordHits(word?)` (the `insight_word` index, joined to each record's envelope;
+every entry with no word),
 `glossaryEntries(word?)` (each module block's `glossary`, through `json_each`
 over the module records alone), `concernRows()` (each module, word and concern
 once) and `withoutWords()`. A record is placed in its module by its natural
@@ -299,6 +301,14 @@ print them (cli.md). On the synthetic store of Broadleaf's size,
 `--word order` prints its 549 entries and 19 glossary entries in under half a
 second, Node's start included.
 
+One reader opens the store BESIDE the model: `codegraph vocabulary
+--drift-insights` (cli.md). Only the model says which class the deterministic
+rules give a word in an entity (the entity's language is the model's), so that
+comparison cannot live in `insights`, which never opens the model. It reads
+every entry through `wordHits()`, hands them to `@codegraph/vocabulary` in that
+package's own shape (neither package imports the other), and sets the model's
+concern beside the list's class, word by word, with the entities.
+
 `@codegraph/insights` stays pure: `store.ts` is a port, and `store-sqlite.ts`
 is handed an OPEN database, so the path, the file and `node:sqlite` itself
 (one import site, in the analyzer's `store/sqlite.ts`) stay with the CLI.

packages/cli/src/args.ts

@@ -435,6 +435,19 @@ export const VOCABULARY_SPEC: CommandSpec = {
         "Compare with a glossary, one business term per line: the terms no scope of --level says (exit 3), and the domain words no term contains.",
       placeholder: "FILE",
     },
+    {
+      name: "drift-insights",
+      type: "boolean",
+      describe:
+        "Compare the words' classes with what 'codegraph explain' judged them: the technical words a model called domain somewhere, " +
+        "and the reverse, with the entities; with --drift, the business terms against both.",
+    },
+    {
+      name: "insights",
+      type: "string",
+      describe: "The insights store --drift-insights reads. Default: <model>.insights.db beside the first model.",
+      placeholder: "FILE",
+    },
     {
       name: "out",
       type: "string",
@@ -1278,6 +1291,10 @@ export interface VocabularyOptions extends ModelInputOptions, ViewOptions, Cache
   readonly technical: string | undefined;
   /** `--drift FILE`: the glossary to compare with. */
   readonly drift: string | undefined;
+  /** `--drift-insights`: the classes against the concerns a model judged, from the insights store. */
+  readonly driftInsights: boolean;
+  /** `--insights FILE`: that store; undefined = beside the first model. */
+  readonly insights: string | undefined;
   /** `--out FILE`: the artifact, beside the report. */
   readonly out: string | undefined;
   /** `--svg FILE`: the cloud of the one scope reported, beside the report. */
@@ -2205,6 +2222,26 @@ function vocabularyOf(values: ParsedValues): Omit<VocabularyOptions, "models"> {
   }
   const svg = stringOf(values, "svg");
   const level = stringOf(values, "level");
+  const driftInsights = flagOf(values, "drift-insights");
+  const insights = stringOf(values, "insights");
+  if (insights !== undefined && !driftInsights) {
+    throw new UsageError("--insights does not apply without --drift-insights", "--insights names the store --drift-insights reads.");
+  }
+  if (driftInsights) {
+    const read = [
+      [scope !== undefined, "--scope"],
+      [level !== undefined, "--level"],
+      [stringOf(values, "class") !== undefined, "--class"],
+      [svg !== undefined, "--svg"],
+    ] as const;
+    for (const [given, flag] of read) {
+      if (!given) continue;
+      throw new UsageError(
+        `${flag} does not apply with --drift-insights`,
+        "--drift-insights compares every word a model judged, over the corpus, with the class the list gives it.",
+      );
+    }
+  }
   const many = drift !== undefined ? "--drift" : level !== undefined && level !== "corpus" ? `--level ${level}` : undefined;
   if (svg !== undefined && many !== undefined) {
     throw new UsageError(`--svg does not apply with ${many}`, "--svg draws one scope's cloud: the corpus, or the one --scope names.");
@@ -2217,6 +2254,8 @@ function vocabularyOf(values: ParsedValues): Omit<VocabularyOptions, "models"> {
     weights: weightsOf(stringOf(values, "weights")),
     technical: stringOf(values, "technical"),
     drift,
+    driftInsights,
+    insights,
     out: stringOf(values, "out"),
     svg,
     json: flagOf(values, "json"),

packages/cli/src/commands/insights.ts

@@ -1,12 +1,8 @@
-import { existsSync } from "node:fs";
-import { loadSqlite } from "@codegraph/analyzer";
 import {
-  InsightsStoreError,
   WORD_CONCERNS,
   answerInsight,
   concernsReport,
   polysemyReport,
-  sqliteInsightsStore,
   wordReport,
   wordsAbsence,
   type FailureRecord,
@@ -19,7 +15,7 @@ import {
 import type { InsightsOptions } from "../args.js";
 import { EXIT, UsageError, type ExitCode } from "../exit.js";
 import { errLine, outLine, type IoSink } from "../io.js";
-import { storePathForModel } from "../insights-paths.js";
+import { openInsightsReader, storePathForModel } from "../insights-paths.js";
 
 /**
  * `codegraph insights [model.jsonl] [--id ID | --level … --concept … | --failures | --runs] [--json]`
@@ -42,7 +38,7 @@ import { storePathForModel } from "../insights-paths.js";
  */
 export function insightsCommand(options: InsightsOptions, io: IoSink): ExitCode {
   const path = options.store ?? storePathForModel(options.model);
-  const store = openReader(path);
+  const store = openInsightsReader(path, "--store");
   try {
     switch (options.mode) {
       case "summary":
@@ -69,24 +65,6 @@ export function insightsCommand(options: InsightsOptions, io: IoSink): ExitCode
   }
 }
 
-function openReader(path: string): InsightsStore {
-  // Checked first: opening a path that does not exist would CREATE it, and asking must not leave a file behind.
-  if (!existsSync(path)) {
-    throw new UsageError(
-      `no insights store at ${path}`,
-      "'codegraph explain <model.jsonl>' creates it (and imports a side-car it finds there). Pass --store FILE if it is elsewhere.",
-    );
-  }
-  const db = loadSqlite().open(path);
-  try {
-    return sqliteInsightsStore(db, { readOnly: true });
-  } catch (error) {
-    db.close();
-    if (!(error instanceof InsightsStoreError)) throw error;
-    throw new UsageError(`${path}: ${error.message}`, "Pass --store FILE to read another store.", { cause: error });
-  }
-}
-
 const plural = (count: number, noun: string, many = `${noun}s`): string => `${count} ${count === 1 ? noun : many}`;
 const usd = (cost: number | undefined): string => (cost === undefined ? "" : `, $${cost.toFixed(4)}`);
 
@@ -212,7 +190,7 @@ function failureReason(failure: FailureRecord): string {
 }
 
 /** What every word answer is blind to, on stderr, where a pipe does not read it. */
-function noteWithoutWords(count: number, io: IoSink): void {
+export function noteWithoutWords(count: number, io: IoSink): void {
   if (count === 0) return;
   errLine(io, `note: ${plural(count, "record")} a model wrote under prompt 1 define${count === 1 ? "s" : ""} no words; the next 'codegraph explain' re-explains ${count === 1 ? "it" : "them"}.`);
 }