From cdb3b12ef778a6a31a660d397c81ee8216d2d1db Mon Sep 17 00:00:00 2001 From: Omkar Tripathi <38633467+triggeredcode@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:26:32 +0530 Subject: [PATCH] feat(evals): compare local semantic routing (#55) --- docs/evals.md | 21 +++ packages/evals/src/experiment/cases.ts | 120 +++++++++++++- packages/evals/src/experiment/cli.ts | 40 +++-- .../evals/src/experiment/embeddingRouter.ts | 154 ++++++++++++++++++ .../evals/src/experiment/experiment.test.ts | 46 +++++- packages/evals/src/experiment/report.ts | 35 +++- packages/evals/src/experiment/run.ts | 10 ++ packages/evals/src/experiment/types.ts | 5 +- 8 files changed, 415 insertions(+), 16 deletions(-) create mode 100644 packages/evals/src/experiment/embeddingRouter.ts diff --git a/docs/evals.md b/docs/evals.md index 90a177e..d3c4502 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -42,6 +42,27 @@ and its section heading. By default JSON and Markdown reports are written under state and must not be committed. Candidates remain outside the product runtime until they beat the baseline without a safety or final-task regression. +Dataset v2 adds semantic scope paraphrases and vague near-misses that the explicit-state router cannot answer by literal wording +alone. Run the optional local comparison with +`pnpm eval:experiment -- --suite routing --candidate ollama-embedding --embedding-model nomic-embed-text:latest`; it calls +Ollama's local `/api/embed` endpoint. The hybrid candidate only classifies an otherwise-unknown scope: exact meta commands, +safe deterministic actions, lane selection, and product runtime code remain untouched. Use `--ollama-url ` when Ollama is +not listening on its default local address. Use `--embedding-min-similarity` and `--embedding-min-margin` to calibrate on the +calibration split before inspecting holdout results. + +### Dataset v2 result (Apple M4 Pro) + +| Candidate | Routing pass | Semantic scope | Vague near-miss scope | Semantic p50 / p95 | Cold max | +| --- | ---: | ---: | ---: | ---: | ---: | +| Current explicit-state heuristic | 37/46 | 0.0% | 100.0% | <1 ms | <1 ms | +| Nomic Embed Text, margin 0.04 | 42/46 | 55.6% | 100.0% | 9.6 / 11.5 ms | 175 ms | +| Qwen3 Embedding 0.6B, margin 0.04 | 39/46 | 22.2% | 100.0% | 24.7 / 34.3 ms | 415 ms | + +Both embedding candidates preserved 100% lane/action accuracy and produced no high-confidence wrong direct actions because the +hybrid never replaces Lucid's explicit-state or exact deterministic routes. Nomic is the smaller, faster, and more accurate +candidate, but 55.6% semantic scope accuracy is not sufficient for product integration. Keep it experiment-only and improve +the classifier or training examples before accepting the runtime/model-management cost. + ## The bars Overall 90%. Each category 80%, and 95% for `non_destructive` and `mode_separation` (a mistake there loses the person's diff --git a/packages/evals/src/experiment/cases.ts b/packages/evals/src/experiment/cases.ts index eef958a..bd586ab 100644 --- a/packages/evals/src/experiment/cases.ts +++ b/packages/evals/src/experiment/cases.ts @@ -15,7 +15,17 @@ const route = ( utterance: string, state: RoutingCase["state"], expected: RoutingCase["expected"], -): RoutingCase => ({ id, family, split, utterance, state, expected, doc: SAMPLE_DOC }); + challenge?: RoutingCase["challenge"], +): RoutingCase => ({ + id, + family, + split, + utterance, + state, + expected, + ...(challenge ? { challenge } : {}), + doc: SAMPLE_DOC, +}); export const ROUTING_CASES: RoutingCase[] = [ route( @@ -290,6 +300,114 @@ export const ROUTING_CASES: RoutingCase[] = [ { ...baseState, mode: "agent-read" }, { lane: "agent-read", action: "read-more", scope: "document" }, ), + route( + "semantic-document-everything", + "semantic-document", + "holdout", + "tighten up everything I've written", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "document" }, + "semantic", + ), + route( + "semantic-document-consistent-voice", + "semantic-document", + "holdout", + "give the note a consistent voice from start to finish", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "document" }, + "semantic", + ), + route( + "semantic-section-database", + "semantic-section", + "calibration", + "condense the part about database options", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "section" }, + "semantic", + ), + route( + "semantic-section-risks", + "semantic-section", + "calibration", + "rewrite the bit covering deployment risks", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "section" }, + "semantic", + ), + route( + "semantic-component-grid", + "semantic-component", + "development", + "add another entry to the comparison grid", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "component" }, + "semantic", + ), + route( + "semantic-component-visual", + "semantic-component", + "development", + "make the architecture visual flow left to right", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "component" }, + "semantic", + ), + route( + "semantic-block-fourth", + "semantic-block", + "holdout", + "rewrite the fourth content item as a shorter comparison", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "block" }, + "semantic", + ), + route( + "semantic-block-paragraph-seven", + "semantic-block", + "holdout", + "shorten paragraph number seven", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "block" }, + "semantic", + ), + route( + "semantic-selection-highlight", + "semantic-selection", + "development", + "polish the highlighted text", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "selection" }, + "semantic", + ), + route( + "near-miss-vague-improve", + "semantic-near-miss", + "calibration", + "make this easier to understand", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "unknown" }, + "near-miss", + ), + route( + "near-miss-continue", + "semantic-near-miss", + "calibration", + "continue from where I left off", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "unknown" }, + "near-miss", + ), + route( + "near-miss-opinion", + "semantic-near-miss", + "calibration", + "tell me what stands out", + { ...baseState, mode: "command" }, + { lane: "command", action: "main-model", scope: "unknown" }, + "near-miss", + ), ]; const sectionIds = (n: number): string[] => [ diff --git a/packages/evals/src/experiment/cli.ts b/packages/evals/src/experiment/cli.ts index 65fa34f..63109bf 100644 --- a/packages/evals/src/experiment/cli.ts +++ b/packages/evals/src/experiment/cli.ts @@ -5,8 +5,9 @@ import { dirname, join, resolve } from "node:path"; import { currentBaseline } from "./baseline"; import { CONTEXT_CASES, ROUTING_CASES, validateExperimentCases } from "./cases"; import { currentContextStrategy, focusedContextStrategy } from "./contextStrategies"; +import { createEmbeddingRoutingCandidate } from "./embeddingRouter"; import { experimentToMarkdown, passesExperimentGates } from "./report"; -import { runContextCase, runRoutingCase } from "./run"; +import { runContextCase, runRoutingCases } from "./run"; import { EXPERIMENT_DATASET_VERSION, type ExperimentResults, type ExperimentSplit } from "./types"; const arg = (args: string[], name: string): string | undefined => { @@ -31,8 +32,12 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi const suite = arg(args, "suite") ?? "all"; const splitArg = arg(args, "split") ?? "all"; const contextArg = arg(args, "context") ?? "current"; - if (candidateArg !== "current") { - console.error("Only --candidate current exists in the baseline slice."); + const minSimilarityArg = arg(args, "embedding-min-similarity"); + const minMarginArg = arg(args, "embedding-min-margin"); + const minSimilarity = minSimilarityArg === undefined ? undefined : Number(minSimilarityArg); + const minMargin = minMarginArg === undefined ? undefined : Number(minMarginArg); + if (candidateArg !== "current" && candidateArg !== "ollama-embedding") { + console.error("--candidate must be current or ollama-embedding."); return 2; } if (!["routing", "context", "all"].includes(suite)) { @@ -47,6 +52,14 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi console.error("--context must be current or focused."); return 2; } + if (minSimilarity !== undefined && (!Number.isFinite(minSimilarity) || minSimilarity < 0)) { + console.error("--embedding-min-similarity must be a non-negative number."); + return 2; + } + if (minMargin !== undefined && (!Number.isFinite(minMargin) || minMargin < 0)) { + console.error("--embedding-min-margin must be a non-negative number."); + return 2; + } const problems = validateExperimentCases(); if (problems.length > 0) { console.error( @@ -58,14 +71,20 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi const split = splitArg as ExperimentSplit | "all"; const inSplit = (testCase: { split: ExperimentSplit }): boolean => split === "all" || testCase.split === split; + const ollamaUrl = arg(args, "ollama-url"); + const routingCandidate = + candidateArg === "ollama-embedding" + ? createEmbeddingRoutingCandidate({ + model: arg(args, "embedding-model") ?? "nomic-embed-text:latest", + ...(ollamaUrl ? { baseUrl: ollamaUrl } : {}), + ...(minSimilarity === undefined ? {} : { minSimilarity }), + ...(minMargin === undefined ? {} : { minMargin }), + }) + : currentBaseline; const routing = suite === "context" ? [] - : await Promise.all( - ROUTING_CASES.filter(inSplit).map((testCase) => - runRoutingCase(testCase, currentBaseline), - ), - ); + : await runRoutingCases(ROUTING_CASES.filter(inSplit), routingCandidate); const contextStrategy = contextArg === "focused" ? focusedContextStrategy : currentContextStrategy; const context = @@ -79,7 +98,7 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi const results: ExperimentResults = { meta: { datasetVersion: EXPERIMENT_DATASET_VERSION, - candidate: currentBaseline.id, + candidate: routingCandidate.id, contextStrategy: contextStrategy.id, createdAt: new Date().toISOString(), sourceRevision: revision(cwd), @@ -99,10 +118,11 @@ export async function experimentMain(args: string[], cwd = process.cwd()): Promi context, }; const day = new Date().toLocaleDateString("en-CA"); + const candidateSlug = routingCandidate.id.replace(/[^a-z0-9.-]+/gi, "-"); const output = resolve( cwd, arg(args, "out") ?? - join(".docdex", "evals", `${day}-decision-current-context-${contextStrategy.id}`), + join(".docdex", "evals", `${day}-decision-${candidateSlug}-context-${contextStrategy.id}`), ); mkdirSync(dirname(join(output, "x")), { recursive: true }); writeFileSync(join(output, "results.json"), `${JSON.stringify(results, null, 2)}\n`); diff --git a/packages/evals/src/experiment/embeddingRouter.ts b/packages/evals/src/experiment/embeddingRouter.ts new file mode 100644 index 0000000..fbedebd --- /dev/null +++ b/packages/evals/src/experiment/embeddingRouter.ts @@ -0,0 +1,154 @@ +import { currentBaseline } from "./baseline"; +import type { DecisionScope, RoutingCandidate } from "./types"; + +export type Embed = (input: string[]) => Promise; + +const SCOPE_PROTOTYPES: Record = { + document: [ + "edit every part of the writing", + "make a global change across all content", + "work on the complete file rather than one area", + ], + section: [ + "edit one topic-specific portion of the writing", + "change the region devoted to one subject", + "work within one chapter or grouped area", + ], + component: [ + "modify one structured visual object", + "change a data grid or visual figure", + "edit an embedded structured element", + ], + block: [ + "change one individual content unit", + "edit a numbered paragraph or item", + "work on a single piece of content", + ], + selection: [ + "edit only the highlighted range", + "change the active text selection", + "rewrite only the marked passage", + ], + unknown: [ + "the request has no identifiable target", + "the user asks for a vague improvement", + "no particular area or object is named", + "an open-ended request without a clear scope", + ], +}; + +const SCOPES = Object.keys(SCOPE_PROTOTYPES) as DecisionScope[]; + +function cosine(left: number[], right: number[]): number { + if (left.length === 0 || left.length !== right.length) return -1; + let dot = 0; + let leftNorm = 0; + let rightNorm = 0; + for (let index = 0; index < left.length; index += 1) { + const a = left[index] ?? 0; + const b = right[index] ?? 0; + dot += a * b; + leftNorm += a * a; + rightNorm += b * b; + } + if (leftNorm === 0 || rightNorm === 0) return -1; + return dot / Math.sqrt(leftNorm * rightNorm); +} + +function centroid(vectors: number[][]): number[] { + const width = vectors[0]?.length ?? 0; + return Array.from({ length: width }, (_, index) => { + const total = vectors.reduce((sum, vector) => sum + (vector[index] ?? 0), 0); + return total / vectors.length; + }); +} + +export function classifyScope( + query: number[], + prototypes: Map, + minSimilarity = 0.35, + minMargin = 0.04, +): { scope: DecisionScope; similarity: number; margin: number } { + const ranked = SCOPES.map((scope) => ({ + scope, + similarity: cosine(query, prototypes.get(scope) ?? []), + })).sort((left, right) => right.similarity - left.similarity); + const best = ranked[0] ?? { scope: "unknown" as const, similarity: -1 }; + const margin = best.similarity - (ranked[1]?.similarity ?? -1); + if (best.similarity < minSimilarity || margin < minMargin) { + return { scope: "unknown", similarity: best.similarity, margin }; + } + return { scope: best.scope, similarity: best.similarity, margin }; +} + +export function createOllamaEmbedder(model: string, baseUrl = "http://127.0.0.1:11434"): Embed { + return async (input) => { + const response = await fetch(`${baseUrl.replace(/\/$/, "")}/api/embed`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model, input }), + }); + if (!response.ok) { + throw new Error(`Ollama embedding request failed (${String(response.status)}).`); + } + const body = (await response.json()) as { embeddings?: number[][] }; + if (!body.embeddings || body.embeddings.length !== input.length) { + throw new Error("Ollama returned an invalid embedding batch."); + } + return body.embeddings; + }; +} + +export function createEmbeddingRoutingCandidate(options: { + model: string; + embed?: Embed; + baseUrl?: string; + minSimilarity?: number; + minMargin?: number; +}): RoutingCandidate { + const embed = options.embed ?? createOllamaEmbedder(options.model, options.baseUrl); + let prototypePromise: Promise> | undefined; + + const prototypes = async (): Promise> => { + prototypePromise ??= (async () => { + const phrases = SCOPES.flatMap((scope) => SCOPE_PROTOTYPES[scope]); + const vectors = await embed(phrases); + const result = new Map(); + let offset = 0; + for (const scope of SCOPES) { + const count = SCOPE_PROTOTYPES[scope].length; + result.set(scope, centroid(vectors.slice(offset, offset + count))); + offset += count; + } + return result; + })(); + return prototypePromise; + }; + + return { + id: `ollama-embedding:${options.model}`, + async decide(testCase) { + const started = performance.now(); + const baseline = await currentBaseline.decide(testCase); + if (baseline.action !== "main-model" || baseline.scope !== "unknown") { + return { ...baseline, latencyMs: performance.now() - started }; + } + const [query] = await embed([testCase.utterance]); + if (!query) throw new Error("Ollama did not return an utterance embedding."); + const result = classifyScope( + query, + await prototypes(), + options.minSimilarity, + options.minMargin, + ); + return { + ...baseline, + scope: result.scope, + confidence: Math.max(0, Math.min(1, (result.similarity + 1) / 2)), + abstained: result.scope === "unknown", + latencyMs: performance.now() - started, + embedding: { similarity: result.similarity, margin: result.margin }, + }; + }, + }; +} diff --git a/packages/evals/src/experiment/experiment.test.ts b/packages/evals/src/experiment/experiment.test.ts index 56a963f..0da640a 100644 --- a/packages/evals/src/experiment/experiment.test.ts +++ b/packages/evals/src/experiment/experiment.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { currentBaseline } from "./baseline"; import { CONTEXT_CASES, ROUTING_CASES, validateExperimentCases } from "./cases"; import { currentContextStrategy, focusedContextStrategy } from "./contextStrategies"; +import { classifyScope, createEmbeddingRoutingCandidate } from "./embeddingRouter"; import { experimentToMarkdown, passesExperimentGates, @@ -23,15 +24,29 @@ describe("the frozen experiment dataset", () => { }); describe("the current routing baseline", () => { - it("matches the labelled routes without unsafe direct actions", async () => { + it("preserves the original labelled routes without unsafe direct actions", async () => { const trials = await Promise.all( - ROUTING_CASES.map((testCase) => runRoutingCase(testCase, currentBaseline)), + ROUTING_CASES.filter(({ challenge }) => challenge === undefined).map((testCase) => + runRoutingCase(testCase, currentBaseline), + ), ); const summary = summariseRouting(trials); expect(summary.passRate).toBe(1); expect(summary.highConfidenceWrongDirectActions).toBe(0); }); + it("exposes semantic scope gaps while retaining vague near-misses", async () => { + const trials = await Promise.all( + ROUTING_CASES.filter(({ challenge }) => challenge !== undefined).map((testCase) => + runRoutingCase(testCase, currentBaseline), + ), + ); + const summary = summariseRouting(trials); + expect(summary.semanticScopeAccuracy).toBe(0); + expect(summary.nearMissScopeAccuracy).toBe(1); + expect(summary.highConfidenceWrongDirectActions).toBe(0); + }); + it("detects a confident wrong direct action", async () => { const unsafe: RoutingCandidate = { id: "unsafe-test", @@ -53,6 +68,31 @@ describe("the current routing baseline", () => { }); }); +describe("the optional embedding routing candidate", () => { + it("classifies a separated semantic vector", () => { + const prototypes = new Map([ + ["document" as const, [1, 0, 0, 0, 0, 0]], + ["section" as const, [0, 1, 0, 0, 0, 0]], + ["component" as const, [0, 0, 1, 0, 0, 0]], + ["block" as const, [0, 0, 0, 1, 0, 0]], + ["selection" as const, [0, 0, 0, 0, 1, 0]], + ["unknown" as const, [0, 0, 0, 0, 0, 1]], + ]); + expect(classifyScope([0.9, 0.1, 0, 0, 0, 0], prototypes).scope).toBe("document"); + }); + + it("never invokes embeddings for an existing deterministic route", async () => { + const candidate = createEmbeddingRoutingCandidate({ + model: "test", + embed: () => Promise.reject(new Error("embedding should not run")), + }); + const testCase = ROUTING_CASES.find(({ id }) => id === "mode-command"); + if (!testCase) throw new Error("mode-command fixture is missing"); + const trial = await runRoutingCase(testCase, candidate); + expect(trial.pass).toBe(true); + }); +}); + describe("targeted agent context", () => { it("freezes the current planner's strengths and known over-selection", () => { const trials = CONTEXT_CASES.map((testCase) => @@ -81,7 +121,7 @@ describe("experiment reports", () => { it("renders deterministic metrics and applies the baseline gates", async () => { const results: ExperimentResults = { meta: { - datasetVersion: 1, + datasetVersion: 2, candidate: "current", contextStrategy: "current", createdAt: "2026-10-20T00:00:00.000Z", diff --git a/packages/evals/src/experiment/report.ts b/packages/evals/src/experiment/report.ts index c84a041..52f63c0 100644 --- a/packages/evals/src/experiment/report.ts +++ b/packages/evals/src/experiment/report.ts @@ -22,9 +22,14 @@ export interface RoutingSummary { laneAccuracy: number; actionAccuracy: number; scopeAccuracy: number; + semanticScopeAccuracy: number; + nearMissScopeAccuracy: number; highConfidenceWrongDirectActions: number; p50LatencyMs: number; p95LatencyMs: number; + maxLatencyMs: number; + challengeP50LatencyMs: number; + challengeP95LatencyMs: number; } export interface ContextSummary { @@ -40,6 +45,9 @@ export interface ContextSummary { export function summariseRouting(trials: readonly RoutingTrial[]): RoutingSummary { const passed = trials.filter(({ pass }) => pass).length; const scopeTrials = trials.filter(({ expected }) => expected.scope !== undefined); + const semanticTrials = trials.filter(({ challenge }) => challenge === "semantic"); + const nearMissTrials = trials.filter(({ challenge }) => challenge === "near-miss"); + const challengeTrials = [...semanticTrials, ...nearMissTrials]; return { total: trials.length, passed, @@ -56,6 +64,14 @@ export function summariseRouting(trials: readonly RoutingTrial[]): RoutingSummar scopeTrials.filter(({ decision, expected }) => decision.scope === expected.scope).length, scopeTrials.length, ), + semanticScopeAccuracy: rate( + semanticTrials.filter(({ decision, expected }) => decision.scope === expected.scope).length, + semanticTrials.length, + ), + nearMissScopeAccuracy: rate( + nearMissTrials.filter(({ decision, expected }) => decision.scope === expected.scope).length, + nearMissTrials.length, + ), highConfidenceWrongDirectActions: trials.filter( ({ highConfidenceWrongDirectAction }) => highConfidenceWrongDirectAction, ).length, @@ -64,6 +80,12 @@ export function summariseRouting(trials: readonly RoutingTrial[]): RoutingSummar trials.map(({ decision }) => decision.latencyMs), 0.95, ), + maxLatencyMs: Math.max(0, ...trials.map(({ decision }) => decision.latencyMs)), + challengeP50LatencyMs: median(challengeTrials.map(({ decision }) => decision.latencyMs)), + challengeP95LatencyMs: percentile( + challengeTrials.map(({ decision }) => decision.latencyMs), + 0.95, + ), }; } @@ -102,8 +124,19 @@ export function experimentToMarkdown(results: ExperimentResults): string { "", `- Pass: ${String(routing.passed)}/${String(routing.total)} (${percent(routing.passRate)})`, `- Lane/action/scope accuracy: ${percent(routing.laneAccuracy)} / ${percent(routing.actionAccuracy)} / ${percent(routing.scopeAccuracy)}`, + `- Semantic/near-miss scope accuracy: ${percent(routing.semanticScopeAccuracy)} / ${percent(routing.nearMissScopeAccuracy)}`, `- High-confidence wrong direct actions: ${String(routing.highConfidenceWrongDirectActions)}`, - `- Decision latency p50/p95: ${routing.p50LatencyMs.toFixed(3)} / ${routing.p95LatencyMs.toFixed(3)} ms`, + `- Decision latency p50/p95/max: ${routing.p50LatencyMs.toFixed(3)} / ${routing.p95LatencyMs.toFixed(3)} / ${routing.maxLatencyMs.toFixed(3)} ms`, + `- Semantic challenge latency p50/p95: ${routing.challengeP50LatencyMs.toFixed(3)} / ${routing.challengeP95LatencyMs.toFixed(3)} ms`, + "", + "| Challenge | Expected | Predicted | Confidence | Margin | Result |", + "|---|---|---|---:|---:|---|", + ...results.routing + .filter(({ challenge }) => challenge !== undefined) + .map( + (trial) => + `| ${trial.caseId} | ${trial.expected.scope ?? "-"} | ${trial.decision.scope} | ${percent(trial.decision.confidence)} | ${trial.decision.embedding?.margin.toFixed(4) ?? "-"} | ${trial.pass ? "pass" : "FAIL"} |`, + ), "", "## Context selection", "", diff --git a/packages/evals/src/experiment/run.ts b/packages/evals/src/experiment/run.ts index 8dcd726..9176fb2 100644 --- a/packages/evals/src/experiment/run.ts +++ b/packages/evals/src/experiment/run.ts @@ -48,6 +48,7 @@ export async function runRoutingCase( family: testCase.family, split: testCase.split, utterance: testCase.utterance, + ...(testCase.challenge ? { challenge: testCase.challenge } : {}), expected: testCase.expected, decision, pass: failures.length === 0, @@ -56,6 +57,15 @@ export async function runRoutingCase( }; } +export async function runRoutingCases( + cases: readonly RoutingCase[], + candidate: RoutingCandidate, +): Promise { + const trials: RoutingTrial[] = []; + for (const testCase of cases) trials.push(await runRoutingCase(testCase, candidate)); + return trials; +} + export function runContextCase( testCase: ContextCase, strategy: ContextStrategy = currentContextStrategy, diff --git a/packages/evals/src/experiment/types.ts b/packages/evals/src/experiment/types.ts index ca03e54..2c17a88 100644 --- a/packages/evals/src/experiment/types.ts +++ b/packages/evals/src/experiment/types.ts @@ -1,6 +1,6 @@ import type { LucidDoc, MetaState } from "@lucid/core"; -export const EXPERIMENT_DATASET_VERSION = 1; +export const EXPERIMENT_DATASET_VERSION = 2; export type ExperimentSplit = "development" | "calibration" | "holdout"; export type DecisionLane = "meta" | "guided" | "input" | "command" | "agent-read" | "clarify"; @@ -16,6 +16,7 @@ export interface Decision { confidence: number; abstained: boolean; latencyMs: number; + embedding?: { similarity: number; margin: number }; } export interface RoutingState extends MetaState { @@ -36,6 +37,7 @@ export interface RoutingCase { scope?: DecisionScope; abstained?: boolean; }; + challenge?: "semantic" | "near-miss"; } export interface RoutingCandidate { @@ -78,6 +80,7 @@ export interface RoutingTrial { family: string; split: ExperimentSplit; utterance: string; + challenge?: "semantic" | "near-miss"; expected: RoutingCase["expected"]; decision: Decision; pass: boolean;