From 5ea02f908d12a0088172792f53de48917d1eef9e Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 17 Oct 2026 18:02:30 -0700 Subject: [PATCH 1/7] feat(telemetry): add a masked error message template to `error_fingerprint` `error_fingerprint` carried only a hash of the error. It now also carries `error_message`: an allowlist template of the error's first line in which identifiers, numbers, quoted text, names and credential values become ``. `maskString` and the fingerprint hash input are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/docs/reference/telemetry.md | 6 +- .../src/altimate/telemetry/error-detail.ts | 151 ++++++++++++++++++ .../opencode/src/altimate/telemetry/index.ts | 2 + packages/opencode/src/session/prompt.ts | 17 +- .../test/telemetry/error-detail.test.ts | 99 ++++++++++++ 5 files changed, 259 insertions(+), 16 deletions(-) create mode 100644 packages/opencode/src/altimate/telemetry/error-detail.ts create mode 100644 packages/opencode/test/telemetry/error-detail.test.ts diff --git a/docs/docs/reference/telemetry.md b/docs/docs/reference/telemetry.md index 8f4f9d8803..5b87b53184 100644 --- a/docs/docs/reference/telemetry.md +++ b/docs/docs/reference/telemetry.md @@ -49,7 +49,7 @@ We collect the following categories of events: | `task_outcome_signal` | Behavioral quality signal at session end — accepted, error, abandoned, or cancelled. Includes tool count, step count, duration, and last tool category. No user content. | | `task_classified` | Intent classification of the first user message using keyword matching — category (e.g. `debug_dbt`, `write_sql`, `optimize_query`), confidence score, and detected warehouse type. No user text is sent — only the classified category. | | `tool_chain_outcome` | Aggregated tool execution sequence at session end — ordered tool names (capped at 50), error count, recovery count, final outcome, duration, and cost. No tool arguments or outputs. | -| `error_fingerprint` | Hashed error pattern for anonymous grouping — SHA-256 hash of masked error message, error class, tool name, and whether recovery succeeded. Raw error content is never sent. | +| `error_fingerprint` | Error pattern for grouping — SHA-256 hash of the masked error message (`error_hash`), plus separate fields for error class, tool name, whether recovery succeeded, and `error_message`. `error_message` is a template of the error's first line: only short plain-letter words and basic punctuation are kept, and identifiers containing digits, `_`, `.`, `/` or `@`, numbers, paths, URLs, quoted values, capitalized names and values after credential keywords are replaced with `` (max 200 characters). Plain lowercase words such as an unquoted table name can survive. | | `sql_fingerprint` | SQL structural shape via AST parsing — statement types, table count, function count, subquery/aggregation/window function presence, and AST node count. No table names, column names, or SQL content. | | `schema_complexity` | Warehouse schema structural metrics from introspection — bucketed table, column, and schema counts plus average columns per table. No schema names or content. | | `validator_check` | A completion-gate validator ran on session end — validator name, `ok` boolean, step, retry count, `enforced` flag (false in shadow mode), and structured `details` (model counts, elapsed time, concurrency limit — no SQL or model content). Only emitted when `ALTIMATE_VALIDATORS_ENABLED=1` or `ALTIMATE_VALIDATORS_SHADOW=1`. See [Validators](../data-engineering/validators.md). | @@ -161,7 +161,7 @@ We take your privacy seriously. Altimate Code telemetry **never** collects: - Credentials, API keys, or tokens - Database connection strings or hostnames - Personally identifiable information (your email is SHA-256 hashed before sending and is used only for anonymous user correlation) -- Tool arguments or outputs +- Tool arguments or outputs, except the first line of tool errors in `error_fingerprint` (the `error_fingerprint` error template keeps only plain English words from the first line of an error; identifiers, numbers, paths and quoted values are replaced with ``. Residual risk: a statement or message written entirely in plain letters, such as an unquoted SQL fragment of simple words, can survive the template) - AI prompt content or responses Error messages are truncated to 500 characters and scrubbed of file paths before sending. @@ -210,7 +210,7 @@ Event type names use **snake_case** with a `domain_action` pattern: - `error_recovered` for error recovery events - `task_outcome_signal`, `task_classified` for session quality signals - `tool_chain_outcome` for tool execution chain aggregation -- `error_fingerprint` for anonymous error pattern grouping +- `error_fingerprint` for error pattern grouping, with a masked one-line error message - `sql_fingerprint` for SQL structural analysis - `schema_complexity` for warehouse schema metrics diff --git a/packages/opencode/src/altimate/telemetry/error-detail.ts b/packages/opencode/src/altimate/telemetry/error-detail.ts new file mode 100644 index 0000000000..e437455086 --- /dev/null +++ b/packages/opencode/src/altimate/telemetry/error-detail.ts @@ -0,0 +1,151 @@ +import { Telemetry } from "./index" + +const MAX_MESSAGE = 200 +const PLACEHOLDER = "" +const PEEL = /^([:,.()'"\-?!]*)(.*?)([:,.()'"\-?!]*)$/ +const WORD = /^(?:[a-z]{1,24}|[A-Z][a-z]{0,23}|[A-Z]{1,3})$/ +const SENTENCE_STARTERS = new Set([ + "error", + "cannot", + "can", + "could", + "failed", + "failure", + "unable", + "invalid", + "unknown", + "missing", + "no", + "not", + "the", + "a", + "an", + "permission", + "access", + "connection", + "timeout", + "request", + "query", + "command", + "file", + "table", + "column", + "unexpected", + "expected", + "syntax", + "authentication", + "operation", + "http", + "tool", + "bad", + "login", + "user", + "this", + "it", + "there", + "too", + "failed", +]) +const CREDENTIAL_WORDS = new Set([ + "password", + "passwd", + "pwd", + "secret", + "token", + "key", + "auth", + "authorization", + "credential", + "credentials", + "apikey", + "bearer", +]) +const CREDENTIAL_SPAN = 2 + +/** + * Allowlist template of an error's first line. Only short plain-letter words and + * a little punctuation survive; every other token (digits, `_`, `/`, `.` inside, + * `@`, `=`, quoted content, capitalized names) becomes ``. English-like words + * that happen to spell a SQL statement are not recognised and can survive. + */ +export function templateErrorMessage(raw: string): string { + const first = raw + .split(/[\r\n]/, 1)[0] + .replace(/'[^']*'|"[^"]*"/g, ` ${PLACEHOLDER} `) + .replace(/['"].*$/, ` ${PLACEHOLDER}`) + const out: string[] = [] + let position = 0 + let secretSpan = 0 + let secretToEol = false + for (const chunk of first.split(/\s+/)) { + if (!chunk) continue + if (chunk === PLACEHOLDER) { + out.push(PLACEHOLDER) + position++ + continue + } + const [, lead, core, trail] = PEEL.exec(chunk)! + // the tokens after a credential keyword are values, however plain they look + if (secretSpan > 0 && core !== "") { + out.push(lead + PLACEHOLDER + trail) + position++ + secretSpan = secretToEol ? 1 : /[,.)!?]/.test(trail) ? 0 : secretSpan - 1 + continue + } + if ( + core + .toLowerCase() + .split(/[_\-.]/) + .some((part) => CREDENTIAL_WORDS.has(part)) + ) { + secretSpan = CREDENTIAL_SPAN + // `label: value` / `label=value`: the value may be a multi-word passphrase, so mask to end of line + secretToEol = /[:=]/.test(trail) || /[:=]/.test(lead) + } + const plainWord = WORD.test(core) && !(core.length > 3 && core === core.toUpperCase()) + const lower = core.toLowerCase() + const name = /^[A-Z]/.test(core) && (position > 0 || !(SENTENCE_STARTERS.has(lower) || CREDENTIAL_WORDS.has(lower))) + out.push(core === "" || (plainWord && !name) ? chunk : lead + PLACEHOLDER + trail) + position++ + } + return out + .join(" ") + .replace(/(?:[:,.()'"\-?!]*\s+)+/g, PLACEHOLDER) + .slice(0, MAX_MESSAGE) +} + +/** Hash input is unchanged from before the detail fields existed, so fingerprints stay stable. */ +export function describeToolError(raw: string) { + return { + errorHash: Telemetry.hashError(Telemetry.maskString(raw).slice(0, 500)), + errorMessage: templateErrorMessage(raw), + } +} + +export function buildErrorFingerprintEvent( + sessionID: string, + err: { + errorHash: string + errorClass: string + toolName: string + toolCategory: string + errorMessage: string + recovered: boolean + recoveryTool: string + }, +): Telemetry.Event { + return { + type: "error_fingerprint", + timestamp: Date.now(), + session_id: sessionID, + error_hash: err.errorHash, + error_class: err.errorClass, + tool_name: err.toolName, + tool_category: err.toolCategory, + error_message: err.errorMessage, + recovery_successful: err.recovered, + recovery_tool: err.recoveryTool, + } +} + +export * as ErrorDetail from "./error-detail" diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 5c16568c1b..b28a8db1ba 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -942,6 +942,8 @@ export namespace Telemetry { tool_name: string /** Tool category */ tool_category: string + /** Allowlist template of the error first line, max 200 chars: plain lowercase words are kept, identifiers, numbers, quoted values, capitalized names and credential values become */ + error_message?: string /** Whether a subsequent tool call succeeded (error was recovered) */ recovery_successful: boolean /** Tool that succeeded after the error (if recovered) */ diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 2711b6dd81..207c761aca 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -113,6 +113,7 @@ import { stampRegistryToolSource, describeMcpTool } from "../altimate/tool-sourc // altimate_change end // altimate_change end import { Telemetry } from "@/telemetry" // altimate_change — session telemetry +import { ErrorDetail } from "@/altimate/telemetry/error-detail" // altimate_change — error_fingerprint payload import { TraceContext } from "@/altimate/observability/trace-context" // altimate_change — client trace per turn import * as OnboardingTelemetry from "@/altimate/telemetry/onboarding" // altimate_change — onboarding funnel // altimate_change start — keyless public Zen predicate (flat module) @@ -801,6 +802,7 @@ export namespace SessionPrompt { toolCategory: string errorClass: string errorHash: string + errorMessage: string recovered: boolean recoveryTool: string } @@ -2176,12 +2178,11 @@ export namespace SessionPrompt { lastToolWasError = true const errorMsg = part.state.status === "error" && typeof part.state.error === "string" ? part.state.error : "unknown" - const masked = Telemetry.maskString(errorMsg).slice(0, 500) pendingError = { toolName: part.tool, toolCategory, errorClass: Telemetry.classifyError(errorMsg), - errorHash: Telemetry.hashError(masked), + ...ErrorDetail.describeToolError(errorMsg), } } else { if (lastToolWasError && pendingError) { @@ -2533,17 +2534,7 @@ export namespace SessionPrompt { } // Error fingerprints — one event per unique error (capped at 20) for (const err of errorRecords.slice(0, 20)) { - Telemetry.track({ - type: "error_fingerprint", - timestamp: Date.now(), - session_id: sessionID, - error_hash: err.errorHash, - error_class: err.errorClass, - tool_name: err.toolName, - tool_category: err.toolCategory, - recovery_successful: err.recovered, - recovery_tool: err.recoveryTool, - }) + Telemetry.track(ErrorDetail.buildErrorFingerprintEvent(sessionID, err)) } // altimate_change end — emit quality signal, tool chain, and error fingerprint events // altimate_change start — populate agent_outcome diagnostic fields diff --git a/packages/opencode/test/telemetry/error-detail.test.ts b/packages/opencode/test/telemetry/error-detail.test.ts new file mode 100644 index 0000000000..6e946a7f95 --- /dev/null +++ b/packages/opencode/test/telemetry/error-detail.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, test } from "bun:test" +import { Telemetry } from "../../src/altimate/telemetry" +import { ErrorDetail } from "../../src/altimate/telemetry/error-detail" + +const { buildErrorFingerprintEvent, describeToolError, templateErrorMessage } = ErrorDetail +// built at runtime so no credential-shaped literal is committed +const basicAuth = Buffer.from(["adm", "pw"].join(":")).toString("base64") +const pgUri = ["postgres:", "//", "u", ":", "p", "@db:5432/x"].join("") + +describe("templateErrorMessage", () => { + test.each([ + ["column ssn_number does not exist in table hr.salaries", "column does not exist in table "], + ["password: hunter2", "password: "], + ["Cannot delete file: permission denied", "Cannot delete file: permission denied"], + ["failed (code 12)!", "failed (code )!"], + ])("%s", (raw, expected) => { + expect(templateErrorMessage(raw)).toBe(expected) + }) + + test("Basic auth header keeps no secret", () => { + const out = templateErrorMessage(`Authorization: Basic ${basicAuth}`) + expect(out).toBe("Authorization: ") + }) + + test("unquoted SQL keeps no identifiers, numbers or names", () => { + const out = templateErrorMessage("SELECT ssn FROM prod.hr.salaries WHERE id = 4417 AND name = Alice") + for (const leak of ["prod", "hr.salaries", "4417", "Alice"]) expect(out).not.toContain(leak) + }) + + test.each([ + "/Users/bob/proj/a.ts", + "C:\\Users\\bob\\a.ts", + "bob@acme.com", + "https://h.example.com/p?token=zzz", + pgUri, + "AKIAIOSFODNN7EXAMPLE", + "ghp_abcdefghijklmnopqrstuvwxyz0123456789", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.sig", + "deadbeefdeadbeefdeadbeefdeadbeef0123", + "'hunter2' \"s3cret\"", + "stg_salesforce_history_2024.sql", + ])("replaces identifier-like token in %s", (raw) => { + const out = templateErrorMessage(`bad ${raw} here`) + expect(out).toBe("bad here") + }) + + test.each([ + ["login failed password: supersecret", "login failed password: "], + ["Authorization: Bearer secrettoken", "Authorization: "], + ["bad api key hunter, retry", "bad api key , retry"], + ["client_secret: supersecret", ""], + ["access_token: secrettoken", ""], + ["api-key: secretvalue", ""], + ["password: correct horse battery staple", "password: "], + ["Alice does not have access", " does not have access"], + ["failed\raccount alice unavailable", "failed"], + ["Error: value 'private customer phrase is invalid", "Error: value "], + ["xKqPzLmNaBcD failed", " failed"], + ["DROP TABLE prod.customer_records", ""], + ])("masks alphabetic credential values in %s", (raw, expected) => { + expect(templateErrorMessage(raw)).toBe(expected) + }) + + test("only the first line, capped at 200 chars", () => { + expect(templateErrorMessage("first line\nsecond line")).toBe("first line") + expect(templateErrorMessage("word ".repeat(200)).length).toBeLessThanOrEqual(200) + }) +}) + +describe("error_fingerprint wiring", () => { + const raw = "column ssn_number does not exist in table hr.salaries\n at run (/w/packages/opencode/src/a.ts:1:1)" + + test("error_hash is identical to the pre-existing computation", () => { + expect(describeToolError(raw).errorHash).toBe("3a0bc6389647e452") + }) + + test("event carries the template", () => { + const d = describeToolError(raw) + const base = { + ...d, + errorClass: "unknown", + toolName: "bash", + toolCategory: "shell", + recovered: false, + recoveryTool: "", + } + const ev: any = buildErrorFingerprintEvent("s1", base) + expect(ev.type).toBe("error_fingerprint") + expect(ev.error_message).toBe("column does not exist in table ") + expect(ev.error_hash).toBe(d.errorHash) + expect("stack" in ev).toBe(false) + }) + + test("prompt.ts emits through these helpers", async () => { + const src = await Bun.file(new URL("../../src/session/prompt.ts", import.meta.url)).text() + expect(src).toContain("...ErrorDetail.describeToolError(errorMsg)") + expect(src).toContain("Telemetry.track(ErrorDetail.buildErrorFingerprintEvent(sessionID, err))") + }) +}) From 8e6309d63c1abb4b440477922ed081a5e3d6348f Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 17 Oct 2026 18:05:50 -0700 Subject: [PATCH 2/7] fix(telemetry): mask the rest of the line after any credential label A token with a part starting with a credential stem (pass, secret, token, key, auth, credential, bearer, pwd, cookie, session) now masks everything after it on the line. `client_secret: a b c`, `access_token=x y` and `passphrase a b c` sent their values before. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/altimate/telemetry/error-detail.ts | 48 ++++++------------- .../test/telemetry/error-detail.test.ts | 8 +++- 2 files changed, 20 insertions(+), 36 deletions(-) diff --git a/packages/opencode/src/altimate/telemetry/error-detail.ts b/packages/opencode/src/altimate/telemetry/error-detail.ts index e437455086..c71e2531d7 100644 --- a/packages/opencode/src/altimate/telemetry/error-detail.ts +++ b/packages/opencode/src/altimate/telemetry/error-detail.ts @@ -46,21 +46,14 @@ const SENTENCE_STARTERS = new Set([ "too", "failed", ]) -const CREDENTIAL_WORDS = new Set([ - "password", - "passwd", - "pwd", - "secret", - "token", - "key", - "auth", - "authorization", - "credential", - "credentials", - "apikey", - "bearer", -]) -const CREDENTIAL_SPAN = 2 +// A token with a part starting with one of these stems (password, passphrase, client_secret, +// access_token=..., apikey, Authorization, ...) ends the readable part of the line. +const CREDENTIAL_STEMS = ["pass", "pwd", "secret", "token", "key", "auth", "credential", "bearer", "cookie", "session"] +const isCredential = (token: string) => + token + .toLowerCase() + .split(/[^a-z]+/) + .some((part) => CREDENTIAL_STEMS.some((stem) => part.startsWith(stem))) /** * Allowlist template of an error's first line. Only short plain-letter words and @@ -75,8 +68,6 @@ export function templateErrorMessage(raw: string): string { .replace(/['"].*$/, ` ${PLACEHOLDER}`) const out: string[] = [] let position = 0 - let secretSpan = 0 - let secretToEol = false for (const chunk of first.split(/\s+/)) { if (!chunk) continue if (chunk === PLACEHOLDER) { @@ -85,26 +76,15 @@ export function templateErrorMessage(raw: string): string { continue } const [, lead, core, trail] = PEEL.exec(chunk)! - // the tokens after a credential keyword are values, however plain they look - if (secretSpan > 0 && core !== "") { - out.push(lead + PLACEHOLDER + trail) - position++ - secretSpan = secretToEol ? 1 : /[,.)!?]/.test(trail) ? 0 : secretSpan - 1 - continue - } - if ( - core - .toLowerCase() - .split(/[_\-.]/) - .some((part) => CREDENTIAL_WORDS.has(part)) - ) { - secretSpan = CREDENTIAL_SPAN - // `label: value` / `label=value`: the value may be a multi-word passphrase, so mask to end of line - secretToEol = /[:=]/.test(trail) || /[:=]/.test(lead) + // everything after a credential label is a value, however plain it looks: mask to end of line + if (isCredential(core)) { + const label = WORD.test(core) ? lead + core + trail : lead + PLACEHOLDER + trail + out.push(label, PLACEHOLDER) + break } const plainWord = WORD.test(core) && !(core.length > 3 && core === core.toUpperCase()) const lower = core.toLowerCase() - const name = /^[A-Z]/.test(core) && (position > 0 || !(SENTENCE_STARTERS.has(lower) || CREDENTIAL_WORDS.has(lower))) + const name = /^[A-Z]/.test(core) && (position > 0 || !SENTENCE_STARTERS.has(lower)) out.push(core === "" || (plainWord && !name) ? chunk : lead + PLACEHOLDER + trail) position++ } diff --git a/packages/opencode/test/telemetry/error-detail.test.ts b/packages/opencode/test/telemetry/error-detail.test.ts index 6e946a7f95..7ec1dea6bf 100644 --- a/packages/opencode/test/telemetry/error-detail.test.ts +++ b/packages/opencode/test/telemetry/error-detail.test.ts @@ -31,7 +31,6 @@ describe("templateErrorMessage", () => { "/Users/bob/proj/a.ts", "C:\\Users\\bob\\a.ts", "bob@acme.com", - "https://h.example.com/p?token=zzz", pgUri, "AKIAIOSFODNN7EXAMPLE", "ghp_abcdefghijklmnopqrstuvwxyz0123456789", @@ -47,7 +46,12 @@ describe("templateErrorMessage", () => { test.each([ ["login failed password: supersecret", "login failed password: "], ["Authorization: Bearer secrettoken", "Authorization: "], - ["bad api key hunter, retry", "bad api key , retry"], + ["bad api key hunter, retry", "bad api key "], + ["fetch https://h.example.com/p?token=zzz failed", "fetch "], + ["auth failed: client_secret: abc def ghi", "auth "], + ["bad access_token=hunter two three", "bad "], + ["password correct horse battery staple", "password "], + ["login failed for passphrase correct horse battery staple", "login failed for passphrase "], ["client_secret: supersecret", ""], ["access_token: secrettoken", ""], ["api-key: secretvalue", ""], From 856aac5ff5eecad6c1cf08d91b75d6a7300f90e3 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 17 Oct 2026 18:26:03 -0700 Subject: [PATCH 3/7] fix(telemetry): cut the error line at the first credential word, path or escape Quoted labels (`"password": x`), backslash-escaped quotes and paths containing spaces could carry values past the token-by-token template. The line is now cut at the token holding the first credential stem or path separator and the rest masked. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/altimate/telemetry/error-detail.ts | 20 +++++++++++++++++-- .../test/telemetry/error-detail.test.ts | 10 +++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/packages/opencode/src/altimate/telemetry/error-detail.ts b/packages/opencode/src/altimate/telemetry/error-detail.ts index c71e2531d7..5cd6a6324d 100644 --- a/packages/opencode/src/altimate/telemetry/error-detail.ts +++ b/packages/opencode/src/altimate/telemetry/error-detail.ts @@ -61,9 +61,25 @@ const isCredential = (token: string) => * `@`, `=`, quoted content, capitalized names) becomes ``. English-like words * that happen to spell a SQL statement are not recognised and can survive. */ +// The line is cut at the token holding the first of: a credential stem (also inside quotes or a +// compound key), a path separator, or an escape. Everything from there on is masked, because a +// value, path or escaped string can run on past whitespace and quote boundaries. +const CUT = new RegExp(`${CREDENTIAL_STEMS.join("|")}|[\\\\/]`, "i") + +function cutAtUnsafe(line: string): string { + const hit = CUT.exec(line) + if (!hit) return line + const tokenStart = line.lastIndexOf(" ", hit.index) + 1 + const lead = line.slice(0, tokenStart) + const token = line.slice(tokenStart).split(/\s/, 1)[0] + const unquoted = token.replace(/['"]/g, "") + const label = unquoted.replace(/[:=,]+$/, "") + // keep a plain credential label (`password:`) so the shape stays readable + return isCredential(label) && WORD.test(label) ? `${lead}${unquoted} ${PLACEHOLDER}` : `${lead}${PLACEHOLDER}` +} + export function templateErrorMessage(raw: string): string { - const first = raw - .split(/[\r\n]/, 1)[0] + const first = cutAtUnsafe(raw.split(/[\r\n]/, 1)[0]) .replace(/'[^']*'|"[^"]*"/g, ` ${PLACEHOLDER} `) .replace(/['"].*$/, ` ${PLACEHOLDER}`) const out: string[] = [] diff --git a/packages/opencode/test/telemetry/error-detail.test.ts b/packages/opencode/test/telemetry/error-detail.test.ts index 7ec1dea6bf..7c88fb0288 100644 --- a/packages/opencode/test/telemetry/error-detail.test.ts +++ b/packages/opencode/test/telemetry/error-detail.test.ts @@ -28,10 +28,7 @@ describe("templateErrorMessage", () => { }) test.each([ - "/Users/bob/proj/a.ts", - "C:\\Users\\bob\\a.ts", "bob@acme.com", - pgUri, "AKIAIOSFODNN7EXAMPLE", "ghp_abcdefghijklmnopqrstuvwxyz0123456789", "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.sig", @@ -47,6 +44,13 @@ describe("templateErrorMessage", () => { ["login failed password: supersecret", "login failed password: "], ["Authorization: Bearer secrettoken", "Authorization: "], ["bad api key hunter, retry", "bad api key "], + ["bad /Users/bob/proj/a.ts here", "bad "], + ["bad C:\\Users\\bob\\a.ts here", "bad "], + [`bad ${pgUri} here`, "bad "], + ['bad "password": hunter here', "bad password: "], + ['open "C:\\Program Files\\acme corp\\a.ts" failed', "open "], + ["open /home/x/my customer name/a.ts failed", "open "], + ['value "a \\" customer name" bad', "value "], ["fetch https://h.example.com/p?token=zzz failed", "fetch "], ["auth failed: client_secret: abc def ghi", "auth "], ["bad access_token=hunter two three", "bad "], From 4db44b8b99575129b12ffa9b06cbb0e08234ea54 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 17 Oct 2026 18:37:45 -0700 Subject: [PATCH 4/7] docs: qualify the SQL privacy claim for the error_fingerprint template Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/docs/reference/telemetry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/reference/telemetry.md b/docs/docs/reference/telemetry.md index 5b87b53184..b9fb0820b5 100644 --- a/docs/docs/reference/telemetry.md +++ b/docs/docs/reference/telemetry.md @@ -156,7 +156,7 @@ Two escape hatches exist for reporting from a test run deliberately: We take your privacy seriously. Altimate Code telemetry **never** collects: -- SQL queries or query results +- SQL queries or query results (except that the `error_fingerprint` template can keep a fragment made only of plain words, see below) - Code content, file contents, or file paths - Credentials, API keys, or tokens - Database connection strings or hostnames From 19ed7d272d5ebce077e783a4ef90f8932de08091 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 17 Oct 2026 18:39:11 -0700 Subject: [PATCH 5/7] fix(telemetry): cut the error line at a backtick-quoted span Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/opencode/src/altimate/telemetry/error-detail.ts | 4 ++-- packages/opencode/test/telemetry/error-detail.test.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/altimate/telemetry/error-detail.ts b/packages/opencode/src/altimate/telemetry/error-detail.ts index 5cd6a6324d..0d06a67b6a 100644 --- a/packages/opencode/src/altimate/telemetry/error-detail.ts +++ b/packages/opencode/src/altimate/telemetry/error-detail.ts @@ -62,9 +62,9 @@ const isCredential = (token: string) => * that happen to spell a SQL statement are not recognised and can survive. */ // The line is cut at the token holding the first of: a credential stem (also inside quotes or a -// compound key), a path separator, or an escape. Everything from there on is masked, because a +// compound key), a path separator, an escape, or a backtick-quoted span. Everything from there on is masked, because a // value, path or escaped string can run on past whitespace and quote boundaries. -const CUT = new RegExp(`${CREDENTIAL_STEMS.join("|")}|[\\\\/]`, "i") +const CUT = new RegExp(`${CREDENTIAL_STEMS.join("|")}|[\\\\/\x60]`, "i") function cutAtUnsafe(line: string): string { const hit = CUT.exec(line) diff --git a/packages/opencode/test/telemetry/error-detail.test.ts b/packages/opencode/test/telemetry/error-detail.test.ts index 7c88fb0288..7f3c1c2376 100644 --- a/packages/opencode/test/telemetry/error-detail.test.ts +++ b/packages/opencode/test/telemetry/error-detail.test.ts @@ -44,6 +44,7 @@ describe("templateErrorMessage", () => { ["login failed password: supersecret", "login failed password: "], ["Authorization: Bearer secrettoken", "Authorization: "], ["bad api key hunter, retry", "bad api key "], + ["unknown column `customer name` in field list", "unknown column "], ["bad /Users/bob/proj/a.ts here", "bad "], ["bad C:\\Users\\bob\\a.ts here", "bad "], [`bad ${pgUri} here`, "bad "], From 232a2c83694f10d8a702303952ea93a2c52ed90d Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 17 Oct 2026 18:55:08 -0700 Subject: [PATCH 6/7] fix(telemetry): cut the error line at the first non-ASCII character Typographic quotes and other non-ASCII delimiters wrapped user content the ASCII quote handling did not see. Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/opencode/src/altimate/telemetry/error-detail.ts | 4 ++-- packages/opencode/test/telemetry/error-detail.test.ts | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/altimate/telemetry/error-detail.ts b/packages/opencode/src/altimate/telemetry/error-detail.ts index 0d06a67b6a..02b4944256 100644 --- a/packages/opencode/src/altimate/telemetry/error-detail.ts +++ b/packages/opencode/src/altimate/telemetry/error-detail.ts @@ -62,9 +62,9 @@ const isCredential = (token: string) => * that happen to spell a SQL statement are not recognised and can survive. */ // The line is cut at the token holding the first of: a credential stem (also inside quotes or a -// compound key), a path separator, an escape, or a backtick-quoted span. Everything from there on is masked, because a +// compound key), a path separator, an escape, a backtick, or any non-ASCII character (typographic quotes and the like). Everything from there on is masked, because a // value, path or escaped string can run on past whitespace and quote boundaries. -const CUT = new RegExp(`${CREDENTIAL_STEMS.join("|")}|[\\\\/\x60]`, "i") +const CUT = new RegExp(`${CREDENTIAL_STEMS.join("|")}|[\\\\/\x60]|[^\\x20-\\x7e]`, "i") function cutAtUnsafe(line: string): string { const hit = CUT.exec(line) diff --git a/packages/opencode/test/telemetry/error-detail.test.ts b/packages/opencode/test/telemetry/error-detail.test.ts index 7f3c1c2376..1652d12094 100644 --- a/packages/opencode/test/telemetry/error-detail.test.ts +++ b/packages/opencode/test/telemetry/error-detail.test.ts @@ -45,6 +45,8 @@ describe("templateErrorMessage", () => { ["Authorization: Bearer secrettoken", "Authorization: "], ["bad api key hunter, retry", "bad api key "], ["unknown column `customer name` in field list", "unknown column "], + ["bad value \u201cprivate customer phrase\u201d here", "bad value "], + ["bad \u00abprivate phrase\u00bb here", "bad "], ["bad /Users/bob/proj/a.ts here", "bad "], ["bad C:\\Users\\bob\\a.ts here", "bad "], [`bad ${pgUri} here`, "bad "], From 8feb417668f6d4c7df1cf4e8f6d06fa3017b7ec4 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 17 Oct 2026 18:58:04 -0700 Subject: [PATCH 7/7] refactor: build `error_message` from a fixed vocabulary of error words Split the first line into letter runs and keep only words in `ERROR_VOCABULARY`; every other run becomes ``. Drops the punctuation/quote/credential-stem heuristics, which could not be made complete. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/docs/reference/telemetry.md | 8 +- .../src/altimate/telemetry/error-detail.ts | 454 ++++++++++++++---- .../test/telemetry/error-detail.test.ts | 121 ++--- 3 files changed, 434 insertions(+), 149 deletions(-) diff --git a/docs/docs/reference/telemetry.md b/docs/docs/reference/telemetry.md index b9fb0820b5..4168488bb1 100644 --- a/docs/docs/reference/telemetry.md +++ b/docs/docs/reference/telemetry.md @@ -49,7 +49,7 @@ We collect the following categories of events: | `task_outcome_signal` | Behavioral quality signal at session end — accepted, error, abandoned, or cancelled. Includes tool count, step count, duration, and last tool category. No user content. | | `task_classified` | Intent classification of the first user message using keyword matching — category (e.g. `debug_dbt`, `write_sql`, `optimize_query`), confidence score, and detected warehouse type. No user text is sent — only the classified category. | | `tool_chain_outcome` | Aggregated tool execution sequence at session end — ordered tool names (capped at 50), error count, recovery count, final outcome, duration, and cost. No tool arguments or outputs. | -| `error_fingerprint` | Error pattern for grouping — SHA-256 hash of the masked error message (`error_hash`), plus separate fields for error class, tool name, whether recovery succeeded, and `error_message`. `error_message` is a template of the error's first line: only short plain-letter words and basic punctuation are kept, and identifiers containing digits, `_`, `.`, `/` or `@`, numbers, paths, URLs, quoted values, capitalized names and values after credential keywords are replaced with `` (max 200 characters). Plain lowercase words such as an unquoted table name can survive. | +| `error_fingerprint` | Error pattern for grouping — SHA-256 hash of the masked error message, error class, tool name, whether recovery succeeded, plus `error_message`. `error_message` contains only words from a fixed list of common error terms taken from the first line of the error; every other word is replaced with ``, so no user text, identifiers, paths, values or SQL can appear (max 200 characters). | | `sql_fingerprint` | SQL structural shape via AST parsing — statement types, table count, function count, subquery/aggregation/window function presence, and AST node count. No table names, column names, or SQL content. | | `schema_complexity` | Warehouse schema structural metrics from introspection — bucketed table, column, and schema counts plus average columns per table. No schema names or content. | | `validator_check` | A completion-gate validator ran on session end — validator name, `ok` boolean, step, retry count, `enforced` flag (false in shadow mode), and structured `details` (model counts, elapsed time, concurrency limit — no SQL or model content). Only emitted when `ALTIMATE_VALIDATORS_ENABLED=1` or `ALTIMATE_VALIDATORS_SHADOW=1`. See [Validators](../data-engineering/validators.md). | @@ -156,12 +156,12 @@ Two escape hatches exist for reporting from a test run deliberately: We take your privacy seriously. Altimate Code telemetry **never** collects: -- SQL queries or query results (except that the `error_fingerprint` template can keep a fragment made only of plain words, see below) +- SQL queries or query results - Code content, file contents, or file paths - Credentials, API keys, or tokens - Database connection strings or hostnames - Personally identifiable information (your email is SHA-256 hashed before sending and is used only for anonymous user correlation) -- Tool arguments or outputs, except the first line of tool errors in `error_fingerprint` (the `error_fingerprint` error template keeps only plain English words from the first line of an error; identifiers, numbers, paths and quoted values are replaced with ``. Residual risk: a statement or message written entirely in plain letters, such as an unquoted SQL fragment of simple words, can survive the template) +- Tool arguments or outputs - AI prompt content or responses Error messages are truncated to 500 characters and scrubbed of file paths before sending. @@ -210,7 +210,7 @@ Event type names use **snake_case** with a `domain_action` pattern: - `error_recovered` for error recovery events - `task_outcome_signal`, `task_classified` for session quality signals - `tool_chain_outcome` for tool execution chain aggregation -- `error_fingerprint` for error pattern grouping, with a masked one-line error message +- `error_fingerprint` for error pattern grouping, with a one-line error message built from a fixed vocabulary of common error words - `sql_fingerprint` for SQL structural analysis - `schema_complexity` for warehouse schema metrics diff --git a/packages/opencode/src/altimate/telemetry/error-detail.ts b/packages/opencode/src/altimate/telemetry/error-detail.ts index 02b4944256..dc761dd3dc 100644 --- a/packages/opencode/src/altimate/telemetry/error-detail.ts +++ b/packages/opencode/src/altimate/telemetry/error-detail.ts @@ -2,112 +2,394 @@ import { Telemetry } from "./index" const MAX_MESSAGE = 200 const PLACEHOLDER = "" -const PEEL = /^([:,.()'"\-?!]*)(.*?)([:,.()'"\-?!]*)$/ -const WORD = /^(?:[a-z]{1,24}|[A-Z][a-z]{0,23}|[A-Z]{1,3})$/ -const SENTENCE_STARTERS = new Set([ - "error", - "cannot", +/** + * Fixed vocabulary of common error and diagnostic words. Credential words (password, + * secret, token, key, auth, cookie, session, ...) are deliberately absent. Keep sorted. + */ +export const ERROR_VOCABULARY: ReadonlySet = new Set([ + "a", + "abort", + "aborted", + "about", + "above", + "access", + "across", + "active", + "add", + "after", + "again", + "against", + "all", + "allowed", + "already", + "also", + "always", + "an", + "and", + "another", + "any", + "are", + "argument", + "arguments", + "array", + "as", + "at", + "attempt", + "available", + "bad", + "because", + "been", + "before", + "being", + "between", + "binary", + "blocked", + "both", + "bound", + "buffer", + "busy", + "but", + "by", + "byte", + "bytes", + "call", "can", + "cancel", + "canceled", + "cancelled", + "cannot", + "case", + "changed", + "check", + "class", + "client", + "close", + "closed", + "code", + "column", + "command", + "completed", + "config", + "configuration", + "conflict", + "connect", + "connection", + "constraint", + "content", + "context", "could", + "create", + "created", + "current", + "data", + "database", + "default", + "delete", + "denied", + "dependency", + "did", + "directory", + "disabled", + "disk", + "does", + "done", + "download", + "drop", + "duplicate", + "during", + "each", + "either", + "empty", + "enabled", + "encoding", + "end", + "ended", + "entry", + "error", + "errors", + "exceeded", + "exceeds", + "exist", + "exists", + "exit", + "expect", + "expected", + "expired", + "extension", + "fail", "failed", "failure", - "unable", + "false", + "field", + "file", + "files", + "find", + "first", + "for", + "format", + "found", + "from", + "full", + "function", + "get", + "given", + "hang", + "has", + "have", + "header", + "host", + "http", + "hung", + "if", + "illegal", + "import", + "in", + "incomplete", + "incorrect", + "index", + "input", + "insert", + "instance", + "insufficient", + "internal", + "into", "invalid", - "unknown", + "is", + "it", + "item", + "kill", + "killed", + "language", + "large", + "later", + "least", + "length", + "less", + "like", + "limit", + "line", + "load", + "loaded", + "local", + "lock", + "locked", + "long", + "lost", + "malformed", + "many", + "match", + "max", + "maximum", + "may", + "memory", + "message", + "method", + "might", + "mismatch", "missing", + "mode", + "model", + "module", + "more", + "must", + "name", + "need", + "network", + "new", "no", + "node", + "non", + "none", "not", - "the", - "a", - "an", + "null", + "number", + "object", + "of", + "off", + "offset", + "on", + "one", + "only", + "open", + "operation", + "option", + "options", + "or", + "order", + "other", + "out", + "output", + "overflow", + "page", + "parameter", + "parse", + "parsed", + "parsing", + "part", + "path", "permission", - "access", - "connection", - "timeout", - "request", + "pipe", + "plan", + "port", + "position", + "previous", + "process", + "project", + "protocol", + "provider", "query", - "command", - "file", - "table", - "column", - "unexpected", - "expected", + "quota", + "range", + "rate", + "read", + "readonly", + "reason", + "received", + "record", + "refused", + "rejected", + "relation", + "remote", + "request", + "requests", + "required", + "reset", + "resolve", + "resource", + "response", + "responses", + "result", + "retry", + "return", + "returned", + "root", + "row", + "rows", + "rule", + "running", + "same", + "save", + "schema", + "script", + "select", + "send", + "server", + "service", + "set", + "shell", + "should", + "signal", + "since", + "size", + "skipped", + "socket", + "some", + "source", + "space", + "specified", + "stack", + "start", + "started", + "state", + "statement", + "status", + "stopped", + "stream", + "string", + "success", + "supported", "syntax", - "authentication", - "operation", - "http", - "tool", - "bad", - "login", - "user", - "this", - "it", + "system", + "table", + "target", + "task", + "text", + "than", + "that", + "the", + "then", "there", + "these", + "this", + "those", + "thread", + "through", + "time", + "timed", + "timeout", + "timestamp", + "to", "too", - "failed", + "tool", + "tools", + "top", + "trace", + "transaction", + "true", + "try", + "type", + "unable", + "unavailable", + "undefined", + "unexpected", + "unexpectedly", + "unique", + "unknown", + "unrecognized", + "unsupported", + "until", + "up", + "update", + "upload", + "url", + "use", + "used", + "user", + "valid", + "value", + "values", + "version", + "view", + "wait", + "waiting", + "was", + "were", + "what", + "when", + "where", + "which", + "while", + "who", + "will", + "with", + "within", + "without", + "worker", + "write", + "written", + "wrong", + "yet", + "you", ]) -// A token with a part starting with one of these stems (password, passphrase, client_secret, -// access_token=..., apikey, Authorization, ...) ends the readable part of the line. -const CREDENTIAL_STEMS = ["pass", "pwd", "secret", "token", "key", "auth", "credential", "bearer", "cookie", "session"] -const isCredential = (token: string) => - token - .toLowerCase() - .split(/[^a-z]+/) - .some((part) => CREDENTIAL_STEMS.some((stem) => part.startsWith(stem))) /** - * Allowlist template of an error's first line. Only short plain-letter words and - * a little punctuation survive; every other token (digits, `_`, `/`, `.` inside, - * `@`, `=`, quoted content, capitalized names) becomes ``. English-like words - * that happen to spell a SQL statement are not recognised and can survive. + * Fixed-vocabulary template of an error's first line. Words are split on any non-letter + * character and lowercased; a word survives only if it is in ERROR_VOCABULARY, every other + * run becomes ``. The output holds only vocabulary words, `` and spaces, so user + * text, identifiers, paths, values and SQL cannot appear. */ -// The line is cut at the token holding the first of: a credential stem (also inside quotes or a -// compound key), a path separator, an escape, a backtick, or any non-ASCII character (typographic quotes and the like). Everything from there on is masked, because a -// value, path or escaped string can run on past whitespace and quote boundaries. -const CUT = new RegExp(`${CREDENTIAL_STEMS.join("|")}|[\\\\/\x60]|[^\\x20-\\x7e]`, "i") - -function cutAtUnsafe(line: string): string { - const hit = CUT.exec(line) - if (!hit) return line - const tokenStart = line.lastIndexOf(" ", hit.index) + 1 - const lead = line.slice(0, tokenStart) - const token = line.slice(tokenStart).split(/\s/, 1)[0] - const unquoted = token.replace(/['"]/g, "") - const label = unquoted.replace(/[:=,]+$/, "") - // keep a plain credential label (`password:`) so the shape stays readable - return isCredential(label) && WORD.test(label) ? `${lead}${unquoted} ${PLACEHOLDER}` : `${lead}${PLACEHOLDER}` -} - export function templateErrorMessage(raw: string): string { - const first = cutAtUnsafe(raw.split(/[\r\n]/, 1)[0]) - .replace(/'[^']*'|"[^"]*"/g, ` ${PLACEHOLDER} `) - .replace(/['"].*$/, ` ${PLACEHOLDER}`) + const first = raw.split(/[\r\n\u2028\u2029]/, 1)[0] const out: string[] = [] - let position = 0 - for (const chunk of first.split(/\s+/)) { - if (!chunk) continue - if (chunk === PLACEHOLDER) { - out.push(PLACEHOLDER) - position++ - continue - } - const [, lead, core, trail] = PEEL.exec(chunk)! - // everything after a credential label is a value, however plain it looks: mask to end of line - if (isCredential(core)) { - const label = WORD.test(core) ? lead + core + trail : lead + PLACEHOLDER + trail - out.push(label, PLACEHOLDER) - break - } - const plainWord = WORD.test(core) && !(core.length > 3 && core === core.toUpperCase()) - const lower = core.toLowerCase() - const name = /^[A-Z]/.test(core) && (position > 0 || !SENTENCE_STARTERS.has(lower)) - out.push(core === "" || (plainWord && !name) ? chunk : lead + PLACEHOLDER + trail) - position++ + for (const token of first.split(/[^A-Za-z]+/)) { + if (!token) continue + const word = token.toLowerCase() + const piece = ERROR_VOCABULARY.has(word) ? word : PLACEHOLDER + if (piece === PLACEHOLDER && out[out.length - 1] === PLACEHOLDER) continue + out.push(piece) } - return out - .join(" ") - .replace(/(?:[:,.()'"\-?!]*\s+)+/g, PLACEHOLDER) - .slice(0, MAX_MESSAGE) + // non-letter characters that carried a value (digits, symbols) leave a gap that must stay visible + return out.join(" ").slice(0, MAX_MESSAGE) } /** Hash input is unchanged from before the detail fields existed, so fingerprints stay stable. */ diff --git a/packages/opencode/test/telemetry/error-detail.test.ts b/packages/opencode/test/telemetry/error-detail.test.ts index 1652d12094..58daf34d5c 100644 --- a/packages/opencode/test/telemetry/error-detail.test.ts +++ b/packages/opencode/test/telemetry/error-detail.test.ts @@ -2,79 +2,82 @@ import { describe, expect, test } from "bun:test" import { Telemetry } from "../../src/altimate/telemetry" import { ErrorDetail } from "../../src/altimate/telemetry/error-detail" -const { buildErrorFingerprintEvent, describeToolError, templateErrorMessage } = ErrorDetail -// built at runtime so no credential-shaped literal is committed -const basicAuth = Buffer.from(["adm", "pw"].join(":")).toString("base64") -const pgUri = ["postgres:", "//", "u", ":", "p", "@db:5432/x"].join("") +const { buildErrorFingerprintEvent, describeToolError, templateErrorMessage, ERROR_VOCABULARY } = ErrorDetail + +const CREDENTIAL_WORDS = [ + "password", + "passwd", + "pwd", + "secret", + "token", + "key", + "auth", + "credential", + "bearer", + "cookie", + "session", +] describe("templateErrorMessage", () => { test.each([ - ["column ssn_number does not exist in table hr.salaries", "column does not exist in table "], - ["password: hunter2", "password: "], - ["Cannot delete file: permission denied", "Cannot delete file: permission denied"], - ["failed (code 12)!", "failed (code )!"], - ])("%s", (raw, expected) => { + ["Cannot delete file: permission denied", "cannot delete file permission denied"], + ["connection refused", "connection refused"], + ["column ssn_number does not exist in table hr.salaries", "column number does not exist in table "], + ["Request failed with status 502", "request failed with status"], + ])("readable: %s", (raw, expected) => { expect(templateErrorMessage(raw)).toBe(expected) }) - test("Basic auth header keeps no secret", () => { - const out = templateErrorMessage(`Authorization: Basic ${basicAuth}`) - expect(out).toBe("Authorization: ") - }) + const adversarial = [ + "password: hunter", + "password: supersecret", + "Authorization: Bearer secrettoken", + "client_secret=abcdef api_key: zzz access-token hunter", + "the passphrase is correct horse battery", + "`password` is `hunter`", + "\u201cpassword\u201d: \u201chunter\u201d", + 'say \\"password\\": \\"hunter\\"', + "cannot open /Users/bob smith/My Projects/Acme Corp/data.csv", + "C:\\Users\\bob smith\\a.ts failed", + "SELECT ssn FROM prod.hr.salaries WHERE id = 4417 AND name = Alice", + "Alice Bob Carol failed", + "ERROR FAILED CaNnOt", + "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo0MTIzNDU2Nzg5", + "bob@acme.com not found", + "https://h.example.com/p?token=zzz failed", + "postgres://admin:S3cret@db:5432/x refused", + "\u00e9t\u00e9 \u4e2d\u6587 \u0434\u0430\u043d\u043d\u044b\u0435 failed", + "line one\nsecret line two", + "line one\u2028secret", + "AKIAIOSFODNN7EXAMPLE ghp_abcdefghijklmnopqrstuvwxyz0123456789", + ] - test("unquoted SQL keeps no identifiers, numbers or names", () => { - const out = templateErrorMessage("SELECT ssn FROM prod.hr.salaries WHERE id = 4417 AND name = Alice") - for (const leak of ["prod", "hr.salaries", "4417", "Alice"]) expect(out).not.toContain(leak) + test.each(adversarial)("only vocabulary words or for %j", (raw) => { + const out = templateErrorMessage(raw) + expect(out).toMatch(/^[a-z<>x ]*$/) + for (const tok of out.split(" ").filter(Boolean)) { + expect(tok === "" || ERROR_VOCABULARY.has(tok)).toBe(true) + } + expect(out).not.toContain(" ") + for (const w of CREDENTIAL_WORDS) expect(out).not.toContain(w) }) - test.each([ - "bob@acme.com", - "AKIAIOSFODNN7EXAMPLE", - "ghp_abcdefghijklmnopqrstuvwxyz0123456789", - "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.sig", - "deadbeefdeadbeefdeadbeefdeadbeef0123", - "'hunter2' \"s3cret\"", - "stg_salesforce_history_2024.sql", - ])("replaces identifier-like token in %s", (raw) => { - const out = templateErrorMessage(`bad ${raw} here`) - expect(out).toBe("bad here") + test("user words never survive", () => { + const out = templateErrorMessage(adversarial.join(" ")) + for (const leak of ["hunter", "supersecret", "bob", "smith", "acme", "alice", "salaries", "prod"]) { + expect(out).not.toContain(leak) + } }) - test.each([ - ["login failed password: supersecret", "login failed password: "], - ["Authorization: Bearer secrettoken", "Authorization: "], - ["bad api key hunter, retry", "bad api key "], - ["unknown column `customer name` in field list", "unknown column "], - ["bad value \u201cprivate customer phrase\u201d here", "bad value "], - ["bad \u00abprivate phrase\u00bb here", "bad "], - ["bad /Users/bob/proj/a.ts here", "bad "], - ["bad C:\\Users\\bob\\a.ts here", "bad "], - [`bad ${pgUri} here`, "bad "], - ['bad "password": hunter here', "bad password: "], - ['open "C:\\Program Files\\acme corp\\a.ts" failed', "open "], - ["open /home/x/my customer name/a.ts failed", "open "], - ['value "a \\" customer name" bad', "value "], - ["fetch https://h.example.com/p?token=zzz failed", "fetch "], - ["auth failed: client_secret: abc def ghi", "auth "], - ["bad access_token=hunter two three", "bad "], - ["password correct horse battery staple", "password "], - ["login failed for passphrase correct horse battery staple", "login failed for passphrase "], - ["client_secret: supersecret", ""], - ["access_token: secrettoken", ""], - ["api-key: secretvalue", ""], - ["password: correct horse battery staple", "password: "], - ["Alice does not have access", " does not have access"], - ["failed\raccount alice unavailable", "failed"], - ["Error: value 'private customer phrase is invalid", "Error: value "], - ["xKqPzLmNaBcD failed", " failed"], - ["DROP TABLE prod.customer_records", ""], - ])("masks alphabetic credential values in %s", (raw, expected) => { - expect(templateErrorMessage(raw)).toBe(expected) + test("vocabulary is sorted and holds no credential words", () => { + const list = [...ERROR_VOCABULARY] + expect(list).toEqual([...list].sort()) + for (const w of CREDENTIAL_WORDS) expect(ERROR_VOCABULARY.has(w)).toBe(false) }) test("only the first line, capped at 200 chars", () => { expect(templateErrorMessage("first line\nsecond line")).toBe("first line") - expect(templateErrorMessage("word ".repeat(200)).length).toBeLessThanOrEqual(200) + expect(templateErrorMessage("error ".repeat(200)).length).toBeLessThanOrEqual(200) }) }) @@ -97,7 +100,7 @@ describe("error_fingerprint wiring", () => { } const ev: any = buildErrorFingerprintEvent("s1", base) expect(ev.type).toBe("error_fingerprint") - expect(ev.error_message).toBe("column does not exist in table ") + expect(ev.error_message).toBe("column number does not exist in table ") expect(ev.error_hash).toBe(d.errorHash) expect("stack" in ev).toBe(false) })