diff --git a/docs/docs/reference/telemetry.md b/docs/docs/reference/telemetry.md index 8f4f9d8803..4168488bb1 100644 --- a/docs/docs/reference/telemetry.md +++ b/docs/docs/reference/telemetry.md @@ -49,7 +49,7 @@ We collect the following categories of events: | `task_outcome_signal` | Behavioral quality signal at session end — accepted, error, abandoned, or cancelled. Includes tool count, step count, duration, and last tool category. No user content. | | `task_classified` | Intent classification of the first user message using keyword matching — category (e.g. `debug_dbt`, `write_sql`, `optimize_query`), confidence score, and detected warehouse type. No user text is sent — only the classified category. | | `tool_chain_outcome` | Aggregated tool execution sequence at session end — ordered tool names (capped at 50), error count, recovery count, final outcome, duration, and cost. No tool arguments or outputs. | -| `error_fingerprint` | Hashed error pattern for anonymous grouping — SHA-256 hash of masked error message, error class, tool name, and whether recovery succeeded. Raw error content is never sent. | +| `error_fingerprint` | Error pattern for grouping — SHA-256 hash of the masked error message, error class, tool name, whether recovery succeeded, plus `error_message`. `error_message` contains only words from a fixed list of common error terms taken from the first line of the error; every other word is replaced with ``, so no user text, identifiers, paths, values or SQL can appear (max 200 characters). | | `sql_fingerprint` | SQL structural shape via AST parsing — statement types, table count, function count, subquery/aggregation/window function presence, and AST node count. No table names, column names, or SQL content. | | `schema_complexity` | Warehouse schema structural metrics from introspection — bucketed table, column, and schema counts plus average columns per table. No schema names or content. | | `validator_check` | A completion-gate validator ran on session end — validator name, `ok` boolean, step, retry count, `enforced` flag (false in shadow mode), and structured `details` (model counts, elapsed time, concurrency limit — no SQL or model content). Only emitted when `ALTIMATE_VALIDATORS_ENABLED=1` or `ALTIMATE_VALIDATORS_SHADOW=1`. See [Validators](../data-engineering/validators.md). | @@ -210,7 +210,7 @@ Event type names use **snake_case** with a `domain_action` pattern: - `error_recovered` for error recovery events - `task_outcome_signal`, `task_classified` for session quality signals - `tool_chain_outcome` for tool execution chain aggregation -- `error_fingerprint` for anonymous error pattern grouping +- `error_fingerprint` for error pattern grouping, with a one-line error message built from a fixed vocabulary of common error words - `sql_fingerprint` for SQL structural analysis - `schema_complexity` for warehouse schema metrics diff --git a/packages/opencode/src/altimate/telemetry/error-detail.ts b/packages/opencode/src/altimate/telemetry/error-detail.ts new file mode 100644 index 0000000000..dc761dd3dc --- /dev/null +++ b/packages/opencode/src/altimate/telemetry/error-detail.ts @@ -0,0 +1,429 @@ +import { Telemetry } from "./index" + +const MAX_MESSAGE = 200 +const PLACEHOLDER = "" +/** + * Fixed vocabulary of common error and diagnostic words. Credential words (password, + * secret, token, key, auth, cookie, session, ...) are deliberately absent. Keep sorted. + */ +export const ERROR_VOCABULARY: ReadonlySet = new Set([ + "a", + "abort", + "aborted", + "about", + "above", + "access", + "across", + "active", + "add", + "after", + "again", + "against", + "all", + "allowed", + "already", + "also", + "always", + "an", + "and", + "another", + "any", + "are", + "argument", + "arguments", + "array", + "as", + "at", + "attempt", + "available", + "bad", + "because", + "been", + "before", + "being", + "between", + "binary", + "blocked", + "both", + "bound", + "buffer", + "busy", + "but", + "by", + "byte", + "bytes", + "call", + "can", + "cancel", + "canceled", + "cancelled", + "cannot", + "case", + "changed", + "check", + "class", + "client", + "close", + "closed", + "code", + "column", + "command", + "completed", + "config", + "configuration", + "conflict", + "connect", + "connection", + "constraint", + "content", + "context", + "could", + "create", + "created", + "current", + "data", + "database", + "default", + "delete", + "denied", + "dependency", + "did", + "directory", + "disabled", + "disk", + "does", + "done", + "download", + "drop", + "duplicate", + "during", + "each", + "either", + "empty", + "enabled", + "encoding", + "end", + "ended", + "entry", + "error", + "errors", + "exceeded", + "exceeds", + "exist", + "exists", + "exit", + "expect", + "expected", + "expired", + "extension", + "fail", + "failed", + "failure", + "false", + "field", + "file", + "files", + "find", + "first", + "for", + "format", + "found", + "from", + "full", + "function", + "get", + "given", + "hang", + "has", + "have", + "header", + "host", + "http", + "hung", + "if", + "illegal", + "import", + "in", + "incomplete", + "incorrect", + "index", + "input", + "insert", + "instance", + "insufficient", + "internal", + "into", + "invalid", + "is", + "it", + "item", + "kill", + "killed", + "language", + "large", + "later", + "least", + "length", + "less", + "like", + "limit", + "line", + "load", + "loaded", + "local", + "lock", + "locked", + "long", + "lost", + "malformed", + "many", + "match", + "max", + "maximum", + "may", + "memory", + "message", + "method", + "might", + "mismatch", + "missing", + "mode", + "model", + "module", + "more", + "must", + "name", + "need", + "network", + "new", + "no", + "node", + "non", + "none", + "not", + "null", + "number", + "object", + "of", + "off", + "offset", + "on", + "one", + "only", + "open", + "operation", + "option", + "options", + "or", + "order", + "other", + "out", + "output", + "overflow", + "page", + "parameter", + "parse", + "parsed", + "parsing", + "part", + "path", + "permission", + "pipe", + "plan", + "port", + "position", + "previous", + "process", + "project", + "protocol", + "provider", + "query", + "quota", + "range", + "rate", + "read", + "readonly", + "reason", + "received", + "record", + "refused", + "rejected", + "relation", + "remote", + "request", + "requests", + "required", + "reset", + "resolve", + "resource", + "response", + "responses", + "result", + "retry", + "return", + "returned", + "root", + "row", + "rows", + "rule", + "running", + "same", + "save", + "schema", + "script", + "select", + "send", + "server", + "service", + "set", + "shell", + "should", + "signal", + "since", + "size", + "skipped", + "socket", + "some", + "source", + "space", + "specified", + "stack", + "start", + "started", + "state", + "statement", + "status", + "stopped", + "stream", + "string", + "success", + "supported", + "syntax", + "system", + "table", + "target", + "task", + "text", + "than", + "that", + "the", + "then", + "there", + "these", + "this", + "those", + "thread", + "through", + "time", + "timed", + "timeout", + "timestamp", + "to", + "too", + "tool", + "tools", + "top", + "trace", + "transaction", + "true", + "try", + "type", + "unable", + "unavailable", + "undefined", + "unexpected", + "unexpectedly", + "unique", + "unknown", + "unrecognized", + "unsupported", + "until", + "up", + "update", + "upload", + "url", + "use", + "used", + "user", + "valid", + "value", + "values", + "version", + "view", + "wait", + "waiting", + "was", + "were", + "what", + "when", + "where", + "which", + "while", + "who", + "will", + "with", + "within", + "without", + "worker", + "write", + "written", + "wrong", + "yet", + "you", +]) + +/** + * Fixed-vocabulary template of an error's first line. Words are split on any non-letter + * character and lowercased; a word survives only if it is in ERROR_VOCABULARY, every other + * run becomes ``. The output holds only vocabulary words, `` and spaces, so user + * text, identifiers, paths, values and SQL cannot appear. + */ +export function templateErrorMessage(raw: string): string { + const first = raw.split(/[\r\n\u2028\u2029]/, 1)[0] + const out: string[] = [] + for (const token of first.split(/[^A-Za-z]+/)) { + if (!token) continue + const word = token.toLowerCase() + const piece = ERROR_VOCABULARY.has(word) ? word : PLACEHOLDER + if (piece === PLACEHOLDER && out[out.length - 1] === PLACEHOLDER) continue + out.push(piece) + } + // non-letter characters that carried a value (digits, symbols) leave a gap that must stay visible + return out.join(" ").slice(0, MAX_MESSAGE) +} + +/** Hash input is unchanged from before the detail fields existed, so fingerprints stay stable. */ +export function describeToolError(raw: string) { + return { + errorHash: Telemetry.hashError(Telemetry.maskString(raw).slice(0, 500)), + errorMessage: templateErrorMessage(raw), + } +} + +export function buildErrorFingerprintEvent( + sessionID: string, + err: { + errorHash: string + errorClass: string + toolName: string + toolCategory: string + errorMessage: string + recovered: boolean + recoveryTool: string + }, +): Telemetry.Event { + return { + type: "error_fingerprint", + timestamp: Date.now(), + session_id: sessionID, + error_hash: err.errorHash, + error_class: err.errorClass, + tool_name: err.toolName, + tool_category: err.toolCategory, + error_message: err.errorMessage, + recovery_successful: err.recovered, + recovery_tool: err.recoveryTool, + } +} + +export * as ErrorDetail from "./error-detail" diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 5c16568c1b..b28a8db1ba 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -942,6 +942,8 @@ export namespace Telemetry { tool_name: string /** Tool category */ tool_category: string + /** Allowlist template of the error first line, max 200 chars: plain lowercase words are kept, identifiers, numbers, quoted values, capitalized names and credential values become */ + error_message?: string /** Whether a subsequent tool call succeeded (error was recovered) */ recovery_successful: boolean /** Tool that succeeded after the error (if recovered) */ diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 2711b6dd81..207c761aca 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -113,6 +113,7 @@ import { stampRegistryToolSource, describeMcpTool } from "../altimate/tool-sourc // altimate_change end // altimate_change end import { Telemetry } from "@/telemetry" // altimate_change — session telemetry +import { ErrorDetail } from "@/altimate/telemetry/error-detail" // altimate_change — error_fingerprint payload import { TraceContext } from "@/altimate/observability/trace-context" // altimate_change — client trace per turn import * as OnboardingTelemetry from "@/altimate/telemetry/onboarding" // altimate_change — onboarding funnel // altimate_change start — keyless public Zen predicate (flat module) @@ -801,6 +802,7 @@ export namespace SessionPrompt { toolCategory: string errorClass: string errorHash: string + errorMessage: string recovered: boolean recoveryTool: string } @@ -2176,12 +2178,11 @@ export namespace SessionPrompt { lastToolWasError = true const errorMsg = part.state.status === "error" && typeof part.state.error === "string" ? part.state.error : "unknown" - const masked = Telemetry.maskString(errorMsg).slice(0, 500) pendingError = { toolName: part.tool, toolCategory, errorClass: Telemetry.classifyError(errorMsg), - errorHash: Telemetry.hashError(masked), + ...ErrorDetail.describeToolError(errorMsg), } } else { if (lastToolWasError && pendingError) { @@ -2533,17 +2534,7 @@ export namespace SessionPrompt { } // Error fingerprints — one event per unique error (capped at 20) for (const err of errorRecords.slice(0, 20)) { - Telemetry.track({ - type: "error_fingerprint", - timestamp: Date.now(), - session_id: sessionID, - error_hash: err.errorHash, - error_class: err.errorClass, - tool_name: err.toolName, - tool_category: err.toolCategory, - recovery_successful: err.recovered, - recovery_tool: err.recoveryTool, - }) + Telemetry.track(ErrorDetail.buildErrorFingerprintEvent(sessionID, err)) } // altimate_change end — emit quality signal, tool chain, and error fingerprint events // altimate_change start — populate agent_outcome diagnostic fields diff --git a/packages/opencode/test/telemetry/error-detail.test.ts b/packages/opencode/test/telemetry/error-detail.test.ts new file mode 100644 index 0000000000..58daf34d5c --- /dev/null +++ b/packages/opencode/test/telemetry/error-detail.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, test } from "bun:test" +import { Telemetry } from "../../src/altimate/telemetry" +import { ErrorDetail } from "../../src/altimate/telemetry/error-detail" + +const { buildErrorFingerprintEvent, describeToolError, templateErrorMessage, ERROR_VOCABULARY } = ErrorDetail + +const CREDENTIAL_WORDS = [ + "password", + "passwd", + "pwd", + "secret", + "token", + "key", + "auth", + "credential", + "bearer", + "cookie", + "session", +] + +describe("templateErrorMessage", () => { + test.each([ + ["Cannot delete file: permission denied", "cannot delete file permission denied"], + ["connection refused", "connection refused"], + ["column ssn_number does not exist in table hr.salaries", "column number does not exist in table "], + ["Request failed with status 502", "request failed with status"], + ])("readable: %s", (raw, expected) => { + expect(templateErrorMessage(raw)).toBe(expected) + }) + + const adversarial = [ + "password: hunter", + "password: supersecret", + "Authorization: Bearer secrettoken", + "client_secret=abcdef api_key: zzz access-token hunter", + "the passphrase is correct horse battery", + "`password` is `hunter`", + "\u201cpassword\u201d: \u201chunter\u201d", + 'say \\"password\\": \\"hunter\\"', + "cannot open /Users/bob smith/My Projects/Acme Corp/data.csv", + "C:\\Users\\bob smith\\a.ts failed", + "SELECT ssn FROM prod.hr.salaries WHERE id = 4417 AND name = Alice", + "Alice Bob Carol failed", + "ERROR FAILED CaNnOt", + "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo0MTIzNDU2Nzg5", + "bob@acme.com not found", + "https://h.example.com/p?token=zzz failed", + "postgres://admin:S3cret@db:5432/x refused", + "\u00e9t\u00e9 \u4e2d\u6587 \u0434\u0430\u043d\u043d\u044b\u0435 failed", + "line one\nsecret line two", + "line one\u2028secret", + "AKIAIOSFODNN7EXAMPLE ghp_abcdefghijklmnopqrstuvwxyz0123456789", + ] + + test.each(adversarial)("only vocabulary words or for %j", (raw) => { + const out = templateErrorMessage(raw) + expect(out).toMatch(/^[a-z<>x ]*$/) + for (const tok of out.split(" ").filter(Boolean)) { + expect(tok === "" || ERROR_VOCABULARY.has(tok)).toBe(true) + } + expect(out).not.toContain(" ") + for (const w of CREDENTIAL_WORDS) expect(out).not.toContain(w) + }) + + test("user words never survive", () => { + const out = templateErrorMessage(adversarial.join(" ")) + for (const leak of ["hunter", "supersecret", "bob", "smith", "acme", "alice", "salaries", "prod"]) { + expect(out).not.toContain(leak) + } + }) + + test("vocabulary is sorted and holds no credential words", () => { + const list = [...ERROR_VOCABULARY] + expect(list).toEqual([...list].sort()) + for (const w of CREDENTIAL_WORDS) expect(ERROR_VOCABULARY.has(w)).toBe(false) + }) + + test("only the first line, capped at 200 chars", () => { + expect(templateErrorMessage("first line\nsecond line")).toBe("first line") + expect(templateErrorMessage("error ".repeat(200)).length).toBeLessThanOrEqual(200) + }) +}) + +describe("error_fingerprint wiring", () => { + const raw = "column ssn_number does not exist in table hr.salaries\n at run (/w/packages/opencode/src/a.ts:1:1)" + + test("error_hash is identical to the pre-existing computation", () => { + expect(describeToolError(raw).errorHash).toBe("3a0bc6389647e452") + }) + + test("event carries the template", () => { + const d = describeToolError(raw) + const base = { + ...d, + errorClass: "unknown", + toolName: "bash", + toolCategory: "shell", + recovered: false, + recoveryTool: "", + } + const ev: any = buildErrorFingerprintEvent("s1", base) + expect(ev.type).toBe("error_fingerprint") + expect(ev.error_message).toBe("column number does not exist in table ") + expect(ev.error_hash).toBe(d.errorHash) + expect("stack" in ev).toBe(false) + }) + + test("prompt.ts emits through these helpers", async () => { + const src = await Bun.file(new URL("../../src/session/prompt.ts", import.meta.url)).text() + expect(src).toContain("...ErrorDetail.describeToolError(errorMsg)") + expect(src).toContain("Telemetry.track(ErrorDetail.buildErrorFingerprintEvent(sessionID, err))") + }) +})