From 1c410a12332249ed1d10031ef1fd320136cb7092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Fri, 23 Oct 2026 17:04:53 +0200 Subject: [PATCH 1/6] feat(vocabulary): split a name into its words MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New package @codegraph/vocabulary (PLAN.md §20.1). splitIdentifier(name, kind, lexicon) breaks on every character that is not a letter, a mark or a digit, then on camel-case boundaries read from Unicode categories: lower→Upper, digit→Upper, and an Upper run before Upper+lower. A digit run stays with its word (utf8, md5, v2); a piece of digits only is dropped and counted. The lexicon adds the conventional names skipped whole, the file kinds whose extension is no word, and C#'s I prefix. Hand tables per language, read through each shipped lexicon, the unicode fixture, and fast-check properties: words are lowercase slices of the name, a separator splits a name into its parts' words. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 6 +- eslint.config.js | 1 + packages/vocabulary/package.json | 31 +++ packages/vocabulary/src/index.ts | 9 + packages/vocabulary/src/split.ts | 87 +++++++ packages/vocabulary/test/split.test.ts | 326 +++++++++++++++++++++++++ packages/vocabulary/tsconfig.json | 8 + packages/vocabulary/tsup.config.ts | 10 + pnpm-lock.yaml | 22 ++ 9 files changed, 499 insertions(+), 1 deletion(-) create mode 100644 packages/vocabulary/package.json create mode 100644 packages/vocabulary/src/index.ts create mode 100644 packages/vocabulary/src/split.ts create mode 100644 packages/vocabulary/test/split.test.ts create mode 100644 packages/vocabulary/tsconfig.json create mode 100644 packages/vocabulary/tsup.config.ts diff --git a/CLAUDE.md b/CLAUDE.md index c7c5ac9..18795ef 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -66,6 +66,10 @@ packages/navigator/ @codegraph/navigator — the navigator MODEL: the browsable tree (modules → types → operations/attributes) plus one classified dependency row per base edge, carrying member, provenance and anchor included. Pure computation. +packages/vocabulary/ @codegraph/vocabulary — the words of the model's NAMES + (never source): the universal splitter driven by each + profile's lexicon, the word classes (domain, technical, + keyword, short) and the role weights. Pure computation. packages/insights/ @codegraph/insights — the bottom-up EXPLANATION walk: units (operations → types → modules), SCC-condensed walk order, context packs, prompts, Merkle fingerprints, plan/run, the @@ -107,7 +111,7 @@ fixtures/ Reference corpora + expected model.jsonl snapshots. ### Hard boundaries (convenience does not override architecture) -- **`core`, `analyzer`, `city` and `navigator` never import Three.js or React** +- **`core`, `analyzer`, `city`, `navigator` and `vocabulary` never import Three.js or React** — they must run in Node with no DOM. `viz` reads the city model and `navigator-ui` the navigator model; neither mutates its model and neither re-derives graph facts the analyzer already computes. diff --git a/eslint.config.js b/eslint.config.js index 79946fa..261d4d2 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -58,6 +58,7 @@ export default tseslint.config( "packages/analyzer/**/*.ts", "packages/city/**/*.ts", "packages/navigator/**/*.ts", + "packages/vocabulary/**/*.ts", "packages/insights/**/*.ts", "packages/cli/**/*.ts", ], diff --git a/packages/vocabulary/package.json b/packages/vocabulary/package.json new file mode 100644 index 0000000..d451e86 --- /dev/null +++ b/packages/vocabulary/package.json @@ -0,0 +1,31 @@ +{ + "name": "@codegraph/vocabulary", + "version": "0.2.0", + "license": "MIT", + "type": "module", + "description": "Codegraph vocabulary: the words of a model's names, classified and weighted by the role of what carries them", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": ["dist"], + "scripts": { + "build": "tsup", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run" + }, + "dependencies": { + "@codegraph/core": "workspace:*" + }, + "devDependencies": { + "@types/node": "catalog:", + "fast-check": "catalog:", + "tsup": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:" + } +} diff --git a/packages/vocabulary/src/index.ts b/packages/vocabulary/src/index.ts new file mode 100644 index 0000000..cb3ed45 --- /dev/null +++ b/packages/vocabulary/src/index.ts @@ -0,0 +1,9 @@ +/** + * `@codegraph/vocabulary` — the words a corpus speaks (PLAN.md §20): every + * name the model carries, split into words, each tagged with a class and + * weighted by the role of the entity that carries it. + * + * PURE COMPUTATION, like `@codegraph/city`: reads names and a profile's + * lexicon, never source, and touches no DOM. + */ +export * from "./split.js"; diff --git a/packages/vocabulary/src/split.ts b/packages/vocabulary/src/split.ts new file mode 100644 index 0000000..351e26d --- /dev/null +++ b/packages/vocabulary/src/split.ts @@ -0,0 +1,87 @@ +import type { Lexicon } from "@codegraph/core"; + +/** + * The split (PLAN.md §20.1): one name → the words it is made of. The lexicon + * says which names are conventional, which kinds are named by a file path and + * whether an interface marker leads the name; every word boundary is universal, + * so the algorithm knows no language. + */ + +export interface SplitName { + /** The name's words, lowercased, in the order the name spells them. */ + readonly tokens: readonly string[]; + /** Pieces made of digits only, dropped (`2024` of `promo#2024`). */ + readonly numeric: number; + /** The whole name is one the language dictates (`toString`): it has no word. */ + readonly conventional: boolean; +} + +const CONVENTIONAL: SplitName = Object.freeze({ tokens: Object.freeze([]), numeric: 0, conventional: true }); + +/** Maximal runs of letters, marks and digits: every other character breaks a word. */ +const PIECE = /[\p{L}\p{M}\p{N}]+/gu; +const UPPER = /^[\p{Lu}\p{Lt}]$/u; +const DIGIT = /^\p{N}$/u; +const MARK = /^\p{M}$/u; +const DIGITS_ONLY = /^\p{N}+$/u; +/** `I` + Upper + lower: `IOrderService`, never `IO` or `IOError`. */ +const INTERFACE_PREFIX = /^I(?=[\p{Lu}\p{Lt}]\p{Ll})/u; + +export function splitIdentifier(name: string, kind: string, lexicon: Lexicon | undefined): SplitName { + if (lexicon?.conventional.includes(name) === true) return CONVENTIONAL; + + let text = name; + if (lexicon?.fileKinds?.includes(kind) === true) text = dropExtension(text); + if (lexicon?.interfacePrefix === true) text = text.replace(INTERFACE_PREFIX, ""); + + const tokens: string[] = []; + let numeric = 0; + for (const [piece] of text.matchAll(PIECE)) { + for (const word of camelWords(piece)) { + if (DIGITS_ONLY.test(word)) numeric += 1; + // `toLowerCase` is locale-independent: a Turkish machine writes the same bytes. + else tokens.push(word.toLowerCase()); + } + } + return { tokens, numeric, conventional: false }; +} + +/** The extension of the last path segment; a leading dot (`.eslintrc`) is not one. */ +function dropExtension(path: string): string { + const dot = path.lastIndexOf("."); + return dot > path.lastIndexOf("/") + 1 ? path.slice(0, dot) : path; +} + +type Case = "upper" | "lower" | "digit"; + +/** + * Camel-case boundaries on Unicode categories: lower → Upper, digit → Upper, + * and an Upper run followed by Upper + lower splits before its last letter + * (`HTTPServer` → HTTP, Server). A caseless letter (`𠀀`, `注`) counts as + * lower; a mark takes the case of the letter it sits on, so it never splits. + */ +function camelWords(piece: string): string[] { + const units: { text: string; kind: Case }[] = []; + for (const char of piece) { + const last = units.at(-1); + if (MARK.test(char) && last !== undefined) last.text += char; + else units.push({ text: char, kind: UPPER.test(char) ? "upper" : DIGIT.test(char) ? "digit" : "lower" }); + } + + const words: string[] = []; + let word = ""; + units.forEach((unit, i) => { + const prev = units[i - 1]?.kind; + const next = units[i + 1]?.kind; + const boundary = + unit.kind === "upper" && + (prev === "lower" || prev === "digit" || (prev === "upper" && next === "lower")); + if (boundary && word !== "") { + words.push(word); + word = ""; + } + word += unit.text; + }); + if (word !== "") words.push(word); + return words; +} diff --git a/packages/vocabulary/test/split.test.ts b/packages/vocabulary/test/split.test.ts new file mode 100644 index 0000000..1b5c907 --- /dev/null +++ b/packages/vocabulary/test/split.test.ts @@ -0,0 +1,326 @@ +import { fileURLToPath } from "node:url"; +import { getProfile, readModelFileSync, type Lexicon } from "@codegraph/core"; +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; +import { splitIdentifier } from "../src/split.js"; + +/** + * The splitter (PLAN.md §20.1) turns ONE name into the words it is made of. + * It knows no language: the lexicon brings the conventional names, the file + * kinds and the interface prefix; every word boundary is universal. Each + * table below is one language's idiom, read through that language's shipped + * lexicon, so a lexicon edit that changes a split shows up here by name. + */ + +function lexiconOf(lang: string): Lexicon { + const lexicon = getProfile(lang)?.lexicon; + if (lexicon === undefined) throw new Error(`${lang} ships no lexicon`); + return lexicon; +} + +/** [kind, name, tokens, numeric pieces dropped]; `null` tokens = a conventional name. */ +type Row = readonly [kind: string, name: string, tokens: readonly string[] | null, numeric?: number]; + +function check(lang: string | undefined, rows: readonly Row[]): void { + const lexicon = lang === undefined ? undefined : lexiconOf(lang); + for (const [kind, name, tokens, numeric = 0] of rows) { + it(`${kind} ${name}`, () => { + const split = splitIdentifier(name, kind, lexicon); + if (tokens === null) { + expect(split).toEqual({ tokens: [], numeric: 0, conventional: true }); + } else { + expect(split).toEqual({ tokens, numeric, conventional: false }); + } + }); + } +} + +describe("camel case and acronyms, with no lexicon", () => { + check(undefined, [ + ["class", "OrderManagerRepository", ["order", "manager", "repository"]], + ["class", "HTTPServer", ["http", "server"]], + ["method", "getURLString", ["get", "url", "string"]], + ["class", "XMLHttpRequest", ["xml", "http", "request"]], + ["class", "IO", ["io"]], + ["class", "IOError", ["io", "error"]], + ["field", "MAX_LINES", ["max", "lines"]], + ["field", "x", ["x"]], + // A digit run stays with the word before it; a digit then an upper case + // letter is a camel boundary like any other. + ["class", "Utf8Encoder", ["utf8", "encoder"]], + ["method", "computeMd5", ["compute", "md5"]], + ["class", "V2Api", ["v2", "api"]], + ["class", "HTTP2Server", ["http2", "server"]], + ["module", "i18n", ["i18n"]], + ["class", "2FA", ["fa"], 1], + ["class", "Promo2024", ["promo2024"]], + ["field", "line_2", ["line"], 1], + // Every character that is not a letter, a mark or a digit breaks a word. + ["method", "->", []], + ["method", "a--b", ["a", "b"]], + ]); +}); + +describe("Unicode letters, not ASCII", () => { + check(undefined, [ + ["class", "Résumé", ["résumé"]], + ["property", "Naïve", ["naïve"]], + // A decomposed é: the combining mark belongs to its letter, never a boundary. + ["class", "RésuméLine", ["résumé", "line"]], + ["class", "ＡFullwidth", ["ａ", "fullwidth"]], + ["class", "𠀀Supplementary", ["𠀀", "supplementary"]], + ["class", "注文明細", ["注文明細"]], + // The token is lowercased as a word: a Greek word ends in ς. + ["class", "ΟΔΟΣ", ["οδος"]], + ["class", "TITLE", ["title"]], + ]); +}); + +describe("Java", () => { + check("java", [ + ["package", "com.acme.order", ["com", "acme", "order"]], + ["class", "EmptyBasketException", ["empty", "basket", "exception"]], + ["method", "notifyShipped", ["notify", "shipped"]], + ["method", "toString", null], + ["method", "hashCode", null], + ["method", "main", null], + ["attribute", "serialVersionUID", null], + // Conventional is the WHOLE name: a word inside one is still a word. + ["method", "toStringOrder", ["to", "string", "order"]], + ]); +}); + +describe("C#", () => { + check("csharp", [ + ["interface", "IOrderService", ["order", "service"]], + ["interface", "IPriceable", ["priceable"]], + ["method", "IsFree", ["is", "free"]], + ["class", "IO", ["io"]], + ["class", "IOError", ["io", "error"]], + ["class", "Repository`1", ["repository"], 1], + ["namespace", "Acme.Order.Adapter", ["acme", "order", "adapter"]], + ["method", "op_Addition", null], + ["property", "this[]", null], + ["method", "ToString", null], + ]); +}); + +describe("TypeScript", () => { + check("ts", [ + ["module", "legacy/acme-report.ts", ["legacy", "acme", "report"]], + ["module", "packages/order/src/promo#2024.ts", ["packages", "order", "src", "promo"], 1], + ["module", "packages/order/src/ambient.d.ts", ["packages", "order", "src", "ambient", "d"]], + ["module", "packages/order/src/ui/order-table.tsx", ["packages", "order", "src", "ui", "order", "table"]], + ["module", "legacy-lib", ["legacy", "lib"]], + // The extension is dropped for a FILE kind only. + ["class", "legacy/acme.ts", ["legacy", "acme", "ts"]], + ["property", "#secret", ["secret"]], + ["property", "max.lines", ["max", "lines"]], + ["method", "as.text", ["as", "text"]], + ["function", "default", ["default"]], + ["property", "[Symbol.iterator]", null], + ["method", "constructor", null], + ]); +}); + +describe("JavaScript", () => { + check("js", [ + ["module", "src/app.mjs", ["src", "app"]], + ["module", ".eslintrc", ["eslintrc"]], + ["method", "toJSON", null], + ]); +}); + +describe("Ruby", () => { + check("rb", [ + ["class", "AcmeOrder::Order::Line", ["acme", "order", "order", "line"]], + ["constant", "AcmeOrder::Order::MAX_LINES", ["acme", "order", "order", "max", "lines"]], + ["method", "valid?", ["valid"]], + ["method", "save!", ["save"]], + ["method", "name=", ["name"]], + ["field", "@total", ["total"]], + ["file", "lib/acme_order/promo#2024.rb", ["lib", "acme", "order", "promo"], 1], + ["file", "Gemfile", ["gemfile"]], + ["file", "acme_order.gemspec", ["acme", "order"]], + ["method", "<=>", null], + ["method", "to_s", null], + ["method", "initialize", null], + ["method", "respond_to_missing?", null], + ]); +}); + +describe("Elixir", () => { + check("ex", [ + // A module's dots are not an extension: `module` is not a file kind here. + ["module", "AcmeOrder.Notifier.Sms.Gateway", ["acme", "order", "notifier", "sms", "gateway"]], + ["file", "lib/acme_order/notifier.ex", ["lib", "acme", "order", "notifier"]], + ["file", "test/acme_order/order_test.exs", ["test", "acme", "order", "order", "test"]], + ["function", "available?", ["available"]], + ["function", "fail!", ["fail"]], + ["attribute", "@max_lines", ["max", "lines"]], + ["function", "handle_call", null], + ["function", "start_link", null], + ["macro", "__using__", null], + ["attribute", "@moduledoc", null], + ]); +}); + +describe("Python", () => { + check("python", [ + ["function", "parse_http_response", ["parse", "http", "response"]], + ["class", "HTTPClient", ["http", "client"]], + ["method", "__init__", null], + ["parameter", "self", null], + ["method", "__private_total", ["private", "total"]], + ]); +}); + +describe("Go", () => { + check("go", [ + ["func", "NewOrderService", ["new", "order", "service"]], + ["struct", "httpServer", ["http", "server"]], + ["method", "String", null], + ["method", "ServeHTTP", null], + ]); +}); + +describe("Rust", () => { + check("rust", [ + ["struct", "HttpServer", ["http", "server"]], + ["function", "as_str", ["as", "str"]], + ["method", "fmt", null], + ["method", "into_iter", null], + ]); +}); + +describe("PHP", () => { + check("php", [ + ["codeFile", "src/Order/OrderRepository.php", ["src", "order", "order", "repository"]], + ["namespace", "App\\Order", ["app", "order"]], + ["property", "$orderTotal", ["order", "total"]], + ["method", "__construct", null], + ["method", "__toString", null], + ]); +}); + +describe("Clojure", () => { + check("clj", [ + ["function", "->Order", ["order"]], + ["function", "order-total", ["order", "total"]], + ["function", "valid?", ["valid"]], + ["var", "*out*", ["out"]], + ["namespace", "acme.order.core", ["acme", "order", "core"]], + ["function", "-main", null], + ]); +}); + +describe("the unicode fixture through the splitter", () => { + // fixtures/unicode exists to break code-unit assumptions: a fullwidth letter + // and a supplementary-plane letter (a surrogate pair in UTF-16). + const model = readModelFileSync( + fileURLToPath(new URL("../../../fixtures/unicode/expected/model.jsonl", import.meta.url)), + ); + const lexicon = lexiconOf(model.lang); + const split = (name: string, kind: string): readonly string[] => + splitIdentifier(name, kind, lexicon).tokens; + + it("splits and lowercases every name without a replacement character", () => { + const names = model.entities.map((entity) => { + const name = (entity as Record)["name"]; + if (typeof name !== "string") throw new Error(`${entity.id} has no name`); + return [name, split(name, entity.kind)] as const; + }); + expect(Object.fromEntries(names)).toEqual({ + sorting: ["sorting"], + Ascii: ["ascii"], + "𠀀Supplementary": ["𠀀", "supplementary"], + ＡFullwidth: ["ａ", "fullwidth"], + }); + for (const [, tokens] of names) { + for (const token of tokens) { + expect(token).not.toContain("�"); + // With the u flag, \p{Cs} matches only a surrogate left without its pair. + expect(token).not.toMatch(/\p{Cs}/u); + } + } + }); +}); + +describe("properties", () => { + // Identifiers the grammars admit, plus the characters that break them. Σ is + // left out: its lower case depends on what follows it, so a token's lower + // case and its slice of the lowered name may differ (pinned by hand above). + const CHARS = [ + ..."abcdefghijklmnopqrstuvwxyz", + ..."ABCDEFGHIJKLMNOPQRSTUVWXYZ", + ..."0123456789", + ..."_-.:/?!@$#`*[]=<> ", + "é", + "É", + "́", + "ß", + "Ａ", + "𠀀", + "注", + "Dž", + "İ", + ]; + const identifier = fc.array(fc.constantFrom(...CHARS), { maxLength: 24 }).map((cs) => cs.join("")); + const separator = fc.constantFrom(..."_-.:/?!@$#`*[]=<> "); + const lexicons = fc.constantFrom( + undefined, + lexiconOf("java"), + lexiconOf("csharp"), + lexiconOf("ts"), + lexiconOf("rb"), + lexiconOf("ex"), + ); + const kinds = fc.constantFrom("class", "module", "file", "method"); + const WORD = /^[\p{L}\p{M}\p{N}]+$/u; + + it("yields non-empty lowercase words, each a slice of the lowered name", () => { + fc.assert( + fc.property(identifier, kinds, lexicons, (name, kind, lexicon) => { + for (const token of splitIdentifier(name, kind, lexicon).tokens) { + expect(token).toMatch(WORD); + expect(token).toBe(token.toLowerCase()); + expect(name.toLowerCase()).toContain(token); + expect(token).not.toMatch(/^\p{N}+$/u); + } + }), + ); + }); + + it("is a function of its inputs", () => { + fc.assert( + fc.property(identifier, kinds, lexicons, (name, kind, lexicon) => { + expect(splitIdentifier(name, kind, lexicon)).toEqual(splitIdentifier(name, kind, lexicon)); + }), + ); + }); + + it("splits a + separator + b into the words of a, then the words of b", () => { + // Without a lexicon: a file kind's extension, the interface prefix and a + // conventional name all look at the WHOLE name, not at its parts. + fc.assert( + fc.property(identifier, separator, identifier, (a, sep, b) => { + const left = splitIdentifier(a, "class", undefined); + const right = splitIdentifier(b, "class", undefined); + expect(splitIdentifier(a + sep + b, "class", undefined)).toEqual({ + tokens: [...left.tokens, ...right.tokens], + numeric: left.numeric + right.numeric, + conventional: false, + }); + }), + ); + }); + + it("gives a conventional name no word at all", () => { + const java = lexiconOf("java"); + fc.assert( + fc.property(fc.constantFrom(...java.conventional), kinds, (name, kind) => { + expect(splitIdentifier(name, kind, java)).toEqual({ tokens: [], numeric: 0, conventional: true }); + }), + ); + }); +}); diff --git a/packages/vocabulary/tsconfig.json b/packages/vocabulary/tsconfig.json new file mode 100644 index 0000000..24b956a --- /dev/null +++ b/packages/vocabulary/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": ".", + "outDir": "dist" + }, + "include": ["src", "test"] +} diff --git a/packages/vocabulary/tsup.config.ts b/packages/vocabulary/tsup.config.ts new file mode 100644 index 0000000..16f4d3e --- /dev/null +++ b/packages/vocabulary/tsup.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["esm"], + dts: true, + clean: true, + sourcemap: true, + target: "node22", +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d3776a7..cb7c213 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -371,6 +371,28 @@ importers: specifier: 'catalog:' version: 3.2.7(@types/node@22.20.1)(tsx@4.23.12) + packages/vocabulary: + dependencies: + '@codegraph/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 22.20.1 + fast-check: + specifier: 'catalog:' + version: 4.9.0 + tsup: + specifier: 'catalog:' + version: 8.5.1(postcss@8.5.26)(tsx@4.23.12)(typescript@5.9.3) + typescript: + specifier: 'catalog:' + version: 5.9.3 + vitest: + specifier: 'catalog:' + version: 3.2.7(@types/node@22.20.1)(tsx@4.23.12) + website: devDependencies: '@codegraph/viz': From 60c31486448878f798abd160260e3bf91867f99c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Fri, 23 Oct 2026 17:07:20 +0200 Subject: [PATCH 2/6] feat(vocabulary): classify words MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit createClassifier(lexicon, technical) tags each word, in this order: keyword (the lexicon reserves it), short (one letter or digit, marks excluded), technical (the shared list ∪ the profile's), else domain. A keyword the splitter never yields whole (`defined?`, `__FILE__`) classifies nothing: the `file` of `upload_file` is not `__FILE__`. DEFAULT_TECHNICAL is the plumbing every language shares. Words two or more lexicons listed (args, opts, ctx, err, callback, json, str, mutex, mixin, async, repo) move into it and out of the profiles, as METAMODEL §5.3 already says a profile's list is what lies beyond the shared one; a test keeps the two disjoint. The website profiles page is regenerated from the CLI. Co-Authored-By: Claude Opus 5.5 --- packages/core/src/profiles/clojure.ts | 2 +- packages/core/src/profiles/csharp.ts | 2 +- packages/core/src/profiles/ecmascript.ts | 2 +- packages/core/src/profiles/elixir.ts | 5 +- packages/core/src/profiles/go.ts | 2 +- packages/core/src/profiles/php.ts | 2 +- packages/core/src/profiles/python.ts | 2 +- packages/core/src/profiles/ruby.ts | 2 +- packages/core/src/profiles/rust.ts | 4 +- packages/vocabulary/src/classify.ts | 66 +++++++++ packages/vocabulary/src/index.ts | 1 + packages/vocabulary/test/classify.test.ts | 154 +++++++++++++++++++++ website/content/docs/reference/profiles.md | 20 +-- 13 files changed, 242 insertions(+), 22 deletions(-) create mode 100644 packages/vocabulary/src/classify.ts create mode 100644 packages/vocabulary/test/classify.test.ts diff --git a/packages/core/src/profiles/clojure.ts b/packages/core/src/profiles/clojure.ts index 320a362..8e054c9 100644 --- a/packages/core/src/profiles/clojure.ts +++ b/packages/core/src/profiles/clojure.ts @@ -139,7 +139,7 @@ export const clojureProfile: Profile = { "-main", "-init", "-post-init", "-toString", "-equals", "-hashCode", ], technical: [ - "coll", "ctx", "kw", "ns", "opts", "seq", "sym", "xf", "xform", + "coll", "kw", "ns", "seq", "sym", "xf", "xform", ], }, }; diff --git a/packages/core/src/profiles/csharp.ts b/packages/core/src/profiles/csharp.ts index 8b089b7..5ab8482 100644 --- a/packages/core/src/profiles/csharp.ts +++ b/packages/core/src/profiles/csharp.ts @@ -246,7 +246,7 @@ export const csharpProfile: Profile = { "op_LeftShift", "op_RightShift", "op_UnsignedRightShift", ], technical: [ - "args", "async", "attribute", "enumerable", "enumerator", "extension", "extensions", "linq", + "attribute", "enumerable", "enumerator", "extension", "extensions", "linq", ], interfacePrefix: true, }, diff --git a/packages/core/src/profiles/ecmascript.ts b/packages/core/src/profiles/ecmascript.ts index f3005ab..0ac1bb9 100644 --- a/packages/core/src/profiles/ecmascript.ts +++ b/packages/core/src/profiles/ecmascript.ts @@ -24,6 +24,6 @@ export const ECMASCRIPT_LEXICON = { "[Symbol.toStringTag]", "[Symbol.dispose]", "[Symbol.asyncDispose]", ], technical: [ - "args", "async", "callback", "cb", "fn", "json", "opts", "promise", "prototype", + "cb", "fn", "promise", "prototype", ], } as const satisfies Lexicon; diff --git a/packages/core/src/profiles/elixir.ts b/packages/core/src/profiles/elixir.ts index dcb28ce..0e884ae 100644 --- a/packages/core/src/profiles/elixir.ts +++ b/packages/core/src/profiles/elixir.ts @@ -155,9 +155,8 @@ export const elixirProfile: Profile = { "@fallback_to_any", "@file", ], technical: [ - "attrs", "behaviour", "callback", "changeset", "conn", "ecto", "ets", "gen", "live", "mix", - "opts", "otp", "phoenix", "pid", "plug", "repo", "schema", "server", "socket", "sup", - "supervisor", "web", + "attrs", "behaviour", "changeset", "conn", "ecto", "ets", "gen", "live", "mix", "otp", + "phoenix", "pid", "plug", "schema", "server", "socket", "sup", "supervisor", "web", ], fileKinds: ["file"], }, diff --git a/packages/core/src/profiles/go.ts b/packages/core/src/profiles/go.ts index 4709355..2218be4 100644 --- a/packages/core/src/profiles/go.ts +++ b/packages/core/src/profiles/go.ts @@ -125,7 +125,7 @@ export const goProfile: Profile = { "MarshalJSON", "UnmarshalJSON", "MarshalText", "UnmarshalText", "ServeHTTP", ], technical: [ - "cfg", "cmd", "ctx", "err", "errs", "internal", "mu", "mutex", "pkg", "srv", "svc", "wg", + "cfg", "cmd", "errs", "internal", "mu", "pkg", "srv", "svc", "wg", ], }, }; diff --git a/packages/core/src/profiles/php.ts b/packages/core/src/profiles/php.ts index 07abacd..7dbf31f 100644 --- a/packages/core/src/profiles/php.ts +++ b/packages/core/src/profiles/php.ts @@ -148,7 +148,7 @@ export const phpProfile: Profile = { "__set_state", "__clone", "__debugInfo", ], technical: [ - "arr", "closure", "facade", "middleware", "php", "str", + "arr", "closure", "facade", "middleware", "php", ], fileKinds: ["codeFile"], }, diff --git a/packages/core/src/profiles/python.ts b/packages/core/src/profiles/python.ts index 1686077..a749f65 100644 --- a/packages/core/src/profiles/python.ts +++ b/packages/core/src/profiles/python.ts @@ -131,7 +131,7 @@ export const pythonProfile: Profile = { "__doc__", "__dict__", ], technical: [ - "args", "dict", "kwargs", "metaclass", "mixin", "py", "str", "tuple", + "dict", "kwargs", "metaclass", "py", "tuple", ], }, }; diff --git a/packages/core/src/profiles/ruby.ts b/packages/core/src/profiles/ruby.ts index 727ccdc..48f6239 100644 --- a/packages/core/src/profiles/ruby.ts +++ b/packages/core/src/profiles/ruby.ts @@ -133,7 +133,7 @@ export const rubyProfile: Profile = { ], technical: [ "accessor", "attr", "concern", "concerns", "ext", "gem", "gemspec", "klass", "lambda", - "mixin", "proc", "rake", "rakefile", + "proc", "rake", "rakefile", ], fileKinds: ["file"], }, diff --git a/packages/core/src/profiles/rust.ts b/packages/core/src/profiles/rust.ts index 5c943ed..a45d1a6 100644 --- a/packages/core/src/profiles/rust.ts +++ b/packages/core/src/profiles/rust.ts @@ -124,8 +124,8 @@ export const rustProfile: Profile = { "borrow_mut", "index_mut", "default", "poll", ], technical: [ - "arc", "buf", "cell", "ctx", "err", "ffi", "iter", "mutex", "ok", "opt", "rc", "refcell", - "rs", "rx", "sys", "tx", "vec", + "arc", "buf", "cell", "ffi", "iter", "ok", "opt", "rc", "refcell", "rs", "rx", "sys", + "tx", "vec", ], }, }; diff --git a/packages/vocabulary/src/classify.ts b/packages/vocabulary/src/classify.ts new file mode 100644 index 0000000..738bb2d --- /dev/null +++ b/packages/vocabulary/src/classify.ts @@ -0,0 +1,66 @@ +import type { Lexicon } from "@codegraph/core"; + +/** + * The classes (PLAN.md §20.1): every word is kept and tagged, never filtered — + * drift needs the technical words too, and "not shown" must stay + * distinguishable from "not there". + */ + +export const TERM_CLASSES = ["domain", "technical", "keyword", "short"] as const; +export type TermClass = (typeof TERM_CLASSES)[number]; + +/** + * Plumbing every language shares. Contestable by construction, which is why + * it is data in one place, overridable whole, and the artifact names the list + * it used. A language's own plumbing (Java's `bean`, Ruby's `concern`) is its + * profile's `lexicon.technical`, never repeated here. + */ +export const DEFAULT_TECHNICAL: readonly string[] = Object.freeze([ + "abstract", "adapter", "add", "and", "api", "app", "arg", "args", "as", "async", + "at", "base", "build", "builder", "by", "callback", "can", "cli", "com", "common", + "config", "configuration", "context", "controller", "core", "create", "ctx", "dao", "data", "default", + "delete", "dto", "entity", "env", "err", "error", "errors", "exception", "factory", "fake", + "fetch", "find", "fixture", "for", "from", "generic", "get", "handle", "handler", "has", + "helper", "helpers", "http", "id", "impl", "in", "info", "init", "into", "io", + "is", "json", "lib", "listener", "load", "main", "make", "manager", "misc", "mixin", + "mock", "model", "mutex", "net", "new", "not", "obj", "of", "on", "options", + "opts", "or", "org", "param", "params", "provider", "proxy", "remove", "repo", "repository", + "req", "request", "res", "resp", "response", "result", "run", "save", "service", "set", + "settings", "setup", "shared", "should", "spec", "src", "str", "stub", "teardown", "temp", + "test", "tests", "tmp", "to", "update", "uri", "url", "util", "utils", "uuid", + "val", "visitor", "with", "wrapper", "xml", +]); + +export type Classify = (token: string) => TermClass; + +const WORD = /^[\p{L}\p{M}\p{N}]+$/u; +const MARK = /\p{M}/u; + +/** + * Decides, in this order: `keyword` (the lexicon reserves the word), `short` + * (one letter or digit), `technical` (the shared list ∪ the profile's), else + * `domain`. A keyword the splitter never yields whole (`defined?`, `__FILE__`) + * classifies nothing: the `file` of `upload_file` is not `__FILE__`. + */ +export function createClassifier( + lexicon: Lexicon | undefined, + technical: readonly string[] = DEFAULT_TECHNICAL, +): Classify { + const keywords = new Set( + (lexicon?.keywords ?? []).map((keyword) => keyword.toLowerCase()).filter((keyword) => WORD.test(keyword)), + ); + const plumbing = new Set([...technical, ...(lexicon?.technical ?? [])]); + return (token) => { + if (keywords.has(token)) return "keyword"; + if (isShort(token)) return "short"; + if (plumbing.has(token)) return "technical"; + return "domain"; + }; +} + +/** One code point that is not a mark: `x`, `𠀀`, a decomposed `é`. */ +function isShort(token: string): boolean { + let letters = 0; + for (const char of token) if (!MARK.test(char) && ++letters > 1) return false; + return true; +} diff --git a/packages/vocabulary/src/index.ts b/packages/vocabulary/src/index.ts index cb3ed45..8fc1bec 100644 --- a/packages/vocabulary/src/index.ts +++ b/packages/vocabulary/src/index.ts @@ -7,3 +7,4 @@ * lexicon, never source, and touches no DOM. */ export * from "./split.js"; +export * from "./classify.js"; diff --git a/packages/vocabulary/test/classify.test.ts b/packages/vocabulary/test/classify.test.ts new file mode 100644 index 0000000..ed284d3 --- /dev/null +++ b/packages/vocabulary/test/classify.test.ts @@ -0,0 +1,154 @@ +import { PROFILES, getProfile, type Lexicon } from "@codegraph/core"; +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; +import { DEFAULT_TECHNICAL, TERM_CLASSES, createClassifier, type TermClass } from "../src/classify.js"; + +/** + * Classification, not filtering (PLAN.md §20 principle 3): every word is kept + * and tagged; a consumer decides what to show. The order is fixed — keyword, + * short, technical, domain — so a word the language reserves is a keyword + * even where the shared list calls it plumbing. + */ + +function lexiconOf(lang: string): Lexicon { + const lexicon = getProfile(lang)?.lexicon; + if (lexicon === undefined) throw new Error(`${lang} ships no lexicon`); + return lexicon; +} + +const WORD = /^[\p{L}\p{M}\p{N}]+$/u; + +describe("the classes", () => { + it("are a closed set of four", () => { + expect(TERM_CLASSES).toEqual(["domain", "technical", "keyword", "short"]); + }); +}); + +describe("one word, per language", () => { + const rows: readonly (readonly [lang: string, token: string, expected: TermClass])[] = [ + ["java", "order", "domain"], + ["java", "new", "keyword"], + ["java", "default", "keyword"], + // A contextual keyword is an identifier: Java's `record`, TypeScript's `type`. + ["java", "type", "domain"], + ["java", "record", "domain"], + ["java", "get", "technical"], + ["java", "bean", "technical"], + ["java", "args", "technical"], + ["java", "i", "short"], + // Two letters are a word: `id`, `io` and `db` are never short. + ["java", "id", "technical"], + ["java", "io", "technical"], + ["java", "db", "domain"], + ["go", "type", "keyword"], + ["go", "ctx", "technical"], + ["go", "wg", "technical"], + ["rust", "self", "keyword"], + ["rust", "async", "keyword"], + ["rust", "new", "technical"], + ["rust", "vec", "technical"], + ["rb", "end", "keyword"], + ["rb", "begin", "keyword"], + ["rb", "concern", "technical"], + ["rb", "bean", "domain"], + // A keyword the splitter can never yield whole classifies nothing: + // the `file` of `upload_file` is not `__FILE__`. + ["rb", "file", "domain"], + ["rb", "defined", "domain"], + ["php", "once", "domain"], + ["clj", "monitor", "domain"], + ["clj", "recur", "keyword"], + ["ts", "type", "domain"], + ["ts", "default", "keyword"], + ["ts", "promise", "technical"], + ["ts", "json", "technical"], + ["csharp", "event", "keyword"], + ["csharp", "string", "keyword"], + ["csharp", "async", "technical"], + ["csharp", "linq", "technical"], + ["php", "list", "keyword"], + ["python", "new", "technical"], + ["ex", "changeset", "technical"], + ["ex", "repo", "technical"], + ]; + + it.each(rows)("%s: %s is %s", (lang, token, expected) => { + expect(createClassifier(lexiconOf(lang))(token)).toBe(expected); + }); +}); + +describe("short", () => { + const classify = createClassifier(undefined); + + it("is one letter or digit, counted in code points, marks excluded", () => { + for (const token of ["x", "𠀀", "ａ", "é", "注"]) expect(classify(token), token).toBe("short"); + for (const token of ["id", "io", "db", "注文"]) expect(classify(token), token).not.toBe("short"); + }); +}); + +describe("the technical list", () => { + it("is overridable whole; the profile's own words still apply", () => { + const classify = createClassifier(lexiconOf("java"), ["order"]); + expect(classify("order")).toBe("technical"); + expect(classify("get")).toBe("domain"); + expect(classify("bean")).toBe("technical"); + expect(classify("new")).toBe("keyword"); + }); + + it("applies alone when there is no lexicon", () => { + const classify = createClassifier(undefined); + expect(classify("new")).toBe("technical"); + expect(classify("order")).toBe("domain"); + }); +}); + +describe("DEFAULT_TECHNICAL", () => { + it("is sorted, unique, and holds only words the splitter can yield", () => { + expect([...DEFAULT_TECHNICAL]).toEqual([...new Set(DEFAULT_TECHNICAL)].sort()); + for (const word of DEFAULT_TECHNICAL) { + expect(word, word).toMatch(WORD); + expect(word, word).toBe(word.toLowerCase()); + // A one-letter word is short before it could be technical. + expect([...word].length, word).toBeGreaterThan(1); + } + }); + + it("holds the plan's seed list", () => { + const seed = `get set is has to from of with by and or not new create build make init impl + abstract base default util utils helper manager service repository controller handler + factory provider dto dao entity model data info config exception error test mock spec + com org net io src lib main app`.split(/\s+/); + expect(DEFAULT_TECHNICAL).toEqual(expect.arrayContaining(seed)); + }); + + // A profile's `technical` is the plumbing of THAT language, beyond the shared + // list (METAMODEL §5.3): a word in both is one fact kept in two places. + it.each(Object.keys(PROFILES).sort())("shares no word with %s's own list", (lang) => { + const own = lexiconOf(lang).technical ?? []; + expect(own.filter((word) => DEFAULT_TECHNICAL.includes(word))).toEqual([]); + }); +}); + +describe("properties", () => { + const lexicons = fc.constantFrom(...Object.keys(PROFILES).sort()).map(lexiconOf); + const token = fc.stringMatching(/^[a-z0-9é]{1,8}$/); + + it("calls a word a keyword only when the lexicon reserves that word", () => { + fc.assert( + fc.property(lexicons, token, (lexicon, word) => { + const reserved = lexicon.keywords.map((keyword) => keyword.toLowerCase()).includes(word); + expect(createClassifier(lexicon)(word) === "keyword").toBe(reserved); + }), + ); + }); + + it("gives every shipped keyword that is a word the keyword class", () => { + for (const lang of Object.keys(PROFILES)) { + const lexicon = lexiconOf(lang); + const classify = createClassifier(lexicon); + for (const keyword of lexicon.keywords.filter((k) => WORD.test(k))) { + expect(classify(keyword.toLowerCase()), `${lang} ${keyword}`).toBe("keyword"); + } + } + }); +}); diff --git a/website/content/docs/reference/profiles.md b/website/content/docs/reference/profiles.md index f453331..f54494c 100644 --- a/website/content/docs/reference/profiles.md +++ b/website/content/docs/reference/profiles.md @@ -56,7 +56,7 @@ A profile that is specified but has no shipped extractor is complete, not defici - Reserved words: `def` · `if` · `do` · `let` · `quote` · `var` · `fn` · `loop` · `recur` · `throw` · `try` · `catch` · `finally` · `monitor-enter` · `monitor-exit` · `new` · `set!` · `nil` · `true` · `false` - Conventional names: `-main` · `-init` · `-post-init` · `-toString` · `-equals` · `-hashCode` -- Technical words: `coll` · `ctx` · `kw` · `ns` · `opts` · `seq` · `sym` · `xf` · `xform` +- Technical words: `coll` · `kw` · `ns` · `seq` · `sym` · `xf` · `xform` **Notes** @@ -100,7 +100,7 @@ A profile that is specified but has no shipped extractor is complete, not defici - Reserved words: `abstract` · `as` · `base` · `bool` · `break` · `byte` · `case` · `catch` · `char` · `checked` · `class` · `const` · `continue` · `decimal` · `default` · `delegate` · `do` · `double` · `else` · `enum` · `event` · `explicit` · `extern` · `false` · `finally` · `fixed` · `float` · `for` · `foreach` · `goto` · `if` · `implicit` · `in` · `int` · `interface` · `internal` · `is` · `lock` · `long` · `namespace` · `new` · `null` · `object` · `operator` · `out` · `override` · `params` · `private` · `protected` · `public` · `readonly` · `ref` · `return` · `sbyte` · `sealed` · `short` · `sizeof` · `stackalloc` · `static` · `string` · `struct` · `switch` · `this` · `throw` · `true` · `try` · `typeof` · `uint` · `ulong` · `unchecked` · `unsafe` · `ushort` · `using` · `virtual` · `void` · `volatile` · `while` - Conventional names: `ToString` · `Equals` · `GetHashCode` · `Finalize` · `Main` · `CompareTo` · `Dispose` · `DisposeAsync` · `GetEnumerator` · `GetAsyncEnumerator` · `MoveNext` · `MoveNextAsync` · `Clone` · `Deconstruct` · `this[]` · `op_Addition` · `op_Subtraction` · `op_Multiply` · `op_Division` · `op_Modulus` · `op_Equality` · `op_Inequality` · `op_LessThan` · `op_GreaterThan` · `op_LessThanOrEqual` · `op_GreaterThanOrEqual` · `op_Implicit` · `op_Explicit` · `op_UnaryNegation` · `op_UnaryPlus` · `op_LogicalNot` · `op_OnesComplement` · `op_Increment` · `op_Decrement` · `op_True` · `op_False` · `op_BitwiseAnd` · `op_BitwiseOr` · `op_ExclusiveOr` · `op_LeftShift` · `op_RightShift` · `op_UnsignedRightShift` -- Technical words: `args` · `async` · `attribute` · `enumerable` · `enumerator` · `extension` · `extensions` · `linq` +- Technical words: `attribute` · `enumerable` · `enumerator` · `extension` · `extensions` · `linq` - Interface prefix: a leading `I` before Upper + lower is a marker, not a word **Notes** @@ -147,7 +147,7 @@ A profile that is specified but has no shipped extractor is complete, not defici - Reserved words: `true` · `false` · `nil` · `when` · `and` · `or` · `not` · `in` · `fn` · `do` · `end` · `catch` · `rescue` · `after` · `else` - Conventional names: `__using__` · `__before_compile__` · `__after_compile__` · `__after_verify__` · `__on_definition__` · `__struct__` · `__info__` · `__deriving__` · `__impl__` · `__protocol__` · `start_link` · `child_spec` · `init` · `handle_call` · `handle_cast` · `handle_info` · `handle_continue` · `terminate` · `code_change` · `format_status` · `to_string` · `inspect` · `@moduledoc` · `@doc` · `@typedoc` · `@spec` · `@type` · `@typep` · `@opaque` · `@callback` · `@macrocallback` · `@optional_callbacks` · `@impl` · `@behaviour` · `@derive` · `@enforce_keys` · `@external_resource` · `@compile` · `@deprecated` · `@dialyzer` · `@on_load` · `@vsn` · `@before_compile` · `@after_compile` · `@after_verify` · `@on_definition` · `@nifs` · `@fallback_to_any` · `@file` -- Technical words: `attrs` · `behaviour` · `callback` · `changeset` · `conn` · `ecto` · `ets` · `gen` · `live` · `mix` · `opts` · `otp` · `phoenix` · `pid` · `plug` · `repo` · `schema` · `server` · `socket` · `sup` · `supervisor` · `web` +- Technical words: `attrs` · `behaviour` · `changeset` · `conn` · `ecto` · `ets` · `gen` · `live` · `mix` · `otp` · `phoenix` · `pid` · `plug` · `schema` · `server` · `socket` · `sup` · `supervisor` · `web` - File kinds: `file` **Notes** @@ -195,7 +195,7 @@ A profile that is specified but has no shipped extractor is complete, not defici - Reserved words: `break` · `case` · `chan` · `const` · `continue` · `default` · `defer` · `else` · `fallthrough` · `for` · `func` · `go` · `goto` · `if` · `import` · `interface` · `map` · `package` · `range` · `return` · `select` · `struct` · `switch` · `type` · `var` - Conventional names: `init` · `main` · `String` · `GoString` · `Error` · `Unwrap` · `Is` · `As` · `Len` · `Less` · `Swap` · `MarshalJSON` · `UnmarshalJSON` · `MarshalText` · `UnmarshalText` · `ServeHTTP` -- Technical words: `cfg` · `cmd` · `ctx` · `err` · `errs` · `internal` · `mu` · `mutex` · `pkg` · `srv` · `svc` · `wg` +- Technical words: `cfg` · `cmd` · `errs` · `internal` · `mu` · `pkg` · `srv` · `svc` · `wg` **Notes** @@ -273,7 +273,7 @@ A profile that is specified but has no shipped extractor is complete, not defici - Reserved words: `await` · `break` · `case` · `catch` · `class` · `const` · `continue` · `debugger` · `default` · `delete` · `do` · `else` · `enum` · `export` · `extends` · `false` · `finally` · `for` · `function` · `if` · `import` · `in` · `instanceof` · `new` · `null` · `return` · `super` · `switch` · `this` · `throw` · `true` · `try` · `typeof` · `var` · `void` · `while` · `with` · `yield` · `implements` · `interface` · `let` · `package` · `private` · `protected` · `public` · `static` - Conventional names: `constructor` · `toString` · `toLocaleString` · `valueOf` · `toJSON` · `[Symbol.iterator]` · `[Symbol.asyncIterator]` · `[Symbol.hasInstance]` · `[Symbol.toPrimitive]` · `[Symbol.toStringTag]` · `[Symbol.dispose]` · `[Symbol.asyncDispose]` -- Technical words: `args` · `async` · `callback` · `cb` · `fn` · `json` · `opts` · `promise` · `prototype` +- Technical words: `cb` · `fn` · `promise` · `prototype` - File kinds: `module` **Notes** @@ -315,7 +315,7 @@ A profile that is specified but has no shipped extractor is complete, not defici - Reserved words: `__halt_compiler` · `abstract` · `and` · `array` · `as` · `break` · `callable` · `case` · `catch` · `class` · `clone` · `const` · `continue` · `declare` · `default` · `die` · `do` · `echo` · `else` · `elseif` · `empty` · `enddeclare` · `endfor` · `endforeach` · `endif` · `endswitch` · `endwhile` · `eval` · `exit` · `extends` · `final` · `finally` · `fn` · `for` · `foreach` · `function` · `global` · `goto` · `if` · `implements` · `include` · `include_once` · `instanceof` · `insteadof` · `interface` · `isset` · `list` · `match` · `namespace` · `new` · `or` · `print` · `private` · `protected` · `public` · `readonly` · `require` · `require_once` · `return` · `static` · `switch` · `throw` · `trait` · `try` · `unset` · `use` · `var` · `while` · `xor` · `yield` · `__CLASS__` · `__DIR__` · `__FILE__` · `__FUNCTION__` · `__LINE__` · `__METHOD__` · `__NAMESPACE__` · `__PROPERTY__` · `__TRAIT__` · `true` · `false` · `null` - Conventional names: `__construct` · `__destruct` · `__call` · `__callStatic` · `__get` · `__set` · `__isset` · `__unset` · `__sleep` · `__wakeup` · `__serialize` · `__unserialize` · `__toString` · `__invoke` · `__set_state` · `__clone` · `__debugInfo` -- Technical words: `arr` · `closure` · `facade` · `middleware` · `php` · `str` +- Technical words: `arr` · `closure` · `facade` · `middleware` · `php` - File kinds: `codeFile` **Notes** @@ -351,7 +351,7 @@ A profile that is specified but has no shipped extractor is complete, not defici - Reserved words: `False` · `None` · `True` · `and` · `as` · `assert` · `async` · `await` · `break` · `class` · `continue` · `def` · `del` · `elif` · `else` · `except` · `finally` · `for` · `from` · `global` · `if` · `import` · `in` · `is` · `lambda` · `nonlocal` · `not` · `or` · `pass` · `raise` · `return` · `try` · `while` · `with` · `yield` - Conventional names: `self` · `cls` · `__init__` · `__new__` · `__del__` · `__repr__` · `__str__` · `__bytes__` · `__format__` · `__hash__` · `__bool__` · `__eq__` · `__ne__` · `__lt__` · `__le__` · `__gt__` · `__ge__` · `__len__` · `__iter__` · `__next__` · `__aiter__` · `__anext__` · `__reversed__` · `__contains__` · `__getitem__` · `__setitem__` · `__delitem__` · `__missing__` · `__getattr__` · `__getattribute__` · `__setattr__` · `__delattr__` · `__dir__` · `__get__` · `__set__` · `__delete__` · `__set_name__` · `__call__` · `__enter__` · `__exit__` · `__aenter__` · `__aexit__` · `__await__` · `__init_subclass__` · `__class_getitem__` · `__post_init__` · `__add__` · `__sub__` · `__mul__` · `__matmul__` · `__truediv__` · `__floordiv__` · `__mod__` · `__pow__` · `__neg__` · `__pos__` · `__abs__` · `__invert__` · `__and__` · `__or__` · `__xor__` · `__lshift__` · `__rshift__` · `__int__` · `__float__` · `__index__` · `__round__` · `__copy__` · `__deepcopy__` · `__reduce__` · `__getstate__` · `__setstate__` · `__slots__` · `__all__` · `__name__` · `__main__` · `__doc__` · `__dict__` -- Technical words: `args` · `dict` · `kwargs` · `metaclass` · `mixin` · `py` · `str` · `tuple` +- Technical words: `dict` · `kwargs` · `metaclass` · `py` · `tuple` **Notes** @@ -384,7 +384,7 @@ A profile that is specified but has no shipped extractor is complete, not defici - Reserved words: `__ENCODING__` · `__LINE__` · `__FILE__` · `BEGIN` · `END` · `alias` · `and` · `begin` · `break` · `case` · `class` · `def` · `defined?` · `do` · `else` · `elsif` · `end` · `ensure` · `false` · `for` · `if` · `in` · `module` · `next` · `nil` · `not` · `or` · `redo` · `rescue` · `retry` · `return` · `self` · `super` · `then` · `true` · `undef` · `unless` · `until` · `when` · `while` · `yield` - Conventional names: `initialize` · `initialize_copy` · `to_s` · `to_str` · `to_a` · `to_ary` · `to_h` · `to_hash` · `to_proc` · `inspect` · `hash` · `eql?` · `==` · `===` · `=~` · `!~` · `!=` · `<=>` · `<` · `>` · `<=` · `>=` · `+` · `-` · `*` · `/` · `%` · `**` · `<<` · `>>` · `&` · `|` · `^` · `~` · `!` · `+@` · `-@` · `[]` · `[]=` · `each` · `coerce` · `method_missing` · `respond_to_missing?` · `respond_to?` · `included` · `extended` · `inherited` · `prepended` · `method_added` · `const_missing` -- Technical words: `accessor` · `attr` · `concern` · `concerns` · `ext` · `gem` · `gemspec` · `klass` · `lambda` · `mixin` · `proc` · `rake` · `rakefile` +- Technical words: `accessor` · `attr` · `concern` · `concerns` · `ext` · `gem` · `gemspec` · `klass` · `lambda` · `proc` · `rake` · `rakefile` - File kinds: `file` **Notes** @@ -441,7 +441,7 @@ A profile that is specified but has no shipped extractor is complete, not defici - Reserved words: `as` · `async` · `await` · `break` · `const` · `continue` · `crate` · `dyn` · `else` · `enum` · `extern` · `false` · `fn` · `for` · `if` · `impl` · `in` · `let` · `loop` · `match` · `mod` · `move` · `mut` · `pub` · `ref` · `return` · `self` · `Self` · `static` · `struct` · `super` · `trait` · `true` · `type` · `unsafe` · `use` · `where` · `while` · `abstract` · `become` · `box` · `do` · `final` · `gen` · `macro` · `override` · `priv` · `try` · `typeof` · `unsized` · `virtual` · `yield` - Conventional names: `main` · `fmt` · `drop` · `clone` · `clone_from` · `eq` · `ne` · `partial_cmp` · `cmp` · `deref` · `deref_mut` · `into_iter` · `from_str` · `try_from` · `try_into` · `as_ref` · `as_mut` · `borrow_mut` · `index_mut` · `default` · `poll` -- Technical words: `arc` · `buf` · `cell` · `ctx` · `err` · `ffi` · `iter` · `mutex` · `ok` · `opt` · `rc` · `refcell` · `rs` · `rx` · `sys` · `tx` · `vec` +- Technical words: `arc` · `buf` · `cell` · `ffi` · `iter` · `ok` · `opt` · `rc` · `refcell` · `rs` · `rx` · `sys` · `tx` · `vec` **Notes** @@ -485,7 +485,7 @@ Declaration spaces: `abstractClass`, `arrowFunction`, `class`, `enum`, `function - Reserved words: `await` · `break` · `case` · `catch` · `class` · `const` · `continue` · `debugger` · `default` · `delete` · `do` · `else` · `enum` · `export` · `extends` · `false` · `finally` · `for` · `function` · `if` · `import` · `in` · `instanceof` · `new` · `null` · `return` · `super` · `switch` · `this` · `throw` · `true` · `try` · `typeof` · `var` · `void` · `while` · `with` · `yield` · `implements` · `interface` · `let` · `package` · `private` · `protected` · `public` · `static` - Conventional names: `constructor` · `toString` · `toLocaleString` · `valueOf` · `toJSON` · `[Symbol.iterator]` · `[Symbol.asyncIterator]` · `[Symbol.hasInstance]` · `[Symbol.toPrimitive]` · `[Symbol.toStringTag]` · `[Symbol.dispose]` · `[Symbol.asyncDispose]` -- Technical words: `args` · `async` · `callback` · `cb` · `fn` · `json` · `opts` · `promise` · `prototype` +- Technical words: `cb` · `fn` · `promise` · `prototype` - File kinds: `module` **Notes** From 26fee22c62285f0d212b829e5f9c837ed2a0a4ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Fri, 23 Oct 2026 17:09:15 +0200 Subject: [PATCH 3/6] feat(vocabulary): roles and weights MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit roleOf(entity, slots) reads traits, never the kind: TModule → module, TType → type, TInvocable → operation, TStructural → parameter or local when an entity lists it in parameters[] / localVariables[], else attribute. A named entity with none of these (a TS namespace, a PHP code file, a Python decorator) is `other`: principle 1 says its words still count. DEFAULT_WEIGHTS: module and type 8, attribute 4, operation 3, parameter, local and other 1. On the fixtures, every listed parameter and local resolves; the C# indexer's parameter is listed nowhere (a property has no TWithParameters in that profile), weighs as an attribute, and is pinned by id until the model lists it. Co-Authored-By: Claude Opus 5.5 --- packages/vocabulary/src/index.ts | 1 + packages/vocabulary/src/weight.ts | 58 ++++++++++ packages/vocabulary/test/weight.test.ts | 138 ++++++++++++++++++++++++ 3 files changed, 197 insertions(+) create mode 100644 packages/vocabulary/src/weight.ts create mode 100644 packages/vocabulary/test/weight.test.ts diff --git a/packages/vocabulary/src/index.ts b/packages/vocabulary/src/index.ts index 8fc1bec..c7b58de 100644 --- a/packages/vocabulary/src/index.ts +++ b/packages/vocabulary/src/index.ts @@ -8,3 +8,4 @@ */ export * from "./split.js"; export * from "./classify.js"; +export * from "./weight.js"; diff --git a/packages/vocabulary/src/weight.ts b/packages/vocabulary/src/weight.ts new file mode 100644 index 0000000..95b9f83 --- /dev/null +++ b/packages/vocabulary/src/weight.ts @@ -0,0 +1,58 @@ +import type { Entity, EntityId } from "@codegraph/core"; + +/** + * Roles and weights (PLAN.md §20.1). A word counts by the role of the entity + * whose name carries it, and the role comes from the entity's traits, never + * its kind. Weights are data: the artifact records the ones it was built + * under, since a count without its scheme is not a fact. + */ + +/** From the outermost container in; `other` is a named entity no role trait describes. */ +export const ROLES = ["module", "type", "operation", "attribute", "parameter", "local", "other"] as const; +export type Role = (typeof ROLES)[number]; +export type Weights = Readonly>; + +export const DEFAULT_WEIGHTS: Weights = Object.freeze({ + module: 8, // a concept the corpus names once and refers to everywhere + type: 8, + operation: 3, // what it does; the prefix is usually technical (get, find, build) + attribute: 4, // what a type holds: nouns, usually domain + parameter: 1, // incidental names + local: 1, + other: 1, // a TS namespace, a PHP code file: no trait says what it is +}); + +/** The structural entities some entity lists as its parameters or its locals. */ +export interface Slots { + readonly parameters: ReadonlySet; + readonly locals: ReadonlySet; +} + +export function structuralSlots(entities: Iterable): Slots { + const parameters = new Set(); + const locals = new Set(); + for (const entity of entities) { + for (const id of idList(entity, "parameters")) parameters.add(id); + for (const id of idList(entity, "localVariables")) locals.add(id); + } + return { parameters, locals }; +} + +/** First match wins: a C# delegate (TType + TInvocable) is a type. */ +export function roleOf(entity: Entity, slots: Slots): Role { + const { traits } = entity; + if (traits.includes("TModule")) return "module"; + if (traits.includes("TType")) return "type"; + if (traits.includes("TInvocable")) return "operation"; + if (traits.includes("TStructural")) { + if (slots.parameters.has(entity.id)) return "parameter"; + if (slots.locals.has(entity.id)) return "local"; + return "attribute"; + } + return "other"; +} + +function idList(entity: Entity, key: "parameters" | "localVariables"): readonly EntityId[] { + const value = (entity as Record)[key]; + return Array.isArray(value) ? value.filter((id): id is EntityId => typeof id === "string") : []; +} diff --git a/packages/vocabulary/test/weight.test.ts b/packages/vocabulary/test/weight.test.ts new file mode 100644 index 0000000..b1de65f --- /dev/null +++ b/packages/vocabulary/test/weight.test.ts @@ -0,0 +1,138 @@ +import { fileURLToPath } from "node:url"; +import { PROFILES, TRAIT_NAMES, readModelFileSync, type Entity, type TraitName } from "@codegraph/core"; +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; +import { DEFAULT_WEIGHTS, ROLES, roleOf, structuralSlots, type Role } from "../src/weight.js"; + +/** + * The role of the entity carrying a name decides the weight of its words + * (PLAN.md §20.1). It comes from TRAITS, never from the kind: a kind is a + * profile's word, a trait is the metamodel's — the navigator's rule. + */ + +function entity(id: string, traits: readonly TraitName[], extra: Record = {}): Entity { + return { id, kind: "anything", traits: [...traits], ...extra } as Entity; +} + +const NO_SLOTS = structuralSlots([]); + +describe("the roles and their default weights", () => { + it("are a closed set, from the outermost container in", () => { + expect(ROLES).toEqual(["module", "type", "operation", "attribute", "parameter", "local", "other"]); + }); + + it("weigh a named concept most and an incidental name least", () => { + expect(DEFAULT_WEIGHTS).toEqual({ + module: 8, + type: 8, + operation: 3, + attribute: 4, + parameter: 1, + local: 1, + other: 1, + }); + expect(Object.keys(DEFAULT_WEIGHTS).sort()).toEqual([...ROLES].sort()); + }); +}); + +describe("roleOf", () => { + it("reads the traits in a fixed order: module, type, operation, structural", () => { + expect(roleOf(entity("a", ["TNamed", "TModule"]), NO_SLOTS)).toBe("module"); + expect(roleOf(entity("a", ["TNamed", "TModule", "TType"]), NO_SLOTS)).toBe("module"); + // A C# delegate is a type that can be invoked. + expect(roleOf(entity("a", ["TNamed", "TType", "TInvocable", "TWithParameters"]), NO_SLOTS)).toBe("type"); + // A Clojure fn-var: TNamed + TStructural + TInvocable (invariant 1). + expect(roleOf(entity("a", ["TNamed", "TStructural", "TInvocable"]), NO_SLOTS)).toBe("operation"); + expect(roleOf(entity("a", ["TNamed", "TStructural"]), NO_SLOTS)).toBe("attribute"); + expect(roleOf(entity("a", ["TNamed"]), NO_SLOTS)).toBe("other"); + }); + + it("tells a parameter and a local from an attribute by who lists them", () => { + const op = entity("op", ["TNamed", "TInvocable", "TWithParameters", "TWithLocalVariables"], { + parameters: ["p"], + localVariables: ["l"], + }); + const slots = structuralSlots([op]); + expect(roleOf(entity("p", ["TNamed", "TStructural"]), slots)).toBe("parameter"); + expect(roleOf(entity("l", ["TNamed", "TStructural"]), slots)).toBe("local"); + expect(roleOf(entity("f", ["TNamed", "TStructural"]), slots)).toBe("attribute"); + }); + + it("never reads the kind", () => { + const traits = fc.subarray([...TRAIT_NAMES]); + fc.assert( + fc.property(traits, fc.string(), fc.string(), (picked, a, b) => { + const one = { ...entity("x", picked), kind: a || "a" } as Entity; + const two = { ...entity("x", picked), kind: b || "b" } as Entity; + expect(roleOf(one, NO_SLOTS)).toBe(roleOf(two, NO_SLOTS)); + }), + ); + }); +}); + +describe("the shipped profiles", () => { + // Every kind's role, read from its required traits — slots aside, which only + // a model decides. The `other` row is the honest one: a kind no role trait + // describes still has a name, and its words still count. + const roles = new Map(); + for (const [lang, profile] of Object.entries(PROFILES)) { + for (const [kind, spec] of Object.entries(profile.kinds)) { + roles.set(`${lang}:${kind}`, roleOf(entity(kind, spec.required), NO_SLOTS)); + } + } + const having = (role: Role): readonly string[] => + [...roles].filter(([, r]) => r === role).map(([key]) => key).sort(); + + it("give no role trait to five kinds, three of them named", () => { + expect(having("other")).toEqual([ + "clj:implBlock", + "php:codeFile", + "python:decorator", + "rust:impl", + "ts:namespace", + ]); + }); + + it("make every file kind a module except PHP's code file", () => { + for (const [lang, profile] of Object.entries(PROFILES)) { + for (const kind of profile.lexicon?.fileKinds ?? []) { + expect(roles.get(`${lang}:${kind}`), `${lang}:${kind}`).toBe(kind === "codeFile" ? "other" : "module"); + } + } + }); + + it("make a delegate a type and a Clojure function an operation", () => { + expect(roles.get("csharp:delegate")).toBe("type"); + expect(roles.get("clj:function")).toBe("operation"); + expect(roles.get("clj:multimethod")).toBe("operation"); + expect(roles.get("ex:module")).toBe("type"); + expect(roles.get("ex:file")).toBe("module"); + }); +}); + +describe("the fixtures", () => { + const FIXTURES = ["java", "csharp", "typescript", "elixir", "ruby"] as const; + + it.each(FIXTURES)("%s: every parameter and local variable is found by its parent's list", (fixture) => { + const model = readModelFileSync( + fileURLToPath(new URL(`../../../fixtures/${fixture}/expected/model.jsonl`, import.meta.url)), + ); + const slots = structuralSlots(model.entities); + const roles = (kind: string): Map => + new Map(model.entities.filter((e) => e.kind === kind).map((e) => [e.id, roleOf(e, slots)])); + const unlisted = [...roles("parameter")].filter(([, role]) => role !== "parameter").map(([id]) => id); + // The C# indexer's parameter is a child of its `property`, and the C# + // profile gives a property no TWithParameters: the model lists it nowhere, + // so it weighs as an attribute. A gap in the model, not guessed around + // from the kind — pinned so that its fix shows here. + expect(unlisted).toEqual( + fixture === "csharp" ? ["csharp:Acme.Order/Basket.Item(System.Int32)#param:index"] : [], + ); + expect([...new Set(roles("localVariable").values())]).toEqual( + model.entities.some((e) => e.kind === "localVariable") ? ["local"] : [], + ); + // TypeScript's namespace is the one fixture kind without a role trait. + const other = model.entities.filter((e) => roleOf(e, slots) === "other").map((e) => `${model.lang}:${e.kind}`); + expect([...new Set(other)]).toEqual(fixture === "typescript" ? ["ts:namespace"] : []); + }); +}); From c5bfa7a4dda3eb6e461f538792175e664123953b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Fri, 23 Oct 2026 17:10:02 +0200 Subject: [PATCH 4/6] docs(plan): split, classes and roles as built MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §20.1 states the digit and caseless-letter rules of the split, the properties as tested, the keyword spellings that classify nothing, the shared technical list (125 words, disjoint from every profile's), the `other` role and the C# indexer parameter the model lists nowhere. §20.4 ticks the three modules and keeps aggregate, artifact and drift open. Co-Authored-By: Claude Opus 5.5 --- PLAN.md | 83 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/PLAN.md b/PLAN.md index d9dc667..720b7ab 100644 --- a/PLAN.md +++ b/PLAN.md @@ -4951,47 +4951,68 @@ lexicon)`, in six steps, each pinned by a table of hand cases per idiom: universal, no data: `@total` → `total`, `valid?` → `valid`, `save!`, `name=`, `Order::Line`, `max_lines`, `acme-report`, `Repository\`1` → `repository`, `1`; -3. camel-case boundaries with the acronym rule — lower→Upper, and an - Upper run followed by Upper + lower splits before its last letter +3. camel-case boundaries with the acronym rule — lower→Upper, digit→Upper, + and an Upper run followed by Upper + lower splits before its last letter (`HTTPServer` → `http`, `server`; `getURLString` → `get`, `url`, `string`; `XMLHttpRequest` → `xml`, `http`, `request`) — on Unicode letter categories, not ASCII (`fixtures/unicode` is the test: `ＡFullwidth` and `𠀀Supplementary` must split and lowercase without a replacement - character); -4. a digit run attaches to the token before it (`utf8`, `md5`, `v2`); a - token that is only digits is dropped and counted (`skipped.numeric`); + character). A caseless letter (`𠀀`, `注`) counts as lower; a mark takes + the case of the letter it sits on, so a decomposed `é` never splits; +4. a digit run stays with the letters around it — only an Upper after it + breaks (`utf8`, `md5`, `v2`, `i18n`, `http2`; `Utf8Encoder` → `utf8`, + `encoder`); a token that is only digits is dropped and counted + (`skipped.numeric`); 5. lowercase (`toLowerCase`, locale-independent: a Turkish-locale machine must write the same bytes); 6. a `conventional` identifier is skipped whole BEFORE step 1 and counted (`skipped.conventional`): `toString` must not contribute `string`, `handle_call` must not contribute `call`. -Properties (fast-check over identifiers the grammar admits): no empty token; -every token lowercase; every token is, case-insensitively, a substring of -the input with the affixes removed; the same input gives the same tokens; -`split(a + sep + b) = split(a) ++ split(b)` for any lexicon separator. +Properties (fast-check over identifiers the grammar admits): every token is +a non-empty run of letters, marks and digits, lowercase, never digits only, +and a slice of the lowered name; the same input gives the same tokens; +`split(a + sep + b) = split(a) ++ split(b)` for any character that is not a +letter, a mark or a digit — without a lexicon, since the file kinds, the +interface prefix and a conventional name read the WHOLE name. A Σ is left +out of the generator: its lower case depends on what follows it, so a +token's lower case and its slice of the lowered name may differ (`ΟΔΟΣ` → +`οδος`, pinned by hand). **The classes** (`classify.ts`), a closed set `TERM_CLASSES = ["domain", "technical", "keyword", "short"]`, decided per TOKEN in this order: `keyword` when the lowercased token equals a lowercased lexicon keyword -(`type`, `default`, `case`, `new`, `end`); `short` when it is one character -(`i`, `e`, `x` — length 2 stays: `id`, `db`, `io` are words); `technical` -when it is in the shared list ∪ the profile's (`DEFAULT_TECHNICAL`, ~80 -words in the package: `get set is has to from of with by and or not new -create build make init impl abstract base default util utils helper manager -service repository controller handler factory provider dto dao entity model -data info config exception error test mock spec com org net io src lib main -app`…, overridable whole with `--technical FILE`, its digest recorded in the -artifact); else `domain`. The list is contestable by construction — which is -why it is data in one place and the artifact says which one it used. +(`type`, `default`, `case`, `new`, `end`) — a keyword the splitter never +yields whole (`defined?`, `__FILE__`, `set!`) classifies nothing, since the +`file` of `upload_file` is not `__FILE__`; `short` when it is one code point +that is not a mark (`i`, `e`, `x`, `𠀀` — length 2 stays: `id`, `db`, `io` +are words); `technical` when it is in the shared list ∪ the profile's +(`DEFAULT_TECHNICAL`, 125 words in the package: `get set is has to from of +with by and or not new create build make init impl abstract base default +util utils helper manager service repository controller handler factory +provider dto dao entity model data info config exception error test mock +spec com org net io src lib main app`, the CRUD verbs, the web and format +words…, overridable whole with `--technical FILE`, its digest recorded in +the artifact — the profile's own words apply either way); else `domain`. The +list is contestable by construction — which is why it is data in one place +and the artifact says which one it used. It holds what every language +shares: a word two lexicons listed (`args`, `opts`, `ctx`, `err`, +`callback`, `str`, `mutex`, `mixin`, `async`) moved into it, and a test +keeps it disjoint from every profile's `technical`. **Roles and weights** (`weight.ts`). The role of the entity carrying a name -comes from its traits, never its kind (the navigator's rule): `module` -(TModule), `type` (TType), `operation` (TInvocable), `parameter` / `local` -(TStructural listed in the parent's `parameters[]` / `localVariables[]`), -`attribute` (every other TStructural: fields, constants, properties, module -attributes). Default weights, recorded in the artifact and overridable with -`--weights role=n,…`: +comes from its traits, never its kind (the navigator's rule), first match +winning: `module` (TModule), `type` (TType — a C# delegate, TType + +TInvocable, is a type), `operation` (TInvocable — a Clojure fn-var too), +`parameter` / `local` (TStructural listed in some entity's `parameters[]` / +`localVariables[]`), `attribute` (every other TStructural: fields, +constants, properties, module attributes), and `other` — a named entity no +role trait describes (TypeScript's `namespace`, PHP's `codeFile`, Python's +`decorator`), whose words still count (principle 1). Built on the fixtures, +one parameter is listed nowhere: the C# indexer's, a child of its +`property`, which the C# profile gives no `TWithParameters`; it weighs as +an attribute until the model lists it, pinned by id. Default weights, +recorded in the artifact and overridable with `--weights role=n,…`: | role | weight | why | |---|---|---| @@ -4999,6 +5020,7 @@ attributes). Default weights, recorded in the artifact and overridable with | `attribute` | 4 | a thing the type holds — nouns, usually domain | | `operation` | 3 | something it does; the prefix is usually technical (`get`, `find`, `build`) | | `parameter`, `local` | 1 | incidental: the "variable buried deep in a method" of the issue | +| `other` | 1 | no trait says what it is | An occurrence is one token in one name; a name carrying a word twice counts twice. Both the weighted sum and the raw occurrence count are kept — one is @@ -5256,12 +5278,13 @@ never looks like a fact. specification; conventional names from its runtime), `codegraph profiles --lang` printing it; the lexicon's validation codes and tests; `schemas/` unchanged, proven by the drift check. -- [ ] `packages/vocabulary/` (`@codegraph/vocabulary`, depends on core and +- [x] `packages/vocabulary/` (`@codegraph/vocabulary`, depends on core and analyzer only): `split.ts`, `classify.ts` with `DEFAULT_TECHNICAL`, - `weight.ts`, `aggregate.ts` on `containingType`/`containingModule`, - `artifact.ts` (`codegraph.vocabulary/1`), `drift.ts`; the hand tables - per idiom and the properties of §20.1; the unicode fixture through the - splitter. + `weight.ts`; the hand tables per idiom and the properties of §20.1; + the unicode fixture through the splitter; the shared words out of + the profiles. +- [ ] `aggregate.ts` on `containingType`/`containingModule`, `artifact.ts` + (`codegraph.vocabulary/1`), `drift.ts`. - [ ] `codegraph vocabulary` (§20.1): the report, `--level`, `--scope`, `--class`, `--weights`, `--technical`, `--drift`, `--out`, `--json`, the view flags; usage errors for an unknown scope, role or class. From a07bf74bbf3e4c828d503efeb4a2b05e143aebbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Tue, 27 Oct 2026 15:28:14 +0100 Subject: [PATCH 5/6] test(insights): build record arbitraries once The round-trip property drew a fresh fc.record, and a fresh fc.stringMatching regex arbitrary, inside .chain for every record. Generation was 380 of the 440 ms per 100 runs, and under the CI runners' parallel load the property took 5.3 s against Vitest's 5 s timeout (PR #32, run 40485233210). The three envelopes are now built once and picked with fc.oneof: the round trip drops from 650 to 180 ms locally, and the projection properties from about 400 to 60 ms. A store that drops metadata on read, or projects the wrong JSON path, still fails them. Co-Authored-By: Claude Opus 5.5 --- packages/insights/test/store.test.ts | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/insights/test/store.test.ts b/packages/insights/test/store.test.ts index 3498096..99f6cd4 100644 --- a/packages/insights/test/store.test.ts +++ b/packages/insights/test/store.test.ts @@ -90,15 +90,20 @@ const blockArb = { }; const strip = (o: T): T => Object.fromEntries(Object.entries(o).filter(([, v]) => v !== undefined)) as T; +// Built once, never per draw: a regex arbitrary costs a parse, and drawing a fresh one per record timed this suite out on CI. +const fingerprintArb = fc.stringMatching(/^[0-9a-f]{64}$/u); +const envelopeArb = (level: "operation" | "type" | "module") => + fc.record({ + key: maybe(keyArb), kind: text, name: maybe(text), file: maybe(text), scc: maybe(fc.array(text, { maxLength: 3 })), + origin: fc.constantFrom("llm", "template"), block: blockArb[level as "operation"], fingerprint: fingerprintArb, + model: maybe(text), usage: maybe(usageArb), metadata: maybe(fc.dictionary(text, text, { maxKeys: 3 })), + }); +const ENVELOPES = { operation: envelopeArb("operation"), type: envelopeArb("type"), module: envelopeArb("module") }; const recordArb = (id: string): fc.Arbitrary => - fc.constantFrom("operation", "type", "module").chain((level) => - fc - .record({ - key: maybe(keyArb), kind: text, name: maybe(text), file: maybe(text), scc: maybe(fc.array(text, { maxLength: 3 })), - origin: fc.constantFrom("llm", "template"), block: blockArb[level as "operation"], fingerprint: fc.stringMatching(/^[0-9a-f]{64}$/u), - model: maybe(text), usage: maybe(usageArb), metadata: maybe(fc.dictionary(text, text, { maxKeys: 3 })), - }) - .map((r) => strip({ t: "i", id, level, ...r }) as unknown as InsightRecord), + fc.oneof( + ...(["operation", "type", "module"] as const).map((level) => + ENVELOPES[level].map((r) => strip({ t: "i", id, level, ...r }) as unknown as InsightRecord), + ), ); const failureArb = (id: string): fc.Arbitrary => From 631c682f5530618bd4a1b1818f8327e017674782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Grodziski?= Date: Tue, 27 Oct 2026 16:01:41 +0100 Subject: [PATCH 6/6] test(extractor-typescript): 30 s test timeout Every extraction builds a TypeScript program over the fixture. That takes 0.2-0.5 s locally but 2-5 s on a CI runner under `pnpm -r test`. On green runs `determinism > writes the same bytes twice`, which extracts twice, already took 3.5-4.4 s; on #33 (run 40522284488) it took 5.1 s and hit Vitest's 5 s default. The CLI and built-bin snapshot tests ran at 3.1 and 3.6 s in the same job. The work is what these tests check, so the package states a timeout that fits it rather than the default. Co-Authored-By: Claude Opus 5.5 --- extractors/typescript/vitest.config.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extractors/typescript/vitest.config.ts b/extractors/typescript/vitest.config.ts index 8bee2f8..ac277ac 100644 --- a/extractors/typescript/vitest.config.ts +++ b/extractors/typescript/vitest.config.ts @@ -13,4 +13,7 @@ export default defineConfig({ "@codegraph/core": fileURLToPath(new URL("../../packages/core/src/index.ts", import.meta.url)), }, }, + // An extraction builds a TypeScript program: 0.2–0.5 s here, 2–5 s on a CI runner + // under `pnpm -r test`, where the 5 s default timed out a test that extracts twice. + test: { testTimeout: 30_000 }, });